§
    ‚Štjs�  ã                   ó  — d dl mZ d dlZd dlmZ ddlmZ ddlmZm	Z	 ddl
mZ ddlmZmZ dd	lmZmZmZmZ dd
lmZ ddlmZ ddlmZmZmZmZ ddlmZ ddlm Z m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4 ddl5m6Z6m7Z7  ej8        e9¦  «        Z:	 	 dSdej;        dej<        dej<        dej<        dej<        dz  de=dz  de=dee         fd„Z> G d„ de$¦  «        Z? G d „ d!e*¦  «        Z@ G d"„ d#e+¦  «        ZA G d$„ d%e,¦  «        ZB G d&„ d'e#¦  «        ZC G d(„ d)e&¦  «        ZD G d*„ d+e(¦  «        ZE G d,„ d-e'¦  «        ZF G d.„ d/e%¦  «        ZG G d0„ d1e)¦  «        ZHe G d2„ d3e¦  «        ¦   «         ZI G d4„ d5eI¦  «        ZJ G d6„ d7ej;        ¦  «        ZK G d8„ d9ej;        ¦  «        ZL G d:„ d;ej;        ¦  «        ZMee G d<„ d=e¦  «        ¦   «         ¦   «         ZN G d>„ d?ej;        ¦  «        ZO G d@„ dAej;        ¦  «        ZP G dB„ dCe2¦  «        ZQ G dD„ dEe3¦  «        ZR G dF„ dGe0¦  «        ZS G dH„ dIe.¦  «        ZT G dJ„ dKe/¦  «        ZU G dL„ dMe1¦  «        ZV G dN„ dOeV¦  «        ZW G dP„ dQeVe¦  «        ZXg dR¢ZYdS )Té    )Ú	dataclassN)Únné   )Úinitialization)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚBaseModelOutputWithPastÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚCausalLMOutputWithPastÚModelOutput)ÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )
ÚEsmAttentionÚEsmEmbeddingsÚ
EsmEncoderÚEsmIntermediateÚEsmLayerÚ	EsmOutputÚ	EsmPoolerÚEsmRotaryEmbeddingÚEsmSelfAttentionÚEsmSelfOutput)ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaMLPÚLlamaPreTrainedModelÚLlamaRMSNormÚLlamaRotaryEmbeddingÚ	repeat_kvé   )ÚEvollaConfigÚSaProtConfigç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — |€|                      d¦  «        dz  }t          | dd¦  «        }t          ||¦  «        }t          ||¦  «        }t          j        ||                     dd¦  «        ¦  «        |z  }	|�|	|z   }	t          j                             |	d¬¦  «        }	t          j         	                    |	|| j
        ¬¦  «        }	t          j        |	|¦  «        }
|
                     dd¦  «                             ¦   «         }
|
|	fS )	Néÿÿÿÿç      à¿Únum_key_value_groupsr+   r   r   ©Údim)ÚpÚtraining)ÚsizeÚgetattrr*   ÚtorchÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr5   r>   Ú
contiguous)r/   r0   r1   r2   r3   r4   r5   r6   Ún_repÚattn_weightsÚattn_outputs              úg/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/evolla/modular_evolla.pyÚeager_attention_forwardrK   C   sú   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ
 �FÐ2°AÑ6Ô6€EÝ
�C˜Ñ
Ô
€CÝ�e˜UÑ#Ô#€Eõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$ó    c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚEvollaSaProtEmbeddingsc                 óX   •— t          ¦   «                              |¦  «         d | _        d S ©N)ÚsuperÚ__init__Úposition_ids©ÚselfÚconfigÚ	__class__s     €rJ   rR   zEvollaSaProtEmbeddings.__init__g   s)   ø€ Ý‰Œ×Ò˜Ñ Ô Ð à ˆÔÐÐrL   )Ú__name__Ú
__module__Ú__qualname__rR   Ú__classcell__©rW   s   @rJ   rN   rN   f   s8   ø€ € € € € ð!ð !ð !ð !ð !ð !ð !ð !ð !rL   rN   c                   óv   ‡ — e Zd Zd
defˆ fd„Ze	 	 	 ddedz  dddedz  dedef         fˆ fd	„¦   «         Z	ˆ xZ
S )ÚEvollaSaProtRotaryEmbeddingNrV   c                 óL   •— t          ¦   «                              ||¦  «         d S rP   )rQ   rR   )rU   rV   ÚdevicerW   s      €rJ   rR   z$EvollaSaProtRotaryEmbedding.__init__n   s#   ø€ Ý‰Œ×Ò˜ Ñ(Ô(Ð(Ð(Ð(rL   r`   ztorch.device | NoneÚseq_lenÚreturnztorch.Tensorc                 óJ   •— t          ¦   «                              | ||¦  «        S rP   )rQ   Úcompute_default_rope_parameters)rV   r`   ra   rW   s      €rJ   rd   z;EvollaSaProtRotaryEmbedding.compute_default_rope_parametersq   s!   ø€ õ ‰wŒw×6Ò6°v¸vÀwÑOÔOÐOrL   rP   ©NNN)rX   rY   rZ   r-   rR   ÚstaticmethodÚintÚtupleÚfloatrd   r[   r\   s   @rJ   r^   r^   m   sÇ   ø€ € € € € ð)ð )˜|ð )ð )ð )ð )ð )ð )ð à&*Ø(,Ø"ðPð PØ˜tÑ#ðPà%ðPð �t‘ðPð 
ˆ~˜uÐ$Ô	%ð	Pð Pð Pð Pð Pñ „\ðPð Pð Pð Pð PrL   r^   c                   ó   — e Zd ZdS )ÚEvollaSaProtSelfAttentionN©rX   rY   rZ   © rL   rJ   rk   rk   z   ó   € € € € € Ø€DrL   rk   c                   ó   — e Zd ZdS )ÚEvollaSaProtSelfOutputNrl   rm   rL   rJ   rp   rp   ~   rn   rL   rp   c                   ó   — e Zd ZdS )ÚEvollaSaProtAttentionNrl   rm   rL   rJ   rr   rr   ‚   rn   rL   rr   c                   ó   — e Zd ZdS )ÚEvollaSaProtIntermediateNrl   rm   rL   rJ   rt   rt   †   rn   rL   rt   c                   ó   — e Zd ZdS )ÚEvollaSaProtOutputNrl   rm   rL   rJ   rv   rv   Š   rn   rL   rv   c                   ó   — e Zd ZdS )ÚEvollaSaProtLayerNrl   rm   rL   rJ   rx   rx   Ž   rn   rL   rx   c                   ó   — e Zd ZdS )ÚEvollaSaProtEncoderNrl   rm   rL   rJ   rz   rz   ’   rn   rL   rz   c                   ó   — e Zd ZdS )ÚEvollaSaProtPoolerNrl   rm   rL   rJ   r|   r|   –   rn   rL   r|   c                   óª   ‡ — e Zd ZU eed<   dgZdZdZdZdZ	e
 eedd¬¦  «        g eedd¬¦  «        gdœZ ej        ¦   «         ˆ fd	„¦   «         Zˆ xZS )
ÚEvollaSaProtPreTrainedModelrV   rx   Tr+   Ú	attention)ÚindexÚ
layer_nameÚcrossattention)Úhidden_statesÚ
attentionsÚcross_attentionsc                 óø   •— t          ¦   «                              |¦  «         t          |t          ¦  «        rB|                     |j        ¦  «        \  }}t          j        t          |d¦  «        |¦  «         d S d S )NÚinv_freq)	rQ   Ú_init_weightsÚ
isinstancer^   rd   rV   ÚinitÚcopy_r@   )rU   r/   Úcurr_inv_freqÚ_rW   s       €rJ   rˆ   z)EvollaSaProtPreTrainedModel._init_weights«   sw   ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ9Ñ:Ô:ð 	CØ%×EÒEÀfÄmÑTÔTÑˆM˜1ÝŒJ•w˜v zÑ2Ô2°MÑBÔBÐBÐBÐBð	Cð 	CrL   )rX   rY   rZ   r-   Ú__annotations__Ú_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendrx   r   rk   Ú_can_record_outputsrA   Úno_gradrˆ   r[   r\   s   @rJ   r~   r~   š   sÉ   ø€ € € € € € àÐÐÑØ,Ð-ÐØÐØ€NØÐØ"&Ðð +Ø%�~Ð&?ÀqÐU`ÐaÑaÔaÐbàˆNÐ4¸AÐJZÐ[Ñ[Ô[ð
ðð Ðð €U„]�_„_ðCð Cð Cð Cñ „_ðCð Cð Cð Cð CrL   r~   c            
       ó¤   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 d
de	j
        dz  de	j
        dz  dee	j
                 ez  fd	„¦   «         ¦   «         Zˆ xZS )ÚEvollaSaProtProteinEncoderrV   c                 óì   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¬¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S ©N©rV   )	rQ   rR   rN   Ú
embeddingsr^   Úrotary_embeddingsrz   ÚencoderÚ	post_initrT   s     €rJ   rR   z#EvollaSaProtProteinEncoder.__init__´   sb   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý0°Ñ8Ô8ˆŒÝ!<ÀFÐ!KÑ!KÔ!KˆÔÝ*¨6Ñ2Ô2ˆŒØ�ŠÑÔÐÐÐrL   c                 ó   — | j         j        S rP   ©r›   Úword_embeddings©rU   s    rJ   Úget_input_embeddingsz/EvollaSaProtProteinEncoder.get_input_embeddings»   s   € ØŒÔ.Ð.rL   c                 ó   — || j         _        d S rP   r    ©rU   r2   s     rJ   Úset_input_embeddingsz/EvollaSaProtProteinEncoder.set_input_embeddings¾   s   € Ø*/ˆŒÔ'Ð'Ð'rL   NÚ	input_idsr3   rb   c                 óÆ  — |                      ¦   «         }|\  }}|j        }|€t          j        ||f|¬¦  «        }|                      ||¬¦  «        }t          | j        ||¬¦  «        }t          j        ||¬¦  «                             d¦  «        }	|  	                    ||	¦  «        }
 | j
        |f||
dœ|¤Ž}|d         }t          ||j        |j        |j        ¬¦  «        S )N©r`   ©r§   r3   )rV   Úinputs_embedsr3   r   )r3   Úposition_embeddings)Úlast_hidden_staterƒ   r„   r…   )r?   r`   rA   Úonesr›   r
   rV   ÚarangeÚ	unsqueezerœ   r�   r   rƒ   r„   r…   )rU   r§   r3   r6   Úinput_shapeÚ
batch_sizeÚ
seq_lengthr`   r«   rS   r¬   Úencoder_outputsÚsequence_outputs                rJ   Úforwardz"EvollaSaProtProteinEncoder.forwardÁ   s  € ð  —n’nÑ&Ô&ˆØ!,Ñˆ
�JàÔ!ˆØÐ!Ý"œZ¨*°jÐ)AÈ6ÐRÑRÔRˆNØŸš°)ÈN˜Ñ[Ô[ˆå2Ø”;Ø'Ø)ð
ñ 
ô 
ˆõ ”| J°vÐ>Ñ>Ô>×HÒHÈÑKÔKˆØ"×4Ò4°]ÀLÑQÔQÐà&˜$œ,Øð
Ø*8ÐNað
ð 
Øekð
ð 
ˆð *¨!Ô,ˆå;Ø-Ø)Ô7Ø&Ô1Ø,Ô=ð	
ñ 
ô 
ð 	
rL   rP   )rX   rY   rZ   r-   rR   r£   r¦   r   r   rA   ÚTensorrh   r   r¶   r[   r\   s   @rJ   r—   r—   ³   sÎ   ø€ € € € € ð˜|ð ð ð ð ð ð ð/ð /ð /ð0ð 0ð 0ð  Øð /3ð!
ð !
à”< $Ñ&ð!
ð œ tÑ+ð!
ð
 
ˆuŒ|Ô	ÐKÑ	Kð!
ð !
ð !
ñ „_ñ  Ôð!
ð !
ð !
ð !
ð !
rL   r—   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )Ú!EvollaSequenceCompressorAttentioné@   é   c                 ó†  •— t          ¦   «                              ¦   «          |dz  | _        || _        ||z  }t	          j        |¦  «        | _        t	          j        |¦  «        | _        t	          j        ||d¬¦  «        | _	        t	          j        ||dz  d¬¦  «        | _
        t	          j        ||d¬¦  «        | _        d S )Nr9   F©Úbiasr   )rQ   rR   ÚscaleÚheadsr   Ú	LayerNormÚ
norm_mediaÚnorm_latentsÚLinearÚto_qÚto_kvÚto_out)rU   r<   Údim_headrÀ   Ú	inner_dimrW   s        €rJ   rR   z*EvollaSequenceCompressorAttention.__init__è   s¨   ø€ Ý‰Œ×ÒÑÔÐØ˜t‘^ˆŒ
ØˆŒ
Ø˜uÑ$ˆ	åœ, sÑ+Ô+ˆŒÝœL¨Ñ-Ô-ˆÔå”I˜c 9°5Ð9Ñ9Ô9ˆŒ	Ý”Y˜s I°¡M¸Ð>Ñ>Ô>ˆŒ
Ý”i 	¨3°UÐ;Ñ;Ô;ˆŒˆˆrL   c                 ó
  — |                       |¦  «        }|                      |¦  «        }| j        }|                      |¦  «        }t	          j        ||fd¬¦  «        }|                      |¦  «                             dd¬¦  «        \  }}|                     | 	                    d¦  «        | 	                    d¦  «        |d¦  «         
                    dddd¦  «        }|                     | 	                    d¦  «        | 	                    d¦  «        |d¦  «         
                    dddd¦  «        }|                     | 	                    d¦  «        | 	                    d¦  «        |d¦  «         
                    dddd¦  «        }|| j        z  }t	          j        ||                     dd¦  «        ¦  «        }	|	|	                     dd¬	¦  «                             ¦   «         z
  }	|	j        \  }
}}}t	          j        ||¦  «                             |j        ¦  «        }|d
d
…d
d
d
d
…f         }|d
d
d
…d
d
…d
f         }||z  }|	                     d|z
                       ¦   «         d¦  «        }	|	                     d¬¦  «        }t	          j        ||¦  «        }| 
                    dddd¦  «        }|                     | 	                    d¦  «        | 	                    d¦  «        d¦  «        }|                      |¦  «        S )zÔ
        Args:
            x (torch.Tensor): image features
                shape (b, n1, D)
            latent (torch.Tensor): latent features
                shape (b, n2, D);  n2: num of latent tokens
        éþÿÿÿr;   r   r8   r   r+   r   T©r<   ÚkeepdimNg     ˆÃÀ)rÂ   rÃ   rÀ   rÅ   rA   ÚcatrÆ   ÚchunkÚviewr?   Úpermuter¿   rB   rC   ÚamaxÚdetachÚshaper®   Útor`   Úmasked_fillÚboolrE   ÚreshaperÇ   )rU   ÚxÚlatentsÚmaskÚhÚqÚkv_inputÚkÚvÚsimÚbsÚnhÚskdÚokdr®   Úmask_expÚones_expÚattnÚouts                      rJ   r¶   z)EvollaSequenceCompressorAttention.forwardõ   s¢  € ð �OŠO˜AÑÔˆØ×#Ò# GÑ,Ô,ˆàŒJˆà�IŠI�gÑÔˆÝ”9˜a ˜\¨rÐ2Ñ2Ô2ˆØ�zŠz˜(Ñ#Ô#×)Ò)Ø�2ð *ñ 
ô 
‰ˆˆ1ð �FŠF�1—6’6˜!‘9”9˜aŸfšf Q™iœi¨¨BÑ/Ô/×7Ò7¸¸1¸aÀÑCÔCˆØ�FŠF�1—6’6˜!‘9”9˜aŸfšf Q™iœi¨¨BÑ/Ô/×7Ò7¸¸1¸aÀÑCÔCˆØ�FŠF�1—6’6˜!‘9”9˜aŸfšf Q™iœi¨¨BÑ/Ô/×7Ò7¸¸1¸aÀÑCÔCˆØ�”
‰Nˆõ Œl˜1˜aŸkšk¨"¨bÑ1Ô1Ñ2Ô2ˆØ�C—H’H ¨T�HÑ2Ô2×9Ò9Ñ;Ô;Ñ;ˆØœ9ÑˆˆB��SÝŒz˜"˜cÑ"Ô"×%Ò% d¤kÑ2Ô2ˆØ˜˜˜˜4  q q qÐ(Ô)ˆØ˜˜a˜a˜a    DÐ(Ô)ˆØ˜(Ñ"ˆà�oŠo˜q 4™xŸošoÑ/Ô/°Ñ6Ô6ˆØ�{Š{˜rˆ{Ñ"Ô"ˆÝŒl˜4 Ñ#Ô#ˆØ�kŠk˜!˜Q  1Ñ%Ô%ˆð �kŠk˜#Ÿ(š( 1™+œ+ s§x¢x°¡{¤{°BÑ7Ô7ˆà�{Š{˜3ÑÔÐrL   )rº   r»   ©rX   rY   rZ   rR   r¶   r[   r\   s   @rJ   r¹   r¹   ç   sL   ø€ € € € € ð<ð <ð <ð <ð <ð <ð) ð ) ð ) ð ) ð ) ð ) ð ) rL   r¹   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚEvollaFeedForwardé   c                 ó>  •— t          ¦   «                              ¦   «          t          ||z  ¦  «        }t          j        |¦  «        | _        t          j        ||d¬¦  «        | _        t          j        ¦   «         | _	        t          j        ||d¬¦  «        | _
        d S ©NFr½   )rQ   rR   rg   r   rÁ   ÚnormrÄ   Úfc1ÚGELUÚ
activationÚfc2)rU   r<   ÚmultrÉ   rW   s       €rJ   rR   zEvollaFeedForward.__init__"  sz   ø€ Ý‰Œ×ÒÑÔÐÝ˜˜d™
‘O”Oˆ	å”L Ñ%Ô%ˆŒ	Ý”9˜S )°%Ð8Ñ8Ô8ˆŒÝœ'™)œ)ˆŒÝ”9˜Y¨°%Ð8Ñ8Ô8ˆŒˆˆrL   c           	      óž   — |                       |                      |                      |                      |¦  «        ¦  «        ¦  «        ¦  «        S rP   )rô   ró   rñ   rð   )rU   rÙ   s     rJ   r¶   zEvollaFeedForward.forward+  s6   € Ø�xŠx˜Ÿš¨¯ª°·²¸1±´Ñ(>Ô(>Ñ?Ô?Ñ@Ô@Ð@rL   )rí   rê   r\   s   @rJ   rì   rì   !  sS   ø€ € € € € ð9ð 9ð 9ð 9ð 9ð 9ðAð Að Að Að Að Að ArL   rì   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )Ú!EvollaSequenceCompressorResamplerrV   c           
      óp  •— t          ¦   «                              ¦   «          |j        j        }|j        | _        t          j        t          j	        | j        |¦  «        d¬¦  «        | _
        t          j        g ¦  «        | _        t          |j        ¦  «        D ]^}| j                             t          j        t!          ||j        |j        ¬¦  «        t'          ||j        ¬¦  «        g¦  «        ¦  «         Œ_t          j        |j        ¦  «        | _        t          j        ||j        ¦  «        | _        d S )NT)Úrequires_grad)r<   rÈ   rÀ   )r<   rõ   )rQ   rR   Úprotein_encoder_configÚhidden_sizeÚresampler_num_latentsÚnum_latentsr   Ú	ParameterrA   ÚrandnrÚ   Ú
ModuleListÚlayersÚrangeÚresampler_depthÚappendr¹   Úresampler_dim_headÚresampler_headsrì   Úresampler_ff_multrÁ   rð   rÄ   Úprotein_projector)rU   rV   Úprotein_repr_dimr�   rW   s       €rJ   rR   z*EvollaSequenceCompressorResampler.__init__0  s  ø€ Ý‰Œ×ÒÑÔÐØ!Ô8ÔDÐØ!Ô7ˆÔÝ”|¥E¤K°Ô0@ÐBRÑ$SÔ$SÐcgÐhÑhÔhˆŒÝ”m BÑ'Ô'ˆŒÝ�vÔ-Ñ.Ô.ð 
	ð 
	ˆAØŒK×ÒÝ”å9Ø 0¸6Ô;TÐ\bÔ\rðñ ô õ *Ð.>ÀVÔE]Ð^Ñ^Ô^ð	ñô ñ	ô 	ð 	ð 	õ ”L Ô!3Ñ4Ô4ˆŒ	Ý!#¤Ð+;¸VÔ=OÑ!PÔ!PˆÔÐÐrL   c                 óN  — |j         d         }|j         \  }}t          j        || j        ¦  «                             |j        ¦  «        }t          j        ||fd¬¦  «        }t          j        |¦  «                             | j        j        ¦  «        }| j        d          |                     ddd¦  «        z  }|                     |j	        ¦  «        }| j
        D ]#\  }	}
 |	|||¦  «        |z   } |
|¦  «        |z   }Œ$|                      |¦  «        }|                      |¦  «        S )Nr   r+   r;   r8   )rÔ   rA   r®   rþ   rÕ   r`   rÎ   rÚ   rÐ   Údtyper  r	  rð   )rU   ÚembedsrÛ   Úbrâ   r�   Úlatent_maskr®   rÚ   rè   ÚffÚtransformed_features               rJ   r¶   z)EvollaSequenceCompressorResampler.forwardE  s  € ØŒL˜ŒOˆà”
‰ˆˆAÝ”j  TÔ%5Ñ6Ô6×9Ò9¸$¼+ÑFÔFˆÝŒy˜$ Ð,°!Ð4Ñ4Ô4ˆõ Œz˜!‰}Œ}×Ò ¤Ô 3Ñ4Ô4ˆØ”,˜tÔ$ t§y¢y°°Q¸Ñ':Ô':Ñ:ˆØ—*’*˜Vœ\Ñ*Ô*ˆØœð 	,ð 	,‰HˆD�"Ø�d˜6 7¨DÑ1Ô1°GÑ;ˆGØ�b˜‘k”k GÑ+ˆGˆGà"×4Ò4°WÑ=Ô=Ðà�yŠyÐ,Ñ-Ô-Ð-rL   )rX   rY   rZ   r,   rR   r¶   r[   r\   s   @rJ   rø   rø   /  sZ   ø€ € € € € ðQ˜|ð Qð Qð Qð Qð Qð Qð*.ð .ð .ð .ð .ð .ð .rL   rø   c                   ó¬   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej        df         dz  ed<   dZe
ej        df         dz  ed<   dS )ÚEvollaProteinEncoderModelOutputaN  
    sequence_compressor_output (`torch.FloatTensor` of shape `(batch_size, compressed_seq_len, hidden_size)`, *optional*):
        Compressed sequence representation produced by the sequence compressor module. The sequence length is
        reduced from the original input length to `compressed_seq_len` via learned compression.
    NÚsequence_compressor_outputr­   .rƒ   r„   )rX   rY   rZ   Ú__doc__r  rA   ÚFloatTensorrŽ   r­   rƒ   rh   r„   rm   rL   rJ   r  r  Y  s•   € € € € € € ðð ð <@Ð Ô 1°DÑ 8Ð?Ð?Ñ?Ø26Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø:>€M�5˜Ô*¨CÐ/Ô0°4Ñ7Ð>Ð>Ñ>Ø7;€J��eÔ'¨Ð,Ô-°Ñ4Ð;Ð;Ñ;Ð;Ð;rL   r  c                   óX   ‡ — e Zd Zdefˆ fd„Zedej        dej        fd„¦   «         Z	ˆ xZ
S )ÚEvollaProteinEncoderrV   c                 ó¦   •— t          ¦   «                              ¦   «          t          |j        ¬¦  «        | _        t          |¬¦  «        | _        d S r™   )rQ   rR   r—   rû   Úmodelrø   Úsequence_compressor_resamplerrT   s     €rJ   rR   zEvollaProteinEncoder.__init__i  sH   ø€ Ý‰Œ×ÒÑÔÐÝ/°vÔ7TÐUÑUÔUˆŒ
Ý-NÐV\Ð-]Ñ-]Ô-]ˆÔ*Ð*Ð*rL   r§   r3   c                 ó–   — |                       ||¬¦  «        }|j        }|                      ||¦  «        }t          ||j        ¬¦  «        S )Nrª   )r  r­   )r  r­   r  r  )rU   r§   r3   r6   Úprotein_outputÚprotein_embedsÚsequence_reprs          rJ   r¶   zEvollaProteinEncoder.forwardn  sT   € àŸš¨iÈ˜ÑWÔWˆØ'Ô9ˆØ×:Ò:¸>È>ÑZÔZˆå.Ø'4Ø,Ô>ð
ñ 
ô 
ð 	
rL   )rX   rY   rZ   r,   rR   r   rA   Ú
LongTensorr  r¶   r[   r\   s   @rJ   r  r  h  s€   ø€ € € € € ð^˜|ð ^ð ^ð ^ð ^ð ^ð ^ð
 ð
 Ô!1ð 
À5ÔCTð 
ð 
ð 
ñ Ôð
ð 
ð 
ð 
ð 
rL   r  c                   ób   ‡ — e Zd Z	 	 	 ddedz  dedz  dedz  fˆ fd„Zd„ Z	 	 	 	 	 	 	 d	d„Zˆ xZS )
Ú#EvollaSequenceAlignerCrossAttentionNÚprotein_encoder_dimÚstructure_encoder_dimÚmsa_encoder_dimc                 óà  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        | j        dz  | _        t          | j        | j        z  ¦  «        | _        | j        | j        z  | _        |j        }|j	        }|j
        }t          j        | j        | j        ¦  «        | _        |�?t          j        || j        ¦  «        | _        t          j        || j        ¦  «        | _        nd | _        d | _        |�?t          j        || j        ¦  «        | _        t          j        || j        ¦  «        | _        nd | _        d | _        |�?t          j        || j        ¦  «        | _        t          j        || j        ¦  «        | _        nd | _        d | _        t)          | j        ¦  «        | _        t          j        |¦  «        | _        t          j        | j        | j        |¬¦  «        | _        t3          | j        |¦  «        | _        t          j        t9          j        dg¦  «        ¦  «        | _        t          j        t9          j        dg¦  «        ¦  «        | _        d S )Nr9   r½   r.   ) rQ   rR   rü   Únum_attention_headsr¿   rg   Úattention_head_sizeÚall_head_sizeÚ$aligner_attention_probs_dropout_probÚaligner_enable_biasÚaligner_ffn_multr   rÄ   r0   Úkey_proteinÚvalue_proteinÚkey_structureÚvalue_structureÚkey_msaÚ	value_msaÚEvollaRMSNormÚattention_normÚDropoutr5   Úout_projrì   r  rÿ   rA   ÚtensorÚgate_attentionÚgate_ffw)	rU   rV   r#  r$  r%  Úattention_probs_dropout_probÚenable_biasÚffn_multrW   s	           €rJ   rR   z,EvollaSequenceAlignerCrossAttention.__init__{  s
  ø€ õ 	‰Œ×ÒÑÔÐà!Ô-ˆÔØ#)Ô#=ˆÔ ØÔ-¨tÑ3ˆŒ
Ý#& tÔ'7¸$Ô:RÑ'RÑ#SÔ#SˆÔ Ø!Ô5¸Ô8PÑPˆÔà'-Ô'RÐ$ØÔ0ˆØÔ*ˆå”Y˜tÔ/°Ô1CÑDÔDˆŒ
ØÐ*Ý!œyÐ)<¸dÔ>PÑQÔQˆDÔÝ!#¤Ð+>ÀÔ@RÑ!SÔ!SˆDÔÐà#ˆDÔØ!%ˆDÔà Ð,Ý!#¤Ð+@À$ÔBTÑ!UÔ!UˆDÔÝ#%¤9Ð-BÀDÔDVÑ#WÔ#WˆDÔ Ð à!%ˆDÔØ#'ˆDÔ àÐ&Ýœ9 _°dÔ6HÑIÔIˆDŒLÝœY ¸Ô8JÑKÔKˆDŒNˆNàˆDŒLØ!ˆDŒNå+¨DÔ,<Ñ=Ô=ˆÔå”zÐ">Ñ?Ô?ˆŒåœ	 $Ô"2°DÔ4DÈ;ÐWÑWÔWˆŒå# DÔ$4°hÑ?Ô?ˆŒÝ œl­5¬<¸¸Ñ+>Ô+>Ñ?Ô?ˆÔÝœ¥U¤\°3°%Ñ%8Ô%8Ñ9Ô9ˆŒˆˆrL   c	                 óŽ  — |||g}	d„ |	D ¦   «         }	|	st          d¦  «        ‚t          j        |	d¬¦  «        }	|                      |¦  «        }
|                      |
¦  «        }
| j        �G| j        �@|                     |¦  «        }|                      |¦  «        }|                      |¦  «        }nd}d}| j        �G| j	        �@|                     |¦  «        }|                      |¦  «        }|  	                    |¦  «        }nd}d}| j
        �G| j        �@|                     |¦  «        }|  
                    |¦  «        }|                      |¦  «        }nd}d}|||g}d„ |D ¦   «         }t          j        |d¬¦  «        }|||g}d„ |D ¦   «         }t          j        |d¬¦  «        }|
                     ¦   «         dd…         | j        | j        fz   } |
j        |Ž                      d	d
dd¦  «        }
|                     ¦   «         dd…         | j        | j        fz   } |j        |Ž                      d	d
dd¦  «        }|                     ¦   «         dd…         | j        | j        fz   } |j        |Ž                      d	d
dd¦  «        }|
| j        z  }
|€St          j        |                     d	¦  «        |                     d¦  «        ¦  «                             |j        ¦  «        }|dd…ddd…df         |	dd…dddd…f         z  }t          j        |
|                     dd¦  «        ¦  «        }||                     dd¬¦  «                             ¦   «         z
  }|                     d|z
                       ¦   «         t          j        |j        ¦  «        j        ¦  «        } t;          j        d¬¦  «        |¦  «        }t          j        ||¦  «        }|                     d	d
dd¦  «                             ¦   «         }|                     ¦   «         dd…         | j         fz   } |j        |Ž }|  !                    |¦  «        }|S )zù
        query_states: text
        key_value_states: protein
        query_states: [bs, query_seq_len, dim]
        key_value_states: [bs, kv_seq_len, dim]
        query_attn_mask: [bs, query_seq_len]
        kv_attn_mask: [bs, kv_seq_len]
        c                 ó   — g | ]}|®|‘ŒS rP   rm   ©Ú.0r�   s     rJ   ú
<listcomp>zGEvollaSequenceAlignerCrossAttention.cross_attention.<locals>.<listcomp>Ä  s   € ÐAÐAÐA˜a°1°=˜°=°=°=rL   z=At least one modality should be provided for cross attention.r+   r;   Nc                 ó   — g | ]}|®|‘ŒS rP   rm   r?  s     rJ   rA  zGEvollaSequenceAlignerCrossAttention.cross_attention.<locals>.<listcomp>ê  s   € Ð;Ð;Ð;˜1¨Q¨]�Q¨]¨]¨]rL   c                 ó   — g | ]}|®|‘ŒS rP   rm   r?  s     rJ   rA  zGEvollaSequenceAlignerCrossAttention.cross_attention.<locals>.<listcomp>î  s   € Ð?Ð?Ð?˜Q°°�q°°°rL   r8   r   r   r   rË   TrÌ   )"Ú
ValueErrorrA   rÎ   r4  r0   r-  r.  rÕ   r/  r0  r1  r2  r?   r'  r(  rÐ   rÑ   r¿   r®   r`   rB   rC   rÒ   rÓ   rÖ   r×   Úfinfor  Úminr   ÚSoftmaxrF   r)  r6  )rU   Úquery_statesÚprotein_key_value_statesÚstructure_key_value_statesÚmsa_key_value_statesÚquery_attn_maskÚprotein_kv_attn_maskÚstructure_kv_attn_maskÚmsa_kv_attn_maskÚkv_attn_maskÚquery_layerÚkey_layer_proteinÚvalue_layer_proteinÚkey_layer_structureÚvalue_layer_structureÚkey_layer_msaÚvalue_layer_msaÚ	key_layerÚvalue_layerÚnew_query_layer_shapeÚnew_key_layer_shapeÚnew_value_layer_shaper3   rH   Úattention_scoresÚattention_probsÚcontext_layerÚnew_context_layer_shapes                               rJ   Úcross_attentionz3EvollaSequenceAlignerCrossAttention.cross_attention®  sª  € ð* -Ð.DÐFVÐWˆØAÐA <ÐAÑAÔAˆØð 	^ÝÐ\Ñ]Ô]Ð]Ý”y °1Ð5Ñ5Ô5ˆà×)Ò)¨,Ñ7Ô7ˆð —j’j Ñ-Ô-ˆàÔÐ'¨DÔ,>Ð,JØ'?×'BÒ'BÀ<Ñ'PÔ'PÐ$Ø $× 0Ò 0Ð1IÑ JÔ JÐØ"&×"4Ò"4Ð5MÑ"NÔ"NÐÐà $ÐØ"&ÐàÔÐ)¨dÔ.BÐ.NØ)C×)FÒ)FÀ|Ñ)TÔ)TÐ&Ø"&×"4Ò"4Ð5OÑ"PÔ"PÐØ$(×$8Ò$8Ð9SÑ$TÔ$TÐ!Ð!à"&ÐØ$(Ð!àŒ<Ð#¨¬Ð(BØ#7×#:Ò#:¸<Ñ#HÔ#HÐ Ø ŸLšLÐ)=Ñ>Ô>ˆMØ"ŸnšnÐ-AÑBÔBˆOˆOà ˆMØ"ˆOà&Ð(;¸]ÐKˆ	Ø;Ð; 	Ð;Ñ;Ô;ˆ	Ý”I˜i¨QÐ/Ñ/Ô/ˆ	à*Ð,AÀ?ÐSˆØ?Ð? +Ð?Ñ?Ô?ˆÝ”i °Ð3Ñ3Ô3ˆà +× 0Ò 0Ñ 2Ô 2°3°B°3Ô 7ØÔ$ØÔ$ð;
ñ !
Ðð '�kÔ&Ð(=Ð>×FÒFÀqÈ!ÈQÐPQÑRÔRˆà'ŸnšnÑ.Ô.¨s°¨sÔ3ØÔ$ØÔ$ð7
ñ 
Ðð #�I”NÐ$7Ð8×@Ò@ÀÀAÀqÈ!ÑLÔLˆ	à +× 0Ò 0Ñ 2Ô 2°3°B°3Ô 7ØÔ$ØÔ$ð;
ñ !
Ðð '�kÔ&Ð(=Ð>×FÒFÀqÈ!ÈQÐPQÑRÔRˆà! D¤JÑ.ˆð Ð"Ý#œj¨×):Ò):¸1Ñ)=Ô)=¸|×?PÒ?PÐQRÑ?SÔ?SÑTÔT×WÒWÐXdÔXkÑlÔlˆOØ(¨¨¨¨D°!°!°!°TÐ)9Ô:¸\È!È!È!ÈTÐSWÐYZÐYZÐYZÐJZÔ=[Ñ[ˆå”| K°×1DÒ1DÀRÈÑ1LÔ1LÑMÔMˆØ# l×&7Ò&7¸BÈÐ&7Ñ&MÔ&M×&TÒ&TÑ&VÔ&VÑVˆØ'×3Ò3Ø�Ñ×%Ò%Ñ'Ô'­¬°\Ô5GÑ)HÔ)HÔ)Lñ
ô 
Ðð -�"œ*¨Ð,Ñ,Ô,Ð-=Ñ>Ô>ˆõ œ _°kÑBÔBˆà%×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔFˆØ"/×"4Ò"4Ñ"6Ô"6°s¸°sÔ";¸tÔ?QÐ>SÑ"SÐØ*˜Ô*Ð,CÐDˆàŸš mÑ4Ô4ˆàÐrL   c           
      ó8  — |�q|j         \  }}}|€ct          j        ||¦  «                             |	j        ¦  «        |	                     ||f¬¦  «        j        z                       |j        ¦  «        }nd }|�q|j         \  }}}|€ct          j        ||¦  «                             |	j        ¦  «        |
                     ||f¬¦  «        j        z                       |j        ¦  «        }nd }|�q|j         \  }}}|€ct          j        ||¦  «                             |	j        ¦  «        |                     ||f¬¦  «        j        z                       |j        ¦  «        }nd }|}|�|                     ¦   «         s,|�|                     ¦   «         s|�Š|                     ¦   «         rv|}|                      ||||||||¬¦  «        }t          j	        | j
        ¦  «        |z  }||z   }|}|                      |¦  «        t          j	        | j        ¦  «        z  }||z   }|S )N)r?   )rH  rI  rJ  rK  rL  rM  rN  rO  )rÔ   rA   r®   rÕ   r`   ÚexpandÚTÚanyra  Útanhr8  r  r9  )rU   rH  Úprotein_kv_statesÚstructure_kv_statesÚmsa_kv_statesrL  rM  rN  rO  Úprotein_batch_maskÚstructure_batch_maskÚmsa_batch_maskÚpast_key_valuesrâ   Úprotein_kv_seq_lenr<   Ústructure_kv_seq_lenÚmsa_kv_seq_lenrƒ   Úresiduals                       rJ   r¶   z+EvollaSequenceAlignerCrossAttention.forward  s`  € ð Ð(Ø*;Ô*AÑ'ˆBÐ" CØ#Ð+å”J˜rÐ#5Ñ6Ô6×9Ò9Ð:LÔ:SÑTÔTØ(×/Ò/Ð6HÈ"Ð5MÐ/ÑNÔNÔPñQç’"Ð&Ô-Ñ.Ô.ð %øð
 $(Ð àÐ*Ø,?Ô,EÑ)ˆBÐ$ cØ%Ð-å”J˜rÐ#7Ñ8Ô8×;Ò;Ð<NÔ<UÑVÔVØ*×1Ò1Ð8LÈbÐ7QÐ1ÑRÔRÔTñUç’"Ð(Ô/Ñ0Ô0ð 'øð
 &*Ð"àÐ$Ø&3Ô&9Ñ#ˆB� ØÐ'å”J˜r >Ñ2Ô2×5Ò5Ð6HÔ6OÑPÔPØ$×+Ò+°.À"Ð1EÐ+ÑFÔFÔHñIç’"�]Ô)Ñ*Ô*ð !øð
  $ÐØ$ˆð Ð*Ð/C×/GÒ/GÑ/IÔ/IÐ*Ø#Ð/Ð4J×4NÒ4NÑ4PÔ4PÐ/ØÐ)Ð.>×.BÒ.BÑ.DÔ.DÐ)à$ˆHØ ×0Ò0Ø*Ø):Ø+>Ø%2Ø /Ø%9Ø'=Ø!1ð 1ñ 	ô 	ˆMõ "œJ tÔ':Ñ;Ô;¸mÑKˆMà$ }Ñ4ˆMà$ˆHØ ŸGšG MÑ2Ô2µU´ZÀÄÑ5NÔ5NÑNˆMØ$ }Ñ4ˆMàÐrL   re   )NNNNNNN)rX   rY   rZ   rg   rR   ra  r¶   r[   r\   s   @rJ   r"  r"  z  s¾   ø€ € € € € ð +/Ø,0Ø&*ð1:ð 1:ð ! 4™Zð1:ð  # T™zð	1:ð
 ˜t™ð1:ð 1:ð 1:ð 1:ð 1:ð 1:ðfnð nð nðn "Ø#ØØØ!ØØðGð Gð Gð Gð Gð Gð Gð GrL   r"  c                   ó   — e Zd ZdS )r3  Nrl   rm   rL   rJ   r3  r3  h  rn   rL   r3  c                   ó   — e Zd ZdS )ÚEvollaRotaryEmbeddingNrl   rm   rL   rJ   rt  rt  l  rn   rL   rt  c                   ó   — e Zd ZdS )Ú	EvollaMLPNrl   rm   rL   rJ   rv  rv  p  rn   rL   rv  c                   ó   — e Zd ZdS )ÚEvollaAttentionNrl   rm   rL   rJ   rx  rx  t  rn   rL   rx  c                   óN  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 	 	 	 	 	 	 ddej        deej        ej        f         dz  dej        dz  d	ej	        dz  d
e
dz  dedz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  fd„Zˆ xZS )ÚEvollaDecoderLayerrV   Ú	layer_idxc                 óÔ   •— t          ¦   «                              ||¦  «         |dz   t          |j        |j        z  d¦  «        z  dk    rt          ||j        ¬¦  «        | _        d S d S )Nr+   r   )r#  )rQ   rR   ÚmaxÚnum_hidden_layersÚaligner_num_add_layersr"  rü   Úadapter)rU   rV   r{  rW   s      €rJ   rR   zEvollaDecoderLayer.__init__y  st   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+Ø˜‰M�S Ô!9¸VÔ=ZÑ!ZÐ\]Ñ^Ô^Ñ^ÐbcÒcÐcÝ>ØØ$*Ô$6ðñ ô ˆDŒLˆLˆLð dÐcrL   NFrƒ   r¬   r3   rS   rm  Ú	use_cacherg  rh  ri  rj  rk  rl  rL  c           
      ó(  — |}|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }t	          | d¦  «        r|                      ||||	||
||¬¦  «        }|S )N)rƒ   r3   rS   rm  r�  r¬   r€  )rH  rg  rh  ri  rL  rj  rk  rl  rm   )Úinput_layernormÚ	self_attnÚpost_attention_layernormÚmlpÚhasattrr€  )rU   rƒ   r¬   r3   rS   rm  r�  rg  rh  ri  rj  rk  rl  rL  r6   rq  r�   s                    rJ   r¶   zEvollaDecoderLayer.forward�  sç   € ð" !ˆà×,Ò,¨]Ñ;Ô;ˆð *˜4œ>ð 
Ø'Ø)Ø%Ø+ØØ 3ð
ð 
ð ð
ð 
Ñˆ�qð ! =Ñ0ˆð !ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆå�4˜Ñ#Ô#ð 
	Ø ŸLšLØ*Ø"3Ø$7Ø+Ø /Ø#5Ø%9Ø-ð )ñ 	ô 	ˆMð ÐrL   )NNNNFNNNNNNN)rX   rY   rZ   r,   rg   rR   rA   r·   rh   r   r   r×   r¶   r[   r\   s   @rJ   rz  rz  x  si  ø€ € € € € ð˜|ð ¸ð ð ð ð ð ð ð IMØ.2Ø04Ø(,Ø!&Ø15Ø37Ø-1Ø26Ø48Ø.2Ø/3ð3ð 3à”|ð3ð # 5¤<°´Ð#=Ô>ÀÑEð3ð œ tÑ+ð	3ð
 Ô&¨Ñ-ð3ð  ™ð3ð ˜$‘;ð3ð !œ<¨$Ñ.ð3ð #œ\¨DÑ0ð3ð ”| dÑ*ð3ð "œL¨4Ñ/ð3ð $œl¨TÑ1ð3ð œ tÑ+ð3ð œ¨Ñ,ð3ð 3ð 3ð 3ð 3ð 3ð 3ð 3rL   rz  c                   óR   — e Zd ZdZdZdZg d¢Z ej        ¦   «         d„ ¦   «         Z	dS )ÚEvollaPreTrainedModelF)rz  rx   rø   r"  c                 ó|  — | j         j        }t          j        | |¦  «         t	          |t
          ¦  «        rRt          j        |j        ¦  «         t          j        |j	        ¦  «         t          j
        |j        j        ¦  «         d S t	          |t          ¦  «        rt          j        |j        d|¬¦  «         d S d S )Nr.   )ÚmeanÚstd)rV   Úinitializer_ranger   rˆ   r‰   r"  rŠ   Úzeros_r8  r9  Úones_r4  Úweightrø   Únormal_rÚ   )rU   r/   rŒ  s      rJ   rˆ   z#EvollaPreTrainedModel._init_weightsÂ  s¯   € àŒkÔ+ˆÝÔ% d¨FÑ3Ô3Ð3Ý�fÕAÑBÔBð 	<ÝŒK˜Ô-Ñ.Ô.Ð.ÝŒK˜œÑ(Ô(Ð(ÝŒJ�vÔ,Ô3Ñ4Ô4Ð4Ð4Ð4Ý˜Õ AÑBÔBð 	<ÝŒL˜œ¨c°sÐ;Ñ;Ô;Ð;Ð;Ð;ð	<ð 	<rL   N)
rX   rY   rZ   r�   r’   r“   r�   rA   r•   rˆ   rm   rL   rJ   r‰  r‰  ·  s[   € € € € € Ø ÐØÐØ"'Ððð ð Ðð €U„]�_„_ð<ð <ñ „_ð<ð <ð <rL   r‰  c                   óh  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zeee		 	 	 	 	 	 	 	 	 	 	 	 dde
j        dz  de
j        dz  de
j        dz  d	edz  d
e
j        dz  dedz  de
j        dz  de
j        dz  de
j        dz  de
j        dz  de
j        dz  de
j        dz  deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚEvollaModelrV   c                 ó&  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        | j        ‰j        | j        ¦  «        | _        t          ‰¬¦  «        | _
        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t%          ‰dd¦  «        | _        t)          ‰¬¦  «        | _        |                      ¦   «          d S )Nrš   c                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS ))rV   r{  )rz  )r@  r{  rV   s     €rJ   rA  z(EvollaModel.__init__.<locals>.<listcomp>Ö  s@   ø€ ð ð ð ð
 õ	 #Ø!Ø'ðñ ô ðð ð rL   )ÚepsÚgradient_checkpointingF)rQ   rR   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingrü   Úembed_tokensr  Úprotein_encoderr  r  r~  r  r3  Úrms_norm_epsrð   r@   r—  rt  Ú
rotary_embrž   rT   s    `€rJ   rR   zEvollaModel.__init__Ï  s  øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒÝœL¨¬¸&Ô:LÈdÔN^Ñ_Ô_ˆÔÝ3¸6ÐBÑBÔBˆÔÝ”mðð ð ð õ
 "' vÔ'?Ñ!@Ô!@ðñ ô ñ
ô 
ˆŒõ " &Ô"4¸&Ô:MÐNÑNÔNˆŒ	Ý&-¨fÐ6NÐPUÑ&VÔ&VˆÔ#Ý/°vÐ>Ñ>Ô>ˆŒØ�ŠÑÔÐÐÐrL   c                 ó   — | j         S rP   ©rœ  r¢   s    rJ   r£   z EvollaModel.get_input_embeddingsä  s   € ØÔ Ð rL   c                 ó   — || _         d S rP   r¡  r¥   s     rJ   r¦   z EvollaModel.set_input_embeddingsç  s   € Ø!ˆÔÐÐrL   Nr§   r3   rS   rm  r«   r�  Úprotein_input_idsÚprotein_attention_maskÚstructure_featsÚ	msa_featsrk  rl  rb   c                 óâ  — |du |duz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }d}d}|�Q|�O|  
                    ||¬¦  «        }|j        }t          j        |j        d         |j        t
          j        ¬¦  «        }t          | j        |||¬	¦  «        }|}|                      ||¬
¦  «        }| j        D ]} ||f||||||	|
|||||dœ|¤Ž}Œ|                      |¦  «        }t%          ||¬¦  «        }|S )a;  
        protein_input_ids (torch.LongTensor):
            The input IDs for the protein sequence in structure-aware tokens. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
        protein_attention_mask (torch.Tensor):
            The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.
        structure_feats (torch.FloatTensor):
            The input IDs for purely structure-based features. Should be of shape `(batch_size, structure_seq_length, structure_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
        msa_feats (torch.FloatTensor):
            The input IDs for purely MSA-based features. Should be of shape `(batch_size, msa_seq_length, msa_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
        structure_batch_mask (torch.Tensor):
            The batch mask to decide which protein sequences are purely structure-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `structure_feats`. Dummpy input for now.
        msa_batch_mask (torch.Tensor):
            The batch mask to decide which protein sequences are purely MSA-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `msa_feats`. Dummpy input for now.
        Nz:You must specify exactly one of input_ids or inputs_embedsrš   r   r+   r©   rª   )r`   r  )rV   r«   r3   rm  )rS   )r3   rS   rm  r�  rg  rh  ri  rj  rk  rl  rL  r¬   )r­   rm  )rD  rœ  r   rV   Úget_seq_lengthrA   r¯   rÔ   r`   r°   r�  r  r®   r×   r   rŸ  r  rð   r   )rU   r§   r3   rS   rm  r«   r�  r£  r¤  r¥  r¦  rk  rl  r6   Úpast_seen_tokensÚprotein_featsrj  Úprotein_outputsÚcausal_maskrƒ   r¬   Údecoder_layerÚoutputs                          rJ   r¶   zEvollaModel.forwardê  s  € ðB ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLàˆØ!ÐàÐ(Ð-CÐ-OØ"×2Ò2Ø+Ø5ð 3ñ ô ˆOð ,ÔFˆMÝ!&¤Ø!Ô'¨Ô*Ø(Ô/Ý”jð"ñ "ô "Ðõ )Ø”;Ø'Ø)Ø+ð	
ñ 
ô 
ˆð &ˆØ"Ÿošo¨mÈ,˜oÑWÔWÐà!œ[ð 	ð 	ˆMØ)˜MØðà*Ø)Ø /Ø#Ø"/Ø$3Ø'Ø#5Ø%9Ø-Ø .Ø$7ðð ð ðð ˆMˆMð" Ÿ	š	 -Ñ0Ô0ˆå(Ø+Ø+ð
ñ 
ô 
ˆð ˆrL   )NNNNNNNNNNNN)rX   rY   rZ   r,   rR   r£   r¦   r   r   r   rA   r   r·   r   r  r×   rh   r   r¶   r[   r\   s   @rJ   r“  r“  Î  s²  ø€ € € € € ð˜|ð ð ð ð ð ð ð*!ð !ð !ð"ð "ð "ð ØØð .2Ø.2Ø04Ø(,Ø26Ø!%Ø59Ø6:Ø48Ø.2Ø48Ø.2ð]ð ]àÔ# dÑ*ð]ð œ tÑ+ð]ð Ô&¨Ñ-ð	]ð
  ™ð]ð Ô(¨4Ñ/ð]ð ˜$‘;ð]ð !Ô+¨dÑ2ð]ð !&¤¨tÑ 3ð]ð Ô*¨TÑ1ð]ð Ô$ tÑ+ð]ð $œl¨TÑ1ð]ð œ tÑ+ð]ð 
Ð(Ñ	(ð]ð ]ð ]ñ „_ñ  Ôñ „^ð]ð ]ð ]ð ]ð ]rL   r“  c                   óú   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 ddej	        dz  dej
        dz  dej        dz  d	ej	        dz  d
ej	        dz  dej
        dz  dedz  deej
        z  fd„¦   «         ¦   «         Zˆ xZS )ÚEvollaForProteinText2Textc                 óþ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        | j        d¬¦  «        | _        |  	                    ¦   «          d S rï   )
rQ   rR   r“  r  rš  r   rÄ   rü   Úlm_headrž   rT   s     €rJ   rR   z"EvollaForProteinText2Text.__init__N  sg   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý  Ñ(Ô(ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°T´_È5ÐQÑQÔQˆŒà�ŠÑÔÐÐÐrL   c                 ó4   — | j                              ¦   «         S rP   )r  r£   r¢   s    rJ   r£   z.EvollaForProteinText2Text.get_input_embeddingsV  s   € ØŒz×.Ò.Ñ0Ô0Ð0rL   c                 ó6   — | j                              |¦  «        S rP   )r  r¦   r¥   s     rJ   r¦   z.EvollaForProteinText2Text.set_input_embeddingsY  s   € ØŒz×.Ò.¨uÑ5Ô5Ð5rL   Nr   r§   r3   r«   Úlabelsr£  r¤  r�  Úlogits_to_keepc	           
      óJ  —  | j         d||||||dœ|	¤Ž}
|
j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        d||| j        dœ|	¤Ž}t          |||
j	        |
j
        |
j        ¬¦  «        }|S )a,  
        protein_input_ids (torch.LongTensor):
            The input IDs for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
        protein_attention_mask (torch.Tensor):
            The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.

        Example:

        ```python
        >>> from transformers import EvollaProcessor, EvollaForProteinText2Text
        >>> model = EvollaForProteinText2Text.from_pretrained("westlake/Evolla-10B-hf")
        >>> processor = EvollaProcessor.from_pretrained("westlake/Evolla-10B-hf")

        >>> protein_information = {
            "aa_seq": "your amino acid sequence",
            "foldseek": "your foldseek sequence",
        }
        >>> question = "What is the function of this protein?"
        >>> message = [
            {"role": "system", "content": "You are an AI expert that can answer any questions about protein."},
            {"role": "user", "content": question},
        ]

        >>> inputs = processor(proteins=[protein_information], messages_list=[message], return_tensors="pt", padding="longest")
        >>> outputs = model.generate(**inputs)

        >>> print(processor.batch_decode(outputs, skip_special_tokens=True))
        ```)r§   r3   r«   r£  r¤  r�  N)Úlogitsrµ  rš  )Úlossr¸  rm  rƒ   r„   rm   )r  r­   r‰   rg   Úslicer²  Úloss_functionrš  r   rm  rƒ   r„   )rU   r§   r3   r«   rµ  r£  r¤  r�  r¶  r6   Úoutputsrƒ   Úslice_indicesr¸  r¹  Ú
lm_outputss                   rJ   r¶   z!EvollaForProteinText2Text.forward\  s÷   € ðT ,6¨4¬:ð ,
ØØ)Ø'Ø/Ø#9Øð,
ð ,
ð ð,
ð ,
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%Ði¨V¸FÈtÌÐiÐiÐbhÐiÐiˆDå+ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ˆ
ð ÐrL   )NNNNNNNr   )rX   rY   rZ   rR   r£   r¦   r   r   rA   r   r·   r  r×   rg   r¶   r[   r\   s   @rJ   r°  r°  M  s8  ø€ € € € € ðð ð ð ð ð1ð 1ð 1ð6ð 6ð 6ð Øð .2Ø.2Ø26Ø*.Ø59Ø6:Ø!%Ø-.ðBð BàÔ# dÑ*ðBð œ tÑ+ðBð Ô(¨4Ñ/ð	Bð
 Ô  4Ñ'ðBð !Ô+¨dÑ2ðBð !&¤¨tÑ 3ðBð ˜$‘;ðBð ˜eœlÑ*ðBð Bð Bñ „^ñ ÔðBð Bð Bð Bð BrL   r°  )r°  r“  r‰  )Nr.   )ZÚdataclassesr   rA   r   Ú r   rŠ   Úcache_utilsr   r   Ú
generationr	   Úmasking_utilsr
   r   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úesm.modeling_esmr   r   r   r   r   r   r    r!   r"   r#   Úllama.modeling_llamar$   r%   r&   r'   r(   r)   r*   Úconfiguration_evollar,   r-   Ú
get_loggerrX   ÚloggerÚModuler·   ri   rK   rN   r^   rk   rp   rr   rt   rv   rx   rz   r|   r~   r—   r¹   rì   rø   r  r  r"  r3  rt  rv  rx  rz  r‰  r“  r°  Ú__all__rm   rL   rJ   ú<module>rÑ     sœ  ðð "Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø JÐ JÐ JÐ JÐ JÐ JÐ JÐ Jðð ð ð ð ð ð ð ð ð ð ð ð .Ð -Ð -Ð -Ð -Ð -Ø &Ð &Ð &Ð &Ð &Ð &ðð ð ð ð ð ð ð ð ð ð ð ð 8Ð 7Ð 7Ð 7Ð 7Ð 7Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ Eðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð =Ð <Ð <Ð <Ð <Ð <Ð <Ð <ð 
ˆÔ	˜HÑ	%Ô	%€ð !Øð %ð  %ØŒIð %àŒ<ð %ð 
Œð %ð Œ<ð	 %ð
 ”L 4Ñ'ð %ð �T‰\ð %ð ð %ð Ð'Ô(ð %ð  %ð  %ð  %ðF!ð !ð !ð !ð !˜]ñ !ô !ð !ð
Pð 
Pð 
Pð 
Pð 
PÐ"4ñ 
Pô 
Pð 
Pð	ð 	ð 	ð 	ð 	Ð 0ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	˜]ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	˜Lñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	˜ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	˜ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	˜ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	˜*ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	˜ñ 	ô 	ð 	ð ðCð Cð Cð Cð C /ñ Cô Cñ „ðCð01
ð 1
ð 1
ð 1
ð 1
Ð!<ñ 1
ô 1
ð 1
ðh7 ð 7 ð 7 ð 7 ð 7 ¨¬	ñ 7 ô 7 ð 7 ðtAð Að Að Að A˜œ	ñ Aô Að Að'.ð '.ð '.ð '.ð '.¨¬	ñ '.ô '.ð '.ðT Ø
ð
<ð 
<ð 
<ð 
<ð 
< kñ 
<ô 
<ñ „ñ „ð
<ð
ð 
ð 
ð 
ð 
˜2œ9ñ 
ô 
ð 
ð$kð kð kð kð k¨"¬)ñ kô kð kð\	ð 	ð 	ð 	ð 	�Lñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð0ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	�ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	�nñ 	ô 	ð 	ð<ð <ð <ð <ð <Ð*ñ <ô <ð <ð~<ð <ð <ð <ð <Ð0ñ <ô <ð <ð.|ð |ð |ð |ð |Ð'ñ |ô |ð |ð~Sð Sð Sð Sð SÐ 5°ñ Sô Sð Sðl PÐ
OÐ
O€€€rL   