§
    ‚ŠtjBþ  ã                   óN  — d dl Z d dlmZ d dlmZ d dlmZ d dlZd dlmZ ddl	m
Z ddlmZ dd	lmZmZ dd
lmZ ddlmZmZmZ ddlmZmZ ddlmZ ddlmZmZmZm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddl*m+Z+m,Z,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2m3Z3 ddl4m5Z5m6Z6 d„ Z7 G d„ dej8        ¦  «        Z9 G d„ dej8        ¦  «        Z:d„ Z; ed¦  «        d`d„¦   «         Z<	 	 dad ej8        d!ej=        d"ej=        d#ej=        d$ej=        dz  d%e>dz  d&e>d'e)e+         fd(„Z? ee<¦  «         G d)„ d*ej8        ¦  «        ¦   «         Z@ G d+„ d,ej8        ¦  «        ZA G d-„ d.ej8        ¦  «        ZBd/„ ZC G d0„ d1ej8        ¦  «        ZD G d2„ d3ej8        ¦  «        ZE G d4„ d5e¦  «        ZF G d6„ d7ej8        ¦  «        ZG G d8„ d9ej8        ¦  «        ZHe, G d:„ d;e'¦  «        ¦   «         ZI G d<„ d=eI¦  «        ZJ G d>„ d?ej8        ¦  «        ZK G d@„ dAej8        ¦  «        ZL G dB„ dCej8        ¦  «        ZMe,e G dD„ dEe!¦  «        ¦   «         ¦   «         ZN G dF„ dGej8        ¦  «        ZO G dH„ dIej8        ¦  «        ZP edJ¦  «         G dK„ dLej8        ¦  «        ¦   «         ZQ G dM„ dNej8        ¦  «        ZR G dO„ dPej8        ¦  «        ZSdQej=        dReTdSej=        fdT„ZU ee<¦  «         G dU„ dVej8        ¦  «        ¦   «         ZV G dW„ dXe¦  «        ZWe, G dY„ dZe'¦  «        ¦   «         ZX G d[„ d\eX¦  «        ZY G d]„ d^eXe¦  «        ZZg d_¢Z[dS )bé    N)ÚCallable)Ú	dataclass)ÚOptional)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)Ú"BaseModelOutputWithCrossAttentionsÚBaseModelOutputWithPastÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚCausalLMOutputWithPastÚModelOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚEvollaConfigÚSaProtConfigc                 óÖ   — |                       |¦  «                             ¦   «         }t          j        |d¬¦  «                             |¦  «        |z  }|                     ¦   «         |z   S )a  
    Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
    are ignored. This is modified from fairseq's `utils.make_positions`.

    Args:
        x: torch.Tensor x:

    Returns: torch.Tensor
    r$   ©Údim)ÚneÚintÚtorchÚcumsumÚtype_asÚlong)Ú	input_idsÚpadding_idxÚmaskÚincremental_indicess       úh/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/evolla/modeling_evolla.pyÚ"create_position_ids_from_input_idsr5   4   s`   € ð �<Š<˜Ñ$Ô$×(Ò(Ñ*Ô*€DÝœ, t°Ð3Ñ3Ô3×;Ò;¸DÑAÔAÀDÑHÐØ×#Ò#Ñ%Ô%¨Ñ3Ð3ó    c                   ó8   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 dd„Zd„ Zˆ xZS )ÚEvollaSaProtEmbeddingszV
    Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
    c                 ó´  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        ¬¦  «        | _        |j        r&t          j	        |j        |j
        ¬¦  «        | _        nd | _        t          j        |j        ¦  «        | _        t          |dd¦  «        | _        |                      dt%          j        |j        ¦  «                             d¦  «        d¬¦  «         |j        | _        | j        dk    r+t          j        |j        |j        | j        ¬¦  «        | _        |j        | _        |j        | _        d | _        d S )	N)r1   ©ÚepsÚposition_embedding_typeÚabsoluteÚposition_ids)r$   éÿÿÿÿF©Ú
persistent)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚemb_layer_norm_beforeÚ	LayerNormÚlayer_norm_epsÚ
layer_normÚDropoutÚhidden_dropout_probÚdropoutÚgetattrr<   Úregister_bufferr,   ÚarangeÚmax_position_embeddingsÚexpandr1   Úposition_embeddingsÚtoken_dropoutÚmask_token_idr>   ©ÚselfÚconfigÚ	__class__s     €r4   rC   zEvollaSaProtEmbeddings.__init__I   sB  ø€ Ý‰Œ×ÒÑÔÐÝ!œ|¨FÔ,=¸vÔ?QÐ_eÔ_rÐsÑsÔsˆÔàÔ'ð 	#Ý œl¨6Ô+=À6ÔCXÐYÑYÔYˆDŒOˆOà"ˆDŒOÝ”z &Ô"<Ñ=Ô=ˆŒå'.¨vÐ7PÐR\Ñ']Ô']ˆÔ$Ø×ÒØ�EœL¨Ô)GÑHÔH×OÒOÐPWÑXÔXÐejð 	ñ 	
ô 	
ð 	
ð "Ô.ˆÔØÔ'¨:Ò5Ð5Ý')¤|ØÔ.°Ô0BÐPTÔP`ð(ñ (ô (ˆDÔ$ð $Ô1ˆÔØ#Ô1ˆÔà ˆÔÐÐr6   Nc                 ó  — |€-|�t          || j        ¦  «        }n|                      |¦  «        }|€|                      |¦  «        }|}| j        r¾|�¼|                     || j        k                         d¦  «        d¦  «        }d}|�|                     d¦  «        n|j	        d         }|| j        k                         d¦  «         
                    ¦   «         |z  }|d|z
  z  d|z
  d d …d d f         z                       |j        ¦  «        }| j        dk    r|                      |¦  «        }	||	z   }| j        �|                      |¦  «        }|�0||                     d¦  «        z                       |j        ¦  «        }|S )Nr?   ç        g¸…ëQ¸¾?r$   r=   )r5   r1   Ú&create_position_ids_from_inputs_embedsrH   rV   Úmasked_fillrW   Ú	unsqueezeÚsumÚshapeÚfloatÚtoÚdtyper<   rU   rL   )
rY   r0   Úattention_maskr>   Úinputs_embedsÚ
embeddingsÚmask_ratio_trainÚsrc_lengthsÚmask_ratio_observedrU   s
             r4   ÚforwardzEvollaSaProtEmbeddings.forwardb   s¶  € ð ÐØÐ$åAÀ)ÈTÔM]Ñ^Ô^��à#×JÒJÈ=ÑYÔY�àÐ Ø ×0Ò0°Ñ;Ô;ˆMð #ˆ
ð Ôð 	 )Ð"7Ø#×/Ò/°¸dÔ>PÒ1P×0[Ò0[Ð\^Ñ0_Ô0_ÐadÑeÔeˆJØ)ÐØ4BÐ4N˜.×,Ò,¨RÑ0Ô0Ð0ÐT]ÔTcÐdeÔTfˆKØ#,°Ô0BÒ#B×"GÒ"GÈÑ"KÔ"K×"QÒ"QÑ"SÔ"SÐVaÑ"aÐØ$¨Ð,<Ñ(<Ñ=ÀÐEXÑAXÐZ[ÐZ[ÐZ[Ð]aÐcgÐZgÔ@hÑh×lÒlØÔ ñô ˆJð Ô'¨:Ò5Ð5Ø"&×":Ò":¸<Ñ"HÔ"HÐØ#Ð&9Ñ9ˆJàŒ?Ð&ØŸš¨Ñ4Ô4ˆJØÐ%Ø$ ~×'?Ò'?ÀÑ'CÔ'CÑC×GÒGÈ
ÔHXÑYÔYˆJð Ðr6   c                 ó  — |                      ¦   «         dd…         }|d         }t          j        | j        dz   || j        z   dz   t          j        |j        ¬¦  «        }|                     d¦  «                             |¦  «        S )z×
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nr?   r$   )re   Údevicer   )Úsizer,   rR   r1   r/   rn   r`   rT   )rY   rg   Úinput_shapeÚsequence_lengthr>   s        r4   r^   z=EvollaSaProtEmbeddings.create_position_ids_from_inputs_embeds“   s‡   € ð $×(Ò(Ñ*Ô*¨3¨B¨3Ô/ˆØ% aœ.ˆå”|ØÔ˜qÑ  /°DÔ4DÑ"DÀqÑ"HÕPUÔPZÐcpÔcwð
ñ 
ô 
ˆð ×%Ò% aÑ(Ô(×/Ò/°Ñ<Ô<Ð<r6   ©NNNN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__rC   rl   r^   Ú__classcell__©r[   s   @r4   r8   r8   D   st   ø€ € € € € ðð ð!ð !ð !ð !ð !ð6 ØØØð/ð /ð /ð /ðb=ð =ð =ð =ð =ð =ð =r6   r8   c                   óÎ   ‡ — e Zd ZU dZej        ed<   ddefˆ fd„Ze		 	 	 ddedz  ddde
dz  d	ed
ef         fd„¦   «         Z ej        ¦   «         edd„¦   «         ¦   «         Zˆ xZS )ÚEvollaSaProtRotaryEmbeddingzú
    Rotary position embeddings.
    Implementation based on [ModernBERT's RotaryEmbedding](https://github.com/huggingface/transformers/blob/aad13b87ed59f2afcfaebc985f403301887a35fc/src/transformers/models/modernbert/modeling_modernbert.py#L94).
    Úinv_freqNrZ   c                 óî   •— t          ¦   «                              ¦   «          || _        i | _        |                      | j        |¦  «        \  }}|                      d|¦  «         t          | d|¦  «         d S )Nr{   Úattention_scaling)rB   rC   rZ   Ú	rope_typeÚcompute_default_rope_parametersrQ   Úsetattr)rY   rZ   rn   Úcurr_inv_freqÚcurr_attention_scalingr[   s        €r4   rC   z$EvollaSaProtRotaryEmbedding.__init__­   sw   ø€ Ý‰Œ×ÒÑÔÐàˆŒØˆŒà04×0TÒ0TÐUYÔU`ÐbhÑ0iÔ0iÑ-ˆÐ-Ø×Ò˜Z¨Ñ7Ô7Ð7Ý�Ð)Ð+AÑBÔBÐBÐBÐBr6   rn   ztorch.device | NoneÚseq_lenÚreturnútorch.Tensorc                 óð   — | j         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «                             |t          j        ¬¦  «        |z  z  z  }||fS )a©  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Úhead_dimNç      ð?r   é   ©re   ©rn   re   )	Ú
rope_thetarP   rF   Únum_attention_headsr,   rR   Úint64rd   rc   ©rZ   rn   rƒ   Úbaser)   Úattention_factorr{   s          r4   r   z;EvollaSaProtRotaryEmbedding.compute_default_rope_parameters·   s‡   € ð( Ô ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐð Ø•U”\ ! S¨!µ5´;Ð?Ñ?Ô?×BÒBÈ&ÕX]ÔXcÐBÑdÔdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r6   c                 óp  — t          | d¦  «        }t          | d¦  «        }|d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬	¦  «        5  |                     ¦   «         |                     ¦   «         z   
                    dd
¦  «        }	t          j        |	|	fd¬¦  «        }
|
                     ¦   «         |z  }|
                     ¦   «         |z  }d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |                     |j        ¬¦  «        fS )Nr{   r}   r   r?   r$   ÚmpsÚcpuF©Údevice_typeÚenabledr‰   r(   rŠ   )rP   rc   rT   rb   rd   rn   Ú
isinstanceÚtypeÚstrr    Ú	transposer,   ÚcatÚcosÚsinre   )rY   Úxr>   Ú
layer_typer{   r}   Úinv_freq_expandedÚposition_ids_expandedr–   ÚfreqsÚembr�   rž   s                r4   rl   z#EvollaSaProtRotaryEmbedding.forwardÖ   sÑ  € õ ˜4 Ñ,Ô,ˆÝ# DÐ*=Ñ>Ô>Ðà$ T¨1¨1¨1¨d ]Ô3×9Ò9Ñ;Ô;×BÒBÀ<ÔCUÐVWÔCXÐZ\Ð^_Ñ`Ô`×cÒcÐdeÔdlÑmÔmÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	0ð 	0Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)Ð/Ñ/ˆCØ—'’'‘)”)Ð/Ñ/ˆCð		0ð 	0ð 	0ñ 	0ô 	0ð 	0ð 	0ð 	0ð 	0ð 	0ð 	0øøøð 	0ð 	0ð 	0ð 	0ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   Ã'BE7Å7E;Å>E;©N©NNN)rs   rt   ru   rv   r,   ÚTensorÚ__annotations__r&   rC   Ústaticmethodr+   Útuplerc   r   Úno_gradr   rl   rw   rx   s   @r4   rz   rz   ¥   s  ø€ € € € € € ðð ð
 ŒlÐÐÑðCð C˜|ð Cð Cð Cð Cð Cð Cð à&*Ø(,Ø"ð*ð *Ø˜tÑ#ð*à%ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ô	%ð	*ð *ð *ñ „\ð*ð< €U„]�_„_Øð<ð <ð <ñ Ôñ „_ð<ð <ð <ð <ð <r6   rz   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr?   r‰   r(   )rb   r,   rœ   )rŸ   Úx1Úx2s      r4   Úrotate_halfr¯   é   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r6   Úrotary_pos_embc                 ó¨  — | j         }|                     |¦  «        }|                     |¦  «        }|                      ¦   «         |z  t          |                      ¦   «         ¦  «        |z  z   }|                     ¦   «         |z  t          |                     ¦   «         ¦  «        |z  z   }|                     |¦  «        |                     |¦  «        fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )re   r`   rc   r¯   rd   )ÚqÚkr�   rž   Úunsqueeze_dimÚoriginal_dtypeÚq_embedÚk_embeds           r4   Úapply_rotary_pos_embr¸   ð   s¦   € ð& ”W€NØ
�-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�wŠw‰yŒy˜3‰¥;¨q¯wªw©y¬yÑ#9Ô#9¸CÑ#?Ñ@€GØ�wŠw‰yŒy˜3‰¥;¨q¯wªw©y¬yÑ#9Ô#9¸CÑ#?Ñ@€GØ�:Š:�nÑ%Ô% w§z¢z°.Ñ'AÔ'AÐAÐAr6   r]   ÚmoduleÚqueryÚkeyÚvaluerf   ÚscalingrO   Úkwargsc                 ó  — |€|                      d¦  «        dz  }t          | dd¦  «        }t          ||¦  «        }t          ||¦  «        }t          j        ||                     dd¦  «        ¦  «        |z  }	|�|	|z   }	t          j                             |	d¬¦  «        }	t          j         	                    |	|| j
        ¬¦  «        }	t          j        |	|¦  «        }
|
                     dd¦  «                             ¦   «         }
|
|	fS )	Nr?   ç      à¿Únum_key_value_groupsr$   r‰   r   r(   )ÚpÚtraining)ro   rP   Ú	repeat_kvr,   Úmatmulr›   r   Ú
functionalÚsoftmaxrO   rÃ   Ú
contiguous)r¹   rº   r»   r¼   rf   r½   rO   r¾   Ún_repÚattn_weightsÚattn_outputs              r4   Úeager_attention_forwardrÌ     sú   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ
 �FÐ2°AÑ6Ô6€EÝ
�C˜Ñ
Ô
€CÝ�e˜UÑ#Ô#€Eõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r6   c                   óº   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej        dej        dz  dej        dz  dej        dz  dej        dz  d	ee         d
e	ej                 fd„Z
ˆ xZS )ÚEvollaSaProtSelfAttentionNFc                 óÐ  •— t          ¦   «                              ¦   «          || _        |j        |j        z  dk    r0t          |d¦  «        s t          d|j        › d|j        › d�¦  «        ‚|j        | _        t          |j        |j        z  ¦  «        | _        | j        | j        z  | _	        t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        |j        | _        |pt#          |dd¦  «        | _        d| _        |j        | _        || _        | j        o| | _        d S )	Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)r<   r=   rˆ   )rB   rC   rZ   rF   r�   ÚhasattrÚ
ValueErrorr+   Úattention_head_sizeÚall_head_sizer   ÚLinearrº   r»   r¼   Úattention_probs_dropout_probrO   rP   r<   r½   Ú
is_decoderÚ	layer_idxÚ	is_causal)rY   rZ   r<   rÙ   Úis_cross_attentionr[   s        €r4   rC   z"EvollaSaProtSelfAttention.__init__0  se  ø€ Ý‰Œ×ÒÑÔÐØˆŒàÔ Ô :Ñ:¸aÒ?Ð?ÍÐPVÐXhÑHiÔHiÐ?Ýð8 FÔ$6ð 8ð 8Ø Ô4ð8ð 8ð 8ñô ð ð
 $*Ô#=ˆÔ Ý#& vÔ'9¸FÔ<VÑ'VÑ#WÔ#WˆÔ Ø!Ô5¸Ô8PÑPˆÔå”Y˜vÔ1°4Ô3EÑFÔFˆŒ
Ý”9˜VÔ/°Ô1CÑDÔDˆŒÝ”Y˜vÔ1°4Ô3EÑFÔFˆŒ
àÔ:ˆŒà'>ð (
Å'ØÐ-¨zñC
ô C
ˆÔ$ð ˆŒØ Ô+ˆŒØ"ˆŒØœÐCÐ1CÐ-CˆŒˆˆr6   Úhidden_statesrf   Úencoder_hidden_statesÚencoder_attention_maskrU   r¾   r„   c                 ó
  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }	|d u}
|
r|n|}|
r|n|}|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|	| j        dz  z  }	| j        dk    r|\  }}t          |	|||d¬¦  «        \  }	}t          j
        | j        j        t          ¦  «        } || |	|||f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }||fS )	Nr?   r$   r‰   rÀ   Úrotary)r´   r]   ©rO   r½   )rb   rÔ   rº   Úviewr›   r»   r¼   r<   r¸   r   Úget_interfacerZ   Ú_attn_implementationrÌ   rÃ   rO   r½   ÚreshaperÈ   )rY   rÜ   rf   rÝ   rÞ   rU   r¾   rp   Úhidden_shapeÚquery_layerrÛ   Úcurrent_statesÚ	key_layerÚvalue_layerr�   rž   Úattention_interfacerË   rÊ   s                      r4   rl   z!EvollaSaProtSelfAttention.forwardL  sÛ  € ð $Ô)¨#¨2¨#Ô.ˆØC˜ÐC bÐC¨$Ô*BÐCÐCˆà—j’j Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆà2¸$Ð>ÐØ2DÐWÐ.Ð.È-ˆØ3EÐYÐ/Ð/È>ˆØ—H’H˜^Ñ,Ô,×1Ò1°,Ñ?Ô?×IÒIÈ!ÈQÑOÔOˆ	Ø—j’j Ñ0Ô0×5Ò5°lÑCÔC×MÒMÈaÐQRÑSÔSˆð " DÔ$<¸dÑ$BÑBˆàÔ'¨8Ò3Ð3Ø*‰HˆC�Ý%9¸+ÀyÐRUÐWZÐjkÐ%lÑ%lÔ%lÑ"ˆK˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}Ð>�C�C°$´,Ø”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ˜LÐ(Ð(r6   )NNFrr   )rs   rt   ru   rC   r,   r§   ÚFloatTensorr   r   rª   rl   rw   rx   s   @r4   rÎ   rÎ   .  sÜ   ø€ € € € € ðDð Dð Dð Dð Dð Dð> 48Ø:>Ø;?Ø37ð.)ð .)à”|ð.)ð Ô)¨DÑ0ð.)ð  %Ô0°4Ñ7ð	.)ð
 !&Ô 1°DÑ 8ð.)ð #œ\¨DÑ0ð.)ð Ð+Ô,ð.)ð 
ˆuŒ|Ô	ð.)ð .)ð .)ð .)ð .)ð .)ð .)ð .)r6   rÎ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEvollaSaProtSelfOutputc                 óÌ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        d S r¥   )	rB   rC   r   rÖ   rF   ÚdenserM   rN   rO   rX   s     €r4   rC   zEvollaSaProtSelfOutput.__init__~  sJ   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒˆˆr6   c                 ód   — |                       |¦  «        }|                      |¦  «        }||z   }|S r¥   ©rð   rO   ©rY   rÜ   Úinput_tensors      r4   rl   zEvollaSaProtSelfOutput.forwardƒ  ó4   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØ%¨Ñ4ˆØÐr6   ©rs   rt   ru   rC   rl   rw   rx   s   @r4   rî   rî   }  óG   ø€ € € € € ð>ð >ð >ð >ð >ð
ð ð ð ð ð ð r6   rî   c                   óV   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej        dz  dee         fd„Zˆ xZ	S )	ÚEvollaSaProtAttentionNFc                 óè   •— t          ¦   «                              ¦   «          t          |||¬¦  «        | _        t	          |¦  «        | _        t          j        |j        |j	        ¬¦  «        | _        d S )N)rÙ   rÛ   r:   )
rB   rC   rÎ   rY   rî   Úoutputr   rJ   rF   rK   )rY   rZ   rÙ   rÛ   r[   s       €r4   rC   zEvollaSaProtAttention.__init__‹  s_   ø€ Ý‰Œ×ÒÑÔÐÝ-¨fÀ	Ð^pÐqÑqÔqˆŒ	Ý,¨VÑ4Ô4ˆŒåœ fÔ&8¸fÔ>SÐTÑTÔTˆŒˆˆr6   rU   r¾   c                 óˆ   — |                       |¦  «        } | j        |f||||dœ|¤Ž\  }}	|                      ||¦  «        }|S )N©rf   rÝ   rÞ   rU   )rJ   rY   rû   )
rY   rÜ   rf   rÝ   rÞ   rU   r¾   Úhidden_states_lnrË   Ú_s
             r4   rl   zEvollaSaProtAttention.forward’  sk   € ð  Ÿ>š>¨-Ñ8Ô8ÐØ"˜œØð
à)Ø"7Ø#9Ø 3ð
ð 
ð ð
ð 
‰ˆ�Qð —k’k +¨}Ñ=Ô=ˆØÐr6   )NFrr   )
rs   rt   ru   rC   r,   r§   r   r   rl   rw   rx   s   @r4   rù   rù   Š  s‹   ø€ € € € € ðUð Uð Uð Uð Uð Uð Ø"Ø#Ø37ðð ð #œ\¨DÑ0ðð Ð+Ô,ðð ð ð ð ð ð ð r6   rù   c                 óf   — | dz  dt          j        | t          j        d¦  «        z  ¦  «        z   z  S )zz
    This is the gelu implementation from the original EVOLLA_SA_PROT repo. Using F.gelu yields subtly wrong results.
    g      à?rˆ   g       @)r,   ÚerfÚmathÚsqrt)rŸ   s    r4   Úgelur  ¨  s/   € ð ˆs‰7�c�EœI a­$¬)°C©.¬.Ñ&8Ñ9Ô9Ñ9Ñ:Ð:r6   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚEvollaSaProtIntermediatec                 ó�   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        d S r¥   )rB   rC   r   rÖ   rF   Úintermediate_sizerð   rX   s     €r4   rC   z!EvollaSaProtIntermediate.__init__°  s6   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3KÑLÔLˆŒ
ˆ
ˆ
r6   rÜ   r„   c                 óN   — |                       |¦  «        }t          |¦  «        }|S r¥   )rð   r  )rY   rÜ   s     r4   rl   z EvollaSaProtIntermediate.forward´  s&   € ØŸ
š
 =Ñ1Ô1ˆÝ˜]Ñ+Ô+ˆØÐr6   ©rs   rt   ru   rC   r,   r§   rl   rw   rx   s   @r4   r  r  ¯  sc   ø€ € € € € ðMð Mð Mð Mð Mð U¤\ð °e´lð ð ð ð ð ð ð ð r6   r  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEvollaSaProtOutputc                 óÌ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _	        d S r¥   )
rB   rC   r   rÖ   r  rF   rð   rM   rN   rO   rX   s     €r4   rC   zEvollaSaProtOutput.__init__»  sJ   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ7¸Ô9KÑLÔLˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒˆˆr6   c                 ód   — |                       |¦  «        }|                      |¦  «        }||z   }|S r¥   rò   ró   s      r4   rl   zEvollaSaProtOutput.forwardÀ  rõ   r6   rö   rx   s   @r4   r  r  º  r÷   r6   r  c                   óZ   ‡ — e Zd Zˆ fd„Z	 	 	 	 ddej        dz  dee         fd„Zd„ Z	ˆ xZ
S )ÚEvollaSaProtLayerc                 óÌ  •— t          ¦   «                              ¦   «          |j        | _        d| _        t	          |¦  «        | _        |j        | _        |j        | _        | j        r/| j        st          | › d�¦  «        ‚t	          |d¬¦  «        | _	        t          |¦  «        | _        t          |¦  «        | _        t          j        |j        |j        ¬¦  «        | _        d S )Nr$   z> should be used as a decoder model if cross attention is addedT)rÛ   r:   )rB   rC   Úchunk_size_feed_forwardÚseq_len_dimrù   Ú	attentionrØ   Úadd_cross_attentionÚRuntimeErrorÚcrossattentionr  Úintermediater  rû   r   rJ   rF   rK   rX   s     €r4   rC   zEvollaSaProtLayer.__init__È  sÓ   ø€ Ý‰Œ×ÒÑÔÐØ'-Ô'EˆÔ$ØˆÔÝ.¨vÑ6Ô6ˆŒØ Ô+ˆŒØ#)Ô#=ˆÔ ØÔ#ð 	YØ”?ð lÝ" dÐ#jÐ#jÐ#jÑkÔkÐkÝ"7¸ÐSWÐ"XÑ"XÔ"XˆDÔÝ4°VÑ<Ô<ˆÔÝ(¨Ñ0Ô0ˆŒÝœ fÔ&8¸fÔ>SÐTÑTÔTˆŒˆˆr6   NrU   r¾   c                 óÐ   —  | j         |f||dœ|¤Ž}| j        r8|�6t          | d¦  «        st          d| › d�¦  «        ‚ | j        |f||||dœ|¤Ž}|                      |¦  «        }|S )N©rf   rU   r  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`rý   )r  rØ   rÒ   ÚAttributeErrorr  Úfeed_forward_chunk)	rY   rÜ   rf   rÝ   rÞ   rU   r¾   Úattention_outputÚlayer_outputs	            r4   rl   zEvollaSaProtLayer.forward×  sÜ   € ð *˜4œ>Øð
à)Ø 3ð
ð 
ð ð	
ð 
Ðð Œ?ð 	Ð4Ð@Ý˜4Ð!1Ñ2Ô2ð Ý$ð`¸dð `ð `ð `ñô ð ð
  3˜tÔ2Ø ð à-Ø&;Ø'=Ø$7ð ð  ð ð ð  Ðð ×.Ò.Ð/?Ñ@Ô@ˆØÐr6   c                 ó†   — |                       |¦  «        }|                      |¦  «        }|                      ||¦  «        }|S r¥   )rJ   r  rû   )rY   r  Úattention_output_lnÚintermediate_outputr  s        r4   r  z$EvollaSaProtLayer.feed_forward_chunkú  sE   € Ø"ŸnšnÐ-=Ñ>Ô>ÐØ"×/Ò/Ð0CÑDÔDÐØ—{’{Ð#6Ð8HÑIÔIˆØÐr6   rr   )rs   rt   ru   rC   r,   r§   r   r   rl   r  rw   rx   s   @r4   r  r  Ç  s•   ø€ € € € € ðUð Uð Uð Uð Uð$ Ø"Ø#Ø37ð!ð !ð #œ\¨DÑ0ð!ð Ð+Ô,ð!ð !ð !ð !ðFð ð ð ð ð ð r6   r  c                   ód   ‡ — e Zd Zˆ fd„Ze	 	 	 	 ddej        dz  dee         fd„¦   «         Z	ˆ xZ
S )ÚEvollaSaProtEncoderc                 ó  •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j	        ‰j
        ¬¦  «        | _        d| _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS © )r  )Ú.0rÿ   rZ   s     €r4   ú
<listcomp>z0EvollaSaProtEncoder.__init__.<locals>.<listcomp>  s"   ø€ Ð#gÐ#gÐ#gÀ!Õ$5°fÑ$=Ô$=Ð#gÐ#gÐ#gr6   r:   F)rB   rC   rZ   r   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayerrJ   rF   rK   Úemb_layer_norm_afterÚgradient_checkpointingrX   s    `€r4   rC   zEvollaSaProtEncoder.__init__  s}   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”]Ð#gÐ#gÐ#gÐ#gÅuÈVÔMeÑGfÔGfÐ#gÑ#gÔ#gÑhÔhˆŒ
Ý$&¤L°Ô1CÈÔI^Ð$_Ñ$_Ô$_ˆÔ!Ø&+ˆÔ#Ð#Ð#r6   NrU   r¾   c           	      óª   — t          | j        ¦  «        D ]\  }} ||f||||dœ|¤Ž}Œ| j        r|                      |¦  «        }t          |¬¦  «        S )Nrý   )Úlast_hidden_state)Ú	enumerater,  r-  r   )	rY   rÜ   rf   rÝ   rÞ   rU   r¾   ÚiÚlayer_modules	            r4   rl   zEvollaSaProtEncoder.forward	  s‰   € õ  )¨¬Ñ4Ô4ð 	ð 	‰OˆAˆ|Ø(˜LØðà-Ø&;Ø'=Ø$7ðð ð ðð ˆMˆMð Ô$ð 	EØ ×5Ò5°mÑDÔDˆMå1ÀMÐRÑRÔRÐRr6   rr   )rs   rt   ru   rC   r   r,   r§   r   r   rl   rw   rx   s   @r4   r#  r#    s˜   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð ð Ø"Ø#Ø37ðSð Sð #œ\¨DÑ0ðSð Ð+Ô,ðSð Sð Sñ ÔðSð Sð Sð Sð Sr6   r#  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚEvollaSaProtPoolerc                 óÀ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        ¦   «         | _        d S r¥   )rB   rC   r   rÖ   rF   rð   ÚTanhÚ
activationrX   s     €r4   rC   zEvollaSaProtPooler.__init__$  sC   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ'™)œ)ˆŒˆˆr6   rÜ   r„   c                 ór   — |d d …df         }|                       |¦  «        }|                      |¦  «        }|S )Nr   )rð   r8  )rY   rÜ   Úfirst_token_tensorÚpooled_outputs       r4   rl   zEvollaSaProtPooler.forward)  s@   € ð +¨1¨1¨1¨a¨4Ô0ÐØŸ
š
Ð#5Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐr6   r
  rx   s   @r4   r5  r5  #  s^   ø€ € € € € ð$ð $ð $ð $ð $ð
 U¤\ð °e´lð ð ð ð ð ð ð ð r6   r5  c                   óª   ‡ — e Zd ZU eed<   dgZdZdZdZdZ	e
 eedd¬¦  «        g eedd¬¦  «        gdœZ ej        ¦   «         ˆ fd	„¦   «         Zˆ xZS )
ÚEvollaSaProtPreTrainedModelrZ   r  Tr$   r  )ÚindexÚ
layer_namer  )rÜ   Ú
attentionsÚcross_attentionsc                 óø   •— t          ¦   «                              |¦  «         t          |t          ¦  «        rB|                     |j        ¦  «        \  }}t          j        t          |d¦  «        |¦  «         d S d S )Nr{   )	rB   Ú_init_weightsr˜   rz   r   rZ   ÚinitÚcopy_rP   )rY   r¹   r�   rÿ   r[   s       €r4   rC  z)EvollaSaProtPreTrainedModel._init_weightsC  sw   ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ9Ñ:Ô:ð 	CØ%×EÒEÀfÄmÑTÔTÑˆM˜1ÝŒJ•w˜v zÑ2Ô2°MÑBÔBÐBÐBÐBð	Cð 	Cr6   )rs   rt   ru   r&   r¨   Ú_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr  r"   rÎ   Ú_can_record_outputsr,   r«   rC  rw   rx   s   @r4   r=  r=  2  sÉ   ø€ € € € € € àÐÐÑØ,Ð-ÐØÐØ€NØÐØ"&Ðð +Ø%�~Ð&?ÀqÐU`ÐaÑaÔaÐbàˆNÐ4¸AÐJZÐ[Ñ[Ô[ð
ðð Ðð €U„]�_„_ðCð Cð Cð Cñ „_ðCð Cð Cð Cð Cr6   r=  c            
       ó¤   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 d
de	j
        dz  de	j
        dz  dee	j
                 ez  fd	„¦   «         ¦   «         Zˆ xZS )ÚEvollaSaProtProteinEncoderrZ   c                 óì   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¬¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S ©N©rZ   )	rB   rC   r8   rh   rz   Úrotary_embeddingsr#  ÚencoderÚ	post_initrX   s     €r4   rC   z#EvollaSaProtProteinEncoder.__init__L  sb   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý0°Ñ8Ô8ˆŒÝ!<ÀFÐ!KÑ!KÔ!KˆÔÝ*¨6Ñ2Ô2ˆŒØ�ŠÑÔÐÐÐr6   c                 ó   — | j         j        S r¥   ©rh   rH   ©rY   s    r4   Úget_input_embeddingsz/EvollaSaProtProteinEncoder.get_input_embeddingsS  s   € ØŒÔ.Ð.r6   c                 ó   — || j         _        d S r¥   rU  ©rY   r¼   s     r4   Úset_input_embeddingsz/EvollaSaProtProteinEncoder.set_input_embeddingsV  s   € Ø*/ˆŒÔ'Ð'Ð'r6   Nr0   rf   r„   c                 óÆ  — |                      ¦   «         }|\  }}|j        }|€t          j        ||f|¬¦  «        }|                      ||¬¦  «        }t          | j        ||¬¦  «        }t          j        ||¬¦  «                             d¦  «        }	|  	                    ||	¦  «        }
 | j
        |f||
dœ|¤Ž}|d         }t          ||j        |j        |j        ¬¦  «        S )N©rn   ©r0   rf   )rZ   rg   rf   r   r  )r0  rÜ   r@  rA  )ro   rn   r,   Úonesrh   r   rZ   rR   r`   rQ  rR  r   rÜ   r@  rA  )rY   r0   rf   r¾   rp   Ú
batch_sizeÚ
seq_lengthrn   rg   r>   rU   Úencoder_outputsÚsequence_outputs                r4   rl   z"EvollaSaProtProteinEncoder.forwardY  s  € ð  —n’nÑ&Ô&ˆØ!,Ñˆ
�JàÔ!ˆØÐ!Ý"œZ¨*°jÐ)AÈ6ÐRÑRÔRˆNØŸš°)ÈN˜Ñ[Ô[ˆå2Ø”;Ø'Ø)ð
ñ 
ô 
ˆõ ”| J°vÐ>Ñ>Ô>×HÒHÈÑKÔKˆØ"×4Ò4°]ÀLÑQÔQÐà&˜$œ,Øð
Ø*8ÐNað
ð 
Øekð
ð 
ˆð *¨!Ô,ˆå;Ø-Ø)Ô7Ø&Ô1Ø,Ô=ð	
ñ 
ô 
ð 	
r6   r¥   )rs   rt   ru   r&   rC   rW  rZ  r!   r#   r,   r§   rª   r   rl   rw   rx   s   @r4   rM  rM  K  sÎ   ø€ € € € € ð˜|ð ð ð ð ð ð ð/ð /ð /ð0ð 0ð 0ð  Øð /3ð!
ð !
à”< $Ñ&ð!
ð œ tÑ+ð!
ð
 
ˆuŒ|Ô	ÐKÑ	Kð!
ð !
ð !
ñ „_ñ  Ôð!
ð !
ð !
ð !
ð !
r6   rM  c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )Ú!EvollaSequenceCompressorAttentioné@   é   c                 ó†  •— t          ¦   «                              ¦   «          |dz  | _        || _        ||z  }t	          j        |¦  «        | _        t	          j        |¦  «        | _        t	          j        ||d¬¦  «        | _	        t	          j        ||dz  d¬¦  «        | _
        t	          j        ||d¬¦  «        | _        d S )NrÀ   F©Úbiasr‰   )rB   rC   ÚscaleÚheadsr   rJ   Ú
norm_mediaÚnorm_latentsrÖ   Úto_qÚto_kvÚto_out)rY   r)   Údim_headrk  Ú	inner_dimr[   s        €r4   rC   z*EvollaSequenceCompressorAttention.__init__€  s¨   ø€ Ý‰Œ×ÒÑÔÐØ˜t‘^ˆŒ
ØˆŒ
Ø˜uÑ$ˆ	åœ, sÑ+Ô+ˆŒÝœL¨Ñ-Ô-ˆÔå”I˜c 9°5Ð9Ñ9Ô9ˆŒ	Ý”Y˜s I°¡M¸Ð>Ñ>Ô>ˆŒ
Ý”i 	¨3°UÐ;Ñ;Ô;ˆŒˆˆr6   c                 ó
  — |                       |¦  «        }|                      |¦  «        }| j        }|                      |¦  «        }t	          j        ||fd¬¦  «        }|                      |¦  «                             dd¬¦  «        \  }}|                     | 	                    d¦  «        | 	                    d¦  «        |d¦  «         
                    dddd¦  «        }|                     | 	                    d¦  «        | 	                    d¦  «        |d¦  «         
                    dddd¦  «        }|                     | 	                    d¦  «        | 	                    d¦  «        |d¦  «         
                    dddd¦  «        }|| j        z  }t	          j        ||                     dd¦  «        ¦  «        }	|	|	                     dd¬	¦  «                             ¦   «         z
  }	|	j        \  }
}}}t	          j        ||¦  «                             |j        ¦  «        }|d
d
…d
d
d
d
…f         }|d
d
d
…d
d
…d
f         }||z  }|	                     d|z
                       ¦   «         d¦  «        }	|	                     d¬¦  «        }t	          j        ||¦  «        }| 
                    dddd¦  «        }|                     | 	                    d¦  «        | 	                    d¦  «        d¦  «        }|                      |¦  «        S )zÔ
        Args:
            x (torch.Tensor): image features
                shape (b, n1, D)
            latent (torch.Tensor): latent features
                shape (b, n2, D);  n2: num of latent tokens
        éþÿÿÿr(   r‰   r?   r   r$   r   T©r)   ÚkeepdimNg     ˆÃÀ)rl  rm  rk  rn  r,   rœ   ro  Úchunkrâ   ro   Úpermuterj  rÅ   r›   ÚamaxÚdetachrb   r^  rd   rn   r_   ÚboolrÇ   rå   rp  )rY   rŸ   Úlatentsr2   Úhr²   Úkv_inputr³   ÚvÚsimÚbsÚnhÚskdÚokdr^  Úmask_expÚones_expÚattnÚouts                      r4   rl   z)EvollaSequenceCompressorAttention.forward�  s¢  € ð �OŠO˜AÑÔˆØ×#Ò# GÑ,Ô,ˆàŒJˆà�IŠI�gÑÔˆÝ”9˜a ˜\¨rÐ2Ñ2Ô2ˆØ�zŠz˜(Ñ#Ô#×)Ò)Ø�2ð *ñ 
ô 
‰ˆˆ1ð �FŠF�1—6’6˜!‘9”9˜aŸfšf Q™iœi¨¨BÑ/Ô/×7Ò7¸¸1¸aÀÑCÔCˆØ�FŠF�1—6’6˜!‘9”9˜aŸfšf Q™iœi¨¨BÑ/Ô/×7Ò7¸¸1¸aÀÑCÔCˆØ�FŠF�1—6’6˜!‘9”9˜aŸfšf Q™iœi¨¨BÑ/Ô/×7Ò7¸¸1¸aÀÑCÔCˆØ�”
‰Nˆõ Œl˜1˜aŸkšk¨"¨bÑ1Ô1Ñ2Ô2ˆØ�C—H’H ¨T�HÑ2Ô2×9Ò9Ñ;Ô;Ñ;ˆØœ9ÑˆˆB��SÝŒz˜"˜cÑ"Ô"×%Ò% d¤kÑ2Ô2ˆØ˜˜˜˜4  q q qÐ(Ô)ˆØ˜˜a˜a˜a    DÐ(Ô)ˆØ˜(Ñ"ˆà�oŠo˜q 4™xŸošoÑ/Ô/°Ñ6Ô6ˆØ�{Š{˜rˆ{Ñ"Ô"ˆÝŒl˜4 Ñ#Ô#ˆØ�kŠk˜!˜Q  1Ñ%Ô%ˆð �kŠk˜#Ÿ(š( 1™+œ+ s§x¢x°¡{¤{°BÑ7Ô7ˆà�{Š{˜3ÑÔÐr6   )re  rf  rö   rx   s   @r4   rd  rd    sL   ø€ € € € € ð<ð <ð <ð <ð <ð <ð) ð ) ð ) ð ) ð ) ð ) ð ) r6   rd  c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚEvollaFeedForwardé   c                 ó>  •— t          ¦   «                              ¦   «          t          ||z  ¦  «        }t          j        |¦  «        | _        t          j        ||d¬¦  «        | _        t          j        ¦   «         | _	        t          j        ||d¬¦  «        | _
        d S ©NFrh  )rB   rC   r+   r   rJ   ÚnormrÖ   Úfc1ÚGELUr8  Úfc2)rY   r)   Úmultrr  r[   s       €r4   rC   zEvollaFeedForward.__init__º  sz   ø€ Ý‰Œ×ÒÑÔÐÝ˜˜d™
‘O”Oˆ	å”L Ñ%Ô%ˆŒ	Ý”9˜S )°%Ð8Ñ8Ô8ˆŒÝœ'™)œ)ˆŒÝ”9˜Y¨°%Ð8Ñ8Ô8ˆŒˆˆr6   c           	      óž   — |                       |                      |                      |                      |¦  «        ¦  «        ¦  «        ¦  «        S r¥   )r‘  r8  r�  rŽ  )rY   rŸ   s     r4   rl   zEvollaFeedForward.forwardÃ  s6   € Ø�xŠx˜Ÿš¨¯ª°·²¸1±´Ñ(>Ô(>Ñ?Ô?Ñ@Ô@Ð@r6   )r‹  rö   rx   s   @r4   rŠ  rŠ  ¹  sS   ø€ € € € € ð9ð 9ð 9ð 9ð 9ð 9ðAð Að Að Að Að Að Ar6   rŠ  c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )Ú!EvollaSequenceCompressorResamplerrZ   c           
      óp  •— t          ¦   «                              ¦   «          |j        j        }|j        | _        t          j        t          j	        | j        |¦  «        d¬¦  «        | _
        t          j        g ¦  «        | _        t          |j        ¦  «        D ]^}| j                             t          j        t!          ||j        |j        ¬¦  «        t'          ||j        ¬¦  «        g¦  «        ¦  «         Œ_t          j        |j        ¦  «        | _        t          j        ||j        ¦  «        | _        d S )NT)Úrequires_grad)r)   rq  rk  )r)   r’  )rB   rC   Úprotein_encoder_configrF   Úresampler_num_latentsÚnum_latentsr   Ú	Parameterr,   Úrandnr|  r)  Úlayersr*  Úresampler_depthÚappendrd  Úresampler_dim_headÚresampler_headsrŠ  Úresampler_ff_multrJ   rŽ  rÖ   Úprotein_projector)rY   rZ   Úprotein_repr_dimrÿ   r[   s       €r4   rC   z*EvollaSequenceCompressorResampler.__init__È  s  ø€ Ý‰Œ×ÒÑÔÐØ!Ô8ÔDÐØ!Ô7ˆÔÝ”|¥E¤K°Ô0@ÐBRÑ$SÔ$SÐcgÐhÑhÔhˆŒÝ”m BÑ'Ô'ˆŒÝ�vÔ-Ñ.Ô.ð 
	ð 
	ˆAØŒK×ÒÝ”å9Ø 0¸6Ô;TÐ\bÔ\rðñ ô õ *Ð.>ÀVÔE]Ð^Ñ^Ô^ð	ñô ñ	ô 	ð 	ð 	õ ”L Ô!3Ñ4Ô4ˆŒ	Ý!#¤Ð+;¸VÔ=OÑ!PÔ!PˆÔÐÐr6   c                 óN  — |j         d         }|j         \  }}t          j        || j        ¦  «                             |j        ¦  «        }t          j        ||fd¬¦  «        }t          j        |¦  «                             | j        j        ¦  «        }| j        d          |                     ddd¦  «        z  }|                     |j	        ¦  «        }| j
        D ]#\  }	}
 |	|||¦  «        |z   } |
|¦  «        |z   }Œ$|                      |¦  «        }|                      |¦  «        S )Nr   r$   r(   r?   )rb   r,   r^  rš  rd   rn   rœ   r|  râ   re   r�  r£  rŽ  )rY   Úembedsr2   Úbr�  rÿ   Úlatent_maskr^  r|  r‡  ÚffÚtransformed_features               r4   rl   z)EvollaSequenceCompressorResampler.forwardÝ  s  € ØŒL˜ŒOˆà”
‰ˆˆAÝ”j  TÔ%5Ñ6Ô6×9Ò9¸$¼+ÑFÔFˆÝŒy˜$ Ð,°!Ð4Ñ4Ô4ˆõ Œz˜!‰}Œ}×Ò ¤Ô 3Ñ4Ô4ˆØ”,˜tÔ$ t§y¢y°°Q¸Ñ':Ô':Ñ:ˆØ—*’*˜Vœ\Ñ*Ô*ˆØœð 	,ð 	,‰HˆD�"Ø�d˜6 7¨DÑ1Ô1°GÑ;ˆGØ�b˜‘k”k GÑ+ˆGˆGà"×4Ò4°WÑ=Ô=Ðà�yŠyÐ,Ñ-Ô-Ð-r6   )rs   rt   ru   r%   rC   rl   rw   rx   s   @r4   r•  r•  Ç  sZ   ø€ € € € € ðQ˜|ð Qð Qð Qð Qð Qð Qð*.ð .ð .ð .ð .ð .ð .r6   r•  c                   ó¬   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej        df         dz  ed<   dZe
ej        df         dz  ed<   dS )ÚEvollaProteinEncoderModelOutputaN  
    sequence_compressor_output (`torch.FloatTensor` of shape `(batch_size, compressed_seq_len, hidden_size)`, *optional*):
        Compressed sequence representation produced by the sequence compressor module. The sequence length is
        reduced from the original input length to `compressed_seq_len` via learned compression.
    NÚsequence_compressor_outputr0  .rÜ   r@  )rs   rt   ru   rv   r­  r,   rì   r¨   r0  rÜ   rª   r@  r&  r6   r4   r¬  r¬  ñ  s•   € € € € € € ðð ð <@Ð Ô 1°DÑ 8Ð?Ð?Ñ?Ø26Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø:>€M�5˜Ô*¨CÐ/Ô0°4Ñ7Ð>Ð>Ñ>Ø7;€J��eÔ'¨Ð,Ô-°Ñ4Ð;Ð;Ñ;Ð;Ð;r6   r¬  c                   óX   ‡ — e Zd Zdefˆ fd„Zedej        dej        fd„¦   «         Z	ˆ xZ
S )ÚEvollaProteinEncoderrZ   c                 ó¦   •— t          ¦   «                              ¦   «          t          |j        ¬¦  «        | _        t          |¬¦  «        | _        d S rO  )rB   rC   rM  r˜  Úmodelr•  Úsequence_compressor_resamplerrX   s     €r4   rC   zEvollaProteinEncoder.__init__  sH   ø€ Ý‰Œ×ÒÑÔÐÝ/°vÔ7TÐUÑUÔUˆŒ
Ý-NÐV\Ð-]Ñ-]Ô-]ˆÔ*Ð*Ð*r6   r0   rf   c                 ó–   — |                       ||¬¦  «        }|j        }|                      ||¦  «        }t          ||j        ¬¦  «        S )Nr]  )r­  r0  )r±  r0  r²  r¬  )rY   r0   rf   r¾   Úprotein_outputÚprotein_embedsÚsequence_reprs          r4   rl   zEvollaProteinEncoder.forward  sT   € àŸš¨iÈ˜ÑWÔWˆØ'Ô9ˆØ×:Ò:¸>È>ÑZÔZˆå.Ø'4Ø,Ô>ð
ñ 
ô 
ð 	
r6   )rs   rt   ru   r%   rC   r   r,   Ú
LongTensorrì   rl   rw   rx   s   @r4   r¯  r¯     s€   ø€ € € € € ð^˜|ð ^ð ^ð ^ð ^ð ^ð ^ð
 ð
 Ô!1ð 
À5ÔCTð 
ð 
ð 
ñ Ôð
ð 
ð 
ð 
ð 
r6   r¯  c                   ób   ‡ — e Zd Z	 	 	 ddedz  dedz  dedz  fˆ fd„Zd„ Z	 	 	 	 	 	 	 d	d„Zˆ xZS )
Ú#EvollaSequenceAlignerCrossAttentionNÚprotein_encoder_dimÚstructure_encoder_dimÚmsa_encoder_dimc                 óà  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        | j        dz  | _        t          | j        | j        z  ¦  «        | _        | j        | j        z  | _        |j        }|j	        }|j
        }t          j        | j        | j        ¦  «        | _        |�?t          j        || j        ¦  «        | _        t          j        || j        ¦  «        | _        nd | _        d | _        |�?t          j        || j        ¦  «        | _        t          j        || j        ¦  «        | _        nd | _        d | _        |�?t          j        || j        ¦  «        | _        t          j        || j        ¦  «        | _        nd | _        d | _        t)          | j        ¦  «        | _        t          j        |¦  «        | _        t          j        | j        | j        |¬¦  «        | _        t3          | j        |¦  «        | _        t          j        t9          j        dg¦  «        ¦  «        | _        t          j        t9          j        dg¦  «        ¦  «        | _        d S )NrÀ   rh  r]   ) rB   rC   rF   r�   rj  r+   rÔ   rÕ   Ú$aligner_attention_probs_dropout_probÚaligner_enable_biasÚaligner_ffn_multr   rÖ   rº   Úkey_proteinÚvalue_proteinÚkey_structureÚvalue_structureÚkey_msaÚ	value_msaÚEvollaRMSNormÚattention_normrM   rO   Úout_projrŠ  r©  r›  r,   ÚtensorÚgate_attentionÚgate_ffw)	rY   rZ   rº  r»  r¼  r×   Úenable_biasÚffn_multr[   s	           €r4   rC   z,EvollaSequenceAlignerCrossAttention.__init__  s
  ø€ õ 	‰Œ×ÒÑÔÐà!Ô-ˆÔØ#)Ô#=ˆÔ ØÔ-¨tÑ3ˆŒ
Ý#& tÔ'7¸$Ô:RÑ'RÑ#SÔ#SˆÔ Ø!Ô5¸Ô8PÑPˆÔà'-Ô'RÐ$ØÔ0ˆØÔ*ˆå”Y˜tÔ/°Ô1CÑDÔDˆŒ
ØÐ*Ý!œyÐ)<¸dÔ>PÑQÔQˆDÔÝ!#¤Ð+>ÀÔ@RÑ!SÔ!SˆDÔÐà#ˆDÔØ!%ˆDÔà Ð,Ý!#¤Ð+@À$ÔBTÑ!UÔ!UˆDÔÝ#%¤9Ð-BÀDÔDVÑ#WÔ#WˆDÔ Ð à!%ˆDÔØ#'ˆDÔ àÐ&Ýœ9 _°dÔ6HÑIÔIˆDŒLÝœY ¸Ô8JÑKÔKˆDŒNˆNàˆDŒLØ!ˆDŒNå+¨DÔ,<Ñ=Ô=ˆÔå”zÐ">Ñ?Ô?ˆŒåœ	 $Ô"2°DÔ4DÈ;ÐWÑWÔWˆŒå# DÔ$4°hÑ?Ô?ˆŒÝ œl­5¬<¸¸Ñ+>Ô+>Ñ?Ô?ˆÔÝœ¥U¤\°3°%Ñ%8Ô%8Ñ9Ô9ˆŒˆˆr6   c	                 óŽ  — |||g}	d„ |	D ¦   «         }	|	st          d¦  «        ‚t          j        |	d¬¦  «        }	|                      |¦  «        }
|                      |
¦  «        }
| j        �G| j        �@|                     |¦  «        }|                      |¦  «        }|                      |¦  «        }nd}d}| j        �G| j	        �@|                     |¦  «        }|                      |¦  «        }|  	                    |¦  «        }nd}d}| j
        �G| j        �@|                     |¦  «        }|  
                    |¦  «        }|                      |¦  «        }nd}d}|||g}d„ |D ¦   «         }t          j        |d¬¦  «        }|||g}d„ |D ¦   «         }t          j        |d¬¦  «        }|
                     ¦   «         dd…         | j        | j        fz   } |
j        |Ž                      d	d
dd¦  «        }
|                     ¦   «         dd…         | j        | j        fz   } |j        |Ž                      d	d
dd¦  «        }|                     ¦   «         dd…         | j        | j        fz   } |j        |Ž                      d	d
dd¦  «        }|
| j        z  }
|€St          j        |                     d	¦  «        |                     d¦  «        ¦  «                             |j        ¦  «        }|dd…ddd…df         |	dd…dddd…f         z  }t          j        |
|                     dd¦  «        ¦  «        }||                     dd¬¦  «                             ¦   «         z
  }|                     d|z
                       ¦   «         t          j        |j        ¦  «        j        ¦  «        } t;          j        d¬¦  «        |¦  «        }t          j        ||¦  «        }|                     d	d
dd¦  «                             ¦   «         }|                     ¦   «         dd…         | j         fz   } |j        |Ž }|  !                    |¦  «        }|S )zù
        query_states: text
        key_value_states: protein
        query_states: [bs, query_seq_len, dim]
        key_value_states: [bs, kv_seq_len, dim]
        query_attn_mask: [bs, query_seq_len]
        kv_attn_mask: [bs, kv_seq_len]
        c                 ó   — g | ]}|®|‘ŒS r¥   r&  ©r'  rÿ   s     r4   r(  zGEvollaSequenceAlignerCrossAttention.cross_attention.<locals>.<listcomp>\  s   € ÐAÐAÐA˜a°1°=˜°=°=°=r6   z=At least one modality should be provided for cross attention.r$   r(   Nc                 ó   — g | ]}|®|‘ŒS r¥   r&  rÑ  s     r4   r(  zGEvollaSequenceAlignerCrossAttention.cross_attention.<locals>.<listcomp>‚  s   € Ð;Ð;Ð;˜1¨Q¨]�Q¨]¨]¨]r6   c                 ó   — g | ]}|®|‘ŒS r¥   r&  rÑ  s     r4   r(  zGEvollaSequenceAlignerCrossAttention.cross_attention.<locals>.<listcomp>†  s   € Ð?Ð?Ð?˜Q°°�q°°°r6   r?   r   r‰   r   rt  Tru  )"rÓ   r,   rœ   rÈ  rº   rÁ  rÂ  rd   rÃ  rÄ  rÅ  rÆ  ro   r�   rÔ   râ   rx  rj  r^  rn   rÅ   r›   ry  rz  r_   r{  Úfinfore   Úminr   ÚSoftmaxrÈ   rÕ   rÉ  )rY   Úquery_statesÚprotein_key_value_statesÚstructure_key_value_statesÚmsa_key_value_statesÚquery_attn_maskÚprotein_kv_attn_maskÚstructure_kv_attn_maskÚmsa_kv_attn_maskÚkv_attn_maskrç   Úkey_layer_proteinÚvalue_layer_proteinÚkey_layer_structureÚvalue_layer_structureÚkey_layer_msaÚvalue_layer_msaré   rê   Únew_query_layer_shapeÚnew_key_layer_shapeÚnew_value_layer_shaperf   rÊ   Úattention_scoresÚattention_probsÚcontext_layerÚnew_context_layer_shapes                               r4   Úcross_attentionz3EvollaSequenceAlignerCrossAttention.cross_attentionF  sª  € ð* -Ð.DÐFVÐWˆØAÐA <ÐAÑAÔAˆØð 	^ÝÐ\Ñ]Ô]Ð]Ý”y °1Ð5Ñ5Ô5ˆà×)Ò)¨,Ñ7Ô7ˆð —j’j Ñ-Ô-ˆàÔÐ'¨DÔ,>Ð,JØ'?×'BÒ'BÀ<Ñ'PÔ'PÐ$Ø $× 0Ò 0Ð1IÑ JÔ JÐØ"&×"4Ò"4Ð5MÑ"NÔ"NÐÐà $ÐØ"&ÐàÔÐ)¨dÔ.BÐ.NØ)C×)FÒ)FÀ|Ñ)TÔ)TÐ&Ø"&×"4Ò"4Ð5OÑ"PÔ"PÐØ$(×$8Ò$8Ð9SÑ$TÔ$TÐ!Ð!à"&ÐØ$(Ð!àŒ<Ð#¨¬Ð(BØ#7×#:Ò#:¸<Ñ#HÔ#HÐ Ø ŸLšLÐ)=Ñ>Ô>ˆMØ"ŸnšnÐ-AÑBÔBˆOˆOà ˆMØ"ˆOà&Ð(;¸]ÐKˆ	Ø;Ð; 	Ð;Ñ;Ô;ˆ	Ý”I˜i¨QÐ/Ñ/Ô/ˆ	à*Ð,AÀ?ÐSˆØ?Ð? +Ð?Ñ?Ô?ˆÝ”i °Ð3Ñ3Ô3ˆà +× 0Ò 0Ñ 2Ô 2°3°B°3Ô 7ØÔ$ØÔ$ð;
ñ !
Ðð '�kÔ&Ð(=Ð>×FÒFÀqÈ!ÈQÐPQÑRÔRˆà'ŸnšnÑ.Ô.¨s°¨sÔ3ØÔ$ØÔ$ð7
ñ 
Ðð #�I”NÐ$7Ð8×@Ò@ÀÀAÀqÈ!ÑLÔLˆ	à +× 0Ò 0Ñ 2Ô 2°3°B°3Ô 7ØÔ$ØÔ$ð;
ñ !
Ðð '�kÔ&Ð(=Ð>×FÒFÀqÈ!ÈQÐPQÑRÔRˆà! D¤JÑ.ˆð Ð"Ý#œj¨×):Ò):¸1Ñ)=Ô)=¸|×?PÒ?PÐQRÑ?SÔ?SÑTÔT×WÒWÐXdÔXkÑlÔlˆOØ(¨¨¨¨D°!°!°!°TÐ)9Ô:¸\È!È!È!ÈTÐSWÐYZÐYZÐYZÐJZÔ=[Ñ[ˆå”| K°×1DÒ1DÀRÈÑ1LÔ1LÑMÔMˆØ# l×&7Ò&7¸BÈÐ&7Ñ&MÔ&M×&TÒ&TÑ&VÔ&VÑVˆØ'×3Ò3Ø�Ñ×%Ò%Ñ'Ô'­¬°\Ô5GÑ)HÔ)HÔ)Lñ
ô 
Ðð -�"œ*¨Ð,Ñ,Ô,Ð-=Ñ>Ô>ˆõ œ _°kÑBÔBˆà%×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔFˆØ"/×"4Ò"4Ñ"6Ô"6°s¸°sÔ";¸tÔ?QÐ>SÑ"SÐØ*˜Ô*Ð,CÐDˆàŸš mÑ4Ô4ˆàÐr6   c           
      ó8  — |�q|j         \  }}}|€ct          j        ||¦  «                             |	j        ¦  «        |	                     ||f¬¦  «        j        z                       |j        ¦  «        }nd }|�q|j         \  }}}|€ct          j        ||¦  «                             |	j        ¦  «        |
                     ||f¬¦  «        j        z                       |j        ¦  «        }nd }|�q|j         \  }}}|€ct          j        ||¦  «                             |	j        ¦  «        |                     ||f¬¦  «        j        z                       |j        ¦  «        }nd }|}|�|                     ¦   «         s,|�|                     ¦   «         s|�Š|                     ¦   «         rv|}|                      ||||||||¬¦  «        }t          j	        | j
        ¦  «        |z  }||z   }|}|                      |¦  «        t          j	        | j        ¦  «        z  }||z   }|S )N)ro   )r×  rØ  rÙ  rÚ  rÛ  rÜ  rÝ  rÞ  )rb   r,   r^  rd   rn   rT   ÚTÚanyrí  ÚtanhrË  r©  rÌ  )rY   r×  Úprotein_kv_statesÚstructure_kv_statesÚmsa_kv_statesrÛ  rÜ  rÝ  rÞ  Úprotein_batch_maskÚstructure_batch_maskÚmsa_batch_maskÚpast_key_valuesr�  Úprotein_kv_seq_lenr)   Ústructure_kv_seq_lenÚmsa_kv_seq_lenrÜ   Úresiduals                       r4   rl   z+EvollaSequenceAlignerCrossAttention.forward¶  s`  € ð Ð(Ø*;Ô*AÑ'ˆBÐ" CØ#Ð+å”J˜rÐ#5Ñ6Ô6×9Ò9Ð:LÔ:SÑTÔTØ(×/Ò/Ð6HÈ"Ð5MÐ/ÑNÔNÔPñQç’"Ð&Ô-Ñ.Ô.ð %øð
 $(Ð àÐ*Ø,?Ô,EÑ)ˆBÐ$ cØ%Ð-å”J˜rÐ#7Ñ8Ô8×;Ò;Ð<NÔ<UÑVÔVØ*×1Ò1Ð8LÈbÐ7QÐ1ÑRÔRÔTñUç’"Ð(Ô/Ñ0Ô0ð 'øð
 &*Ð"àÐ$Ø&3Ô&9Ñ#ˆB� ØÐ'å”J˜r >Ñ2Ô2×5Ò5Ð6HÔ6OÑPÔPØ$×+Ò+°.À"Ð1EÐ+ÑFÔFÔHñIç’"�]Ô)Ñ*Ô*ð !øð
  $ÐØ$ˆð Ð*Ð/C×/GÒ/GÑ/IÔ/IÐ*Ø#Ð/Ð4J×4NÒ4NÑ4PÔ4PÐ/ØÐ)Ð.>×.BÒ.BÑ.DÔ.DÐ)à$ˆHØ ×0Ò0Ø*Ø):Ø+>Ø%2Ø /Ø%9Ø'=Ø!1ð 1ñ 	ô 	ˆMõ "œJ tÔ':Ñ;Ô;¸mÑKˆMà$ }Ñ4ˆMà$ˆHØ ŸGšG MÑ2Ô2µU´ZÀÄÑ5NÔ5NÑNˆMØ$ }Ñ4ˆMàÐr6   r¦   )NNNNNNN)rs   rt   ru   r+   rC   rí  rl   rw   rx   s   @r4   r¹  r¹    s¾   ø€ € € € € ð +/Ø,0Ø&*ð1:ð 1:ð ! 4™Zð1:ð  # T™zð	1:ð
 ˜t™ð1:ð 1:ð 1:ð 1:ð 1:ð 1:ðfnð nð nðn "Ø#ØØØ!ØØðGð Gð Gð Gð Gð Gð Gð Gr6   r¹  ÚRMSNormc                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
rÇ  ç�íµ ÷Æ°>r;   r„   Nc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z<
        EvollaRMSNorm is equivalent to T5LayerNorm
        N)rB   rC   r   r›  r,   r^  ÚweightÚvariance_epsilon)rY   rF   r;   r[   s      €r4   rC   zEvollaRMSNorm.__init__  sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr6   rÜ   c                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )Nr‰   r?   T)rv  )	re   rd   r,   Úfloat32ÚpowÚmeanÚrsqrtr  r  )rY   rÜ   Úinput_dtypeÚvariances       r4   rl   zEvollaRMSNorm.forward
  s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r6   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)rª   r  rb   r  rV  s    r4   Ú
extra_reprzEvollaRMSNorm.extra_repr  s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr6   )rÿ  )
rs   rt   ru   rc   rC   r,   r§   rl   r  rw   rx   s   @r4   rÇ  rÇ     sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð Jr6   rÇ  c                   óÔ   ‡ — e Zd ZU ej        ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d         de
dz  ded	ef         fd
„¦   «         Z ej        ¦   «         ed„ ¦   «         ¦   «         Zˆ xZS )ÚEvollaRotaryEmbeddingr{   NrZ   c                 ó²  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        |                      d|d¬¦  «         |                      d|                     ¦   «         d¬¦  «         d S )Nr~   Údefaultr{   Fr@   Úoriginal_inv_freq)rB   rC   rS   Úmax_seq_len_cachedÚoriginal_max_seq_lenrZ   Úrope_parametersr~   r   r   r}   rQ   Úclone)rY   rZ   rn   Úrope_init_fnr{   r[   s        €r4   rC   zEvollaRotaryEmbedding.__init__  sÊ   ø€ Ý‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(à×Ò˜Z¨¸eÐÑDÔDÐDØ×ÒÐ0°(·.².Ñ2BÔ2BÈuÐÑUÔUÐUÐUÐUr6   rn   ztorch.devicerƒ   r„   r…   c                 óü   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «                             |t          j        ¬¦  «        |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        rŒ   r‡   Nrˆ   r   r‰   rŠ   r‹   )	r  rP   rF   r�   r,   rR   rŽ   rd   rc   r�   s          r4   r   z5EvollaRotaryEmbedding.compute_default_rope_parameters(  sŒ   € ð& Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐð Ø•U”\ ! S¨!µ5´;Ð?Ñ?Ô?×BÒBÈ&ÕX]ÔXcÐBÑdÔdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r6   c                 óN  — | j         d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  |                     ¦   «         |                     ¦   «         z   
                    dd¦  «        }t          j        ||fd¬	¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬
¦  «        |	                     |j        ¬
¦  «        fS )Nr   r?   r$   r“   r”   Fr•   r‰   r(   rŠ   )r{   rc   rT   rb   rd   rn   r˜   r™   rš   r    r›   r,   rœ   r�   r}   rž   re   )
rY   rŸ   r>   r¡   r¢   r–   r£   r¤   r�   rž   s
             r4   rl   zEvollaRotaryEmbedding.forwardF  s·  € ð !œM¨$°°°°4¨-Ô8×>Ò>Ñ@Ô@×GÒGÈÔHZÐ[\ÔH]Ð_aÐcdÑeÔe×hÒhÐijÔiqÑrÔrÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	5ð 	5Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃBE&Å&E*Å-E*r¥   r¦   )rs   rt   ru   r,   r§   r¨   r%   rC   r©   r   r+   rª   rc   r   r«   r   rl   rw   rx   s   @r4   r  r    sù   ø€ € € € € € ØŒlÐÐÑðVð V˜|ð Vð Vð Vð Vð Vð Vð  à&*Ø+/Ø"ð*ð *Ø˜tÑ#ð*à˜Ô(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ô	%ð	*ð *ð *ñ „\ð*ð: €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r6   r  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú	EvollaMLPc                 ó¶  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        t          j        | j        | j        |j        ¬¦  «        | _        t          j        | j        | j        |j        ¬¦  «        | _	        t          j        | j        | j        |j        ¬¦  «        | _
        t          |j                 | _        d S )Nrh  )rB   rC   rZ   rF   r  r   rÖ   Úmlp_biasÚ	gate_projÚup_projÚ	down_projr	   Ú
hidden_actÚact_fnrX   s     €r4   rC   zEvollaMLP.__init__W  s¯   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔÝœ 4Ô#3°TÔ5KÐRXÔRaÐbÑbÔbˆŒÝ”y Ô!1°4Ô3IÐPVÔP_Ð`Ñ`Ô`ˆŒÝœ 4Ô#9¸4Ô;KÐRXÔRaÐbÑbÔbˆŒÝ˜VÔ.Ô/ˆŒˆˆr6   c                 ó¨   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        }|S r¥   )r  r   r  r  )rY   rŸ   r  s      r4   rl   zEvollaMLP.forwarda  sA   € Ø—N’N 4§;¢;¨t¯~ª~¸aÑ/@Ô/@Ñ#AÔ#AÀDÇLÂLÐQRÁOÄOÑ#SÑTÔTˆ	ØÐr6   rö   rx   s   @r4   r  r  V  sG   ø€ € € € € ð0ð 0ð 0ð 0ð 0ðð ð ð ð ð ð r6   r  rÜ   rÉ   r„   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r$   N)rb   rT   rå   )rÜ   rÉ   ÚbatchÚnum_key_value_headsÚslenr‡   s         r4   rÄ   rÄ   f  s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr6   c                   óÎ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej        de	ej        ej        f         dz  dej        dz  d	e
dz  d
ee         de	ej        ej        f         fd„Zˆ xZS )ÚEvollaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrZ   rÙ   c                 ó®  •— t          ¦   «                              ¦   «          || _        || _        t	          |d|j        |j        z  ¦  «        | _        |j        |j        z  | _	        | j        dz  | _
        |j        | _        d| _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        | j        z  |j        |j        ¬¦  «        | _        d S )Nr‡   rÀ   Trh  )rB   rC   rZ   rÙ   rP   rF   r�   r‡   r$  rÁ   r½   Úattention_dropoutrÚ   r   rÖ   Úattention_biasÚq_projÚk_projÚv_projÚo_proj©rY   rZ   rÙ   r[   s      €r4   rC   zEvollaAttention.__init__v  sB  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!Ø”} dÑ*ˆŒØ!'Ô!9ˆÔØˆŒå”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒˆˆr6   NrÜ   rU   rf   rø  r¾   r„   c                 ó"  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|\  }}t          ||	||¦  «        \  }}	|�|                     |	|
| j	        ¦  «        \  }	}
t          j        | j        j        t          ¦  «        } || ||	|
|f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr?   r$   r‰   r]   rá   )rb   r‡   r+  râ   r›   r,  r-  r¸   ÚupdaterÙ   r   rã   rZ   rä   rÌ   rÃ   r)  r½   rå   rÈ   r.  )rY   rÜ   rU   rf   rø  r¾   rp   ræ   r×  Ú
key_statesÚvalue_statesr�   rž   rë   rË   rÊ   s                   r4   rl   zEvollaAttention.forward�  sÀ  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r6   r¦   )rs   rt   ru   rv   r%   r+   rC   r,   r§   rª   r
   r   r   rl   rw   rx   s   @r4   r'  r'  r  så   ø€ € € € € àGÐGð
˜|ð 
¸ð 
ð 
ð 
ð 
ð 
ð 
ð4 IMØ.2Ø(,ð&)ð &)à”|ð&)ð # 5¤<°´Ð#=Ô>ÀÑEð&)ð œ tÑ+ð	&)ð
  ™ð&)ð Ð+Ô,ð&)ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð&)ð &)ð &)ð &)ð &)ð &)ð &)ð &)r6   r'  c                   ó\  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 	 	 	 	 	 	 ddej        deej        ej        f         dz  dej        dz  d	ej	        dz  d
e
dz  dedz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        fd„Zˆ xZS )ÚEvollaDecoderLayerrZ   rÙ   c                 ó¼  •— t          ¦   «                              ¦   «          |j        | _        t          ||¬¦  «        | _        t          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          |j        |j        ¬¦  «        | _
        |dz   t          |j        |j        z  d¦  «        z  dk    rt          ||j        ¬¦  «        | _        d S d S )N©rZ   rÙ   r:   r$   r   )rº  )rB   rC   rF   r'  Ú	self_attnr  ÚmlprÇ  Úrms_norm_epsÚinput_layernormÚpost_attention_layernormÚmaxr+  Úaligner_num_add_layersr¹  Úadapterr/  s      €r4   rC   zEvollaDecoderLayer.__init__·  sÔ   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔå(°À)ÐLÑLÔLˆŒå˜VÑ$Ô$ˆŒÝ,¨VÔ-?ÀVÔEXÐYÑYÔYˆÔÝ(5°fÔ6HÈfÔNaÐ(bÑ(bÔ(bˆÔ%Ø˜‰M�S Ô!9¸VÔ=ZÑ!ZÐ\]Ñ^Ô^Ñ^ÐbcÒcÐcÝ>ØØ$*Ô$6ðñ ô ˆDŒLˆLˆLð dÐcr6   NFrÜ   rU   rf   r>   rø  Ú	use_cacherò  ró  rô  rõ  rö  r÷  rÛ  r„   c           
      ó(  — |}|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }t	          | d¦  «        r|                      ||||	||
||¬¦  «        }|S )N)rÜ   rf   r>   rø  r@  rU   r?  )r×  rò  ró  rô  rÛ  rõ  rö  r÷  r&  )r;  r8  r<  r9  rÒ   r?  )rY   rÜ   rU   rf   r>   rø  r@  rò  ró  rô  rõ  rö  r÷  rÛ  r¾   rü  rÿ   s                    r4   rl   zEvollaDecoderLayer.forwardÆ  sç   € ð" !ˆà×,Ò,¨]Ñ;Ô;ˆð *˜4œ>ð 
Ø'Ø)Ø%Ø+ØØ 3ð
ð 
ð ð
ð 
Ñˆ�qð ! =Ñ0ˆð !ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆå�4˜Ñ#Ô#ð 
	Ø ŸLšLØ*Ø"3Ø$7Ø+Ø /Ø#5Ø%9Ø-ð )ñ 	ô 	ˆMð Ðr6   )NNNNFNNNNNNN)rs   rt   ru   r%   r+   rC   r,   r§   rª   r·  r
   r{  rl   rw   rx   s   @r4   r5  r5  ¶  su  ø€ € € € € ð˜|ð ¸ð ð ð ð ð ð ð$ IMØ.2Ø04Ø(,Ø!&Ø15Ø37Ø-1Ø26Ø48Ø.2Ø/3ð3ð 3à”|ð3ð # 5¤<°´Ð#=Ô>ÀÑEð3ð œ tÑ+ð	3ð
 Ô&¨Ñ-ð3ð  ™ð3ð ˜$‘;ð3ð !œ<¨$Ñ.ð3ð #œ\¨DÑ0ð3ð ”| dÑ*ð3ð "œL¨4Ñ/ð3ð $œl¨TÑ1ð3ð œ tÑ+ð3ð œ¨Ñ,ð3ð  
Œð!3ð 3ð 3ð 3ð 3ð 3ð 3ð 3r6   r5  c                   óˆ   ‡ — e Zd ZU eed<   dZdZg d¢ZdgZdZ	dZ
dZdZdZeedœZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )	ÚEvollaPreTrainedModelrZ   r±  T)r5  r  r•  r¹  rø  F)rÜ   r@  c                 ó–  •— | j         j        }t          ¦   «                              |¦  «         t	          |t
          ¦  «        rRt          j        |j        ¦  «         t          j        |j	        ¦  «         t          j
        |j        j        ¦  «         d S t	          |t          ¦  «        rt          j        |j        d|¬¦  «         d S d S )Nr]   )r  Ústd)rZ   Úinitializer_rangerB   rC  r˜   r¹  rD  Úzeros_rË  rÌ  Úones_rÈ  r  r•  Únormal_r|  )rY   r¹   rE  r[   s      €r4   rC  z#EvollaPreTrainedModel._init_weights  sµ   ø€ àŒkÔ+ˆÝ‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕAÑBÔBð 	<ÝŒK˜Ô-Ñ.Ô.Ð.ÝŒK˜œÑ(Ô(Ð(ÝŒJ�vÔ,Ô3Ñ4Ô4Ð4Ð4Ð4Ý˜Õ AÑBÔBð 	<ÝŒL˜œ¨c°sÐ;Ñ;Ô;Ð;Ð;Ð;ð	<ð 	<r6   )rs   rt   ru   r%   r¨   Úbase_model_prefixÚsupports_gradient_checkpointingrF  Ú_skip_keys_device_placementrG  rH  rI  Ú_can_compile_fullgraphrJ  r5  r'  rK  r,   r«   rC  rw   rx   s   @r4   rC  rC  ü  s³   ø€ € € € € € àÐÐÑØÐØ&*Ð#ðð ð Ðð $5Ð"5ÐØ ÐØ€NØÐà!ÐØ"'Ðà+Ø%ðð Ðð
 €U„]�_„_ð<ð <ð <ð <ñ „_ð<ð <ð <ð <ð <r6   rC  c                   óh  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zeee		 	 	 	 	 	 	 	 	 	 	 	 dde
j        dz  de
j        dz  de
j        dz  d	edz  d
e
j        dz  dedz  de
j        dz  de
j        dz  de
j        dz  de
j        dz  de
j        dz  de
j        dz  deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚEvollaModelrZ   c                 ó&  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        | j        ‰j        | j        ¦  «        | _        t          ‰¬¦  «        | _
        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t%          ‰dd¦  «        | _        t)          ‰¬¦  «        | _        |                      ¦   «          d S )NrP  c                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS )r7  )r5  )r'  rÙ   rZ   s     €r4   r(  z(EvollaModel.__init__.<locals>.<listcomp>'  s@   ø€ ð ð ð ð
 õ	 #Ø!Ø'ðñ ô ðð ð r6   r:   r.  F)rB   rC   rG   r1   rE   r   rD   rF   Úembed_tokensr¯  Úprotein_encoderr)  r*  r+  r�  rÇ  r:  rŽ  rP   r.  r  Ú
rotary_embrS  rX   s    `€r4   rC   zEvollaModel.__init__   s  øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒÝœL¨¬¸&Ô:LÈdÔN^Ñ_Ô_ˆÔÝ3¸6ÐBÑBÔBˆÔÝ”mðð ð ð õ
 "' vÔ'?Ñ!@Ô!@ðñ ô ñ
ô 
ˆŒõ " &Ô"4¸&Ô:MÐNÑNÔNˆŒ	Ý&-¨fÐ6NÐPUÑ&VÔ&VˆÔ#Ý/°vÐ>Ñ>Ô>ˆŒØ�ŠÑÔÐÐÐr6   c                 ó   — | j         S r¥   ©rR  rV  s    r4   rW  z EvollaModel.get_input_embeddings5  s   € ØÔ Ð r6   c                 ó   — || _         d S r¥   rV  rY  s     r4   rZ  z EvollaModel.set_input_embeddings8  s   € Ø!ˆÔÐÐr6   Nr0   rf   r>   rø  rg   r@  Úprotein_input_idsÚprotein_attention_maskÚstructure_featsÚ	msa_featsrö  r÷  r„   c                 óâ  — |du |duz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }d}d}|�Q|�O|  
                    ||¬¦  «        }|j        }t          j        |j        d         |j        t
          j        ¬¦  «        }t          | j        |||¬	¦  «        }|}|                      ||¬
¦  «        }| j        D ]} ||f||||||	|
|||||dœ|¤Ž}Œ|                      |¦  «        }t%          ||¬¦  «        }|S )a;  
        protein_input_ids (torch.LongTensor):
            The input IDs for the protein sequence in structure-aware tokens. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
        protein_attention_mask (torch.Tensor):
            The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.
        structure_feats (torch.FloatTensor):
            The input IDs for purely structure-based features. Should be of shape `(batch_size, structure_seq_length, structure_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
        msa_feats (torch.FloatTensor):
            The input IDs for purely MSA-based features. Should be of shape `(batch_size, msa_seq_length, msa_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
        structure_batch_mask (torch.Tensor):
            The batch mask to decide which protein sequences are purely structure-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `structure_feats`. Dummpy input for now.
        msa_batch_mask (torch.Tensor):
            The batch mask to decide which protein sequences are purely MSA-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `msa_feats`. Dummpy input for now.
        Nz:You must specify exactly one of input_ids or inputs_embedsrP  r   r$   r\  r]  r‹   )rZ   rg   rf   rø  )r>   )rf   r>   rø  r@  rò  ró  rô  rõ  rö  r÷  rÛ  rU   )r0  rø  )rÓ   rR  r   rZ   Úget_seq_lengthr,   rR   rb   rn   r`   rS  r­  r^  r{  r   rT  r�  rŽ  r   )rY   r0   rf   r>   rø  rg   r@  rX  rY  rZ  r[  rö  r÷  r¾   Úpast_seen_tokensÚprotein_featsrõ  Úprotein_outputsÚcausal_maskrÜ   rU   Údecoder_layerrû   s                          r4   rl   zEvollaModel.forward;  s  € ðB ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLàˆØ!ÐàÐ(Ð-CÐ-OØ"×2Ò2Ø+Ø5ð 3ñ ô ˆOð ,ÔFˆMÝ!&¤Ø!Ô'¨Ô*Ø(Ô/Ý”jð"ñ "ô "Ðõ )Ø”;Ø'Ø)Ø+ð	
ñ 
ô 
ˆð &ˆØ"Ÿošo¨mÈ,˜oÑWÔWÐà!œ[ð 	ð 	ˆMØ)˜MØðà*Ø)Ø /Ø#Ø"/Ø$3Ø'Ø#5Ø%9Ø-Ø .Ø$7ðð ð ðð ˆMˆMð" Ÿ	š	 -Ñ0Ô0ˆå(Ø+Ø+ð
ñ 
ô 
ˆð ˆr6   )NNNNNNNNNNNN)rs   rt   ru   r%   rC   rW  rZ  r   r!   r#   r,   r·  r§   r
   rì   r{  rª   r   rl   rw   rx   s   @r4   rO  rO    s²  ø€ € € € € ð˜|ð ð ð ð ð ð ð*!ð !ð !ð"ð "ð "ð ØØð .2Ø.2Ø04Ø(,Ø26Ø!%Ø59Ø6:Ø48Ø.2Ø48Ø.2ð]ð ]àÔ# dÑ*ð]ð œ tÑ+ð]ð Ô&¨Ñ-ð	]ð
  ™ð]ð Ô(¨4Ñ/ð]ð ˜$‘;ð]ð !Ô+¨dÑ2ð]ð !&¤¨tÑ 3ð]ð Ô*¨TÑ1ð]ð Ô$ tÑ+ð]ð $œl¨TÑ1ð]ð œ tÑ+ð]ð 
Ð(Ñ	(ð]ð ]ð ]ñ „_ñ  Ôñ „^ð]ð ]ð ]ð ]ð ]r6   rO  c                   óú   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 ddej	        dz  dej
        dz  dej        dz  d	ej	        dz  d
ej	        dz  dej
        dz  dedz  deej
        z  fd„¦   «         ¦   «         Zˆ xZS )ÚEvollaForProteinText2Textc                 óþ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        | j        d¬¦  «        | _        |  	                    ¦   «          d S r�  )
rB   rC   rO  r±  rE   r   rÖ   rF   Úlm_headrS  rX   s     €r4   rC   z"EvollaForProteinText2Text.__init__Ÿ  sg   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý  Ñ(Ô(ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°T´_È5ÐQÑQÔQˆŒà�ŠÑÔÐÐÐr6   c                 ó4   — | j                              ¦   «         S r¥   )r±  rW  rV  s    r4   rW  z.EvollaForProteinText2Text.get_input_embeddings§  s   € ØŒz×.Ò.Ñ0Ô0Ð0r6   c                 ó6   — | j                              |¦  «        S r¥   )r±  rZ  rY  s     r4   rZ  z.EvollaForProteinText2Text.set_input_embeddingsª  s   € ØŒz×.Ò.¨uÑ5Ô5Ð5r6   Nr   r0   rf   rg   ÚlabelsrX  rY  r@  Úlogits_to_keepc	           
      óJ  —  | j         d||||||dœ|	¤Ž}
|
j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        d||| j        dœ|	¤Ž}t          |||
j	        |
j
        |
j        ¬¦  «        }|S )a,  
        protein_input_ids (torch.LongTensor):
            The input IDs for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
        protein_attention_mask (torch.Tensor):
            The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.

        Example:

        ```python
        >>> from transformers import EvollaProcessor, EvollaForProteinText2Text
        >>> model = EvollaForProteinText2Text.from_pretrained("westlake/Evolla-10B-hf")
        >>> processor = EvollaProcessor.from_pretrained("westlake/Evolla-10B-hf")

        >>> protein_information = {
            "aa_seq": "your amino acid sequence",
            "foldseek": "your foldseek sequence",
        }
        >>> question = "What is the function of this protein?"
        >>> message = [
            {"role": "system", "content": "You are an AI expert that can answer any questions about protein."},
            {"role": "user", "content": question},
        ]

        >>> inputs = processor(proteins=[protein_information], messages_list=[message], return_tensors="pt", padding="longest")
        >>> outputs = model.generate(**inputs)

        >>> print(processor.batch_decode(outputs, skip_special_tokens=True))
        ```)r0   rf   rg   rX  rY  r@  N)Úlogitsri  rE   )Úlossrl  rø  rÜ   r@  r&  )r±  r0  r˜   r+   Úslicerf  Úloss_functionrE   r   rø  rÜ   r@  )rY   r0   rf   rg   ri  rX  rY  r@  rj  r¾   ÚoutputsrÜ   Úslice_indicesrl  rm  Ú
lm_outputss                   r4   rl   z!EvollaForProteinText2Text.forward­  s÷   € ðT ,6¨4¬:ð ,
ØØ)Ø'Ø/Ø#9Øð,
ð ,
ð ð,
ð ,
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%Ði¨V¸FÈtÌÐiÐiÐbhÐiÐiˆDå+ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ˆ
ð Ðr6   )NNNNNNNr   )rs   rt   ru   rC   rW  rZ  r   r   r,   r·  r§   rì   r{  r+   rl   rw   rx   s   @r4   rd  rd  ž  s8  ø€ € € € € ðð ð ð ð ð1ð 1ð 1ð6ð 6ð 6ð Øð .2Ø.2Ø26Ø*.Ø59Ø6:Ø!%Ø-.ðBð BàÔ# dÑ*ðBð œ tÑ+ðBð Ô(¨4Ñ/ð	Bð
 Ô  4Ñ'ðBð !Ô+¨dÑ2ðBð !&¤¨tÑ 3ðBð ˜$‘;ðBð ˜eœlÑ*ðBð Bð Bñ „^ñ ÔðBð Bð Bð Bð Br6   rd  )rd  rO  rC  )r$   )Nr]   )\r  Úcollections.abcr   Údataclassesr   Útypingr   r,   r   Ú r   rD  Úactivationsr	   Úcache_utilsr
   r   Ú
generationr   Úintegrationsr   r   r   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr    r!   Úutils.output_capturingr"   r#   Úconfiguration_evollar%   r&   r5   ÚModuler8   rz   r¯   r¸   r§   rc   rÌ   rÎ   rî   rù   r  r  r  r  r#  r5  r=  rM  rd  rŠ  r•  r¬  r¯  r¹  rÇ  r  r  r+   rÄ   r'  r5  rC  rO  rd  Ú__all__r&  r6   r4   ú<module>r‡     sj  ðð* €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø fÐ fÐ fÐ fÐ fÐ fÐ fÐ fÐ fÐ fØ JÐ JÐ JÐ JÐ JÐ JÐ JÐ JØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð LÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ <Ð <Ð <Ð <Ð <Ð <Ð <Ð <ð4ð 4ð 4ð ^=ð ^=ð ^=ð ^=ð ^=˜RœYñ ^=ô ^=ð ^=ðBA<ð A<ð A<ð A<ð A< "¤)ñ A<ô A<ð A<ðH(ð (ð (ð ÐÐ*Ñ+Ô+ðBð Bð Bñ ,Ô+ðBð@ !Øð %ð  %ØŒIð %àŒ<ð %ð 
Œð %ð Œ<ð	 %ð
 ”L 4Ñ'ð %ð �T‰\ð %ð ð %ð Ð'Ô(ð %ð  %ð  %ð  %ðF ÐÐ)Ñ*Ô*ðK)ð K)ð K)ð K)ð K) ¤	ñ K)ô K)ñ +Ô*ðK)ð\
ð 
ð 
ð 
ð 
˜RœYñ 
ô 
ð 
ðð ð ð ð ˜BœIñ ô ð ð<;ð ;ð ;ðð ð ð ð ˜rœyñ ô ð ð
ð 
ð 
ð 
ð 
˜œñ 
ô 
ð 
ð7ð 7ð 7ð 7ð 7Ð2ñ 7ô 7ð 7ðtSð Sð Sð Sð S˜"œ)ñ Sô Sð SðDð ð ð ð ˜œñ ô ð ð ðCð Cð Cð Cð C /ñ Cô Cñ „ðCð01
ð 1
ð 1
ð 1
ð 1
Ð!<ñ 1
ô 1
ð 1
ðh7 ð 7 ð 7 ð 7 ð 7 ¨¬	ñ 7 ô 7 ð 7 ðtAð Að Að Að A˜œ	ñ Aô Að Að'.ð '.ð '.ð '.ð '.¨¬	ñ '.ô '.ð '.ðT Ø
ð
<ð 
<ð 
<ð 
<ð 
< kñ 
<ô 
<ñ „ñ „ð
<ð
ð 
ð 
ð 
ð 
˜2œ9ñ 
ô 
ð 
ð$kð kð kð kð k¨"¬)ñ kô kð kð\ Ð˜YÑ'Ô'ðJð Jð Jð Jð J�B”Iñ Jô Jñ (Ô'ðJð(><ð ><ð ><ð ><ð ><˜BœIñ ><ô ><ð ><ðBð ð ð ð �”	ñ ô ð ð 	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð ÐÐ)Ñ*Ô*ð@)ð @)ð @)ð @)ð @)�b”iñ @)ô @)ñ +Ô*ð@)ðFCð Cð Cð Cð CÐ3ñ Cô Cð CðL ð<ð <ð <ð <ð <˜Oñ <ô <ñ „ð<ðD|ð |ð |ð |ð |Ð'ñ |ô |ð |ð~Sð Sð Sð Sð SÐ 5°ñ Sô Sð Sðl PÐ
OÐ
O€€€r6   