§
    ‚ŠtjÑM ã                   ó¼  — d dl Z d dlZd dlmZmZ d dlmZ d dlmZ d dl	Z
d dlZd dlmZ d dlmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZ ddlmZ ddlmZmZ ddlm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+  ej,        e-¦  «        Z. ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z/d„ Z0d„ Z1dUde2ej3                 de4dej3        fd„Z5dej3        de6fd„Z7dej3        de2e6         fd„Z8d„ Z9 G d „ d!ej:        ¦  «        Z; G d"„ d#ej<        ¦  «        Z=ej>        j?        dVdej3        d%e6dej3        fd&„¦   «         Z@ G d'„ d(ej<        ¦  «        ZA G d)„ d*ej<        ¦  «        ZB G d+„ d,ej<        ¦  «        ZC G d-„ d.e¦  «        ZD G d/„ d0ej<        ¦  «        ZE G d1„ d2ej<        ¦  «        ZF G d3„ d4ej<        ¦  «        ZG G d5„ d6ej<        ¦  «        ZH G d7„ d8ej<        ¦  «        ZI G d9„ d:ej<        ¦  «        ZJ G d;„ d<¦  «        ZKdWd>„ZLd?„ ZM G d@„ dAej<        ¦  «        ZN G dB„ dCej<        ¦  «        ZO G dD„ dEej<        ¦  «        ZP G dF„ dGej<        ¦  «        ZQ G dH„ dIej<        ¦  «        ZR G dJ„ dKej<        ¦  «        ZS G dL„ dMej<        ¦  «        ZT G dN„ dOej<        ¦  «        ZU G dP„ dQej<        ¦  «        ZV edR¬¦  «         G dS„ dTe¦  «        ¦   «         ZWdTd.gZXdS )Xé    N)ÚCallableÚSequence)Ú	dataclass)Úpartial)Ú	LayerNormé   )Úinitialization)Úis_deepspeed_available)ÚModelOutput)ÚContextManagersÚauto_docstringÚlogging)Úmaybe_autocasté   )ÚEsmModelÚEsmPreTrainedModel)Ú	OFProteinÚRigidÚRotationÚatom14_to_atom37Úchunk_layerÚcompute_predicted_aligned_errorÚ
compute_tmÚ-frames_and_literature_positions_to_atom14_posÚmake_atom14_masksÚresidue_constantsÚto_pdbÚtorsion_angles_to_framesz8
    Output type of [`EsmForProteinFoldingOutput`].
    )Úcustom_introc                   óÆ  — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dZ
ej        dz  ed<   dZej        dz  ed<   dZej        dz  ed<   dZej        dz  ed	<   dZej        dz  ed
<   dZej        dz  ed<   dZej        dz  ed<   dZej        dz  ed<   dZej        dz  ed<   dZej        dz  ed<   dZej        dz  ed<   dZej        dz  ed<   dZej        dz  ed<   dZej        dz  ed<   dZej        dz  ed<   dZej        dz  ed<   dZej        dz  ed<   dZej        dz  ed<   dZej        dz  ed<   dZej        dz  ed<   dS )ÚEsmForProteinFoldingOutputa#
  
    frames (`torch.FloatTensor`):
        Output frames.
    sidechain_frames (`torch.FloatTensor`):
        Output sidechain frames.
    unnormalized_angles (`torch.FloatTensor`):
        Predicted unnormalized backbone and side chain torsion angles.
    angles (`torch.FloatTensor`):
        Predicted backbone and side chain torsion angles.
    positions (`torch.FloatTensor`):
        Predicted positions of the backbone and side chain atoms.
    states (`torch.FloatTensor`):
        Hidden states from the protein folding trunk.
    s_s (`torch.FloatTensor`):
        Per-residue embeddings derived by concatenating the hidden states of each layer of the ESM-2 LM stem.
    s_z (`torch.FloatTensor`):
        Pairwise residue embeddings.
    distogram_logits (`torch.FloatTensor`):
        Input logits to the distogram used to compute residue distances.
    lm_logits (`torch.FloatTensor`):
        Logits output by the ESM-2 protein language model stem.
    aatype (`torch.FloatTensor`):
        Input amino acids (AlphaFold2 indices).
    atom14_atom_exists (`torch.FloatTensor`):
        Whether each atom exists in the atom14 representation.
    residx_atom14_to_atom37 (`torch.FloatTensor`):
        Mapping between atoms in the atom14 and atom37 representations.
    residx_atom37_to_atom14 (`torch.FloatTensor`):
        Mapping between atoms in the atom37 and atom14 representations.
    atom37_atom_exists (`torch.FloatTensor`):
        Whether each atom exists in the atom37 representation.
    residue_index (`torch.FloatTensor`):
        The index of each residue in the protein chain. Unless internal padding tokens are used, this will just be
        a sequence of integers from 0 to `sequence_length`.
    lddt_head (`torch.FloatTensor`):
        Raw outputs from the lddt head used to compute plddt.
    plddt (`torch.FloatTensor`):
        Per-residue confidence scores. Regions of low confidence may indicate areas where the model's prediction is
        uncertain, or where the protein structure is disordered.
    ptm_logits (`torch.FloatTensor`):
        Raw logits used for computing ptm.
    ptm (`torch.FloatTensor`):
        TM-score output representing the model's high-level confidence in the overall structure.
    aligned_confidence_probs (`torch.FloatTensor`):
        Per-residue confidence scores for the aligned structure.
    predicted_aligned_error (`torch.FloatTensor`):
        Predicted error between the model's prediction and the ground truth.
    max_predicted_aligned_error (`torch.FloatTensor`):
        Per-sample maximum predicted error.
    NÚframesÚsidechain_framesÚunnormalized_anglesÚanglesÚ	positionsÚstatesÚs_sÚs_zÚdistogram_logitsÚ	lm_logitsÚaatypeÚatom14_atom_existsÚresidx_atom14_to_atom37Úresidx_atom37_to_atom14Úatom37_atom_existsÚresidue_indexÚ	lddt_headÚplddtÚ
ptm_logitsÚptmÚaligned_confidence_probsÚpredicted_aligned_errorÚmax_predicted_aligned_error)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r"   ÚtorchÚFloatTensorÚ__annotations__r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   © ó    úf/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/esm/modeling_esmfold.pyr!   r!   6   sQ  € € € € € € ð1ð 1ðf (,€FˆEÔ Ñ$Ð+Ð+Ñ+Ø15Ð�eÔ'¨$Ñ.Ð5Ð5Ñ5Ø48Ð˜Ô*¨TÑ1Ð8Ð8Ñ8Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø*.€IˆuÔ  4Ñ'Ð.Ð.Ñ.Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø$(€CˆÔ	˜TÑ	!Ð(Ð(Ñ(Ø$(€CˆÔ	˜TÑ	!Ð(Ð(Ñ(Ø15Ð�eÔ'¨$Ñ.Ð5Ð5Ñ5Ø*.€IˆuÔ  4Ñ'Ð.Ð.Ñ.Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø37Ð˜Ô)¨DÑ0Ð7Ð7Ñ7Ø8<Ð˜UÔ.°Ñ5Ð<Ð<Ñ<Ø8<Ð˜UÔ.°Ñ5Ð<Ð<Ñ<Ø37Ð˜Ô)¨DÑ0Ð7Ð7Ñ7Ø.2€M�5Ô$ tÑ+Ð2Ð2Ñ2Ø*.€IˆuÔ  4Ñ'Ð.Ð.Ñ.Ø&*€Eˆ5Ô˜tÑ#Ð*Ð*Ñ*Ø+/€J�Ô! DÑ(Ð/Ð/Ñ/Ø$(€CˆÔ	˜TÑ	!Ð(Ð(Ñ(Ø9=Ð˜eÔ/°$Ñ6Ð=Ð=Ñ=Ø8<Ð˜UÔ.°Ñ5Ð<Ð<Ñ<Ø<@Ð Ô!2°TÑ!9Ð@Ð@Ñ@Ð@Ð@rA   r!   c                 óz   — t          j        | ¦  «        }|t           j        k    }|ot          j        | ¦  «        }|S ©N)r=   Úget_autocast_dtypeÚfloat16Úis_autocast_enabled)Údevice_typeÚautocast_dtypeÚfp16_enableds      rB   Úis_fp16_enabledrK   ‰   s9   € åÔ-¨kÑ:Ô:€NØ!¥U¤]Ò2€LØÐJ¥EÔ$=¸kÑ$JÔ$J€LàÐrA   c                  ó€   — t          ¦   «         rdS 	 dd l} | j                             ¦   «         S # t          $ r Y dS w xY w)NFr   )r
   Ú	deepspeedÚutilsÚis_initializedÚ	Exception)rM   s    rB   Úis_deepspeed_initializedrQ   ’   s^   € ÝÑÔð 	Øˆuð	ØÐÐÐð ”?×1Ò1Ñ3Ô3Ð3øÝð 	ð 	ð 	Ø�5�5ð	øøøs   ’/ ¯
=¼=ÚsamplesÚpad_vÚreturnc                 óZ  — t          | ¦  «        dk    rt          j        ¦   «         S t          d„ | D ¦   «         ¦  «        dk    rt          dd„ | D ¦   «         › �¦  «        ‚t	          d„ | D ¦   «         ¦  «        \  }d„ t          d„ | D ¦   «         Ž D ¦   «         }t          j        t          | ¦  «        g|¢R | d         j        |d	œŽ}|                     |¦  «         t          t          | ¦  «        ¦  «        D ]3}||         }| |         }||t	          d
„ |j
        D ¦   «         ¦  «        <   Œ4|S )zõ
    Takes a list of tensors with the following dimensions:
        [(d_11, ..., d_1K),
         (d_21, ..., d_2K), ..., (d_N1, ..., d_NK)]
    and stack + pads them into a single tensor of:
    (N, max_i=1,N { d_i1 }, ..., max_i=1,N {diK})
    r   c                 ó6   — h | ]}|                      ¦   «         ’ŒS r@   ©Údim©Ú.0Úxs     rB   ú	<setcomp>z(collate_dense_tensors.<locals>.<setcomp>©   s    € Ð%Ð%Ð%˜ˆA�EŠE‰GŒGÐ%Ð%Ð%rA   r   z Samples has varying dimensions: c                 ó6   — g | ]}|                      ¦   «         ‘ŒS r@   rW   rY   s     rB   ú
<listcomp>z)collate_dense_tensors.<locals>.<listcomp>ª   s    € Ð>XÐ>XÐ>XÈ1¸q¿uºu¹w¼wÐ>XÐ>XÐ>XrA   c                 ó   — h | ]	}|j         ’Œ
S r@   ©ÚdevicerY   s     rB   r\   z(collate_dense_tensors.<locals>.<setcomp>«   s   € Ð1Ð1Ð1 A�q”xÐ1Ð1Ð1rA   c                 ó,   — g | ]}t          |¦  «        ‘ŒS r@   )Úmax)rZ   Úlsts     rB   r^   z)collate_dense_tensors.<locals>.<listcomp>¬   s   € ÐFÐFÐF˜c•�S‘”ÐFÐFÐFrA   c                 ó   — g | ]	}|j         ‘Œ
S r@   ©ÚshaperY   s     rB   r^   z)collate_dense_tensors.<locals>.<listcomp>¬   s   € Ð*DÐ*DÐ*D°q¨1¬7Ð*DÐ*DÐ*DrA   )Údtypera   c              3   ó6   K  — | ]}t          d |¦  «        V — ŒdS )r   N©Úslice)rZ   Úks     rB   ú	<genexpr>z(collate_dense_tensors.<locals>.<genexpr>²   s*   è è € Ð4Ð4 q•u˜Q ‘{”{Ð4Ð4Ð4Ð4Ð4Ð4rA   )Úlenr=   ÚTensorÚRuntimeErrorÚtupleÚzipÚemptyrh   Úfill_Úrangerg   )rR   rS   ra   Ú	max_shapeÚresultÚiÚresult_iÚts           rB   Úcollate_dense_tensorsr{   Ÿ   sJ  € õ ˆ7�|„|�qÒÐÝŒ|‰~Œ~ÐÝ
Ð%Ð%˜WÐ%Ñ%Ô%Ñ&Ô&¨!Ò+Ð+ÝÐZÐ>XÐ>XÐPWÐ>XÑ>XÔ>XÐZÐZÑ[Ô[Ð[ÝÐ1Ð1¨Ð1Ñ1Ô1Ñ2Ô2�I€VØFÐF¥SÐ*DÐ*D¸GÐ*DÑ*DÔ*DÐ%EÐFÑFÔF€IÝŒ[�˜W™œÐY¨	ÐYÐY¸À¼Ô9IÐRXÐYÐYÐY€FØ
‡L‚L�ÑÔÐÝ•3�w‘<”<Ñ Ô ð :ð :ˆØ˜!”9ˆØ�AŒJˆØ89ˆ•Ð4Ð4¨A¬GÐ4Ñ4Ô4Ñ4Ô4Ñ5Ð5Ø€MrA   rz   Úno_dimsc                 óN   — |                       | j        d | …         dz   ¦  «        S )N©éÿÿÿÿ)Úreshaperg   )rz   r|   s     rB   Úflatten_final_dimsr�   ¶   s&   € Ø�9Š9�Q”W˜Y˜w˜h˜YÔ'¨%Ñ/Ñ0Ô0Ð0rA   ÚtensorÚindsc           	      óÜ   ‡— dt          |¦  «        z  Št          t          t          | j        d ‰…         ¦  «        ¦  «        ¦  «        }|                      |ˆfd„|D ¦   «         z   ¦  «        S )Nr   c                 ó   •— g | ]}‰|z   ‘ŒS r@   r@   )rZ   rx   Ú
zero_indexs     €rB   r^   z&permute_final_dims.<locals>.<listcomp>½   s   ø€ Ð'EÐ'EÐ'E¸1¨
°Q©Ð'EÐ'EÐ'ErA   )rn   Úlistru   rg   Úpermute)r‚   rƒ   Ú
first_indsr†   s      @rB   Úpermute_final_dimsrŠ   º   sf   ø€ Ø•c˜$‘i”i‘€JÝ•e�C ¤¨[¨j¨[Ô 9Ñ:Ô:Ñ;Ô;Ñ<Ô<€JØ�>Š>˜*Ð'EÐ'EÐ'EÐ'EÀÐ'EÑ'EÔ'EÑEÑFÔFÐFrA   c                 óÚ   ‡— |d         }i }|                      ¦   «         D ]J\  Š}ˆfd„|D ¦   «         }t          |t          ¦  «        rt          | |¦  «        |‰<   Œ< | |¦  «        |‰<   ŒK|S )Nr   c                 ó    •— g | ]
}|‰         ‘ŒS r@   r@   )rZ   Údrl   s     €rB   r^   z!dict_multimap.<locals>.<listcomp>Ä   s   ø€ Ð%Ð%Ð%˜!��1”Ð%Ð%Ð%rA   )ÚitemsÚ
isinstanceÚdictÚdict_multimap)ÚfnÚdictsÚfirstÚnew_dictÚvÚall_vrl   s         @rB   r‘   r‘   À   s„   ø€ Ø�!ŒH€EØ€HØ—’‘”ð $ð $‰ˆˆ1Ø%Ð%Ð%Ð%˜uÐ%Ñ%Ô%ˆÝ�a�ÑÔð 	$Ý'¨¨EÑ2Ô2ˆH�Q‰KˆKà˜"˜U™)œ)ˆH�Q‰KˆKà€OrA   c                   ón   ‡ — e Zd ZdZ	 	 	 ddedededed	eej	        ej	        gdf         dz  f
ˆ fd
„Z
ˆ xZS )ÚEsmFoldLinearz¾
    A Linear layer with built-in nonstandard initializations. Called just like torch.nn.Linear.

    Implements the initializers in 1.11.4, plus some additional ones found in the code.
    TÚdefaultNÚin_dimÚout_dimÚbiasÚinitÚinit_fnc                 ó   •— t          ¦   «                              |||¬¦  «         |rEt          j        ¦   «         5  | j                             d¦  «         ddd¦  «         n# 1 swxY w Y   || _        || _        |dvrt          d¦  «        ‚dS )aM  
        Args:
            in_dim:
                The final dimension of inputs to the layer
            out_dim:
                The final dimension of layer outputs
            bias:
                Whether to learn an additive bias. True by default
            init:
                The initializer to use. Choose from:

                "default": LeCun fan-in truncated normal initialization "relu": He initialization w/ truncated normal
                distribution "glorot": Fan-average Glorot uniform initialization "gating": Weights=0, Bias=1 "normal":
                Normal initialization with std=1/sqrt(fan_in) "final": Weights=0, Bias=0

                Overridden by init_fn if the latter is not None.
            init_fn:
                A custom initializer taking weight and bias as inputs. Overrides init if not None.
        ©r�   r   N)rš   ÚreluÚglorotÚgatingÚnormalÚfinalzInvalid init string.)	ÚsuperÚ__init__r=   Úno_gradr�   rt   rž   rŸ   Ú
ValueError)Úselfr›   rœ   r�   rž   rŸ   Ú	__class__s         €rB   r¨   zEsmFoldLinear.__init__Ô   sÓ   ø€ õ6 	‰Œ×Ò˜ ¨tÐÑ4Ô4Ð4àð 	#Ý”‘”ð #ð #Ø”	—’ Ñ"Ô"Ð"ð#ð #ð #ñ #ô #ð #ð #ð #ð #ð #ð #øøøð #ð #ð #ð #àˆŒ	ØˆŒàÐQÐQÐQÝÐ3Ñ4Ô4Ð4ð RÐQs   »A"Á"A&Á)A&)Trš   N)r9   r:   r;   r<   ÚintÚboolÚstrr   r=   ro   r¨   Ú__classcell__©r¬   s   @rB   r™   r™   Í   sŸ   ø€ € € € € ðð ð ØØGKð$5ð $5àð$5ð ð$5ð ð	$5ð
 ð$5ð ˜5œ<¨¬Ð6¸Ð<Ô=ÀÑDð$5ð $5ð $5ð $5ð $5ð $5ð $5ð $5ð $5ð $5rA   r™   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚEsmFoldLayerNormçñhãˆµøä>c                 ó  •— t          ¦   «                              ¦   «          |f| _        || _        t	          j        t          j        |¦  «        ¦  «        | _        t	          j        t          j	        |¦  «        ¦  «        | _
        d S rD   )r§   r¨   Úc_inÚepsÚnnÚ	Parameterr=   ÚonesÚweightÚzerosr�   )r«   r¶   r·   r¬   s      €rB   r¨   zEsmFoldLayerNorm.__init__ü   sc   ø€ Ý‰Œ×ÒÑÔÐà�GˆŒ	ØˆŒå”l¥5¤:¨dÑ#3Ô#3Ñ4Ô4ˆŒÝ”L¥¤¨TÑ!2Ô!2Ñ3Ô3ˆŒ	ˆ	ˆ	rA   c           	      óÌ  — |j         }|t          j        u r—t          ¦   «         s‰t	          dd¬¦  «        5  t
          j                             || j        | j	         
                    |¬¦  «        | j         
                    |¬¦  «        | j        ¦  «        }d d d ¦  «         n# 1 swxY w Y   n7t
          j                             || j        | j	        | j        | j        ¦  «        }|S )NÚcudaF©rH   Úenabled©rh   )rh   r=   Úbfloat16rQ   r   r¸   Ú
functionalÚ
layer_normr¶   r»   Útor�   r·   )r«   r[   r�   Úouts       rB   ÚforwardzEsmFoldLayerNorm.forward  s  € ØŒGˆØ•”ÐÐÕ'?Ñ'AÔ'AÐÝ¨F¸EÐBÑBÔBð wð wÝ”m×.Ò.¨q°$´)¸T¼[¿^º^ÐRS¸^Ñ=TÔ=TÐVZÔV_×VbÒVbÐijÐVbÑVkÔVkÐmqÔmuÑvÔv�ðwð wð wñ wô wð wð wð wð wð wð wøøøð wð wð wð wøõ ”-×*Ò*¨1¨d¬i¸¼ÀdÄiÐQUÔQYÑZÔZˆCàˆ
s   µA B!Â!B%Â(B%)r´   ©r9   r:   r;   r¨   rÇ   r°   r±   s   @rB   r³   r³   û   sL   ø€ € € € € ð4ð 4ð 4ð 4ð 4ð 4ðð ð ð ð ð ð rA   r³   r   rX   c                 ó8  — | j         }|t          j        u r^t          ¦   «         sPt	          dd¬¦  «        5  t          j        j                             | |¬¦  «        }ddd¦  «         n# 1 swxY w Y   n&t          j        j                             | |¬¦  «        }|S )z[
    Softmax, but without automatic casting to fp32 when the input is of type bfloat16
    r¾   Fr¿   rW   N)rh   r=   rÂ   rQ   r   r¸   rÃ   Úsoftmax)rz   rX   r�   Úss       rB   Úsoftmax_no_castrÌ     sÎ   € ð
 	
Œ€AØ�EŒNÐÐÕ#;Ñ#=Ô#=ÐÝ¨¸Ð>Ñ>Ô>ð 	8ð 	8Ý”Ô#×+Ò+¨A°3Ð+Ñ7Ô7ˆAð	8ð 	8ð 	8ñ 	8ô 	8ð 	8ð 	8ð 	8ð 	8ð 	8ð 	8øøøð 	8ð 	8ð 	8ð 	8øõ ŒHÔ×'Ò'¨¨sÐ'Ñ3Ô3ˆà€Hs   µ'A(Á(A,Á/A,c                   óT  ‡ — e Zd ZdZ	 ddedededededefˆ fd	„Zd
ej        dej        de	ej        ej        ej        f         fd„Z
dej        d
ej        dej        fd„Z	 	 	 	 	 	 	 dd
ej        dej        deej                 dz  dedededededej        dz  dej        fd„Zˆ xZS )ÚEsmFoldAttentionzu
    Standard multi-head attention using AlphaFold's default layer initialization. Allows multiple bias vectors.
    TÚc_qÚc_kÚc_vÚc_hiddenÚno_headsr¤   c                 óˆ  •— t          ¦   «                              ¦   «          || _        || _        || _        || _        || _        || _        t          | j        | j        | j        z  dd¬¦  «        | _	        t          | j        | j        | j        z  dd¬¦  «        | _
        t          | j        | j        | j        z  dd¬¦  «        | _        t          | j        | j        z  | j        d¬¦  «        | _        d| _        | j        r)t          | j        | j        | j        z  d¬¦  «        | _        t          j        ¦   «         | _        dS )aª  
        Args:
            c_q:
                Input dimension of query data
            c_k:
                Input dimension of key data
            c_v:
                Input dimension of value data
            c_hidden:
                Per-head hidden dimension
            no_heads:
                Number of attention heads
            gating:
                Whether the output should be gated using query data
        Fr£   ©r�   rž   r¦   ©rž   Nr¤   )r§   r¨   rÏ   rÐ   rÑ   rÒ   rÓ   r¤   r™   Úlinear_qÚlinear_kÚlinear_vÚlinear_oÚlinear_gr¸   ÚSigmoidÚsigmoid)r«   rÏ   rÐ   rÑ   rÒ   rÓ   r¤   r¬   s          €rB   r¨   zEsmFoldAttention.__init__$  s  ø€ õ0 	‰Œ×ÒÑÔÐàˆŒØˆŒØˆŒØ ˆŒØ ˆŒØˆŒõ
 & d¤h°´ÀÄÑ0MÐTYÐ`hÐiÑiÔiˆŒÝ% d¤h°´ÀÄÑ0MÐTYÐ`hÐiÑiÔiˆŒÝ% d¤h°´ÀÄÑ0MÐTYÐ`hÐiÑiÔiˆŒÝ% d¤m°d´mÑ&CÀTÄXÐT[Ð\Ñ\Ô\ˆŒàˆŒØŒ;ð 	bÝ)¨$¬(°D´MÀDÄMÑ4QÐX`ÐaÑaÔaˆDŒMå”z‘|”|ˆŒˆˆrA   Úq_xÚkv_xrT   c                 óN  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                     |j        d d…         | j        dfz   ¦  «        }|                     |j        d d…         | j        dfz   ¦  «        }|                     |j        d d…         | j        dfz   ¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|t          j        | j	        ¦  «        z  }|||fS )Nr   éþÿÿÿéýÿÿÿ)
r×   rØ   rÙ   Úviewrg   rÓ   Ú	transposeÚmathÚsqrtrÒ   )r«   rÞ   rß   Úqrl   r–   s         rB   Ú	_prep_qkvzEsmFoldAttention._prep_qkvS  s  € à�MŠM˜#ÑÔˆØ�MŠM˜$ÑÔˆØ�MŠM˜$ÑÔˆð �FŠF�1”7˜3˜B˜3”< 4¤=°"Ð"5Ñ5Ñ6Ô6ˆØ�FŠF�1”7˜3˜B˜3”< 4¤=°"Ð"5Ñ5Ñ6Ô6ˆØ�FŠF�1”7˜3˜B˜3”< 4¤=°"Ð"5Ñ5Ñ6Ô6ˆð �KŠK˜˜BÑÔˆØ�KŠK˜˜BÑÔˆØ�KŠK˜˜BÑÔˆà	�TŒY�t”}Ñ%Ô%Ñ%ˆà�!�QˆwˆrA   Úoc                 ó  — | j         �Y|                      |                       |¦  «        ¦  «        }|                     |j        d d…         | j        dfz   ¦  «        }||z  }t          |d¦  «        }|                      |¦  «        }|S )Nr   é   )rÛ   rÝ   rã   rg   rÓ   r�   rÚ   )r«   ré   rÞ   Úgs       rB   Ú_wrap_upzEsmFoldAttention._wrap_upg  s‚   € ØŒ=Ð$Ø—’˜TŸ]š]¨3Ñ/Ô/Ñ0Ô0ˆAð —’�q”w˜s ˜s”| t¤}°bÐ&9Ñ9Ñ:Ô:ˆAØ�A‘ˆAõ ˜q !Ñ$Ô$ˆð �MŠM˜!ÑÔˆàˆrA   NFé   é   ÚbiasesÚuse_memory_efficient_kernelÚuse_lmaÚlma_q_chunk_sizeÚlma_kv_chunk_sizeÚ	use_flashÚ
flash_maskc
                 óà  — |r|�|€t          d¦  «        ‚|r|�t          d¦  «        ‚|||g}
t          |
¦  «        dk    rt          d¦  «        ‚|€g }|                      ||¦  «        \  }}}t          |d¦  «        }t	          j        ||¦  «        }|D ]}||z  }Œt          |d¦  «        }t	          j        ||¦  «        }|                     dd	¦  «        }|                      ||¦  «        }|S )
a{  
        Args:
            q_x:
                [*, Q, C_q] query data
            kv_x:
                [*, K, C_k] key data
            biases:
                List of biases that broadcast to [*, H, Q, K]
            use_memory_efficient_kernel:
                Whether to use a custom memory-efficient attention kernel. This should be the default choice for most.
                If none of the "use_<...>" flags are True, a stock PyTorch implementation is used instead
            use_lma:
                Whether to use low-memory attention (Staats & Rabe 2021). If none of the "use_<...>" flags are True, a
                stock PyTorch implementation is used instead
            lma_q_chunk_size:
                Query chunk size (for LMA)
            lma_kv_chunk_size:
                Key/Value chunk size (for LMA)
        Returns
            [*, Q, C_q] attention update
        NzPIf use_lma is specified, lma_q_chunk_size and lma_kv_chunk_size must be providedzSuse_flash is incompatible with the bias option. For masking, use flash_mask insteadr   z2Choose at most one alternative attention algorithm)r   r   r   rá   râ   )	rª   Úsumrè   rŠ   r=   ÚmatmulrÌ   rä   rí   )r«   rÞ   rß   rð   rñ   rò   ró   rô   rõ   rö   Úattn_optionsÚqueryÚkeyÚvalueÚoutputÚbs                   rB   rÇ   zEsmFoldAttention.forwardw  s%  € ðB ð 	qÐ(Ð0Ð4EÐ4MÝÐoÑpÔpÐpàð 	t˜Ð+ÝÐrÑsÔsÐsà3°W¸iÐHˆÝˆ|ÑÔ˜qÒ Ð ÝÐQÑRÔRÐRàˆ>ØˆFð !ŸNšN¨3°Ñ5Ô5Ñˆˆs�EÝ   fÑ-Ô-ˆõ ”˜e SÑ)Ô)ˆØð 	ð 	ˆAØ�a‰KˆFˆFÝ  ¨Ñ,Ô,ˆõ ”˜f eÑ,Ô,ˆØ×!Ò! " bÑ)Ô)ˆØ—’˜v sÑ+Ô+ˆàˆrA   ©T)NFFrî   rï   FN)r9   r:   r;   r<   r­   r®   r¨   r=   ro   rq   rè   rí   r‡   rÇ   r°   r±   s   @rB   rÎ   rÎ     s©  ø€ € € € € ðð ð ð-$ð -$àð-$ð ð-$ð ð	-$ð
 ð-$ð ð-$ð ð-$ð -$ð -$ð -$ð -$ð -$ð^˜Uœ\ð °´ð À%ÈÌÐV[ÔVbÐdiÔdpÐHpÔBqð ð ð ð ð(˜%œ,ð ¨U¬\ð ¸e¼lð ð ð ð ð( -1Ø,1ØØ $Ø!%ØØ*.ð=ð =àŒ\ð=ð Œlð=ð �U”\Ô" TÑ)ð	=ð
 &*ð=ð ð=ð ð=ð ð=ð ð=ð ”L 4Ñ'ð=ð 
Œð=ð =ð =ð =ð =ð =ð =ð =rA   rÎ   c                   óô   ‡ — e Zd Zdˆ fd„	Zej        j        	 	 	 ddej        deej                 de	de
d	e
d
e
dej        fd„¦   «         Z	 	 	 	 	 ddej        dej        dz  de	dz  de
d	e
d
e
dej        fd„Zˆ xZS )ÚEsmFoldTriangleAttentionTç    eÍÍAc                 ó\  •— t          ¦   «                              ¦   «          || _        || _        || _        || _        || _        t          | j        ¦  «        | _        t          || j        dd¬¦  «        | _
        t          | j        | j        | j        | j        | j        ¦  «        | _        dS )zç
        Args:
            c_in:
                Input channel dimension
            c_hidden:
                Overall hidden channel dimension (not per-head)
            no_heads:
                Number of attention heads
        Fr¥   rÕ   N)r§   r¨   r¶   rÒ   rÓ   ÚstartingÚinfr   rÄ   r™   ÚlinearrÎ   Úmha)r«   r¶   rÒ   rÓ   r  r  r¬   s         €rB   r¨   z!EsmFoldTriangleAttention.__init__¸  s�   ø€ õ 	‰Œ×ÒÑÔÐàˆŒ	Ø ˆŒØ ˆŒØ ˆŒØˆŒå# D¤IÑ.Ô.ˆŒå# D¨$¬-¸eÈ(ÐSÑSÔSˆŒå# D¤I¨t¬y¸$¼)ÀTÄ]ÐTXÔTaÑbÔbˆŒˆˆrA   Fr[   rð   Ú
chunk_sizerñ   rò   Úinplace_saferT   c           
      óœ   — |||dœ}t          t          | j        ||¬¦  «        ||t          |j        dd…         ¦  «        |r|nd¬¦  «        S )ztriangle! triangle!)rÞ   rß   rð   )rñ   rò   Nrá   )r	  Úno_batch_dimsÚ_out)r   r   r  rn   rg   )r«   r[   rð   r	  rñ   rò   r
  Ú
mha_inputss           rB   Ú_chunkzEsmFoldTriangleAttention._chunkÐ  so   € ð ØØð
ð 
ˆ
õ Ý�D”HÐ:UÐ_fÐgÑgÔgØØ!Ý˜aœg c r cœlÑ+Ô+Ø"Ð,��¨ð
ñ 
ô 
ð 	
rA   NÚmaskc                 ó8  — |€"|                      |j        dd…         ¦  «        }| j        s,|                     dd¦  «        }|                     dd¦  «        }|                      |¦  «        }| j        |dz
  z  ddd…dddd…f         }t          |                      |¦  «        d¦  «        }|                     d¦  «        }||g}	|�|  	                    ||	||||¬	¦  «        }n|  
                    |||	||¬
¦  «        }| j        s|                     dd¦  «        }|S )z­
        Args:
            x:
                [*, I, J, C_in] input tensor (e.g. the pair representation)
        Returns:
            [*, I, J, C_in] output tensor
        Nr   rá   râ   r   .©rë   r   r   éüÿÿÿ)rñ   rò   r
  )rÞ   rß   rð   rñ   rò   )Únew_onesrg   r  rä   rÄ   r  rŠ   r  Ú	unsqueezer  r  )
r«   r[   r  r	  rñ   rò   r
  Ú	mask_biasÚtriangle_biasrð   s
             rB   rÇ   z EsmFoldTriangleAttention.forwardé  sO  € ð  ˆ<à—:’:Ø”˜˜˜”ñô ˆDð Œ}ð 	*Ø—’˜B Ñ#Ô#ˆAØ—>’> " bÑ)Ô)ˆDð �OŠO˜AÑÔˆð ”X ¨¡Ñ*¨C°°°°D¸$ÀÀÀÐ,AÔBˆ	õ +¨4¯;ª;°q©>¬>¸9ÑEÔEˆð &×/Ò/°Ñ3Ô3ˆà˜]Ð+ˆàÐ!Ø—’ØØØØ,GØØ)ð ñ ô ˆAˆAð —’Ø˜A fÐJeÐovð ñ ô ˆAð Œ}ð 	$Ø—’˜B Ñ#Ô#ˆAàˆrA   )Tr  )FFF)NNFFF)r9   r:   r;   r¨   r=   ÚjitÚignorero   r‡   r­   r®   r  rÇ   r°   r±   s   @rB   r  r  ·  sE  ø€ € € € € ðcð cð cð cð cð cð0 „YÔð -2ØØ"ð
ð 
àŒ<ð
ð �U”\Ô"ð
ð ð	
ð
 &*ð
ð ð
ð ð
ð 
Œð
ð 
ð 
ñ Ôð
ð6 %)Ø!%Ø,1ØØ"ð9ð 9àŒ<ð9ð Œl˜TÑ!ð9ð ˜$‘Jð	9ð
 &*ð9ð ð9ð ð9ð 
Œð9ð 9ð 9ð 9ð 9ð 9ð 9ð 9rA   r  c                   óø   ‡ — e Zd ZdZdˆ fd„	Z	 ddej        dej        dedz  dej        fd	„Z	 	 	 dd
ej        dej        dz  dedz  de	fd„Z
	 	 	 	 dd
ej        dej        dz  de	de	dedz  dej        fd„Zˆ xZS )Ú#EsmFoldTriangleMultiplicativeUpdatez*
    Implements Algorithms 11 and 12.
    Tc                 óð  •— t          ¦   «                              ¦   «          |j        }|| _        t	          ||¦  «        | _        t	          ||d¬¦  «        | _        t	          ||¦  «        | _        t	          ||d¬¦  «        | _        t	          ||d¬¦  «        | _	        t	          ||d¬¦  «        | _
        t          |¦  «        | _        t          |¦  «        | _        t          j        ¦   «         | _        d S )Nr¤   rÖ   r¦   )r§   r¨   Úpairwise_state_dimÚ	_outgoingr™   Ú
linear_a_pÚ
linear_a_gÚ
linear_b_pÚ
linear_b_grÛ   Úlinear_zr   Úlayer_norm_inÚlayer_norm_outr¸   rÜ   rÝ   )r«   Úconfigr  rÒ   r¬   s       €rB   r¨   z,EsmFoldTriangleMultiplicativeUpdate.__init__*  sÓ   ø€ Ý‰Œ×ÒÑÔÐØÔ,ˆØ"ˆŒå'¨°(Ñ;Ô;ˆŒÝ'¨°(ÀÐJÑJÔJˆŒÝ'¨°(Ñ;Ô;ˆŒÝ'¨°(ÀÐJÑJÔJˆŒÝ% h°¸xÐHÑHÔHˆŒÝ% h°¸wÐGÑGÔGˆŒå& xÑ0Ô0ˆÔÝ'¨Ñ1Ô1ˆÔå”z‘|”|ˆŒˆˆrA   NÚarÿ   Ú_inplace_chunk_sizerT   c                 óÂ  — | j         r!t          |d¦  «        }t          |d¦  «        }n t          |d¦  «        }t          |d¦  «        }|�qt          d|j        d         |¦  «        D ]Q}|d|||z   …d d …d d …f         }|d|||z   …d d …d d …f         }t	          j        ||¦  «        |d|||z   …d d …d d …f<   ŒR|}nt	          j        ||¦  «        }t          |d¦  «        S )Nr  )rë   r   r   r   râ   .©r   rë   r   )r  rŠ   ru   rg   r=   rù   )r«   r'  rÿ   r(  rx   Úa_chunkÚb_chunkÚps           rB   Ú_combine_projectionsz8EsmFoldTriangleMultiplicativeUpdate._combine_projections;  s%  € ð Œ>ð 	1Ý" 1 iÑ0Ô0ˆAÝ" 1 iÑ0Ô0ˆAˆAå" 1 iÑ0Ô0ˆAÝ" 1 iÑ0Ô0ˆAàÐ*å˜1˜aœg bœkÐ+>Ñ?Ô?ð ð �Ø˜C  QÐ)<Ñ%<Ð!<¸a¸a¸aÀÀÀÐBÔC�Ø˜C  QÐ)<Ñ%<Ð!<¸a¸a¸aÀÀÀÐBÔC�Ý<A¼LØØñ=ô =��#�q˜1Ð2Ñ2Ð2°A°A°A°q°q°qÐ8Ñ9Ð9ð
 ˆAˆAå”˜Q Ñ"Ô"ˆAå! ! YÑ/Ô/Ð/rA   Úzr  Úinplace_chunk_sizeÚwith_addc                 óp  ‡ ‡‡ ‡!‡"‡#‡$‡%‡&— |€"|                      |j        dd…         ¦  «        }|                     d¦  «        }dˆ fd„	Š!dˆ!ˆˆ fd„	} |||dd¬¦  «        }‰��Î|j        d         Š$‰$dz  ‰$dz  z   Š#dŠ%d	Š ‰ j        r‰%n‰ }d
„ Š"ˆ"fd„Š&ˆ ˆ"ˆ#ˆ$ˆ%ˆ&fd„}t	          |j        ¦  «        }	‰#|	‰ <   |                     |	¦  «        }
 ‰"|
¦  «        }t          d‰#¦  «        |‰ <   |
                     ||         ¦  «         d}t	          t          d‰#‰¦  «        ¦  «        }d„ t          ||dd…         ‰#gz   ¦  «        D ¦   «         }t	          t          ‰#‰$‰¦  «        ¦  «        }ˆfd„|D ¦   «         }t          ||z   ||z   ¦  «        }|D �]¡\  }}|s|‰#k    r ||
|¦  «        }
d} ‰&||||z   |¦  «        } ‰&||||z   |¦  «        }| 
                    ¦   «         }|‰ k    r ‰&||||z   ‰ ¦  «        }nK|s3 ‰"|¦  «        }t          d‰#¦  «        |‰ <    ‰&|
|||z   ‰%¦  «        ||<   n|‰#z
  } ‰&|
|||z   ‰%¦  «        } |||dd¬¦  «        }~t          j        ||¦  «        }t          |d¦  «        }‰                      |¦  «        }‰                      |¦  «        } ‰&||||z   ‰ ¦  «        }‰                      ‰                      |¦  «        ¦  «        }|                     ¦   «          ~||z  } ‰"|¦  «        }t          |||z   ¦  «        |‰ <   |r||xx         |z  cc<   �Œœ|||<   �Œ£n… |||dd¦  «        }t          j        ||¦  «        }‰                      |¦  «        }‰                      |¦  «        }‰                      |¦  «        }|                     ¦   «          ||z  }|r||z  }n|}|S )a™  
        Args:
            z:
                A [*, N, N, C_z] pair representation
            mask:
                A [*, N, N] pair mask
            inplace_chunk_size:
                Size of chunks used in the main computation. Increase to trade memory for speed.
            with_add:
                If True, z is overwritten with (z + update). Otherwise, it is overwritten with (update).
        Returns:
            A reference to the overwritten z

        More memory-efficient, inference-only version of the forward function. Uses in-place operations, fusion of the
        addition that happens after this module in the Evoformer, a smidge of recomputation, and a cache of overwritten
        values to lower peak memory consumption of this module from 5x the size of the input tensor z to 2.5x its size.
        Useful for inference on extremely long sequences.

        It works as follows. We will make reference to variables used in the default forward implementation below.
        Naively, triangle multiplication attention requires the manifestation of 5 tensors the size of z: 1) z, the
        "square" input tensor, 2) a, the first projection of z, 3) b, the second projection of b, 4) g, a z-sized mask,
        and 5) a z-sized tensor for intermediate computations. For large N, this is prohibitively expensive; for
        N=4000, for example, z is more than 8GB alone. To avoid this problem, we compute b, g, and all intermediate
        tensors in small chunks, noting that the chunks required to compute a chunk of the output depend only on the
        tensor a and corresponding vertical and horizontal chunks of z. This suggests an algorithm that loops over
        pairs of chunks of z: hereafter "columns" and "rows" of z, even though each "column" and "row" in fact contains
        inplace_chunk_size contiguous true columns and rows of z. Writing output chunks to a new tensor would bring
        total memory consumption down to 3x the size of z. However, more memory can be saved by writing output chunks
        directly to z in-place. WLOG, we choose to write output chunks vertically, overwriting the ith "column" of z at
        the end of the ith iteration of the main loop. Despite this overwriting, the ith column is always one column
        ahead of previously overwritten columns and can be recovered directly from z. After the first iteration,
        however, the ith row of z is always at least partially overwritten. For this reason, we introduce the z-cache,
        a tensor one-half the size of z. The z-cache initially contains the left half (2nd and 3rd quadrants) of z. For
        0 < i < N/2, the missing left part of the ith row of z is recovered from this cache at the beginning of the ith
        iteration. Once i exceeds n/2, the cache is "reoriented" to encompass the 3rd and 4th quadrants of z instead.
        Though the 3rd quadrant of the original z is entirely overwritten at this point, it can be recovered from the
        z-cache itself. Thereafter, the ith row of z can be recovered in its entirety from the reoriented z-cache.
        After the final iteration, z has been completely overwritten and contains the triangular multiplicative update.
        If with_add is True, it instead contains the sum of z and the triangular multiplicative update. In either case,
        peak memory consumption is just 2.5x the size of z, disregarding memory used for chunks and other small
        variables.
        Nr   Tc                 óô   •— |r‰j         }‰j        }n‰j        }‰j        }‰                     | ¦  «        }  || ¦  «        }|                     ¦   «          | || ¦  «        z  }||z  }t          |d¦  «        }|S )Nr  )r   r  r"  r!  r$  Úsigmoid_rŠ   )Úpairr  r'  rÛ   Úlinear_pr-  r«   s         €rB   Úcompute_projection_helperzYEsmFoldTriangleMultiplicativeUpdate._inference_forward.<locals>.compute_projection_helper‹  s…   ø€ Øð +Øœ?�Øœ?��àœ?�Øœ?�à×%Ò% dÑ+Ô+ˆDØ�˜‘”ˆAØ�JŠJ‰LŒLˆLØ��˜$‘”ÑˆAØ�‰IˆAÝ" 1 iÑ0Ô0ˆAØˆHrA   c           
      óX  •— ‰j         |z  }|s& ‰| ||¦  «        }|r|                     dd¦  «        }nö|r‰j        n‰j        }|j        j        d         }| j        d d…         |fz   | j        dd…         z   }|                      |¦  «        }t          d| j        d         ‰¦  «        D ]}	| d|	|	‰z   …d d …d d …f         }
 ‰| d|	|	‰z   …d d …d d …f         |d|	|	‰z   …d d …d d …f         |¦  «        }
|r#|
                     dd¦  «        }
|
|d|	|	‰z   …f<   n|
|d|	|	‰z   …d d …f<   ~
Œ€|S )Nr   rá   râ   r   .)r  rä   r   r"  r�   rg   Ú	new_zerosru   )r5  r  r'  ÚchunkedÚneed_transposer-  rÛ   ÚcÚ	out_shaperx   Ú
pair_chunkr7  r0  r«   s              €€€rB   Úcompute_projectionzREsmFoldTriangleMultiplicativeUpdate._inference_forward.<locals>.compute_projection›  s­  ø€ Ø!œ^¨aÑ/ˆNØð #Ø-Ð-¨d°D¸!Ñ<Ô<�Ø!ð ,ØŸš B¨Ñ+Ô+�Aøð /0ÐD˜4œ?˜?°T´_�Ø”MÔ'¨Ô+�Ø œJ s¨ sœO¨q¨dÑ2°T´ZÀÀ2ÀÔ5FÑF�	Ø—N’N 9Ñ-Ô-�Ý˜q $¤*¨R¤.Ð2DÑEÔEð #ð #�AØ!% c¨1¨qÐ3EÑ/EÐ+EÀqÀqÀqÈ!È!È!Ð&KÔ!L�JØ!:Ð!:Ø˜S ! aÐ*<Ñ&<Ð"<¸a¸a¸aÀÀÀÐBÔCØ˜S ! aÐ*<Ñ&<Ð"<¸a¸a¸aÀÀÀÐBÔCØñ"ô "�Jð
 &ð KØ%/×%9Ò%9¸"¸bÑ%AÔ%A˜
Ø=G˜˜#˜q 1Ð'9Ñ#9Ð9Ð9Ñ:Ð:à@J˜˜#˜q 1Ð'9Ñ#9Ð9¸1¸1¸1Ð<Ñ=à"˜
àˆHrA   )r:  rë   râ   rá   c                 ó$   — d„ | j         D ¦   «         S )Nc                 ó,   — g | ]}t          d ¦  «        ‘ŒS rD   rj   )rZ   Ú_s     rB   r^   z`EsmFoldTriangleMultiplicativeUpdate._inference_forward.<locals>.empty_slicer.<locals>.<listcomp>Æ  s   € Ð5Ð5Ð5¨�˜d™œÐ5Ð5Ð5rA   rf   )rz   s    rB   Úempty_slicerzLEsmFoldTriangleMultiplicativeUpdate._inference_forward.<locals>.empty_slicerÅ  s   € Ø5Ð5¨Q¬WÐ5Ñ5Ô5Ð5rA   c                 óP   •—  ‰| ¦  «        }t          ||¦  «        ||<   | |         S rD   rj   )rz   ÚstartÚendrX   rË   rC  s        €rB   Úslice_tensorzLEsmFoldTriangleMultiplicativeUpdate._inference_forward.<locals>.slice_tensorÈ  s,   ø€ à �L ‘O”O�Ý˜u cÑ*Ô*��#‘Ø˜”t�rA   c                 ó>  •—  ‰| ‰d ‰
¦  «        }|                       ‰
‰¦  «        } | dd ‰	dz  …d d …d d …f         }  ‰| ¦  «        }t          d‰¦  «        |‰<   || |<    ‰|‰d ‰
¦  «        } ‰|‰d ‰¦  «        } ‰| ¦  «        }t          ‰d ¦  «        |‰<   || |<   | S )N.rë   r   )rä   rk   )Úz_cacher/  Ú
quadrant_3Úfirst_half_slicerÚ
quadrant_4Úquadrant_3_slicerÚcol_dimrC  Úhalf_nÚnÚrow_dimrG  s         €€€€€€rB   Úflip_z_cache_zMEsmFoldTriangleMultiplicativeUpdate._inference_forward.<locals>.flip_z_cache_Î  sé   ø€ ð *˜\¨'°6¸4ÀÑIÔI�
Ø!×+Ò+¨G°WÑ=Ô=�ð " # z¨!¨q©& z°1°1°1°a°a°aÐ"7Ô8�ð %1 L°Ñ$9Ô$9Ð!Ý-2°1°fÑ-=Ô-=Ð! 'Ñ*Ø-7�Ð)Ñ*ð *˜\¨!¨V°T¸7ÑCÔC�
Ø)˜\¨*°f¸dÀGÑLÔL�
ð %1 L°Ñ$9Ô$9Ð!Ý-2°6¸4Ñ-@Ô-@Ð! 'Ñ*à-7�Ð)Ñ*à�rA   r   Fc                 ó   — g | ]
\  }}||z
  ‘ŒS r@   r@   )rZ   Úi_1Úi_2s      rB   r^   zJEsmFoldTriangleMultiplicativeUpdate._inference_forward.<locals>.<listcomp>ö  s    € Ð^Ð^Ð^©X¨S°#˜s S™yÐ^Ð^Ð^rA   r   c                 ó   •— g | ]}‰‘ŒS r@   r@   )rZ   rB  r0  s     €rB   r^   zJEsmFoldTriangleMultiplicativeUpdate._inference_forward.<locals>.<listcomp>ø  s   ø€ Ð!IÐ!IÐ!I¸Ð"4Ð!IÐ!IÐ!IrA   )r'  r:  r*  r   )TT)r  rg   r  r  r‡   r9  rk   Úcopy_ru   rr   Úcloner=   rù   rŠ   r%  r#  rÛ   r$  r4  )'r«   r/  r  r0  r1  r?  r'  Úb_chunk_dimrR  Úz_cache_shaperI  Úz_cache_slicerÚz_cache_rotatedÚi_rangeÚinitial_offsetsÚ
after_halfÚafter_half_offsetsÚcombined_range_with_offsetsrx   ÚoffsetÚ	z_chunk_bÚ
mask_chunkÚz_chunk_slicerÚz_cache_offsetr,  Úx_chunkÚ	z_chunk_gÚg_chunkÚz_slicerrÿ   r[   rì   rN  r7  rC  rO  rP  rQ  rG  s'   `  `                            @@@@@@@rB   Ú_inference_forwardz6EsmFoldTriangleMultiplicativeUpdate._inference_forwardU  só  øøøøøøøøø€ ðb ˆ<Ø—:’:˜aœg c r cœlÑ+Ô+ˆDà�~Š~˜bÑ!Ô!ˆð	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ðB Ð˜q $¨°dÐ;Ñ;Ô;ˆàÑ)Ø”˜”ˆAØ˜!‘V˜a !™e‘^ˆFØˆGØˆGØ%)¤^Ð@˜'˜'¸ˆKð6ð 6ð 6ðð ð ð ð ðð ð ð ð ð ð ð ð ð õ8 ! ¤™MœMˆMØ%+ˆM˜'Ñ"Ø—k’k -Ñ0Ô0ˆGØ)˜\¨'Ñ2Ô2ˆNÝ&+¨A¨vÑ&6Ô&6ˆN˜7Ñ#Ø�MŠM˜!˜NÔ+Ñ,Ô,Ð,Ø#ˆOõ
 �5  FÐ,>Ñ?Ô?Ñ@Ô@ˆGØ^Ð^½¸WÀgÈaÈbÈbÄkÐU[ÐT\ÑF\Ñ9]Ô9]Ð^Ñ^Ô^ˆOÝ�e F¨AÐ/AÑBÔBÑCÔCˆJØ!IÐ!IÐ!IÐ!I¸jÐ!IÑ!IÔ!IÐÝ*-¨g¸
Ñ.BÀOÐVhÑDhÑ*iÔ*iÐ'Ø8ð .*ñ .*‘	��6Ø&ð +¨1°ª;¨;Ø+˜m¨G°QÑ7Ô7�GØ&*�Oà(˜L¨¨A¨q°6©z¸;ÑGÔG�	Ø)˜\¨$°°1°v±:¸{ÑKÔK�
à%ŸOšOÑ-Ô-�	Ø 'Ò)Ð)Ø , ¨Q°°1°v±:¸wÑ GÔ G�I�Ið
 +ð lØ)5¨°iÑ)@Ô)@˜Ý27¸¸6Ñ2BÔ2B˜ wÑ/Ø4@°LÀÈ!ÈQÐQWÉZÐY`Ñ4aÔ4a˜	 .Ñ1Ð1à)*¨V©˜Ø$0 L°¸.È.Ð[aÑJaÐcjÑ$kÔ$k˜	à,Ð,¨Y¸
ÀeÐUZÐ[Ñ[Ô[�Øåœ, q¨'Ñ2Ô2�Ý,¨W°iÑ@Ô@�Ø×-Ò-¨gÑ6Ô6�ØŸ-š-¨Ñ0Ô0�ð )˜L¨¨A¨q°6©z¸7ÑCÔC�	ØŸ-š-¨×(:Ò(:¸9Ñ(EÔ(EÑFÔF�Ø× Ò Ñ"Ô"Ð"Øà˜7Ñ"�ð (˜<¨™?œ?�Ý$)¨!¨Q°©ZÑ$8Ô$8�˜Ñ!Øð *Ø�h�K�K”K 7Ñ*�K�K‘K‘Kà")�A�h‘K‘Kð].*ð` #Ð" 1 d¨E°5Ñ9Ô9ˆAÝ”˜Q Ñ"Ô"ˆAØ×#Ò# AÑ&Ô&ˆAØ—’˜aÑ Ô ˆAØ—’˜aÑ Ô ˆAØ�JŠJ‰LŒLˆLØ�‰FˆAØð Ø�Q‘��à�àˆrA   Fé   r
  Ú_add_with_inplacec                 óð  — |r|                       ||||¬¦  «        }|S |€"|                     |j        dd…         ¦  «        }|                     d¦  «        }|                      |¦  «        }|}||                      |                      |¦  «        ¦  «        z  }||                      |¦  «        z  }|}||                      |                      |¦  «        ¦  «        z  }||  	                    |¦  «        z  }|j
        j        dk    r|j
        j        nd}	t          |	¦  «        rdt          |	d¬¦  «        5  |                      |                     ¦   «         |                     ¦   «         ¦  «        }ddd¦  «         n# 1 swxY w Y   n|                      ||¦  «        }~~|                      |¦  «        }|                      |¦  «        }|                      |                      |¦  «        ¦  «        }
||
z  }|S )zÛ
        Args:
            x:
                [*, N_res, N_res, C_z] input tensor
            mask:
                [*, N_res, N_res] input mask
        Returns:
            [*, N_res, N_res, C_z] output tensor
        )r0  r1  Nr   ÚmpsÚcpuFr¿   )rk  r  rg   r  r$  rÝ   r   r  r"  r!  ra   ÚtyperK   r   r.  Úfloatr%  r#  rÛ   )r«   r/  r  r
  rm  r(  r[   r'  rÿ   rH   rì   s              rB   rÇ   z+EsmFoldTriangleMultiplicativeUpdate.forward8  s,  € ð" ð 	Ø×'Ò'ØØØ#6Ø*ð	 (ñ ô ˆAð ˆHàˆ<Ø—:’:˜aœg c r cœlÑ+Ô+ˆDà�~Š~˜bÑ!Ô!ˆà×Ò˜qÑ!Ô!ˆØˆØ�—’˜TŸ_š_¨QÑ/Ô/Ñ0Ô0Ñ0ˆØ�—’ Ñ"Ô"Ñ"ˆØˆØ�—’˜TŸ_š_¨QÑ/Ô/Ñ0Ô0Ñ0ˆØ�—’ Ñ"Ô"Ñ"ˆà'(¤x¤}¸Ò'=Ð'=�a”h”m�mÀ5ˆÝ˜;Ñ'Ô'ð 	0Ý¨KÀÐGÑGÔGð Dð DØ×-Ò-¨a¯gªg©i¬i¸¿º¹¼ÑCÔC�ðDð Dð Dñ Dô Dð Dð Dð Dð Dð Dð Døøøð Dð Dð Dð Døð ×)Ò)¨!¨QÑ/Ô/ˆAàˆqØ×Ò Ñ"Ô"ˆØ�MŠM˜!ÑÔˆØ�LŠL˜Ÿš qÑ)Ô)Ñ*Ô*ˆØ�‰Eˆàˆs   Ä4;E;Å;E?ÆE?r   rD   )NNT)NFFrl  )r9   r:   r;   r<   r¨   r=   ro   r­   r.  r®   rk  rÇ   r°   r±   s   @rB   r  r  %  sb  ø€ € € € € ðð ð$ð $ð $ð $ð $ð $ð$ SWð0ð 0Ø”ð0Ø"'¤,ð0ØEHÈ4ÁZð0à	Œð0ð 0ð 0ð 0ð: %)Ø)-Øðað aàŒ<ðað Œl˜TÑ!ðað   $™Jð	að
 ðað að að aðL %)Ø"Ø"'Ø*-ð4ð 4àŒ<ð4ð Œl˜TÑ!ð4ð ð	4ð
  ð4ð ! 4™Zð4ð 
Œð4ð 4ð 4ð 4ð 4ð 4ð 4ð 4rA   r  c                   óL   ‡ — e Zd ZdZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚEsmFoldPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    c                 ó¨	  •— t          |t          ¦  «        �rÏt          j        ¦   «         5  |j        �"|                     |j        |j        ¦  «         �nx|j        dk    rV|j        j        }dt          d|d         ¦  «        z  }t          j        |¦  «        }t          j        |j        |¬¦  «         �n|j        dk    rU|j        j        }dt          d|d         ¦  «        z  }t          j        |¦  «        }t          j        |j        |¬¦  «         n·|j        dk    rt          j        |j        d¬	¦  «         n�|j        d
k    r:t          j        |j        ¦  «         |j        rt          j        |j        ¦  «         nK|j        dk    rt          j        |j        d¬¦  «         n$|j        dk    rt          j        |j        ¦  «         ddd¦  «         dS # 1 swxY w Y   dS t          |t"          ¦  «        rd}t          j        |j        |¦  «         dS t          |t(          ¦  «        �r�t          j        |j        j        j        ¦  «         t          j        |j        j        j        ¦  «         t          j        |j        j        j        ¦  «         t          j        |j        j        j        ¦  «         t          j        |j        j        j        j        ¦  «         t          j        |j        j        j        j        ¦  «         t          j        |j        j        j        j        ¦  «         t          j        |j        j        j        j        ¦  «         t          j        |j        j        j        ¦  «         t          j        |j        j        j        ¦  «         t          j        |j        j        j        ¦  «         t          j        |j         j        j        ¦  «         t          j        |j         j        j        ¦  «         t          j        |j!        j"        d         j        ¦  «         t          j        |j!        j"        d         j        ¦  «         t          j        |j#        j"        d         j        ¦  «         t          j        |j#        j"        d         j        ¦  «         dS tI          ¦   «          %                    |¦  «         dS )zInitialize the weightsNrš   ç      ð?r   )Ústdr¢   g       @r£   )Úgainr¤   r¥   r  )Únonlinearityr¦   gËab€ˆRá?rá   )&r�   r™   r=   r©   rŸ   r»   r�   rž   rg   rc   rå   ræ   Únormal_Úxavier_uniform_Úzeros_Úones_Úkaiming_normal_ÚEsmFoldInvariantPointAttentionÚ	constant_Úhead_weightsÚ#EsmFoldTriangularSelfAttentionBlockÚ
tri_mul_inr#  Útri_mul_outÚtri_att_startr  rÚ   Útri_att_endÚsequence_to_pairÚo_projÚpair_to_sequencer  Úseq_attentionÚmlp_seqÚmlpÚmlp_pairr§   Ú_init_weights)r«   Úmodulerg   Úscalerw  Úsoftplus_inverse_1r¬   s         €rB   rŽ  z$EsmFoldPreTrainedModel._init_weightsv  s8  ø€ õ �f�mÑ,Ô,ñ /	*Ý”‘”ð /ð /Ø”>Ð-Ø—N’N 6¤=°&´+Ñ>Ô>Ð>Ñ>Ø”[ IÒ-Ð-Ø"œMÔ/�EØ¥# a¨¨q¬Ñ"2Ô"2Ñ2�EÝœ) EÑ*Ô*�CÝ”L ¤°CÐ8Ñ8Ô8Ð8Ñ8Ø”[ FÒ*Ð*Ø"œMÔ/�EØ¥# a¨¨q¬Ñ"2Ô"2Ñ2�EÝœ) EÑ*Ô*�CÝ”L ¤°CÐ8Ñ8Ô8Ð8Ð8Ø”[ HÒ,Ð,ÝÔ(¨¬¸QÐ?Ñ?Ô?Ð?Ð?Ø”[ HÒ,Ð,Ý”K ¤Ñ.Ô.Ð.Ø”{ð 0Ýœ
 6¤;Ñ/Ô/Ð/øØ”[ HÒ,Ð,ÝÔ(¨¬ÀXÐNÑNÔNÐNÐNØ”[ GÒ+Ð+Ý”K ¤Ñ.Ô.Ð.ð-/ð /ð /ñ /ô /ð /ð /ð /ð /ð /ð /ð /øøøð /ð /ð /ð /ð /ð /õ. ˜Õ >Ñ?Ô?ð 	*Ø!2ÐÝŒN˜6Ô.Ð0BÑCÔCÐCÐCÐCÝ˜Õ CÑDÔDñ 	*ÝŒK˜Ô)Ô2Ô9Ñ:Ô:Ð:ÝŒK˜Ô)Ô2Ô7Ñ8Ô8Ð8ÝŒK˜Ô*Ô3Ô:Ñ;Ô;Ð;ÝŒK˜Ô*Ô3Ô8Ñ9Ô9Ð9ÝŒK˜Ô,Ô0Ô9Ô@ÑAÔAÐAÝŒK˜Ô,Ô0Ô9Ô>Ñ?Ô?Ð?ÝŒK˜Ô*Ô.Ô7Ô>Ñ?Ô?Ð?ÝŒK˜Ô*Ô.Ô7Ô<Ñ=Ô=Ð=åŒK˜Ô/Ô6Ô=Ñ>Ô>Ð>ÝŒK˜Ô/Ô6Ô;Ñ<Ô<Ð<ÝŒK˜Ô/Ô6Ô=Ñ>Ô>Ð>ÝŒK˜Ô,Ô3Ô:Ñ;Ô;Ð;ÝŒK˜Ô,Ô3Ô8Ñ9Ô9Ð9ÝŒK˜œÔ*¨2Ô.Ô5Ñ6Ô6Ð6ÝŒK˜œÔ*¨2Ô.Ô3Ñ4Ô4Ð4ÝŒK˜œÔ+¨BÔ/Ô6Ñ7Ô7Ð7ÝŒK˜œÔ+¨BÔ/Ô4Ñ5Ô5Ð5Ð5Ð5å‰GŒG×!Ò! &Ñ)Ô)Ð)Ð)Ð)s   «F"GÇGÇ!G)r9   r:   r;   r<   r=   r©   rŽ  r°   r±   s   @rB   rt  rt  o  sV   ø€ € € € € ðð ð €U„]�_„_ð1*ð 1*ð 1*ð 1*ñ „_ð1*ð 1*ð 1*ð 1*ð 1*rA   rt  c                   ó(   ‡ — e Zd Zdˆ fd„	Zdd„Zˆ xZS )ÚEsmFoldSelfAttentionFc                 ó  •— t          ¦   «                              ¦   «          |||z  k    sJ ‚|| _        || _        || _        t          j        ||dz  d¬¦  «        | _        t          j        ||d¬¦  «        | _        || _	        |rVt          j        ||¦  «        | _
        t          j        | j
        j        ¦  «         t          j        | j
        j        ¦  «         | j        dz  | _        t          j        | j        j        ¦  «         d S )Nr   Fr¡   Tç      à¿)r§   r¨   Ú	embed_dimÚ	num_headsÚ
head_widthr¸   ÚLinearÚprojrˆ  ÚgatedÚg_projrž   r|  r»   r}  r�   Úrescale_factor)r«   r–  r—  r˜  r›  r¬   s        €rB   r¨   zEsmFoldSelfAttention.__init__¬  sî   ø€ Ý‰Œ×ÒÑÔÐØ˜I¨
Ñ2Ò2Ð2Ð2Ð2à"ˆŒØ"ˆŒØ$ˆŒå”I˜i¨°Q©¸UÐCÑCÔCˆŒ	Ý”i 	¨9¸4Ð@Ñ@Ô@ˆŒØˆŒ
Øð 	)Ýœ) I¨yÑ9Ô9ˆDŒKÝŒK˜œÔ*Ñ+Ô+Ð+ÝŒJ�t”{Ô'Ñ(Ô(Ð(à"œo¨tÑ3ˆÔåŒ�D”KÔ$Ñ%Ô%Ð%Ð%Ð%rA   Nc                 ó  —  |                       |¦  «        j        g |j        dd…         ¢| j        ‘d‘R Ž }|                     dddd¦  «        }|                     dd¬¦  «        \  }}}| j        |z  }t          j        d||¦  «        }	|�|	|                     dddd¦  «        z   }	|�2|dd…ddf         }|	 	                    |d	k    t          j         ¦  «        }	t          j                             |	d¬¦  «        }	t          j        d
|	|¦  «        }
 |
j        g |
j        dd…         ¢d‘R Ž }
| j        r*|                      |¦  «                             ¦   «         |
z  }
|                      |
¦  «        }
|
|	                     dddd¦  «        fS )aä  
        Basic self attention with optional mask and external pairwise bias. To handle sequences of different lengths,
        use mask.

        Inputs:
            x: batch of input sequences (.. x L x C) mask: batch of boolean masks where 1=valid, 0=padding position (..
            x L_k) bias: batch of scalar pairwise attention biases (.. x Lq x Lk x num_heads)

        Outputs:
          sequence projection (B x L x embed_dim), attention maps (B x L x L x num_heads)
        Nrë   r   r   r   r   rW   z...qc,...kc->...qkFz...hqk,...hkc->...qhc)rš  rã   rg   r—  rˆ   Úchunkr�  r=   ÚeinsumÚmasked_fillÚnpr  r¸   rÃ   rÊ   r€   r›  rœ  rÝ   rˆ  )r«   r[   r  r�   Úindicesrz   rç   rl   r–   r'  Úys              rB   rÇ   zEsmFoldSelfAttention.forwardÀ  s”  € ð ˆD�IŠI�a‰LŒLÔÐ?˜qœw r¨ rœ{Ð?¨D¬NÐ?¸BÐ?Ð?Ð?ˆØ�IŠI�a˜˜A˜qÑ!Ô!ˆØ—'’'˜! �'Ñ$Ô$‰ˆˆ1ˆaàÔ !Ñ#ˆÝŒLÐ-¨q°!Ñ4Ô4ˆð ÐØ�D—L’L  A q¨!Ñ,Ô,Ñ,ˆAð ÐØ˜˜˜˜4 ˜Ô&ˆDØ—’˜d ešm­b¬f¨WÑ5Ô5ˆAåŒM×!Ò! !¨Ð!Ñ,Ô,ˆåŒLÐ0°!°QÑ7Ô7ˆØˆAŒIÐ'�q”w˜r ˜r”{Ð' BÐ'Ð'Ð'ˆàŒ:ð 	-Ø—’˜A‘”×&Ò&Ñ(Ô(¨1Ñ,ˆAØ�KŠK˜‰NŒNˆà�!—)’)˜A˜q ! QÑ'Ô'Ð'Ð'rA   )F)NNNrÈ   r±   s   @rB   r“  r“  «  sQ   ø€ € € € € ð&ð &ð &ð &ð &ð &ð(&(ð &(ð &(ð &(ð &(ð &(ð &(ð &(rA   r“  c                   ób   ‡ — e Zd ZdZdedeee         z  fˆ fd„Zdej	        dej	        fd„Z
ˆ xZS )ÚEsmFoldDropoutzl
    Implementation of dropout with the ability to share the dropout mask along a particular dimension.
    ÚrÚ	batch_dimc                 óÐ   •— t          ¦   «                              ¦   «          || _        t          |t          ¦  «        r|g}|| _        t          j        | j        ¦  «        | _        d S rD   )	r§   r¨   r§  r�   r­   r¨  r¸   ÚDropoutÚdropout)r«   r§  r¨  r¬   s      €rB   r¨   zEsmFoldDropout.__init__î  sW   ø€ Ý‰Œ×ÒÑÔÐàˆŒÝ�i¥Ñ%Ô%ð 	$Ø"˜ˆIØ"ˆŒÝ”z $¤&Ñ)Ô)ˆŒˆˆrA   r[   rT   c                 ó¬   — t          |j        ¦  «        }| j        �| j        D ]}d||<   Œ||                      |                     |¦  «        ¦  «        z  S )Nr   )r‡   rg   r¨  r«  r  )r«   r[   rg   Úbds       rB   rÇ   zEsmFoldDropout.forward÷  sV   € Ý�Q”W‘”ˆØŒ>Ð%Ø”nð ð �Ø��b‘	�	Ø�4—<’< §
¢
¨5Ñ 1Ô 1Ñ2Ô2Ñ2Ð2rA   )r9   r:   r;   r<   rr  r­   r‡   r¨   r=   ro   rÇ   r°   r±   s   @rB   r¦  r¦  é  sƒ   ø€ € € € € ðð ð*˜%ð *¨C°$°s´)©Oð *ð *ð *ð *ð *ð *ð3˜œð 3¨%¬,ð 3ð 3ð 3ð 3ð 3ð 3ð 3ð 3rA   r¦  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEsmFoldSequenceToPairc                 ón  •— t          ¦   «                              ¦   «          t          j        |¦  «        | _        t          j        ||dz  d¬¦  «        | _        t          j        d|z  |d¬¦  «        | _        t          j	        | j        j
        ¦  «         t          j	        | j        j
        ¦  «         d S )Nrë   Tr¡   )r§   r¨   r¸   r   Ú	layernormr™  rš  rˆ  rž   r|  r�   )r«   Úsequence_state_dimÚ	inner_dimr  r¬   s       €rB   r¨   zEsmFoldSequenceToPair.__init__   s”   ø€ Ý‰Œ×ÒÑÔÐåœÐ&8Ñ9Ô9ˆŒÝ”IÐ0°)¸a±-ÀdÐKÑKÔKˆŒ	Ý”i  I¡Ð/AÈÐMÑMÔMˆŒåŒ�D”I”NÑ#Ô#Ð#ÝŒ�D”KÔ$Ñ%Ô%Ð%Ð%Ð%rA   c                 ó°  — t          |j        ¦  «        dk    sJ ‚|                      |¦  «        }|                      |¦  «        }|                     dd¬¦  «        \  }}|dd…ddd…dd…f         |dd…dd…ddd…f         z  }|dd…ddd…dd…f         |dd…dd…ddd…f         z
  }t          j        ||gd¬¦  «        }|                      |¦  «        }|S )z×
        Inputs:
          sequence_state: B x L x sequence_state_dim

        Output:
          pairwise_state: B x L x L x pairwise_state_dim

        Intermediate state:
          B x L x L x 2*inner_dim
        r   rë   r   rW   N)rn   rg   r±  rš  rŸ  r=   Úcatrˆ  )r«   Úsequence_staterË   rç   rl   ÚprodÚdiffr[   s           rB   rÇ   zEsmFoldSequenceToPair.forward
  sý   € õ �>Ô'Ñ(Ô(¨AÒ-Ð-Ð-Ð-à�NŠN˜>Ñ*Ô*ˆØ�IŠI�a‰LŒLˆØ�wŠw�q˜bˆwÑ!Ô!‰ˆˆ1à����D˜!˜!˜!˜Q˜Q˜Q�Ô ! A A A q q q¨$°°° MÔ"2Ñ2ˆØ����D˜!˜!˜!˜Q˜Q˜Q�Ô ! A A A q q q¨$°°° MÔ"2Ñ2ˆåŒI�t˜T�l¨Ð+Ñ+Ô+ˆØ�KŠK˜‰NŒNˆàˆrA   rÈ   r±   s   @rB   r¯  r¯  ÿ  sG   ø€ € € € € ð&ð &ð &ð &ð &ðð ð ð ð ð ð rA   r¯  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEsmFoldPairToSequencec                 ó²   •— t          ¦   «                              ¦   «          t          j        |¦  «        | _        t          j        ||d¬¦  «        | _        d S )NFr¡   )r§   r¨   r¸   r   r±  r™  r  )r«   r  r—  r¬   s      €rB   r¨   zEsmFoldPairToSequence.__init__&  sH   ø€ Ý‰Œ×ÒÑÔÐåœÐ&8Ñ9Ô9ˆŒÝ”iÐ 2°IÀEÐJÑJÔJˆŒˆˆrA   c                 óŽ   — t          |j        ¦  «        dk    sJ ‚|                      |¦  «        }|                      |¦  «        }|S )z’
        Inputs:
          pairwise_state: B x L x L x pairwise_state_dim

        Output:
          pairwise_bias: B x L x L x num_heads
        é   )rn   rg   r±  r  )r«   Úpairwise_stater/  Úpairwise_biass       rB   rÇ   zEsmFoldPairToSequence.forward,  sF   € õ �>Ô'Ñ(Ô(¨AÒ-Ð-Ð-Ð-Ø�NŠN˜>Ñ*Ô*ˆØŸš A™œˆØÐrA   rÈ   r±   s   @rB   rº  rº  %  sL   ø€ € € € € ðKð Kð Kð Kð Kðð ð ð ð ð ð rA   rº  c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚEsmFoldResidueMLPr   c           	      ó8  •— t          ¦   «                              ¦   «          t          j        t          j        |¦  «        t          j        ||¦  «        t          j        ¦   «         t          j        ||¦  «        t          j        |¦  «        ¦  «        | _        d S rD   )	r§   r¨   r¸   Ú
Sequentialr   r™  ÚReLUrª  rŒ  )r«   r–  r³  r«  r¬   s       €rB   r¨   zEsmFoldResidueMLP.__init__;  sq   ø€ Ý‰Œ×ÒÑÔÐå”=ÝŒL˜Ñ#Ô#ÝŒI�i Ñ+Ô+ÝŒG‰IŒIÝŒI�i Ñ+Ô+ÝŒJ�wÑÔñ
ô 
ˆŒˆˆrA   c                 ó2   — ||                       |¦  «        z   S rD   )rŒ  )r«   r[   s     rB   rÇ   zEsmFoldResidueMLP.forwardF  s   € Ø�4—8’8˜A‘;”;‰ÐrA   ©r   rÈ   r±   s   @rB   rÁ  rÁ  :  sL   ø€ € € € € ð	
ð 	
ð 	
ð 	
ð 	
ð 	
ðð ð ð ð ð ð rA   rÁ  c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )r‚  c                 ó^  •— t          ¦   «                              ¦   «          || _        |j        }|j        }||j        z  }||j        z  }t          j        |¦  «        | _	        t          ||dz  |¦  «        | _        t          ||¦  «        | _        t          |||j        d¬¦  «        | _        t!          |d¬¦  «        | _        t!          |d¬¦  «        | _        t'          ||j        |dd¬¦  «        | _        t'          ||j        |dd¬¦  «        | _        t-          |d|z  |j        ¬	¦  «        | _        t-          |d|z  |j        ¬	¦  «        | _        t          j        |j        ¦  «        | _        t9          |j        dz  d¦  «        | _        t9          |j        dz  d
¦  «        | _        d S )Nrë   T)r›  )r  Fr  )r  r  r½  )r«  r   )r§   r¨   r&  r²  r  Úsequence_head_widthÚpairwise_head_widthr¸   r   Úlayernorm_1r¯  r‡  rº  r‰  r“  rŠ  r  r„  rƒ  r  r…  r†  rÁ  r«  r‹  r�  rª  Údropr¦  Úrow_dropÚcol_drop)r«   r&  r²  r  Úsequence_num_headsÚpairwise_num_headsr¬   s         €rB   r¨   z,EsmFoldTriangularSelfAttentionBlock.__init__K  sÁ  ø€ Ý‰Œ×ÒÑÔÐØˆŒà#Ô6ÐØ#Ô6ÐØ/°6Ô3MÑMÐØ/°6Ô3MÑMÐåœ<Ð(:Ñ;Ô;ˆÔå 5Ð6HÐJ\Ð`aÑJaÐcuÑ vÔ vˆÔÝ 5Ð6HÐJ\Ñ ]Ô ]ˆÔå1ØÐ 2°FÔ4NÐVZð
ñ 
ô 
ˆÔõ ?¸vÐQUÐVÑVÔVˆÔÝ=¸fÐPUÐVÑVÔVˆŒå5Ø Ô :Ð<NÐTWÐbfð
ñ 
ô 
ˆÔõ 4Ø Ô :Ð<NÐTWÐbgð
ñ 
ô 
ˆÔõ )Ð);¸QÐASÑ=SÐ]cÔ]kÐlÑlÔlˆŒÝ)Ð*<¸aÐBTÑ>TÐ^dÔ^lÐmÑmÔmˆŒå”J˜vœ~Ñ.Ô.ˆŒ	Ý& v¤~¸Ñ'9¸1Ñ=Ô=ˆŒÝ& v¤~¸Ñ'9¸1Ñ=Ô=ˆŒˆˆrA   Nc           	      óz  — t          |j        ¦  «        dk    r%t          dt          |j        ¦  «        › d�¦  «        ‚t          |j        ¦  «        dk    r%t          dt          |j        ¦  «        › d�¦  «        ‚|�=t          |j        ¦  «        dk    r%t          dt          |j        ¦  «        › d�¦  «        ‚|j        \  }}}|j        d         }	|| j        j        k    r t          d	|› d
| j        j        › d�¦  «        ‚|	| j        j        k    r t          d|	› d
| j        j        › d�¦  «        ‚||j        d         k    r!t          d|› d
|j        d         › d�¦  «        ‚||j        d         k    s||j        d         k    r/t          d|› d
|j        d         › d|j        d         › d�¦  «        ‚|                      |¦  «        }
|                      |¦  «        }|                      |||
¬¦  «        \  }}||  	                    |¦  «        z   }|  
                    |¦  «        }||                      |¦  «        z   }|�+|                     d¦  «        |                     d¦  «        z  nd}||                      |                      ||¬¦  «        ¦  «        z   }||                      |                      ||¬¦  «        ¦  «        z   }||                      |                      |||¬¦  «        ¦  «        z   }||                      |                      |||¬¦  «        ¦  «        z   }|                      |¦  «        }||fS )a*  
        Inputs:
          sequence_state: B x L x sequence_state_dim pairwise_state: B x L x L x pairwise_state_dim mask: B x L boolean
          tensor of valid positions

        Output:
          sequence_state: B x L x sequence_state_dim pairwise_state: B x L x L x pairwise_state_dim
        r   z,`sequence_state` should be a 3d-tensor, got ú dims.r½  z,`pairwise_state` should be a 4d-tensor, got Nrë   ú"`mask` should be a 2d-tensor, got zR`sequence_state` last dimension should be equal to `self.sequence_state_dim`. Got ú != ú.zR`pairwise_state` last dimension should be equal to `self.pairwise_state_dim`. Got r   zD`sequence_state` and `pairwise_state` have inconsistent batch size: r   zI`sequence_state` and `pairwise_state` have inconsistent sequence length: z or )r  r�   ©r  )r  r	  )rn   rg   rª   r&  r²  r  r‰  rË  rŠ  rÌ  r‹  r‡  r  rÍ  r„  rÎ  rƒ  r…  r†  r�  )r«   r¶  r¾  r  r	  Ú,_EsmFoldTriangularSelfAttentionBlock__kwargsr¨  Úseq_dimr²  r  r�   r¤  rB  Útri_masks                 rB   rÇ   z+EsmFoldTriangularSelfAttentionBlock.forwardm  sÚ  € õ ˆ~Ô#Ñ$Ô$¨Ò)Ð)ÝÐmÍCÐP^ÔPdÑLeÔLeÐmÐmÐmÑnÔnÐnÝˆ~Ô#Ñ$Ô$¨Ò)Ð)ÝÐmÍCÐP^ÔPdÑLeÔLeÐmÐmÐmÑnÔnÐnØÐ¥ D¤J¡¤°1Ò 4Ð 4ÝÐYÅ#ÀdÄjÁ/Ä/ÐYÐYÐYÑZÔZÐZà1?Ô1EÑ.ˆ	�7Ð.Ø+Ô1°!Ô4Ðà ¤Ô!?Ò?Ð?ÝðMØ%ðMð MØ+/¬;Ô+IðMð Mð Mñô ð ð  ¤Ô!?Ò?Ð?ÝðMØ%ðMð MØ+/¬;Ô+IðMð Mð Mñô ð ð ˜Ô,¨QÔ/Ò/Ð/Ýð.ÐW`ð .ð .Ø!Ô'¨Ô*ð.ð .ð .ñô ð ð �nÔ*¨1Ô-Ò-Ð-°¸NÔ<PÐQRÔ<SÒ1SÐ1SÝðKÐ\cð Kð KØ!Ô'¨Ô*ðKð KØ0>Ô0DÀQÔ0GðKð Kð Kñô ð ð ×$Ò$ ^Ñ4Ô4ˆð ×Ò˜^Ñ,Ô,ˆØ×!Ò! !¨$°TÐ!Ñ:Ô:‰ˆˆ1Ø'¨$¯)ª)°A©,¬,Ñ6ˆØŸš nÑ5Ô5ˆð (¨$×*?Ò*?ÀÑ*OÔ*OÑOˆð =AÐ<L�4—>’> !Ñ$Ô$ t§~¢~°aÑ'8Ô'8Ñ8Ð8ÐRVˆØ'¨$¯-ª-¸×8HÒ8HÈÐ^fÐ8HÑ8gÔ8gÑ*hÔ*hÑhˆØ'¨$¯-ª-¸¿ºÈÐ]e¸Ñ8fÔ8fÑ*gÔ*gÑgˆØ'¨$¯-ª-Ø×Ò˜~°HÈÐÑTÔTñ+
ô +
ñ 
ˆð (¨$¯-ª-Ø×Ò˜^°(ÀzÐÑRÔRñ+
ô +
ñ 
ˆð
 Ÿš ~Ñ6Ô6ˆà˜~Ð-Ð-rA   )NNrÈ   r±   s   @rB   r‚  r‚  J  sU   ø€ € € € € ð >ð  >ð  >ð  >ð  >ðDB.ð B.ð B.ð B.ð B.ð B.ð B.ð B.rA   r‚  c                   ó"   — e Zd Zdd„Zd„ Zd„ ZdS )	ÚEsmCategoricalMixtureé2   r   r   c                 ó®   — || _         t          j        |||dz   | j         j        | j         j        ¬¦  «        }|d d…         |dd …         z   dz  | _        d S )Nr   ©ra   rh   r   rë   )Úlogitsr=   Úlinspacera   rh   Úv_bins)r«   ÚparamÚbinsrE  rF  s        rB   r¨   zEsmCategoricalMixture.__init__³  sX   € àˆŒÝŒ~˜e S¨$°©(¸4¼;Ô;MÐUYÔU`ÔUfÐgÑgÔgˆØ˜C˜R˜C”y 4¨¨¨¤8Ñ+¨qÑ0ˆŒˆˆrA   c                 óR  — |                      d¦  «        | j        d g|j        z           z
                       ¦   «                              d¦  «        }| j                             d¦  «        }t          j        ||                      d¦  «        d¬¦  «         	                    d¦  «        S )Nr   rW   )
r  rá  ÚndimÚabsÚargminrß  Úlog_softmaxr=   Útake_along_dimÚsqueeze)r«   ÚtrueÚ
true_indexÚnlls       rB   Úlog_probzEsmCategoricalMixture.log_prob¹  s�   € ð —n’n RÑ(Ô(¨4¬;¸°vÀÄ	Ñ7IÔ+JÑJ×OÒOÑQÔQ×XÒXÐY[Ñ\Ô\ˆ
ØŒk×%Ò% bÑ)Ô)ˆÝÔ# C¨×)=Ò)=¸bÑ)AÔ)AÀrÐJÑJÔJ×RÒRÐSUÑVÔVÐVrA   c                 ó’   — | j                              d¦  «        | j                             d¦  «        z                       d¦  «        S )Nr   r   )rß  rÊ   rá  r  rê  )r«   s    rB   ÚmeanzEsmCategoricalMixture.meanÁ  s;   € Ø”×#Ò# BÑ'Ô'¨$¬+×*?Ò*?ÀÑ*BÔ*BÑB×KÒKÈBÑOÔOÐOrA   N)rÜ  r   r   )r9   r:   r;   r¨   rî  rð  r@   rA   rB   rÛ  rÛ  ²  sN   € € € € € ð1ð 1ð 1ð 1ðWð Wð WðPð Pð Pð Pð PrA   rÛ  rÜ  c                 óH   — t          | |¬¦  «                             ¦   «         S )N©rã  )rÛ  rð  )rß  rã  s     rB   Úcategorical_lddtró  Å  s!   € å  ¨dÐ3Ñ3Ô3×8Ò8Ñ:Ô:Ð:rA   c                 ó"  — | €dS t          | j        ¦  «        dk    r%t          dt          | j        ¦  «        › d�¦  «        ‚| j        \  }}|                      d¦  «                             |||¦  «        }|                     ||z  |¦  «        }|S )zÍ
    Helper to convert B x L mask of valid positions to axial mask used in row column attentions.

    Input:
      mask: B x L tensor of booleans

    Output:
      mask: B x L x L tensor of booleans
    Nrë   rÓ  rÒ  r   )rn   rg   rª   r  Úexpandr€   )r  r¨  rØ  Úms       rB   Úget_axial_maskr÷  Ê  s�   € ð €|Øˆtå
ˆ4Œ:�„˜!ÒÐÝÐU½cÀ$Ä*¹o¼oÐUÐUÐUÑVÔVÐVØœÑ€IˆwØ�Š�qÑÔ× Ò  ¨G°WÑ=Ô=€AØ	�	Š	�)˜gÑ% wÑ/Ô/€AØ€HrA   c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )ÚEsmFoldRelativePositionc                 óÊ   •— t          ¦   «                              ¦   «          |j        | _        t          j                             d| j        z  dz   |j        ¦  «        | _        d S ©Nrë   )	r§   r¨   Úposition_binsrã  r=   r¸   Ú	Embeddingr  Ú	embedding©r«   r&  r¬   s     €rB   r¨   z EsmFoldRelativePosition.__init__á  sP   ø€ Ý‰Œ×ÒÑÔÐØÔ(ˆŒ	õ œ×+Ò+¨A°´	©M¸AÑ,=¸vÔ?XÑYÔYˆŒˆˆrA   Nc                 óÜ  — |j         t          j        k    rt          d|j         › d�¦  «        ‚|�0|j        |j        k    r t          d|j        › d|j        › d�¦  «        ‚|dd…ddd…f         |dd…dd…df         z
  }|                     | j         | j        ¦  «        }|| j        z   dz   }|�(|dd…ddd…f         |dd…dd…df         z  }d||d	k    <   |                      |¦  «        }|S )
zÚ
        Input:
          residue_index: B x L tensor of indices (dtype=torch.long) mask: B x L tensor of booleans

        Output:
          pairwise_state: B x L x L x pairwise_state_dim tensor of embeddings
        z`residue_index` has dtype z, it should be `torch.long`.Nz5`residue_index` and `mask` have inconsistent shapes: rÔ  rÕ  r   r   F)rh   r=   Úlongrª   rg   Úclamprã  rþ  )r«   r1   r  r¸  rþ   s        rB   rÇ   zEsmFoldRelativePosition.forwardé  s+  € ð Ô¥%¤*Ò,Ð,ÝÐk¸-Ô:MÐkÐkÐkÑlÔlÐlØÐ Ô 3°t´zÒ AÐ AÝØnÈÔH[ÐnÐnÐaeÔakÐnÐnÐnñô ð ð ˜Q˜Q˜Q  a a a˜ZÔ(¨=¸¸¸¸A¸A¸A¸t¸Ô+DÑDˆØ�zŠz˜4œ9˜* d¤iÑ0Ô0ˆØ�d”iÑ !Ñ#ˆàÐØ˜˜˜˜4   ˜
Ô# d¨1¨1¨1¨a¨a¨a°¨:Ô&6Ñ6ˆDØ"#ˆD�˜’Ñà—’ Ñ%Ô%ˆØˆrA   rD   rÈ   r±   s   @rB   rù  rù  à  sQ   ø€ € € € € ðZð Zð Zð Zð Zðð ð ð ð ð ð ð rA   rù  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚEsmFoldAngleResnetBlockc                 óü   •— t          ¦   «                              ¦   «          t          |j        |j        d¬¦  «        | _        t          |j        |j        d¬¦  «        | _        t          j        ¦   «         | _        d S ©Nr¢   rÖ   r¦   )	r§   r¨   r™   Ú
resnet_dimÚlinear_1Úlinear_2r¸   rÄ  r¢   rÿ  s     €rB   r¨   z EsmFoldAngleResnetBlock.__init__  sd   ø€ Ý‰Œ×ÒÑÔÐå% fÔ&7¸Ô9JÐQWÐXÑXÔXˆŒÝ% fÔ&7¸Ô9JÐQXÐYÑYÔYˆŒå”G‘I”IˆŒ	ˆ	ˆ	rA   r'  rT   c                 ó¸   — |}|                       |¦  «        }|                      |¦  «        }|                       |¦  «        }|                      |¦  «        }||z   S rD   )r¢   r  r	  )r«   r'  Ú	s_initials      rB   rÇ   zEsmFoldAngleResnetBlock.forward  sQ   € Øˆ	à�IŠI�a‰LŒLˆØ�MŠM˜!ÑÔˆØ�IŠI�a‰LŒLˆØ�MŠM˜!ÑÔˆà�9‰}ÐrA   )r9   r:   r;   r¨   r=   ro   rÇ   r°   r±   s   @rB   r  r    s^   ø€ € € € € ðð ð ð ð ð˜œð ¨%¬,ð ð ð ð ð ð ð ð rA   r  c                   ón   ‡ — e Zd ZdZˆ fd„Zdej        dej        deej        ej        f         fd„Zˆ xZ	S )ÚEsmFoldAngleResnetz.
    Implements Algorithm 20, lines 11-14
    c                 óö  •— t          ¦   «                              ¦   «          || _        t          |j        |j        ¦  «        | _        t          |j        |j        ¦  «        | _        t          j	        ¦   «         | _
        t          |j        ¦  «        D ]+}t          |¦  «        }| j
                             |¦  «         Œ,t          |j        |j        dz  ¦  «        | _        t          j        ¦   «         | _        d S rû  )r§   r¨   r&  r™   Úsequence_dimr  Ú	linear_inÚlinear_initialr¸   Ú
ModuleListÚlayersru   Únum_resnet_blocksr  ÚappendÚ
num_anglesÚ
linear_outrÄ  r¢   )r«   r&  rB  Úlayerr¬   s       €rB   r¨   zEsmFoldAngleResnet.__init__  sÉ   ø€ Ý‰Œ×ÒÑÔÐØˆŒå& vÔ':¸FÔ<MÑNÔNˆŒÝ+¨FÔ,?ÀÔARÑSÔSˆÔå”m‘o”oˆŒÝ�vÔ/Ñ0Ô0ð 	&ð 	&ˆAÝ+¨FÑ3Ô3ˆEØŒK×Ò˜uÑ%Ô%Ð%Ð%å'¨Ô(9¸6Ô;LÈqÑ;PÑQÔQˆŒå”G‘I”IˆŒ	ˆ	ˆ	rA   rË   r  rT   c           	      ó&  — |                       |¦  «        }|                      |¦  «        }|                       |¦  «        }|                      |¦  «        }||z   }| j        D ]} ||¦  «        }Œ|                       |¦  «        }|                      |¦  «        }|                     |j        dd…         dz   ¦  «        }|}t          j        t          j	        t          j
        |dz  dd¬¦  «        | j        j        ¬¦  «        ¦  «        }||z  }||fS )a  
        Args:
            s:
                [*, C_hidden] single embedding
            s_initial:
                [*, C_hidden] single embedding as of the start of the StructureModule
        Returns:
            [*, no_angles, 2] predicted angles
        Nr   )r   rë   rë   T)rX   Úkeepdim)Úmin)r¢   r  r  r  r  rã   rg   r=   ræ   r  rø   r&  Úepsilon)r«   rË   r  ÚlÚunnormalized_sÚ
norm_denoms         rB   rÇ   zEsmFoldAngleResnet.forward-  s  € ð —I’I˜iÑ(Ô(ˆ	Ø×'Ò'¨	Ñ2Ô2ˆ	Ø�IŠI�a‰LŒLˆØ�NŠN˜1ÑÔˆØ�	‰Mˆà”ð 	ð 	ˆAØ��!‘”ˆAˆAà�IŠI�a‰LŒLˆð �OŠO˜AÑÔˆð �FŠF�1”7˜3˜B˜3”< 'Ñ)Ñ*Ô*ˆàˆÝ”ZÝŒKÝ”	˜!˜Q™$ B°Ð5Ñ5Ô5Ø”KÔ'ðñ ô ñ
ô 
ˆ
ð �
‰Nˆà˜qÐ Ð rA   ©
r9   r:   r;   r<   r¨   r=   ro   rq   rÇ   r°   r±   s   @rB   r  r    s   ø€ € € € € ðð ðð ð ð ð ð )!˜œð )!°%´,ð )!À5ÈÌÐW\ÔWcÐIcÔCdð )!ð )!ð )!ð )!ð )!ð )!ð )!ð )!rA   r  c                   ó–   ‡ — e Zd ZdZˆ fd„Z	 	 ddej        dej        dz  dedej        d	ed
e	ej                 dz  dej        fd„Z
ˆ xZS )r  z"
    Implements Algorithm 22.
    c                 óV  •— t          ¦   «                              ¦   «          || _        |j        }|j        }|j        | _        |j        | _        |j	        | _	        |j
        | _
        |j        |j        z  }t          ||¦  «        | _        t          |d|z  ¦  «        | _        |j        |j	        z  dz  }t          ||¦  «        | _        |j        |j	        |j
        z   z  dz  }t          ||¦  «        | _        t          ||j        ¦  «        | _        t#          j        t'          j        |j        ¦  «        ¦  «        | _        |j        ||j        z   |j
        dz  z   z  }t          ||d¬¦  «        | _        t#          j        d¬¦  «        | _        t#          j        ¦   «         | _        d S )Nrë   r   r½  r¦   rÖ   r   rW   )r§   r¨   r&  r  Úpairwise_dimÚipa_dimÚ
hidden_dimÚnum_heads_ipar—  Únum_qk_pointsÚnum_v_pointsr™   r×   Ú	linear_kvÚlinear_q_pointsÚlinear_kv_pointsÚlinear_br¸   r¹   r=   r¼   r�  r  ÚSoftmaxrÊ   ÚSoftplusÚsoftplus)	r«   r&  Úc_sÚc_zÚhcÚhpqÚhpkvÚconcat_out_dimr¬   s	           €rB   r¨   z'EsmFoldInvariantPointAttention.__init__^  sw  ø€ Ý‰Œ×ÒÑÔÐØˆŒàÔ!ˆØÔ!ˆØ œ.ˆŒØÔ-ˆŒØ#Ô1ˆÔØ"Ô/ˆÔð Œ^˜fÔ2Ñ2ˆÝ% c¨2Ñ.Ô.ˆŒÝ& s¨A°©FÑ3Ô3ˆŒàÔ" VÔ%9Ñ9¸AÑ=ˆÝ,¨S°#Ñ6Ô6ˆÔàÔ# vÔ';¸fÔ>QÑ'QÑRÐUVÑVˆÝ -¨c°4Ñ 8Ô 8ˆÔå% c¨6Ô+?Ñ@Ô@ˆŒåœL­¬°VÔ5IÑ)JÔ)JÑKÔKˆÔàÔ-°°v´~Ñ1EÈÔH[Ð^_ÑH_Ñ1_Ñ`ˆÝ'¨¸À'ÐJÑJÔJˆŒå”z bÐ)Ñ)Ô)ˆŒÝœ™œˆŒˆˆrA   FNrË   r/  r§  r  Ú_offload_inferenceÚ_z_reference_listrT   c           	      óî  — |g}|                       |¦  «        }|                      |¦  «        }|                     |j        dd…         | j        dfz   ¦  «        }|                     |j        dd…         | j        dfz   ¦  «        }t          j        || j        d¬¦  «        \  }	}
|                      |¦  «        }t          j        ||j        d         dz  d¬¦  «        }t          j	        |d¬¦  «        }|d          
                    |¦  «        }|                     |j        dd…         | j        | j        dfz   ¦  «        }|                      |¦  «        }t          j        ||j        d         dz  d¬¦  «        }t          j	        |d¬¦  «        }|d          
                    |¦  «        }|                     |j        dd…         | j        ddfz   ¦  «        }t          j        || j        | j        gd¬¦  «        \  }}|                      |d         ¦  «        }|r=t          j        |d         ¦  «        dk    sJ ‚|d                              ¦   «         |d<   |j        j        d	k    r|j        j        nd
}t)          |¦  «        rt+          |d¬¦  «        5  t          j        t/          |                     ¦   «         d¦  «        t/          |	                     ¦   «         d¦  «        ¦  «        }ddd¦  «         n# 1 swxY w Y   n1t          j        t/          |d¦  «        t/          |	d¦  «        ¦  «        }|t3          j        dd| j        z  z  ¦  «        z  }|t3          j        d¦  «        t/          |d¦  «        z  z  }|                     d¦  «        |                     d¦  «        z
  }|dz  }t9          t          j        |d¬¦  «        ¦  «        } |                      | j        ¦  «        j        dtA          |j        dd…         ¦  «        z  dz   Ž }|t3          j        dd| j        dz  dz  z  z  ¦  «        z  }||z  }t          j        |d¬¦  «        dz  }|                     d¦  «        |                     d¦  «        z  }| j!        j"        |dz
  z  }t/          |d¦  «        }||z   }||                     d¦  «        z   }|  #                    |¦  «        }t          j        ||
 $                    dd¦  «         %                    |j&        ¬¦  «        ¦  «         $                    dd¦  «        }tO          |d¦  «        }t          j        |dddd…dd…df         t/          |d¦  «        dddd…dd…f         z  d¬¦  «        }t/          |d¦  «        }|d          (                    |¦  «        }tO          t          j        t          j        |dz  d¬¦  «        | j!        j)        z   ¦  «        d¦  «        } |j*        g |j        dd…         ¢d‘d‘R Ž }|r#|d          %                    |j        ¦  «        |d<   t          j        | $                    dd¦  «        |d          %                    |j&        ¬¦  «        ¦  «        }tO          |d¦  «        }|  +                    t          j,        |gt          j        |d¬¦  «        ¢|‘|‘R d¬¦  «         %                    |d         j&        ¬¦  «        ¦  «        }|S )ab  
        Args:
            s:
                [*, N_res, C_s] single representation
            z:
                [*, N_res, N_res, C_z] pair representation
            r:
                [*, N_res] transformation object
            mask:
                [*, N_res] mask
        Returns:
            [*, N_res, C_s] single representation update
        Nr   rW   r   ).Nrá   r   rë   ro  rp  Fr¿   )r   r   rë   r*  rv  gUUUUUUÕ?r  r  éûÿÿÿ)r   )r   r   g      "@r•  r   râ   rÁ   .)r   r   r   rë   )rë   r   r   r   ).NN)-r×   r)  rã   rg   r—  r=   Úsplitr%  r*  ÚstackÚapplyr'  r+  r(  r,  ÚsysÚgetrefcountrp  ra   rq  rK   r   rù   rŠ   rr  rå   ræ   r  rø   Úunbindr/  r�  rn   r&  r  rÊ   rä   rÅ   rh   r�   Úinvert_applyr  r€   r  rµ  )r«   rË   r/  r§  r  r6  r7  rç   Úkvrl   r–   Úq_ptsÚkv_ptsÚk_ptsÚv_ptsrÿ   rH   r'  Úpt_attr�  Úsquare_maskré   Úo_ptÚ	o_pt_normÚo_pairs                            rB   rÇ   z&EsmFoldInvariantPointAttention.forward�  sñ  € ð, ˆCˆð �MŠM˜!ÑÔˆØ�^Š^˜AÑÔˆð �FŠF�1”7˜3˜B˜3”< 4¤>°2Ð"6Ñ6Ñ7Ô7ˆð �WŠW�R”X˜c˜r˜c”] d¤n°bÐ%9Ñ9Ñ:Ô:ˆõ Œ{˜2˜tœ°BÐ7Ñ7Ô7‰ˆˆ1ð ×$Ò$ QÑ'Ô'ˆõ ”˜E 5¤;¨r¤?°aÑ#7¸RÐ@Ñ@Ô@ˆÝ”˜E rÐ*Ñ*Ô*ˆØ�)”×"Ò" 5Ñ)Ô)ˆð —
’
˜5œ; s¨ sÔ+¨t¬~¸tÔ?QÐSTÐ.UÑUÑVÔVˆð ×&Ò& qÑ)Ô)ˆõ ”˜V V¤\°"Ô%5¸Ñ%:ÀÐCÑCÔCˆÝ”˜V¨Ð,Ñ,Ô,ˆØ�9”×#Ò# FÑ+Ô+ˆð —’˜Vœ\¨#¨2¨#Ô.°$´.À"ÀaÐ1HÑHÑIÔIˆõ ”{ 6¨DÔ,>ÀÔ@QÐ+RÐXZÐ[Ñ[Ô[‰ˆˆuð �MŠM˜!˜Aœ$ÑÔˆàð 	Ý”? 1 Q¤4Ñ(Ô(¨AÒ-Ð-Ð-Ð-Ø�Q”4—8’8‘:”:ˆAˆa‰Dð ()¤x¤}¸Ò'=Ð'=�a”h”m�mÀ5ˆÝ˜;Ñ'Ô'ð 
	Ý¨KÀÐGÑGÔGð ð Ý”LÝ& q§w¢w¡y¤y°)Ñ<Ô<Ý& q§w¢w¡y¤y°)Ñ<Ô<ñô �ðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð øõ ”Ý" 1 iÑ0Ô0Ý" 1 iÑ0Ô0ñô ˆAð
 	
�TŒY�s˜a $¤/Ñ1Ñ2Ñ3Ô3Ñ3ˆØ	�TŒY�wÑÔÕ"4°Q¸	Ñ"BÔ"BÑBÑBˆð —’ Ñ$Ô$ u§¢°rÑ':Ô':Ñ:ˆØ˜‘ˆõ •U”\ &¨bÐ1Ñ1Ô1Ñ2Ô2ˆØ<�t—}’} TÔ%6Ñ7Ô7Ô<¸tÅcÈ&Ì,ÐWZÐXZÐWZÔJ[ÑF\ÔF\Ñ?\Ð_fÑ?fÐhˆØ#¥d¤i°°q¸DÔ<NÐQTÑ<TÐWXÑ<XÑ7YÑ0ZÑ&[Ô&[Ñ[ˆØ˜,Ñ&ˆõ ”˜6 rÐ*Ñ*Ô*¨dÑ3ˆà—n’n RÑ(Ô(¨4¯>ª>¸"Ñ+=Ô+=Ñ=ˆØ”k”o¨°q©Ñ9ˆõ $ F¨IÑ6Ô6ˆà�‰JˆØ�×%Ò% bÑ)Ô)Ñ)ˆØ�LŠL˜‰OŒOˆõ ŒL˜˜AŸKšK¨¨BÑ/Ô/×2Ò2¸¼Ð2ÑAÔAÑBÔB×LÒLÈRÐQSÑTÔTˆõ ˜q !Ñ$Ô$ˆõ ŒyØˆs�D˜!˜!˜!˜Q˜Q˜Q Ð$Ô%Õ(:¸5À,Ñ(OÔ(OÐPSÐUYÐ[\Ð[\Ð[\Ð^_Ð^_Ð^_ÐP_Ô(`Ñ`Øð
ñ 
ô 
ˆõ " $¨Ñ5Ô5ˆØ�Ô!×.Ò.¨tÑ4Ô4ˆõ '¥u¤zµ%´)¸DÀ!¹GÈÐ2LÑ2LÔ2LÈtÌ{ÔObÑ2bÑ'cÔ'cÐefÑgÔgˆ	ð ˆtŒ|Ð4˜TœZ¨¨¨œ_Ð4¨bÐ4°!Ð4Ð4Ð4ˆàð 	(Ø�Q”4—7’7˜4œ;Ñ'Ô'ˆAˆa‰Dõ ”˜aŸkšk¨"¨bÑ1Ô1°1°Q´4·7²7ÀÄ°7Ñ3IÔ3IÑJÔJˆõ $ F¨AÑ.Ô.ˆð �OŠOÝŒI�qÐI�5œ<¨°"Ð5Ñ5Ô5ÐI°yÐIÀ&ÐIÐIÈrÐRÑRÔR×UÒUÐ\]Ð^_Ô\`Ô\fÐUÑgÔgñ
ô 
ˆð ˆs   ÊAK:Ë:K>ÌK>)FN)r9   r:   r;   r<   r¨   r=   ro   r   r®   r   rÇ   r°   r±   s   @rB   r  r  Y  sÑ   ø€ € € € € ðð ð!&ð !&ð !&ð !&ð !&ðR $)Ø;?ð[ð [àŒ<ð[ð Œ<˜$Ñð[ð ð	[ð
 Œlð[ð !ð[ð $ E¤LÔ1°DÑ8ð[ð 
Œð[ð [ð [ð [ð [ð [ð [ð [rA   r  c                   ó`   ‡ — e Zd ZdZˆ fd„Zdej        deej        ej        f         fd„Zˆ xZ	S )ÚEsmFoldBackboneUpdatez*
    Implements part of Algorithm 23.
    c                 ó€   •— t          ¦   «                              ¦   «          t          |j        dd¬¦  «        | _        d S )Né   r¦   rÖ   )r§   r¨   r™   r  r  rÿ  s     €rB   r¨   zEsmFoldBackboneUpdate.__init__$  s6   ø€ Ý‰Œ×ÒÑÔÐå# FÔ$7¸ÀÐIÑIÔIˆŒˆˆrA   rË   rT   c                 ó0   — |                       |¦  «        }|S )z‚
        Args:
            [*, N_res, C_s] single representation
        Returns:
            [*, N_res, 6] update vector
        )r  )r«   rË   Úupdates      rB   rÇ   zEsmFoldBackboneUpdate.forward)  s   € ð —’˜Q‘”ˆàˆrA   r   r±   s   @rB   rL  rL    sy   ø€ € € € € ðð ðJð Jð Jð Jð Jð

˜œð 
¨%°´¸e¼lÐ0JÔ*Kð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
rA   rL  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú%EsmFoldStructureModuleTransitionLayerc                 ó>  •— t          ¦   «                              ¦   «          t          |j        |j        d¬¦  «        | _        t          |j        |j        d¬¦  «        | _        t          |j        |j        d¬¦  «        | _        t          j        ¦   «         | _	        d S r  )
r§   r¨   r™   r  r  r	  Úlinear_3r¸   rÄ  r¢   rÿ  s     €rB   r¨   z.EsmFoldStructureModuleTransitionLayer.__init__7  s�   ø€ Ý‰Œ×ÒÑÔÐå% fÔ&9¸6Ô;NÐU[Ð\Ñ\Ô\ˆŒÝ% fÔ&9¸6Ô;NÐU[Ð\Ñ\Ô\ˆŒÝ% fÔ&9¸6Ô;NÐU\Ð]Ñ]Ô]ˆŒå”G‘I”IˆŒ	ˆ	ˆ	rA   c                 óæ   — |}|                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z   }|S rD   )r  r¢   r	  rT  )r«   rË   r  s      rB   rÇ   z-EsmFoldStructureModuleTransitionLayer.forward@  sf   € Øˆ	Ø�MŠM˜!ÑÔˆØ�IŠI�a‰LŒLˆØ�MŠM˜!ÑÔˆØ�IŠI�a‰LŒLˆØ�MŠM˜!ÑÔˆà�	‰MˆàˆrA   rÈ   r±   s   @rB   rR  rR  6  sG   ø€ € € € € ðð ð ð ð ð
ð 
ð 
ð 
ð 
ð 
ð 
rA   rR  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú EsmFoldStructureModuleTransitionc                 ót  •— t          ¦   «                              ¦   «          || _        t          j        ¦   «         | _        t          |j        ¦  «        D ]+}t          |¦  «        }| j         	                    |¦  «         Œ,t          j
        |j        ¦  «        | _        t          |j        ¦  «        | _        d S rD   )r§   r¨   r&  r¸   r  r  ru   Únum_transition_layersrR  r  rª  Údropout_rater«  r   r  rÄ   )r«   r&  rB  r  r¬   s       €rB   r¨   z)EsmFoldStructureModuleTransition.__init__N  s—   ø€ Ý‰Œ×ÒÑÔÐØˆŒå”m‘o”oˆŒÝ�vÔ3Ñ4Ô4ð 	"ð 	"ˆAÝ5°fÑ=Ô=ˆAØŒK×Ò˜qÑ!Ô!Ð!Ð!å”z &Ô"5Ñ6Ô6ˆŒÝ# FÔ$7Ñ8Ô8ˆŒˆˆrA   c                 ó„   — | j         D ]} ||¦  «        }Œ|                      |¦  «        }|                      |¦  «        }|S rD   )r  r«  rÄ   )r«   rË   r  s      rB   rÇ   z(EsmFoldStructureModuleTransition.forwardZ  sG   € Ø”ð 	ð 	ˆAØ��!‘”ˆAˆAà�LŠL˜‰OŒOˆØ�OŠO˜AÑÔˆàˆrA   rÈ   r±   s   @rB   rW  rW  M  sG   ø€ € € € € ð
9ð 
9ð 
9ð 
9ð 
9ðð ð ð ð ð ð rA   rW  c                   ó<   ‡ — e Zd Zˆ fd„Z	 	 dd„Zd„ Zd„ Zd„ Zˆ xZS )	ÚEsmFoldStructureModulec                 ó  •— t          ¦   «                              ¦   «          || _        t          |j        ¦  «        | _        t          |j        ¦  «        | _        t          |j        |j        ¦  «        | _	        t          |¦  «        | _        t          j        |j        ¦  «        | _        t          |j        ¦  «        | _        t#          |¦  «        | _        t'          |¦  «        | _        t+          |¦  «        | _        d S rD   )r§   r¨   r&  r   r  Úlayer_norm_sr#  Úlayer_norm_zr™   r  r  Úipar¸   rª  rZ  Úipa_dropoutÚlayer_norm_iparW  Ú
transitionrL  Ú	bb_updater  Úangle_resnetrÿ  s     €rB   r¨   zEsmFoldStructureModule.__init__e  sÊ   ø€ Ý‰Œ×ÒÑÔÐØˆŒõ & fÔ&9Ñ:Ô:ˆÔÝ% fÔ&9Ñ:Ô:ˆÔå& vÔ':¸FÔ<OÑPÔPˆŒå1°&Ñ9Ô9ˆŒåœ: fÔ&9Ñ:Ô:ˆÔÝ'¨Ô(;Ñ<Ô<ˆÔå:¸6ÑBÔBˆŒÝ.¨vÑ6Ô6ˆŒÝ.¨vÑ6Ô6ˆÔÐÐrA   NFc           
      ó  — |d         }|€"|                      |j        dd…         ¦  «        }|                      |¦  «        }|                      |d         ¦  «        }d}|rBt	          j        |d         ¦  «        dk    sJ ‚|d                              ¦   «         |d<   |g}d}|}|                      |¦  «        }t          j	        |j        dd…         |j
        |j        | j        d¬¦  «        }	g }
t          | j        j        ¦  «        D �]·}||                      |||	|||¬¦  «        z   }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|	                     |                      |¦  «        ¦  «        }	t          t-          |	                     ¦   «                              ¦   «         d¬	¦  «        |	                     ¦   «         ¦  «        }|                     | j        j        ¦  «        }|                      ||¦  «        \  }}|                      |||¦  «        }|                      ||¦  «        }|	                     | j        j        ¦  «        }|                     ¦   «         |                      ¦   «         ||||d
œ}|
 !                    |¦  «         |	 "                    ¦   «         }	�Œ¹~~|r#|d          #                    |j        ¦  «        |d<   tI          tJ          j&        |
¦  «        }
||
d<   |
S )a×  
        Args:
            evoformer_output_dict:
                Dictionary containing:
                    "single":
                        [*, N_res, C_s] single representation
                    "pair":
                        [*, N_res, N_res, C_z] pair representation
            aatype:
                [*, N_res] amino acid indices
            mask:
                Optional [*, N_res] sequence mask
        Returns:
            A dictionary of outputs
        ÚsingleNr   r5  rë   Úquat)Úfmt)r6  r7  )Úrot_matsÚquats)r"   r#   r$   r%   r&   r'   )'r  rg   r_  r`  r=  r>  rp  r  r   Úidentityrh   ra   Útrainingru   r&  Ú
num_blocksra  rb  rc  rd  Úcompose_q_update_vecre  r   Úget_rotsÚget_rot_matsÚ	get_transÚscale_translationÚtrans_scale_factorrf  r   r   Úto_tensor_7Úto_tensor_4x4r  Ústop_rot_gradientrÅ   r‘   r=   r;  )r«   Úevoformer_output_dictr,   r  r6  rË   r/  Úz_reference_listr  ÚrigidsÚoutputsrx   Úbackb_to_globalr$   r%   Úall_frames_to_globalÚpred_xyzÚscaled_rigidsÚpredss                      rB   rÇ   zEsmFoldStructureModule.forward}  s'  € ð, " (Ô+ˆàˆ<à—:’:˜aœg c r cœlÑ+Ô+ˆDð ×Ò˜aÑ Ô ˆð ×ÒÐ3°FÔ;Ñ<Ô<ˆàÐØð 	Ý”?Ð#8¸Ô#@ÑAÔAÀQÒFÐFÐFÐFØ,AÀ&Ô,I×,MÒ,MÑ,OÔ,OÐ! &Ñ)Ø !˜sÐØˆAð ˆ	Ø�NŠN˜1ÑÔˆõ ”ØŒG�C�R�CŒLØŒGØŒHØŒMØð
ñ 
ô 
ˆð ˆÝ�t”{Ô-Ñ.Ô.ð /	0ñ /	0ˆAà�D—H’HØØØØØ#5Ø"2ð ñ ô ñ ˆAð × Ò  Ñ#Ô#ˆAØ×#Ò# AÑ&Ô&ˆAØ—’ Ñ"Ô"ˆAð ×0Ò0°·²ÀÑ1BÔ1BÑCÔCˆFõ
 $Ý &§/¢/Ñ"3Ô"3×"@Ò"@Ñ"BÔ"BÈ$ÐOÑOÔOØ× Ò Ñ"Ô"ñô ˆOð
 .×?Ò?ÀÄÔ@^Ñ_Ô_ˆOð +/×*;Ò*;¸A¸yÑ*IÔ*IÑ'Ð à#'×#@Ò#@ÀÐRXÐZ`Ñ#aÔ#aÐ à×IÒIÐJ^Ð`fÑgÔgˆHà"×4Ò4°T´[Ô5SÑTÔTˆMð (×3Ò3Ñ5Ô5Ø$8×$FÒ$FÑ$HÔ$HØ':Ø Ø%Øðð ˆEð �NŠN˜5Ñ!Ô!Ð!à×-Ò-Ñ/Ô/ˆF‰FàÐàð 	WØ,AÀ&Ô,I×,LÒ,LÈQÌXÑ,VÔ,VÐ! &Ñ)å¥¤¨WÑ5Ô5ˆØˆ�ÑàˆrA   c           	      óH  — t          | d¦  «        s8|                      dt          j        t          j        ||d¬¦  «        d¬¦  «         t          | d¦  «        s7|                      dt          j        t          j        |d¬¦  «        d¬¦  «         t          | d¦  «        s8|                      dt          j        t          j        ||d¬¦  «        d¬¦  «         t          | d¦  «        s:|                      dt          j        t          j        ||d¬¦  «        d¬¦  «         d S d S )	NÚdefault_framesF)rh   ra   Úrequires_grad)Ú
persistentÚ	group_idx)ra   r„  Ú	atom_maskÚlit_positions)	ÚhasattrÚregister_bufferr=   r‚   r   Ú!restype_rigid_group_default_frameÚrestype_atom14_to_rigid_groupÚrestype_atom14_maskÚ$restype_atom14_rigid_group_positions)r«   Úfloat_dtypera   s      rB   Ú_init_residue_constantsz.EsmFoldStructureModule._init_residue_constantsî  s‚  € Ý�tÐ-Ñ.Ô.ð 
	Ø× Ò Ø Ý”Ý%ÔGØ%Ø!Ø"'ð	ñ ô ð !ð !ñ 	ô 	ð 	õ �t˜[Ñ)Ô)ð 		Ø× Ò ØÝ”Ý%ÔCØ!Ø"'ðñ ô ð
 !ð !ñ ô ð õ �t˜[Ñ)Ô)ð 
	Ø× Ò ØÝ”Ý%Ô9Ø%Ø!Ø"'ð	ñ ô ð !ð !ñ 	ô 	ð 	õ �t˜_Ñ-Ô-ð 
	Ø× Ò ØÝ”Ý%ÔJØ%Ø!Ø"'ð	ñ ô ð !ð !ñ 	ô 	ð 	ð 	ð 	ð
	ð 
	rA   c                 óp   — |                       |j        |j        ¦  «         t          |||| j        ¦  «        S rD   )r�  rh   ra   r   rƒ  )r«   r§  ÚalphaÚfs       rB   r   z/EsmFoldStructureModule.torsion_angles_to_frames  s3   € à×$Ò$ U¤[°%´,Ñ?Ô?Ð?å'¨¨5°!°TÔ5HÑIÔIÐIrA   c                 óÚ   — |                       |                     ¦   «         j        |                     ¦   «         j        ¦  «         t	          ||| j        | j        | j        | j        ¦  «        S rD   )	r�  rq  rh   ra   r   rƒ  r†  r‡  rˆ  )r«   r§  r“  s      rB   r   zDEsmFoldStructureModule.frames_and_literature_positions_to_atom14_pos!  s\   € à×$Ò$ Q§Z¢Z¡\¤\Ô%7¸¿º¹¼Ô9LÑMÔMÐMÝ<ØØØÔØŒNØŒNØÔñ
ô 
ð 	
rA   )NF)	r9   r:   r;   r¨   rÇ   r�  r   r   r°   r±   s   @rB   r]  r]  d  sŠ   ø€ € € € € ð7ð 7ð 7ð 7ð 7ð8 Ø ðoð oð oð oðb+ð +ð +ðZJð Jð Jð

ð 

ð 

ð 

ð 

ð 

ð 

rA   r]  c                   ó@   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zed„ ¦   «         Zˆ xZS )ÚEsmFoldingTrunkc                 ó   •‡— t          ¦   «                              ¦   «          ‰| _        ‰j        }‰j        }t          ‰¦  «        | _        t          j        ˆfd„t          ‰j
        ¦  «        D ¦   «         ¦  «        | _        d| _        t          j        |¦  «        | _        t          j        |¦  «        | _        t          j        | j        |¦  «        | _        | j        j        d                              ¦   «                              ¦   «          t+          ‰j        ¦  «        | _        t          j        |‰j        j        ¦  «        | _        t          j        |‰j        j        ¦  «        | _        ‰j        | _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r@   )r‚  )rZ   rB  r&  s     €rB   r^   z,EsmFoldingTrunk.__init__.<locals>.<listcomp>8  s#   ø€ Ð$sÐ$sÐ$sÐUVÕ%HÈÑ%PÔ%PÐ$sÐ$sÐ$srA   é   r   )r§   r¨   r&  r²  r  rù  Úpairwise_positional_embeddingr¸   r  ru   ro  ÚblocksÚrecycle_binsr   Úrecycle_s_normÚrecycle_z_normrý  Úrecycle_distor»   ÚdetachÚzero_r]  Ústructure_moduler™  r  Ú
trunk2sm_sr#  Ú
trunk2sm_zr	  )r«   r&  r0  r1  r¬   s    `  €rB   r¨   zEsmFoldingTrunk.__init__/  s3  øø€ Ý‰Œ×ÒÑÔÐØˆŒàÔ'ˆØÔ'ˆå-DÀVÑ-LÔ-LˆÔ*å”mÐ$sÐ$sÐ$sÐ$sÕZ_Ð`fÔ`qÑZrÔZrÐ$sÑ$sÔ$sÑtÔtˆŒàˆÔÝ œl¨3Ñ/Ô/ˆÔÝ œl¨3Ñ/Ô/ˆÔÝœ\¨$Ô*;¸SÑAÔAˆÔØÔÔ! !Ô$×+Ò+Ñ-Ô-×3Ò3Ñ5Ô5Ð5å 6°vÔ7NÑ OÔ OˆÔÝœ) C¨Ô)@Ô)MÑNÔNˆŒÝœ) C¨Ô)@Ô)MÑNÔNˆŒà Ô+ˆŒˆˆrA   c                 ó   — || _         d S rD   )r	  )r«   r	  s     rB   Úset_chunk_sizezEsmFoldingTrunk.set_chunk_sizeF  s   € ð
 %ˆŒˆˆrA   c           	      óž  ‡ — |j         }|}|}	|€‰ j        j        }n|dk     rt          d¦  «        ‚|dz  }ˆ fd„}
|}|	}t	          j        |¦  «        }t	          j        |¦  «        }t	          j        |j        dd…         |t          j        dœŽ}t          |¦  «        D �]¢}t          ||dz
  k    rg nt	          j        ¦   «         g¦  «        5  ‰                      |                     ¦   «         ¦  «                             |¦  «        }‰                      |                     ¦   «         ¦  «                             |¦  «        }|‰                      |                     ¦   «         ¦  «                             |¦  «        z  } |
||z   |	|z   ||¦  «        \  }}‰                      ‰                      |¦  «        ‰                      |¦  «        dœ||                     ¦   «         ¦  «        }|}|}t*                               |d	         d         dd…dd…dd
…f         dd‰ j        ¦  «        }ddd¦  «         n# 1 swxY w Y   �Œ¤||d<   ||d<   |S )a~  
        Inputs:
          seq_feats: B x L x C tensor of sequence features pair_feats: B x L x L x C tensor of pair features residx: B
          x L long tensor giving the position in the sequence mask: B x L boolean tensor indicating valid residues

        Output:
          predicted_structure: B x L x (num_atoms_per_residue * 3) tensor wrapped in a Coordinates object
        Nr   z(Number of recycles must not be negative.r   c                 ó„   •— |‰                      ||¬¦  «        z   }‰j        D ]} || |||‰j        ¬¦  «        \  } }Œ| |fS )NrÖ  )r  r1   r	  )rš  r›  r	  )rË   r/  Úresidxr  Úblockr«   s        €rB   Ú
trunk_iterz+EsmFoldingTrunk.forward.<locals>.trunk_iterb  s^   ø€ Ø�D×6Ò6°vÀDÐ6ÑIÔIÑIˆAàœð `ð `�Ø�u˜Q ¨¸FÈtÌÐ_Ñ_Ô_‘��1�1Ø�a�4ˆKrA   r   rÞ  )rh  r5  r&   r   g      @g     `5@r(   r)   )ra   r&  Úmax_recyclesrª   r=   Ú
zeros_liker¼   rg   Úint64ru   r   r©   r�  r   rÅ   rž  rŸ  r¢  r£  r¤  rr  r–  Ú	distogramrœ  )r«   Ú	seq_featsÚ
pair_featsÚtrue_aar©  r  Úno_recyclesra   Ús_s_0Ús_z_0r«  r(   r)   Ú	recycle_sÚ	recycle_zrœ  Úrecycle_idxÚ	structures   `                 rB   rÇ   zEsmFoldingTrunk.forwardM  s­  ø€ ð Ô!ˆØˆØˆàÐØœ+Ô2ˆKˆKà˜QŠˆÝ Ð!KÑLÔLÐLØ˜1ÑˆKð	ð 	ð 	ð 	ð 	ð ˆØˆÝÔ$ SÑ)Ô)ˆ	ÝÔ$ SÑ)Ô)ˆ	Ý”{ C¤I¨c¨r¨c¤N¸6ÍÌÐUÐUÐUˆå  Ñ-Ô-ð 	ñ 	ˆKÝ  {°kÀA±oÒ'EÐ'E  ÍEÌMÉOÌOÐK\Ñ]Ô]ð ð à ×/Ò/°	×0@Ò0@Ñ0BÔ0BÑCÔC×FÒFÀvÑNÔN�	Ø ×/Ò/°	×0@Ò0@Ñ0BÔ0BÑCÔC×FÒFÀvÑNÔN�	Ø˜T×/Ò/°×0CÒ0CÑ0EÔ0EÑFÔF×IÒIÈ&ÑQÔQÑQ�	à%˜: e¨iÑ&7¸ÀÑ9JÈFÐTXÑYÔY‘��Sð !×1Ò1Ø#Ÿš¨sÑ3Ô3¸T¿_º_ÈSÑ=QÔ=QÐRÐRØØ—J’J‘L”Lñô �	ð  �	Ø�	å.×8Ò8Ø˜kÔ*¨2Ô.¨q¨q¨q°!°!°!°R°a°R¨xÔ8ØØØÔ%ñ	 ô  �ð%ð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ùð2 ˆ	�%ÑØˆ	�%ÑàÐs   ÃEH6È6H:	È=H:	c                 óÊ  — t          j        |||dz
  | j        ¬¦  «        }|dz  }d„ |                      dd¬¦  «        D ¦   «         \  }}}||z
  }||z
  }	|                     |	d¬¦  «        }
d	|
z  d
|z  z   d|	z  z
  |z   }|dd d d …d d …f         |dd d …d d d …f         z
                       d¦  «                             dd¬¦  «        }t          j        ||k    d¬¦  «        }|S )Nr   r`   rë   c                 ó8   — g | ]}|                      d ¦  «        ‘ŒS )rá   )rê  rY   s     rB   r^   z-EsmFoldingTrunk.distogram.<locals>.<listcomp>˜  s"   € ÐCÐCÐC a�A—I’I˜b‘M”MÐCÐCÐCrA   r   rá   rW   r   gÆ vlÂ¥â¿g‡O[ŸI-â?g»:Ïñ4Má?.T)rX   Úkeepdims)r=   rà  ra   rŸ  ÚcrossÚpowrø   )ÚcoordsÚmin_binÚmax_binÚnum_binsÚ
boundariesÚNÚCAÚCrÿ   r<  r'  ÚCBÚdistsrã  s                 rB   r¯  zEsmFoldingTrunk.distogramŽ  s  € õ ”^ØØØ�q‰LØ”=ð	
ñ 
ô 
ˆ
ð   ‘]ˆ
ØCÐC¨6¯<ª<¸¸r¨<Ñ+BÔ+BÐCÑCÔC‰ˆˆ2ˆqà�‰FˆØ�‰FˆØ�GŠG�A˜2ˆGÑÔˆØ˜1‰_˜z¨A™~Ñ-°
¸Q±Ñ>ÀÑCˆØ�C˜˜q˜q˜q ! ! !�OÔ$ r¨#¨q¨q¨q°$¸¸¸¨/Ô':Ñ:×?Ò?ÀÑBÔB×FÒFÈ2ÐX\ÐFÑ]Ô]ˆÝŒy˜ Ò+°Ð4Ñ4Ô4ˆØˆrA   )	r9   r:   r;   r¨   r¦  rÇ   Ústaticmethodr¯  r°   r±   s   @rB   r–  r–  .  sr   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð.%ð %ð %ð?ð ?ð ?ðB ðð ñ „\ðð ð ð ð rA   r–  a}  
    ESMForProteinFolding is the HuggingFace port of the original ESMFold model. It consists of an ESM-2 "stem" followed
    by a protein folding "head", although unlike most other output heads, this "head" is similar in size and runtime to
    the rest of the model combined! It outputs a dictionary containing predicted structural information about the input
    protein(s).
    c                   óØ  ‡ — e Zd ZddgZdZdZdZdZˆ fd„Zˆ fd„Z	e
dee         dej        fd	„¦   «         Ze	 	 	 	 	 dd
ej        dej        dz  dej        dz  dej        dz  dedz  dedz  defd„¦   «         Zd„ Zdej        dej        fd„Zd„ Z ej        ¦   «         	 ddeee         z  fd„¦   «         Ze
dedee         fd„¦   «         Zdefd„Zdee         dee         fd„Zˆ xZS )ÚEsmForProteinFoldingr]  r‚  FNc                 óæ   •— t          ¦   «                              |¦  «         t          |t          ¦  «        r9t	          j        |j        |                     |j        j	        ¦  «        ¦  «         d S d S rD   )
r§   rŽ  r�   rË  rž   rW  Ú
af2_to_esmÚ_af2_to_esm_from_vocab_listr&  Ú
vocab_list)r«   r�  r¬   s     €rB   rŽ  z"EsmForProteinFolding._init_weights·  si   ø€ Ý‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ2Ñ3Ô3ð 	hÝŒJ�vÔ(¨&×*LÒ*LÈVÌ]ÔMeÑ*fÔ*fÑgÔgÐgÐgÐgð	hð 	hrA   c           
      óô  •— t          ¦   «                              |¦  «         || _        d| _        t	          |d¬¦  «        | _        | j                             d¦  «         | j        j        j        r| j         	                    ¦   «          | j        j
        | _        | j        j        | j        j        z  | _        | j        j        | _        |                      d|                      |j        ¦  «        ¦  «         t'          j        t+          j        | j        dz   ¦  «        ¦  «        | _        | j        j        j        }|j        }|j        }t'          j        t9          | j        ¦  «        t'          j        | j        |¦  «        t'          j        ¦   «         t'          j        ||¦  «        ¦  «        | _        t@          j!        dz   | _"        d| _#        | j"        dz
  | _$        | j"        dz
  | _%        | j        j         &                    d	¦  «        | _'        | j        j         &                    d
¦  «        | _(        | j        j         &                    d¦  «        | _)        | j        j         &                    d¦  «        | _*        | j        j        j+        r!t'          j,        | j"        |d¬¦  «        | _-        t]          |¦  «        | _        t'          j        || j        ¦  «        | _/        t'          j        || j        ¦  «        | _0        t'          j        || j"        ¦  «        | _1        d| _2        |j3        }t'          j        t'          j        |j4        ¦  «        t'          j        |j4        | j        j        j5        ¦  «        t'          j        | j        j        j5        | j        j        j5        ¦  «        t'          j        | j        j        j5        d| j2        z  ¦  «        ¦  «        | _6        |  7                    ¦   «          d S )Né@   F)Úadd_pooling_layerrÍ  r   r   r   rë   z<cls>z<mask>z<eos>ú<pad>)Úpadding_idxrÜ  é%   )8r§   r¨   r&  Údistogram_binsr   ÚesmÚrequires_grad_Úesmfold_configÚfp16_esmÚhalfÚhidden_sizeÚ	esm_featsÚnum_hidden_layersÚnum_attention_headsÚ	esm_attnsÚ
esm_layersrŠ  rÎ  rÏ  r¸   r¹   r=   r¼   Úesm_s_combineÚtrunkr²  r  rÃ  r   r™  rÄ  Ú	esm_s_mlpr   Úrestype_numÚn_tokens_embedÚpad_idxÚunk_idxÚmask_idxÚindexÚesm_dict_cls_idxÚesm_dict_mask_idxÚesm_dict_eos_idxÚesm_dict_padding_idxÚembed_aarý  rþ  r–  Údistogram_headÚptm_headÚlm_headÚ	lddt_binsr¢  r  Úlddt_head_hid_dimr2   Ú	post_init)r«   r&  Útrunk_configr0  r1  Ústructure_module_configr¬   s         €rB   r¨   zEsmForProteinFolding.__init__¼  s  ø€ Ý‰Œ×Ò˜Ñ Ô Ð àˆŒà ˆÔå˜F°eÐ<Ñ<Ô<ˆŒàŒ×Ò Ñ&Ô&Ð&ØŒ;Ô%Ô.ð 	ØŒH�MŠM‰OŒOˆOàœÔ0ˆŒØœÔ6¸¼Ô9XÑXˆŒØœ+Ô7ˆŒØ×Ò˜\¨4×+KÒ+KÈFÔL]Ñ+^Ô+^Ñ_Ô_Ð_Ýœ\­%¬+°d´oÈÑ6IÑ*JÔ*JÑKÔKˆÔà”{Ô1Ô7ˆØÔ-ˆØÔ-ˆÝœÝ�d”nÑ%Ô%ÝŒI�d”n cÑ*Ô*ÝŒG‰IŒIÝŒI�c˜3ÑÔñ	
ô 
ˆŒõ 0Ô;¸aÑ?ˆÔØˆŒØÔ*¨QÑ.ˆŒØÔ+¨aÑ/ˆŒØ $¤Ô 6× <Ò <¸WÑ EÔ EˆÔØ!%¤Ô!7×!=Ò!=¸hÑ!GÔ!GˆÔØ $¤Ô 6× <Ò <¸WÑ EÔ EˆÔØ$(¤KÔ$:×$@Ò$@ÀÑ$IÔ$IˆÔ!ØŒ;Ô%Ô.ð 	SÝœ\¨$Ô*=¸sÐPQÐRÑRÔRˆDŒNå$ \Ñ2Ô2ˆŒ
å œi¨¨TÔ-@ÑAÔAˆÔÝœ	 # tÔ':Ñ;Ô;ˆŒÝ”y  dÔ&9Ñ:Ô:ˆŒØˆŒØ".Ô"?ÐÝœÝŒLÐ0Ô=Ñ>Ô>ÝŒIÐ-Ô:¸D¼KÔ<VÔ<hÑiÔiÝŒI�d”kÔ0ÔBÀDÄKÔD^ÔDpÑqÔqÝŒI�d”kÔ0ÔBÀBÈÌÑDWÑXÔXñ	
ô 
ˆŒð 	�ŠÑÔÐÐÐrA   rÏ  rT   c                 óŠ   ‡ — ‰                       d¦  «        gˆ fd„t          j        D ¦   «         z   }t          j        |¦  «        S )NrÓ  c                 ó:   •— g | ]}‰                      |¦  «        ‘ŒS r@   )rê  )rZ   r–   rÏ  s     €rB   r^   zDEsmForProteinFolding._af2_to_esm_from_vocab_list.<locals>.<listcomp>ø  s'   ø€ Ð4tÐ4tÐ4tÈQ°Z×5EÒ5EÀaÑ5HÔ5HÐ4tÐ4tÐ4trA   )rê  r   Úrestypes_with_xr=   r‚   )rÏ  Úesm_reorders   ` rB   rÎ  z0EsmForProteinFolding._af2_to_esm_from_vocab_listõ  sI   ø€ ð "×'Ò'¨Ñ0Ô0Ð1Ð4tÐ4tÐ4tÐ4tÕRcÔRsÐ4tÑ4tÔ4tÑtˆÝŒ|˜KÑ(Ô(Ð(rA   Ú	input_idsÚattention_maskÚposition_idsÚmasking_patternÚnum_recyclesÚoutput_hidden_statesc                 ó¼  — | j         j        }|}	|	j        d         }
|	j        d         }|j        }|€t	          j        |	|¬¦  «        }|€)t	          j        ||¬¦  «                             |¦  «        }|                      |	|¦  «        }|�|  	                    |	|||¦  «        \  }}}n|	}d}|  
                    |¦  «        }|                     | j        j        ¦  «        }|j        r|dz  }|                     ¦   «         }| j                             d¦  «                             d¦  «        |z                       d¦  «        }|                      |¦  «        }|                     |
|||j        j        ¦  «        }| j         j        j        r||                      |¦  «        z  }|                      |||	|||¬¦  «        }d„ |                     ¦   «         D ¦   «         }|r||d<   |                      |d	         ¦  «        }||                     dd¦  «        z   dz  }||d
<   |                      |d         ¦  «        }||d<   |	|d<   t;          |¦  «         dD ]%}||xx         |                     d¦  «        z  cc<   Œ&||d<   |                      |d         ¦  «                             |d         j        d         |
|d| j         ¦  «        }||d<   tC          |d         | j         ¬¦  «        }||d<   |  "                    |d	         ¦  «        }||d<   tG          |d| j$        ¬¦  «        |d<   | %                    tM          |d| j$        ¬¦  «        ¦  «         tO          di |¤ŽS )ae  
        masking_pattern (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Locations of tokens to mask during training as a form of regularization. Mask values selected in `[0, 1]`.
        num_recycles (`int`, *optional*, defaults to `None`):
            Number of times to recycle the input sequence. If `None`, defaults to `config.num_recycles`. "Recycling"
            consists of passing the output of the folding trunk back in as input to the trunk. During training, the
            number of recycles should vary with each batch, to ensure that the model learns to output valid predictions
            after each recycle. During inference, num_recycles should be set to the highest value that the model was
            trained with for maximum accuracy. Accordingly, when this value is set to `None`, config.max_recycles is
            used.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, EsmForProteinFolding

        >>> model = EsmForProteinFolding.from_pretrained("facebook/esmfold_v1")
        >>> tokenizer = AutoTokenizer.from_pretrained("facebook/esmfold_v1")
        >>> inputs = tokenizer(["MLKNVQVQLV"], return_tensors="pt", add_special_tokens=False)  # A tiny random peptide
        >>> outputs = model(**inputs)
        >>> folded_positions = outputs.positions
        ```

        r   r   Nr`   rë   )r³  c                 ó"   — i | ]\  }}|d v ¯	||“ŒS ))r)   r(   r"   r#   r$   r%   r&   r'   r@   ©rZ   rl   r–   s      rB   ú
<dictcomp>z0EsmForProteinFolding.forward.<locals>.<dictcomp>N  s@   € ð 
ð 
ð 
á��1Øð	ð
ð 
ð ˆqð
ð 
ð 
rA   Úmlm_targetsr)   r*   r(   r+   r,   )r-   r0   r   r1   r'   r2   rò  r3   r4   é   )rÁ  Úno_binsr5   r@   )(r&  rÙ  rg   ra   r=   Ú	ones_likeÚarangeÚ	expand_asÚaf2_idx_to_esm_idxÚ	bert_maskÚ&compute_language_model_representationsrÅ   râ  rh   Úesm_ablate_sequencer   rÊ   r  rê  rä  r9  rã  r  rï  rþ  rŽ   rð  rä   rò  r   r2   r€   ró  ró  rñ  r   rÖ  rP  r   r!   )r«   rü  rý  rþ  rÿ  r   r  ÚkwargsÚcfgÚaaÚBÚLra   ÚesmaaÚ	masked_aar  Úesm_sr´  rµ  r¹  Údisto_logitsr+   rl   r2   r3   r4   s                             rB   rÇ   zEsmForProteinFolding.forwardû  s“  € ðF ŒkÔ(ˆàˆØŒH�QŒKˆØŒH�QŒKˆØÔ!ˆØÐ!Ý"œ_¨R¸Ð?Ñ?Ô?ˆNØÐÝ œ<¨°&Ð9Ñ9Ô9×CÒCÀIÑNÔNˆLð ×'Ò'¨¨NÑ;Ô;ˆàÐ&Ø,0¯NªN¸2¸uÀnÐVeÑ,fÔ,fÑ)ˆI�u˜k˜kàˆIØˆKð ×;Ò;¸EÑBÔBˆð
 —’˜Ô+Ô1Ñ2Ô2ˆàÔ"ð 	Ø˜A‘IˆEà—’‘”ˆð Ô#×+Ò+¨AÑ.Ô.×8Ò8¸Ñ;Ô;¸eÑC×LÒLÈQÑOÔOˆØ—’˜uÑ%Ô%ˆà—’  1 a¨¬Ô)EÑFÔFˆàŒ;Ô%Ô.ð 	/Ø�T—^’^ IÑ.Ô.Ñ.ˆEàŸ*š* U¨E°2°|À^Ðam˜*ÑnÔnˆ	ð
ð 
à!ŸšÑ)Ô)ð
ñ 
ô 
ˆ	ð" ð 	3Ø'2ˆI�mÑ$à×*Ò*¨9°UÔ+;Ñ<Ô<ˆØ$ |×'=Ò'=¸aÀÑ'CÔ'CÑCÀqÑHˆØ(4ˆ	Ð$Ñ%à—L’L ¨5Ô!1Ñ2Ô2ˆ	Ø!*ˆ	�+Ñà ˆ	�(ÑÝ˜)Ñ$Ô$Ð$ð

ð 	9ð 	9ˆAð �aˆLˆLŒL˜N×4Ò4°RÑ8Ô8Ñ8ˆLˆL‰LˆLØ%1ˆ	�/Ñ"à—N’N 9¨XÔ#6Ñ7Ô7×?Ò?À	È(Ô@SÔ@YÐZ[Ô@\Ð^_ÐabÐdfÐhlÔhvÑwÔwˆ	Ø!*ˆ	�+ÑÝ  ¨2¤°T´^ÐDÑDÔDˆØ"ˆ	�'Ñà—]’] 9¨UÔ#3Ñ4Ô4ˆ
Ø",ˆ	�,ÑÝ% j¸"ÀdÔFYÐZÑZÔZˆ	�%ÑØ×ÒÕ8¸ÈRÐY]ÔYlÐmÑmÔmÑnÔnÐnå)Ð6Ð6¨IÐ6Ð6Ð6rA   c                 óÈ   — | j         j        |j        k    r$| j                              |j        ¦  «        | _         |dz                        |dk    d¦  «        }| j         |         S ©Nr   r   )rÍ  ra   rÅ   r¡  )r«   r  r  s      rB   r  z'EsmForProteinFolding.af2_idx_to_esm_idx‚  sW   € àŒ?Ô! R¤YÒ.Ð.Ø"œo×0Ò0°´Ñ;Ô;ˆDŒOØ�1‰f×!Ò! $¨!¢)¨QÑ/Ô/ˆØŒ˜rÔ"Ð"rA   r  c                 ó„  — t          |                      ¦   «         ¦  «        j        }|j        \  }}| j        j        j        r1t          j        ||| j	        j
        d         d| j        |¬¦  «        }|S | j        | j        }}|                     |df|¦  «        }|                     |df| j        ¦  «        }	t          j        |||	gd¬¦  «        }||t#          |¦  «        |dk                         d¦  «        f<   |                      ||dk    d¬¦  «        d         }
t          j        |
d	¬¦  «        }|d d …dd…f         }|S )
Nr   r   r`   r   rW   T)rý  r  Úhidden_statesrë   )ÚnextÚ
parametersra   rg   r&  rÙ  Ú	bypass_lmr=   r¼   râ  ÚsizerÝ  rë  rí  Únew_fullrî  rµ  ru   rø   r×  r;  )r«   r  ra   r  r  r  ÚbosiÚeosiÚbosÚeosÚesm_hidden_statess              rB   r  z;EsmForProteinFolding.compute_language_model_representations‰  s<  € Ý�d—o’oÑ'Ô'Ñ(Ô(Ô/ˆØŒ{‰ˆˆ1àŒ;Ô%Ô/ð 	Ý”K  1 dÔ&8Ô&=¸aÔ&@À"ÀdÄnÐ]cÐdÑdÔdˆEØˆLàÔ*¨DÔ,AˆdˆØ�nŠn˜a ˜V TÑ*Ô*ˆØ�nŠn˜a ˜V TÔ%>Ñ?Ô?ˆÝ”	˜3  sÐ+°Ð3Ñ3Ô3ˆà/3ˆ�e�A‰hŒh˜ !š×(Ò(¨Ñ+Ô+Ð+Ñ,ð
 !ŸHšH U¸5ÀAº:Ð\`˜HÑaÔaÐbqÔrÐÝ”Ð-°1Ð5Ñ5Ô5ˆà�a�a�a˜˜2˜�g”ˆàˆrA   c                 óÎ   — |                      ¦   «         }|                      ¦   «         }|                      ¦   «         }| j        ||dk    <   d||dk    <   | j        ||dk    <   |||fS r  )rX  ré  rì  )r«   r  r  r  ÚpatternÚnew_aaÚtargetÚ	new_esmaas           rB   r  zEsmForProteinFolding.bert_mask¢  se   € Ø—’‘”ˆØ—’‘”ˆØ—K’K‘M”Mˆ	Ø#œ}ˆˆw˜!Š|ÑØ ˆˆw˜!Š|ÑØ"&Ô"8ˆ	�'˜Q’,ÑØ�y &Ð(Ð(rA   Úseqsc                 ó  ‡‡— t          |t          ¦  «        r|g}n|}t          |                      ¦   «         ¦  «        j        Št          ˆfd„|D ¦   «         ¦  «        Št          ˆfd„|D ¦   «         ¦  «        }|€Bt          j        ‰j        d         ‰¬¦  «         	                    t          |¦  «        d¦  «        n|                     ‰¦  «        }|j        dk    r|                     d¦  «        }|                      ‰||¬¦  «        S )Nc           	      óÄ   •— g | ]\}t          j        t          j        |t          j        d ¬¦  «        ¦  «                             ‰¦  «                             d¬¦  «        ‘Œ]S )T)ÚsequenceÚmappingÚmap_unknown_to_xr   rW   )r=   Ú
from_numpyr   Úsequence_to_onehotÚrestype_order_with_xrÅ   Úargmax)rZ   Úseqra   s     €rB   r^   z.EsmForProteinFolding.infer.<locals>.<listcomp>¸  ss   ø€ ð ð ð ð õ Ô Ý%Ô8Ø!$Ý 1Ô FØ)-ðñ ô ñô ÷ ’�F‘”ß’˜A�‘”ðð ð rA   c                 óT   •— g | ]$}‰                      t          |¦  «        ¦  «        ‘Œ%S r@   )r  rn   )rZ   r6  r,   s     €rB   r^   z.EsmForProteinFolding.infer.<locals>.<listcomp>Å  s+   ø€ Ð%OÐ%OÐ%OÀC f§o¢oµc¸#±h´hÑ&?Ô&?Ð%OÐ%OÐ%OrA   r   r`   r   r   )rþ  )r�   r¯   r  r  ra   r{   r=   r
  rg   rõ  rn   rÅ   rå  r  rÇ   )r«   r,  rþ  rd   r  r,   ra   s        @@rB   ÚinferzEsmForProteinFolding.infer«  s1  øø€ õ �d�CÑ Ô ð 	Ø�&ˆCˆCàˆCå�d—o’oÑ'Ô'Ñ(Ô(Ô/ˆÝ&ðð ð ð ð ðñ ô ñ
ô 
ˆõ %Ð%OÐ%OÐ%OÐ%OÈ3Ð%OÑ%OÔ%OÑPÔPˆð Ð#õ ŒL˜œ aœ°Ð8Ñ8Ô8×?Ò?ÅÀCÁÄÈ"ÑMÔMÐMà—’ Ñ(Ô(ð 	ð
 Ô Ò!Ð!Ø'×1Ò1°!Ñ4Ô4ˆLØ�|Š|ØØØ%ð ñ 
ô 
ð 	
rA   rþ   c           	      ó¶  — d„ |                       ¦   «         D ¦   «         } g }t          | d         d         | ¦  «        }| d         }t          | d         j        d         ¦  «        D ]s}| d         |         }||         }||         }| d         |         dz   }t	          ||||| d	         |         ¬
¦  «        }	|                     t          |	¦  «        ¦  «         Œt|S )zDReturns the pdb (file) string from the model given the model output.c                 ód   — i | ]-\  }}||                      d ¦  «                             ¦   «         “Œ.S )rp  )rÅ   Únumpyr  s      rB   r  z6EsmForProteinFolding.output_to_pdb.<locals>.<dictcomp>Ö  s4   € ÐDÐDÐD©T¨Q°�!�Q—T’T˜%‘[”[×&Ò&Ñ(Ô(ÐDÐDÐDrA   r&   r   r0   r,   r   r1   r   r3   )r,   Úatom_positionsr‡  r1   Ú	b_factors)rŽ   r   ru   rg   r   r  r   )
rþ   ÚpdbsÚfinal_atom_positionsÚfinal_atom_maskrx   r  Úpred_posr  ÚresidÚpreds
             rB   Úoutput_to_pdbz"EsmForProteinFolding.output_to_pdbÓ  só   € ð EÐD°V·\²\±^´^ÐDÑDÔDˆØˆÝ/°°{Ô0CÀBÔ0GÈÑPÔPÐØ Ð!5Ô6ˆÝ�v˜hÔ'Ô-¨aÔ0Ñ1Ô1ð 	&ð 	&ˆAØ˜Ô! !Ô$ˆBØ+¨AÔ.ˆHØ" 1Ô%ˆDØ˜?Ô+¨AÔ.°Ñ2ˆEÝØØ'ØØ#Ø  œ/¨!Ô,ðñ ô ˆDð �KŠK�˜t™œÑ%Ô%Ð%Ð%ØˆrA   c                 óˆ   — t          |t          ¦  «        sJ ‚ | j        |g|¢R i |¤Ž}|                      |¦  «        d         S )úEReturns the pdb (file) string from the model given an input sequence.r   )r�   r¯   r8  rD  ©r«   r,  Úargsr  rþ   s        rB   Ú	infer_pdbzEsmForProteinFolding.infer_pdbé  sR   € å˜$¥Ñ$Ô$Ð$Ð$Ð$Ø�”˜DÐ2 4Ð2Ð2Ð2¨6Ð2Ð2ˆØ×!Ò! &Ñ)Ô)¨!Ô,Ð,rA   c                 óN   —  | j         |g|¢R i |¤Ž}|                      |¦  «        S )rF  )r8  rD  rG  s        rB   Ú
infer_pdbszEsmForProteinFolding.infer_pdbsï  s7   € à�”˜DÐ2 4Ð2Ð2Ð2¨6Ð2Ð2ˆØ×!Ò! &Ñ)Ô)Ð)rA   )NNNNFrD   )r9   r:   r;   Ú_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_attention_backendÚ_can_record_outputsrŽ  r¨   rÉ  r‡   r¯   r=   ro   rÎ  r   r­   r®   r!   rÇ   r  r  r  r©   r8  r�   rD  rI  rK  r°   r±   s   @rB   rË  rË  §  sT  ø€ € € € € ð 2Ð3XÐYÐØ ÐØ€NØ"'ÐàÐðhð hð hð hð hð
7ð 7ð 7ð 7ð 7ðr ð)°°S´	ð )¸e¼lð )ð )ð )ñ „\ð)ð
 ð /3Ø,0Ø/3Ø#'Ø,1ðD7ð D7à”<ðD7ð œ tÑ+ðD7ð ”l TÑ)ð	D7ð
 œ¨Ñ,ðD7ð ˜D‘jðD7ð # T™kðD7ð 
$ðD7ð D7ð D7ñ „^ðD7ðL#ð #ð #ð¸E¼Lð ÈUÌ\ð ð ð ð ð2)ð )ð )ð €U„]�_„_ð ð%
ð %
à�D˜”I‰oð%
ð %
ð %
ñ „_ð%
ðN ð˜dð  t¨C¤yð ð ð ñ „\ðð*-°#ð -ð -ð -ð -ð*˜t Cœyð *¸dÀ3¼ið *ð *ð *ð *ð *ð *ð *ð *rA   rË  rÆ  r~   )rÜ  )Yrå   r=  Úcollections.abcr   r   Údataclassesr   Ú	functoolsr   r;  r¢  r=   Útorch.nnr¸   r   Ú r	   rž   Úintegrations.deepspeedr
   Úmodeling_outputsr   rN   r   r   r   Úutils.genericr   Úmodeling_esmr   r   Úopenfold_utilsr   r   r   r   r   r   r   r   r   r   r   r   Ú
get_loggerr9   Úloggerr!   rK   rQ   r‡   ro   rr  r{   r­   r�   rŠ   r‘   r™  r™   ÚModuler³   r  r  rÌ   rÎ   r  r  rt  r“  r¦  r¯  rº  rÁ  r‚  rÛ  ró  r÷  rù  r  r  r  rL  rR  rW  r]  r–  rË  Ú__all__r@   rA   rB   ú<module>r_     sÅ  ðð €€€Ø 
€
€
€
Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð à Ð Ð Ð Ø €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø <Ð <Ð <Ð <Ð <Ð <Ø +Ð +Ð +Ð +Ð +Ð +ðð ð ð ð ð ð ð ð ð ð
 ,Ð +Ð +Ð +Ð +Ð +Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð  
ˆÔ	˜HÑ	%Ô	%€ð €ððñ ô ð
 ðJAð JAð JAð JAð JA ñ JAô JAñ „ñô ðJAðZð ð ð
ð 
ð 
ðð  4¨¬Ô#5ð ¸eð ÈEÌLð ð ð ð ð.1˜%œ,ð 1°ð 1ð 1ð 1ð 1ðG˜uœ|ð G°4¸´9ð Gð Gð Gð Gð
ð 
ð 
ð+5ð +5ð +5ð +5ð +5�B”Iñ +5ô +5ð +5ð\ð ð ð ð �r”yñ ô ð ð* „Ôðð �u”|ð ¨#ð °u´|ð ð ð ñ ÔððUð Uð Uð Uð U�r”yñ Uô Uð Uðpkð kð kð kð k˜rœyñ kô kð kð\Gð Gð Gð Gð G¨"¬)ñ Gô Gð GðT
9*ð 9*ð 9*ð 9*ð 9*Ð/ñ 9*ô 9*ð 9*ðx;(ð ;(ð ;(ð ;(ð ;(˜2œ9ñ ;(ô ;(ð ;(ð|3ð 3ð 3ð 3ð 3�R”Yñ 3ô 3ð 3ð,#ð #ð #ð #ð #˜BœIñ #ô #ð #ðLð ð ð ð ˜BœIñ ô ð ð*ð ð ð ð ˜œ	ñ ô ð ð e.ð e.ð e.ð e.ð e.¨"¬)ñ e.ô e.ð e.ðPPð Pð Pð Pð Pñ Pô Pð Pð&;ð ;ð ;ð ;ð
ð ð ð,!ð !ð !ð !ð !˜bœiñ !ô !ð !ðHð ð ð ð ˜bœiñ ô ð ð(>!ð >!ð >!ð >!ð >!˜œñ >!ô >!ð >!ðBCð Cð Cð Cð C R¤Yñ Cô Cð CðLð ð ð ð ˜BœIñ ô ð ð.ð ð ð ð ¨B¬Iñ ô ð ð.ð ð ð ð  r¤yñ ô ð ð.G
ð G
ð G
ð G
ð G
˜RœYñ G
ô G
ð G
ðTrð rð rð rð r�b”iñ rô rð rðr €ððñ ô ðC*ð C*ð C*ð C*ð C*Ð-ñ C*ô C*ñô ðC*ðL
 "Ð#;Ð
<€€€rA   