§
    ‚Štj¬¬  ã                   óð  — d Z ddlZddlmZ ddlmZ ddlZddlmZ ddlm	Z	m
Z
mZ ddlmZ dd	lmZmZ dd
lmZmZ ddlmZ ddlmZmZmZmZmZ ddlmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z( ddl)m*Z*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0  e(j1        e2¦  «        Z3d„ Z4 ed¦  «        dMd„¦   «         Z5d„ Z6d„ Z7d„ Z8 G d„ dej9        ¦  «        Z: G d„ dej9        ¦  «        Z; G d„ d ej9        ¦  «        Z<	 	 dNd"ej9        d#ej=        d$ej=        d%ej=        d&ej=        dz  d'e>dz  d(e>d)e#e%         fd*„Z? ee5¦  «         G d+„ d,ej9        ¦  «        ¦   «         Z@ G d-„ d.ej9        ¦  «        ZA G d/„ d0ej9        ¦  «        ZB G d1„ d2ej9        ¦  «        ZC G d3„ d4ej9        ¦  «        ZD G d5„ d6e¦  «        ZE G d7„ d8ej9        ¦  «        ZF G d9„ d:ej9        ¦  «        ZGe& G d;„ d<e!¦  «        ¦   «         ZHe& G d=„ d>eH¦  «        ¦   «         ZIe& G d?„ d@eH¦  «        ¦   «         ZJ G dA„ dBej9        ¦  «        ZK e&dC¬D¦  «         G dE„ dFeH¦  «        ¦   «         ZLe& G dG„ dHeH¦  «        ¦   «         ZM G dI„ dJej9        ¦  «        ZNdK„ ZOg dL¢ZPdS )OzPyTorch ESM model.é    N)ÚCallable)ÚOptional)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)Úuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)Ú"BaseModelOutputWithCrossAttentionsÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚMaskedLMOutputÚSequenceClassifierOutputÚTokenClassifierOutput)Údynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Ú	EsmConfigc                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Néÿÿÿÿé   ©Údim)ÚshapeÚtorchÚcat)ÚxÚx1Úx2s      úb/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/esm/modeling_esm.pyÚrotate_halfr/   0   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'ó    Úrotary_pos_embc                 ó¨  — | j         }|                     |¦  «        }|                     |¦  «        }|                      ¦   «         |z  t          |                      ¦   «         ¦  «        |z  z   }|                     ¦   «         |z  t          |                     ¦   «         ¦  «        |z  z   }|                     |¦  «        |                     |¦  «        fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )ÚdtypeÚ	unsqueezeÚfloatr/   Úto)ÚqÚkÚcosÚsinÚunsqueeze_dimÚoriginal_dtypeÚq_embedÚk_embeds           r.   Úapply_rotary_pos_embr?   7   s¦   € ð& ”W€NØ
�-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�wŠw‰yŒy˜3‰¥;¨q¯wªw©y¬yÑ#9Ô#9¸CÑ#?Ñ@€GØ�wŠw‰yŒy˜3‰¥;¨q¯wªw©y¬yÑ#9Ô#9¸CÑ#?Ñ@€GØ�:Š:�nÑ%Ô% w§z¢z°.Ñ'AÔ'AÐAÐAr0   c                 óf   — | dz  dt          j        | t          j        d¦  «        z  ¦  «        z   z  S )zo
    This is the gelu implementation from the original ESM repo. Using F.gelu yields subtly wrong results.
    g      à?ç      ð?g       @)r)   ÚerfÚmathÚsqrt©r+   s    r.   ÚgelurF   R   s/   € ð ˆs‰7�c�EœI a­$¬)°C©.¬.Ñ&8Ñ9Ô9Ñ9Ñ:Ð:r0   c                 ó4   — | |                       dd¦  «        z   S )zJMake layer symmetric in final two dimensions, used for contact prediction.r$   éþÿÿÿ)Ú	transposerE   s    r.   Ú
symmetrizerJ   Y   s   € àˆq�{Š{˜2˜rÑ"Ô"Ñ"Ð"r0   c                 óÎ   — |                       dd¬¦  «        }|                       dd¬¦  «        }|                       dd¬¦  «        }||z  }|                     |¦  «         | |z
  }|S )z=Perform average product correct, used for contact prediction.r$   T)ÚkeepdimsrH   )r$   rH   )ÚsumÚdiv_)r+   Úa1Úa2Úa12ÚavgÚ
normalizeds         r.   Úaverage_product_correctrT   ^   sh   € à	
�Šˆr˜DˆÑ	!Ô	!€BØ	
�Šˆr˜DˆÑ	!Ô	!€BØ
�%Š%� 4ˆ%Ñ
(Ô
(€Cà
ˆr‰'€CØ‡H‚HˆS�M„M€MØ�S‘€JØÐr0   c                   óÚ   ‡ — e Zd ZU dZej        ed<   ddefˆ fd„Ze		 	 	 ddedz  de
d         dedz  d	ed
ef         fd„¦   «         Z ej        ¦   «         edd„¦   «         ¦   «         Zˆ xZS )ÚEsmRotaryEmbeddingzú
    Rotary position embeddings.
    Implementation based on [ModernBERT's RotaryEmbedding](https://github.com/huggingface/transformers/blob/aad13b87ed59f2afcfaebc985f403301887a35fc/src/transformers/models/modernbert/modeling_modernbert.py#L94).
    Úinv_freqNÚconfigc                 óî   •— t          ¦   «                              ¦   «          || _        i | _        |                      | j        |¦  «        \  }}|                      d|¦  «         t          | d|¦  «         d S )NrW   Úattention_scaling)ÚsuperÚ__init__rX   Ú	rope_typeÚcompute_default_rope_parametersÚregister_bufferÚsetattr)ÚselfrX   ÚdeviceÚcurr_inv_freqÚcurr_attention_scalingÚ	__class__s        €r.   r\   zEsmRotaryEmbedding.__init__r   sw   ø€ Ý‰Œ×ÒÑÔÐàˆŒØˆŒà04×0TÒ0TÐUYÔU`ÐbhÑ0iÔ0iÑ-ˆÐ-Ø×Ò˜Z¨Ñ7Ô7Ð7Ý�Ð)Ð+AÑBÔBÐBÐBÐBr0   rb   ztorch.deviceÚseq_lenÚreturnztorch.Tensorc                 óð   — | j         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «                             |t          j        ¬¦  «        |z  z  z  }||fS )a©  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Úhead_dimNrA   r   r%   ©r3   )rb   r3   )	Ú
rope_thetaÚgetattrÚhidden_sizeÚnum_attention_headsr)   ÚarangeÚint64r6   r5   )rX   rb   rf   Úbaser'   Úattention_factorrW   s          r.   r^   z2EsmRotaryEmbedding.compute_default_rope_parameters|   s‡   € ð( Ô ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐð Ø•U”\ ! S¨!µ5´;Ð?Ñ?Ô?×BÒBÈ&ÕX]ÔXcÐBÑdÔdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r0   c                 óp  — t          | d¦  «        }t          | d¦  «        }|d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬	¦  «        5  |                     ¦   «         |                     ¦   «         z   
                    dd
¦  «        }	t          j        |	|	fd¬¦  «        }
|
                     ¦   «         |z  }|
                     ¦   «         |z  }d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |                     |j        ¬¦  «        fS )NrW   rZ   r   r$   r!   ÚmpsÚcpuF)Údevice_typeÚenabledr%   r&   rj   )rl   r5   Úexpandr(   r6   rb   Ú
isinstanceÚtypeÚstrr   rI   r)   r*   r9   r:   r3   )ra   r+   Úposition_idsÚ
layer_typerW   rZ   Úinv_freq_expandedÚposition_ids_expandedrv   ÚfreqsÚembr9   r:   s                r.   ÚforwardzEsmRotaryEmbedding.forward›   sÑ  € õ ˜4 Ñ,Ô,ˆÝ# DÐ*=Ñ>Ô>Ðà$ T¨1¨1¨1¨d ]Ô3×9Ò9Ñ;Ô;×BÒBÀ<ÔCUÐVWÔCXÐZ\Ð^_Ñ`Ô`×cÒcÐdeÔdlÑmÔmÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	0ð 	0Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)Ð/Ñ/ˆCØ—'’'‘)”)Ð/Ñ/ˆCð		0ð 	0ð 	0ñ 	0ô 	0ð 	0ð 	0ð 	0ð 	0ð 	0ð 	0øøøð 	0ð 	0ð 	0ð 	0ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   Ã'BE7Å7E;Å>E;©N)NNN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r)   ÚTensorÚ__annotations__r"   r\   Ústaticmethodr   ÚintÚtupler5   r^   Úno_gradr   r‚   Ú__classcell__©re   s   @r.   rV   rV   j   s
  ø€ € € € € € ðð ð
 ŒlÐÐÑðCð C˜yð Cð Cð Cð Cð Cð Cð à#'Ø+/Ø"ð*ð *Ø˜DÑ ð*à˜Ô(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ô	%ð	*ð *ð *ñ „\ð*ð< €U„]�_„_Øð<ð <ð <ñ Ôñ „_ð<ð <ð <ð <ð <r0   rV   c                   ó8   ‡ — e Zd ZdZ	 	 ddedefˆ fd„Zd„ Zˆ xZS )	ÚEsmContactPredictionHeadzWPerforms symmetrization, apc, and computes a logistic regression on the output featuresTr%   Úin_featuresÚeos_idxc                 óÊ   •— t          ¦   «                              ¦   «          || _        || _        t	          j        |d|¦  «        | _        t	          j        ¦   «         | _        d S )Nr!   )	r[   r\   r’   r“   r   ÚLinearÚ
regressionÚSigmoidÚ
activation)ra   r’   Úbiasr“   re   s       €r.   r\   z!EsmContactPredictionHead.__init__±   sP   ø€ õ 	‰Œ×ÒÑÔÐØ&ˆÔØˆŒÝœ) K°°DÑ9Ô9ˆŒÝœ*™,œ,ˆŒˆˆr0   c                 óª  — |                      | j        ¦  «                             |¦  «        }|                     d¦  «        |                     d¦  «        z  }||d d …d d d d …d d …f         z  }|dd d…d d…f         }|ddd …dd …f         }|                     ¦   «         \  }}}}}|                     |||z  ||¦  «        }|                     | j        j        j        ¦  «        }t          t          |¦  «        ¦  «        }|                     dddd¦  «        }|                      |                      |¦  «                             d¦  «        ¦  «        S )Nr!   r%   .r$   r   r	   )Úner“   r6   r4   ÚsizeÚviewr–   Úweightrb   rT   rJ   Úpermuter˜   Úsqueeze)	ra   ÚtokensÚ
attentionsÚeos_maskÚ
batch_sizeÚlayersÚheadsÚseqlenÚ_s	            r.   r‚   z EsmContactPredictionHead.forward½   sP  € à—9’9˜Tœ\Ñ*Ô*×-Ò-¨jÑ9Ô9ˆØ×%Ò% aÑ(Ô(¨8×+=Ò+=¸aÑ+@Ô+@Ñ@ˆØ (¨1¨1¨1¨d°D¸!¸!¸!¸Q¸Q¸QÐ+>Ô"?Ñ?ˆ
Ø  S b S¨#¨2¨# Ô.ˆ
à  Q R R¨¨¨ Ô,ˆ
Ø/9¯ªÑ/@Ô/@Ñ,ˆ
�F˜E 6¨1Ø—_’_ Z°¸%±ÀÈÑPÔPˆ
ð  —]’]ØŒOÔ"Ô)ñ
ô 
ˆ
õ -­Z¸
Ñ-CÔ-CÑDÔDˆ
Ø×'Ò'¨¨1¨a°Ñ3Ô3ˆ
Ø�Š˜tŸš¨zÑ:Ô:×BÒBÀ1ÑEÔEÑFÔFÐFr0   )Tr%   )r„   r…   r†   r‡   r‹   r\   r‚   rŽ   r�   s   @r.   r‘   r‘   ®   sx   ø€ € € € € ØaÐað
 Øð	
'ð 
'àð
'ð ð	
'ð 
'ð 
'ð 
'ð 
'ð 
'ðGð Gð Gð Gð Gð Gð Gr0   r‘   c                   ó8   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 dd„Zd„ Zˆ xZS )ÚEsmEmbeddingszV
    Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
    c                 ó¦  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        ¬¦  «        | _        |j        r&t          j	        |j        |j
        ¬¦  «        | _        nd | _        t          j        |j        ¦  «        | _        t          |dd¦  «        | _        |                      dt%          j        |j        ¦  «                             d¦  «        d¬¦  «         |j        | _        | j        dk    r+t          j        |j        |j        | j        ¬¦  «        | _        |j        | _        |j        | _        d S )	N)Úpadding_idx©ÚepsÚposition_embedding_typeÚabsoluter|   ©r!   r$   F)Ú
persistent)r[   r\   r   Ú	EmbeddingÚ
vocab_sizerm   Úpad_token_idÚword_embeddingsÚemb_layer_norm_beforeÚ	LayerNormÚlayer_norm_epsÚ
layer_normÚDropoutÚhidden_dropout_probÚdropoutrl   r¯   r_   r)   ro   Úmax_position_embeddingsrx   r¬   Úposition_embeddingsÚtoken_dropoutÚmask_token_id©ra   rX   re   s     €r.   r\   zEsmEmbeddings.__init__Ö   s:  ø€ Ý‰Œ×ÒÑÔÐÝ!œ|¨FÔ,=¸vÔ?QÐ_eÔ_rÐsÑsÔsˆÔàÔ'ð 	#Ý œl¨6Ô+=À6ÔCXÐYÑYÔYˆDŒOˆOà"ˆDŒOÝ”z &Ô"<Ñ=Ô=ˆŒå'.¨vÐ7PÐR\Ñ']Ô']ˆÔ$Ø×ÒØ�EœL¨Ô)GÑHÔH×OÒOÐPWÑXÔXÐejð 	ñ 	
ô 	
ð 	
ð "Ô.ˆÔØÔ'¨:Ò5Ð5Ý')¤|ØÔ.°Ô0BÐPTÔP`ð(ñ (ô (ˆDÔ$ð $Ô1ˆÔØ#Ô1ˆÔÐÐr0   Nc                 ó  — |€-|�t          || j        ¦  «        }n|                      |¦  «        }|€|                      |¦  «        }|}| j        r¾|�¼|                     || j        k                         d¦  «        d¦  «        }d}|�|                     d¦  «        n|j	        d         }|| j        k                         d¦  «         
                    ¦   «         |z  }|d|z
  z  d|z
  d d …d d f         z                       |j        ¦  «        }| j        dk    r|                      |¦  «        }	||	z   }| j        �|                      |¦  «        }|�0||                     d¦  «        z                       |j        ¦  «        }|S )Nr$   ç        g¸…ëQ¸¾?r!   r°   )Ú"create_position_ids_from_input_idsr¬   Ú&create_position_ids_from_inputs_embedsr¶   rÀ   Úmasked_fillrÁ   r4   rM   r(   r5   r6   r3   r¯   r¿   rº   )
ra   Ú	input_idsÚattention_maskr|   Úinputs_embedsÚ
embeddingsÚmask_ratio_trainÚsrc_lengthsÚmask_ratio_observedr¿   s
             r.   r‚   zEsmEmbeddings.forwardí   s¶  € ð ÐØÐ$åAÀ)ÈTÔM]Ñ^Ô^��à#×JÒJÈ=ÑYÔY�àÐ Ø ×0Ò0°Ñ;Ô;ˆMð #ˆ
ð Ôð 	 )Ð"7Ø#×/Ò/°¸dÔ>PÒ1P×0[Ò0[Ð\^Ñ0_Ô0_ÐadÑeÔeˆJØ)ÐØ4BÐ4N˜.×,Ò,¨RÑ0Ô0Ð0ÐT]ÔTcÐdeÔTfˆKØ#,°Ô0BÒ#B×"GÒ"GÈÑ"KÔ"K×"QÒ"QÑ"SÔ"SÐVaÑ"aÐØ$¨Ð,<Ñ(<Ñ=ÀÐEXÑAXÐZ[ÐZ[ÐZ[Ð]aÐcgÐZgÔ@hÑh×lÒlØÔ ñô ˆJð Ô'¨:Ò5Ð5Ø"&×":Ò":¸<Ñ"HÔ"HÐØ#Ð&9Ñ9ˆJàŒ?Ð&ØŸš¨Ñ4Ô4ˆJØÐ%Ø$ ~×'?Ò'?ÀÑ'CÔ'CÑC×GÒGÈ
ÔHXÑYÔYˆJð Ðr0   c                 ó  — |                      ¦   «         dd…         }|d         }t          j        | j        dz   || j        z   dz   t          j        |j        ¬¦  «        }|                     d¦  «                             |¦  «        S )z×
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nr$   r!   )r3   rb   r   )rœ   r)   ro   r¬   Úlongrb   r4   rx   )ra   rÊ   Úinput_shapeÚsequence_lengthr|   s        r.   rÆ   z4EsmEmbeddings.create_position_ids_from_inputs_embeds  s‡   € ð $×(Ò(Ñ*Ô*¨3¨B¨3Ô/ˆØ% aœ.ˆå”|ØÔ˜qÑ  /°DÔ4DÑ"DÀqÑ"HÕPUÔPZÐcpÔcwð
ñ 
ô 
ˆð ×%Ò% aÑ(Ô(×/Ò/°Ñ<Ô<Ð<r0   ©NNNN)r„   r…   r†   r‡   r\   r‚   rÆ   rŽ   r�   s   @r.   rª   rª   Ñ   st   ø€ € € € € ðð ð2ð 2ð 2ð 2ð 2ð2 ØØØð/ð /ð /ð /ðb=ð =ð =ð =ð =ð =ð =r0   rª   rÄ   ÚmoduleÚqueryÚkeyÚvaluerÉ   Úscalingr½   Úkwargsc                 ó®  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |d¬¦  «        }t          j                             ||| j        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «         	                    ¦   «         }	|	|fS )Nr$   ç      à¿r%   r	   r&   )ÚpÚtrainingr!   )
rœ   r)   ÚmatmulrI   r   Ú
functionalÚsoftmaxr½   rÝ   Ú
contiguous)
rÔ   rÕ   rÖ   r×   rÉ   rØ   r½   rÙ   Úattn_weightsÚattn_outputs
             r.   Úeager_attention_forwardrä   1  sÈ   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r0   c                   óº   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej        dej        dz  dej        dz  dej        dz  dej        dz  d	ee         d
e	ej                 fd„Z
ˆ xZS )ÚEsmSelfAttentionNFc                 óÐ  •— t          ¦   «                              ¦   «          || _        |j        |j        z  dk    r0t          |d¦  «        s t          d|j        › d|j        › d�¦  «        ‚|j        | _        t          |j        |j        z  ¦  «        | _        | j        | j        z  | _	        t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        |j        | _        |pt#          |dd¦  «        | _        d| _        |j        | _        || _        | j        o| | _        d S )	Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)r¯   r°   rA   )r[   r\   rX   rm   rn   ÚhasattrÚ
ValueErrorr‹   Úattention_head_sizeÚall_head_sizer   r•   rÕ   rÖ   r×   Úattention_probs_dropout_probr½   rl   r¯   rØ   Ú
is_decoderÚ	layer_idxÚ	is_causal)ra   rX   r¯   rð   Úis_cross_attentionre   s        €r.   r\   zEsmSelfAttention.__init__O  se  ø€ Ý‰Œ×ÒÑÔÐØˆŒàÔ Ô :Ñ:¸aÒ?Ð?ÍÐPVÐXhÑHiÔHiÐ?Ýð8 FÔ$6ð 8ð 8Ø Ô4ð8ð 8ð 8ñô ð ð
 $*Ô#=ˆÔ Ý#& vÔ'9¸FÔ<VÑ'VÑ#WÔ#WˆÔ Ø!Ô5¸Ô8PÑPˆÔå”Y˜vÔ1°4Ô3EÑFÔFˆŒ
Ý”9˜VÔ/°Ô1CÑDÔDˆŒÝ”Y˜vÔ1°4Ô3EÑFÔFˆŒ
àÔ:ˆŒà'>ð (
Å'ØÐ-¨zñC
ô C
ˆÔ$ð ˆŒØ Ô+ˆŒØ"ˆŒØœÐCÐ1CÐ-CˆŒˆˆr0   Úhidden_statesrÉ   Úencoder_hidden_statesÚencoder_attention_maskr¿   rÙ   rg   c                 ó
  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }	|d u}
|
r|n|}|
r|n|}|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|	| j        dz  z  }	| j        dk    r|\  }}t          |	|||d¬¦  «        \  }	}t          j
        | j        j        t          ¦  «        } || |	|||f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }||fS )	Nr$   r!   r%   rÛ   Úrotary)r;   rÄ   )r½   rØ   )r(   rì   rÕ   r�   rI   rÖ   r×   r¯   r?   r   Úget_interfacerX   Ú_attn_implementationrä   rÝ   r½   rØ   Úreshaperá   )ra   ró   rÉ   rô   rõ   r¿   rÙ   rÑ   Úhidden_shapeÚquery_layerrò   Úcurrent_statesÚ	key_layerÚvalue_layerr9   r:   Úattention_interfacerã   râ   s                      r.   r‚   zEsmSelfAttention.forwardk  sÛ  € ð $Ô)¨#¨2¨#Ô.ˆØC˜ÐC bÐC¨$Ô*BÐCÐCˆà—j’j Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆà2¸$Ð>ÐØ2DÐWÐ.Ð.È-ˆØ3EÐYÐ/Ð/È>ˆØ—H’H˜^Ñ,Ô,×1Ò1°,Ñ?Ô?×IÒIÈ!ÈQÑOÔOˆ	Ø—j’j Ñ0Ô0×5Ò5°lÑCÔC×MÒMÈaÐQRÑSÔSˆð " DÔ$<¸dÑ$BÑBˆàÔ'¨8Ò3Ð3Ø*‰HˆC�Ý%9¸+ÀyÐRUÐWZÐjkÐ%lÑ%lÔ%lÑ"ˆK˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}Ð>�C�C°$´,Ø”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ˜LÐ(Ð(r0   )NNFrÓ   )r„   r…   r†   r\   r)   rˆ   ÚFloatTensorr   r   rŒ   r‚   rŽ   r�   s   @r.   ræ   ræ   M  sÜ   ø€ € € € € ðDð Dð Dð Dð Dð Dð> 48Ø:>Ø;?Ø37ð.)ð .)à”|ð.)ð Ô)¨DÑ0ð.)ð  %Ô0°4Ñ7ð	.)ð
 !&Ô 1°DÑ 8ð.)ð #œ\¨DÑ0ð.)ð Ð+Ô,ð.)ð 
ˆuŒ|Ô	ð.)ð .)ð .)ð .)ð .)ð .)ð .)ð .)r0   ræ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEsmSelfOutputc                 óÌ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        d S rƒ   )	r[   r\   r   r•   rm   Údenser»   r¼   r½   rÂ   s     €r.   r\   zEsmSelfOutput.__init__�  sJ   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒˆˆr0   c                 ód   — |                       |¦  «        }|                      |¦  «        }||z   }|S rƒ   ©r  r½   ©ra   ró   Úinput_tensors      r.   r‚   zEsmSelfOutput.forward¢  ó4   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØ%¨Ñ4ˆØÐr0   ©r„   r…   r†   r\   r‚   rŽ   r�   s   @r.   r  r  œ  óG   ø€ € € € € ð>ð >ð >ð >ð >ð
ð ð ð ð ð ð r0   r  c                   óV   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej        dz  dee         fd„Zˆ xZ	S )	ÚEsmAttentionNFc                 óè   •— t          ¦   «                              ¦   «          t          |||¬¦  «        | _        t	          |¦  «        | _        t          j        |j        |j	        ¬¦  «        | _        d S )N)rð   rò   r­   )
r[   r\   ræ   ra   r  Úoutputr   r¸   rm   r¹   )ra   rX   rð   rò   re   s       €r.   r\   zEsmAttention.__init__ª  s_   ø€ Ý‰Œ×ÒÑÔÐÝ$ V°yÐUgÐhÑhÔhˆŒ	Ý# FÑ+Ô+ˆŒåœ fÔ&8¸fÔ>SÐTÑTÔTˆŒˆˆr0   r¿   rÙ   c                 óˆ   — |                       |¦  «        } | j        |f||||dœ|¤Ž\  }}	|                      ||¦  «        }|S )N©rÉ   rô   rõ   r¿   )r¸   ra   r  )
ra   ró   rÉ   rô   rõ   r¿   rÙ   Úhidden_states_lnrã   r¨   s
             r.   r‚   zEsmAttention.forward±  sk   € ð  Ÿ>š>¨-Ñ8Ô8ÐØ"˜œØð
à)Ø"7Ø#9Ø 3ð
ð 
ð ð
ð 
‰ˆ�Qð —k’k +¨}Ñ=Ô=ˆØÐr0   )NFrÓ   )
r„   r…   r†   r\   r)   rˆ   r   r   r‚   rŽ   r�   s   @r.   r  r  ©  s‹   ø€ € € € € ðUð Uð Uð Uð Uð Uð Ø"Ø#Ø37ðð ð #œ\¨DÑ0ðð Ð+Ô,ðð ð ð ð ð ð ð r0   r  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚEsmIntermediatec                 ó�   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        d S rƒ   )r[   r\   r   r•   rm   Úintermediate_sizer  rÂ   s     €r.   r\   zEsmIntermediate.__init__È  s6   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3KÑLÔLˆŒ
ˆ
ˆ
r0   ró   rg   c                 óN   — |                       |¦  «        }t          |¦  «        }|S rƒ   )r  rF   )ra   ró   s     r.   r‚   zEsmIntermediate.forwardÌ  s&   € ØŸ
š
 =Ñ1Ô1ˆÝ˜]Ñ+Ô+ˆØÐr0   ©r„   r…   r†   r\   r)   rˆ   r‚   rŽ   r�   s   @r.   r  r  Ç  sc   ø€ € € € € ðMð Mð Mð Mð Mð U¤\ð °e´lð ð ð ð ð ð ð ð r0   r  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú	EsmOutputc                 óÌ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _	        d S rƒ   )
r[   r\   r   r•   r  rm   r  r»   r¼   r½   rÂ   s     €r.   r\   zEsmOutput.__init__Ó  sJ   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ7¸Ô9KÑLÔLˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒˆˆr0   c                 ód   — |                       |¦  «        }|                      |¦  «        }||z   }|S rƒ   r  r  s      r.   r‚   zEsmOutput.forwardØ  r
  r0   r  r�   s   @r.   r  r  Ò  r  r0   r  c                   óZ   ‡ — e Zd Zˆ fd„Z	 	 	 	 ddej        dz  dee         fd„Zd„ Z	ˆ xZ
S )ÚEsmLayerc                 óÌ  •— t          ¦   «                              ¦   «          |j        | _        d| _        t	          |¦  «        | _        |j        | _        |j        | _        | j        r/| j        st          | › d�¦  «        ‚t	          |d¬¦  «        | _	        t          |¦  «        | _        t          |¦  «        | _        t          j        |j        |j        ¬¦  «        | _        d S )Nr!   z> should be used as a decoder model if cross attention is addedT)rò   r­   )r[   r\   Úchunk_size_feed_forwardÚseq_len_dimr  Ú	attentionrï   Úadd_cross_attentionÚRuntimeErrorÚcrossattentionr  Úintermediater  r  r   r¸   rm   r¹   rÂ   s     €r.   r\   zEsmLayer.__init__à  sÒ   ø€ Ý‰Œ×ÒÑÔÐØ'-Ô'EˆÔ$ØˆÔÝ% fÑ-Ô-ˆŒØ Ô+ˆŒØ#)Ô#=ˆÔ ØÔ#ð 	PØ”?ð lÝ" dÐ#jÐ#jÐ#jÑkÔkÐkÝ".¨vÈ$Ð"OÑ"OÔ"OˆDÔÝ+¨FÑ3Ô3ˆÔÝ Ñ'Ô'ˆŒÝœ fÔ&8¸fÔ>SÐTÑTÔTˆŒˆˆr0   Nr¿   rÙ   c                 óÐ   —  | j         |f||dœ|¤Ž}| j        r8|�6t          | d¦  «        st          d| › d�¦  «        ‚ | j        |f||||dœ|¤Ž}|                      |¦  «        }|S )N)rÉ   r¿   r&  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r  )r#  rï   rê   ÚAttributeErrorr&  Úfeed_forward_chunk)	ra   ró   rÉ   rô   rõ   r¿   rÙ   Úattention_outputÚlayer_outputs	            r.   r‚   zEsmLayer.forwardï  sÜ   € ð *˜4œ>Øð
à)Ø 3ð
ð 
ð ð	
ð 
Ðð Œ?ð 	Ð4Ð@Ý˜4Ð!1Ñ2Ô2ð Ý$ð`¸dð `ð `ð `ñô ð ð
  3˜tÔ2Ø ð à-Ø&;Ø'=Ø$7ð ð  ð ð ð  Ðð ×.Ò.Ð/?Ñ@Ô@ˆØÐr0   c                 ó†   — |                       |¦  «        }|                      |¦  «        }|                      ||¦  «        }|S rƒ   )r¸   r'  r  )ra   r+  Úattention_output_lnÚintermediate_outputr,  s        r.   r*  zEsmLayer.feed_forward_chunk  sE   € Ø"ŸnšnÐ-=Ñ>Ô>ÐØ"×/Ò/Ð0CÑDÔDÐØ—{’{Ð#6Ð8HÑIÔIˆØÐr0   rÓ   )r„   r…   r†   r\   r)   rˆ   r   r   r‚   r*  rŽ   r�   s   @r.   r  r  ß  s•   ø€ € € € € ðUð Uð Uð Uð Uð$ Ø"Ø#Ø37ð!ð !ð #œ\¨DÑ0ð!ð Ð+Ô,ð!ð !ð !ð !ðFð ð ð ð ð ð r0   r  c                   ód   ‡ — e Zd Zˆ fd„Ze	 	 	 	 ddej        dz  dee         fd„¦   «         Z	ˆ xZ
S )Ú
EsmEncoderc                 ó  •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j	        ‰j
        ¬¦  «        | _        d| _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS © )r  )Ú.0r¨   rX   s     €r.   ú
<listcomp>z'EsmEncoder.__init__.<locals>.<listcomp>  s!   ø€ Ð#^Ð#^Ð#^¸¥H¨VÑ$4Ô$4Ð#^Ð#^Ð#^r0   r­   F)r[   r\   rX   r   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayerr¸   rm   r¹   Úemb_layer_norm_afterÚgradient_checkpointingrÂ   s    `€r.   r\   zEsmEncoder.__init__  s}   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”]Ð#^Ð#^Ð#^Ð#^½eÀFÔD\Ñ>]Ô>]Ð#^Ñ#^Ô#^Ñ_Ô_ˆŒ
Ý$&¤L°Ô1CÈÔI^Ð$_Ñ$_Ô$_ˆÔ!Ø&+ˆÔ#Ð#Ð#r0   Nr¿   rÙ   c           	      óª   — t          | j        ¦  «        D ]\  }} ||f||||dœ|¤Ž}Œ| j        r|                      |¦  «        }t          |¬¦  «        S )Nr  )Úlast_hidden_state)Ú	enumerater:  r;  r   )	ra   ró   rÉ   rô   rõ   r¿   rÙ   ÚiÚlayer_modules	            r.   r‚   zEsmEncoder.forward!  s‰   € õ  )¨¬Ñ4Ô4ð 	ð 	‰OˆAˆ|Ø(˜LØðà-Ø&;Ø'=Ø$7ðð ð ðð ˆMˆMð Ô$ð 	EØ ×5Ò5°mÑDÔDˆMå1ÀMÐRÑRÔRÐRr0   rÓ   )r„   r…   r†   r\   r   r)   rˆ   r   r   r‚   rŽ   r�   s   @r.   r1  r1    s˜   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð ð Ø"Ø#Ø37ðSð Sð #œ\¨DÑ0ðSð Ð+Ô,ðSð Sð Sñ ÔðSð Sð Sð Sð Sr0   r1  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )Ú	EsmPoolerc                 óÀ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        ¦   «         | _        d S rƒ   )r[   r\   r   r•   rm   r  ÚTanhr˜   rÂ   s     €r.   r\   zEsmPooler.__init__=  sC   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ'™)œ)ˆŒˆˆr0   ró   rg   c                 ór   — |d d …df         }|                       |¦  «        }|                      |¦  «        }|S ©Nr   )r  r˜   )ra   ró   Úfirst_token_tensorÚpooled_outputs       r.   r‚   zEsmPooler.forwardB  s@   € ð +¨1¨1¨1¨a¨4Ô0ÐØŸ
š
Ð#5Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐr0   r  r�   s   @r.   rC  rC  <  s^   ø€ € € € € ð$ð $ð $ð $ð $ð
 U¤\ð °e´lð ð ð ð ð ð ð ð r0   rC  c                   óÄ   ‡ — e Zd ZU eed<   dZdZdZg d¢ZdgZ	dZ
dZdZdZe eedd¬	¦  «        g eedd
¬	¦  «        gdœZ ej        ¦   «         ˆ fd„¦   «         Zd„ Zˆ xZS )ÚEsmPreTrainedModelrX   ÚesmTF)r  Ú#EsmFoldTriangularSelfAttentionBlockrª   zposition_embeddings.weightr!   r#  )ÚindexÚ
layer_namer&  )ró   r¢   Úcross_attentionsc                 ó$  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rt	          j        |j        ¦  «         dS t          |t          ¦  «        rQt	          j        |j	        t          j        |j	        j        d         ¦  «                             d¦  «        ¦  «         dS t          |t          ¦  «        rB|                     |j        ¦  «        \  }}t	          j        t#          |d¦  «        |¦  «         dS dS )zInitialize the weightsr$   r±   rW   N)r[   Ú_init_weightsry   Ú	EsmLMHeadÚinitÚzeros_r™   rª   Úcopy_r|   r)   ro   r(   rx   rV   r^   rX   rl   )ra   rÔ   rc   r¨   re   s       €r.   rR  z EsmPreTrainedModel._init_weights`  s÷   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�iÑ(Ô(ð 	CÝŒK˜œÑ$Ô$Ð$Ð$Ð$Ý˜¥Ñ.Ô.ð 	CÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÑhÔhÐhÐhÐhÝ˜Õ 2Ñ3Ô3ð 	CØ%×EÒEÀfÄmÑTÔTÑˆM˜1ÝŒJ•w˜v zÑ2Ô2°MÑBÔBÐBÐBÐBð	Cð 	Cr0   c                 ó   — d S rƒ   r4  ©ra   s    r.   Úget_output_embeddingsz(EsmPreTrainedModel.get_output_embeddingsl  s	   € ð ˆtr0   )r„   r…   r†   r"   r‰   Úbase_model_prefixÚsupports_gradient_checkpointingÚaccepts_loss_kwargsÚ_no_split_modulesÚ"_keys_to_ignore_on_load_unexpectedÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr  r   ræ   Ú_can_record_outputsr)   r�   rR  rY  rŽ   r�   s   @r.   rK  rK  K  sñ   ø€ € € € € € àÐÐÑØÐØ&*Ð#ØÐØ\Ð\Ð\ÐØ*FÐ)GÐ&ØÐØ€NØÐØ"&Ðð "Ø%�~Ð&6¸aÈKÐXÑXÔXÐYàˆNÐ+°1ÐAQÐRÑRÔRð
ðð Ðð €U„]�_„_ð	Cð 	Cð 	Cð 	Cñ „_ð	Cðð ð ð ð ð ð r0   rK  c                   ó0  ‡ — e Zd ZdZdˆ fd„	Zd„ Zd„ Zd„ Zee	e
	 	 	 	 	 	 ddej        dz  d	ej        dz  d
ej        dz  dej        dz  dej        dz  dej        dz  dee         deej                 ez  fd„¦   «         ¦   «         ¦   «         Zd„ Zd„ Zˆ xZS )ÚEsmModela  

    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in [Attention is
    all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
    Tc                 óô  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        d| _        t          |dd¦  «        | _        | j        dk    rt          |¬¦  «        | _        t          |¦  «        | _
        |rt          |¦  «        nd| _        t          |j        |j        z  d¬¦  «        | _        |                      | j        ¦  «         |                      ¦   «          dS )zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        Nr¯   r°   r÷   )rX   T)r’   r™   )r[   r\   rX   rª   rË   Úrotary_embeddingsrl   r¯   rV   r1  ÚencoderrC  Úpoolerr‘   r9  rn   Úcontact_headÚ"_register_load_state_dict_pre_hookÚ	load_hookÚ	post_init)ra   rX   Úadd_pooling_layerre   s      €r.   r\   zEsmModel.__init__€  sð   ø€ õ
 	‰Œ×Ò˜Ñ Ô Ð ØˆŒå'¨Ñ/Ô/ˆŒà!%ˆÔÝ'.¨vÐ7PÐR\Ñ']Ô']ˆÔ$ØÔ'¨8Ò3Ð3Ý%7¸vÐ%FÑ%FÔ%FˆDÔ"å! &Ñ)Ô)ˆŒà+<ÐF•i Ñ'Ô'Ð'À$ˆŒå4ØÔ0°6Ô3MÑMÐTXð
ñ 
ô 
ˆÔð 	×/Ò/°´Ñ?Ô?Ð?à�ŠÑÔÐÐÐr0   c                 óÈ   ‡— ‰› d�}||vrUt          ˆfd„t          |                     ¦   «         ¦  «        D ¦   «         ¦  «        }|r||d                  ||<   |D ]}||= ŒdS dS )aÿ  Remap per-layer rotary inv_freq keys from old checkpoints to the new model-level location.

        Old checkpoints stored inv_freq per attention layer at:
            {prefix}encoder.layer.{i}.attention.self.rotary_embeddings.inv_freq
        New code stores a single shared inv_freq at:
            {prefix}rotary_embeddings.inv_freq
        The old checkpoint values must be preserved (not recomputed) because they may
        have been saved in float16, matching the precision used during training.
        zrotary_embeddings.inv_freqc              3   óp   •K  — | ]0}|                      ‰¦  «        ¯|                     d ¦  «        ¯,|V — Œ1dS )z*.attention.self.rotary_embeddings.inv_freqN)Ú
startswithÚendswith)r5  r8   Úprefixs     €r.   ú	<genexpr>z%EsmModel.load_hook.<locals>.<genexpr>§  s`   øè è € ð ð àØ—<’< Ñ'Ô'ðð -.¯JªJÐ7cÑ,dÔ,dðØðð ð ð ð ð r0   r   N)ÚsortedÚlistÚkeys)ra   Ú
state_dictrs  ÚargsÚnew_keyÚold_keysr8   s     `    r.   rl  zEsmModel.load_hook›  s¬   ø€ ð Ð7Ð7Ð7ˆØ˜*Ð$Ð$Ýð ð ð ð å˜jŸošoÑ/Ô/Ñ0Ô0ðñ ô ñ ô ˆHð
 ð >Ø&0°¸!´Ô&=�
˜7Ñ#Øð "ð "�Ø˜q�M�Mð %Ð$ð"ð "r0   c                 ó   — | j         j        S rƒ   ©rË   r¶   rX  s    r.   Úget_input_embeddingszEsmModel.get_input_embeddings±  s   € ØŒÔ.Ð.r0   c                 ó   — || j         _        d S rƒ   r}  )ra   r×   s     r.   Úset_input_embeddingszEsmModel.set_input_embeddings´  s   € Ø*/ˆŒÔ'Ð'Ð'r0   NrÈ   rÉ   r|   rÊ   rô   rõ   rÙ   rg   c                 óö  — |du |duz  rt          d¦  «        ‚|€|                      |||¬¦  «        }|                      ||||d¬¦  «        \  }}| j        dk    rT|€;|j        d         }t          j        ||j        ¬¦  «                             d¦  «        }|  	                    ||¦  «        }	nd}	 | j
        |f||||	d	œ|¤Ž}
|
d         }| j        �|                      |¦  «        nd}t          ||¬
¦  «        S )aV  
        input_ids (`torch.LongTensor` of shape `((batch_size, sequence_length))`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        position_ids (`torch.LongTensor` of shape `((batch_size, sequence_length))`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `((batch_size, sequence_length), hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        Nz:You must specify exactly one of input_ids or inputs_embeds)rÈ   rÉ   r|   )rÉ   rõ   Úembedding_outputrô   Úpast_key_valuesr÷   r!   )rb   r   r  )r>  Úpooler_output)rë   rË   Ú_create_attention_masksr¯   r(   r)   ro   rb   r4   rg  rh  ri  r   )ra   rÈ   rÉ   r|   rÊ   rô   rõ   rÙ   rf   r¿   Úencoder_outputsÚsequence_outputrI  s                r.   r‚   zEsmModel.forward·  sk  € ð> ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ ð !ŸOšOØ#Ø-Ø)ð ,ñ ô ˆMð 26×1MÒ1MØ)Ø#9Ø*Ø"7Ø ð 2Nñ 2
ô 2
Ñ.ˆÐ.ð Ô'¨8Ò3Ð3ØÐ#Ø'Ô-¨aÔ0�Ý$œ|¨G¸MÔ<PÐQÑQÔQ×[Ò[Ð\]Ñ^Ô^�Ø"&×"8Ò"8¸ÈÑ"UÔ"UÐÐà"&Ðà&˜$œ,Øð
à)Ø"7Ø#9Ø 3ð
ð 
ð ð
ð 
ˆð *¨!Ô,ˆØ8<¼Ð8O˜Ÿš OÑ4Ô4Ð4ÐUYˆå;Ø-Ø'ð
ñ 
ô 
ð 	
r0   c                 ó¶   — | j         j        rt          | j         |||¬¦  «        }nt          | j         ||¬¦  «        }|�t          | j         |||¬¦  «        }||fS )N)rX   rÊ   rÉ   rƒ  )rX   rÊ   rÉ   )rX   rÊ   rÉ   rô   )rX   rï   r   r   )ra   rÉ   rõ   r‚  rô   rƒ  s         r.   r…  z EsmModel._create_attention_masks  s�   € ð Œ;Ô!ð 	Ý/Ø”{Ø.Ø-Ø /ð	ñ ô ˆNˆNõ 7Ø”{Ø.Ø-ðñ ô ˆNð "Ð-Ý%>Ø”{Ø.Ø5Ø&;ð	&ñ &ô &Ð"ð Ð5Ð5Ð5r0   c                 óz  —  | ||dd¬¦  «        j         }t          j        |d¬¦  «        }||                     d¦  «                             d¦  «                             d¦  «        z  }||                     d¦  «                             d¦  «                             d¦  «        z  }|                      ||¦  «        S )NT)rÉ   Úreturn_dictÚoutput_attentionsr!   r&   r%   r	   é   )r¢   r)   Ústackr4   rj  )ra   r¡   rÉ   Úattnss       r.   Úpredict_contactszEsmModel.predict_contacts#  s²   € Ø��V¨NÈÐ`dÐeÑeÔeÔpˆÝ”˜E qÐ)Ñ)Ô)ˆð
 	�×)Ò)¨!Ñ,Ô,×6Ò6°qÑ9Ô9×CÒCÀAÑFÔFÑFˆØ�×)Ò)¨!Ñ,Ô,×6Ò6°qÑ9Ô9×CÒCÀAÑFÔFÑFˆØ× Ò  ¨Ñ/Ô/Ð/r0   )T)NNNNNN)r„   r…   r†   r‡   r\   rl  r~  r€  r   r    r   r)   rˆ   r   r   rŒ   r   r‚   r…  r�  rŽ   r�   s   @r.   re  re  r  sz  ø€ € € € € ð
ð 
ðð ð ð ð ð ð6"ð "ð "ð,/ð /ð /ð0ð 0ð 0ð  ØØð *.Ø.2Ø,0Ø-1Ø59Ø6:ðF
ð F
à”< $Ñ&ðF
ð œ tÑ+ðF
ð ”l TÑ)ð	F
ð
 ”| dÑ*ðF
ð  %œ|¨dÑ2ðF
ð !&¤¨tÑ 3ðF
ð Ð+Ô,ðF
ð 
ˆuŒ|Ô	ÐKÑ	KðF
ð F
ð F
ñ „^ñ „_ñ  ÔðF
ðR6ð 6ð 6ð@	0ð 	0ð 	0ð 	0ð 	0ð 	0ð 	0r0   re  c                   ó  ‡ — e Zd ZddiZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 dde	j
        dz  de	j        dz  d	e	j
        dz  d
e	j        dz  de	j        dz  de	j        dz  de	j
        dz  dee         deez  fd„¦   «         ¦   «         Zd„ Zˆ xZS )ÚEsmForMaskedLMzlm_head.decoder.weightz%esm.embeddings.word_embeddings.weightc                 ó  •— t          ¦   «                              |¦  «         |j        rt                               d¦  «         t          |d¬¦  «        | _        t          |¦  «        | _        |  	                    ¦   «          d S )NzjIf you want to use `EsmForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.F©rn  )
r[   r\   rï   ÚloggerÚwarningre  rL  rS  Úlm_headrm  rÂ   s     €r.   r\   zEsmForMaskedLM.__init__3  s|   ø€ Ý‰Œ×Ò˜Ñ Ô Ð àÔð 	Ý�NŠNð1ñô ð õ
 ˜F°eÐ<Ñ<Ô<ˆŒÝ  Ñ(Ô(ˆŒà�ŠÑÔÐÐÐr0   c                 ó   — | j         j        S rƒ   ©r–  ÚdecoderrX  s    r.   rY  z$EsmForMaskedLM.get_output_embeddingsA  s   € ØŒ|Ô#Ð#r0   c                 ó   — || j         _        d S rƒ   r˜  )ra   Únew_embeddingss     r.   Úset_output_embeddingsz$EsmForMaskedLM.set_output_embeddingsD  s   € Ø-ˆŒÔÐÐr0   NrÈ   rÉ   r|   rÊ   rô   rõ   ÚlabelsrÙ   rg   c           	      óp  —  | j         |f|||||dœ|¤Ž}	|	d         }
|                      |
¦  «        }d}|�et          ¦   «         }|                     |j        ¦  «        } ||                     d| j        j        ¦  «        |                     d¦  «        ¦  «        }t          |||	j	        |	j
        ¬¦  «        S )a¢  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        )rÉ   r|   rÊ   rô   rõ   r   Nr$   ©ÚlossÚlogitsró   r¢   )rL  r–  r   r6   rb   r�   rX   r´   r   ró   r¢   )ra   rÈ   rÉ   r|   rÊ   rô   rõ   r�  rÙ   Úoutputsr‡  Úprediction_scoresÚmasked_lm_lossÚloss_fcts                 r.   r‚   zEsmForMaskedLM.forwardG  så   € ð( �$”(Øð
à)Ø%Ø'Ø"7Ø#9ð
ð 
ð ð
ð 
ˆð " !œ*ˆØ ŸLšL¨Ñ9Ô9ÐàˆØÐÝ'Ñ)Ô)ˆHà—Y’YÐ0Ô7Ñ8Ô8ˆFØ%˜XÐ&7×&<Ò&<¸RÀÄÔAWÑ&XÔ&XÐZ`×ZeÒZeÐfhÑZiÔZiÑjÔjˆNåØØ$Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r0   c                 ó:   — | j                              ||¬¦  «        S )N)rÉ   )rL  r�  )ra   r¡   rÉ   s      r.   r�  zEsmForMaskedLM.predict_contactsu  s   € ØŒx×(Ò(¨ÀÐ(ÑOÔOÐOr0   )NNNNNNN)r„   r…   r†   Ú_tied_weights_keysr\   rY  rœ  r   r   r)   Ú
LongTensorrˆ   r  r   r   rŒ   r   r‚   r�  rŽ   r�   s   @r.   r‘  r‘  /  sY  ø€ € € € € à2Ð4[Ð\Ððð ð ð ð ð$ð $ð $ð.ð .ð .ð Øð .2Ø.2Ø04Ø26Ø:>Ø6:Ø*.ð*
ð *
àÔ# dÑ*ð*
ð œ tÑ+ð*
ð Ô&¨Ñ-ð	*
ð
 Ô(¨4Ñ/ð*
ð  %Ô0°4Ñ7ð*
ð !&¤¨tÑ 3ð*
ð Ô  4Ñ'ð*
ð Ð+Ô,ð*
ð 
�Ñ	ð*
ð *
ð *
ñ „^ñ Ôð*
ðXPð Pð Pð Pð Pð Pð Pr0   r‘  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )rS  z&ESM Head for masked language modeling.c                 ó†  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j        |j	        d¬¦  «        | _
        t          j        t          j        |j	        ¦  «        ¦  «        | _        d S )Nr­   F)r™   )r[   r\   r   r•   rm   r  r¸   r¹   rº   r´   r™  Ú	Parameterr)   Úzerosr™   rÂ   s     €r.   r\   zEsmLMHead.__init__|  sŽ   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒå”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒÝ”L¥¤¨VÔ->Ñ!?Ô!?Ñ@Ô@ˆŒ	ˆ	ˆ	r0   c                 ó²   — |                       |¦  «        }t          |¦  «        }|                      |¦  «        }|                      |¦  «        | j        z   }|S rƒ   )r  rF   rº   r™  r™   ©ra   ÚfeaturesrÙ   r+   s       r.   r‚   zEsmLMHead.forward„  sL   € Ø�JŠJ�xÑ Ô ˆÝ�‰GŒGˆØ�OŠO˜AÑÔˆð �LŠL˜‰OŒO˜dœiÑ'ˆØˆr0   ©r„   r…   r†   r‡   r\   r‚   rŽ   r�   s   @r.   rS  rS  y  sR   ø€ € € € € Ø0Ð0ðAð Að Að Að Aðð ð ð ð ð ð r0   rS  z›
    ESM Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    )Úcustom_introc                   óÐ   ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej	        dz  dej        dz  de
e         d	eez  fd
„¦   «         ¦   «         Zˆ xZS )ÚEsmForSequenceClassificationc                 óì   •— t          ¦   «                              |¦  «         |j        | _        || _        t	          |d¬¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S ©NFr“  )	r[   r\   Ú
num_labelsrX   re  rL  ÚEsmClassificationHeadÚ
classifierrm  rÂ   s     €r.   r\   z%EsmForSequenceClassification.__init__•  se   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒØˆŒå˜F°eÐ<Ñ<Ô<ˆŒÝ/°Ñ7Ô7ˆŒà�ŠÑÔÐÐÐr0   NrÈ   rÉ   r|   rÊ   r�  rÙ   rg   c                 óŒ  —  | j         |f|||dœ|¤Ž}|d         }|                      |¦  «        }	d}
|��t|                     |	j        ¦  «        }| j        j        €f| j        dk    rd| j        _        nN| j        dk    r7|j        t          j	        k    s|j        t          j
        k    rd| j        _        nd| j        _        | j        j        dk    rWt          ¦   «         }| j        dk    r1 ||	                     ¦   «         |                     ¦   «         ¦  «        }
nŽ ||	|¦  «        }
n�| j        j        dk    rGt          ¦   «         } ||	                     d| j        ¦  «        |                     d¦  «        ¦  «        }
n*| j        j        dk    rt          ¦   «         } ||	|¦  «        }
t!          |
|	|j        |j        ¬	¦  «        S )
a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        ©rÉ   r|   rÊ   r   Nr!   r–   Úsingle_label_classificationÚmulti_label_classificationr$   rŸ  )rL  r¸  r6   rb   rX   Úproblem_typer¶  r3   r)   rÐ   r‹   r   r    r   r�   r   r   ró   r¢   ©ra   rÈ   rÉ   r|   rÊ   r�  rÙ   r¢  r‡  r¡  r   r¥  s               r.   r‚   z$EsmForSequenceClassification.forwardŸ  s×  € ð$ �$”(Øð
à)Ø%Ø'ð	
ð 
ð
 ð
ð 
ˆð " !œ*ˆØ—’ Ñ1Ô1ˆàˆØÑØ—Y’Y˜vœ}Ñ-Ô-ˆFàŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ñ @Ô @À&Ç+Â+ÈbÁ/Ä/ÑRÔR��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�å'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r0   ©NNNNN)r„   r…   r†   r\   r   r   r)   r¨  rˆ   r  r   r   rŒ   r   r‚   rŽ   r�   s   @r.   r³  r³  Ž  sð   ø€ € € € € ðð ð ð ð ð Øð .2Ø.2Ø04Ø26Ø*.ð8
ð 8
àÔ# dÑ*ð8
ð œ tÑ+ð8
ð Ô&¨Ñ-ð	8
ð
 Ô(¨4Ñ/ð8
ð Ô  4Ñ'ð8
ð Ð+Ô,ð8
ð 
Ð)Ñ	)ð8
ð 8
ð 8
ñ „^ñ Ôð8
ð 8
ð 8
ð 8
ð 8
r0   r³  c                   óÐ   ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej	        dz  dej        dz  de
e         d	eez  fd
„¦   «         ¦   «         Zˆ xZS )ÚEsmForTokenClassificationc                 ó:  •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        t          j        |j        ¦  «        | _        t          j	        |j
        |j        ¦  «        | _        |                      ¦   «          d S rµ  )r[   r\   r¶  re  rL  r   r»   r¼   r½   r•   rm   r¸  rm  rÂ   s     €r.   r\   z"EsmForTokenClassification.__init__Þ  s|   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå˜F°eÐ<Ñ<Ô<ˆŒÝ”z &Ô"<Ñ=Ô=ˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒà�ŠÑÔÐÐÐr0   NrÈ   rÉ   r|   rÊ   r�  rÙ   rg   c                 óŒ  —  | j         |f|||dœ|¤Ž}|d         }|                      |¦  «        }|                      |¦  «        }	d}
|�`t          ¦   «         }|                     |	j        ¦  «        } ||	                     d| j        ¦  «        |                     d¦  «        ¦  «        }
t          |
|	|j	        |j
        ¬¦  «        S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        rº  r   Nr$   rŸ  )rL  r½   r¸  r   r6   rb   r�   r¶  r   ró   r¢   r¾  s               r.   r‚   z!EsmForTokenClassification.forwardè  sá   € ð  �$”(Øð
à)Ø%Ø'ð	
ð 
ð
 ð
ð 
ˆð " !œ*ˆàŸ,š, Ñ7Ô7ˆØ—’ Ñ1Ô1ˆàˆØÐÝ'Ñ)Ô)ˆHà—Y’Y˜vœ}Ñ-Ô-ˆFØ�8˜FŸKšK¨¨D¬OÑ<Ô<¸f¿kºkÈ"¹o¼oÑNÔNˆDå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r0   r¿  )r„   r…   r†   r\   r   r   r)   r¨  rˆ   r  r   r   rŒ   r   r‚   rŽ   r�   s   @r.   rÁ  rÁ  Ü  sð   ø€ € € € € ðð ð ð ð ð Øð .2Ø.2Ø04Ø26Ø*.ð'
ð '
àÔ# dÑ*ð'
ð œ tÑ+ð'
ð Ô&¨Ñ-ð	'
ð
 Ô(¨4Ñ/ð'
ð Ô  4Ñ'ð'
ð Ð+Ô,ð'
ð 
Ð&Ñ	&ð'
ð '
ð '
ñ „^ñ Ôð'
ð '
ð '
ð '
ð '
r0   rÁ  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )r·  z-Head for sentence-level classification tasks.c                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        t          j        |j        |j	        ¦  «        | _
        d S rƒ   )r[   r\   r   r•   rm   r  r»   r¼   r½   r¶  Úout_projrÂ   s     €r.   r\   zEsmClassificationHead.__init__  sc   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒÝœ	 &Ô"4°fÔ6GÑHÔHˆŒˆˆr0   c                 óô   — |d d …dd d …f         }|                       |¦  «        }|                      |¦  «        }t          j        |¦  «        }|                       |¦  «        }|                      |¦  «        }|S rG  )r½   r  r)   ÚtanhrÆ  r®  s       r.   r‚   zEsmClassificationHead.forward  sj   € Ø�Q�Q�Q˜˜1˜1˜1�WÔˆØ�LŠL˜‰OŒOˆØ�JŠJ�q‰MŒMˆÝŒJ�q‰MŒMˆØ�LŠL˜‰OŒOˆØ�MŠM˜!ÑÔˆØˆr0   r°  r�   s   @r.   r·  r·    sR   ø€ € € € € Ø7Ð7ðIð Ið Ið Ið Iðð ð ð ð ð ð r0   r·  c                 óÖ   — |                       |¦  «                             ¦   «         }t          j        |d¬¦  «                             |¦  «        |z  }|                     ¦   «         |z   S )a  
    Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
    are ignored. This is modified from fairseq's `utils.make_positions`.

    Args:
        x: torch.Tensor x:

    Returns: torch.Tensor
    r!   r&   )r›   r‹   r)   ÚcumsumÚtype_asrÐ   )rÈ   r¬   ÚmaskÚincremental_indicess       r.   rÅ   rÅ   '  s`   € ð �<Š<˜Ñ$Ô$×(Ò(Ñ*Ô*€DÝœ, t°Ð3Ñ3Ô3×;Ò;¸DÑAÔAÀDÑHÐØ×#Ò#Ñ%Ô%¨Ñ3Ð3r0   )r‘  r³  rÁ  re  rK  )r!   )NrÄ   )Qr‡   rC   Úcollections.abcr   Útypingr   r)   r   Útorch.nnr   r   r   Ú r
   rT  Úintegrationsr   r   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   r   Úutils.output_capturingr   r    Úconfiguration_esmr"   Ú
get_loggerr„   r”  r/   r?   rF   rJ   rT   ÚModulerV   r‘   rª   rˆ   r5   rä   ræ   r  r  r  r  r  r1  rC  rK  re  r‘  rS  r³  rÁ  r·  rÅ   Ú__all__r4  r0   r.   ú<module>rà     s®  ðð Ð à €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ JÐ JÐ JÐ JÐ JÐ JÐ JÐ JØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð 7Ð 6Ð 6Ð 6Ð 6Ð 6Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ (Ð (Ð (Ð (Ð (Ð (ð 
ˆÔ	˜HÑ	%Ô	%€ð(ð (ð (ð ÐÐ*Ñ+Ô+ðBð Bð Bñ ,Ô+ðBð4;ð ;ð ;ð#ð #ð #ð
	ð 	ð 	ðA<ð A<ð A<ð A<ð A<˜œñ A<ô A<ð A<ðH Gð  Gð  Gð  Gð  G˜rœyñ  Gô  Gð  GðF\=ð \=ð \=ð \=ð \=�B”Iñ \=ô \=ð \=ðL !Øð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð8 ÐÐ)Ñ*Ô*ðK)ð K)ð K)ð K)ð K)�r”yñ K)ô K)ñ +Ô*ðK)ð\
ð 
ð 
ð 
ð 
�B”Iñ 
ô 
ð 
ðð ð ð ð �2”9ñ ô ð ð<ð ð ð ð �b”iñ ô ð ð
ð 
ð 
ð 
ð 
�”	ñ 
ô 
ð 
ð7ð 7ð 7ð 7ð 7Ð)ñ 7ô 7ð 7ðtSð Sð Sð Sð S�”ñ Sô Sð SðFð ð ð ð �”	ñ ô ð ð ð#ð #ð #ð #ð #˜ñ #ô #ñ „ð#ðL ðy0ð y0ð y0ð y0ð y0Ð!ñ y0ô y0ñ „ðy0ðx ðFPð FPð FPð FPð FPÐ'ñ FPô FPñ „ðFPðRð ð ð ð �”	ñ ô ð ð* €ððñ ô ðE
ð E
ð E
ð E
ð E
Ð#5ñ E
ô E
ñô ðE
ðP ð4
ð 4
ð 4
ð 4
ð 4
Ð 2ñ 4
ô 4
ñ „ð4
ðnð ð ð ð ˜BœIñ ô ð ð&4ð 4ð 4ð ð ð €€€r0   