§
    ‚ŠtjóŽ  ã                   ó2  — d dl Z d dlmZ d dlmZ d dlZd dlmZ d dlmZm	Z	m
Z
 ddlmZ ddlmZ dd	lmZmZ dd
lmZmZ ddlmZ ddlmZmZmZmZmZmZ ddlm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z) ddl*m+Z+m,Z,m-Z- ddl.m/Z/ ddl0m1Z1  G d„ dej2        ¦  «        Z3 G d„ dej2        ¦  «        Z4 G d„ dej2        ¦  «        Z5	 dAdej2        dej6        dej6        dej6        d ej6        dz  d!e7d"e7fd#„Z8d$„ Z9 ed%¦  «        dBd&„¦   «         Z: ee:¦  «         G d'„ d(ej2        ¦  «        ¦   «         Z; G d)„ d*e¦  «        Z<e) G d+„ d,e$¦  «        ¦   «         Z=e) G d-„ d.e=¦  «        ¦   «         Z> G d/„ d0ej2        ¦  «        Z? e)d1¬2¦  «         G d3„ d4e=¦  «        ¦   «         Z@ e)d5¬2¦  «         G d6„ d7e=¦  «        ¦   «         ZA e)d8¬2¦  «         G d9„ d:e=¦  «        ¦   «         ZBe) G d;„ d<e=¦  «        ¦   «         ZC e)d=¬2¦  «         G d>„ d?e=¦  «        ¦   «         ZDg d@¢ZEdS )Cé    N)ÚCallable)ÚOptional)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)Úuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚ(create_bidirectional_sliding_window_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstring)Úcan_return_tupleÚmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )ÚModernBertConfigc                   ój   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej        dz  dej        dz  dej        fd„Z	ˆ xZ
S )
ÚModernBertEmbeddingszV
    Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
    Úconfigc                 ó>  •— t          ¦   «                              ¦   «          || _        t          j        |j        |j        |j        ¬¦  «        | _        t          j	        |j        |j
        |j        ¬¦  «        | _        t          j        |j        ¦  «        | _        d S )N)Úpadding_idx©ÚepsÚbias)ÚsuperÚ__init__r&   r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚtok_embeddingsÚ	LayerNormÚnorm_epsÚ	norm_biasÚnormÚDropoutÚembedding_dropoutÚdrop©Úselfr&   Ú	__class__s     €úp/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/modernbert/modeling_modernbert.pyr-   zModernBertEmbeddings.__init__9   s{   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ œl¨6Ô+<¸fÔ>PÐ^dÔ^qÐrÑrÔrˆÔÝ”L Ô!3¸¼ÈvÔO_Ð`Ñ`Ô`ˆŒ	Ý”J˜vÔ7Ñ8Ô8ˆŒ	ˆ	ˆ	ó    NÚ	input_idsÚinputs_embedsÚreturnc                 óÒ   — |�)|                       |                      |¦  «        ¦  «        }n;|                       |                      |                      |¦  «        ¦  «        ¦  «        }|S ©N)r9   r6   r2   )r;   r?   r@   Úhidden_statess       r=   ÚforwardzModernBertEmbeddings.forward@   sZ   € ð Ð$Ø ŸIšI d§i¢i°Ñ&>Ô&>Ñ?Ô?ˆMˆMà ŸIšI d§i¢i°×0CÒ0CÀIÑ0NÔ0NÑ&OÔ&OÑPÔPˆMØÐr>   ©NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r#   r-   ÚtorchÚ
LongTensorÚTensorrE   Ú__classcell__©r<   s   @r=   r%   r%   4   s™   ø€ € € € € ðð ð9Ð/ð 9ð 9ð 9ð 9ð 9ð 9ð _cðð ØÔ)¨DÑ0ðØHMÌÐW[ÑH[ðà	Œðð ð ð ð ð ð ð r>   r%   c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚModernBertMLPa6  Applies the GLU at the end of each ModernBERT layer.

    Compared to the default BERT architecture, this block replaces :class:`~transformers.model.bert.modeling_bert.BertIntermediate`
    and :class:`~transformers.model.bert.modeling_bert.SelfOutput` with a single module that has similar functionality.
    r&   c                 óŒ  •— t          ¦   «                              ¦   «          || _        t          j        |j        t          |j        ¦  «        dz  |j        ¬¦  «        | _	        t          |j                 | _        t          j        |j        ¦  «        | _        t          j        |j        |j        |j        ¬¦  «        | _        d S )Né   ©r+   )r,   r-   r&   r   ÚLinearr0   ÚintÚintermediate_sizeÚmlp_biasÚWir   Úhidden_activationÚactr7   Úmlp_dropoutr9   ÚWor:   s     €r=   r-   zModernBertMLP.__init__Q   s—   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”)˜FÔ.µ°FÔ4LÑ0MÔ0MÐPQÑ0QÐX^ÔXgÐhÑhÔhˆŒÝ˜&Ô2Ô3ˆŒÝ”J˜vÔ1Ñ2Ô2ˆŒ	Ý”)˜FÔ4°fÔ6HÈvÌÐ_Ñ_Ô_ˆŒˆˆr>   rD   rA   c                 óØ   — |                       |¦  «                             dd¬¦  «        \  }}|                      |                      |                      |¦  «        |z  ¦  «        ¦  «        S )NrS   éÿÿÿÿ©Údim)rY   Úchunkr]   r9   r[   )r;   rD   ÚinputÚgates       r=   rE   zModernBertMLP.forwardY   sW   € Ø—g’g˜mÑ,Ô,×2Ò2°1¸"Ð2Ñ=Ô=‰ˆˆtØ�wŠw�t—y’y §¢¨%¡¤°4Ñ!7Ñ8Ô8Ñ9Ô9Ð9r>   )
rG   rH   rI   rJ   r#   r-   rK   rM   rE   rN   rO   s   @r=   rQ   rQ   J   s|   ø€ € € € € ðð ð`Ð/ð `ð `ð `ð `ð `ð `ð: U¤\ð :°e´lð :ð :ð :ð :ð :ð :ð :ð :r>   rQ   c                   óâ   ‡ — e Zd ZU ej        ed<   ddefˆ fd„Ze	 	 	 	 ddedz  de	d         de
dz  dedz  d	ed
ef         f
d„¦   «         Z ej        ¦   «         edd„¦   «         ¦   «         Zˆ xZS )ÚModernBertRotaryEmbeddingÚinv_freqNr&   c                 ó€  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        t          t          |j        ¦  «        ¦  «        | _        i | _	        | j        D ]È}| j        j
        |         }|€Œ|d         | j	        |<   | j        }| j	        |         dk    rt          | j	        |                  } || j        |¬¦  «        \  }}|                      |› d�|d¬¦  «         |                      |› d�|                     ¦   «         d¬¦  «         t          | |› d�|¦  «         ŒÉd S )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqF)Ú
persistentÚ_original_inv_freqÚ_attention_scaling)r,   r-   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr&   ÚlistÚsetÚlayer_typesri   Úrope_parametersÚcompute_default_rope_parametersr   Úregister_bufferÚcloneÚsetattr)	r;   r&   Údevicerl   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingr<   s	           €r=   r-   z"ModernBertRotaryEmbedding.__init__a   s^  ø€ Ý‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!ØˆŒÝ¥ FÔ$6Ñ 7Ô 7Ñ8Ô8ˆÔØˆŒØÔ*ð 	Uð 	UˆJØœ+Ô5°jÔAˆKØÐ"Øà)4°[Ô)AˆDŒN˜:Ñ&Ø%)Ô%IˆLØŒ~˜jÔ)¨YÒ6Ð6Ý2°4´>À*Ô3MÔN�Ø4@°LÀÄÐYcÐ4dÑ4dÔ4dÑ1ˆMÐ1Ø× Ò  JÐ!9Ð!9Ð!9¸=ÐUZÐ Ñ[Ô[Ð[Ø× Ò  JÐ!BÐ!BÐ!BÀM×DWÒDWÑDYÔDYÐfkÐ ÑlÔlÐlÝ�D˜ZÐ;Ð;Ð;Ð=SÑTÔTÐTÐTð	Uð 	Ur>   r|   ztorch.deviceÚseq_lenrl   rA   ztorch.Tensorc                 ó  — | j         |         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «                             |t          j        ¬¦  «        |z  z  z  }||fS )	a|  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNg      ð?r   rS   ©Údtype©r|   r†   )	rw   Úgetattrr0   Únum_attention_headsrK   ÚarangeÚint64ÚtoÚfloat)r&   r|   r�   rl   Úbasera   Úattention_factorrg   s           r=   rx   z9ModernBertRotaryEmbedding.compute_default_rope_parametersv   s‘   € ð2 Ô% jÔ1°,Ô?ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐð Ø•U”\ ! S¨!µ5´;Ð?Ñ?Ô?×BÒBÈ&ÕX]ÔXcÐBÑdÔdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r>   c                 ó|  — t          | |› d�¦  «        }t          | |› d�¦  «        }|d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬	¦  «        5  |                     ¦   «         |                     ¦   «         z   
                    dd
¦  «        }	t          j        |	|	fd¬¦  «        }
|
                     ¦   «         |z  }|
                     ¦   «         |z  }d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |                     |j        ¬¦  «        fS )Nrm   rp   r   r_   r"   ÚmpsÚcpuF)Údevice_typeÚenabledrS   r`   r…   )rˆ   r�   ÚexpandÚshaperŒ   r|   Ú
isinstanceÚtypeÚstrr   Ú	transposerK   ÚcatÚcosÚsinr†   )r;   ÚxÚposition_idsrl   rg   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedr“   ÚfreqsÚembrœ   r�   s                r=   rE   z!ModernBertRotaryEmbedding.forwardš   sâ  € õ ˜4 JÐ!9Ð!9Ð!9Ñ:Ô:ˆÝ# D¨ZÐ*KÐ*KÐ*KÑLÔLÐà$ T¨1¨1¨1¨d ]Ô3×9Ò9Ñ;Ô;×BÒBÀ<ÔCUÐVWÔCXÐZ\Ð^_Ñ`Ô`×cÒcÐdeÔdlÑmÔmÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	0ð 	0Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)Ð/Ñ/ˆCØ—'’'‘)”)Ð/Ñ/ˆCð		0ð 	0ð 	0ñ 	0ô 	0ð 	0ð 	0ð 	0ð 	0ð 	0ð 	0øøøð 	0ð 	0ð 	0ð 	0ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   Ã-BE=Å=FÆFrC   ©NNNN)rG   rH   rI   rK   rM   Ú__annotations__r#   r-   Ústaticmethodr   rV   r™   Útupler�   rx   Úno_gradr   rE   rN   rO   s   @r=   rf   rf   ^   s  ø€ € € € € € ØŒlÐÐÑðUð UÐ/ð Uð Uð Uð Uð Uð Uð* à*.Ø+/Ø"Ø!%ð	!*ð !*Ø  4Ñ'ð!*à˜Ô(ð!*ð �t‘ð!*ð ˜$‘Jð	!*ð
 
ˆ~˜uÐ$Ô	%ð!*ð !*ð !*ñ „\ð!*ðF €U„]�_„_Øð<ð <ð <ñ Ôñ „_ð<ð <ð <ð <ð <r>   rf   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutc                 óÀ  — t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |dt           j        ¬¦  «                             |j        ¦  «        }t          j         	                    ||| j
        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «                             ¦   «         }	|	|fS )NrS   r	   r_   )ra   r†   )ÚpÚtrainingr"   )rK   Úmatmulrš   r   Ú
functionalÚsoftmaxÚfloat32rŒ   r†   r±   r´   Ú
contiguous)
r«   r¬   r­   r®   r¯   r°   r±   ÚkwargsÚattn_weightsÚattn_outputs
             r=   Úeager_attention_forwardr½   ­   sÃ   € õ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€KØ˜Ð$Ð$r>   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr_   rS   r`   )r–   rK   r›   )rž   Úx1Úx2s      r=   Úrotate_halfrÁ   Ã   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r>   Úrotary_pos_embc                 ó¨  — | j         }|                     |¦  «        }|                     |¦  «        }|                      ¦   «         |z  t          |                      ¦   «         ¦  «        |z  z   }|                     ¦   «         |z  t          |                     ¦   «         ¦  «        |z  z   }|                     |¦  «        |                     |¦  «        fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )r†   Ú	unsqueezer�   rÁ   rŒ   )ÚqÚkrœ   r�   Úunsqueeze_dimÚoriginal_dtypeÚq_embedÚk_embeds           r=   Úapply_rotary_pos_embrË   Ê   s¦   € ð& ”W€NØ
�-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�wŠw‰yŒy˜3‰¥;¨q¯wªw©y¬yÑ#9Ô#9¸CÑ#?Ñ@€GØ�wŠw‰yŒy˜3‰¥;¨q¯wªw©y¬yÑ#9Ô#9¸CÑ#?Ñ@€GØ�:Š:�nÑ%Ô% w§z¢z°.Ñ'AÔ'AÐAÐAr>   c                   óÐ   ‡ — e Zd ZdZddededz  fˆ fd„Z	 	 ddej        de	ej        ej        f         dz  dej        dz  d	e
e         d
e	ej        ej        dz  f         f
d„Zˆ xZS )ÚModernBertAttentiona‚  Performs multi-headed self attention on a batch of unpadded sequences.

    If Flash Attention 2 is installed, this module uses Flash Attention to improve throughput.
    If Flash Attention 2 is not installed, the implementation will use PyTorch's SDPA kernel,
    which requires padding and unpadding inputs, adding some overhead.

    See `forward` method for additional details.
    Nr&   Ú	layer_idxc                 óº  •— t          ¦   «                              ¦   «          || _        || _        |j        |j        z  dk    r t          d|j        › d|j        › d�¦  «        ‚|j        | _        |j        | _        |j        |j        z  | _	        t          j        |j        d| j	        z  |j        z  |j        ¬¦  «        | _        |j        |         dk    r|j        dz   | _        nd | _        d	| _        t          j        |j        |j        |j        ¬¦  «        | _        |j        d
k    rt          j        |j        ¦  «        nt          j        ¦   «         | _        d S )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads (ú)r	   rT   Úsliding_attentionr"   Frª   )r,   r-   r&   rÎ   r0   r‰   Ú
ValueErrorÚattention_dropoutÚdeterministic_flash_attnr„   r   rU   Úattention_biasÚWqkvrv   Úsliding_windowÚ	is_causalr]   r7   ÚIdentityÚout_drop©r;   r&   rÎ   r<   s      €r=   r-   zModernBertAttention.__init__ð   sq  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒàÔ Ô :Ñ:¸aÒ?Ð?Ýð L FÔ$6ð  Lð  LÐntô  oIð  Lð  Lð  Lñô ð ð "(Ô!9ˆÔØ(.Ô(GˆÔ%ØÔ*¨fÔ.HÑHˆŒÝ”IØÔ  D¤MÑ 1°FÔ4NÑ NÐU[ÔUjð
ñ 
ô 
ˆŒ	ð Ô˜iÔ(Ð,?Ò?Ð?ð #)Ô"7¸!Ñ";ˆDÔÐà"&ˆDÔàˆŒå”)˜FÔ.°Ô0BÈÔI^Ð_Ñ_Ô_ˆŒØ@FÔ@XÐ[^Ò@^Ð@^�œ
 6Ô#;Ñ<Ô<Ð<ÕdfÔdoÑdqÔdqˆŒˆˆr>   rD   Úposition_embeddingsr¯   rº   rA   c                 ó¶  — |j         d d…         }|                      |¦  «        } |j        g |¢d‘d‘| j        ‘R Ž }|                     d¬¦  «        \  }}}	|                     dd¦  «        }|                     dd¦  «        }|	                     dd¦  «        }	|\  }
}t          |||
|d¬¦  «        \  }}t          j        | j	        j
        t          ¦  «        } || |||	|f| j        r| j        nd| j        d	z  | j        | j        d
œ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |                      |¦  «        ¦  «        }||fS )Nr_   r	   éýÿÿÿr`   r"   rS   )rÇ   rª   ç      à¿)r±   r°   r×   Údeterministic)r–   rÖ   Úviewr„   Úunbindrš   rË   r   Úget_interfacer&   Ú_attn_implementationr½   r´   rÓ   r×   rÔ   Úreshaper¹   rÚ   r]   )r;   rD   rÜ   r¯   rº   Úinput_shapeÚqkvÚquery_statesÚ
key_statesÚvalue_statesrœ   r�   Úattention_interfacer¼   r»   s                  r=   rE   zModernBertAttention.forward  s§  € ð $Ô)¨#¨2¨#Ô.ˆà�iŠi˜Ñ&Ô&ˆØˆcŒhÐ:˜Ð: QÐ:¨Ð:¨D¬MÐ:Ð:Ð:ˆØ14·²À°Ñ1CÔ1CÑ.ˆ�j ,à#×-Ò-¨a°Ñ3Ô3ˆØ×)Ò)¨!¨QÑ/Ô/ˆ
Ø#×-Ò-¨a°Ñ3Ô3ˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÐjkÐ#lÑ#lÔ#lÑ ˆ�jå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð%
ð /3¬mÐD�DÔ*Ð*ÀØ”M 4Ñ'ØÔ.ØÔ7ð%
ð %
ð ð%
ð %
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—m’m D§G¢G¨KÑ$8Ô$8Ñ9Ô9ˆØ˜LÐ(Ð(r>   rC   rF   )rG   rH   rI   rJ   r#   rV   r-   rK   rM   r¨   r   r   rE   rN   rO   s   @r=   rÍ   rÍ   å   sñ   ø€ € € € € ðð ðrð rÐ/ð r¸CÀ$¹Jð rð rð rð rð rð rð@ IMØ.2ð	')ð ')à”|ð')ð # 5¤<°´Ð#=Ô>ÀÑEð')ð œ tÑ+ð	')ð
 Ð+Ô,ð')ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð')ð ')ð ')ð ')ð ')ð ')ð ')ð ')r>   rÍ   c                   ó’   ‡ — e Zd Zddededz  fˆ fd„Z	 	 ddej        dej        dz  dej        dz  dee	         d	ej        f
d
„Z
ˆ xZS )ÚModernBertEncoderLayerNr&   rÎ   c                 óÆ  •— t          ¦   «                              ¦   «          || _        || _        |dk    rt	          j        ¦   «         | _        n+t	          j        |j        |j	        |j
        ¬¦  «        | _        t          ||¬¦  «        | _        t	          j        |j        |j	        |j
        ¬¦  «        | _        t          |¦  «        | _        |j        |         | _        d S )Nr   r)   )r&   rÎ   )r,   r-   r&   rÎ   r   rÙ   Ú	attn_normr3   r0   r4   r5   rÍ   ÚattnÚmlp_normrQ   Úmlprv   Úattention_typerÛ   s      €r=   r-   zModernBertEncoderLayer.__init__8  sº   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ˜Š>ˆ>Ýœ[™]œ]ˆDŒNˆNåœ\¨&Ô*<À&Ä/ÐX^ÔXhÐiÑiÔiˆDŒNÝ'¨vÀÐKÑKÔKˆŒ	Ýœ VÔ%7¸V¼_ÐSYÔScÐdÑdÔdˆŒÝ  Ñ(Ô(ˆŒØ$Ô0°Ô;ˆÔÐÐr>   rD   r¯   rÜ   rº   rA   c                 ó´   —  | j         |                      |¦  «        f||dœ|¤Ž\  }}||z   }||                      |                      |¦  «        ¦  «        z   }|S )N)rÜ   r¯   )rð   rï   rò   rñ   )r;   rD   r¯   rÜ   rº   r¼   Ú_s          r=   rE   zModernBertEncoderLayer.forwardE  sx   € ð #˜œØ�NŠN˜=Ñ)Ô)ð
à 3Ø)ð
ð 
ð ð	
ð 
‰ˆ�Qð &¨Ñ3ˆØ%¨¯ª°·²¸}Ñ1MÔ1MÑ(NÔ(NÑNˆØÐr>   rC   rF   )rG   rH   rI   r#   rV   r-   rK   rM   r   r   rE   rN   rO   s   @r=   rí   rí   7  s¾   ø€ € € € € ð<ð <Ð/ð <¸CÀ$¹Jð <ð <ð <ð <ð <ð <ð  /3Ø37ð	ð à”|ðð œ tÑ+ðð #œ\¨DÑ0ð	ð
 Ð+Ô,ðð 
Œðð ð ð ð ð ð ð r>   rí   c                   óŽ   ‡ — e Zd ZU eed<   dZdZddgZdZdZ	dZ
dZeedœZ ej        ¦   «         dej        fˆ fd„¦   «         Zˆ xZS )	ÚModernBertPreTrainedModelr&   ÚmodelTr%   rí   )rD   Ú
attentionsr«   c                 óT  •‡— t          ¦   «                              |¦  «         | j        j        Š‰€dŠdt          j        dt          fˆfd„}| j        j        | j        j        t          j	        d| j        j
        z  ¦  «        z  | j        j        | j        j        dz  dœ}t          |t          ¦  «        r ||j        |d         ¦  «         d S t          |t          ¦  «        r0 ||j        |d	         ¦  «          ||j        |d
         ¦  «         d S t          |t$          ¦  «        r0 ||j        |d	         ¦  «          ||j        |d
         ¦  «         d S t          |t(          ¦  «        r ||j        |d
         ¦  «         d S t          |t,          ¦  «        r ||j        |d
         ¦  «         d S t          |t0          t2          t4          t6          f¦  «        r ||j        |d         ¦  «         d S t          |t:          ¦  «        r›|j        D ]•}|j        }|j         |         dk    rtB          |j         |                  } ||j        |¬¦  «        \  }}tE          j#        tI          ||› d�¦  «        |¦  «         tE          j#        tI          ||› d�¦  «        |¦  «         Œ”d S d S )Nr	   r«   Ústdc                 óÎ   •— t          j        | j        d|‰ |z  ‰|z  ¬¦  «         t          | t          j        ¦  «        r"| j        �t          j        | j        ¦  «         d S d S d S )Nrª   )Úmeanrû   ÚaÚb)ÚinitÚtrunc_normal_Úweightr—   r   rU   r+   Úzeros_)r«   rû   Úcutoff_factors     €r=   Úinit_weightz<ModernBertPreTrainedModel._init_weights.<locals>.init_weightn  s€   ø€ ÝÔØ”ØØØ �. 3Ñ&Ø #Ñ%ðñ ô ð õ ˜&¥"¤)Ñ,Ô,ð -Ø”;Ð*Ý”K ¤Ñ,Ô,Ð,Ð,Ð,ð-ð -Ø*Ð*r>   g       @rß   )ÚinÚoutÚ	embeddingÚ	final_outr  r  r  r	  rj   rk   rm   ro   )%r,   Ú_init_weightsr&   Úinitializer_cutoff_factorr   ÚModuler�   Úinitializer_rangeÚmathÚsqrtÚnum_hidden_layersr0   r—   r%   r2   rQ   rY   r]   rÍ   rÖ   ÚModernBertPredictionHeadÚdenseÚModernBertForMaskedLMÚdecoderÚ#ModernBertForSequenceClassificationÚModernBertForMultipleChoiceÚ ModernBertForTokenClassificationÚModernBertForQuestionAnsweringÚ
classifierrf   rv   rx   ri   r   r   Úcopy_rˆ   )
r;   r«   r  Ústdsrl   r~   r   rõ   r  r<   s
           @€r=   r
  z'ModernBertPreTrainedModel._init_weightsg  sö  øø€ å‰Œ×Ò˜fÑ%Ô%Ð%ØœÔ=ˆØÐ ØˆMð	-¥¤	ð 	-µð 	-ð 	-ð 	-ð 	-ð 	-ð 	-ð ”+Ô/Ø”;Ô0µ4´9¸SÀ4Ä;ÔC`Ñ=`Ñ3aÔ3aÑaØœÔ6ØœÔ0°$Ñ6ð	
ð 
ˆõ �fÕ2Ñ3Ô3ð 	^ØˆK˜Ô-¨t°KÔ/@ÑAÔAÐAÐAÐAÝ˜¥Ñ.Ô.ð 	^ØˆK˜œ	 4¨¤:Ñ.Ô.Ð.ØˆK˜œ	 4¨¤;Ñ/Ô/Ð/Ð/Ð/Ý˜Õ 3Ñ4Ô4ð 	^ØˆK˜œ T¨$¤ZÑ0Ô0Ð0ØˆK˜œ	 4¨¤;Ñ/Ô/Ð/Ð/Ð/Ý˜Õ 8Ñ9Ô9ð 	^ØˆK˜œ d¨5¤kÑ2Ô2Ð2Ð2Ð2Ý˜Õ 5Ñ6Ô6ð 	^ØˆK˜œ¨¨U¬Ñ4Ô4Ð4Ð4Ð4ÝØå3Ý+Ý0Ý.ð	ñ
ô 
ð 	^ð ˆK˜Ô)¨4°Ô+<Ñ=Ô=Ð=Ð=Ð=Ý˜Õ 9Ñ:Ô:ð 	^Ø$Ô0ð ^ð ^�
Ø%ÔE�ØÔ# JÔ/°9Ò<Ð<Ý#6°vÔ7GÈ
Ô7SÔ#T�LØ#/ <°´È*Ð#UÑ#UÔ#UÑ �˜qÝ”
�7 6¨jÐ+CÐ+CÐ+CÑDÔDÀmÑTÔTÐTÝ”
�7 6¨jÐ+LÐ+LÐ+LÑMÔMÈ}Ñ]Ô]Ð]Ð]ð	^ð 	^ð^ð ^r>   )rG   rH   rI   r#   r¦   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendrí   rÍ   Ú_can_record_outputsrK   r©   r   r  r
  rN   rO   s   @r=   r÷   r÷   W  s·   ø€ € € € € € àÐÐÑØÐØ&*Ð#Ø/Ð1IÐJÐØÐØ€NØÐØ"&Ðð 0Ø)ðð Ðð
 €U„]�_„_ð7^ B¤Ið 7^ð 7^ð 7^ð 7^ð 7^ñ „_ð7^ð 7^ð 7^ð 7^ð 7^r>   r÷   c                   óÖ   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zeee		 	 	 	 dde
j        dz  de
j        dz  de
j        dz  d	e
j        dz  d
ee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚModernBertModelr&   c                 ó¦  •‡— t          ¦   «                              ‰¦  «         ‰| _        t          ‰¦  «        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _	        t          j
        ‰j        ‰j        ‰j        ¬¦  «        | _        t          ‰¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS © )rí   )Ú.0rÎ   r&   s     €r=   ú
<listcomp>z,ModernBertModel.__init__.<locals>.<listcomp>©  s$   ø€ ÐhÐhÐh¸9Õ# F¨IÑ6Ô6ÐhÐhÐhr>   r)   )r&   F)r,   r-   r&   r%   Ú
embeddingsr   Ú
ModuleListÚranger  Úlayersr3   r0   r4   r5   Ú
final_normrf   Ú
rotary_embÚgradient_checkpointingÚ	post_initr:   s    `€r=   r-   zModernBertModel.__init__¤  sº   øø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ.¨vÑ6Ô6ˆŒÝ”mØhÐhÐhÐhÍÈfÔNfÑHgÔHgÐhÑhÔhñ
ô 
ˆŒõ œ, vÔ'9¸v¼ÐU[ÔUeÐfÑfÔfˆŒÝ3¸6ÐBÑBÔBˆŒØ&+ˆÔ#Ø�ŠÑÔÐÐÐr>   c                 ó   — | j         j        S rC   ©r+  r2   ©r;   s    r=   Úget_input_embeddingsz$ModernBertModel.get_input_embeddings°  s   € ØŒÔ-Ð-r>   c                 ó   — || j         _        d S rC   r4  )r;   r®   s     r=   Úset_input_embeddingsz$ModernBertModel.set_input_embeddings³  s   € Ø).ˆŒÔ&Ð&Ð&r>   Nr?   r¯   rŸ   r@   rº   rA   c                 ó–  — |d u |d uz  rt          d¦  «        ‚|�|j        d         n|j        d         }|�|j        n|j        }|€)t          j        ||¬¦  «                             d¦  «        }|                      ||¬¦  «        }t          |x}	t          ¦  «        s$| j	        ||dœ}
t          d
i |
¤Žt          d
i |
¤Ždœ}	i }t          | j	        j        ¦  «        D ]}|                      |||¦  «        ||<   Œ| j        D ]$} ||f|	|j                 ||j                 dœ|¤Ž}Œ%|                      |¦  «        }t%          |¬	¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embedsr"   ©r|   r   )r?   r@   )r&   r@   r¯   )Úfull_attentionrÑ   )r¯   rÜ   )Úlast_hidden_stater(  )rÒ   r–   r|   rK   rŠ   rÄ   r+  r—   Údictr&   r   r   ru   rv   r0  r.  ró   r/  r   )r;   r?   r¯   rŸ   r@   rº   r�   r|   rD   Úattention_mask_mappingÚmask_kwargsrÜ   rl   Úencoder_layers                 r=   rE   zModernBertModel.forward¶  s·  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZà,9Ð,E�-Ô% aÔ(Ð(È9Ì?Ð[\ÔK]ˆØ%.Ð%:�Ô!Ð!ÀÔ@TˆàÐÝ œ<¨¸Ð?Ñ?Ô?×IÒIÈ!ÑLÔLˆLàŸš°)È=˜ÑYÔYˆå°NÐBÐ0ÅDÑIÔIð 		àœ+Ø!.Ø"0ðð ˆKõ #<Ð"JÐ"J¸kÐ"JÐ"JÝ%MÐ%\Ð%\ÐP[Ð%\Ð%\ð&ð &Ð"ð
 !ÐÝ˜dœkÔ5Ñ6Ô6ð 	gð 	gˆJØ.2¯oªo¸mÈ\Ð[eÑ.fÔ.fÐ 
Ñ+Ð+à!œ[ð 	ð 	ˆMØ)˜MØðà5°mÔ6RÔSØ$7¸Ô8TÔ$Uðð ð ð	ð ˆMˆMð Ÿš¨Ñ6Ô6ˆå°Ð?Ñ?Ô?Ð?r>   r¥   )rG   rH   rI   r#   r-   r6  r8  r    r!   r   rK   rL   rM   r   r   r   rE   rN   rO   s   @r=   r%  r%  ¢  s  ø€ € € € € ð
Ð/ð 
ð 
ð 
ð 
ð 
ð 
ð.ð .ð .ð/ð /ð /ð  ØØð .2Ø.2Ø04Ø-1ð,@ð ,@àÔ# dÑ*ð,@ð œ tÑ+ð,@ð Ô&¨Ñ-ð	,@ð
 ”| dÑ*ð,@ð Ð+Ô,ð,@ð 
ð,@ð ,@ð ,@ñ „^ñ „_ñ  Ôð,@ð ,@ð ,@ð ,@ð ,@r>   r%  c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )r  r&   c                 ó.  •— t          ¦   «                              ¦   «          || _        t          j        |j        |j        |j        ¦  «        | _        t          |j	                 | _
        t          j        |j        |j        |j        ¬¦  «        | _        d S )Nr)   )r,   r-   r&   r   rU   r0   Úclassifier_biasr  r   Úclassifier_activationr[   r3   r4   r5   r6   r:   s     €r=   r-   z!ModernBertPredictionHead.__init__é  sq   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”Y˜vÔ1°6Ô3EÀvÔG]Ñ^Ô^ˆŒ
Ý˜&Ô6Ô7ˆŒÝ”L Ô!3¸¼ÈvÔO_Ð`Ñ`Ô`ˆŒ	ˆ	ˆ	r>   rD   rA   c                 óx   — |                       |                      |                      |¦  «        ¦  «        ¦  «        S rC   )r6   r[   r  )r;   rD   s     r=   rE   z ModernBertPredictionHead.forwardð  s,   € Ø�yŠy˜Ÿš $§*¢*¨]Ñ";Ô";Ñ<Ô<Ñ=Ô=Ð=r>   )	rG   rH   rI   r#   r-   rK   rM   rE   rN   rO   s   @r=   r  r  è  sr   ø€ € € € € ðaÐ/ð að að að að að að> U¤\ð >°e´lð >ð >ð >ð >ð >ð >ð >ð >r>   r  zd
    The ModernBert Model with a decoder head on top that is used for masked language modeling.
    )Úcustom_introc                   ó  ‡ — e Zd ZddiZdefˆ fd„Zd„ Zdej        fd„Z	e
e	 	 	 	 	 dd	ej        dz  d
ej        dz  dej        dz  dej        dz  dej        dz  dee         deej                 ez  fd„¦   «         ¦   «         Zˆ xZS )r  zdecoder.weightz&model.embeddings.tok_embeddings.weightr&   c                 ój  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        t          j        |j	        |j
        |j        ¬¦  «        | _        | j        j        | _        | j        j        | _        |                      ¦   «          d S )NrT   )r,   r-   r&   r%  rø   r  Úheadr   rU   r0   r/   Údecoder_biasr  Úsparse_predictionÚsparse_pred_ignore_indexr2  r:   s     €r=   r-   zModernBertForMaskedLM.__init__ü  s“   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ$ VÑ,Ô,ˆŒ
Ý,¨VÑ4Ô4ˆŒ	Ý”y Ô!3°VÔ5FÈVÔM`ÐaÑaÔaˆŒà!%¤Ô!>ˆÔØ(,¬Ô(LˆÔ%ð 	�ŠÑÔÐÐÐr>   c                 ó   — | j         S rC   ©r  r5  s    r=   Úget_output_embeddingsz+ModernBertForMaskedLM.get_output_embeddings	  s
   € ØŒ|Ðr>   Únew_embeddingsc                 ó   — || _         d S rC   rN  )r;   rP  s     r=   Úset_output_embeddingsz+ModernBertForMaskedLM.set_output_embeddings  s   € Ø%ˆŒˆˆr>   Nr?   r¯   rŸ   r@   Úlabelsrº   rA   c                 ó²  —  | j         d||||dœ|¤Ž}|d         }| j        rS|�Q|                     d¦  «        }|                     |j        d         d¦  «        }|| j        k    }	||	         }||	         }|                      |                      |¦  «        ¦  «        }
d }|� | j        |
|fd| j        j	        i|¤Ž}t          ||
|j        |j        ¬¦  «        S )N©r?   r¯   rŸ   r@   r   r_   r/   ©ÚlossÚlogitsrD   rù   r(  )rø   rK  rá   r–   rL  r  rI  Úloss_functionr&   r/   r   rD   rù   )r;   r?   r¯   rŸ   r@   rS  rº   Úoutputsr<  Úmask_tokensrX  rW  s               r=   rE   zModernBertForMaskedLM.forward  s  € ð �$”*ð 
ØØ)Ø%Ø'ð	
ð 
ð
 ð
ð 
ˆð $ AœJÐàÔ!ð 	) fÐ&8à—[’[ ‘_”_ˆFØ 1× 6Ò 6°v´|ÀA´ÈÑ KÔ KÐð ! DÔ$AÒAˆKØ 1°+Ô >ÐØ˜KÔ(ˆFà—’˜dŸišiÐ(9Ñ:Ô:Ñ;Ô;ˆàˆØÐØ%�4Ô% f¨fÐbÐbÀÄÔAWÐbÐ[aÐbÐbˆDåØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r>   ©NNNNN)rG   rH   rI   Ú_tied_weights_keysr#   r-   rO  r   rU   rR  r   r   rK   rL   rM   r   r   r¨   r   rE   rN   rO   s   @r=   r  r  ô  s:  ø€ € € € € ð +Ð,TÐUÐðÐ/ð ð ð ð ð ð ðð ð ð&°B´Ið &ð &ð &ð &ð Øð .2Ø.2Ø,0Ø-1Ø&*ð'
ð '
àÔ# dÑ*ð'
ð œ tÑ+ð'
ð ”l TÑ)ð	'
ð
 ”| dÑ*ð'
ð ”˜tÑ#ð'
ð Ð+Ô,ð'
ð 
ˆuŒ|Ô	˜~Ñ	-ð'
ð '
ð '
ñ „^ñ Ôð'
ð '
ð '
ð '
ð '
r>   r  z`
    The ModernBert Model with a sequence classification head on top that performs pooling.
    c                   óì   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej        dz  dej	        dz  dej	        dz  dej	        dz  dej	        dz  d	e
e         d
eej	                 ez  fd„¦   «         ¦   «         Zˆ xZS )r  r&   c                 ó‚  •— t          ¦   «                              |¦  «         |j        | _        || _        t	          |¦  «        | _        t          |¦  «        | _        t          j	         
                    |j        ¦  «        | _        t          j        |j        |j        ¦  «        | _        |                      ¦   «          d S rC   )r,   r-   Ú
num_labelsr&   r%  rø   r  rI  rK   r   r7   Úclassifier_dropoutr9   rU   r0   r  r2  r:   s     €r=   r-   z,ModernBertForSequenceClassification.__init__A  s•   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒØˆŒå$ VÑ,Ô,ˆŒ
Ý,¨VÑ4Ô4ˆŒ	Ý”H×$Ò$ VÔ%>Ñ?Ô?ˆŒ	Ýœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐr>   Nr?   r¯   rŸ   r@   rS  rº   rA   c                 óø  —  | j         d||||dœ|¤Ž}|d         }| j        j        dk    r|dd…df         }n‰| j        j        dk    ry|€3t          j        |j        dd…         |j        t          j        ¬¦  «        }||                     d¦  «        z   	                    d	¬
¦  «        | 	                    d	d¬¦  «        z  }|  
                    |¦  «        }	|                      |	¦  «        }	|                      |	¦  «        }
d}|��Z| j        j        €f| j        d	k    rd| j        _        nN| j        d	k    r7|j        t          j        k    s|j        t          j        k    rd| j        _        nd| j        _        | j        j        dk    rWt%          ¦   «         }| j        d	k    r1 ||
                     ¦   «         |                     ¦   «         ¦  «        }nŽ ||
|¦  «        }n�| j        j        dk    rGt)          ¦   «         } ||
                     d| j        ¦  «        |                     d¦  «        ¦  «        }n*| j        j        dk    rt-          ¦   «         } ||
|¦  «        }t/          ||
|j        |j        ¬¦  «        S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        rU  r   ÚclsNrý   rS   r‡   r_   r"   r`   T©ra   ÚkeepdimÚ
regressionÚsingle_label_classificationÚmulti_label_classificationrV  r(  )rø   r&   Úclassifier_poolingrK   Úonesr–   r|   ÚboolrÄ   ÚsumrI  r9   r  Úproblem_typer`  r†   ÚlongrV   r   Úsqueezer   rá   r   r   rD   rù   )r;   r?   r¯   rŸ   r@   rS  rº   rZ  r<  Úpooled_outputrX  rW  Úloss_fcts                r=   rE   z+ModernBertForSequenceClassification.forwardN  sª  € ð" �$”*ð 
ØØ)Ø%Ø'ð	
ð 
ð
 ð
ð 
ˆð $ AœJÐàŒ;Ô)¨UÒ2Ð2Ø 1°!°!°!°Q°$Ô 7ÐÐØŒ[Ô+¨vÒ5Ð5ØÐ%Ý!&¤Ø%Ô+¨B¨Q¨BÔ/Ð8IÔ8PÕX]ÔXbð"ñ "ô "�ð "3°^×5MÒ5MÈbÑ5QÔ5QÑ!Q× VÒ VÐ[\Ð VÑ ]Ô ]Ð`n×`rÒ`rØ˜tð asñ aô añ !Ðð Ÿ	š	Ð"3Ñ4Ô4ˆØŸ	š	 -Ñ0Ô0ˆØ—’ Ñ/Ô/ˆàˆØÑØŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ñ @Ô @À&Ç+Â+ÈbÁ/Ä/ÑRÔR��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�å'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r>   r\  )rG   rH   rI   r#   r-   r   r   rK   rL   rM   r   r   r¨   r   rE   rN   rO   s   @r=   r  r  ;  s  ø€ € € € € ðÐ/ð ð ð ð ð ð ð Øð .2Ø.2Ø,0Ø-1Ø&*ðC
ð C
àÔ# dÑ*ðC
ð œ tÑ+ðC
ð ”l TÑ)ð	C
ð
 ”| dÑ*ðC
ð ”˜tÑ#ðC
ð Ð+Ô,ðC
ð 
ˆuŒ|Ô	Ð7Ñ	7ðC
ð C
ð C
ñ „^ñ ÔðC
ð C
ð C
ð C
ð C
r>   r  zv
    The ModernBert Model with a token classification head on top, e.g. for Named Entity Recognition (NER) tasks.
    c                   óì   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej        dz  dej	        dz  dej	        dz  dej	        dz  dej	        dz  d	e
e         d
eej	                 ez  fd„¦   «         ¦   «         Zˆ xZS )r  r&   c                 ót  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          |¦  «        | _        t          j         	                    |j
        ¦  «        | _        t          j        |j        |j        ¦  «        | _        |                      ¦   «          d S rC   ©r,   r-   r`  r%  rø   r  rI  rK   r   r7   ra  r9   rU   r0   r  r2  r:   s     €r=   r-   z)ModernBertForTokenClassification.__init__œ  sŽ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå$ VÑ,Ô,ˆŒ
Ý,¨VÑ4Ô4ˆŒ	Ý”H×$Ò$ VÔ%>Ñ?Ô?ˆŒ	Ýœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐr>   Nr?   r¯   rŸ   r@   rS  rº   rA   c                 ó‚  —  | j         d||||dœ|¤Ž}|d         }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }	d}
|�Ft	          ¦   «         } ||	                     d| j        ¦  «        |                     d¦  «        ¦  «        }
t          |
|	|j        |j	        ¬¦  «        S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        rU  r   Nr_   rV  r(  )
rø   rI  r9   r  r   rá   r`  r   rD   rù   )r;   r?   r¯   rŸ   r@   rS  rº   rZ  r<  rX  rW  rq  s               r=   rE   z(ModernBertForTokenClassification.forward¨  så   € ð �$”*ð 
ØØ)Ø%Ø'ð	
ð 
ð
 ð
ð 
ˆð $ AœJÐà ŸIšIÐ&7Ñ8Ô8ÐØ ŸIšIÐ&7Ñ8Ô8ÐØ—’Ð!2Ñ3Ô3ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬OÑ<Ô<¸f¿kºkÈ"¹o¼oÑNÔNˆDå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r>   r\  )rG   rH   rI   r#   r-   r   r   rK   rL   rM   r   r   r¨   r   rE   rN   rO   s   @r=   r  r  –  s  ø€ € € € € ð
Ð/ð 
ð 
ð 
ð 
ð 
ð 
ð Øð .2Ø.2Ø,0Ø-1Ø&*ð$
ð $
àÔ# dÑ*ð$
ð œ tÑ+ð$
ð ”l TÑ)ð	$
ð
 ”| dÑ*ð$
ð ”˜tÑ#ð$
ð Ð+Ô,ð$
ð 
ˆuŒ|Ô	Ð4Ñ	4ð$
ð $
ð $
ñ „^ñ Ôð$
ð $
ð $
ð $
ð $
r>   r  c                   óì   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	e	e
         d
eej                 ez  fd„¦   «         ¦   «         Zˆ xZS )r  r&   c                 ót  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          |¦  «        | _        t          j         	                    |j
        ¦  «        | _        t          j        |j        |j        ¦  «        | _        |                      ¦   «          d S rC   rt  r:   s     €r=   r-   z'ModernBertForQuestionAnswering.__init__Ó  sŒ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå$ VÑ,Ô,ˆŒ
Ý,¨VÑ4Ô4ˆŒ	Ý”H×$Ò$ VÔ%>Ñ?Ô?ˆŒ	Ýœ) FÔ$6¸Ô8IÑJÔJˆŒà�ŠÑÔÐÐÐr>   Nr?   r¯   rŸ   Ústart_positionsÚend_positionsrº   rA   c                 óì  —  | j         |f||dœ|¤Ž}|d         }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }	|	                     dd¬¦  «        \  }
}|
                     d¦  «                             ¦   «         }
|                     d¦  «                             ¦   «         }d }|�|� | j        |
|||fi |¤Ž}t          ||
||j	        |j
        ¬¦  «        S )N)r¯   rŸ   r   r"   r_   r`   )rW  Ústart_logitsÚ
end_logitsrD   rù   )rø   rI  r9   r  Úsplitro  r¹   rY  r   rD   rù   )r;   r?   r¯   rŸ   rx  ry  rº   rZ  r<  rX  r{  r|  rW  s                r=   rE   z&ModernBertForQuestionAnswering.forwardÞ  s.  € ð �$”*Øð
à)Ø%ð
ð 
ð ð	
ð 
ˆð $ AœJÐà ŸIšIÐ&7Ñ8Ô8ÐØ ŸIšIÐ&7Ñ8Ô8ÐØ—’Ð!2Ñ3Ô3ˆà#)§<¢<°°r <Ñ#:Ô#:Ñ ˆ�jØ#×+Ò+¨BÑ/Ô/×:Ò:Ñ<Ô<ˆØ×'Ò'¨Ñ+Ô+×6Ò6Ñ8Ô8ˆ
àˆØÐ&¨=Ð+DØ%�4Ô% l°JÀÐQ^ÐiÐiÐbhÐiÐiˆDå+ØØ%Ø!Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r>   r\  )rG   rH   rI   r#   r-   r   r   rK   rM   r   r   r¨   r   rE   rN   rO   s   @r=   r  r  Ñ  s   ø€ € € € € ð	Ð/ð 	ð 	ð 	ð 	ð 	ð 	ð Øð *.Ø.2Ø,0Ø/3Ø-1ð#
ð #
à”< $Ñ&ð#
ð œ tÑ+ð#
ð ”l TÑ)ð	#
ð
 œ¨Ñ,ð#
ð ”| dÑ*ð#
ð Ð+Ô,ð#
ð 
ˆuŒ|Ô	Ð;Ñ	;ð#
ð #
ð #
ñ „^ñ Ôð#
ð #
ð #
ð #
ð #
r>   r  z«
    The ModernBert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RocStories/SWAG tasks.
    c                   óì   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej        dz  dej	        dz  dej	        dz  dej	        dz  dej	        dz  d	e
e         d
eej	                 ez  fd„¦   «         ¦   «         Zˆ xZS )r  r&   c                 ó`  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        t          j         	                    |j
        ¦  «        | _        t          j        |j        d¦  «        | _        |                      ¦   «          d S )Nr"   )r,   r-   r&   r%  rø   r  rI  rK   r   r7   ra  r9   rU   r0   r  r2  r:   s     €r=   r-   z$ModernBertForMultipleChoice.__init__  sˆ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒå$ VÑ,Ô,ˆŒ
Ý,¨VÑ4Ô4ˆŒ	Ý”H×$Ò$ VÔ%>Ñ?Ô?ˆŒ	Ýœ) FÔ$6¸Ñ:Ô:ˆŒð 	�ŠÑÔÐÐÐr>   Nr?   r¯   rŸ   r@   rS  rº   rA   c                 ó  — |�|j         d         n|j         d         }|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�=|                     d|                     d¦  «        |                     d¦  «        ¦  «        nd} | j        d||||dœ|¤Ž}|d         }	| j        j        dk    rˆt          j        |	j         d         |	j        ¬¦  «        }
|�/| 	                    d¬	¦  «         
                    |	j        ¦  «        }n&t          j        dt          j        |	j        ¬
¦  «        }|	|
|f         }	nV| j        j        dk    rF|                     dd¬¦  «        }|	|                     d¦  «        z                       d¬	¦  «        |z  }	|                      |	¦  «        }|                      |¦  «        }|                      |¦  «        }|                     d|¦  «        }d}|�t%          j        ¦   «         } |||¦  «        }t)          |||j        |j        ¬¦  «        S )a&  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors.
        Nr"   r_   éþÿÿÿrU  r   rc  r:  r`   )r†   r|   rý   Trd  rV  r(  )r–   rá   Úsizerø   r&   ri  rK   rŠ   r|   ÚargmaxrŒ   Útensorrn  rl  rÄ   rI  r9   r  r   r   r   rD   rù   )r;   r?   r¯   rŸ   r@   rS  rº   Únum_choicesrZ  r<  Ú	indices_0Úcls_maskÚnum_non_pad_tokensrp  rX  Úreshaped_logitsrW  rq  s                     r=   rE   z#ModernBertForMultipleChoice.forward  sº  € ð  -6Ð,A�i”o aÔ(Ð(À}ÔGZÐ[\ÔG]ˆà>GÐ>S�I—N’N 2 y§~¢~°bÑ'9Ô'9Ñ:Ô:Ð:ÐY]ˆ	ØM[ÐMg˜×,Ò,¨R°×1DÒ1DÀRÑ1HÔ1HÑIÔIÐIÐmqˆØGSÐG_�|×(Ò(¨¨\×->Ò->¸rÑ-BÔ-BÑCÔCÐCÐeiˆð Ð(ð ×Ò˜r =×#5Ò#5°bÑ#9Ô#9¸=×;MÒ;MÈbÑ;QÔ;QÑRÔRÐRàð 	ð �$”*ð 
ØØ)Ø%Ø'ð	
ð 
ð
 ð
ð 
ˆð $ AœJÐð Œ;Ô)¨UÒ2Ð2ÝœÐ%6Ô%<¸QÔ%?ÐHYÔH`ÐaÑaÔaˆIàÐ)Ø)×0Ò0°RÐ0Ñ8Ô8×;Ò;Ð<MÔ<TÑUÔU��õ !œ<¨µ´ÐDUÔD\Ð]Ñ]Ô]�à 1°)¸XÐ2EÔ FÐÐð Œ[Ô+¨vÒ5Ð5Ø!/×!3Ò!3¸À4Ð!3Ñ!HÔ!HÐØ!2°^×5MÒ5MÈbÑ5QÔ5QÑ!Q× VÒ VÐ[\Ð VÑ ]Ô ]Ð`rÑ rÐàŸ	š	Ð"3Ñ4Ô4ˆØŸ	š	 -Ñ0Ô0ˆØ—’ Ñ/Ô/ˆà Ÿ+š+ b¨+Ñ6Ô6ˆàˆØÐÝÔ*Ñ,Ô,ˆHØ�8˜O¨VÑ4Ô4ˆDå(ØØ"Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r>   r\  )rG   rH   rI   r#   r-   r   r   rK   rL   rM   r   r   r¨   r   rE   rN   rO   s   @r=   r  r    s  ø€ € € € € ð
Ð/ð 
ð 
ð 
ð 
ð 
ð 
ð Øð .2Ø.2Ø,0Ø-1Ø&*ðC
ð C
àÔ# dÑ*ðC
ð œ tÑ+ðC
ð ”l TÑ)ð	C
ð
 ”| dÑ*ðC
ð ”˜tÑ#ðC
ð Ð+Ô,ðC
ð 
ˆuŒ|Ô	Ð8Ñ	8ðC
ð C
ð C
ñ „^ñ ÔðC
ð C
ð C
ð C
ð C
r>   r  )r%  r÷   r  r  r  r  r  )rª   )r"   )Fr  Úcollections.abcr   Útypingr   rK   r   Útorch.nnr   r   r   Ú r
   r   Úactivationsr   Úintegrationsr   r   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úutils.genericr   r   r    Úutils.output_capturingr!   Úconfiguration_modernbertr#   r  r%   rQ   rf   rM   r�   r½   rÁ   rË   rÍ   rí   r÷   r%  r  r  r  r  r  r  Ú__all__r(  r>   r=   ú<module>r›     sÔ  ðð, €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð LÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6ðð ð ð ð ˜2œ9ñ ô ð ð,:ð :ð :ð :ð :�B”Iñ :ô :ð :ð(L<ð L<ð L<ð L<ð L< ¤	ñ L<ô L<ð L<ðl ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð %ð %ð %ð,(ð (ð (ð ÐÐ*Ñ+Ô+ðBð Bð Bñ ,Ô+ðBð4 ÐÐ)Ñ*Ô*ðN)ð N)ð N)ð N)ð N)˜"œ)ñ N)ô N)ñ +Ô*ðN)ðbð ð ð ð Ð7ñ ô ð ð@ ðG^ð G^ð G^ð G^ð G^ ñ G^ô G^ñ „ðG^ðT ðB@ð B@ð B@ð B@ð B@Ð/ñ B@ô B@ñ „ðB@ðJ	>ð 	>ð 	>ð 	>ð 	>˜rœyñ 	>ô 	>ð 	>ð €ððñ ô ð
?
ð ?
ð ?
ð ?
ð ?
Ð5ñ ?
ô ?
ñô ð
?
ðD €ððñ ô ð
S
ð S
ð S
ð S
ð S
Ð*Cñ S
ô S
ñô ð
S
ðl €ððñ ô ð
3
ð 3
ð 3
ð 3
ð 3
Ð'@ñ 3
ô 3
ñô ð
3
ðl ð1
ð 1
ð 1
ð 1
ð 1
Ð%>ñ 1
ô 1
ñ „ð1
ðh €ððñ ô ð
R
ð R
ð R
ð R
ð R
Ð";ñ R
ô R
ñô ð
R
ðjð ð €€€r>   