§
    ‚Štj¢“  ã                   óZ  — d dl Z d dlmZ d dlmZmZ d dlZd dlmZ d dlm	Z	 d dl
mZmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZmZ ddlmZ ddlmZmZm Z m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z,m-Z-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5 ddl6m7Z7m8Z8  e.j9        e:¦  «        Z; e-d¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z< G d„ de	j=        ¦  «        Z> G d„ de	j=        ¦  «        Z? G d „ d!e7¦  «        Z@ ed"¦  «        d?d$„¦   «         ZA eeA¦  «         G d%„ d&e	j=        ¦  «        ¦   «         ZB G d'„ d(e¦  «        ZCe- G d)„ d*e(¦  «        ¦   «         ZDe- G d+„ d,eD¦  «        ¦   «         ZE G d-„ d.e	j=        ¦  «        ZF e-d/¬0¦  «         G d1„ d2eD¦  «        ¦   «         ZG e-d3¬0¦  «         G d4„ d5eD¦  «        ¦   «         ZH e-d6¬0¦  «         G d7„ d8eD¦  «        ¦   «         ZIe- G d9„ d:eD¦  «        ¦   «         ZJ e-d;¬0¦  «         G d<„ d=eD¦  «        ¦   «         ZKg d>¢ZLdS )@é    N)ÚCallable)ÚLiteralÚOptional)Ústrict)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)ÚPreTrainedConfig)Úuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚ(create_bidirectional_sliding_window_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚROPE_INIT_FUNCTIONS)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputsé   )Úeager_attention_forward)ÚGemma3RotaryEmbeddingÚrotate_halfzanswerdotai/ModernBERT-base)Ú
checkpointc                   óÔ  ‡ — e Zd ZU dZdZdgZdddœZdZee	d<   d	Z
ee	d
<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZedz  e	d<   d Zeee         z  dz  e	d!<   d"Zedz  e	d#<   d"Zedz  e	d$<   d Zedz  e	d%<   dZee	d&<   d'Zeez  e	d(<   dZee         dz  e	d)<   dZ e!e"d*         e!f         dz  e	d+<   d,Z#ee	d-<   d'Z$eez  e	d.<   dZ%ee	d/<   d'Z&eez  e	d0<   d1Z'ee	d2<   d3Z(e"d4         e	d5<   d'Z)eez  e	d6<   dZ*ee	d7<   dZ+ee	d8<   dZ,ee	d9<   dZ-ee	d:<   d;Z.ee	d<<   d1Z/ee	d=<   ˆ fd>„Z0d?„ Z1ˆ fd@„Z2e3dA„ ¦   «         Z4e4j5        dB„ ¦   «         Z4ˆ xZ6S )CÚModernBertConfiga+  
    initializer_cutoff_factor (`float`, *optional*, defaults to 2.0):
        The cutoff factor for the truncated_normal_initializer for initializing all weight matrices.
    norm_eps (`float`, *optional*, defaults to 1e-05):
        The epsilon used by the rms normalization layers.
    norm_bias (`bool`, *optional*, defaults to `False`):
        Whether to use bias in the normalization layers.
    local_attention (`int`, *optional*, defaults to 128):
        The window size for local attention.
    mlp_dropout (`float`, *optional*, defaults to 0.0):
        The dropout ratio for the MLP layers.
    decoder_bias (`bool`, *optional*, defaults to `True`):
        Whether to use bias in the decoder layers.
    classifier_pooling (`str`, *optional*, defaults to `"cls"`):
        The pooling method for the classifier. Should be either `"cls"` or `"mean"`. In local attention layers, the
        CLS token doesn't attend to all tokens on long sequences.
    classifier_bias (`bool`, *optional*, defaults to `False`):
        Whether to use bias in the classifier.
    classifier_activation (`str`, *optional*, defaults to `"gelu"`):
        The activation function for the classifier.
    deterministic_flash_attn (`bool`, *optional*, defaults to `False`):
        Whether to use deterministic flash attention. If `False`, inference will be faster but not deterministic.
    sparse_prediction (`bool`, *optional*, defaults to `False`):
        Whether to use sparse prediction for the masked language model instead of returning the full dense logits.
    sparse_pred_ignore_index (`int`, *optional*, defaults to -100):
        The index to ignore for the sparse prediction.

    Examples:

    ```python
    >>> from transformers import ModernBertModel, ModernBertConfig

    >>> # Initializing a ModernBert style configuration
    >>> configuration = ModernBertConfig()

    >>> # Initializing a model from the modernbert-base style configuration
    >>> model = ModernBertModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Ú
modernbertÚpast_key_valuesg     ˆAg     ˆÃ@)ÚglobalÚlocaliÀÄ  Ú
vocab_sizei   Úhidden_sizei€  Úintermediate_sizeé   Únum_hidden_layersé   Únum_attention_headsÚgeluÚhidden_activationi    Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeç       @Úinitializer_cutoff_factorgñhãˆµøä>Únorm_epsFÚ	norm_biasikÄ  NÚpad_token_idijÄ  Úeos_token_idiiÄ  Úbos_token_idÚcls_token_idÚsep_token_idÚattention_biasç        Úattention_dropoutÚlayer_types©Úfull_attentionÚsliding_attentionÚrope_parametersé€   Úlocal_attentionÚembedding_dropoutÚmlp_biasÚmlp_dropoutTÚdecoder_biasÚcls)rQ   ÚmeanÚclassifier_poolingÚclassifier_dropoutÚclassifier_biasÚclassifier_activationÚdeterministic_flash_attnÚsparse_predictioniœÿÿÿÚsparse_pred_ignore_indexÚtie_word_embeddingsc                 óÀ   •‡— |                      dd¦  «        Š| j        €%ˆfd„t          | j        ¦  «        D ¦   «         | _         t	          ¦   «         j        di |¤Ž d S )NÚglobal_attn_every_n_layersr   c                 ó<   •— g | ]}t          |‰z  ¦  «        rd nd‘ŒS ©rI   rH   )Úbool)Ú.0Úir\   s     €úo/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/modernbert/modular_modernbert.pyú
<listcomp>z2ModernBertConfig.__post_init__.<locals>.<listcomp>‹   sC   ø€ ð  ð  ð  àõ (,¨AÐ0JÑ,JÑ'KÔ'KÐaÐ#Ð#ÐQað ð  ð  ó    © )ÚgetrF   Úranger3   ÚsuperÚ__post_init__)ÚselfÚkwargsr\   Ú	__class__s     @€rb   ri   zModernBertConfig.__post_init__‡   s}   øø€ à%+§Z¢ZÐ0LÈaÑ%PÔ%PÐ"ØÔÐ#ð ð  ð  ð  å˜tÔ5Ñ6Ô6ð ñ  ô  ˆDÔð
 	�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'rd   c                 ó²  — |                      dd ¦  «        }ddiddidœ}| j        �| j        n|| _        |�@| j        d                              |¦  «         | j        d                              |¦  «         | j                             d¦  «        €ddi| j        d<   | j        d                              d|                      d| j        d	         ¦  «        ¦  «         | j                             d¦  «        €ddi| j        d<   | j        d                              d|                      d
| j        d         ¦  «        ¦  «         |                      ¦   «          |S )NÚrope_scalingÚ	rope_typeÚdefaultr^   rH   rI   Ú
rope_thetaÚglobal_rope_thetar-   Úlocal_rope_thetar.   )ÚpoprJ   Úupdaterf   Ú
setdefaultÚdefault_thetaÚstandardize_rope_params)rj   rk   rn   Údefault_rope_paramss       rb   Úconvert_rope_params_to_dictz,ModernBertConfig.convert_rope_params_to_dict’   s|  € Ø—z’z .°$Ñ7Ô7ˆð
 #.¨yÐ!9Ø*¨IÐ6ð
ð 
Ðð 8<Ô7KÐ7W˜tÔ3Ð3Ð]pˆÔØÐ#ØÔ Ð!1Ô2×9Ò9¸,ÑGÔGÐGØÔ Ð!4Ô5×<Ò<¸\ÑJÔJÐJð Ô×#Ò#Ð$4Ñ5Ô5Ð=Ø6AÀ9Ð5MˆDÔ Ð!1Ñ2ØÔÐ-Ô.×9Ò9Ø˜&Ÿ*š*Ð%8¸$Ô:LÈXÔ:VÑWÔWñ	
ô 	
ð 	
ð Ô×#Ò#Ð$7Ñ8Ô8Ð@Ø9DÀiÐ8PˆDÔ Ð!4Ñ5ØÔÐ0Ô1×<Ò<Ø˜&Ÿ*š*Ð%7¸Ô9KÈGÔ9TÑUÔUñ	
ô 	
ð 	
ð
 	×$Ò$Ñ&Ô&Ð&Øˆrd   c                 ót   •— t          ¦   «                              ¦   «         }|                     dd ¦  «         |S )NÚreference_compile)rh   Úto_dictrt   )rj   Úoutputrl   s     €rb   r}   zModernBertConfig.to_dict°   s0   ø€ Ý‘”—’Ñ"Ô"ˆØ�
Š
Ð&¨Ñ-Ô-Ð-Øˆrd   c                 ó   — | j         dz  S )zKHalf-window size: `local_attention` is the total window, so we divide by 2.r$   ©rL   ©rj   s    rb   Úsliding_windowzModernBertConfig.sliding_windowµ   s   € ð Ô# qÑ(Ð(rd   c                 ó   — |dz  | _         dS )z<Set sliding_window by updating local_attention to 2 * value.r$   Nr€   ©rj   Úvalues     rb   r‚   zModernBertConfig.sliding_windowº   s   € ð  % q™yˆÔÐÐrd   )7Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferencerw   r/   ÚintÚ__annotations__r0   r1   r3   r5   r7   Ústrr8   r9   Úfloatr;   r<   r=   r_   r>   r?   Úlistr@   rA   rB   rC   rE   rF   rJ   Údictr   rL   rM   rN   rO   rP   rS   rT   rU   rV   rW   rX   rY   rZ   ri   rz   r}   Úpropertyr‚   ÚsetterÚ__classcell__©rl   s   @rb   r*   r*   4   s5  ø€ € € € € € ð(ð (ðT €JØ#4Ð"5ÐØ(°8Ð<Ð<€Mà€J�ÐÐÑØ€K�ÐÐÑØ!Ð�sÐ!Ð!Ñ!ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø#Ð�sÐ#Ð#Ñ#Ø#'Ð˜SÐ'Ð'Ñ'Ø#Ð�uÐ#Ð#Ñ#Ø'*Ð˜uÐ*Ð*Ñ*Ø€HˆeÐÐÑØ€IˆtÐÐÑØ$€L�#˜‘*Ð$Ð$Ñ$Ø+0€L�#˜˜Sœ	‘/ DÑ(Ð0Ð0Ñ0Ø$€L�#˜‘*Ð$Ð$Ñ$Ø$€L�#˜‘*Ð$Ð$Ñ$Ø$€L�#˜‘*Ð$Ð$Ñ$Ø €N�DÐ Ð Ñ Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø$(€K��c”˜TÑ!Ð(Ð(Ñ(ØY]€O�T˜'Ð"GÔHÈ$ÐNÔOÐRVÑVÐ]Ð]Ñ]Ø€O�SÐÐÑØ%(Ð�u˜s‘{Ð(Ð(Ñ(Ø€HˆdÐÐÑØ"€K�˜‘Ð"Ð"Ñ"Ø€L�$ÐÐÑØ16Ð˜ Ô.Ð6Ð6Ñ6Ø&)Ð˜ ™Ð)Ð)Ñ)Ø!€O�TÐ!Ð!Ñ!Ø!'Ð˜3Ð'Ð'Ñ'Ø%*Ð˜dÐ*Ð*Ñ*Ø#Ð�tÐ#Ð#Ñ#Ø$(Ð˜cÐ(Ð(Ñ(Ø $Ð˜Ð$Ð$Ñ$ð	(ð 	(ð 	(ð 	(ð 	(ðð ð ð<ð ð ð ð ð
 ð)ð )ñ „Xð)ð Ôð)ð )ñ Ôð)ð )ð )ð )ð )rd   r*   c                   ój   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej        dz  dej        dz  dej        fd„Z	ˆ xZ
S )
ÚModernBertEmbeddingszV
    Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
    Úconfigc                 ó>  •— t          ¦   «                              ¦   «          || _        t          j        |j        |j        |j        ¬¦  «        | _        t          j	        |j        |j
        |j        ¬¦  «        | _        t          j        |j        ¦  «        | _        d S )N)Úpadding_idx©ÚepsÚbias)rh   Ú__init__r˜   r   Ú	Embeddingr/   r0   r>   Útok_embeddingsÚ	LayerNormr<   r=   ÚnormÚDropoutrM   Údrop©rj   r˜   rl   s     €rb   rž   zModernBertEmbeddings.__init__Å   s{   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ œl¨6Ô+<¸fÔ>PÐ^dÔ^qÐrÑrÔrˆÔÝ”L Ô!3¸¼ÈvÔO_Ð`Ñ`Ô`ˆŒ	Ý”J˜vÔ7Ñ8Ô8ˆŒ	ˆ	ˆ	rd   NÚ	input_idsÚinputs_embedsÚreturnc                 óÒ   — |�)|                       |                      |¦  «        ¦  «        }n;|                       |                      |                      |¦  «        ¦  «        ¦  «        }|S ©N)r¤   r¢   r    )rj   r¦   r§   Úhidden_statess       rb   ÚforwardzModernBertEmbeddings.forwardÌ   sZ   € ð Ð$Ø ŸIšI d§i¢i°Ñ&>Ô&>Ñ?Ô?ˆMˆMà ŸIšI d§i¢i°×0CÒ0CÀIÑ0NÔ0NÑ&OÔ&OÑPÔPˆMØÐrd   ©NN)r†   r‡   rˆ   r‰   r*   rž   ÚtorchÚ
LongTensorÚTensorr¬   r”   r•   s   @rb   r—   r—   À   s™   ø€ € € € € ðð ð9Ð/ð 9ð 9ð 9ð 9ð 9ð 9ð _cðð ØÔ)¨DÑ0ðØHMÌÐW[ÑH[ðà	Œðð ð ð ð ð ð ð rd   r—   c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚModernBertMLPa6  Applies the GLU at the end of each ModernBERT layer.

    Compared to the default BERT architecture, this block replaces :class:`~transformers.model.bert.modeling_bert.BertIntermediate`
    and :class:`~transformers.model.bert.modeling_bert.SelfOutput` with a single module that has similar functionality.
    r˜   c                 óŒ  •— t          ¦   «                              ¦   «          || _        t          j        |j        t          |j        ¦  «        dz  |j        ¬¦  «        | _	        t          |j                 | _        t          j        |j        ¦  «        | _        t          j        |j        |j        |j        ¬¦  «        | _        d S )Nr$   ©r�   )rh   rž   r˜   r   ÚLinearr0   rŒ   r1   rN   ÚWir   r7   Úactr£   rO   r¤   ÚWor¥   s     €rb   rž   zModernBertMLP.__init__Ý   s—   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”)˜FÔ.µ°FÔ4LÑ0MÔ0MÐPQÑ0QÐX^ÔXgÐhÑhÔhˆŒÝ˜&Ô2Ô3ˆŒÝ”J˜vÔ1Ñ2Ô2ˆŒ	Ý”)˜FÔ4°fÔ6HÈvÌÐ_Ñ_Ô_ˆŒˆˆrd   r«   r¨   c                 óØ   — |                       |¦  «                             dd¬¦  «        \  }}|                      |                      |                      |¦  «        |z  ¦  «        ¦  «        S )Nr$   éÿÿÿÿ©Údim)r¶   Úchunkr¸   r¤   r·   )rj   r«   ÚinputÚgates       rb   r¬   zModernBertMLP.forwardå   sW   € Ø—g’g˜mÑ,Ô,×2Ò2°1¸"Ð2Ñ=Ô=‰ˆˆtØ�wŠw�t—y’y §¢¨%¡¤°4Ñ!7Ñ8Ô8Ñ9Ô9Ð9rd   )
r†   r‡   rˆ   r‰   r*   rž   r®   r°   r¬   r”   r•   s   @rb   r²   r²   Ö   s|   ø€ € € € € ðð ð`Ð/ð `ð `ð `ð `ð `ð `ð: U¤\ð :°e´lð :ð :ð :ð :ð :ð :ð :ð :rd   r²   c                   óŽ   ‡ — e Zd Zddefˆ fd„Ze	 	 	 	 ddedz  ded         dedz  dedz  de	d	e
f         f
ˆ fd
„¦   «         Zˆ xZS )ÚModernBertRotaryEmbeddingNr˜   c                 óL   •— t          ¦   «                              ||¦  «         d S rª   )rh   rž   )rj   r˜   Údevicerl   s      €rb   rž   z"ModernBertRotaryEmbedding.__init__ë   s#   ø€ Ý‰Œ×Ò˜ Ñ(Ô(Ð(Ð(Ð(rd   rÃ   ztorch.deviceÚseq_lenÚ
layer_typer¨   ztorch.Tensorc                 óL   •— t          ¦   «                              | |||¦  «        S rª   )rh   Úcompute_default_rope_parameters)r˜   rÃ   rÄ   rÅ   rl   s       €rb   rÇ   z9ModernBertRotaryEmbedding.compute_default_rope_parametersî   s$   ø€ õ ‰wŒw×6Ò6°v¸vÀwÐPZÑ[Ô[Ð[rd   rª   ©NNNN)r†   r‡   rˆ   r*   rž   Ústaticmethodr   rŒ   rŽ   Útupler�   rÇ   r”   r•   s   @rb   rÁ   rÁ   ê   sß   ø€ € € € € ð)ð )Ð/ð )ð )ð )ð )ð )ð )ð à*.Ø+/Ø"Ø!%ð	\ð \Ø  4Ñ'ð\à˜Ô(ð\ð �t‘ð\ð ˜$‘Jð	\ð
 
ˆ~˜uÐ$Ô	%ð\ð \ð \ð \ð \ñ „\ð\ð \ð \ð \ð \rd   rÁ   Úrotary_pos_embé   c                 ó¨  — | j         }|                     |¦  «        }|                     |¦  «        }|                      ¦   «         |z  t          |                      ¦   «         ¦  «        |z  z   }|                     ¦   «         |z  t          |                     ¦   «         ¦  «        |z  z   }|                     |¦  «        |                     |¦  «        fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )ÚdtypeÚ	unsqueezer�   r'   Úto)ÚqÚkÚcosÚsinÚunsqueeze_dimÚoriginal_dtypeÚq_embedÚk_embeds           rb   Úapply_rotary_pos_embrÙ   ø   s¦   € ð& ”W€NØ
�-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�wŠw‰yŒy˜3‰¥;¨q¯wªw©y¬yÑ#9Ô#9¸CÑ#?Ñ@€GØ�wŠw‰yŒy˜3‰¥;¨q¯wªw©y¬yÑ#9Ô#9¸CÑ#?Ñ@€GØ�:Š:�nÑ%Ô% w§z¢z°.Ñ'AÔ'AÐAÐArd   c                   óÐ   ‡ — e Zd ZdZddededz  fˆ fd„Z	 	 ddej        de	ej        ej        f         dz  dej        dz  d	e
e         d
e	ej        ej        dz  f         f
d„Zˆ xZS )ÚModernBertAttentiona‚  Performs multi-headed self attention on a batch of unpadded sequences.

    If Flash Attention 2 is installed, this module uses Flash Attention to improve throughput.
    If Flash Attention 2 is not installed, the implementation will use PyTorch's SDPA kernel,
    which requires padding and unpadding inputs, adding some overhead.

    See `forward` method for additional details.
    Nr˜   Ú	layer_idxc                 óº  •— t          ¦   «                              ¦   «          || _        || _        |j        |j        z  dk    r t          d|j        › d|j        › d�¦  «        ‚|j        | _        |j        | _        |j        |j        z  | _	        t          j        |j        d| j	        z  |j        z  |j        ¬¦  «        | _        |j        |         dk    r|j        dz   | _        nd | _        d	| _        t          j        |j        |j        |j        ¬¦  «        | _        |j        d
k    rt          j        |j        ¦  «        nt          j        ¦   «         | _        d S )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads (ú)r   r´   rI   rÌ   FrD   )rh   rž   r˜   rÜ   r0   r5   Ú
ValueErrorrE   rW   Úhead_dimr   rµ   rC   ÚWqkvrF   r‚   Ú	is_causalr¸   r£   ÚIdentityÚout_drop©rj   r˜   rÜ   rl   s      €rb   rž   zModernBertAttention.__init__  sq  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒàÔ Ô :Ñ:¸aÒ?Ð?Ýð L FÔ$6ð  Lð  LÐntô  oIð  Lð  Lð  Lñô ð ð "(Ô!9ˆÔØ(.Ô(GˆÔ%ØÔ*¨fÔ.HÑHˆŒÝ”IØÔ  D¤MÑ 1°FÔ4NÑ NÐU[ÔUjð
ñ 
ô 
ˆŒ	ð Ô˜iÔ(Ð,?Ò?Ð?ð #)Ô"7¸!Ñ";ˆDÔÐà"&ˆDÔàˆŒå”)˜FÔ.°Ô0BÈÔI^Ð_Ñ_Ô_ˆŒØ@FÔ@XÐ[^Ò@^Ð@^�œ
 6Ô#;Ñ<Ô<Ð<ÕdfÔdoÑdqÔdqˆŒˆˆrd   r«   Úposition_embeddingsÚattention_maskrk   r¨   c                 ó¶  — |j         d d…         }|                      |¦  «        } |j        g |¢d‘d‘| j        ‘R Ž }|                     d¬¦  «        \  }}}	|                     dd¦  «        }|                     dd¦  «        }|	                     dd¦  «        }	|\  }
}t          |||
|d¬¦  «        \  }}t          j        | j	        j
        t          ¦  «        } || |||	|f| j        r| j        nd| j        d	z  | j        | j        d
œ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |                      |¦  «        ¦  «        }||fS )Nrº   r   éýÿÿÿr»   rÌ   r$   )rÕ   rD   ç      à¿)ÚdropoutÚscalingr‚   Údeterministic)Úshaperá   Úviewrà   ÚunbindÚ	transposerÙ   r   Úget_interfacer˜   Ú_attn_implementationr%   ÚtrainingrE   r‚   rW   ÚreshapeÚ
contiguousrä   r¸   )rj   r«   ræ   rç   rk   Úinput_shapeÚqkvÚquery_statesÚ
key_statesÚvalue_statesrÓ   rÔ   Úattention_interfaceÚattn_outputÚattn_weightss                  rb   r¬   zModernBertAttention.forward;  s§  € ð $Ô)¨#¨2¨#Ô.ˆà�iŠi˜Ñ&Ô&ˆØˆcŒhÐ:˜Ð: QÐ:¨Ð:¨D¬MÐ:Ð:Ð:ˆØ14·²À°Ñ1CÔ1CÑ.ˆ�j ,à#×-Ò-¨a°Ñ3Ô3ˆØ×)Ò)¨!¨QÑ/Ô/ˆ
Ø#×-Ò-¨a°Ñ3Ô3ˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÐjkÐ#lÑ#lÔ#lÑ ˆ�jå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð%
ð /3¬mÐD�DÔ*Ð*ÀØ”M 4Ñ'ØÔ.ØÔ7ð%
ð %
ð ð%
ð %
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—m’m D§G¢G¨KÑ$8Ô$8Ñ9Ô9ˆØ˜LÐ(Ð(rd   rª   r­   )r†   r‡   rˆ   r‰   r*   rŒ   rž   r®   r°   rÊ   r   r   r¬   r”   r•   s   @rb   rÛ   rÛ     sñ   ø€ € € € € ðð ðrð rÐ/ð r¸CÀ$¹Jð rð rð rð rð rð rð@ IMØ.2ð	')ð ')à”|ð')ð # 5¤<°´Ð#=Ô>ÀÑEð')ð œ tÑ+ð	')ð
 Ð+Ô,ð')ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð')ð ')ð ')ð ')ð ')ð ')ð ')ð ')rd   rÛ   c                   ó’   ‡ — e Zd Zddededz  fˆ fd„Z	 	 ddej        dej        dz  dej        dz  dee	         d	ej        f
d
„Z
ˆ xZS )ÚModernBertEncoderLayerNr˜   rÜ   c                 óÆ  •— t          ¦   «                              ¦   «          || _        || _        |dk    rt	          j        ¦   «         | _        n+t	          j        |j        |j	        |j
        ¬¦  «        | _        t          ||¬¦  «        | _        t	          j        |j        |j	        |j
        ¬¦  «        | _        t          |¦  «        | _        |j        |         | _        d S )Nr   r›   )r˜   rÜ   )rh   rž   r˜   rÜ   r   rã   Ú	attn_normr¡   r0   r<   r=   rÛ   ÚattnÚmlp_normr²   ÚmlprF   Úattention_typerå   s      €rb   rž   zModernBertEncoderLayer.__init__f  sº   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ˜Š>ˆ>Ýœ[™]œ]ˆDŒNˆNåœ\¨&Ô*<À&Ä/ÐX^ÔXhÐiÑiÔiˆDŒNÝ'¨vÀÐKÑKÔKˆŒ	Ýœ VÔ%7¸V¼_ÐSYÔScÐdÑdÔdˆŒÝ  Ñ(Ô(ˆŒØ$Ô0°Ô;ˆÔÐÐrd   r«   rç   ræ   rk   r¨   c                 ó´   —  | j         |                      |¦  «        f||dœ|¤Ž\  }}||z   }||                      |                      |¦  «        ¦  «        z   }|S )N)ræ   rç   )r  r  r  r  )rj   r«   rç   ræ   rk   rý   Ú_s          rb   r¬   zModernBertEncoderLayer.forwards  sx   € ð #˜œØ�NŠN˜=Ñ)Ô)ð
à 3Ø)ð
ð 
ð ð	
ð 
‰ˆ�Qð &¨Ñ3ˆØ%¨¯ª°·²¸}Ñ1MÔ1MÑ(NÔ(NÑNˆØÐrd   rª   r­   )r†   r‡   rˆ   r*   rŒ   rž   r®   r°   r   r   r¬   r”   r•   s   @rb   r   r   e  s¾   ø€ € € € € ð<ð <Ð/ð <¸CÀ$¹Jð <ð <ð <ð <ð <ð <ð  /3Ø37ð	ð à”|ðð œ tÑ+ðð #œ\¨DÑ0ð	ð
 Ð+Ô,ðð 
Œðð ð ð ð ð ð ð rd   r   c                   óŽ   ‡ — e Zd ZU eed<   dZdZddgZdZdZ	dZ
dZeedœZ ej        ¦   «         dej        fˆ fd„¦   «         Zˆ xZS )	ÚModernBertPreTrainedModelr˜   ÚmodelTr—   r   )r«   Ú
attentionsÚmodulec                 óT  •‡— t          ¦   «                              |¦  «         | j        j        Š‰€dŠdt          j        dt          fˆfd„}| j        j        | j        j        t          j	        d| j        j
        z  ¦  «        z  | j        j        | j        j        dz  dœ}t          |t          ¦  «        r ||j        |d         ¦  «         d S t          |t          ¦  «        r0 ||j        |d	         ¦  «          ||j        |d
         ¦  «         d S t          |t$          ¦  «        r0 ||j        |d	         ¦  «          ||j        |d
         ¦  «         d S t          |t(          ¦  «        r ||j        |d
         ¦  «         d S t          |t,          ¦  «        r ||j        |d
         ¦  «         d S t          |t0          t2          t4          t6          f¦  «        r ||j        |d         ¦  «         d S t          |t:          ¦  «        r›|j        D ]•}|j        }|j         |         dk    rtB          |j         |                  } ||j        |¬¦  «        \  }}tE          j#        tI          ||› d�¦  «        |¦  «         tE          j#        tI          ||› d�¦  «        |¦  «         Œ”d S d S )Nr   r  Ústdc                 óÎ   •— t          j        | j        d|‰ |z  ‰|z  ¬¦  «         t          | t          j        ¦  «        r"| j        �t          j        | j        ¦  «         d S d S d S )NrD   )rR   r  ÚaÚb)ÚinitÚtrunc_normal_ÚweightÚ
isinstancer   rµ   r�   Úzeros_)r  r  Úcutoff_factors     €rb   Úinit_weightz<ModernBertPreTrainedModel._init_weights.<locals>.init_weightœ  s€   ø€ ÝÔØ”ØØØ �. 3Ñ&Ø #Ñ%ðñ ô ð õ ˜&¥"¤)Ñ,Ô,ð -Ø”;Ð*Ý”K ¤Ñ,Ô,Ð,Ð,Ð,ð-ð -Ø*Ð*rd   r:   rê   )ÚinÚoutÚ	embeddingÚ	final_outr  r  r  r  rp   )rÅ   Ú	_inv_freqÚ_original_inv_freq)%rh   Ú_init_weightsr˜   r;   r   ÚModuler�   r9   ÚmathÚsqrtr3   r0   r  r—   r    r²   r¶   r¸   rÛ   rá   ÚModernBertPredictionHeadÚdenseÚModernBertForMaskedLMÚdecoderÚ#ModernBertForSequenceClassificationÚModernBertForMultipleChoiceÚ ModernBertForTokenClassificationÚModernBertForQuestionAnsweringÚ
classifierrÁ   rF   rÇ   ro   r   r  Úcopy_Úgetattr)
rj   r  r  ÚstdsrÅ   Úrope_init_fnÚcurr_inv_freqr  r  rl   s
           @€rb   r   z'ModernBertPreTrainedModel._init_weights•  sö  øø€ å‰Œ×Ò˜fÑ%Ô%Ð%ØœÔ=ˆØÐ ØˆMð	-¥¤	ð 	-µð 	-ð 	-ð 	-ð 	-ð 	-ð 	-ð ”+Ô/Ø”;Ô0µ4´9¸SÀ4Ä;ÔC`Ñ=`Ñ3aÔ3aÑaØœÔ6ØœÔ0°$Ñ6ð	
ð 
ˆõ �fÕ2Ñ3Ô3ð 	^ØˆK˜Ô-¨t°KÔ/@ÑAÔAÐAÐAÐAÝ˜¥Ñ.Ô.ð 	^ØˆK˜œ	 4¨¤:Ñ.Ô.Ð.ØˆK˜œ	 4¨¤;Ñ/Ô/Ð/Ð/Ð/Ý˜Õ 3Ñ4Ô4ð 	^ØˆK˜œ T¨$¤ZÑ0Ô0Ð0ØˆK˜œ	 4¨¤;Ñ/Ô/Ð/Ð/Ð/Ý˜Õ 8Ñ9Ô9ð 	^ØˆK˜œ d¨5¤kÑ2Ô2Ð2Ð2Ð2Ý˜Õ 5Ñ6Ô6ð 	^ØˆK˜œ¨¨U¬Ñ4Ô4Ð4Ð4Ð4ÝØå3Ý+Ý0Ý.ð	ñ
ô 
ð 	^ð ˆK˜Ô)¨4°Ô+<Ñ=Ô=Ð=Ð=Ð=Ý˜Õ 9Ñ:Ô:ð 	^Ø$Ô0ð ^ð ^�
Ø%ÔE�ØÔ# JÔ/°9Ò<Ð<Ý#6°vÔ7GÈ
Ô7SÔ#T�LØ#/ <°´È*Ð#UÑ#UÔ#UÑ �˜qÝ”
�7 6¨jÐ+CÐ+CÐ+CÑDÔDÀmÑTÔTÐTÝ”
�7 6¨jÐ+LÐ+LÐ+LÑMÔMÈ}Ñ]Ô]Ð]Ð]ð	^ð 	^ð^ð ^rd   )r†   r‡   rˆ   r*   r�   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr   rÛ   Ú_can_record_outputsr®   Úno_gradr   r!  r   r”   r•   s   @rb   r
  r
  …  s·   ø€ € € € € € àÐÐÑØÐØ&*Ð#Ø/Ð1IÐJÐØÐØ€NØÐØ"&Ðð 0Ø)ðð Ðð
 €U„]�_„_ð7^ B¤Ið 7^ð 7^ð 7^ð 7^ð 7^ñ „_ð7^ð 7^ð 7^ð 7^ð 7^rd   r
  c                   óÖ   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zeee		 	 	 	 dde
j        dz  de
j        dz  de
j        dz  d	e
j        dz  d
ee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚModernBertModelr˜   c                 ó¦  •‡— t          ¦   «                              ‰¦  «         ‰| _        t          ‰¦  «        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _	        t          j
        ‰j        ‰j        ‰j        ¬¦  «        | _        t          ‰¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS re   )r   )r`   rÜ   r˜   s     €rb   rc   z,ModernBertModel.__init__.<locals>.<listcomp>×  s$   ø€ ÐhÐhÐh¸9Õ# F¨IÑ6Ô6ÐhÐhÐhrd   r›   )r˜   F)rh   rž   r˜   r—   Ú
embeddingsr   Ú
ModuleListrg   r3   Úlayersr¡   r0   r<   r=   Ú
final_normrÁ   Ú
rotary_embÚgradient_checkpointingÚ	post_initr¥   s    `€rb   rž   zModernBertModel.__init__Ò  sº   øø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ.¨vÑ6Ô6ˆŒÝ”mØhÐhÐhÐhÍÈfÔNfÑHgÔHgÐhÑhÔhñ
ô 
ˆŒõ œ, vÔ'9¸v¼ÐU[ÔUeÐfÑfÔfˆŒÝ3¸6ÐBÑBÔBˆŒØ&+ˆÔ#Ø�ŠÑÔÐÐÐrd   c                 ó   — | j         j        S rª   ©r?  r    r�   s    rb   Úget_input_embeddingsz$ModernBertModel.get_input_embeddingsÞ  s   € ØŒÔ-Ð-rd   c                 ó   — || j         _        d S rª   rG  r„   s     rb   Úset_input_embeddingsz$ModernBertModel.set_input_embeddingsá  s   € Ø).ˆŒÔ&Ð&Ð&rd   Nr¦   rç   Úposition_idsr§   rk   r¨   c                 ó–  — |d u |d uz  rt          d¦  «        ‚|�|j        d         n|j        d         }|�|j        n|j        }|€)t          j        ||¬¦  «                             d¦  «        }|                      ||¬¦  «        }t          |x}	t          ¦  «        s$| j	        ||dœ}
t          d
i |
¤Žt          d
i |
¤Ždœ}	i }t          | j	        j        ¦  «        D ]}|                      |||¦  «        ||<   Œ| j        D ]$} ||f|	|j                 ||j                 dœ|¤Ž}Œ%|                      |¦  «        }t%          |¬	¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embedsrÌ   ©rÃ   r   )r¦   r§   )r˜   r§   rç   rG   )rç   ræ   )Úlast_hidden_statere   )rß   rî   rÃ   r®   ÚarangerÏ   r?  r  r‘   r˜   r   r   ÚsetrF   rC  rA  r  rB  r   )rj   r¦   rç   rK  r§   rk   rÄ   rÃ   r«   Úattention_mask_mappingÚmask_kwargsræ   rÅ   Úencoder_layers                 rb   r¬   zModernBertModel.forwardä  s·  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZà,9Ð,E�-Ô% aÔ(Ð(È9Ì?Ð[\ÔK]ˆØ%.Ð%:�Ô!Ð!ÀÔ@TˆàÐÝ œ<¨¸Ð?Ñ?Ô?×IÒIÈ!ÑLÔLˆLàŸš°)È=˜ÑYÔYˆå°NÐBÐ0ÅDÑIÔIð 		àœ+Ø!.Ø"0ðð ˆKõ #<Ð"JÐ"J¸kÐ"JÐ"JÝ%MÐ%\Ð%\ÐP[Ð%\Ð%\ð&ð &Ð"ð
 !ÐÝ˜dœkÔ5Ñ6Ô6ð 	gð 	gˆJØ.2¯oªo¸mÈ\Ð[eÑ.fÔ.fÐ 
Ñ+Ð+à!œ[ð 	ð 	ˆMØ)˜MØðà5°mÔ6RÔSØ$7¸Ô8TÔ$Uðð ð ð	ð ˆMˆMð Ÿš¨Ñ6Ô6ˆå°Ð?Ñ?Ô?Ð?rd   rÈ   )r†   r‡   rˆ   r*   rž   rH  rJ  r"   r#   r   r®   r¯   r°   r   r   r   r¬   r”   r•   s   @rb   r<  r<  Ð  s  ø€ € € € € ð
Ð/ð 
ð 
ð 
ð 
ð 
ð 
ð.ð .ð .ð/ð /ð /ð  ØØð .2Ø.2Ø04Ø-1ð,@ð ,@àÔ# dÑ*ð,@ð œ tÑ+ð,@ð Ô&¨Ñ-ð	,@ð
 ”| dÑ*ð,@ð Ð+Ô,ð,@ð 
ð,@ð ,@ð ,@ñ „^ñ „_ñ  Ôð,@ð ,@ð ,@ð ,@ð ,@rd   r<  c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )r$  r˜   c                 ó.  •— t          ¦   «                              ¦   «          || _        t          j        |j        |j        |j        ¦  «        | _        t          |j	                 | _
        t          j        |j        |j        |j        ¬¦  «        | _        d S )Nr›   )rh   rž   r˜   r   rµ   r0   rU   r%  r   rV   r·   r¡   r<   r=   r¢   r¥   s     €rb   rž   z!ModernBertPredictionHead.__init__  sq   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”Y˜vÔ1°6Ô3EÀvÔG]Ñ^Ô^ˆŒ
Ý˜&Ô6Ô7ˆŒÝ”L Ô!3¸¼ÈvÔO_Ð`Ñ`Ô`ˆŒ	ˆ	ˆ	rd   r«   r¨   c                 óx   — |                       |                      |                      |¦  «        ¦  «        ¦  «        S rª   )r¢   r·   r%  )rj   r«   s     rb   r¬   z ModernBertPredictionHead.forward  s,   € Ø�yŠy˜Ÿš $§*¢*¨]Ñ";Ô";Ñ<Ô<Ñ=Ô=Ð=rd   )	r†   r‡   rˆ   r*   rž   r®   r°   r¬   r”   r•   s   @rb   r$  r$    sr   ø€ € € € € ðaÐ/ð að að að að að að> U¤\ð >°e´lð >ð >ð >ð >ð >ð >ð >ð >rd   r$  zd
    The ModernBert Model with a decoder head on top that is used for masked language modeling.
    )Úcustom_introc                   ó  ‡ — e Zd ZddiZdefˆ fd„Zd„ Zdej        fd„Z	e
e	 	 	 	 	 dd	ej        dz  d
ej        dz  dej        dz  dej        dz  dej        dz  dee         deej                 ez  fd„¦   «         ¦   «         Zˆ xZS )r&  zdecoder.weightz&model.embeddings.tok_embeddings.weightr˜   c                 ój  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        t          j        |j	        |j
        |j        ¬¦  «        | _        | j        j        | _        | j        j        | _        |                      ¦   «          d S )Nr´   )rh   rž   r˜   r<  r  r$  Úheadr   rµ   r0   r/   rP   r'  rX   rY   rE  r¥   s     €rb   rž   zModernBertForMaskedLM.__init__*  s“   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ$ VÑ,Ô,ˆŒ
Ý,¨VÑ4Ô4ˆŒ	Ý”y Ô!3°VÔ5FÈVÔM`ÐaÑaÔaˆŒà!%¤Ô!>ˆÔØ(,¬Ô(LˆÔ%ð 	�ŠÑÔÐÐÐrd   c                 ó   — | j         S rª   ©r'  r�   s    rb   Úget_output_embeddingsz+ModernBertForMaskedLM.get_output_embeddings7  s
   € ØŒ|Ðrd   Únew_embeddingsc                 ó   — || _         d S rª   r\  )rj   r^  s     rb   Úset_output_embeddingsz+ModernBertForMaskedLM.set_output_embeddings:  s   € Ø%ˆŒˆˆrd   Nr¦   rç   rK  r§   Úlabelsrk   r¨   c                 ó²  —  | j         d||||dœ|¤Ž}|d         }| j        rS|�Q|                     d¦  «        }|                     |j        d         d¦  «        }|| j        k    }	||	         }||	         }|                      |                      |¦  «        ¦  «        }
d }|� | j        |
|fd| j        j	        i|¤Ž}t          ||
|j        |j        ¬¦  «        S )N©r¦   rç   rK  r§   r   rº   r/   ©ÚlossÚlogitsr«   r  re   )r  rX   rï   rî   rY   r'  rZ  Úloss_functionr˜   r/   r   r«   r  )rj   r¦   rç   rK  r§   ra  rk   ÚoutputsrN  Úmask_tokensrf  re  s               rb   r¬   zModernBertForMaskedLM.forward=  s  € ð �$”*ð 
ØØ)Ø%Ø'ð	
ð 
ð
 ð
ð 
ˆð $ AœJÐàÔ!ð 	) fÐ&8à—[’[ ‘_”_ˆFØ 1× 6Ò 6°v´|ÀA´ÈÑ KÔ KÐð ! DÔ$AÒAˆKØ 1°+Ô >ÐØ˜KÔ(ˆFà—’˜dŸišiÐ(9Ñ:Ô:Ñ;Ô;ˆàˆØÐØ%�4Ô% f¨fÐbÐbÀÄÔAWÐbÐ[aÐbÐbˆDåØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rd   ©NNNNN)r†   r‡   rˆ   Ú_tied_weights_keysr*   rž   r]  r   rµ   r`  r!   r   r®   r¯   r°   r   r   rÊ   r   r¬   r”   r•   s   @rb   r&  r&  "  s:  ø€ € € € € ð +Ð,TÐUÐðÐ/ð ð ð ð ð ð ðð ð ð&°B´Ið &ð &ð &ð &ð Øð .2Ø.2Ø,0Ø-1Ø&*ð'
ð '
àÔ# dÑ*ð'
ð œ tÑ+ð'
ð ”l TÑ)ð	'
ð
 ”| dÑ*ð'
ð ”˜tÑ#ð'
ð Ð+Ô,ð'
ð 
ˆuŒ|Ô	˜~Ñ	-ð'
ð '
ð '
ñ „^ñ Ôð'
ð '
ð '
ð '
ð '
rd   r&  z`
    The ModernBert Model with a sequence classification head on top that performs pooling.
    c                   óì   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej        dz  dej	        dz  dej	        dz  dej	        dz  dej	        dz  d	e
e         d
eej	                 ez  fd„¦   «         ¦   «         Zˆ xZS )r(  r˜   c                 ó‚  •— t          ¦   «                              |¦  «         |j        | _        || _        t	          |¦  «        | _        t          |¦  «        | _        t          j	         
                    |j        ¦  «        | _        t          j        |j        |j        ¦  «        | _        |                      ¦   «          d S rª   )rh   rž   Ú
num_labelsr˜   r<  r  r$  rZ  r®   r   r£   rT   r¤   rµ   r0   r,  rE  r¥   s     €rb   rž   z,ModernBertForSequenceClassification.__init__o  s•   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒØˆŒå$ VÑ,Ô,ˆŒ
Ý,¨VÑ4Ô4ˆŒ	Ý”H×$Ò$ VÔ%>Ñ?Ô?ˆŒ	Ýœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐrd   Nr¦   rç   rK  r§   ra  rk   r¨   c                 óø  —  | j         d||||dœ|¤Ž}|d         }| j        j        dk    r|dd…df         }n‰| j        j        dk    ry|€3t          j        |j        dd…         |j        t          j        ¬¦  «        }||                     d¦  «        z   	                    d	¬
¦  «        | 	                    d	d¬¦  «        z  }|  
                    |¦  «        }	|                      |	¦  «        }	|                      |	¦  «        }
d}|��Z| j        j        €f| j        d	k    rd| j        _        nN| j        d	k    r7|j        t          j        k    s|j        t          j        k    rd| j        _        nd| j        _        | j        j        dk    rWt%          ¦   «         }| j        d	k    r1 ||
                     ¦   «         |                     ¦   «         ¦  «        }nŽ ||
|¦  «        }n�| j        j        dk    rGt)          ¦   «         } ||
                     d| j        ¦  «        |                     d¦  «        ¦  «        }n*| j        j        dk    rt-          ¦   «         } ||
|¦  «        }t/          ||
|j        |j        ¬¦  «        S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        rc  r   rQ   NrR   r$   )rÃ   rÎ   rº   rÌ   r»   T©r¼   ÚkeepdimÚ
regressionÚsingle_label_classificationÚmulti_label_classificationrd  re   )r  r˜   rS   r®   Úonesrî   rÃ   r_   rÏ   ÚsumrZ  r¤   r,  Úproblem_typern  rÎ   ÚlongrŒ   r
   Úsqueezer	   rï   r   r   r«   r  )rj   r¦   rç   rK  r§   ra  rk   rh  rN  Úpooled_outputrf  re  Úloss_fcts                rb   r¬   z+ModernBertForSequenceClassification.forward|  sª  € ð" �$”*ð 
ØØ)Ø%Ø'ð	
ð 
ð
 ð
ð 
ˆð $ AœJÐàŒ;Ô)¨UÒ2Ð2Ø 1°!°!°!°Q°$Ô 7ÐÐØŒ[Ô+¨vÒ5Ð5ØÐ%Ý!&¤Ø%Ô+¨B¨Q¨BÔ/Ð8IÔ8PÕX]ÔXbð"ñ "ô "�ð "3°^×5MÒ5MÈbÑ5QÔ5QÑ!Q× VÒ VÐ[\Ð VÑ ]Ô ]Ð`n×`rÒ`rØ˜tð asñ aô añ !Ðð Ÿ	š	Ð"3Ñ4Ô4ˆØŸ	š	 -Ñ0Ô0ˆØ—’ Ñ/Ô/ˆàˆØÑØŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ñ @Ô @À&Ç+Â+ÈbÁ/Ä/ÑRÔR��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�å'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rd   rj  )r†   r‡   rˆ   r*   rž   r!   r   r®   r¯   r°   r   r   rÊ   r   r¬   r”   r•   s   @rb   r(  r(  i  s  ø€ € € € € ðÐ/ð ð ð ð ð ð ð Øð .2Ø.2Ø,0Ø-1Ø&*ðC
ð C
àÔ# dÑ*ðC
ð œ tÑ+ðC
ð ”l TÑ)ð	C
ð
 ”| dÑ*ðC
ð ”˜tÑ#ðC
ð Ð+Ô,ðC
ð 
ˆuŒ|Ô	Ð7Ñ	7ðC
ð C
ð C
ñ „^ñ ÔðC
ð C
ð C
ð C
ð C
rd   r(  zv
    The ModernBert Model with a token classification head on top, e.g. for Named Entity Recognition (NER) tasks.
    c                   óì   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej        dz  dej	        dz  dej	        dz  dej	        dz  dej	        dz  d	e
e         d
eej	                 ez  fd„¦   «         ¦   «         Zˆ xZS )r*  r˜   c                 ót  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          |¦  «        | _        t          j         	                    |j
        ¦  «        | _        t          j        |j        |j        ¦  «        | _        |                      ¦   «          d S rª   ©rh   rž   rn  r<  r  r$  rZ  r®   r   r£   rT   r¤   rµ   r0   r,  rE  r¥   s     €rb   rž   z)ModernBertForTokenClassification.__init__Ê  sŽ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå$ VÑ,Ô,ˆŒ
Ý,¨VÑ4Ô4ˆŒ	Ý”H×$Ò$ VÔ%>Ñ?Ô?ˆŒ	Ýœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐrd   Nr¦   rç   rK  r§   ra  rk   r¨   c                 ó‚  —  | j         d||||dœ|¤Ž}|d         }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }	d}
|�Ft	          ¦   «         } ||	                     d| j        ¦  «        |                     d¦  «        ¦  «        }
t          |
|	|j        |j	        ¬¦  «        S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        rc  r   Nrº   rd  re   )
r  rZ  r¤   r,  r	   rï   rn  r   r«   r  )rj   r¦   rç   rK  r§   ra  rk   rh  rN  rf  re  r{  s               rb   r¬   z(ModernBertForTokenClassification.forwardÖ  så   € ð �$”*ð 
ØØ)Ø%Ø'ð	
ð 
ð
 ð
ð 
ˆð $ AœJÐà ŸIšIÐ&7Ñ8Ô8ÐØ ŸIšIÐ&7Ñ8Ô8ÐØ—’Ð!2Ñ3Ô3ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬OÑ<Ô<¸f¿kºkÈ"¹o¼oÑNÔNˆDå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rd   rj  )r†   r‡   rˆ   r*   rž   r!   r   r®   r¯   r°   r   r   rÊ   r   r¬   r”   r•   s   @rb   r*  r*  Ä  s  ø€ € € € € ð
Ð/ð 
ð 
ð 
ð 
ð 
ð 
ð Øð .2Ø.2Ø,0Ø-1Ø&*ð$
ð $
àÔ# dÑ*ð$
ð œ tÑ+ð$
ð ”l TÑ)ð	$
ð
 ”| dÑ*ð$
ð ”˜tÑ#ð$
ð Ð+Ô,ð$
ð 
ˆuŒ|Ô	Ð4Ñ	4ð$
ð $
ð $
ñ „^ñ Ôð$
ð $
ð $
ð $
ð $
rd   r*  c                   óì   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	e	e
         d
eej                 ez  fd„¦   «         ¦   «         Zˆ xZS )r+  r˜   c                 ót  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          |¦  «        | _        t          j         	                    |j
        ¦  «        | _        t          j        |j        |j        ¦  «        | _        |                      ¦   «          d S rª   r~  r¥   s     €rb   rž   z'ModernBertForQuestionAnswering.__init__  sŒ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå$ VÑ,Ô,ˆŒ
Ý,¨VÑ4Ô4ˆŒ	Ý”H×$Ò$ VÔ%>Ñ?Ô?ˆŒ	Ýœ) FÔ$6¸Ô8IÑJÔJˆŒà�ŠÑÔÐÐÐrd   Nr¦   rç   rK  Ústart_positionsÚend_positionsrk   r¨   c                 óì  —  | j         |f||dœ|¤Ž}|d         }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }	|	                     dd¬¦  «        \  }
}|
                     d¦  «                             ¦   «         }
|                     d¦  «                             ¦   «         }d }|�|� | j        |
|||fi |¤Ž}t          ||
||j	        |j
        ¬¦  «        S )N)rç   rK  r   rÌ   rº   r»   )re  Ústart_logitsÚ
end_logitsr«   r  )r  rZ  r¤   r,  Úsplitry  rö   rg  r   r«   r  )rj   r¦   rç   rK  r‚  rƒ  rk   rh  rN  rf  r…  r†  re  s                rb   r¬   z&ModernBertForQuestionAnswering.forward  s.  € ð �$”*Øð
à)Ø%ð
ð 
ð ð	
ð 
ˆð $ AœJÐà ŸIšIÐ&7Ñ8Ô8ÐØ ŸIšIÐ&7Ñ8Ô8ÐØ—’Ð!2Ñ3Ô3ˆà#)§<¢<°°r <Ñ#:Ô#:Ñ ˆ�jØ#×+Ò+¨BÑ/Ô/×:Ò:Ñ<Ô<ˆØ×'Ò'¨Ñ+Ô+×6Ò6Ñ8Ô8ˆ
àˆØÐ&¨=Ð+DØ%�4Ô% l°JÀÐQ^ÐiÐiÐbhÐiÐiˆDå+ØØ%Ø!Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
rd   rj  )r†   r‡   rˆ   r*   rž   r!   r   r®   r°   r   r   rÊ   r   r¬   r”   r•   s   @rb   r+  r+  ÿ  s   ø€ € € € € ð	Ð/ð 	ð 	ð 	ð 	ð 	ð 	ð Øð *.Ø.2Ø,0Ø/3Ø-1ð#
ð #
à”< $Ñ&ð#
ð œ tÑ+ð#
ð ”l TÑ)ð	#
ð
 œ¨Ñ,ð#
ð ”| dÑ*ð#
ð Ð+Ô,ð#
ð 
ˆuŒ|Ô	Ð;Ñ	;ð#
ð #
ð #
ñ „^ñ Ôð#
ð #
ð #
ð #
ð #
rd   r+  z«
    The ModernBert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RocStories/SWAG tasks.
    c                   óì   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej        dz  dej	        dz  dej	        dz  dej	        dz  dej	        dz  d	e
e         d
eej	                 ez  fd„¦   «         ¦   «         Zˆ xZS )r)  r˜   c                 ó`  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        t          j         	                    |j
        ¦  «        | _        t          j        |j        d¦  «        | _        |                      ¦   «          d S )NrÌ   )rh   rž   r˜   r<  r  r$  rZ  r®   r   r£   rT   r¤   rµ   r0   r,  rE  r¥   s     €rb   rž   z$ModernBertForMultipleChoice.__init__:  sˆ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒå$ VÑ,Ô,ˆŒ
Ý,¨VÑ4Ô4ˆŒ	Ý”H×$Ò$ VÔ%>Ñ?Ô?ˆŒ	Ýœ) FÔ$6¸Ñ:Ô:ˆŒð 	�ŠÑÔÐÐÐrd   Nr¦   rç   rK  r§   ra  rk   r¨   c                 ó  — |�|j         d         n|j         d         }|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�=|                     d|                     d¦  «        |                     d¦  «        ¦  «        nd} | j        d||||dœ|¤Ž}|d         }	| j        j        dk    rˆt          j        |	j         d         |	j        ¬¦  «        }
|�/| 	                    d¬	¦  «         
                    |	j        ¦  «        }n&t          j        dt          j        |	j        ¬
¦  «        }|	|
|f         }	nV| j        j        dk    rF|                     dd¬¦  «        }|	|                     d¦  «        z                       d¬	¦  «        |z  }	|                      |	¦  «        }|                      |¦  «        }|                      |¦  «        }|                     d|¦  «        }d}|�t%          j        ¦   «         } |||¦  «        }t)          |||j        |j        ¬¦  «        S )a&  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors.
        NrÌ   rº   éþÿÿÿrc  r   rQ   rM  r»   )rÎ   rÃ   rR   Trp  rd  re   )rî   rï   Úsizer  r˜   rS   r®   rO  rÃ   ÚargmaxrÐ   Útensorrx  rv  rÏ   rZ  r¤   r,  r   r	   r   r«   r  )rj   r¦   rç   rK  r§   ra  rk   Únum_choicesrh  rN  Ú	indices_0Úcls_maskÚnum_non_pad_tokensrz  rf  Úreshaped_logitsre  r{  s                     rb   r¬   z#ModernBertForMultipleChoice.forwardF  sº  € ð  -6Ð,A�i”o aÔ(Ð(À}ÔGZÐ[\ÔG]ˆà>GÐ>S�I—N’N 2 y§~¢~°bÑ'9Ô'9Ñ:Ô:Ð:ÐY]ˆ	ØM[ÐMg˜×,Ò,¨R°×1DÒ1DÀRÑ1HÔ1HÑIÔIÐIÐmqˆØGSÐG_�|×(Ò(¨¨\×->Ò->¸rÑ-BÔ-BÑCÔCÐCÐeiˆð Ð(ð ×Ò˜r =×#5Ò#5°bÑ#9Ô#9¸=×;MÒ;MÈbÑ;QÔ;QÑRÔRÐRàð 	ð �$”*ð 
ØØ)Ø%Ø'ð	
ð 
ð
 ð
ð 
ˆð $ AœJÐð Œ;Ô)¨UÒ2Ð2ÝœÐ%6Ô%<¸QÔ%?ÐHYÔH`ÐaÑaÔaˆIàÐ)Ø)×0Ò0°RÐ0Ñ8Ô8×;Ò;Ð<MÔ<TÑUÔU��õ !œ<¨µ´ÐDUÔD\Ð]Ñ]Ô]�à 1°)¸XÐ2EÔ FÐÐð Œ[Ô+¨vÒ5Ð5Ø!/×!3Ò!3¸À4Ð!3Ñ!HÔ!HÐØ!2°^×5MÒ5MÈbÑ5QÔ5QÑ!Q× VÒ VÐ[\Ð VÑ ]Ô ]Ð`rÑ rÐàŸ	š	Ð"3Ñ4Ô4ˆØŸ	š	 -Ñ0Ô0ˆØ—’ Ñ/Ô/ˆà Ÿ+š+ b¨+Ñ6Ô6ˆàˆØÐÝÔ*Ñ,Ô,ˆHØ�8˜O¨VÑ4Ô4ˆDå(ØØ"Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rd   rj  )r†   r‡   rˆ   r*   rž   r!   r   r®   r¯   r°   r   r   rÊ   r   r¬   r”   r•   s   @rb   r)  r)  4  s  ø€ € € € € ð
Ð/ð 
ð 
ð 
ð 
ð 
ð 
ð Øð .2Ø.2Ø,0Ø-1Ø&*ðC
ð C
àÔ# dÑ*ðC
ð œ tÑ+ðC
ð ”l TÑ)ð	C
ð
 ”| dÑ*ðC
ð ”˜tÑ#ðC
ð Ð+Ô,ðC
ð 
ˆuŒ|Ô	Ð8Ñ	8ðC
ð C
ð C
ñ „^ñ ÔðC
ð C
ð C
ð C
ð C
rd   r)  )r*   r<  r
  r&  r(  r*  r+  r)  )rÌ   )Mr"  Úcollections.abcr   Útypingr   r   r®   Úhuggingface_hub.dataclassesr   r   Útorch.nnr   r	   r
   Ú r   r  Úactivationsr   Úconfiguration_utilsr   Úintegrationsr   r   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r    Úutils.genericr!   r"   Úutils.output_capturingr#   Úalign.modeling_alignr%   Úgemma3.modeling_gemma3r&   r'   Ú
get_loggerr†   Úloggerr*   r!  r—   r²   rÁ   rÙ   rÛ   r   r
  r<  r$  r&  r(  r*  r+  r)  Ú__all__re   rd   rb   ú<module>rª     sü  ðð  €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø $Ð $Ð $Ð $Ð $Ð $Ð $Ð $à €€€Ø .Ð .Ð .Ð .Ð .Ð .Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 7Ð 6Ð 6Ð 6Ð 6Ð 6Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø :Ð :Ð :Ð :Ð :Ð :Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ Gð 
ˆÔ	˜HÑ	%Ô	%€ð €Ð8Ð9Ñ9Ô9ØðG)ð G)ð G)ð G)ð G)Ð'ñ G)ô G)ñ „ñ :Ô9ðG)ðTð ð ð ð ˜2œ9ñ ô ð ð,:ð :ð :ð :ð :�B”Iñ :ô :ð :ð(\ð \ð \ð \ð \Ð 5ñ \ô \ð \ð ÐÐ*Ñ+Ô+ðBð Bð Bñ ,Ô+ðBð4 ÐÐ)Ñ*Ô*ðN)ð N)ð N)ð N)ð N)˜"œ)ñ N)ô N)ñ +Ô*ðN)ðbð ð ð ð Ð7ñ ô ð ð@ ðG^ð G^ð G^ð G^ð G^ ñ G^ô G^ñ „ðG^ðT ðB@ð B@ð B@ð B@ð B@Ð/ñ B@ô B@ñ „ðB@ðJ	>ð 	>ð 	>ð 	>ð 	>˜rœyñ 	>ô 	>ð 	>ð €ððñ ô ð
?
ð ?
ð ?
ð ?
ð ?
Ð5ñ ?
ô ?
ñô ð
?
ðD €ððñ ô ð
S
ð S
ð S
ð S
ð S
Ð*Cñ S
ô S
ñô ð
S
ðl €ððñ ô ð
3
ð 3
ð 3
ð 3
ð 3
Ð'@ñ 3
ô 3
ñô ð
3
ðl ð1
ð 1
ð 1
ð 1
ð 1
Ð%>ñ 1
ô 1
ñ „ð1
ðh €ððñ ô ð
R
ð R
ð R
ð R
ð R
Ð";ñ R
ô R
ñô ð
R
ðj	ð 	ð 	€€€rd   