§
    ‚ŠtjÆ  ã                   óŒ   — d dl mZ d dlmZ ddlmZ ddlmZ  ed¬¦  «        e G d„ d	e¦  «        ¦   «         ¦   «         Zd	gZ	d
S )é    )ÚLiteral)Ústricté   )ÚPreTrainedConfig)Úauto_docstringzanswerdotai/ModernBERT-base)Ú
checkpointc                   óÔ  ‡ — e Zd ZU dZdZdgZdddœZdZee	d<   d	Z
ee	d
<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZedz  e	d<   d Zeee         z  dz  e	d!<   d"Zedz  e	d#<   d"Zedz  e	d$<   d Zedz  e	d%<   dZee	d&<   d'Zeez  e	d(<   dZee         dz  e	d)<   dZ e!e"d*         e!f         dz  e	d+<   d,Z#ee	d-<   d'Z$eez  e	d.<   dZ%ee	d/<   d'Z&eez  e	d0<   d1Z'ee	d2<   d3Z(e"d4         e	d5<   d'Z)eez  e	d6<   dZ*ee	d7<   dZ+ee	d8<   dZ,ee	d9<   dZ-ee	d:<   d;Z.ee	d<<   d1Z/ee	d=<   ˆ fd>„Z0d?„ Z1ˆ fd@„Z2e3dA„ ¦   «         Z4e4j5        dB„ ¦   «         Z4ˆ xZ6S )CÚModernBertConfiga+  
    initializer_cutoff_factor (`float`, *optional*, defaults to 2.0):
        The cutoff factor for the truncated_normal_initializer for initializing all weight matrices.
    norm_eps (`float`, *optional*, defaults to 1e-05):
        The epsilon used by the rms normalization layers.
    norm_bias (`bool`, *optional*, defaults to `False`):
        Whether to use bias in the normalization layers.
    local_attention (`int`, *optional*, defaults to 128):
        The window size for local attention.
    mlp_dropout (`float`, *optional*, defaults to 0.0):
        The dropout ratio for the MLP layers.
    decoder_bias (`bool`, *optional*, defaults to `True`):
        Whether to use bias in the decoder layers.
    classifier_pooling (`str`, *optional*, defaults to `"cls"`):
        The pooling method for the classifier. Should be either `"cls"` or `"mean"`. In local attention layers, the
        CLS token doesn't attend to all tokens on long sequences.
    classifier_bias (`bool`, *optional*, defaults to `False`):
        Whether to use bias in the classifier.
    classifier_activation (`str`, *optional*, defaults to `"gelu"`):
        The activation function for the classifier.
    deterministic_flash_attn (`bool`, *optional*, defaults to `False`):
        Whether to use deterministic flash attention. If `False`, inference will be faster but not deterministic.
    sparse_prediction (`bool`, *optional*, defaults to `False`):
        Whether to use sparse prediction for the masked language model instead of returning the full dense logits.
    sparse_pred_ignore_index (`int`, *optional*, defaults to -100):
        The index to ignore for the sparse prediction.

    Examples:

    ```python
    >>> from transformers import ModernBertModel, ModernBertConfig

    >>> # Initializing a ModernBert style configuration
    >>> configuration = ModernBertConfig()

    >>> # Initializing a model from the modernbert-base style configuration
    >>> model = ModernBertModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Ú
modernbertÚpast_key_valuesg     ˆAg     ˆÃ@)ÚglobalÚlocaliÀÄ  Ú
vocab_sizei   Úhidden_sizei€  Úintermediate_sizeé   Únum_hidden_layersé   Únum_attention_headsÚgeluÚhidden_activationi    Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeg       @Úinitializer_cutoff_factorgñhãˆµøä>Únorm_epsFÚ	norm_biasikÄ  NÚpad_token_idijÄ  Úeos_token_idiiÄ  Úbos_token_idÚcls_token_idÚsep_token_idÚattention_biasg        Úattention_dropoutÚlayer_types)Úfull_attentionÚsliding_attentionÚrope_parametersé€   Úlocal_attentionÚembedding_dropoutÚmlp_biasÚmlp_dropoutTÚdecoder_biasÚcls)r.   ÚmeanÚclassifier_poolingÚclassifier_dropoutÚclassifier_biasÚclassifier_activationÚdeterministic_flash_attnÚsparse_predictioniœÿÿÿÚsparse_pred_ignore_indexÚtie_word_embeddingsc                 óÀ   •‡— |                      dd¦  «        Š| j        €%ˆfd„t          | j        ¦  «        D ¦   «         | _         t	          ¦   «         j        di |¤Ž d S )NÚglobal_attn_every_n_layersr   c                 ó<   •— g | ]}t          |‰z  ¦  «        rd nd‘ŒS ©r&   r%   )Úbool)Ú.0Úir9   s     €úu/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/modernbert/configuration_modernbert.pyú
<listcomp>z2ModernBertConfig.__post_init__.<locals>.<listcomp>u   sC   ø€ ð  ð  ð  àõ (,¨AÐ0JÑ,JÑ'KÔ'KÐaÐ#Ð#ÐQað ð  ð  ó    © )Úgetr$   Úranger   ÚsuperÚ__post_init__)ÚselfÚkwargsr9   Ú	__class__s     @€r?   rF   zModernBertConfig.__post_init__q   s}   øø€ à%+§Z¢ZÐ0LÈaÑ%PÔ%PÐ"ØÔÐ#ð ð  ð  ð  å˜tÔ5Ñ6Ô6ð ñ  ô  ˆDÔð
 	�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'rA   c                 ó²  — |                      dd ¦  «        }ddiddidœ}| j        �| j        n|| _        |�@| j        d                              |¦  «         | j        d                              |¦  «         | j                             d¦  «        €ddi| j        d<   | j        d                              d|                      d| j        d	         ¦  «        ¦  «         | j                             d¦  «        €ddi| j        d<   | j        d                              d|                      d
| j        d         ¦  «        ¦  «         |                      ¦   «          |S )NÚrope_scalingÚ	rope_typeÚdefaultr;   r%   r&   Ú
rope_thetaÚglobal_rope_thetar   Úlocal_rope_thetar   )Úpopr'   ÚupdaterC   Ú
setdefaultÚdefault_thetaÚstandardize_rope_params)rG   rH   rK   Údefault_rope_paramss       r?   Úconvert_rope_params_to_dictz,ModernBertConfig.convert_rope_params_to_dict|   s|  € Ø—z’z .°$Ñ7Ô7ˆð
 #.¨yÐ!9Ø*¨IÐ6ð
ð 
Ðð 8<Ô7KÐ7W˜tÔ3Ð3Ð]pˆÔØÐ#ØÔ Ð!1Ô2×9Ò9¸,ÑGÔGÐGØÔ Ð!4Ô5×<Ò<¸\ÑJÔJÐJð Ô×#Ò#Ð$4Ñ5Ô5Ð=Ø6AÀ9Ð5MˆDÔ Ð!1Ñ2ØÔÐ-Ô.×9Ò9Ø˜&Ÿ*š*Ð%8¸$Ô:LÈXÔ:VÑWÔWñ	
ô 	
ð 	
ð Ô×#Ò#Ð$7Ñ8Ô8Ð@Ø9DÀiÐ8PˆDÔ Ð!4Ñ5ØÔÐ0Ô1×<Ò<Ø˜&Ÿ*š*Ð%7¸Ô9KÈGÔ9TÑUÔUñ	
ô 	
ð 	
ð
 	×$Ò$Ñ&Ô&Ð&ØˆrA   c                 ót   •— t          ¦   «                              ¦   «         }|                     dd ¦  «         |S )NÚreference_compile)rE   Úto_dictrQ   )rG   ÚoutputrI   s     €r?   rZ   zModernBertConfig.to_dictš   s0   ø€ Ý‘”—’Ñ"Ô"ˆØ�
Š
Ð&¨Ñ-Ô-Ð-ØˆrA   c                 ó   — | j         dz  S )zKHalf-window size: `local_attention` is the total window, so we divide by 2.é   ©r)   )rG   s    r?   Úsliding_windowzModernBertConfig.sliding_windowŸ   s   € ð Ô# qÑ(Ð(rA   c                 ó   — |dz  | _         dS )z<Set sliding_window by updating local_attention to 2 * value.r]   Nr^   )rG   Úvalues     r?   r_   zModernBertConfig.sliding_window¤   s   € ð  % q™yˆÔÐÐrA   )7Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferencerT   r   ÚintÚ__annotations__r   r   r   r   r   Ústrr   r   Úfloatr   r   r   r<   r   r   Úlistr   r    r!   r"   r#   r$   r'   Údictr   r)   r*   r+   r,   r-   r0   r1   r2   r3   r4   r5   r6   r7   rF   rW   rZ   Úpropertyr_   ÚsetterÚ__classcell__)rI   s   @r?   r
   r
      s5  ø€ € € € € € ð(ð (ðT €JØ#4Ð"5ÐØ(°8Ð<Ð<€Mà€J�ÐÐÑØ€K�ÐÐÑØ!Ð�sÐ!Ð!Ñ!ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø#Ð�sÐ#Ð#Ñ#Ø#'Ð˜SÐ'Ð'Ñ'Ø#Ð�uÐ#Ð#Ñ#Ø'*Ð˜uÐ*Ð*Ñ*Ø€HˆeÐÐÑØ€IˆtÐÐÑØ$€L�#˜‘*Ð$Ð$Ñ$Ø+0€L�#˜˜Sœ	‘/ DÑ(Ð0Ð0Ñ0Ø$€L�#˜‘*Ð$Ð$Ñ$Ø$€L�#˜‘*Ð$Ð$Ñ$Ø$€L�#˜‘*Ð$Ð$Ñ$Ø €N�DÐ Ð Ñ Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø$(€K��c”˜TÑ!Ð(Ð(Ñ(ØY]€O�T˜'Ð"GÔHÈ$ÐNÔOÐRVÑVÐ]Ð]Ñ]Ø€O�SÐÐÑØ%(Ð�u˜s‘{Ð(Ð(Ñ(Ø€HˆdÐÐÑØ"€K�˜‘Ð"Ð"Ñ"Ø€L�$ÐÐÑØ16Ð˜ Ô.Ð6Ð6Ñ6Ø&)Ð˜ ™Ð)Ð)Ñ)Ø!€O�TÐ!Ð!Ñ!Ø!'Ð˜3Ð'Ð'Ñ'Ø%*Ð˜dÐ*Ð*Ñ*Ø#Ð�tÐ#Ð#Ñ#Ø$(Ð˜cÐ(Ð(Ñ(Ø $Ð˜Ð$Ð$Ñ$ð	(ð 	(ð 	(ð 	(ð 	(ðð ð ð<ð ð ð ð ð
 ð)ð )ñ „Xð)ð Ôð)ð )ñ Ôð)ð )ð )ð )ð )rA   r
   N)
Útypingr   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úutilsr   r
   Ú__all__rB   rA   r?   ú<module>rv      s¶   ðð, Ð Ð Ð Ð Ð à .Ð .Ð .Ð .Ð .Ð .à 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø #Ð #Ð #Ð #Ð #Ð #ð €Ð8Ð9Ñ9Ô9ØðG)ð G)ð G)ð G)ð G)Ð'ñ G)ô G)ñ „ñ :Ô9ðG)ðT Ð
€€€rA   