§
    ‚ŠtjŒ  ã                   ó�   — d Z ddlmZ ddlmZ ddlmZ ddlmZ  ed¬¦  «        e G d	„ d
e¦  «        ¦   «         ¦   «         Z	d
gZ
dS )zMistral4 model configurationé    )Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringz#mistralai/Mistral-Small-4-119B-2603)Ú
checkpointc            
       óÖ  ‡ — e Zd ZU dZdZdgZddddddddddœ	Zd	gd
gfddgdgfdgdgfdœZdddddœZddiZ	dZ
eed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZedz  ed <   d!Zeed"<   d#Zeed<   d$Zeed%<   d&Zeed'<   d(Zedz  ed)<   d*Zeed+<   d#Zedz  ed,<   d*Zeed-<   d!Zedz  ed.<   d!Zedz  ed/<   d0Zedz  ed1<   d2Zedz  ed3<   d4Z e!dz  ed5<   d6Z"e#ed7<   d8Z$eed9<   d:Z%eed;<   d<Z&eed=<   d4Z'e!ed><   d?Z(edz  ed@<   d!Z)edz  edA<   dBZ*ee+e         z  dz  edC<   d!Z,edz  edD<   dEZ-e!edF<   dZ.e/e0z  dz  edG<   d4Z1e!dz  edH<   dEZ2e!edI<   dJZ3eez  dz  edK<   ˆ fdL„Z4ˆ xZ5S )MÚMistral4Configaã  
    n_group (`int`, *optional*, defaults to 1):
        Number of groups for routed experts.
    first_k_dense_replace (`int`, *optional*, defaults to 0):
        Number of dense layers in shallow layers(embed->dense->dense->...->dense->moe->moe...->lm_head).
                                                        \--k dense layers--/
    rope_interleave (`bool`, *optional*, defaults to `True`):
        Whether to interleave the rotary position embeddings.

    Example:

    ```python
    >>> from transformers import Mistral4Model, Mistral4Config

    >>> # Initializing a Mistral4 style configuration
    >>> configuration = Mistral4Config()

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Úmistral4Úpast_key_valuesÚpacked_colwiseÚrowwiseÚmoe_tp_expertsÚcolwise)	ú!layers.*.mlp.experts.gate_up_projúlayers.*.mlp.experts.down_projúlayers.*.mlp.expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormÚ	ep_routerÚgrouped_gemm)zlayers.*.mlp.gater   r   r   Únum_local_expertsÚn_routed_expertsi   Ú
vocab_sizei   Úhidden_sizei 0  Úintermediate_sizei   Úmoe_intermediate_sizeé$   Únum_hidden_layersé    Únum_attention_headsNÚnum_key_value_headsé   Ún_shared_expertsé€   ç      ð?Úrouted_scaling_factoré   Úkv_lora_ranki   Úq_lora_ranké@   Úqk_rope_head_dimÚ
v_head_dimÚqk_nope_head_dimÚn_groupÚ
topk_groupé   Únum_experts_per_tokr   Úfirst_k_dense_replaceTÚnorm_topk_probÚsiluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeg�íµ ÷Æ°>Úrms_norm_epsÚ	use_cacheé   Úpad_token_idÚbos_token_idé   Úeos_token_idÚpretraining_tpFÚtie_word_embeddingsÚrope_parametersÚrope_interleaveÚattention_biasg        Úattention_dropoutc                 óp  •— | j         €-dddd| j        ddddd| j        | j        | j        z   z  dœ| _         | j        €| j        | _        | j        | j        z   | _        | j        | j        z   | _        | j                              d	| j        | j        z  ¦  «          t          ¦   «         j
        dd
ddhi|¤Ž d S )NÚyarng     ˆÃ@g      `@i    g      @@r+   gš™™™™™¹?)ÚtypeÚ
rope_thetaÚfactorÚ original_max_position_embeddingsr<   Ú	beta_fastÚ	beta_slowÚmscale_all_dimÚmscaleÚllama_4_scaling_betaÚpartial_rotary_factorrV   Úignore_keys_at_rope_validationrU   r<   © )rG   r<   r1   r3   r'   r&   Úqk_head_dimÚhead_dimÚ
setdefaultÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €úq/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/mistral4/configuration_mistral4.pyr]   zMistral4Config.__post_init__n   sô   ø€ ØÔÐ'àØ%ØØ48Ø+/Ô+GØ!Ø Ø"%ØØ(+Ø)-Ô)>À$ÔBWÐZ^ÔZoÑBoÑ)pð$ð $ˆDÔ ð Ô#Ð+Ø'+Ô'?ˆDÔ$àÔ0°4Ô3HÑHˆÔØÔ-°Ô0EÑEˆŒØÔ×'Ò'Ð(?ÀÔAVÐY]ÔYfÑAfÑgÔgÐgØ�‰ŒÔð 	
ð 	
Ø,BÐD]Ð+^ð	
Øbhð	
ð 	
ð 	
ð 	
ð 	
ó    )6Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planÚbase_model_ep_planÚattribute_mapr   ÚintÚ__annotations__r    r!   r"   r$   r&   r'   r)   r   r,   Úfloatr.   r/   r1   r2   r3   r4   r5   r7   r8   r9   Úboolr;   Ústrr<   r=   r>   r?   rA   rB   rD   ÚlistrE   rF   rG   r   ÚdictrH   rI   rJ   r]   Ú__classcell__)r`   s   @ra   r
   r
      sc  ø€ € € € € € ðð ð* €JØ#4Ð"5Ðà-=Ø*3Ø 0Ø1:Ø/8Ø1:Ø"+Ø )Ø"+ð
ð 
Ðð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ðð Ðð )Ø-;Ø*8Ø 0ð	ð Ðð 	Ð/ð€Mð €J�ÐÐÑØ€K�ÐÐÑØ"Ð�sÐ"Ð"Ñ"Ø!%Ð˜3Ð%Ð%Ñ%ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø&(Ð˜˜t™Ð(Ð(Ñ(ØÐ�cÐÐÑØÐ�cÐÐÑØ#&Ð˜5Ð&Ð&Ñ&Ø€L�#ÐÐÑØ"€K��t‘Ð"Ð"Ñ"ØÐ�cÐÐÑØ €J��d‘
Ð Ð Ñ ØÐ�cÐÐÑØ€GˆS�4‰ZÐÐÑØ€J��d‘
ÐÐÑØ&'Ð˜˜t™Ð'Ð'Ñ'Ø()Ð˜3 ™:Ð)Ð)Ñ)Ø"&€N�D˜4‘KÐ&Ð&Ñ&Ø€J�ÐÐÑØ#*Ð˜SÐ*Ð*Ñ*Ø#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØ!€L�#˜‘*Ð!Ð!Ñ!Ø €L�#˜‘*Ð Ð Ñ Ø+,€L�#˜˜Sœ	‘/ DÑ(Ð,Ð,Ñ,Ø!"€N�C˜$‘JÐ"Ð"Ñ"Ø %Ð˜Ð%Ð%Ñ%Ø48€O�^ dÑ*¨TÑ1Ð8Ð8Ñ8Ø#'€O�T˜D‘[Ð'Ð'Ñ'Ø €N�DÐ Ð Ñ Ø,/Ð�u˜s‘{ TÑ)Ð/Ð/Ñ/ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
rb   r
   N)rf   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr   r
   Ú__all__rX   rb   ra   ú<module>rz      s½   ðð #Ð "à .Ð .Ð .Ð .Ð .Ð .à 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø #Ð #Ð #Ð #Ð #Ð #ð €Ð@ÐAÑAÔAØðm
ð m
ð m
ð m
ð m
Ð%ñ m
ô m
ñ „ñ BÔAðm
ð` Ð
€€€rb   