§
    ‚ŠtjŠ  ã                   ó�   — d Z ddlmZ ddlmZ ddlmZ ddlmZ  ed¬¦  «        e G d	„ d
e¦  «        ¦   «         ¦   «         Z	d
gZ
dS )zMixtral model configurationé    )Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringzmistralai/Mixtral-8x7B-v0.1)Ú
checkpointc                   ó  ‡ — e Zd ZU dZdZdgZdZdddddddd	œZd
gdgfddgdgfdgdgfdœZdddddœZ	ddiZ
dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZedz  ed <   d!Zeed"<   d#Zeed$<   d%Zeed&<   d'Zeed(<   d)Zeed*<   dZedz  ed+<   d,Zedz  ed-<   d.Zeee         z  dz  ed/<   d0Z eed1<   dZ!edz  ed2<   d3Z"eez  ed4<   d.Z#eed5<   dZ$eed<   d0Z%eed6<   d7Z&eed8<   d3Z'eed9<   dZ(e)e*z  dz  ed:<   ˆ fd;„Z+ˆ xZ,S )<ÚMixtralConfigaˆ  
    Example:

    ```python
    >>> from transformers import MixtralModel, MixtralConfig

    >>> # Initializing a Mixtral 7B style configuration
    >>> configuration = MixtralConfig()

    >>> # Initializing a model from the Mixtral 7B style configuration
    >>> model = MixtralModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```ÚmixtralÚpast_key_valuesg    €„.AÚcolwiseÚrowwiseÚpacked_colwiseÚmoe_tp_experts)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projú!layers.*.mlp.experts.gate_up_projúlayers.*.mlp.experts.down_projúlayers.*.mlp.expertsÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormÚ	ep_routerÚgrouped_gemm)zlayers.*.mlp.gater   r   r   Únum_expertsÚnum_local_expertsi }  Ú
vocab_sizei   Úhidden_sizei 8  Úintermediate_sizeé    Únum_hidden_layersÚnum_attention_headsé   Únum_key_value_headsNÚhead_dimÚsiluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangegñhãˆµøä>Úrms_norm_epsTÚ	use_cacheÚpad_token_idé   Úbos_token_idé   Úeos_token_idFÚtie_word_embeddingsÚsliding_windowg        Úattention_dropoutÚnum_experts_per_tokÚoutput_router_logitsgü©ñÒMbP?Úrouter_aux_loss_coefÚrouter_jitter_noiseÚrope_parametersc                 ó`   •— | j         €| j        | _          t          ¦   «         j        di |¤Ž d S )N© )r&   r$   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €úo/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/mixtral/configuration_mixtral.pyr>   zMixtralConfig.__post_init__\   s:   ø€ ØÔ#Ð+Ø'+Ô'?ˆDÔ$à�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'ó    )-Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚdefault_thetaÚbase_model_tp_planÚbase_model_pp_planÚbase_model_ep_planÚattribute_mapr   ÚintÚ__annotations__r    r!   r#   r$   r&   r'   r)   Ústrr*   r+   Úfloatr,   r-   Úboolr.   r0   r2   Úlistr3   r4   r5   r6   r   r7   r8   r9   r:   r   Údictr>   Ú__classcell__)rA   s   @rB   r
   r
      s–  ø€ € € € € € ðð ð  €JØ#4Ð"5ÐØ€Mà%.Ø%.Ø%.Ø%.Ø-=Ø*3Ø 0ðð Ðð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ðð Ðð )Ø-;Ø*8Ø 0ð	ð Ðð #Ð$7Ð8€Mà€J�ÐÐÑØ€K�ÐÐÑØ"Ð�sÐ"Ð"Ñ"ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø Ð˜Ð Ð Ñ Ø€Hˆc�D‰jÐÐÑØ€J�ÐÐÑØ#,Ð˜SÐ,Ð,Ñ,Ø#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØ#€L�#˜‘*Ð#Ð#Ñ#Ø €L�#˜‘*Ð Ð Ñ Ø+,€L�#˜˜Sœ	‘/ DÑ(Ð,Ð,Ñ,Ø %Ð˜Ð%Ð%Ñ%Ø!%€N�C˜$‘JÐ%Ð%Ñ%Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø Ð˜Ð Ð Ñ ØÐ�sÐÐÑØ!&Ð˜$Ð&Ð&Ñ&Ø"'Ð˜%Ð'Ð'Ñ'Ø!$Ð˜Ð$Ð$Ñ$Ø48€O�^ dÑ*¨TÑ1Ð8Ð8Ñ8ð(ð (ð (ð (ð (ð (ð (ð (ð (rC   r
   N)rG   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr   r
   Ú__all__r<   rC   rB   ú<module>r\      s¼   ðð "Ð !à .Ð .Ð .Ð .Ð .Ð .à 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø #Ð #Ð #Ð #Ð #Ð #ð €Ð8Ð9Ñ9Ô9ØðG(ð G(ð G(ð G(ð G(Ð$ñ G(ô G(ñ „ñ :Ô9ðG(ðT Ð
€€€rC   