§
    ‚Štj–  ã                   ó�   — d Z ddlmZ ddlmZ ddlmZ ddlmZ  ed¬¦  «        e G d	„ d
e¦  «        ¦   «         ¦   «         Z	d
gZ
dS )zPyTorch Phi-MoE model.é    )Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringzmicrosoft/Phi-3.5-MoE-instruct)Ú
checkpointc                   óò  ‡ — e Zd ZU dZdZdgZdddddœZdZd	Ze	e
d
<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZee
d<   dZe	e
d<   dZee
d<   dZee
d<   dZee
d<   dZe	dz  e
d<   d Ze	dz  e
d!<   d"Ze	ee	         z  dz  e
d#<   d$Zee
d%<   dZeez  dz  e
d&<   dZ e	dz  e
d'<   d(Z!ee	z  e
d)<   d"Z"e	e
d*<   d+Z#e	e
d,<   d$Z$ee
d-<   d.Z%ee
d/<   d0Z&ee
d1<   d(Z'ee
d2<   d$Z(ee
d3<   d$Z)ee
d4<   ˆ fd5„Z*ˆ fd6„Z+ˆ xZ,S )7ÚPhimoeConfiga²  
    num_local_experts (`int`, *optional*, defaults to 16):
        Number of experts per Sparse MLP layer.
    input_jitter_noise (`float`, *optional*, defaults to 0.0):
        Input jitter noise
    lm_head_bias (`bool`, *optional*, defaults to `False`):
        LM head bias

    Example:

    ```python
    >>> from transformers import PhimoeModel, PhimoeConfig
    >>> # Initializing a Phi-3 style configuration
    >>> configuration = PhimoeConfig.from_pretrained("microsoft/Phi-3.5-MoE-instruct")
    >>> # Initializing a model from the configuration
    >>> model = PhimoeModel(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```ÚphimoeÚpast_key_valuesÚ	ep_routerÚgrouped_gemmÚmoe_tp_experts)zlayers.*.mlp.routerz!layers.*.mlp.experts.gate_up_projzlayers.*.mlp.experts.down_projzlayers.*.mlp.expertsg    €„.Ai@}  Ú
vocab_sizei   Úhidden_sizei   Úintermediate_sizeé    Únum_hidden_layersÚnum_attention_headsé   Únum_key_value_headsÚsiluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangegñhãˆµøä>Úrms_norm_epsTÚ	use_cacheNÚpad_token_idé   Úbos_token_idé   Úeos_token_idFÚtie_word_embeddingsÚrope_parametersÚsliding_windowg        Úattention_dropoutÚnum_experts_per_toké   Únum_local_expertsÚoutput_router_logitsgü©ñÒMbP?Úrouter_aux_loss_coefg{®Gáz„?Úrouter_jitter_noiseÚinput_jitter_noiseÚattention_biasÚlm_head_biasc                 ó`   •— | j         €| j        | _          t          ¦   «         j        di |¤Ž d S )N© )r   r   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €úm/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/phimoe/configuration_phimoe.pyr3   zPhimoeConfig.__post_init__T   s:   ø€ ØÔ#Ð+Ø'+Ô'?ˆDÔ$Ø�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'ó    c                 óÈ  •— t          ¦   «                              ¦   «          | j        d         dk    r­d| j        v r| j        d         | _        | j                             dd¦  «        }| j                             dd¦  «        }t          |t          t          f¦  «        st          d|› �¦  «        ‚t          |t          t          f¦  «        st          d|› �¦  «        ‚dS dS )	z?
        Validate the `rope_parameters` configuration.
        Ú	rope_typeÚdefaultÚ original_max_position_embeddingsÚshort_mscaleNÚlong_mscalez=`rope_parameters`'s short_mscale field must be a number, got z<`rope_parameters`'s long_mscale field must be a number, got )	r2   Úvalidate_roper$   r<   ÚgetÚ
isinstanceÚintÚfloatÚ	TypeError)r4   Úrope_parameters_short_mscaleÚrope_parameters_long_mscaler6   s      €r7   r?   zPhimoeConfig.validate_ropeY   s  ø€ õ 	‰Œ×ÒÑÔÐð Ô Ô,°	Ò9Ð9Ø1°TÔ5IÐIÐIØ8<Ô8LÐMoÔ8p�Ô5Ø+/Ô+?×+CÒ+CÀNÐTXÑ+YÔ+YÐ(Ø*.Ô*>×*BÒ*BÀ=ÐRVÑ*WÔ*WÐ'ÝÐ:½SÅ%¸LÑIÔIð ÝØrÐTpÐrÐrñô ð õ Ð9½CÅ¸<ÑHÔHð ÝØpÐSnÐpÐpñô ð ð :Ð9ðð r8   )-Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_ep_planÚdefault_thetar   rB   Ú__annotations__r   r   r   r   r   r   Ústrr   r   rC   r   r   Úboolr   r    r"   Úlistr#   r$   r   Údictr%   r&   r'   r)   r*   r+   r,   r-   r.   r/   r3   r?   Ú__classcell__)r6   s   @r7   r
   r
      sb  ø€ € € € € € ðð ð( €JØ#4Ð"5Ðà*Ø-;Ø*8Ø 0ð	ð Ðð €Mà€J�ÐÐÑØ€K�ÐÐÑØ!Ð�sÐ!Ð!Ñ!ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø Ð˜Ð Ð Ñ Ø€J�ÐÐÑØ#,Ð˜SÐ,Ð,Ñ,Ø#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØ#€L�#˜‘*Ð#Ð#Ñ#Ø €L�#˜‘*Ð Ð Ñ Ø+,€L�#˜˜Sœ	‘/ DÑ(Ð,Ð,Ñ,Ø %Ð˜Ð%Ð%Ñ%Ø48€O�^ dÑ*¨TÑ1Ð8Ð8Ñ8Ø!%€N�C˜$‘JÐ%Ð%Ñ%Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø Ð˜Ð Ð Ñ ØÐ�sÐÐÑØ!&Ð˜$Ð&Ð&Ñ&Ø"'Ð˜%Ð'Ð'Ñ'Ø!%Ð˜Ð%Ð%Ñ%Ø #Ð˜Ð#Ð#Ñ#Ø €N�DÐ Ð Ñ Ø€L�$ÐÐÑð(ð (ð (ð (ð (ð
ð ð ð ð ð ð ð ð r8   r
   N)rJ   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr   r
   Ú__all__r1   r8   r7   ú<module>rZ      s¼   ðð Ð à .Ð .Ð .Ð .Ð .Ð .à 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø #Ð #Ð #Ð #Ð #Ð #ð €Ð;Ð<Ñ<Ô<ØðRð Rð Rð Rð RÐ#ñ Rô Rñ „ñ =Ô<ðRðj Ð
€€€r8   