§
    ‚ŠtjZ  ã                   ó’   — d Z ddlmZ ddlmZ ddlmZ ddlmZ e edd¬	¦  «         G d
„ de¦  «        ¦   «         ¦   «         Z	dgZ
dS )zAFMoE model configurationé    )Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringzÏ
    AFMoE is an Adaptive Feedforward MoE (Mixture of Experts) model with token-choice routing, shared experts, and a
    hybrid attention mechanism combining sliding window and full attention patterns.
    zarcee-ai/Trinity-Mini)Úcustom_introÚ
checkpointc                   ó�  ‡ — e Zd ZU dZdZdgZdgdgfddgdgfdgdgfdœZd	d
d
ddœZdZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	dz  e
d<   dZe	e
d<   dZe	dz  e
d<   dZe	dz  e
d<   dZee
d <   d!Ze	e
d"<   d#Zee
d$<   d%Zee
d&<   d'Zee
d(<   d)Zee
d*<   dZeez  dz  e
d+<   d,Ze	dz  e
d-<   d.Z e	dz  e
d/<   d0Z!e	dz  e
d1<   d2Z"edz  e
d3<   d)Z#ee
d4<   d5Z$e	dz  e
d6<   d7Z%e	dz  e
d8<   dZ&e'e         dz  e
d9<   d:Z(ee	z  dz  e
d;<   d)Z)edz  e
d<<   dZ*e	e'e	         z  dz  e
d=<   dZ+e	dz  e
d><   dZ,e	dz  e
d?<   d)Z-ee
d@<   ˆ fdA„Z.ˆ xZ/S )BÚAfmoeConfiga%  
    global_attn_every_n_layers (`int`, *optional*, defaults to 4):
        The frequency of full attention layers. Every Nth layer will use full attention, while others use sliding
        window attention.
    mup_enabled (`bool`, *optional*, defaults to `False`):
        Whether to enable muP (Maximal Update Parametrization) input scaling. When enabled, input embeddings
        are scaled by `sqrt(hidden_size)`.

    Example:
    ```python
    >>> from transformers import AfmoeModel, AfmoeConfig

    >>> # Initializing an AFMoE configuration
    >>> configuration = AfmoeConfig()

    >>> # Initializing a model from the afmoe-small-sft-v1 style configuration
    >>> model = AfmoeModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    ÚafmoeÚpast_key_valuesÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormÚ	ep_routerÚgrouped_gemmÚmoe_tp_experts)zlayers.*.mlp.routerz!layers.*.mlp.experts.gate_up_projzlayers.*.mlp.experts.down_projzlayers.*.mlp.expertsi  Ú
vocab_sizei   Úhidden_sizei   Úintermediate_sizei€  Úmoe_intermediate_sizeé    Únum_hidden_layersé   NÚnum_dense_layersé   Únum_attention_headsÚnum_key_value_headsé€   Úhead_dimÚsiluÚ
hidden_acti @  Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangegñhãˆµøä>Úrms_norm_epsTÚ	use_cacheFÚtie_word_embeddingsÚrope_parametersé@   Únum_expertsé   Únum_experts_per_toké   Únum_shared_expertsg      ð?Úroute_scaleÚoutput_router_logitsé   Úglobal_attn_every_n_layersi   Úsliding_windowÚlayer_typesg        Úattention_dropoutÚmup_enabledÚeos_token_idÚpad_token_idÚbos_token_idÚattention_biasc                 óº   •‡ — ‰ j         €%ˆ fd„t          ‰ j        ¦  «        D ¦   «         ‰ _         ‰ j        €‰ j        ‰ _         t          ¦   «         j        di |¤Ž d S )Nc                 óL   •— g | ] }t          |d z   ‰j        z  ¦  «        rdnd‘Œ!S )r   Úsliding_attentionÚfull_attention)Úboolr6   )Ú.0ÚiÚselfs     €úk/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/afmoe/configuration_afmoe.pyú
<listcomp>z-AfmoeConfig.__post_init__.<locals>.<listcomp>h   sI   ø€ ð  ð  ð  àõ (,¨Q°©U°dÔ6UÑ,UÑ'VÔ'VÐlÐ#Ð#Ð\lð ð  ð  ó    © )r8   Úranger   r"   r!   ÚsuperÚ__post_init__)rF   ÚkwargsÚ	__class__s   ` €rG   rM   zAfmoeConfig.__post_init__f   s|   øø€ ØÔÐ#ð ð  ð  ð  å˜tÔ5Ñ6Ô6ð ñ  ô  ˆDÔð
 Ô#Ð+Ø'+Ô'?ˆDÔ$à�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'rI   )0Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_pp_planÚbase_model_ep_planr   ÚintÚ__annotations__r   r   r   r   r   r!   r"   r$   r&   Ústrr'   r(   Úfloatr)   r*   rC   r+   r,   r   Údictr.   r0   r2   r3   r4   r6   r7   r8   Úlistr9   r:   r;   r<   r=   r>   rM   Ú__classcell__)rO   s   @rG   r   r      só  ø€ € € € € € ðð ð. €JØ#4Ð"5Ðð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ðð Ðð  +Ø-;Ø*8Ø 0ð	ð Ðð €J�ÐÐÑØ€K�ÐÐÑØ!Ð�sÐ!Ð!Ñ!Ø!%Ð˜3Ð%Ð%Ñ%ØÐ�sÐÐÑØ#$Ð�c˜D‘jÐ$Ð$Ñ$Ø!Ð˜Ð!Ð!Ñ!Ø&*Ð˜˜t™Ð*Ð*Ñ*Ø€Hˆc�D‰jÐÐÑØ€J�ÐÐÑØ#(Ð˜SÐ(Ð(Ñ(Ø#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØ %Ð˜Ð%Ð%Ñ%Ø48€O�^ dÑ*¨TÑ1Ð8Ð8Ñ8Ø €K��t‘Ð Ð Ñ Ø&'Ð˜˜t™Ð'Ð'Ñ'Ø%&Ð˜˜d™
Ð&Ð&Ñ&Ø #€K�˜‘Ð#Ð#Ñ#Ø!&Ð˜$Ð&Ð&Ñ&Ø-.Ð  d¡
Ð.Ð.Ñ.Ø!%€N�C˜$‘JÐ%Ð%Ñ%Ø$(€K��c”˜TÑ!Ð(Ð(Ñ(Ø,/Ð�u˜s‘{ TÑ)Ð/Ð/Ñ/Ø$€K�˜‘Ð$Ð$Ñ$Ø+/€L�#˜˜Sœ	‘/ DÑ(Ð/Ð/Ñ/Ø#€L�#˜‘*Ð#Ð#Ñ#Ø#€L�#˜‘*Ð#Ð#Ñ#Ø €N�DÐ Ð Ñ ð
(ð 
(ð 
(ð 
(ð 
(ð 
(ð 
(ð 
(ð 
(rI   r   N)rS   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr   r   Ú__all__rJ   rI   rG   ú<module>rd      sÊ   ðð  Ð à .Ð .Ð .Ð .Ð .Ð .à 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø #Ð #Ð #Ð #Ð #Ð #ð Ø€ðð 'ðñ ô ðQ(ð Q(ð Q(ð Q(ð Q(Ð"ñ Q(ô Q(ñô ñ „ðQ(ðh ˆ/€€€rI   