§
    ‚Štj¥p  ã                   ó¦  — d Z ddlmZ ddlmZmZ ddlZddlmc m	Z
 ddlZddlmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZmZ ddlmZ ddlmZ ddlmZmZ ddl m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z* ddl+m,Z,m-Z- ddl.m/Z/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5m6Z6 ddl7m8Z8m9Z9  e$d¬¦  «        e G d„ de*¦  «        ¦   «         ¦   «         Z: G d„ de-¦  «        Z; G d „ d!e9¦  «        Z< G d"„ d#ej=        ¦  «        Z> G d$„ d%ej=        ¦  «        Z? G d&„ d'e3¦  «        Z@ G d(„ d)e/¦  «        ZA G d*„ d+ej=        ¦  «        ZB G d,„ d-ej=        ¦  «        ZC G d.„ d/ej=        ¦  «        ZD G d0„ d1e8¦  «        ZE G d2„ d3ej=        ¦  «        ZF G d4„ d5e0¦  «        ZGe$ G d6„ d7e,¦  «        ¦   «         ZH e$d¬¦  «         G d8„ d9e(eG¦  «        ¦   «         ZIg d:¢ZJdS );zPyTorch Zaya model.é    )ÚCallable)ÚAnyÚLiteralN)Ústrict)Únn)Úinité   )ÚCacheÚDynamicCache)ÚPreTrainedConfig)Úcreate_causal_maskÚcreate_recurrent_attention_maskÚ!create_sliding_window_causal_mask)ÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONS)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstring)ÚOutputRecorderé   )ÚAfmoeForCausalLM)ÚLagunaConfig)ÚLagunaModelÚLagunaRotaryEmbedding)ÚLlamaDecoderLayerÚLlamaPreTrainedModelÚ	repeat_kv)ÚPhi3Attention)Úapply_rotary_pos_embÚeager_attention_forward)ÚQwen3MoeExpertsÚQwen3MoeRMSNormzZyphra/ZAYA1-8B)Ú
checkpointc                   ó  — e Zd ZU dZdZdZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZedz  ed<   dZedz  ed<   d	Zedz  ed<   dZeee         z  dz  ed<   dZeed<   dZeed<   dZeed<   dZeed<   d	Zeed<   d	Zeed<    e¦   «         Z e¦   «         Z e¦   «         Z e¦   «         Z e¦   «         Z e¦   «         Z  e¦   «         Z! e¦   «         Z" e¦   «         Z# e¦   «         Z$ e¦   «         Z%d „ Z&d!„ Z'd"„ Z(dS )#Ú
ZayaConfiga€  
    lm_head_bias (`bool`, *optional*, defaults to `False`):
        Whether to add a bias to the language modeling head.
    router_hidden_size (`int`, *optional*, defaults to 256):
        Hidden size used by the ZAYA router.
    cca_time0 (`int`, *optional*, defaults to 2):
        First temporal parameter of the CCA projection.
    cca_time1 (`int`, *optional*, defaults to 2):
        Second temporal parameter of the CCA projection.

    ```python
    >>> from transformers import ZayaConfig, ZayaModel

    >>> configuration = ZayaConfig()
    >>> model = ZayaModel(configuration)

    >>> configuration = model.config
    ```
    Úzayai€  Ú
vocab_sizei   Úmoe_intermediate_sizeé   Únum_attention_headsr   Únum_key_value_headsTÚtie_word_embeddingsgñhãˆµøä>Úrms_norm_epsNÚsliding_windowr   Úpad_token_idÚbos_token_idéj   Úeos_token_idé   Únum_experts_per_toké   Únum_expertsFÚlm_head_biasé   Úrouter_hidden_sizeÚ	cca_time0Ú	cca_time1c                 óÂ   — | j         €dg| j        z  nt          | j         ¦  «        | _         ddddœddddœdœ}| j        €|| _        t	          j        | fi |¤ddd	hi¤Ž d S )
NÚhybridÚdefaultg    ÐSAç      à?)Ú	rope_typeÚ
rope_thetaÚpartial_rotary_factorg     ˆÃ@©r?   Úhybrid_slidingÚignore_keys_at_rope_validationrF   )Úlayer_typesÚnum_hidden_layersÚlistÚrope_parametersr   Ú__post_init__)ÚselfÚkwargsÚdefault_rope_paramss      úc/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/zaya/modular_zaya.pyrL   zZayaConfig.__post_init__n   s«   € ØBFÔBRÐBZ˜H˜:¨Ô(>Ñ>Ð>Õ`dÐeiÔeuÑ`vÔ`vˆÔð 'Ø)Ø),ðð ð 'Ø&Ø),ðð ðZ
ð Z
Ðð ÔÐ'Ø#6ˆDÔ åÔ& tÐsÐs¨vÐsÐsÐW_ÐaqÐVrÐsÐsÐsÐsÐsÐsó    c                 ó   — |S ©N© )rM   rN   s     rP   Úconvert_rope_params_to_dictz&ZayaConfig.convert_rope_params_to_dict‚   s   € àˆrQ   c                 óÀ   — | j         dk    rt          d¦  «        ‚| j        | j        z  dk    rt          d¦  «        ‚d| j        v r| j        €t          d¦  «        ‚d S d S )Nr5   z5ZAYA currently supports `num_experts_per_tok=1` only.r   zB`num_attention_heads` must be a multiple of `num_key_value_heads`.rF   zJ`sliding_window` must be set when `layer_types` contains `hybrid_sliding`.)r6   Ú
ValueErrorr,   r-   rH   r0   )rM   s    rP   Úvalidate_architecturez ZayaConfig.validate_architecture†   sw   € ØÔ# qÒ(Ð(ÝÐTÑUÔUÐUØÔ# dÔ&>Ñ>À!ÒCÐCÝÐaÑbÔbÐbØ˜tÔ/Ð/Ð/°DÔ4GÐ4OÝÐiÑjÔjÐjð 0Ð/Ð4OÐ4OrQ   ))Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typer)   ÚintÚ__annotations__r*   r,   r-   r.   Úboolr/   Úfloatr0   r1   r2   r4   rJ   r6   r8   r9   r;   r<   r=   ÚAttributeErrorÚbase_model_tp_planÚbase_model_pp_planÚintermediate_sizeÚshared_expert_intermediate_sizeÚrouter_aux_loss_coefÚnum_attention_heads_per_layerÚgatingÚmlp_layer_typesÚmoe_routed_scaling_factorÚ moe_apply_router_weight_on_inputÚmoe_router_logit_softcappingrL   rU   rX   rT   rQ   rP   r'   r'   3   s  € € € € € € ðð ð( €Jà€J�ÐÐÑØ!%Ð˜3Ð%Ð%Ñ%Ø Ð˜Ð Ð Ñ Ø Ð˜Ð Ð Ñ Ø $Ð˜Ð$Ð$Ñ$Ø€L�%ÐÐÑØ!%€N�C˜$‘JÐ%Ð%Ñ%Ø €L�#˜‘*Ð Ð Ñ Ø €L�#˜‘*Ð Ð Ñ Ø+.€L�#˜˜Sœ	‘/ DÑ(Ð.Ð.Ñ.à Ð˜Ð Ð Ñ Ø€K�ÐÐÑà€L�$ÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø€IˆsÐÐÑØ€IˆsÐÐÑð
 (˜Ñ)Ô)ÐØ'˜Ñ)Ô)ÐØ&˜Ñ(Ô(ÐØ&4 nÑ&6Ô&6Ð#Ø)˜>Ñ+Ô+ÐØ$2 NÑ$4Ô$4Ð!Øˆ^ÑÔ€FØ$�nÑ&Ô&€OØ . Ñ 0Ô 0ÐØ'5 ~Ñ'7Ô'7Ð$Ø#1 >Ñ#3Ô#3Ð ðtð tð tð(ð ð ðkð kð kð kð krQ   r'   c                   ó   — e Zd ZdS )ÚZayaRotaryEmbeddingN©rY   rZ   r[   rT   rQ   rP   ro   ro   �   ó   € € € € € Ø€DrQ   ro   c                   ó   — e Zd ZdS )ÚZayaRMSNormNrp   rT   rQ   rP   rs   rs   “   rq   rQ   rs   c                   ód   ‡ — e Zd ZdZdedefˆ fd„Z	 d
dej        de	dz  dej        dz  fd	„Z
ˆ xZS )ÚZayaCCAProjectionav  
    Projects hidden states into attention q/k/v states with ZAYA's Compressed Convolutional Attention (CCA) path.
    See https://huggingface.co/papers/2510.04476.

    This follows the usual q/k/v projection flow, with three ZAYA-specific changes: q/k are mixed by a causal 1D
    convolution, q/k keep residual projection paths, and v uses a delayed recurrent state.
    ÚconfigÚ	layer_idxc                 ó¤  •— t          ¦   «                              ¦   «          || _        || _        |j        | _        |j        | _        |j        | _        | j        dz
  | j        dz
  z   | _	        |j
        | _
        |j        | _        |j        | _        | j        | j
        z  | _        | j        | j        z  }| j
        | j        z  }t          j        | j        || j        j        ¬¦  «        | _        t          j        | j        || j        j        ¬¦  «        | _        t          j        | j        |dz  | j        j        ¬¦  «        | _        t          j        | j        |dz  | j        j        ¬¦  «        | _        ||z   }t          j        ||| j        |dd¬¦  «        | _        t          j        ||| j        | j
        | j        z   dd¬¦  «        | _        d S )Nr5   ©Úbiasr   r   )Úin_channelsÚout_channelsÚkernel_sizeÚgroupsÚpaddingÚstride)ÚsuperÚ__init__rv   rw   Úhidden_sizer<   Údepthwise_kernel_sizer=   Úgrouped_kernel_sizeÚconv_kernel_sizer-   r,   Úhead_dimÚnum_key_value_groupsr   ÚLinearÚattention_biasÚq_projÚk_projÚv_proj_currentÚv_proj_delayedÚConv1dÚconv_qk_depthwiseÚconv_qk_grouped)rM   rv   rw   Úquery_hidden_sizeÚkey_value_hidden_sizeÚconv_channelsÚ	__class__s         €rP   r‚   zZayaCCAProjection.__init__    sÈ  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒà!Ô-ˆÔà%+Ô%5ˆÔ"Ø#)Ô#3ˆÔ Ø!%Ô!;¸aÑ!?ÀDÔD\Ð_`ÑD`Ñ aˆÔà#)Ô#=ˆÔ Ø#)Ô#=ˆÔ ØœˆŒØ$(Ô$<ÀÔ@XÑ$XˆÔ!à Ô4°t´}ÑDÐØ $Ô 8¸4¼=Ñ HÐå”i Ô 0Ð2CÈ$Ì+ÔJdÐeÑeÔeˆŒÝ”i Ô 0Ð2GÈdÌkÔNhÐiÑiÔiˆŒÝ œi¨Ô(8Ð:OÐSTÑ:TÐ[_Ô[fÔ[uÐvÑvÔvˆÔÝ œi¨Ô(8Ð:OÐSTÑ:TÐ[_Ô[fÔ[uÐvÑvÔvˆÔà-Ð0AÑAˆÝ!#¤Ø%Ø&ØÔ2Ø ØØð"
ñ "
ô "
ˆÔõ  "œyØ%Ø&ØÔ0ØÔ,¨tÔ/GÑGØØð 
ñ  
ô  
ˆÔÐÐrQ   NÚhidden_statesÚpast_key_valuesÚ	conv_maskc                 ó  — |�*||d d …d d …d f                               |j        ¦  «        z  }|j        d d…         }g |¢d‘| j        ‘R }|                      |¦  «        }|                      |¦  «        }t          j        ||gd¬¦  «        } |j        |Ž }	 |j        |Ž  	                    dd¦  «        }
t          |
| j        ¦  «         	                    dd¦  «        }
|	|
z   dz  }	 |	j        g |¢d‘| j        ‘| j        ‘R Ž                      d¬¦  «        }
| 	                    dd¦  «        }|d uo|                     | j        ¦  «        }|r6|j        | j                 j        d         }t          j        ||gd¬¦  «        }nt#          j        || j        df¦  «        }|�W|d| j         d …f         }t#          j        || j        |j        d         z
  df¦  «        }|                     || j        ¦  «         |                      |¦  «        }|                      |¦  «         	                    dd¦  «        }|	j        d         |	j        d         z  } |dd |…f         j        |Ž |	z   } |d|d …f         j        |Ž |
z   }|                      |¦  «        }|                      |¦  «        }|r1|j        | j                 j        d                              d¦  «        }n5|                      |                     |d         d| j        ¦  «        ¦  «        }t          j        ||d d …d d…f         gd¬¦  «        }|�(|                     |d d …dd d …f         | j        ¦  «          t          j        ||gd¬¦  «        j        |Ž }|||fS )	Néÿÿÿÿ©Údimr5   r   rA   éþÿÿÿr   .)ÚtoÚdtypeÚshaper‡   r‹   rŒ   ÚtorchÚcatÚviewÚ	transposer   rˆ   ÚmeanÚhas_previous_staterw   ÚlayersÚconv_statesÚFÚpadr†   Úupdate_conv_stater�   r‘   r�   rŽ   Úrecurrent_statesÚ	unsqueezeÚ	new_zerosrƒ   Úupdate_recurrent_state)rM   r–   r—   r˜   Úinput_shapeÚhidden_shapeÚprojected_queriesÚprojected_keysÚ	qk_statesÚquery_residualÚkey_residualÚuse_precomputed_statesÚcached_qk_statesÚnew_conv_stater’   ÚqueryÚkeyÚvalue_currentÚdelayed_v_stateÚrecurrent_v_stateÚvalue_delayedÚvalues                         rP   ÚforwardzZayaCCAProjection.forwardÊ   sä  € ð Ð Ø)¨I°a°a°a¸¸¸¸D°jÔ,A×,DÒ,DÀ]ÔEXÑ,YÔ,YÑYˆMà#Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà ŸKšK¨Ñ6Ô6ÐØŸš ]Ñ3Ô3ˆÝ”IÐ0°.ÐAÀrÐJÑJÔJˆ	à/Ð*Ô/°Ð>ˆØ*�~Ô*¨LÐ9×CÒCÀAÀqÑIÔIˆÝ  ¨tÔ/HÑIÔI×SÒSÐTUÐWXÑYÔYˆØ(¨<Ñ7¸3Ñ>ˆØ*�~Ô*Ðf¨KÐf¸Ðf¸TÔ=VÐfÐX\ÔXeÐfÐfÐf×kÒkÐprÐkÑsÔsˆà×'Ò'¨¨1Ñ-Ô-ˆ	Ø!0¸Ð!<Ð!sÀ×AcÒAcÐdhÔdrÑAsÔAsÐØ!ð 	EØ.Ô5°d´nÔEÔQÐRSÔTÐÝœ	Ð#3°YÐ"?ÀRÐHÑHÔHˆIˆIåœ˜i¨$Ô*?ÀÐ)CÑDÔDˆIàÐ&Ø& s¨TÔ-BÐ,BÐ,DÐ,DÐ'DÔEˆNÝœU >°DÔ4IÈNÔL`ÐacÔLdÑ4dÐfgÐ3hÑiÔiˆNØ×-Ò-¨n¸d¼nÑMÔMÐMà×*Ò*¨9Ñ5Ô5ˆ	Ø×(Ò(¨Ñ3Ô3×=Ò=¸aÀÑCÔCˆ	à*Ô0°Ô4°~Ô7KÈBÔ7OÑOÐØ7�	˜#Ð1Ð 1Ð1Ð1Ô2Ô7¸ÐFÈÑWˆØ5ˆi˜Ð.Ð/Ð/Ð/Ô0Ô5°|ÐDÀ|ÑSˆð ×+Ò+¨MÑ:Ô:ˆØ×-Ò-¨mÑ<Ô<ˆØ!ð 	rØ /Ô 6°t´~Ô FÔ WÐXYÔ Z× dÒ dÐefÑ gÔ gÐÐà $× 3Ò 3°M×4KÒ4KÈKÐXYÌNÐ\]Ð_cÔ_oÑ4pÔ4pÑ qÔ qÐÝœ	Ð#4°oÀaÀaÀaÈÈ"ÈÀfÔ6MÐ"NÐTUÐVÑVÔVˆàÐ&Ø×2Ò2°?À1À1À1ÀbÈ!È!È!À8Ô3LÈdÌnÑ]Ô]Ð]àF•”	˜=¨-Ð8¸bÐAÑAÔAÔFÈÐUˆà�c˜5Ð Ð rQ   rS   )rY   rZ   r[   r\   r'   r^   r‚   r¡   ÚTensorr
   rÁ   Ú__classcell__©r•   s   @rP   ru   ru   —   s�   ø€ € € € € ðð ð(
˜zð (
°cð (
ð (
ð (
ð (
ð (
ð (
ð\ *.ð	9!ð 9!à”|ð9!ð  ™ð9!ð ”< $Ñ&ð	9!ð 9!ð 9!ð 9!ð 9!ð 9!ð 9!ð 9!rQ   ru   c                   ót   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        deej        ej        f         fd„Z	ˆ xZ
S )Ú
ZayaQKNormzf
    L2-normalizes q/k states to sqrt(head_dim) and applies ZAYA's learned per-KV-head key scale.
    rv   c                 óÆ   •— t          ¦   «                              ¦   «          |j        dz  | _        t	          j        t          j        |j        ¦  «        ¦  «        | _	        d S )NrA   )
r�   r‚   r‡   Úhead_dim_scaler   Ú	Parameterr¡   Úzerosr-   Útemp©rM   rv   r•   s     €rP   r‚   zZayaQKNorm.__init__  sJ   ø€ Ý‰Œ×ÒÑÔÐØ$œo¨sÑ2ˆÔÝ”L¥¤¨VÔ-GÑ!HÔ!HÑIÔIˆŒ	ˆ	ˆ	rQ   Úquery_statesÚ
key_statesÚreturnc                 óJ  — t          j        |j        ¦  «        j        }|| j        |                     ddd¬¦  «                             |¦  «        z  z  }|| j        |                     ddd¬¦  «                             |¦  «        z  z  }|| j        d d d d …d f         z  }||fS )Nr   rš   T)Úprœ   Úkeepdim)r¡   ÚfinforŸ   ÚepsrÈ   ÚnormÚ	clamp_minrË   )rM   rÍ   rÎ   Únorm_epss       rP   rÁ   zZayaQKNorm.forward  s³   € Ý”;˜|Ô1Ñ2Ô2Ô6ˆØ#ØÔ ,×"3Ò"3°a¸RÈÐ"3Ñ"NÔ"N×"XÒ"XÐYaÑ"bÔ"bÑbñ
ˆð  ØÔ *§/¢/°A¸2Àt /Ñ"LÔ"L×"VÒ"VÐW_Ñ"`Ô"`Ñ`ñ
ˆ
ð   $¤)¨D°$¸¸¸¸4Ð,?Ô"@Ñ@ˆ
Ø˜ZÐ'Ð'rQ   )rY   rZ   r[   r\   r'   r‚   r¡   rÂ   ÚtuplerÁ   rÃ   rÄ   s   @rP   rÆ   rÆ     s”   ø€ € € € € ðð ðJ˜zð Jð Jð Jð Jð Jð Jð
	( E¤Lð 	(¸e¼lð 	(ÈuÐUZÔUaÐchÔcoÐUoÔOpð 	(ð 	(ð 	(ð 	(ð 	(ð 	(ð 	(ð 	(rQ   rÆ   c                   óÖ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej        dee	e
f         dz  dedz  deej        ej        f         dz  d	ee         d
eej        ej        dz  f         fd„Zˆ xZS )ÚZayaAttentionrv   rw   c                 ó˜  •— t          ¦   «                              ||¦  «         ~|j        |         | _        | j        dk    r|j        nd | _        |j        | _        |j        | _        t          j        |j        | j	        z  |j        |j
        ¬¦  «        | _        t          |¦  «        | _        t          | j        |¬¦  «        | _        d S )NrF   ry   )rv   rw   )r�   r‚   rH   Ú
layer_typer0   rƒ   r,   r   r‰   r‡   rŠ   Úo_projrÆ   Úqk_normru   rv   Úqkv_proj)rM   rv   rw   Úop_sizer•   s       €rP   r‚   zZayaAttention.__init__  sÆ   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+ØØ Ô,¨YÔ7ˆŒØ7;´ÐJZÒ7ZÐ7Z˜fÔ3Ð3Ð`dˆÔØ!Ô-ˆÔØ#)Ô#=ˆÔ å”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒõ " &Ñ)Ô)ˆŒÝ)Ø”;Øð
ñ 
ô 
ˆŒˆˆrQ   Nr–   Úattention_maskr—   Úposition_embeddingsrN   rÏ   c                 óÎ  — |j         d d…         }|pi }|                     d¦  «        }|                     d¦  «        }	|                      |||	¦  «        \  }
}}|                      |
|¦  «        \  }
}|
                     dd¦  «        }
|                     dd¦  «        }|                     dd¦  «        }|\  }}t          |
|||¦  «        \  }
}|�|                     ||| j        ¦  «        \  }}t          j	        | j
        j        t          ¦  «        } || |
|||f| j        sdn| j        | j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž }|                      |¦  «        }||fS )Nrš   ÚcausalÚconvr5   r   ç        )ÚdropoutÚscalingr0   )r    Úgetrß   rÞ   r¤   r!   Úupdaterw   r   Úget_interfacerv   Ú_attn_implementationr"   ÚtrainingÚattention_dropoutrè   r0   ÚreshaperÝ   )rM   r–   rá   r—   râ   rN   r°   Úmask_mappingÚcausal_maskr˜   rÍ   rÎ   Úvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                     rP   rÁ   zZayaAttention.forward.  s·  € ð $Ô)¨#¨2¨#Ô.ˆà%Ð+¨ˆØ"×&Ò& xÑ0Ô0ˆØ ×$Ò$ VÑ,Ô,ˆ	ð 26·²¸}ÈoÐ_hÑ1iÔ1iÑ.ˆ�j ,Ø#'§<¢<°¸jÑ#IÔ#IÑ ˆ�jà#×-Ò-¨a°Ñ3Ô3ˆØ×)Ò)¨!¨QÑ/Ô/ˆ
Ø#×-Ò-¨a°Ñ3Ô3ˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}ÐH�C�C°$Ô2HØ”LØÔ.ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;ˆØ—k’k +Ñ.Ô.ˆà˜LÐ(Ð(rQ   )NNN©rY   rZ   r[   r'   r^   r‚   r¡   rÂ   ÚdictÚstrr   r
   rØ   r   r   rÁ   rÃ   rÄ   s   @rP   rÚ   rÚ     sè   ø€ € € € € ð
˜zð 
°cð 
ð 
ð 
ð 
ð 
ð 
ð( 15Ø(,ØHLð.)ð .)à”|ð.)ð ˜S #˜Xœ¨Ñ-ð.)ð  ™ð	.)ð
 # 5¤<°´Ð#=Ô>ÀÑEð.)ð Ð+Ô,ð.)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð.)ð .)ð .)ð .)ð .)ð .)ð .)ð .)rQ   rÚ   c                   óì   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 ddej        dej        dz  dee	e
f         dz  dedz  d	eej        ej        f         dz  d
ee         deej        ej        dz  f         fd„Zˆ xZS )ÚZayaDecoderLayerrv   rw   c                 óÚ   •— t          ¦   «                              ||¦  «         t          ||¦  «        | _        t	          |j        ¦  «        | _        t	          |j        ¦  «        | _        d S rS   )r�   r‚   ÚZayaSparseMoeBlockÚmlpÚZayaResidualScalingrƒ   Úpost_attention_residual_scaleÚpost_mlp_residual_scale©rM   rv   rw   r•   s      €rP   r‚   zZayaDecoderLayer.__init__`  sZ   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+Ý% f¨iÑ8Ô8ˆŒÝ-@ÀÔASÑ-TÔ-TˆÔ*Ý':¸6Ô;MÑ'NÔ'NˆÔ$Ð$Ð$rQ   Nr–   Úprev_router_hidden_statesrá   r—   râ   rN   rÏ   c                 ó¢  — |}|                       |                     | j         j        j        ¬¦  «        ¦  «        } | j        d||||dœ|¤Ž\  }}|                      ||¦  «        }|                      |                     | j        j        j        ¬¦  «        ¦  «        }|                      ||¦  «        \  }}|                      ||¦  «        }||fS )N©rŸ   )r–   rá   r—   râ   rT   )	Úinput_layernormrž   ÚweightrŸ   Ú	self_attnr  Úpost_attention_layernormrÿ   r  )	rM   r–   r  rá   r—   râ   rN   ÚresidualÚ_s	            rP   rÁ   zZayaDecoderLayer.forwardf  sð   € ð !ˆð ×,Ò,¨X¯[ª[¸tÔ?SÔ?ZÔ?`¨[Ñ-aÔ-aÑbÔbˆà)˜4œ>ð 
Ø'Ø)Ø+Ø 3ð	
ð 
ð
 ð
ð 
Ñˆ�qð ×5Ò5°mÀXÑNÔNˆØ×5Ò5°h·k²kÈÔHeÔHlÔHr°kÑ6sÔ6sÑtÔtˆà37·8²8ØØ%ñ4
ô 4
Ñ0ˆÐ0ð
 ×4Ò4°]ÀHÑMÔMˆàÐ7Ð7Ð7rQ   )NNNNrø   rÄ   s   @rP   rü   rü   _  s  ø€ € € € € ðO˜zð O°cð Oð Oð Oð Oð Oð Oð :>Ø04Ø(,ØHLð 8ð  8à”|ð 8ð $)¤<°$Ñ#6ð 8ð ˜S #˜Xœ¨Ñ-ð	 8ð
  ™ð 8ð # 5¤<°´Ð#=Ô>ÀÑEð 8ð Ð+Ô,ð 8ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð 8ð  8ð  8ð  8ð  8ð  8ð  8ð  8rQ   rü   c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )r   rƒ   c                 ó   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        t          j        t	          j        |¦  «        ¦  «        | _        t          j        t	          j        |¦  «        ¦  «        | _	        t          j        t	          j        |¦  «        ¦  «        | _
        d S rS   )r�   r‚   r   rÉ   r¡   ÚonesÚhidden_states_scalerÊ   Úhidden_states_biasÚresidual_scaleÚresidual_bias)rM   rƒ   r•   s     €rP   r‚   zZayaResidualScaling.__init__Š  s�   ø€ Ý‰Œ×ÒÑÔÐÝ#%¤<µ´
¸;Ñ0GÔ0GÑ#HÔ#HˆÔ Ý"$¤,­u¬{¸;Ñ/GÔ/GÑ"HÔ"HˆÔÝ œl­5¬:°kÑ+BÔ+BÑCÔCˆÔÝœ\­%¬+°kÑ*BÔ*BÑCÔCˆÔÐÐrQ   r–   r  c                 óT   — || j         z   | j        z  }|| j        z   | j        z  }||z   S rS   )r  r  r  r  )rM   r–   r  s      rP   rÁ   zZayaResidualScaling.forward‘  s7   € à&¨Ô)@Ñ@ÀDÔD\Ñ\ˆØ˜tÔ1Ñ1°TÔ5HÑHˆØ˜xÑ'Ð'rQ   )	rY   rZ   r[   r^   r‚   r¡   rÂ   rÁ   rÃ   rÄ   s   @rP   r   r   ‰  sq   ø€ € € € € ðD Cð Dð Dð Dð Dð Dð Dð( U¤\ð (¸U¼\ð (ð (ð (ð (ð (ð (ð (ð (rQ   r   c                   óP   ‡ — e Zd Zdededefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚZayaRouterMLPrƒ   r8   r/   c                 óL  •— t          ¦   «                              ¦   «          t          ||¬¦  «        | _        t	          j        ||d¬¦  «        | _        t	          j        ||d¬¦  «        | _        t	          j        ||d¬¦  «        | _        t	          j	        ¦   «         | _
        d S )N)rÔ   Try   F)r�   r‚   rs   rÕ   r   r‰   Úfc1Úfc2Úout_projÚGELUÚact_fn)rM   rƒ   r8   r/   r•   s       €rP   r‚   zZayaRouterMLP.__init__™  s†   ø€ Ý‰Œ×ÒÑÔÐÝ °Ð>Ñ>Ô>ˆŒ	Ý”9˜[¨+¸DÐAÑAÔAˆŒÝ”9˜[¨+¸DÐAÑAÔAˆŒÝœ	 +¨{ÀÐGÑGÔGˆŒÝ”g‘i”iˆŒˆˆrQ   r–   rÏ   c                 óö   — |                       |¦  «        }|                      |                      |¦  «        ¦  «        }|                      |                      |¦  «        ¦  «        }|                      |¦  «        S rS   )rÕ   r  r  r  r  )rM   r–   s     rP   rÁ   zZayaRouterMLP.forward¡  s_   € ØŸ	š	 -Ñ0Ô0ˆØŸš D§H¢H¨]Ñ$;Ô$;Ñ<Ô<ˆØŸš D§H¢H¨]Ñ$;Ô$;Ñ<Ô<ˆØ�}Š}˜]Ñ+Ô+Ð+rQ   )
rY   rZ   r[   r^   ra   r‚   r¡   rÂ   rÁ   rÃ   rÄ   s   @rP   r  r  ˜  sx   ø€ € € € € ð  Cð  °cð  Èð  ð  ð  ð  ð  ð  ð, U¤\ð ,°e´lð ,ð ,ð ,ð ,ð ,ð ,ð ,ð ,rQ   r  c                   ó–   ‡ — e Zd Zdeddfˆ fd„Z	 ddej        dej        dz  deej        ej        ej        ej        f         fd„Zˆ xZ	S )	Ú
ZayaRouterrw   rÏ   Nc                 óŒ  •— t          ¦   «                              ¦   «          || _        |j        | _        || _        |j        | _        |j        dz   | _        |j        | _        |j	        | _	        t          j        | j        | j	        d¬¦  «        | _        | j        dk    | _        | j        r0t          j        t          j        | j	        ¦  «        ¦  «        | _        t%          | j	        | j        |j        ¦  «        | _        |                      dt          j        | j        t          j        ¬¦  «        ¦  «         d| j        d<   d S )	Nr5   Try   r   Úbalancing_biasesr  ç      ð¿rš   )r�   r‚   rv   rƒ   rw   r8   Únum_router_classesr6   Útop_kr;   r   r‰   Ú	down_projÚuse_edarÉ   r¡   r  Úrouter_states_scaler  r/   Ú
router_mlpÚregister_bufferrÊ   Úfloat32r!  r  s      €rP   r‚   zZayaRouter.__init__©  s  ø€ õ
 	‰Œ×ÒÑÔÐàˆŒØ!Ô-ˆÔØ"ˆŒà!Ô-ˆÔà"(Ô"4°qÑ"8ˆÔØÔ/ˆŒ
Ø"(Ô";ˆÔåœ 4Ô#3°TÔ5LÐSWÐXÑXÔXˆŒà”~¨Ò*ˆŒØŒ<ð 	YÝ')¤|µE´J¸tÔ?VÑ4WÔ4WÑ'XÔ'XˆDÔ$å'¨Ô(?ÀÔAXÐZ`ÔZmÑnÔnˆŒà×ÒÐ/µ´¸TÔ=TÕ\aÔ\iÐ1jÑ1jÔ1jÑkÔkÐkØ$(ˆÔ˜bÑ!Ð!Ð!rQ   r–   Úrouter_statesc                 óü  — d| j         f}|j        d         }|                      |¦  «        }| j        r|�||| j        z  z   }|d d …| d …f                              ¦   «         }|                      |¦  «        }t          j        |d¬¦  «        }| 	                    ¦   «          
                    t          j        ¦  «        | j        z   }	t          j        |	| j         d¬¦  «        \  }
}t          j        |d|¬¦  «        }|| j        j        k    }|                     |d¦  «        }|                     |d¦  «        }|                     d| j        ¦  «        |                     |¦  «        |                     |¦  «        |fS )Nrš   r5   r›   r   )rœ   Úindexr   )r$  r    r%  r&  r'  Úcloner(  r¡   ÚsoftmaxÚdetachrž   r*  r!  ÚtopkÚgatherrv   r8   Úmasked_fillrï   r#  )rM   r–   r+  Úfinal_shapeÚ
seq_lengthÚrouter_hidden_statesÚrouter_hidden_states_nextÚrouter_logitsÚrouter_probsÚbiased_router_probsr  Úrouter_indicesÚskip_experts                rP   rÁ   zZayaRouter.forwardÅ  s}  € ð
 ˜4œ:Ð&ˆØ"Ô(¨Ô+ˆ
à#Ÿ~š~¨mÑ<Ô<ÐàŒ<ð 	c˜MÐ5Ø#7¸-È$ÔJbÑ:bÑ#bÐ à$8¸¸¸¸Z¸K¸L¸L¸Ô$I×$OÒ$OÑ$QÔ$QÐ!ØŸšÐ(<Ñ=Ô=ˆÝ”} ]¸Ð;Ñ;Ô;ˆà*×1Ò1Ñ3Ô3×6Ò6µu´}ÑEÔEÈÔH]Ñ]ÐÝ!œJÐ':¸D¼JÈBÐOÑOÔOÑˆˆ>Ý”| L°a¸~ÐNÑNÔNˆð %¨¬Ô(?Ò?ˆØ#×/Ò/°¸QÑ?Ô?ˆØ'×3Ò3°KÀÑCÔCˆð ×!Ò! " dÔ&=Ñ>Ô>Ø× Ò  Ñ-Ô-Ø×"Ò" ;Ñ/Ô/Ø%ð	
ð 	
rQ   rS   ©
rY   rZ   r[   r^   r‚   r¡   rÂ   rØ   rÁ   rÃ   rÄ   s   @rP   r  r  ¨  sª   ø€ € € € € ð)ð ð)ð 
ð	)ð )ð )ð )ð )ð )ð> .2ð
ð 
à”|ð
ð ”| dÑ*ð
ð 
ˆuŒ|˜Uœ\¨5¬<¸¼ÐEÔ	Fð	
ð 
ð 
ð 
ð 
ð 
ð 
ð 
rQ   r  c                   ó   — e Zd ZdS )ÚZayaExpertsNrp   rT   rQ   rP   r?  r?  ç  rq   rQ   r?  c            
       ó€   ‡ — e Zd Zdefˆ fd„Z	 ddej        dej        dz  deej        ej        dz  f         fd„Zˆ xZ	S )	rþ   rw   c                 óš   •— t          ¦   «                              ¦   «          t          ||¦  «        | _        t	          |¦  «        | _        d S rS   )r�   r‚   r  Úgater?  Úexpertsr  s      €rP   r‚   zZayaSparseMoeBlock.__init__ì  s=   ø€ Ý‰Œ×ÒÑÔÐÝ˜v yÑ1Ô1ˆŒ	Ý" 6Ñ*Ô*ˆŒˆˆrQ   Nr–   r  rÏ   c                 óæ   — |                       ||¬¦  «        \  }}}}|j        \  }}}|                     ||z  |¦  «        }	|                      |	||¦  «        }
|
                     |||¦  «        }
|
|fS )N)r+  )rB  r    r£   rC  )rM   r–   r  r  r9  r;  Ú
batch_sizer5  Úemb_dimÚhidden_states_flatÚexpert_outputs              rP   rÁ   zZayaSparseMoeBlock.forwardñ  s—   € ð FJÇYÂYØÐ)Bð FOñ F
ô F
ÑBˆˆ<˜Ð)Bð +8Ô*=Ñ'ˆ
�J Ø*×/Ò/°
¸ZÑ0GÈÑQÔQÐØŸšÐ%7¸ÈÑVÔVˆØ%×*Ò*¨:°zÀ7ÑKÔKˆàÐ7Ð7Ð7rQ   rS   r=  rÄ   s   @rP   rþ   rþ   ë  sš   ø€ € € € € ð+¨#ð +ð +ð +ð +ð +ð +ð :>ð8ð 8à”|ð8ð $)¤<°$Ñ#6ð8ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð	8ð 8ð 8ð 8ð 8ð 8ð 8ð 8rQ   rþ   c                   óp   — e Zd ZU eed<   dZ eed¬¦  «        ee	dœZ
 ej        ¦   «         d„ ¦   «         ZdS )ÚZayaPreTrainedModelrv   Fr   )r-  )r8  r–   Ú
attentionsc                 óæ  — t          j        | |¦  «         t          |t          ¦  «        rft	          j        |j        ¦  «         t	          j        |j        ¦  «         t	          j        |j	        ¦  «         t	          j        |j
        ¦  «         d S t          |t          ¦  «        r4t	          j        |j        ¦  «         t	          j        |j        ¦  «         d S t          |t          ¦  «        rt	          j        |j        ¦  «         d S t          |t           ¦  «        rE|j        rt	          j        |j        ¦  «         t	          j        |j        ¦  «         d|j        d<   d S t          |t(          ¦  «        rF| j        j        }t	          j        |j        d|¬¦  «         t	          j        |j        d|¬¦  «         d S t          |t4          ¦  «        r›|j        D ]•}|j        }|j        |         dk    rt<          |j        |                  } ||j        |¬¦  «        \  }}t?          ||› d�¦  «                              |¦  «         t?          ||› d�¦  «                              |¦  «         Œ”d S d S )	Nr"  rš   ræ   )r¥   Ústdr@   )rÜ   Ú	_inv_freqÚ_original_inv_freq)!r   Ú_init_weightsÚ
isinstancer   r   Úones_r  Úzeros_r  r  r  Ú	ZayaModelÚinput_hidden_states_scaleÚinput_hidden_states_biasrÆ   rË   r  r&  r'  r!  r?  rv   Úinitializer_rangeÚnormal_Úgate_up_projr%  ro   rH   Úcompute_default_rope_parametersrB   r   ÚgetattrÚcopy_)rM   ÚmodulerM  rÜ   Úrope_init_fnÚcurr_inv_freqr  s          rP   rP  z!ZayaPreTrainedModel._init_weights  sk  € åÔ% d¨FÑ3Ô3Ð3Ý�fÕ1Ñ2Ô2ð 	XÝŒJ�vÔ1Ñ2Ô2Ð2ÝŒK˜Ô1Ñ2Ô2Ð2ÝŒJ�vÔ,Ñ-Ô-Ð-ÝŒK˜Ô,Ñ-Ô-Ð-Ð-Ð-Ý˜¥	Ñ*Ô*ð 	XÝŒJ�vÔ7Ñ8Ô8Ð8ÝŒK˜Ô7Ñ8Ô8Ð8Ð8Ð8Ý˜¥
Ñ+Ô+ð 	XÝŒK˜œÑ$Ô$Ð$Ð$Ð$Ý˜¥
Ñ+Ô+ð 	XØŒ~ð 7Ý”
˜6Ô5Ñ6Ô6Ð6ÝŒK˜Ô/Ñ0Ô0Ð0Ø*.ˆFÔ# BÑ'Ð'Ð'Ý˜¥Ñ,Ô,ð 	XØ”+Ô/ˆCÝŒL˜Ô,°3¸CÐ@Ñ@Ô@Ð@ÝŒL˜Ô)°¸Ð=Ñ=Ô=Ð=Ð=Ð=Ý˜Õ 3Ñ4Ô4ð 	XØ$Ô0ð Xð X�
Ø%ÔE�ØÔ# JÔ/°9Ò<Ð<Ý#6°vÔ7GÈ
Ô7SÔ#T�LØ#/ <°´È*Ð#UÑ#UÔ#UÑ �˜qÝ˜ :Ð 8Ð 8Ð 8Ñ9Ô9×?Ò?ÀÑNÔNÐNÝ˜ :Ð AÐ AÐ AÑBÔB×HÒHÈÑWÔWÐWÐWð	Xð 	XðXð XrQ   N)rY   rZ   r[   r'   r_   Ú_can_compile_fullgraphr   r  rü   rÚ   Ú_can_record_outputsr¡   Úno_gradrP  rT   rQ   rP   rJ  rJ    su   € € € € € € ØÐÐÑà"Ðà'˜¨
¸!Ð<Ñ<Ô<Ø)Ø#ðð Ðð €U„]�_„_ðXð Xñ „_ðXð Xð XrQ   rJ  c                   ó²   ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dedz  dej	        dz  d	e
dz  d
ee         defd„Zˆ xZS )rT  rv   c                 ó
  •— t          ¦   «                              |¦  «         t          j        t	          j        |j        ¦  «        ¦  «        | _        t          j        t	          j        |j        ¦  «        ¦  «        | _	        d S rS   )
r�   r‚   r   rÉ   r¡   r  rƒ   rU  rÊ   rV  rÌ   s     €rP   r‚   zZayaModel.__init__/  s_   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý)+¬µe´jÀÔASÑ6TÔ6TÑ)UÔ)UˆÔ&Ý(*¬µU´[ÀÔASÑ5TÔ5TÑ(UÔ(UˆÔ%Ð%Ð%rQ   NÚ	input_idsrá   Úposition_idsr—   Úinputs_embedsÚ	use_cacherN   rÏ   c                 ó  ‡ ‡‡‡‡— |d u |d uz  rt          d¦  «        ‚|€‰                      |¦  «        }|r|€t          ‰ j        ¬¦  «        }‰€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   Š‰ 	                    d¦  «        Št          |x}	t          ¦  «        sL‰ j        |||‰dœŠˆfd„ˆfd„d	œŠˆfd
„t          ‰ j        j        ¦  «        D ¦   «         }	t          di ‰¤Ž|	d<   |Šˆˆˆ fd„t          ‰ j        j        ¦  «        D ¦   «         }
‰‰ j        z   ‰ j        z                       t
          j        ¦  «        Šd }t'          ‰ j        ¦  «        D ]J\  }}‰ j        j        |         } |‰|f|	|         |	                     d¦  «        dœ||
|         dœ|¤Ž\  Š}ŒK‰                      ‰                     ‰ j        j        j        ¬¦  «        ¦  «        Št3          ‰|r|nd ¬¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embeds)rv   r   r5   )Údevice©rv   rg  rá   r—   rf  c                  ó   •— t          di ‰ ¤ŽS ©NrT   ©r   ©Úmask_kwargss   €rP   ú<lambda>z#ZayaModel.forward.<locals>.<lambda>U  s   ø€ Õ"4Ð"CÐ"C°{Ð"CÐ"C€ rQ   c                  ó   •— t          di ‰ ¤ŽS rm  ©r   ro  s   €rP   rq  z#ZayaModel.forward.<locals>.<lambda>V  s   ø€ Õ*KÐ*ZÐ*ZÈkÐ*ZÐ*Z€ rQ   rE   c                 ó2   •— i | ]}| ‰|         ¦   «         “ŒS rT   rT   ©Ú.0rÜ   Úmask_creation_functionss     €rP   ú
<dictcomp>z%ZayaModel.forward.<locals>.<dictcomp>X  s7   ø€ ð #ð #ð #ØFP�
Ð?Ð3°JÔ?ÑAÔAð#ð #ð #rQ   rå   c                 ó@   •— i | ]}|‰                      ‰‰|¦  «        “ŒS rT   )Ú
rotary_emb)rv  rÜ   r–   rf  rM   s     €€€rP   rx  z%ZayaModel.forward.<locals>.<dictcomp>_  s;   ø€ ð 
ð 
ð 
àð ˜Ÿš¨°|ÀZÑPÔPð
ð 
ð 
rQ   )rä   rå   )rá   r—   râ   r  )Úlast_hidden_stater—   rT   )rW   Úembed_tokensr   rv   Úget_seq_lengthr¡   Úaranger    rj  r­   rQ  rù   ÚsetrH   r   rV  rU  rž   r*  Ú	enumerater§   ré   rÕ   r  rŸ   r   )rM   re  rá   rf  r—   rg  rh  rN   Úpast_seen_tokensÚcausal_mask_mappingrâ   r  ÚidxÚdecoder_layerrÜ   r–   rw  rp  s   `  `           @@@rP   rÁ   zZayaModel.forward4  sÖ  øøøøø€ ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå°Ð?Ð-ÅÑFÔFð 	Yàœ+Ø!.Ø"0Ø#2Ø ,ðð ˆKð DÐCÐCÐCØ"ZÐ"ZÐ"ZÐ"Zð'ð 'Ð#ð#ð #ð #ð #ÝTWÐX\ÔXcÔXoÑTpÔTpð#ñ #ô #Ðõ +JÐ*XÐ*XÈKÐ*XÐ*XÐ Ñ'à%ˆð
ð 
ð 
ð 
ð 
ð 
å! $¤+Ô"9Ñ:Ô:ð
ñ 
ô 
Ðð (¨$Ô*GÑGÈ4ÔKiÑi×mÒmÝŒMñ
ô 
ˆð %)Ð!å"+¨D¬KÑ"8Ô"8ð 	ð 	ÑˆC�ØœÔ0°Ô5ˆJð 8E°}ØØ)ð
8ð 2°*Ô=Ø/×3Ò3°FÑ;Ô;ð ð  ð !0Ø$7¸
Ô$Cð
8ð 
8ð ð
8ð 
8Ñ4ˆMÐ4Ð4ð Ÿ	š	 -×"2Ò"2¸¼Ô9IÔ9OÐ"2Ñ"PÔ"PÑQÔQˆå%Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
rQ   )NNNNNN)rY   rZ   r[   r'   r‚   r¡   Ú
LongTensorrÂ   r
   ÚFloatTensorr`   r   r   r   rÁ   rÃ   rÄ   s   @rP   rT  rT  -  s  ø€ € € € € ðV˜zð Vð Vð Vð Vð Vð Vð .2Ø.2Ø04Ø(,Ø26Ø!%ðL
ð L
àÔ# dÑ*ðL
ð œ tÑ+ðL
ð Ô&¨Ñ-ð	L
ð
  ™ðL
ð Ô(¨4Ñ/ðL
ð ˜$‘;ðL
ð Ð+Ô,ðL
ð 
 ðL
ð L
ð L
ð L
ð L
ð L
ð L
ð L
rQ   rT  c                   ój   ‡ — e Zd ZddiZdZ e¦   «         Z e¦   «         Zˆ fd„Ze	dd„¦   «         Z
ˆ xZS )ÚZayaForCausalLMzlm_head.weightzmodel.embed_tokens.weightTc                 óœ   •—  t          ¦   «         j        |fi |¤Ž t          j        |j        |j        | j        j        ¬¦  «        | _        d S )Nry   )	r�   r‚   r   r‰   rƒ   r)   rv   r9   Úlm_head)rM   rv   rN   r•   s      €rP   r‚   zZayaForCausalLM.__init__‹  sI   ø€ Ø�‰ŒÔ˜Ð*Ð* 6Ð*Ð*Ð*Ý”y Ô!3°VÔ5FÈTÌ[ÔMeÐfÑfÔfˆŒˆˆrQ   Nc                 ó¶   ‡‡	— |                       ¦   «         }|||||dœŠ	ˆ	fd„ˆ	fd„dœŠˆfd„t          |j        ¦  «        D ¦   «         }t          di ‰	¤Ž|d<   |S )Nrk  c                  ó   •— t          di ‰ ¤ŽS rm  rn  ro  s   €rP   rq  z;ZayaForCausalLM.create_masks_for_generate.<locals>.<lambda>œ  s   ø€ Õ0Ð?Ð?°;Ð?Ð?€ rQ   c                  ó   •— t          di ‰ ¤ŽS rm  rs  ro  s   €rP   rq  z;ZayaForCausalLM.create_masks_for_generate.<locals>.<lambda>�  s   ø€ Õ&GÐ&VÐ&VÈ+Ð&VÐ&V€ rQ   rE   c                 ó2   •— i | ]}| ‰|         ¦   «         “ŒS rT   rT   ru  s     €rP   rx  z=ZayaForCausalLM.create_masks_for_generate.<locals>.<dictcomp>Ÿ  s7   ø€ ð 
ð 
ð 
ØBLˆJÐ;Ð/°
Ô;Ñ=Ô=ð
ð 
ð 
rQ   rå   rT   )Úget_text_configr  rH   r   )
rv   rg  rá   r—   rf  r  Útext_configrð   rw  rp  s
           @@rP   Úcreate_masks_for_generatez)ZayaForCausalLM.create_masks_for_generate�  s®   øø€ ð ×,Ò,Ñ.Ô.ˆà!Ø*Ø,Ø.Ø(ð
ð 
ˆð @Ð?Ð?Ð?ØVÐVÐVÐVð#
ð #
Ðð
ð 
ð 
ð 
ÝPSÐT_ÔTkÑPlÔPlð
ñ 
ô 
ˆõ  ?ÐMÐMÀÐMÐMˆ�VÑØÐrQ   rS   )rY   rZ   r[   Ú_tied_weights_keysÚ_is_statefulrb   Ú_tp_planÚ_pp_planr‚   Ústaticmethodr‘  rÃ   rÄ   s   @rP   rˆ  rˆ  ƒ  sˆ   ø€ € € € € à*Ð,GÐHÐØ€Làˆ~ÑÔ€HØˆ~ÑÔ€Hðgð gð gð gð gð ðð ð ñ „\ðð ð ð ð rQ   rˆ  )r'   rJ  rT  rˆ  )Kr\   Úcollections.abcr   Útypingr   r   r¡   Útorch.nn.functionalr   Ú
functionalr©   Útorch.utils.checkpointÚhuggingface_hub.dataclassesr   Útorch.nnr   Úcache_utilsr
   r   Úconfiguration_utilsr   Úmasking_utilsr   r   r   Úmodeling_outputsr   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úutils.output_capturingr   Úafmoe.modeling_afmoer   Úlaguna.configuration_lagunar   Úlaguna.modeling_lagunar   r   Úllama.modeling_llamar   r   r   Úphi3.modeling_phi3r    Ú qwen3_5_moe.modeling_qwen3_5_moer!   r"   Úqwen3_moe.modeling_qwen3_moer#   r$   r'   ro   rs   ÚModuleru   rÆ   rÚ   rü   r   r  r  r?  rþ   rJ  rT  rˆ  Ú__all__rT   rQ   rP   ú<module>r°     s@  ðð Ð à $Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ø .Ð .Ð .Ð .Ð .Ð .Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø sÐ sÐ sÐ sÐ sÐ sÐ sÐ sÐ sÐ sØ 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &ðð ð ð ð ð ð ð ð 5Ð 4Ð 4Ð 4Ð 4Ð 4Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ UÐ UÐ UÐ UÐ UÐ UÐ UÐ UÐ UÐ UØ .Ð .Ð .Ð .Ð .Ð .ðð ð ð ð ð ð ð ð LÐ KÐ KÐ KÐ KÐ KÐ KÐ Kð €Ð,Ð-Ñ-Ô-ØðWkð Wkð Wkð Wkð Wk�ñ Wkô Wkñ „ñ .Ô-ðWkðt	ð 	ð 	ð 	ð 	Ð/ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	�/ñ 	ô 	ð 	ðl!ð l!ð l!ð l!ð l!˜œ	ñ l!ô l!ð l!ð^(ð (ð (ð (ð (�”ñ (ô (ð (ð,@)ð @)ð @)ð @)ð @)�Mñ @)ô @)ð @)ðF'8ð '8ð '8ð '8ð '8Ð(ñ '8ô '8ð '8ðT(ð (ð (ð (ð (˜"œ)ñ (ô (ð (ð,ð ,ð ,ð ,ð ,�B”Iñ ,ô ,ð ,ð <
ð <
ð <
ð <
ð <
�”ñ <
ô <
ð <
ð~	ð 	ð 	ð 	ð 	�/ñ 	ô 	ð 	ð8ð 8ð 8ð 8ð 8˜œñ 8ô 8ð 8ð0'Xð 'Xð 'Xð 'Xð 'XÐ.ñ 'Xô 'Xð 'XðT ðR
ð R
ð R
ð R
ð R
�ñ R
ô R
ñ „ðR
ðj €Ð,Ð-Ñ-Ô-ðð ð ð ð Ð&Ð(;ñ ô ñ .Ô-ððDð ð €€€rQ   