§
    ‚Štj)Q  ã                   ó$  — d Z ddlmZ ddlmZmZmZ ddlZddlm	c m
Z ddlmZ ddlm	Z	 ddlmZ dd	lmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddl!m"Z" ddl#m$Z$m%Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5m6Z6m7Z7m8Z8 ddl9m:Z:m;Z; ddl<m=Z=m>Z>  e%j?        e@¦  «        ZA e$d¬¦  «        e G d„ de3¦  «        ¦   «         ¦   «         ZB G d „ d!e8¦  «        ZC G d"„ d#e,¦  «        ZD G d$„ d%e6¦  «        ZE G d&„ d'e:¦  «        ZF G d(„ d)e=¦  «        ZG G d*„ d+e>¦  «        ZHe( G d,„ d-e*¦  «        ¦   «         ZI G d.„ d/e.¦  «        ZJ G d0„ d1e7¦  «        ZK G d2„ d3e0¦  «        ZL G d4„ d5e5¦  «        ZMg d6¢ZNdS )7zPyTorch Laguna model.é    )ÚCallable)ÚAnyÚLiteralÚOptionalN)Ústrict)Únné   )Úinitialization)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONS)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)Úauto_docstringÚlogging)ÚTransformersKwargsÚno_inherit_decoratoré   )ÚAfmoeAttention)ÚGemma3RotaryEmbedding)ÚGlm4MoeLiteDecoderLayer)Ú
LlamaModelÚeager_attention_forward)ÚQwen2MoeConfig)ÚQwen2MoeForCausalLMÚQwen2MoeMLPÚQwen2MoePreTrainedModelÚQwen2MoeRMSNorm)ÚQwen3_5MoeTopKRouterÚapply_rotary_pos_emb)ÚQwen3MoeExpertsÚQwen3MoeSparseMoeBlockzpoolside/laguna-XS.2)Ú
checkpointc                   ó:  — e Zd ZU dZdZi dd“dd“dd“dd“dd	“d
d“dd“dd“dd“dd	“dd“dd	“dd“dd“dd“dd	“ZdZeed<   dZ	eed<   dZ
eed<   dZeed<   d Zeed!<   d"Zeed#<   d$Zeed%<   d Zeed&<   d'Zeed(<   d'Zeed)<   d'Zeed*<   d+Zeed,<   d-Zeed.<   d/Zeez  ed0<   d1Zee         d1z  ed2<   d1Zee         d1z  ed3<   d4Zeed5<   d-Zeed6<   d7Zeed8<    e¦   «         Z  e¦   «         Z! e¦   «         Z" e¦   «         Z# e¦   «         Z$ e¦   «         Z%d9„ Z&d:„ Z'd;„ Z(d1S )<ÚLagunaConfigu¤  
    gating (`bool` or `str`, *optional*, defaults to `True`):
        Softplus output-gate granularity. ``True`` or ``"per-head"`` applies one gate per head,
        broadcast across ``head_dim``; ``"per-element"`` applies one gate per ``(head, head_dim)``
        channel.
    num_attention_heads_per_layer (`list[int]`, *optional*):
        Per-layer override for ``num_attention_heads``. Length must equal ``num_hidden_layers``.
    mlp_layer_types (`list[str]`, *optional*):
        Per-layer MLP type â€” ``"dense"`` or ``"sparse"``. Length must equal
        ``num_hidden_layers``. Defaults to first layer dense, rest sparse.
    moe_routed_scaling_factor (`float`, *optional*, defaults to 1.0):
        Scalar applied to routed-expert output before combining with the shared-expert output.
    moe_apply_router_weight_on_input (`bool`, *optional*, defaults to `False`):
        Whether to apply router weights to the MoE input rather than the output. Not supported
        in transformers yet; ``True`` will raise a ``NotImplementedError`` for now.
    moe_router_logit_softcapping (`float`, *optional*, defaults to 0.0):
        Scaling factor when applying tanh softcapping on the logits of the MoE router logits.

    Example:

    ```python
    >>> from transformers import LagunaModel, LagunaConfig

    >>> configuration = LagunaConfig()
    >>> model = LagunaModel(configuration)
    >>> configuration = model.config
    ```
    Úlagunazlayers.*.self_attn.q_projÚcolwisezlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.g_projzlayers.*.self_attn.o_projÚrowwisezlayers.*.self_attn.q_normÚreplicated_with_grad_allreducezlayers.*.self_attn.k_normzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projz!layers.*.mlp.experts.gate_up_projÚpacked_colwisezlayers.*.mlp.experts.down_projzlayers.*.mlp.expertsÚmoe_tp_expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_proji ˆ Ú
vocab_sizei    Úintermediate_sizeé(   Únum_hidden_layersé0   Únum_attention_headsé   Únum_key_value_headsi   Úmax_position_embeddingsé   Únum_expertsÚnum_experts_per_toki   Úmoe_intermediate_sizeÚshared_expert_intermediate_sizeÚsliding_windowé€   Úhead_dimFÚattention_biasTÚgatingNÚnum_attention_heads_per_layerÚmlp_layer_typesç      ð?Úmoe_routed_scaling_factorÚ moe_apply_router_weight_on_inputç        Úmoe_router_logit_softcappingc                 ó  — | j         €dg| j        z  | _         | j        €dgdg| j        dz
  z  z   | _        | j        €| j        g| j        z  | _        ddddœdd	d
dœdœ}| j        €|| _        t          j        | fi |¤dddhi¤Ž d S )NÚfull_attentionÚdenseÚsparseé   Údefaultg    €„Ag      à?)Ú	rope_typeÚ
rope_thetaÚpartial_rotary_factorg     ˆÃ@rF   ©rL   Úsliding_attentionÚignore_keys_at_rope_validationrU   )Úlayer_typesr4   rE   rD   r6   Úrope_parametersr   Ú__post_init__)ÚselfÚkwargsÚdefault_rope_paramss      úg/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/laguna/modular_laguna.pyrY   zLagunaConfig.__post_init__„   sî   € ØÔÐ#Ø 0Ð1°DÔ4JÑJˆDÔØÔÐ'Ø$+ 9°¨z¸TÔ=SÐVWÑ=WÑ/XÑ#XˆDÔ ØÔ-Ð5Ø26Ô2JÐ1KÈdÔNdÑ1dˆDÔ.ð -6ÀXÐhkÐlÐlØ/8ÈÐjmÐ!nÐ!nðe
ð e
Ðð ÔÐ'Ø#6ˆDÔ õ 	Ô&Øð	
ð 	
Øð	
ð 	
Ø<OÐQaÐ;bð	
ð 	
ð 	
ð 	
ð 	
ð 	
ó    c                 ó   — |S ©N© )rZ   r[   s     r]   Úconvert_rope_params_to_dictz(LagunaConfig.convert_rope_params_to_dict˜   s   € àˆr^   c                 óü  — | j         rt          d¦  «        ‚| j        �Jt          | j        ¦  «        | j        k    r-t          dt          | j        ¦  «        › d| j        › d�¦  «        ‚t          | j        ¦  «        | j        k    r-t          dt          | j        ¦  «        › d| j        › d�¦  «        ‚t          | j        ¦  «        | j        k    r-t          dt          | j        ¦  «        › d| j        › d�¦  «        ‚dS )z'Part of ``@strict``-powered validation.zhmoe_apply_router_weight_on_input=True is not yet supported in the transformers implementation of Laguna.Nz&num_attention_heads_per_layer length (z ) must equal num_hidden_layers (z).zlayer_types length (zmlp_layer_types length ()rH   ÚNotImplementedErrorrD   Úlenr4   Ú
ValueErrorrW   rE   )rZ   s    r]   Úvalidate_architecturez"LagunaConfig.validate_architectureœ   sa  € àÔ0ð 	Ý%ð9ñô ð ð
 Ô.Ð:Ý�DÔ6Ñ7Ô7¸4Ô;QÒQÐQåðL½¸TÔ=_Ñ9`Ô9`ð Lð LØ15Ô1GðLð Lð Lñô ð õ ˆtÔÑ Ô  DÔ$:Ò:Ð:ÝðL¥s¨4Ô+;Ñ'<Ô'<ð Lð LØ15Ô1GðLð Lð Lñô ð õ ˆtÔ#Ñ$Ô$¨Ô(>Ò>Ð>ÝðL­3¨tÔ/CÑ+DÔ+Dð Lð LØ15Ô1GðLð Lð Lñô ð ð ?Ð>r^   ))Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚbase_model_tp_planr1   ÚintÚ__annotations__r2   r4   r6   r8   r9   r;   r<   r=   r>   r?   rA   rB   ÚboolrC   ÚstrrD   ÚlistrE   rG   ÚfloatrH   rJ   ÚAttributeErrorÚdecoder_sparse_stepÚmlp_only_layersÚqkv_biasÚnorm_topk_probÚuse_sliding_windowÚmax_window_layersrY   rb   rg   ra   r^   r]   r*   r*   0   sÅ  € € € € € € ðð ð: €JðØ# Yðà# Yðð 	$ Yðð 	$ Yð	ð
 	$ Yðð 	$Ð%Eðð 	$Ð%Eðð 	! )ðð 	 	ðð 	! )ðð 	,Ð-=ðð 	)¨)ðð 	Ð 0ðð 	0°ðð 	.¨yðð  	0°ð!Ðð& €J�ÐÐÑØ!Ð�sÐ!Ð!Ñ!ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø Ð˜Ð Ð Ñ Ø#)Ð˜SÐ)Ð)Ñ)Ø€K�ÐÐÑØ Ð˜Ð Ð Ñ Ø!$Ð˜3Ð$Ð$Ñ$Ø+.Ð# SÐ.Ð.Ñ.Ø€N�CÐÐÑð €HˆcÐÐÑØ €N�DÐ Ð Ñ Ø€FˆD�3‰JÐÐÑØ6:Ð! 4¨¤9¨tÑ#3Ð:Ð:Ñ:à(,€O�T˜#”Y Ñ%Ð,Ð,Ñ,Ø'*Ð˜uÐ*Ð*Ñ*Ø-2Ð$ dÐ2Ð2Ñ2Ø*-Ð  %Ð-Ð-Ñ-ð )˜.Ñ*Ô*ÐØ$�nÑ&Ô&€OØˆ~ÑÔ€HØ#�^Ñ%Ô%€NØ'˜Ñ)Ô)ÐØ&˜Ñ(Ô(Ðð
ð 
ð 
ð(ð ð ðð ð ð ð r^   r*   c                   ó   — e Zd ZdS )ÚLagunaRMSNormN©rh   ri   rj   ra   r^   r]   r|   r|   ·   ó   € € € € € Ø€Dr^   r|   c                   óˆ   ‡ — e Zd Zdefˆ fd„Ze	 	 	 	 ddedz  ded         dedz  dedz  de	d	e
f         f
d
„¦   «         Zˆ xZS )ÚLagunaRotaryEmbeddingÚconfigc                 óJ   •— t          ¦   «                              |¦  «         d S r`   )ÚsuperÚ__init__©rZ   r�   Ú	__class__s     €r]   r„   zLagunaRotaryEmbedding.__init__¼   s!   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ð Ð r^   NÚdeviceztorch.deviceÚseq_lenÚ
layer_typeÚreturnztorch.Tensorc                 ón  — | j         |         d         }| j         |                              dd¦  «        }t          | dd¦  «        p| j        | j        z  }t          ||z  ¦  «        }d}d|t          j        d|dt          j        ¬¦  «         	                    |t          j
        ¬	¦  «        |z  z  z  }	|	|fS )
a{  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        rR   rS   rF   rA   Nr   r   )Údtype)r‡   rŒ   )rX   ÚgetÚgetattrÚhidden_sizer6   rn   ÚtorchÚarangeÚint64Útors   )
r�   r‡   rˆ   r‰   ÚbaserS   rA   ÚdimÚattention_factorÚinv_freqs
             r]   Úcompute_default_rope_parametersz5LagunaRotaryEmbedding.compute_default_rope_parameters¿   sÄ   € ð. Ô% jÔ1°,Ô?ˆà &Ô 6°zÔ B× FÒ FÐG^Ð`cÑ dÔ dÐÝ˜6 :¨tÑ4Ô4Ðh¸Ô8JÈfÔNhÑ8hˆÝ�(Ð2Ñ2Ñ3Ô3ˆàÐð Ø•U”\ ! S¨!µ5´;Ð?Ñ?Ô?×BÒBÈ&ÕX]ÔXcÐBÑdÔdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r^   )NNNN)rh   ri   rj   r*   r„   Ústaticmethodr   rn   rq   Útuplers   r˜   Ú__classcell__©r†   s   @r]   r€   r€   »   s¿   ø€ € € € € ð!˜|ð !ð !ð !ð !ð !ð !ð à&*Ø+/Ø"Ø!%ð	"*ð "*Ø˜tÑ#ð"*à˜Ô(ð"*ð �t‘ð"*ð ˜$‘Jð	"*ð
 
ˆ~˜uÐ$Ô	%ð"*ð "*ð "*ñ „\ð"*ð "*ð "*ð "*ð "*r^   r€   c                   ó   — e Zd ZdS )Ú	LagunaMLPNr}   ra   r^   r]   rž   rž   å   r~   r^   rž   c                   óh   ‡ — e Zd Zˆ fd„Zdej        deej        ej        ej        f         fd„Zˆ xZS )ÚLagunaTopKRouterc                 óÄ   •— t          ¦   «                              ¦   «          t          j        t	          j        |j        ¦  «        d¬¦  «        | _        |j        | _	        d S )NF)Úrequires_grad)
rƒ   r„   r   Ú	Parameterr�   Úzerosr;   Úe_score_correction_biasrJ   Úrouter_logit_softcappingr…   s     €r]   r„   zLagunaTopKRouter.__init__ê   sO   ø€ Ý‰Œ×ÒÑÔÐÝ')¤|µE´KÀÔ@RÑ4SÔ4SÐchÐ'iÑ'iÔ'iˆÔ$Ø(.Ô(KˆÔ%Ð%Ð%r^   Úhidden_statesrŠ   c                 ó6  — |                      d| j        ¦  «        }t          j        || j        ¦  «                             ¦   «         }| j        dk    r$t          j        || j        z  ¦  «        | j        z  }t          j	        |¦  «        }|| j
                             |j        ¦  «        z   }t          j        || j        d¬¦  «        \  }}|                     d|¦  «        }||                     dd¬¦  «        z  }|                     |j        ¦  «        }|||fS )NéÿÿÿÿrI   )r•   T)r•   Úkeepdim)ÚreshapeÚ
hidden_dimÚFÚlinearÚweightrs   r¦   r�   ÚtanhÚsigmoidr¥   r“   rŒ   ÚtopkÚtop_kÚgatherÚsum)rZ   r§   Úrouter_logitsÚrouting_scoresÚscores_for_selectionÚ_Úselected_expertsÚrouting_weightss           r]   ÚforwardzLagunaTopKRouter.forwardï   s  € ð &×-Ò-¨b°$´/ÑBÔBˆÝœ °´Ñ<Ô<×BÒBÑDÔDˆàÔ(¨3Ò.Ð.Ý!œJ }°tÔ7TÑ'TÑUÔUÐX\ÔXuÑuˆMåœ }Ñ5Ô5ˆà-°Ô0L×0OÒ0OÐP^ÔPdÑ0eÔ0eÑeÐÝ#œjÐ)=¸t¼zÈrÐRÑRÔRÑˆÐØ(×/Ò/°Ð4DÑEÔEˆØ)¨O×,?Ò,?ÀBÐPTÐ,?Ñ,UÔ,UÑUˆØ)×,Ò,¨]Ô-@ÑAÔAˆà˜oÐ/?Ð?Ð?r^   )	rh   ri   rj   r„   r�   ÚTensorrš   r¼   r›   rœ   s   @r]   r    r    é   s�   ø€ € € € € ðLð Lð Lð Lð Lð
@à”|ð@ð 
ˆuŒ|˜Uœ\¨5¬<Ð7Ô	8ð@ð @ð @ð @ð @ð @ð @ð @r^   r    c                   ó   — e Zd ZdS )ÚLagunaExpertsNr}   ra   r^   r]   r¿   r¿     r~   r^   r¿   c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚLagunaSparseMoeBlockr�   c                 ó˜   •— t          ¦   «                              |¦  «         t          ||j        ¬¦  «        | _        |j        | _        d S )N©r2   )rƒ   r„   rž   r>   Úshared_expertsrG   Úrouted_scaling_factorr…   s     €r]   r„   zLagunaSparseMoeBlock.__init__	  sD   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý'¨À&ÔBhÐiÑiÔiˆÔØ%+Ô%EˆÔ"Ð"Ð"r^   r§   rŠ   c                 ó  — |j         \  }}}|                     d|¦  «        }|                      |¦  «        }|                      |¦  «        \  }}}|                      |||¦  «        }|| j        z  }||z   }|                     |||¦  «        }|S )Nr©   )ÚshapeÚviewrÄ   ÚgateÚexpertsrÅ   r«   )	rZ   r§   Ú
batch_sizeÚsequence_lengthr¬   Úshared_outputr¹   r»   rº   s	            r]   r¼   zLagunaSparseMoeBlock.forward  sœ   € Ø2?Ô2EÑ/ˆ
�O ZØ%×*Ò*¨2¨zÑ:Ô:ˆØ×+Ò+¨MÑ:Ô:ˆà/3¯yªy¸Ñ/GÔ/GÑ,ˆˆ?Ð,ØŸš ]Ð4DÀoÑVÔVˆà%¨Ô(BÑBˆØ%¨Ñ5ˆà%×-Ò-¨j¸/È:ÑVÔVˆØÐr^   )	rh   ri   rj   r*   r„   r�   r½   r¼   r›   rœ   s   @r]   rÁ   rÁ     sq   ø€ € € € € ðF˜|ð Fð Fð Fð Fð Fð Fð
 U¤\ð °e´lð ð ð ð ð ð ð ð r^   rÁ   c                   óÎ   ‡ — e Zd ZdZdededefˆ fd„Z	 ddej        de	ej        ej        f         d	ej        dz  d
e
dz  dee         de	ej        ej        dz  f         fd„Zˆ xZS )ÚLagunaAttentionzSAfmoe-style SWA/GQA attention with Laguna-specific gating and per-layer head count.r�   Ú	layer_idxÚ	num_headsc                 ó   •— || _         t          ¦   «                              ||¦  «         | j         |j        z  | _        t          j        |j        | j         | j        z  |j	        ¬¦  «        | _
        t          j        | j         | j        z  |j        |j	        ¬¦  «        | _        | `|j        du p
|j        dk    | _        | j        r| j         n| j         | j        z  }t          j        |j        |d¬¦  «        | _        d S )N)ÚbiasTzper-headF)rÑ   rƒ   r„   r8   Únum_key_value_groupsr   ÚLinearr�   rA   rB   Úq_projÚo_projÚ	gate_projrC   Úgate_per_headÚg_proj)rZ   r�   rÐ   rÑ   Ú
g_proj_dimr†   s        €r]   r„   zLagunaAttention.__init__!  sã   ø€ à"ˆŒå‰Œ×Ò˜ Ñ+Ô+Ð+Ø$(¤N°fÔ6PÑ$PˆÔ!å”i Ô 2°D´NÀTÄ]Ñ4RÐY_ÔYnÐoÑoÔoˆŒÝ”i ¤°´Ñ >ÀÔ@RÐY_ÔYnÐoÑoÔoˆŒàˆNØ#œ]¨dÐ2ÐQ°f´mÀzÒ6QˆÔØ'+Ô'9Ð]�T”^�^¸t¼~ÐPTÔP]Ñ?]ˆ
Ý”i Ô 2°JÀUÐKÑKÔKˆŒˆˆr^   Nr§   Úposition_embeddingsÚattention_maskÚpast_key_valuesr[   rŠ   c                 ó´  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «        }|                      |¦  «                             |¦  «        }	|                      |¦  «                             |¦  «        }
|                      |¦  «                             dd¦  «        }|                      |	¦  «                             dd¦  «        }	|
                     dd¦  «        }
|\  }}t          ||	||¦  «        \  }}	|�| 
                    |	|
| j        ¦  «        \  }	}
t          j        | j        j        t           ¦  «        } || ||	|
|f| j        sdn| j        | j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }t/          j        |                      |¦  «                             ¦   «         ¦  «                             |j        ¦  «        }| j        r:  |j        g |¢d‘| j        ‘R Ž |                     d¦  «        z  j        g |¢d‘R Ž }n||z  }|                      |¦  «        }||fS )Nr©   rO   r   rI   )ÚdropoutÚscalingr?   ) rÇ   rA   rÖ   rÈ   Úk_projÚv_projÚq_normÚ	transposeÚk_normr%   ÚupdaterÐ   r   Úget_interfacer�   Ú_attn_implementationr   ÚtrainingÚattention_dropoutrá   r?   r«   Ú
contiguousr­   ÚsoftplusrÚ   rs   r“   rŒ   rÙ   Ú	unsqueezer×   )rZ   r§   rÜ   rÝ   rÞ   r[   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightsrÉ   s                    r]   r¼   zLagunaAttention.forward0  s–  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔBˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆà—{’{ <Ñ0Ô0×:Ò:¸1¸aÑ@Ô@ˆØ—[’[ Ñ,Ô,×6Ò6°q¸!Ñ<Ô<ˆ
Ø#×-Ò-¨a°Ñ3Ô3ˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}ÐH�C�C°$Ô2HØ”LØÔ.ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆåŒz˜$Ÿ+š+ mÑ4Ô4×:Ò:Ñ<Ô<Ñ=Ô=×@Ò@ÀÔARÑSÔSˆØÔð 	-ØgÐ+˜;Ô+ÐL¨[ÐL¸"ÐL¸d¼mÐLÐLÐLÈtÏ~Ê~Ð^`ÑOaÔOaÑaÔgð ØðØ ðð ð ˆKˆKð &¨Ñ,ˆKà—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r^   r`   )rh   ri   rj   rk   r*   rn   r„   r�   r½   rš   r   r   r   r¼   r›   rœ   s   @r]   rÏ   rÏ     sí   ø€ € € € € à]Ð]ðL˜|ð L¸ð LÈð Lð Lð Lð Lð Lð Lð( )-ð3)ð 3)à”|ð3)ð # 5¤<°´Ð#=Ô>ð3)ð œ tÑ+ð	3)ð
  ™ð3)ð Ð-Ô.ð3)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð3)ð 3)ð 3)ð 3)ð 3)ð 3)ð 3)ð 3)r^   rÏ   c                   ó   — e Zd Zdedefd„ZdS )ÚLagunaDecoderLayerr�   rÐ   c                 ó   — t           j                             | ¦  «         |j        | _        t	          |||j        |         ¦  «        | _        |j        |         dk    rt          |¦  «        | _	        nt          ||j        ¬¦  «        | _	        t          |j        |j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        d S )NrN   rÃ   )Úeps)r   ÚModuler„   r�   rÏ   rD   Ú	self_attnrE   rÁ   Úmlprž   r2   r|   Úrms_norm_epsÚinput_layernormÚpost_attention_layernorm)rZ   r�   rÐ   s      r]   r„   zLagunaDecoderLayer.__init__g  s¶   € Ý
Œ	×Ò˜4Ñ Ô Ð Ø!Ô-ˆÔÝ(¨°¸FÔ<`ÐajÔ<kÑlÔlˆŒØÔ! )Ô,°Ò8Ð8Ý+¨FÑ3Ô3ˆDŒHˆHå  ¸6Ô;SÐTÑTÔTˆDŒHÝ,¨VÔ-?ÀVÔEXÐYÑYÔYˆÔÝ(5°fÔ6HÈfÔNaÐ(bÑ(bÔ(bˆÔ%Ð%Ð%r^   N)rh   ri   rj   r*   rn   r„   ra   r^   r]   rú   rú   f  s>   € € € € € ð	c˜|ð 	c¸ð 	cð 	cð 	cð 	cð 	cð 	cr^   rú   c                   óH   ‡ — e Zd Z ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚLagunaPreTrainedModelc                 ó.  •— t          ¦   «                              |¦  «         t          |t          ¦  «        r+t          j        j                             |j        ¦  «         d S t          |t          ¦  «        r›|j
        D ]•}|j        }|j        |         dk    rt          |j        |                  } ||j        |¬¦  «        \  }}t          j        t!          ||› d�¦  «        |¦  «         t          j        t!          ||› d�¦  «        |¦  «         Œ”d S d S )NrP   )r‰   Ú	_inv_freqÚ_original_inv_freq)rƒ   Ú_init_weightsÚ
isinstancer    r�   r   ÚinitÚzeros_r¥   r€   rW   r˜   rQ   r   r�   Úcopy_rŽ   )rZ   Úmoduler‰   Úrope_init_fnÚcurr_inv_freqr¹   r†   s         €r]   r  z#LagunaPreTrainedModel._init_weightst  s%  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ.Ñ/Ô/ð 		^ÝŒHŒM× Ò  Ô!?Ñ@Ô@Ð@Ð@Ð@Ý˜Õ 5Ñ6Ô6ð 	^Ø$Ô0ð ^ð ^�
Ø%ÔE�ØÔ# JÔ/°9Ò<Ð<Ý#6°vÔ7GÈ
Ô7SÔ#T�LØ#/ <°´È*Ð#UÑ#UÔ#UÑ �˜qÝ”
�7 6¨jÐ+CÐ+CÐ+CÑDÔDÀmÑTÔTÐTÝ”
�7 6¨jÐ+LÐ+LÐ+LÑMÔMÈ}Ñ]Ô]Ð]Ð]ð	^ð 	^ð^ð ^r^   )rh   ri   rj   r�   Úno_gradr  r›   rœ   s   @r]   r  r  s  sS   ø€ € € € € Ø€U„]�_„_ð^ð ^ð ^ð ^ñ „_ð^ð ^ð ^ð ^ð ^r^   r  c                   óœ   — e Zd Z	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dedz  dej        dz  dedz  de	e
         d	efd
„ZdS )ÚLagunaModelNÚ	input_idsrÝ   Úposition_idsrÞ   Úinputs_embedsÚ	use_cacher[   rŠ   c           	      ó˜  ‡— |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          |x}	t          ¦  «        sI| j        ||||dœŠˆfd„ˆfd„d	œ}
i }	t          | j        j        ¦  «        D ]} |
|         ¦   «         |	|<   Œ|}i }t          | j        j        ¦  «        D ]}|                      |||¦  «        ||<   Œt          | j        d | j        j        …         ¦  «        D ]?\  }} ||f|	| j        j        |                  || j        j        |                  ||d
œ|¤Ž}Œ@|                      |¦  «        }t'          ||r|nd ¬¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embeds)r�   r   rO   )r‡   )r�   r  rÝ   rÞ   r  c                  ó   •— t          di ‰ ¤ŽS ©Nra   )r   ©Úmask_kwargss   €r]   ú<lambda>z%LagunaModel.forward.<locals>.<lambda>¥  s   ø€ Õ*<Ð*KÐ*K¸{Ð*KÐ*K€ r^   c                  ó   •— t          di ‰ ¤ŽS r  )r   r  s   €r]   r  z%LagunaModel.forward.<locals>.<lambda>¦  s   ø€ Õ-NÐ-]Ð-]ÐQ\Ð-]Ð-]€ r^   rT   )rÝ   rÜ   r  rÞ   )Úlast_hidden_staterÞ   )rf   Úembed_tokensr   r�   Úget_seq_lengthr�   r‘   rÇ   r‡   rî   r	  ÚdictÚsetrW   Ú
rotary_embÚ	enumerateÚlayersr4   Únormr   )rZ   r  rÝ   r  rÞ   r  r  r[   Úpast_seen_tokensÚcausal_mask_mappingÚmask_creation_functionsr‰   r§   rÜ   ÚiÚdecoder_layerr  s                   @r]   r¼   zLagunaModel.forward„  sc  ø€ ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå°Ð?Ð-ÅÑFÔFð 	Xàœ+Ø!.Ø"0Ø#2Ø ,ðð ˆKð #LÐ"KÐ"KÐ"KØ%]Ð%]Ð%]Ð%]ð'ð 'Ð#ð #%ÐÝ! $¤+Ô"9Ñ:Ô:ð Xð X�
Ø2UÐ2IÈ*Ô2UÑ2WÔ2WÐ# JÑ/Ð/à%ˆØ ÐÝ˜dœkÔ5Ñ6Ô6ð 	gð 	gˆJØ.2¯oªo¸mÈ\Ð[eÑ.fÔ.fÐ 
Ñ+Ð+å )¨$¬+Ð6U¸¼Ô8UÐ6UÔ*VÑ WÔ Wð 	ð 	ÑˆAˆ}Ø)˜MØðà2°4´;Ô3JÈ1Ô3MÔNØ$7¸¼Ô8OÐPQÔ8RÔ$SØ)Ø /ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå%Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
r^   )NNNNNN)rh   ri   rj   r�   Ú
LongTensorr½   r   ÚFloatTensorrp   r   r   r   r¼   ra   r^   r]   r  r  ƒ  s¼   € € € € € ð .2Ø.2Ø04Ø(,Ø26Ø!%ð<
ð <
àÔ# dÑ*ð<
ð œ tÑ+ð<
ð Ô&¨Ñ-ð	<
ð
  ™ð<
ð Ô(¨4Ñ/ð<
ð ˜$‘;ð<
ð Ð+Ô,ð<
ð 
 ð<
ð <
ð <
ð <
ð <
ð <
r^   r  c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚLagunaForCausalLMc                 ó6   •—  t          ¦   «         j        di |¤ŽS )a­  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        ra   )rƒ   r¼   )rZ   Úsuper_kwargsr†   s     €r]   r¼   zLagunaForCausalLM.forwardÄ  s!   ø€ ð �u‰wŒwŒÐ.Ð. Ð.Ð.Ð.r^   )rh   ri   rj   r¼   r›   rœ   s   @r]   r/  r/  Ã  s8   ø€ € € € € ð/ð /ð /ð /ð /ð /ð /ð /ð /r^   r/  )r*   r/  r  r  )Ork   Úcollections.abcr   Útypingr   r   r   r�   Útorch.nn.functionalr   Ú
functionalr­   Úhuggingface_hub.dataclassesr   Ú r
   r
  Úcache_utilsr   r   Úconfiguration_utilsr   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_outputsr   Úmodeling_rope_utilsr   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   Úutils.genericr   r   Úafmoe.modeling_afmoer   Úgemma3.modeling_gemma3r   Ú$glm4_moe_lite.modeling_glm4_moe_liter   Úllama.modeling_llamar   r   Ú!qwen2_moe.configuration_qwen2_moer   Úqwen2_moe.modeling_qwen2_moer    r!   r"   r#   Ú qwen3_5_moe.modeling_qwen3_5_moer$   r%   Úqwen3_moe.modeling_qwen3_moer&   r'   Ú
get_loggerrh   Úloggerr*   r|   r€   rž   r    r¿   rÁ   rÏ   rú   r  r  r/  Ú__all__ra   r^   r]   ú<module>rM     sœ  ðð Ð à $Ð $Ð $Ð $Ð $Ð $Ø )Ð )Ð )Ð )Ð )Ð )Ð )Ð )Ð )Ð )à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø .Ð .Ð .Ð .Ð .Ð .Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ BÐ BÐ BÐ BÐ BÐ BØ 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø &Ð &Ð &Ð &Ð &Ð &Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø :Ð :Ð :Ð :Ð :Ð :Ø JÐ JÐ JÐ JÐ JÐ JØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ >Ð >Ð >Ð >Ð >Ð >Ø uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uØ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YØ RÐ RÐ RÐ RÐ RÐ RÐ RÐ Rð 
ˆÔ	˜HÑ	%Ô	%€ð €Ð1Ð2Ñ2Ô2ØðBð Bð Bð Bð B�>ñ Bô Bñ „ñ 3Ô2ðBðJ	ð 	ð 	ð 	ð 	�Oñ 	ô 	ð 	ð'*ð '*ð '*ð '*ð '*Ð1ñ '*ô '*ð '*ðT	ð 	ð 	ð 	ð 	�ñ 	ô 	ð 	ð@ð @ð @ð @ð @Ð+ñ @ô @ð @ð6	ð 	ð 	ð 	ð 	�Oñ 	ô 	ð 	ðð ð ð ð Ð1ñ ô ð ð* ðE)ð E)ð E)ð E)ð E)�nñ E)ô E)ñ ÔðE)ðP
cð 
cð 
cð 
cð 
cÐ0ñ 
cô 
cð 
cð^ð ^ð ^ð ^ð ^Ð3ñ ^ô ^ð ^ð =
ð =
ð =
ð =
ð =
�*ñ =
ô =
ð =
ð@/ð /ð /ð /ð /Ð+ñ /ô /ð /ðð ð €€€r^   