§
    ‚Štj¯  ã                   óÆ  — d dl Z d dlmZ ddlmZ ddlmZ ddlm	Z	m
Z
 ddlmZmZ dd	lmZmZmZ dd
lmZ ddlmZmZmZmZ  ej        e¦  «        Z ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z G d„ de¦  «        Z G d„ de¦  «        Z G d„ de¦  «        Z G d„ de¦  «        Z  G d„ de¦  «        Z! G d„ de¦  «        Z" G d„ de¦  «        Z#g d¢Z$dS )é    N)Ústricté   )Úinitialization)ÚPreTrainedConfig)ÚROPE_INIT_FUNCTIONSÚRopeParameters)Úauto_docstringÚloggingé   )ÚLagunaDecoderLayerÚLagunaModelÚLagunaRotaryEmbedding)ÚQwen3MoeConfig)ÚQwen3MoeAttentionÚQwen3MoeForCausalLMÚQwen3MoePreTrainedModelÚQwen3MoeSparseMoeBlockz JetBrains/Mellum2-12B-A2.5B-Base)Ú
checkpointc                   óL  — e Zd ZU dZdZdZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZedz  ed<   dZeed<   dZeed<   dZeed<   dZee         dz  ed<   dZee         dz  ed<   dZeez  dz  ed<    e¦   «         Z e¦   «         Z e¦   «         Zd„ Zd„ ZdS )ÚMellumConfigu}  
    mlp_layer_types (`list[str]`, *optional*):
        Per-layer MLP type â€” `"dense"` or `"sparse"`. Length must equal
        `num_hidden_layers`. Defaults to all sparse.

    ```python
    >>> from transformers import MellumModel, MellumConfig

    >>> configuration = MellumConfig()
    >>> model = MellumModel(configuration)
    >>> configuration = model.config
    ```
    Úmellumi € Ú
vocab_sizei 	  Úhidden_sizei   Úintermediate_sizeé   Únum_hidden_layersé€   Úhead_dimi   Úmax_position_embeddingsi   NÚsliding_windowé@   Únum_expertsi€  Úmoe_intermediate_sizeTÚnorm_topk_probÚlayer_typesÚmlp_layer_typesÚrope_parametersc                 óÀ   — | j         €dg| j        z  | _         | j        €dg| j        z  | _        | j        €dddœdddœdœ| _        t	          j        | fi |¤dd	dhi¤Ž d S )
NÚfull_attentionÚsparseÚdefaultg    €„A)Ú	rope_typeÚ
rope_thetag     ˆÃ@)r)   Úsliding_attentionÚignore_keys_at_rope_validationr.   )r%   r   r&   r'   r   Ú__post_init__©ÚselfÚkwargss     úg/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/mellum/modular_mellum.pyr0   zMellumConfig.__post_init__K   s³   € ØÔÐ#Ø 0Ð1°DÔ4JÑJˆDÔØÔÐ'Ø$, :°Ô0FÑ#FˆDÔ àÔÐ'à09ÈÐ"RÐ"RØ3<ÈGÐ%TÐ%Tð$ð $ˆDÔ õ
 	Ô&Øð	
ð 	
àð	
ð 	
ð -@ÐAQÐ+Rð	
ð 	
ð 	
ð 	
ð 	
ð 	
ó    c                 ó   — |S ©N© r1   s     r4   Úconvert_rope_params_to_dictz(MellumConfig.convert_rope_params_to_dict]   s   € àˆr5   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typer   ÚintÚ__annotations__r   r   r   r   r   r    r"   r#   r$   Úboolr%   ÚlistÚstrr&   r'   Údictr   ÚAttributeErrorÚuse_sliding_windowÚdecoder_sparse_stepÚmlp_only_layersr0   r9   r8   r5   r4   r   r   &   sb  € € € € € € ðð ð €Jà€J�ÐÐÑØ€K�ÐÐÑØ!Ð�sÐ!Ð!Ñ!ØÐ�sÐÐÑØ€HˆcÐÐÑØ#)Ð˜SÐ)Ð)Ñ)Ø!%€N�C˜$‘JÐ%Ð%Ñ%Ø€K�ÐÐÑØ!$Ð˜3Ð$Ð$Ñ$Ø€N�DÐÐÑØ$(€K��c”˜TÑ!Ð(Ð(Ñ(Ø(,€O�T˜#”Y Ñ%Ð,Ð,Ñ,Ø48€O�T˜NÑ*¨TÑ1Ð8Ð8Ñ8à'˜Ñ)Ô)ÐØ(˜.Ñ*Ô*ÐØ$�nÑ&Ô&€Oð
ð 
ð 
ð$ð ð ð ð r5   r   c                   ó   — e Zd ZdS )ÚMellumRotaryEmbeddingN©r:   r;   r<   r8   r5   r4   rJ   rJ   b   ó   € € € € € Ø€Dr5   rJ   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚMellumAttentionÚconfigÚ	layer_idxc                 óŠ   •— t          ¦   «                              ||¦  «         |j        |         dk    r|j        nd | _        d S )Nr.   )ÚsuperÚ__init__r%   r    ©r2   rO   rP   Ú	__class__s      €r4   rS   zMellumAttention.__init__g   sG   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+Ø7=Ô7IÈ)Ô7TÐXkÒ7kÐ7k˜fÔ3Ð3ÐquˆÔÐÐr5   ©r:   r;   r<   r   r?   rS   Ú__classcell__©rU   s   @r4   rN   rN   f   sW   ø€ € € € € ðv˜|ð v¸ð vð vð vð vð vð vð vð vð vð vr5   rN   c                   ó   — e Zd ZdS )ÚMellumSparseMoeBlockNrK   r8   r5   r4   rZ   rZ   l   rL   r5   rZ   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚMellumDecoderLayerrO   rP   c                 ór   •— t          ¦   «                              ¦   «          t          ||¦  «        | _        d S r7   )rR   rS   rN   Ú	self_attnrT   s      €r4   rS   zMellumDecoderLayer.__init__q   s.   ø€ Ý‰Œ×ÒÑÔÐÝ(¨°Ñ;Ô;ˆŒˆˆr5   rV   rX   s   @r4   r\   r\   p   sK   ø€ € € € € ð<˜|ð <¸ð <ð <ð <ð <ð <ð <ð <ð <ð <ð <r5   r\   c                   óH   ‡ — e Zd Z ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚMellumPreTrainedModelc                 ó®  •— t          ¦   «                              |¦  «         t          |t          ¦  «        r›|j        D ]•}|j        }|j        |         dk    rt          |j        |                  } ||j        |¬¦  «        \  }}t          j
        t          ||› d�¦  «        |¦  «         t          j
        t          ||› d�¦  «        |¦  «         Œ”d S d S )Nr+   )Ú
layer_typeÚ	_inv_freqÚ_original_inv_freq)rR   Ú_init_weightsÚ
isinstancerJ   r%   Úcompute_default_rope_parametersr,   r   rO   ÚinitÚcopy_Úgetattr)r2   Úmodulerb   Úrope_init_fnÚcurr_inv_freqÚ_rU   s         €r4   re   z#MellumPreTrainedModel._init_weightsw   sð   ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ3Ñ4Ô4ð 	^Ø$Ô0ð ^ð ^�
Ø%ÔE�ØÔ# JÔ/°9Ò<Ð<Ý#6°vÔ7GÈ
Ô7SÔ#T�LØ#/ <°´È*Ð#UÑ#UÔ#UÑ �˜qÝ”
�7 6¨jÐ+CÐ+CÐ+CÑDÔDÀmÑTÔTÐTÝ”
�7 6¨jÐ+LÐ+LÐ+LÑMÔMÈ}Ñ]Ô]Ð]Ð]ð	^ð 	^ð^ð ^r5   )r:   r;   r<   ÚtorchÚno_gradre   rW   rX   s   @r4   r`   r`   v   sS   ø€ € € € € Ø€U„]�_„_ð	^ð 	^ð 	^ð 	^ñ „_ð	^ð 	^ð 	^ð 	^ð 	^r5   r`   c                   ó   — e Zd ZdS )ÚMellumModelNrK   r8   r5   r4   rr   rr   „   rL   r5   rr   c                   ó   — e Zd ZdS )ÚMellumForCausalLMNrK   r8   r5   r4   rt   rt   ˆ   rL   r5   rt   )r   rt   rr   r`   )%ro   Úhuggingface_hub.dataclassesr   Ú r   rh   Úconfiguration_utilsr   Úmodeling_rope_utilsr   r   Úutilsr	   r
   Úlaguna.modeling_lagunar   r   r   Ú!qwen3_moe.configuration_qwen3_moer   Úqwen3_moe.modeling_qwen3_moer   r   r   r   Ú
get_loggerr:   Úloggerr   rJ   rN   rZ   r\   r`   rr   rt   Ú__all__r8   r5   r4   ú<module>r€      s©  ðð €€€Ø .Ð .Ð .Ð .Ð .Ð .à &Ð &Ð &Ð &Ð &Ð &Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3ðð ð ð ð ð ð ð ð -Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø [Ð [Ð [Ð [Ð [Ð [Ð [Ð [Ð [Ð [Ø >Ð >Ð >Ð >Ð >Ð >ðð ð ð ð ð ð ð ð ð ð ð ð 
ˆÔ	˜HÑ	%Ô	%€ð €Ð=Ð>Ñ>Ô>Øð7ð 7ð 7ð 7ð 7�>ñ 7ô 7ñ „ñ ?Ô>ð7ðt	ð 	ð 	ð 	ð 	Ð1ñ 	ô 	ð 	ðvð vð vð vð vÐ'ñ vô vð vð	ð 	ð 	ð 	ð 	Ð1ñ 	ô 	ð 	ð<ð <ð <ð <ð <Ð+ñ <ô <ð <ð^ð ^ð ^ð ^ð ^Ð3ñ ^ô ^ð ^ð	ð 	ð 	ð 	ð 	�+ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð+ñ 	ô 	ð 	ðð ð €€€r5   