§
    ‚ŠtjòW  ã                   ó8  — d Z ddlZddlmc mZ ddlmZ ddlmZ ddlm	Z
 ddlmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddl*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5  e"j6        e7¦  «        Z8 e!d¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z9 G d„ de3¦  «        Z: G d„ de¦  «        Z; G d„ dej<        ¦  «        Z= G d „ d!e)¦  «        Z> G d"„ d#e+¦  «        Z? G d$„ d%e5¦  «        Z@ G d&„ d'e4¦  «        ZA G d(„ d)e,e¦  «        ZB G d*„ d+e2¦  «        ZC G d,„ d-e1¦  «        ZD G d.„ d/e-¦  «        ZE G d0„ d1e/¦  «        ZF G d2„ d3e0¦  «        ZG G d4„ d5e.¦  «        ZHg d6¢ZIdS )7zPyTorch MiniMax model.é    N)Ústrict)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚMoeModelOutputWithPast)ÚRopeParameters)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚGemma2RotaryEmbedding)ÚMixtralAttentionÚMixtralDecoderLayerÚMixtralForCausalLMÚMixtralForQuestionAnsweringÚ MixtralForSequenceClassificationÚMixtralForTokenClassificationÚMixtralModelÚMixtralPreTrainedModelÚMixtralRMSNormÚMixtralSparseMoeBlockÚMixtralTopKRouterzMiniMaxAI/MiniMax-Text-01-hf)Ú
checkpointc                   óº  ‡ — e Zd ZU dZdZdgZdZdddddddd	œZd
gdgfddgdgfdgdgfdœZdddddœZ	ddiZ
dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZedz  ed <   d!Zeed"<   d#Zeed$<   d%Zeed&<   d'Zeed(<   d)Zeed*<   dZedz  ed+<   d,Zedz  ed-<   d.Zeee         z  dz  ed/<   d0Z eed1<   dZ!edz  ed2<   d3Z"eez  ed4<   d.Z#eed5<   dZ$eed<   d0Z%eed6<   d7Z&eed8<   d3Z'eed9<   dZ(e)e*z  dz  ed:<   dZ+ee         dz  ed;<   d<Z,eed=<   d,Z-eez  ed><   d,Z.eez  ed?<   d,Z/eez  ed@<   d,Z0eez  edA<   d,Z1eez  edB<   d,Z2eez  edC<   ˆ fdD„Z3ˆ xZ4S )EÚMiniMaxConfiga¡  
    block_size (`int`, *optional*, defaults to 256):
        The length of each attention block, determining how queries, keys, and values
        are grouped and processed for intra- and inter-block attention.
    full_attn_alpha_factor (`float`, *optional*, defaults to 1):
        Weight for residual value in residual connection after normal attention.
    full_attn_beta_factor (`float`, *optional*, defaults to 1):
        Weight for hidden state value in residual connection after normal attention.
    linear_attn_alpha_factor (`float`, *optional*, defaults to 1):
        Weight for residual value in residual connection after lightning attention.
    linear_attn_beta_factor (`float`, *optional*, defaults to 1):
        Weight for hidden state value in residual connection after lightning attention.
    mlp_alpha_factor (`float`, *optional*, defaults to 1):
        Weight for residual value in residual connection after MLP.
    mlp_beta_factor (`float`, *optional*, defaults to 1):
        Weight for hidden state value in residual connection after MLP.

    ```python
    >>> from transformers import MiniMaxModel, MiniMaxConfig

    >>> # Initializing a MiniMax style configuration
    >>> configuration = MiniMaxConfig()

    >>> # Initializing a model from the MiniMax style configuration
    >>> model = MiniMaxModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```ÚminimaxÚpast_key_valuesg    €„.AÚcolwiseÚrowwiseÚpacked_colwiseÚmoe_tp_experts)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projú!layers.*.mlp.experts.gate_up_projúlayers.*.mlp.experts.down_projúlayers.*.mlp.expertsÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormÚ	ep_routerÚgrouped_gemm)zlayers.*.mlp.gater.   r/   r0   Únum_expertsÚnum_local_expertsi }  Ú
vocab_sizei   Úhidden_sizei 8  Úintermediate_sizeé    Únum_hidden_layersÚnum_attention_headsé   Únum_key_value_headsNÚhead_dimÚsiluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeçñhãˆµøä>Úrms_norm_epsTÚ	use_cacheÚpad_token_idé   Úbos_token_idr   Úeos_token_idFÚtie_word_embeddingsÚsliding_windowg        Úattention_dropoutÚnum_experts_per_tokÚoutput_router_logitsgü©ñÒMbP?Úrouter_aux_loss_coefÚrouter_jitter_noiseÚrope_parametersÚlayer_typesé   Ú
block_sizeÚfull_attn_alpha_factorÚfull_attn_beta_factorÚlinear_attn_alpha_factorÚlinear_attn_beta_factorÚmlp_alpha_factorÚmlp_beta_factorc                 ó´   •— | j         €| j        | _         | j        €#d„ t          | j        ¦  «        D ¦   «         | _         t          ¦   «         j        di |¤Ž d S )Nc                 ó@   — g | ]}t          |d z   dz  ¦  «        rdnd‘ŒS )rM   r   Úfull_attentionÚlinear_attention)Úbool)Ú.0Úis     úi/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/minimax/modular_minimax.pyú
<listcomp>z/MiniMaxConfig.__post_init__.<locals>.<listcomp>–   sA   € ð  ð  ð  ØRS¥D¨!¨a©%°1©Ñ$5Ô$5ÐMÐ Ð Ð;Mð ð  ð  ó    © )rC   rA   rX   Úranger@   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €rh   rn   zMiniMaxConfig.__post_init__‘   sr   ø€ ØÔ#Ð+Ø'+Ô'?ˆDÔ$àÔÐ#ð ð  ÝW\Ð]aÔ]sÑWtÔWtð ñ  ô  ˆDÔð 	�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'rj   )5Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚdefault_thetaÚbase_model_tp_planÚbase_model_pp_planÚbase_model_ep_planÚattribute_mapr<   ÚintÚ__annotations__r=   r>   r@   rA   rC   rD   rF   ÚstrrG   rH   ÚfloatrJ   rK   re   rL   rN   rO   ÚlistrP   rQ   rR   rS   r;   rT   rU   rV   rW   r   ÚdictrX   rZ   r[   r\   r]   r^   r_   r`   rn   Ú__classcell__©rq   s   @rh   r'   r'   6   s<  ø€ € € € € € ðð ð< €JØ#4Ð"5ÐØ€Mà%.Ø%.Ø%.Ø%.Ø-=Ø*3Ø 0ðð Ðð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ðð Ðð )Ø-;Ø*8Ø 0ð	ð Ðð #Ð$7Ð8€Mà€J�ÐÐÑØ€K�ÐÐÑØ"Ð�sÐ"Ð"Ñ"ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø Ð˜Ð Ð Ñ Ø€Hˆc�D‰jÐÐÑØ€J�ÐÐÑØ#,Ð˜SÐ,Ð,Ñ,Ø#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØ#€L�#˜‘*Ð#Ð#Ñ#Ø €L�#˜‘*Ð Ð Ñ Ø+,€L�#˜˜Sœ	‘/ DÑ(Ð,Ð,Ñ,Ø %Ð˜Ð%Ð%Ñ%Ø!%€N�C˜$‘JÐ%Ð%Ñ%Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø Ð˜Ð Ð Ñ ØÐ�sÐÐÑØ!&Ð˜$Ð&Ð&Ñ&Ø"'Ð˜%Ð'Ð'Ñ'Ø!$Ð˜Ð$Ð$Ñ$Ø48€O�^ dÑ*¨TÑ1Ð8Ð8Ñ8Ø$(€K��c”˜TÑ!Ð(Ð(Ñ(Ø€J�ÐÐÑØ*+Ð˜C %™KÐ+Ð+Ñ+Ø)*Ð˜3 ™;Ð*Ð*Ñ*Ø,-Ð˜c E™kÐ-Ð-Ñ-Ø+,Ð˜S 5™[Ð,Ð,Ñ,Ø$%Ð�c˜E‘kÐ%Ð%Ñ%Ø#$€O�S˜5‘[Ð$Ð$Ñ$ð	(ð 	(ð 	(ð 	(ð 	(ð 	(ð 	(ð 	(ð 	(rj   r'   c                   ó   — e Zd ZdS )ÚMiniMaxRMSNormN©rr   rs   rt   rk   rj   rh   r†   r†   �   ó   € € € € € Ø€Drj   r†   c                   óh   ‡ — e Zd Zˆ fd„Zd„ Zdefd„Zˆ fd„Zdefd„Zde	j
        fd	„Zd
efd„Zˆ xZS )ÚMiniMaxCachec                 óV   •— t          ¦   «                              ¦   «          g | _        d S ©N)rm   Ú__init__Úlinear_cache©ro   rq   s    €rh   r�   zMiniMaxCache.__init__¢   s'   ø€ Ý‰Œ×ÒÑÔÐØ02ˆÔÐÐrj   c                 óž   — t          t          | j        ¦  «        |dz   ¦  «        D ]}| j                             g ¦  «         Œ|| j        |<   d S )NrM   )rl   ÚlenrŽ   Úappend)ro   Ú	layer_idxrŽ   Ú_s       rh   Úset_linear_cachezMiniMaxCache.set_linear_cache¦   sW   € å•s˜4Ô,Ñ-Ô-¨y¸1©}Ñ=Ô=ð 	)ð 	)ˆAØÔ×$Ò$ RÑ(Ô(Ð(Ð(Ø'3ˆÔ˜)Ñ$Ð$Ð$rj   r“   c                 óF   — |t          | ¦  «        k     r| j        |         S d S rŒ   )r‘   rŽ   )ro   r“   s     rh   Úget_linear_cachezMiniMaxCache.get_linear_cache¬   s&   € Ø•s˜4‘y”yÒ Ð ØÔ$ YÔ/Ð/Øˆtrj   c                 ó„   •— t          t          ¦   «                              ¦   «         t          | j        ¦  «        ¦  «        S rŒ   )Úmaxrm   Ú__len__r‘   rŽ   r�   s    €rh   rš   zMiniMaxCache.__len__±   s,   ø€ Ý•5‘7”7—?’?Ñ$Ô$¥c¨$Ô*;Ñ&<Ô&<Ñ=Ô=Ð=rj   Úrepeatsc                 óü   — t          t          | ¦  «        ¦  «        D ]^}| j        |         g k    r+| j        |                              |d¬¦  «        | j        |<   Œ>| j        |                              |¦  «         Œ_d S )Nr   ©Údim)rl   r‘   rŽ   Úrepeat_interleaver6   Úbatch_repeat_interleave)ro   r›   r“   s      rh   r    z$MiniMaxCache.batch_repeat_interleave´   s‹   € Ý�s 4™yœyÑ)Ô)ð 	Hð 	HˆIØÔ  Ô+¨rÒ1Ð1Ø/3Ô/@ÀÔ/K×/]Ò/]Ð^eÐklÐ/]Ñ/mÔ/m�Ô! )Ñ,Ð,à”˜IÔ&×>Ò>¸wÑGÔGÐGÐGð		Hð 	Hrj   Úindicesc                 óâ   — t          t          | ¦  «        ¦  «        D ]Q}| j        |         g k    r| j        |         |df         | j        |<   Œ1| j        |                              |¦  «         ŒRd S )N.)rl   r‘   rŽ   r6   Úbatch_select_indices)ro   r¡   r“   s      rh   r£   z!MiniMaxCache.batch_select_indices»   s€   € Ý�s 4™yœyÑ)Ô)ð 	Eð 	EˆIØÔ  Ô+¨rÒ1Ð1Ø/3Ô/@ÀÔ/KÈGÐUXÈLÔ/Y�Ô! )Ñ,Ð,à”˜IÔ&×;Ò;¸GÑDÔDÐDÐDð		Eð 	Erj   Ú
max_lengthc                 ó    — t          d¦  «        ‚)Nz*MiniMaxCache doesnot support `crop` method)ÚRuntimeError)ro   r¤   s     rh   ÚcropzMiniMaxCache.cropÂ   s   € ÝÐGÑHÔHÐHrj   )rr   rs   rt   r�   r•   r}   r—   rš   r    ÚtorchÚTensorr£   r§   rƒ   r„   s   @rh   rŠ   rŠ   ¡   sá   ø€ € € € € ð3ð 3ð 3ð 3ð 3ð4ð 4ð 4ð¨#ð ð ð ð ð
>ð >ð >ð >ð >ðH¨sð Hð Hð Hð HðE¨E¬Lð Eð Eð Eð EðI˜sð Ið Ið Ið Ið Ið Ið Ið Irj   rŠ   c                   óð   ‡ — e Zd Zdedefˆ fd„Zd„ Zd„ Z	 ddej	        de
ej	        ej	        f         d	ej	        dz  d
edz  dee         de
ej	        ej	        dz  e
ej	                 dz  f         fd„Zˆ xZS )ÚMiniMaxLightningAttentionÚconfigr“   c                 ó   •— t          ¦   «                              ¦   «          || _        t          |dd ¦  «        p|j        |j        z  | _        |j        | _        |j        | _        |j        | _        t          |j
                 | _        t          | j        | j        z  ¦  «        | _        t          j        |j        | j        | j        z  dz  d¬¦  «        | _        t          j        | j        | j        z  |j        d¬¦  «        | _        t          j        |j        | j        | j        z  d¬¦  «        | _        |                      ¦   «         }|                      |¦  «        \  }}}|                      d|¦  «         |                      d|¦  «         |                      d|¦  «         |                      d|¦  «         |j        |         | _        d S )	NrD   r   F)ÚbiasÚ
slope_rateÚquery_decayÚ	key_decayÚdiagonal_decay)rm   r�   r“   Úgetattrr=   rA   rD   r@   rZ   r   rF   Úact_fnr†   r7   r   ÚLinearÚqkv_projÚout_projÚoutput_gateÚget_slope_rateÚdecay_factorsÚregister_bufferrX   Ú
layer_type)ro   r¬   r“   r¯   r°   r±   r²   rq   s          €rh   r�   z"MiniMaxLightningAttention.__init__Ç   s   ø€ Ý‰Œ×ÒÑÔÐØ"ˆŒÝ ¨
°DÑ9Ô9Ðm¸VÔ=OÐSYÔSmÑ=mˆŒØ#)Ô#=ˆÔ Ø!'Ô!9ˆÔØ Ô+ˆŒå˜VÔ.Ô/ˆŒÝ" 4¤=°4Ô3KÑ#KÑLÔLˆŒ	Ýœ	 &Ô"4°dÔ6NÐQUÔQ^Ñ6^ÐabÑ6bÐinÐoÑoÔoˆŒÝœ	 $Ô":¸T¼]Ñ"JÈFÔL^ÐejÐkÑkÔkˆŒÝœ9 VÔ%7¸Ô9QÐTXÔTaÑ9aÐhmÐnÑnÔnˆÔà×(Ò(Ñ*Ô*ˆ
Ø15×1CÒ1CÀJÑ1OÔ1OÑ.ˆ�Y à×Ò˜\¨:Ñ6Ô6Ð6Ø×Ò˜]¨KÑ8Ô8Ð8Ø×Ò˜[¨)Ñ4Ô4Ð4Ø×ÒÐ-¨~Ñ>Ô>Ð>à Ô,¨YÔ7ˆŒˆˆrj   c                 óÂ   — ddd| j         z  z  z  }t          j        | j         ¦  «        dz   }d| j        | j        dz
  dz   z  z
  dz   }||z  }||z  }|d d …d d f         }|S )NrM   r   rB   rI   )rA   r¨   Úaranger“   r@   )ro   ÚbaseÚexponentÚfactorÚrates        rh   r¹   z(MiniMaxLightningAttention.get_slope_rateß   s~   € Ø�A˜!˜dÔ6Ñ6Ñ7Ñ8ˆÝ”< Ô 8Ñ9Ô9¸AÑ=ˆØ�T”^ tÔ'=ÀÑ'AÀDÑ'HÑIÑIÈDÑPˆà�X‰~ˆØ�f‰}ˆØ�A�A�A�t˜T�MÔ"ˆàˆrj   c                 ó°  — t          j        | j        ¦  «        dz   }t          j        | |d d …d f         z  ¦  «        }t          j        | | j        |d d …d f         z
  z  ¦  «        }|d d …d f         |d d d …f         z
  }|d d d d …d d …f         }||z  }t          j        |dk    | t          d¦  «        ¦  «        }t          j        |¦  «        }|||fS )NrM   r   z-inf)r¨   r¾   rZ   ÚexpÚwherer€   )ro   r¯   Úblock_size_ranger°   r±   r²   s         rh   rº   z'MiniMaxLightningAttention.decay_factorsê   sô   € Ý œ<¨¬Ñ8Ô8¸1Ñ<Ðå”i  Ð.>¸q¸q¸qÀ$¸wÔ.GÑ GÑHÔHˆÝ”I˜z˜k¨T¬_Ð?OÐPQÐPQÐPQÐSWÐPWÔ?XÑ-XÑYÑZÔZˆ	à)¨!¨!¨!¨T¨'Ô2Ð5EÀdÈAÈAÈAÀgÔ5NÑNˆØ'¨¨d°A°A°A°q°q°qÐ(8Ô9ˆØ# nÑ4ˆÝœ ^°qÒ%8¸>¸/Í5ÐQWÉ=Ì=ÑYÔYˆÝœ >Ñ2Ô2ˆà˜I ~Ð5Ð5rj   Nr3   Úposition_embeddingsr4   r)   rp   Úreturnc                 ó	  — |j         \  }}}|| j        z   dz
  | j        z  }	|                      |                      |¦  «        ¦  «        }
|
                     ||| j        d| j        z  ¦  «        }
t          j        |
| j        d¬¦  «        \  }}}| 	                    dd¦  «        }| 	                    dd¦  «        }| 	                    dd¦  «        }d }|�| 
                    | j        ¦  «        }|�€t          j        || j        | j        | j        ¦  «                             |¦  «        }|�]|                     t          j        ¬¦  «        }|                     |                     d¦  «                             d¦  «         d¦  «        }g }t#          |	¦  «        D �]a}|| j        z  }t%          || j        z   |¦  «        }||z
  }|d d …d d …||…f         }|d d …d d …||…f         }|d d …d d …||…f         }| j        d d …d |…f         }| j        d d …| d …f         }| j        d d …d d …d |…d |…f         }t          j        | j         |z  ¦  «        }t          j        || 	                    dd¦  «        ¦  «        }t          j        ||z  |¦  «        }t          j        ||z  |¦  «        }||z   }|                     |¦  «         t          j        ||z   	                    dd¦  «        |¦  «        }||z  |z   }�ŒcnÅt          j        | j         ¦  «        } g }t#          |¦  «        D ]™}|d d …d d …||dz   …f         }|d d …d d …||dz   …f         }|d d …d d …||dz   …f         }t          j        | 	                    dd¦  «        |¦  «        }!| |z  |!z   }t          j        ||¦  «        }|                     |¦  «         Œšt          j        |d¬¦  «        }| 	                    dd¦  «        }|                     ||| j        | j        z  ¦  «        }|                      |¦  «        }t9          j        |                      |¦  «        ¦  «        |z  }|                      |¦  «        }|�|                      | j        |¦  «         ||fS )	NrM   r   r�   r   )Údtypeéÿÿÿÿr   éþÿÿÿ)!ÚshaperZ   r´   r¶   ÚreshaperA   rD   r¨   ÚsplitÚ	transposer—   r“   ÚzerosÚtore   Úmasked_fillÚ	unsqueezerl   Úminr°   r±   r²   rÄ   r¯   Úmatmulr’   Úcatr7   ÚFÚsigmoidr¸   r·   r•   )"ro   r3   rÇ   r4   r)   rp   Ú
batch_sizeÚseq_lenr=   Ú
num_blocksÚ
qkv_statesÚquery_statesÚ
key_statesÚvalue_statesÚattn_weights_interÚattn_outputrg   Ú	start_idxÚend_idxÚcurrent_block_sizeÚcurrent_query_statesÚcurrent_key_statesÚcurrent_value_statesÚcurrent_query_decayÚcurrent_key_decayÚcurrent_diagonal_decayÚblock_decayÚattn_weights_intraÚattn_output_intraÚattn_output_interÚcurrent_attn_outputÚnext_attn_weights_interÚratioÚcurrent_attn_weights_inters"                                     rh   Úforwardz!MiniMaxLightningAttention.forwardø   s5  € ð ,9Ô+>Ñ(ˆ
�G˜[Ø ¤Ñ/°!Ñ3¸¼ÑGˆ
à—[’[ §¢¨}Ñ!=Ô!=Ñ>Ô>ˆ
Ø×'Ò'¨
°G¸TÔ=UÐWXÐ[_Ô[hÑWhÑiÔiˆ
å16´¸ZÈÌÐ\]Ð1^Ñ1^Ô1^Ñ.ˆ�j ,à#×-Ò-¨a°Ñ3Ô3ˆØ×)Ò)¨!¨QÑ/Ô/ˆ
Ø#×-Ò-¨a°Ñ3Ô3ˆð "ÐØÐ&Ø!0×!AÒ!AÀ$Ä.Ñ!QÔ!QÐàÑ%Ý!&¤¨Z¸Ô9QÐSWÔS`ÐbfÔboÑ!pÔ!p×!sÒ!sØñ"ô "Ðð
 Ð)Ø!/×!2Ò!2½¼Ð!2Ñ!DÔ!D�Ø+×7Ò7¸×9QÒ9QÐRSÑ9TÔ9T×9^Ò9^Ð_aÑ9bÔ9bÐ8bÐdeÑfÔf�àˆKÝ˜:Ñ&Ô&ð `ñ `�Ø ¤Ñ/�	Ý˜i¨$¬/Ñ9¸7ÑCÔC�Ø%,¨yÑ%8Ð"à'3°A°A°A°q°q°q¸)ÀGÐ:KÐ4KÔ'LÐ$Ø%/°°°°1°1°1°iÀÐ6GÐ0GÔ%HÐ"Ø'3°A°A°A°q°q°q¸)ÀGÐ:KÐ4KÔ'LÐ$à&*Ô&6°q°q°qÐ:MÐ;MÐ:MÐ7MÔ&NÐ#Ø$(¤N°1°1°1Ð7IÐ6IÐ6JÐ6JÐ3JÔ$KÐ!Ø)-Ô)<¸Q¸Q¸QÀÀÀÐCVÐDVÐCVÐXkÐYkÐXkÐ=kÔ)lÐ&Ý#œi¨¬Ð(8Ð;MÑ(MÑNÔN�õ &+¤\Ð2FÐHZ×HdÒHdÐegÐikÑHlÔHlÑ%mÔ%mÐ"Ý$)¤LÐ1CÐF\Ñ1\Ð^rÑ$sÔ$sÐ!õ %*¤LÐ1EÐH[Ñ1[Ð]oÑ$pÔ$pÐ!ð '8Ð:KÑ&KÐ#Ø×"Ò"Ð#6Ñ7Ô7Ð7õ +0¬,Ø'Ð*;Ñ;×FÒFÀrÈ2ÑNÔNÐPdñ+ô +Ð'ð &8¸+Ñ%EÐH_Ñ%_Ð"Ñ"ð;`õ@ ”I˜tœÐ.Ñ/Ô/ˆEØˆKÝ˜7‘^”^ð 	8ð 	8�Ø'3°A°A°A°q°q°q¸!¸aÀ!¹e¸)°OÔ'DÐ$Ø%/°°°°1°1°1°a¸!¸a¹%°i°Ô%@Ð"Ø'3°A°A°A°q°q°q¸!¸aÀ!¹e¸)°OÔ'DÐ$å-2¬\Ð:L×:VÒ:VÐWYÐ[]Ñ:^Ô:^Ð`tÑ-uÔ-uÐ*Ø%*Ð-?Ñ%?ÐB\Ñ%\Ð"Ý&+¤lÐ3GÐI[Ñ&\Ô&\Ð#à×"Ò"Ð#6Ñ7Ô7Ð7Ð7õ ”i °Ð4Ñ4Ô4ˆð "×+Ò+¨A¨qÑ1Ô1ˆØ!×)Ò)¨*°g¸tÔ?WÐZ^ÔZgÑ?gÑhÔhˆØ—i’i Ñ,Ô,ˆÝ”i × 0Ò 0°Ñ ?Ô ?Ñ@Ô@À;ÑNˆØ—m’m KÑ0Ô0ˆð Ð&Ø×,Ò,¨T¬^Ð=OÑPÔPÐPàÐ.Ð.Ð.rj   rŒ   )rr   rs   rt   r'   r}   r�   r¹   rº   r¨   r©   Útupler   r   r   rô   rƒ   r„   s   @rh   r«   r«   Æ   s  ø€ € € € € ð8˜}ð 8¸ð 8ð 8ð 8ð 8ð 8ð 8ð0	ð 	ð 	ð6ð 6ð 6ð& )-ð_/ð _/à”|ð_/ð # 5¤<°´Ð#=Ô>ð_/ð œ tÑ+ð	_/ð
  ™ð_/ð Ð-Ô.ð_/ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð_/ð _/ð _/ð _/ð _/ð _/ð _/ð _/rj   r«   c                   ó   — e Zd ZdS )ÚMiniMaxRotaryEmbeddingNr‡   rk   rj   rh   r÷   r÷   Z  rˆ   rj   r÷   c                   ó   — e Zd ZdS )ÚMiniMaxAttentionNr‡   rk   rj   rh   rù   rù   ^  rˆ   rj   rù   c                   ó   — e Zd ZdS )ÚMiniMaxTopKRouterNr‡   rk   rj   rh   rû   rû   b  rˆ   rj   rû   c                   ó   — e Zd ZdS )ÚMiniMaxSparseMoeBlockNr‡   rk   rj   rh   rý   rý   f  rˆ   rj   rý   c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej        deej        ej        f         dz  dej        dz  d	ej	        dz  d
e
dz  dedz  dee         deej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚMiniMaxDecoderLayerr¬   r“   c                 óÌ  •— t          ¦   «                              ||¦  «         || _        t          |d¦  «        r|j        |         nd | _        |j        | _        |j        | _        | `t          |¦  «        | _        | j        dk    r/t          ||¦  «        | _        |j        | _        |j        | _        d S t!          ||¦  «        | _        |j        | _        |j        | _        d S )NrX   rd   )rm   r�   r“   ÚhasattrrX   Ú
block_typer_   r`   Úmlprý   r«   Ú	self_attnr]   Úattn_alpha_factorr^   Úattn_beta_factorrù   r[   r\   )ro   r¬   r“   rq   s      €rh   r�   zMiniMaxDecoderLayer.__init__k  sÚ   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+à"ˆŒÝ;BÀ6È=Ñ;YÔ;YÐc˜&Ô,¨YÔ7Ð7Ð_cˆŒØ &Ô 7ˆÔØ%Ô5ˆÔØˆHÝ(¨Ñ0Ô0ˆŒØŒ?Ð0Ò0Ð0Ý6°v¸yÑIÔIˆDŒNØ%+Ô%DˆDÔ"Ø$*Ô$BˆDÔ!Ð!Ð!å-¨f°iÑ@Ô@ˆDŒNØ%+Ô%BˆDÔ"Ø$*Ô$@ˆDÔ!Ð!Ð!rj   NFr3   rÇ   r4   Úposition_idsr)   rK   rp   rÈ   c           
      ó  — |                       |¦  «        }|} | j        d||||||dœ|¤Ž\  }}	|| j        z  || j        z  z   }|                      |¦  «        }|}|                      |¦  «        }|| j        z  || j        z  z   }|S )N)r3   rÇ   r4   r  r)   rK   rk   )Úinput_layernormr  r  r  Úpost_attention_layernormr  r_   r`   )
ro   r3   rÇ   r4   r  r)   rK   rp   Úresidualr”   s
             rh   rô   zMiniMaxDecoderLayer.forward}  s¿   € ð ×,Ò,¨]Ñ;Ô;ˆØ ˆØ)˜4œ>ð 
Ø'Ø 3Ø)Ø%Ø+Øð
ð 
ð ð
ð 
Ñˆ�qð ! 4Ô#9Ñ9¸MÈDÔLaÑ<aÑaˆØ×5Ò5°mÑDÔDˆØ ˆØŸš Ñ/Ô/ˆØ  4Ô#8Ñ8¸=È4ÔK_Ñ;_Ñ_ˆàÐrj   )NNNNF)rr   rs   rt   r'   r}   r�   r¨   r©   rõ   Ú
LongTensorr   re   r   r   ÚFloatTensorrô   rƒ   r„   s   @rh   rÿ   rÿ   j  s"  ø€ € € € € ðA˜}ð A¸ð Að Að Að Að Að Að* IMØ.2Ø04Ø(,Ø!&ðð à”|ðð # 5¤<°´Ð#=Ô>ÀÑEðð œ tÑ+ð	ð
 Ô&¨Ñ-ðð  ™ðð ˜$‘;ðð Ð-Ô.ðð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uðð ð ð ð ð ð ð rj   rÿ   c                   óJ   ‡ — e Zd ZdZ eedd¬¦  «        eeegdœZ	ˆ fd„Z
ˆ xZS )ÚMiniMaxPreTrainedModelFzmlp.gater   )Ú
layer_nameÚindex)Úrouter_logitsr3   Ú
attentionsc                 ó¢  •— t          ¦   «                              |¦  «         t          |t          ¦  «        r—|                     ¦   «         }|                     |¦  «        \  }}}t          j        |j        |¦  «         t          j        |j	        |¦  «         t          j        |j
        |¦  «         t          j        |j        |¦  «         d S d S rŒ   )rm   Ú_init_weightsÚ
isinstancer«   r¹   rº   ÚinitÚcopy_r¯   r°   r±   r²   )ro   Úmoduler¯   r°   r±   r²   rq   s         €rh   r  z$MiniMaxPreTrainedModel._init_weights£  s½   ø€ Ý‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ7Ñ8Ô8ð 	>Ø×.Ò.Ñ0Ô0ˆJØ5;×5IÒ5IÈ*Ñ5UÔ5UÑ2ˆK˜ NÝŒJ�vÔ(¨*Ñ5Ô5Ð5ÝŒJ�vÔ)¨;Ñ7Ô7Ð7ÝŒJ�vÔ'¨Ñ3Ô3Ð3ÝŒJ�vÔ,¨nÑ=Ô=Ð=Ð=Ð=ð	>ð 	>rj   )rr   rs   rt   Ú_can_compile_fullgraphr   rû   rÿ   rù   r«   Ú_can_record_outputsr  rƒ   r„   s   @rh   r  r  ›  sm   ø€ € € € € Ø"Ðà'˜Ð(9ÀjÐXYÐZÑZÔZØ,Ø'Ð)BÐCðð Ðð>ð >ð >ð >ð >ð >ð >ð >ð >rj   r  c                   óÂ   — e Zd Zee	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dedz  dej	        dz  de
dz  dee         d	eez  fd
„¦   «         ¦   «         ZdS )ÚMiniMaxModelNr1   r4   r  r)   r2   rK   rp   rÈ   c           
      ó  — |d u |d uz  rt          d¦  «        ‚|r|€t          ¦   «         }n7|r5t          |t          ¦  «        s t          dt          |¦  «        › d�¦  «        ‚|€|                      |¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j	        ¬¦  «        |z   }| 
                    d¦  «        }| j        j        €t          nt          }	 |	| j        ||||¬¦  «        }
|}|                      ||¦  «        }t!          | j        ¦  «        D ]/\  }}| j        j        |         dk    r|
}n|} ||f|||||d	œ|¤Ž}Œ0|                      |¦  «        }t)          ||¬
¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embedszSMiniMax uses cache of its own and is not compatible with `past_key_values` of type ú.r   rM   )Údevice)r¬   r2   r4   r)   r  rc   )r4   rÇ   r  r)   rK   )Úlast_hidden_stater)   )Ú
ValueErrorrŠ   r  Útyper5   Úget_seq_lengthr¨   r¾   rÍ   r   rÔ   r¬   rQ   r   r   Ú
rotary_embÚ	enumerater6   rX   r7   r   )ro   r1   r4   r  r)   r2   rK   rp   Úpast_seen_tokensÚmask_functionÚcausal_maskr3   rÇ   rg   Údecoder_layerÚinput_attention_masks                   rh   rô   zMiniMaxModel.forward¯  s  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàð 	˜Ð0Ý*™nœnˆOˆOØð 	�z¨/½<ÑHÔHð 	ÝØ~ÕfjÐkzÑf{Ôf{Ð~Ð~Ð~ñô ð ð Ð Ø ×-Ò-¨iÑ8Ô8ˆMàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLà.2¬kÔ.HÐ.PÕ*Ð*ÕVwˆØ#�mØ”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆð &ˆØ"Ÿošo¨m¸\ÑJÔJÐå )¨$¬+Ñ 6Ô 6ð 	ð 	ÑˆAˆ}ØŒ{Ô& qÔ)Ð-=Ò=Ð=Ø'2Ð$Ð$ð (6Ð$à)˜MØðà3Ø$7Ø)Ø /Ø#ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå%Ø+Ø+ð
ñ 
ô 
ð 	
rj   )NNNNNN)rr   rs   rt   r   r   r¨   r  r©   rŠ   r  re   r   r   rõ   r   rô   rk   rj   rh   r  r  ®  sØ   € € € € € ØØð .2Ø.2Ø04Ø/3Ø26Ø!%ð>
ð >
àÔ# dÑ*ð>
ð œ tÑ+ð>
ð Ô&¨Ñ-ð	>
ð
 &¨Ñ,ð>
ð Ô(¨4Ñ/ð>
ð ˜$‘;ð>
ð Ð+Ô,ð>
ð 
Ð'Ñ	'ð>
ð >
ð >
ñ „_ñ  Ôð>
ð >
ð >
rj   r  c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚMiniMaxForCausalLMc                 ó6   •—  t          ¦   «         j        di |¤ŽS )a€  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, MiniMaxForCausalLM

        >>> model = MiniMaxForCausalLM.from_pretrained("MiniMaxAI/MiniMax-Text-01-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("MiniMaxAI/MiniMax-Text-01-hf")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```rk   )rm   rô   )ro   Úsuper_kwargsrq   s     €rh   rô   zMiniMaxForCausalLM.forwardó  s!   ø€ ð. �u‰wŒwŒÐ.Ð. Ð.Ð.Ð.rj   )rr   rs   rt   rô   rƒ   r„   s   @rh   r-  r-  ò  s8   ø€ € € € € ð/ð /ð /ð /ð /ð /ð /ð /ð /rj   r-  c                   ó   — e Zd ZdS )Ú MiniMaxForSequenceClassificationNr‡   rk   rj   rh   r1  r1    rˆ   rj   r1  c                   ó   — e Zd ZdS )ÚMiniMaxForTokenClassificationNr‡   rk   rj   rh   r3  r3    rˆ   rj   r3  c                   ó   — e Zd ZdS )ÚMiniMaxForQuestionAnsweringNr‡   rk   rj   rh   r5  r5    rˆ   rj   r5  )r'   r  r  r-  r1  r3  r5  )Jru   r¨   Útorch.nn.functionalr   Ú
functionalrØ   Úhuggingface_hub.dataclassesr   Ú r   r  Úactivationsr   Úcache_utilsr   r	   Úconfiguration_utilsr
   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   Úmodeling_rope_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úgemma2.modeling_gemma2r   Úmixtral.modeling_mixtralr   r   r   r   r   r   r    r!   r"   r#   r$   Ú
get_loggerrr   Úloggerr'   r†   rŠ   ÚModuler«   r÷   rù   rû   rý   rÿ   r  r  r-  r1  r3  r5  Ú__all__rk   rj   rh   ú<module>rL     sê  ðð Ð à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø .Ð .Ð .Ð .Ð .Ð .Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø &Ð &Ð &Ð &Ð &Ð &Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ :Ð :Ð :Ð :Ð :Ð :ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 
ˆÔ	˜HÑ	%Ô	%€ð €Ð9Ð:Ñ:Ô:Øðb(ð b(ð b(ð b(ð b(Ð$ñ b(ô b(ñ „ñ ;Ô:ðb(ðJ	ð 	ð 	ð 	ð 	�^ñ 	ô 	ð 	ð"Ið "Ið "Ið "Ið "I�<ñ "Iô "Ið "IðJQ/ð Q/ð Q/ð Q/ð Q/ ¤	ñ Q/ô Q/ð Q/ðh	ð 	ð 	ð 	ð 	Ð2ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð'ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð)ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð1ñ 	ô 	ð 	ð.ð .ð .ð .ð .Ð-Ð/Iñ .ô .ð .ðb>ð >ð >ð >ð >Ð3ñ >ô >ð >ð&A
ð A
ð A
ð A
ð A
�<ñ A
ô A
ð A
ðH/ð /ð /ð /ð /Ð+ñ /ô /ð /ð6	ð 	ð 	ð 	ð 	Ð'Gñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð$Añ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð"=ñ 	ô 	ð 	ðð ð €€€rj   