
    <hAk                     Z   S r SSKJr  SSKrSSKJs  Jr  SSKJr  SSKJ	r	  SSK
JrJr  SSKJr  SS	KJrJr  SS
KJr  SSKJr  SSKJr  SSKJr  SSKJrJr  SSKJr  SSKJ r   SSK!J"r"J#r#J$r$J%r%J&r&J'r'J(r(J)r)J*r*J+r+  \RX                  " \-5      r. " S S\ 5      r/ " S S\*5      r0 " S S\5      r1 " S S\Rd                  5      r3 " S S\"5      r4 " S S\+5      r5 " S S \#\5      r6 " S! S"\)5      r7 " S# S$\(5      r8 " S% S&\$5      r9 " S' S(\&5      r: " S) S*\'5      r; " S+ S,\%5      r</ S-Qr=g).zPyTorch MiniMax model.    )OptionalN)nn   )ACT2FN)CacheDynamicCache)layer_type_validation)create_causal_mask!create_sliding_window_causal_mask)FlashAttentionKwargs)GradientCheckpointingLayer)MoeModelOutputWithPast)Unpack)TransformersKwargslogging)OutputRecorder   )MixtralConfig)
MixtralAttentionMixtralDecoderLayerMixtralForCausalLMMixtralForQuestionAnswering MixtralForSequenceClassificationMixtralForTokenClassificationMixtralModelMixtralPreTrainedModelMixtralRMSNormMixtralSparseMoeBlockc                   @   ^  \ rS rSrSr        SU 4S jjrSrU =r$ )MiniMaxConfig4   ao  
This is the configuration class to store the configuration of a [`MiniMaxModel`]. It is used to instantiate an
MiniMax model according to the specified arguments, defining the model architecture. Instantiating a configuration
with the defaults will yield a similar configuration to that of the MiniMax.

[MiniMaxAI/MiniMax-Text-01-hf](https://huggingface.co/MiniMaxAI/MiniMax-Text-01-hf)

Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PretrainedConfig`] for more information.


Args:
    vocab_size (`int`, *optional*, defaults to 32000):
        Vocabulary size of the MiniMax model. Defines the number of different tokens that can be represented by the
        `inputs_ids` passed when calling [`MiniMaxModel`]
    hidden_size (`int`, *optional*, defaults to 4096):
        Dimension of the hidden representations.
    intermediate_size (`int`, *optional*, defaults to 14336):
        Dimension of the MLP representations.
    num_hidden_layers (`int`, *optional*, defaults to 32):
        Number of hidden layers in the Transformer encoder.
    num_attention_heads (`int`, *optional*, defaults to 32):
        Number of attention heads for each attention layer in the Transformer encoder.
    num_key_value_heads (`int`, *optional*, defaults to 8):
        This is the number of key_value heads that should be used to implement Grouped Query Attention. If
        `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
        `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
        converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
        by meanpooling all the original heads within that group. For more details, check out [this
        paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to `8`.
    head_dim (`int`, *optional*, defaults to `hidden_size // num_attention_heads`):
        The attention head dimension.
    hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
        The non-linear activation function (function or string) in the decoder.
    max_position_embeddings (`int`, *optional*, defaults to `4096*32`):
        The maximum sequence length that this model might ever be used with. MiniMax's sliding window attention
        allows sequence of up to 4096*32 tokens.
    initializer_range (`float`, *optional*, defaults to 0.02):
        The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
    rms_norm_eps (`float`, *optional*, defaults to 1e-05):
        The epsilon used by the rms normalization layers.
    use_cache (`bool`, *optional*, defaults to `True`):
        Whether or not the model should return the last key/values attentions (not used by all models). Only
        relevant if `config.is_decoder=True`.
    pad_token_id (`int`, *optional*):
        The id of the padding token.
    bos_token_id (`int`, *optional*, defaults to 1):
        The id of the "beginning-of-sequence" token.
    eos_token_id (`int`, *optional*, defaults to 2):
        The id of the "end-of-sequence" token.
    tie_word_embeddings (`bool`, *optional*, defaults to `False`):
        Whether the model's input and output word embeddings should be tied.
    rope_theta (`float`, *optional*, defaults to 1000000.0):
        The base period of the RoPE embeddings.
    sliding_window (`int`, *optional*):
        Sliding window attention window size. If not specified, will default to `4096`.
    attention_dropout (`float`, *optional*, defaults to 0.0):
        The dropout ratio for the attention probabilities.
    num_experts_per_tok (`int`, *optional*, defaults to 2):
        The number of experts to route per-token, can be also interpreted as the `top-k` routing
        parameter
    num_local_experts (`int`, *optional*, defaults to 8):
        Number of experts per Sparse MLP layer.
    output_router_logits (`bool`, *optional*, defaults to `False`):
        Whether or not the router logits should be returned by the model. Enabeling this will also
        allow the model to output the auxiliary loss. See [here]() for more details
    router_aux_loss_coef (`float`, *optional*, defaults to 0.001):
        The aux loss factor for the total loss.
    router_jitter_noise (`float`, *optional*, defaults to 0.0):
        Amount of noise to add to the router.
    layer_types (`list`, *optional*):
        Attention pattern for each layer.
    block_size (`int`, *optional*, defaults to 256):
        The length of each attention block, determining how queries, keys, and values
        are grouped and processed for intra- and inter-block attention.
    full_attn_alpha_factor (`float`, *optional*, defaults to 1):
        Weight for residual value in residual connection after normal attention.
    full_attn_beta_factor (`float`, *optional*, defaults to 1):
        Weight for hidden state value in residual connection after normal attention.
    linear_attn_alpha_factor (`float`, *optional*, defaults to 1):
        Weight for residual value in residual connection after lightning attention.
    linear_attn_beta_factor (`float`, *optional*, defaults to 1):
        Weight for hidden state value in residual connection after lightning attention.
    mlp_alpha_factor (`float`, *optional*, defaults to 1):
        Weight for residual value in residual connection after MLP.
    mlp_beta_factor (`float`, *optional*, defaults to 1):
        Weight for hidden state value in residual connection after MLP.

```python
>>> from transformers import MiniMaxModel, MiniMaxConfig

>>> # Initializing a MiniMax style configuration
>>> configuration = MiniMaxConfig()

>>> # Initializing a model from the MiniMax style configuration
>>> model = MiniMaxModel(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```c	                 X  > [         TU ]  " S0 U	D6  Xl        X l        X0l        X@l        XPl        X`l        Xpl        Xl	        U R                  cB  [        U R                  5       V
s/ sH  n
[        U
S-   S-  5      (       a  SOSPM     sn
U l        [        U R                  5        g s  sn
f )N   r   full_attentionlinear_attention )super__init__layer_types
block_sizefull_attn_alpha_factorfull_attn_beta_factorlinear_attn_alpha_factorlinear_attn_beta_factormlp_alpha_factormlp_beta_factorrangenum_hidden_layersboolr	   )selfr)   r*   r+   r,   r-   r.   r/   r0   super_kwargsi	__class__s              c/var/www/html/shao/venv/lib/python3.13/site-packages/transformers/models/minimax/modular_minimax.pyr(   MiniMaxConfig.__init__   s     	(<(&$&<#%:"(@%'>$ 0.#W\]a]s]sWt WtRSD!a%1$5$5 ;MMWt D 	d../ s   &#B')r*   r+   r,   r)   r-   r.   r/   r0   )N   r#   r#   r#   r#   r#   r#   )__name__
__module____qualname____firstlineno____doc__r(   __static_attributes____classcell__r7   s   @r8   r    r    4   s.    cN  !" !0 0    r    c                       \ rS rSrSrg)MiniMaxRMSNorm   r&   Nr;   r<   r=   r>   r@   r&   rC   r8   rE   rE          rC   rE   c                      ^  \ rS rSrU 4S jrS rS\4S jrU 4S jrS\4U 4S jjr	S r
S	\4S
 jrS\R                  4S jrS\4S jrSrU =r$ )MiniMaxCache   c                 0   > [         TU ]  5         / U l        g N)r'   r(   linear_cacher4   r7   s    r8   r(   MiniMaxCache.__init__   s    02rC   c                     [        [        U R                  5      US-   5       H  nU R                  R                  / 5        M      X R                  U'   g )Nr#   )r1   lenrN   append)r4   	layer_idxrN   _s       r8   set_linear_cacheMiniMaxCache.set_linear_cache   sD    s4,,-y1}=A$$R( >'3)$rC   rT   c                 @    U[        U 5      :  a  U R                  U   $ g rM   )rR   rN   r4   rT   s     r8   get_linear_cacheMiniMaxCache.get_linear_cache   s"    s4y $$Y//rC   c                 Z   > [        [        TU ]	  5       [        U R                  5      5      $ rM   )maxr'   __len__rR   rN   rO   s    r8   r^   MiniMaxCache.__len__   s"    57?$c$*;*;&<==rC   c                    > U[        U R                  5      :  a#  U R                  U   / :w  a  U R                  U   4$ [        TU ]  U5      $ rM   )rR   rN   r'   __getitem__)r4   rT   r7   s     r8   ra   MiniMaxCache.__getitem__   sM    s4,,--$2C2CI2NRT2T%%i022w"9--rC   c              #   N   #    [        [        U 5      5       H	  nX   v   M     g 7frM   )r1   rR   rY   s     r8   __iter__MiniMaxCache.__iter__   s      s4y)I/! *s   #%repeatsc                     [        [        U 5      5       H`  nU R                  U   / :w  a,  U R                  U   R                  USS9U R                  U'   MB  U R                  U   R                  U5        Mb     g )Nr   dim)r1   rR   rN   repeat_interleavelayersbatch_repeat_interleave)r4   rf   rT   s      r8   rl   $MiniMaxCache.batch_repeat_interleave   sl    s4y)I  +r1/3/@/@/K/]/]^ekl/]/m!!),I&>>wG	 *rC   indicesc                     [        [        U 5      5       HW  nU R                  U   / :w  a#  U R                  U   US4   U R                  U'   M9  U R                  U   R	                  U5        MY     g )N.)r1   rR   rN   rk   batch_select_indices)r4   rn   rT   s      r8   rp   !MiniMaxCache.batch_select_indices   sd    s4y)I  +r1/3/@/@/KGUXL/Y!!),I&;;GD	 *rC   
max_lengthc                     [        S5      e)Nz*MiniMaxCache doesnot support `crop` method)RuntimeError)r4   rr   s     r8   cropMiniMaxCache.crop   s    GHHrC   )rN   )r;   r<   r=   r>   r(   rV   intrZ   r^   ra   rd   rl   torchTensorrp   ru   r@   rA   rB   s   @r8   rJ   rJ      sc    34# 
>.S .
"Hs HEELL EIs I IrC   rJ   c                   N  ^  \ rS rSrS\S\4U 4S jjrS rS r  SS\	R                  S\\	R                  \	R                  4   S	\\	R                     S
\\   S\\	R                     S\\   S\\	R                  \\	R                     \\\	R                        4   4S jjrSrU =r$ )MiniMaxLightningAttention   configrT   c                   > [         TU ]  5         X l        [        USS 5      =(       d    UR                  UR
                  -  U l        UR
                  U l        UR                  U l        UR                  U l        [        UR                     U l        [        U R                  U R
                  -  5      U l        [        R                  " UR                  U R
                  U R                  -  S-  SS9U l        [        R                  " U R
                  U R                  -  UR                  SS9U l        [        R                  " UR                  U R
                  U R                  -  SS9U l        U R'                  5       nU R)                  U5      u  pEnU R+                  SU5        U R+                  SU5        U R+                  SU5        U R+                  SU5        g )	Nhead_dimr   F)bias
slope_ratequery_decay	key_decaydiagonal_decay)r'   r(   rT   getattrhidden_sizenum_attention_headsr   r2   r*   r   
hidden_actact_fnrE   normr   Linearqkv_projout_projoutput_gateget_slope_ratedecay_factorsregister_buffer)r4   r}   rT   r   r   r   r   r7   s          r8   r(   "MiniMaxLightningAttention.__init__   s   "
D9mV=O=OSYSmSm=m#)#=#= !'!9!9 ++V../"4==43K3K#KL			&"4"4d6N6NQUQ^Q^6^ab6bino		$":":T]]"JFL^L^ejk99V%7%79Q9QTXTaTa9ahmn((*
151C1CJ1O.\:6]K8[)4-~>rC   c                     SSSU R                   -  -  -  n[        R                  " U R                   5      S-   nSU R                  U R                  S-
  S-   -  -
  S-   nX-  nXC-  nUS S 2S S 4   nU$ )Nr#   r      gh㈵>)r   rx   arangerT   r2   )r4   baseexponentfactorrates        r8   r   (MiniMaxLightningAttention.get_slope_rate   s    A!d66678<< 8 89A=T^^t'='='AD'HIIDP~}AtTM"rC   c                    [         R                  " U R                  5      S-   n[         R                  " U* US S 2S 4   -  5      n[         R                  " U* U R                  US S 2S 4   -
  -  5      nUS S 2S 4   US S S 24   -
  nUS S S S 2S S 24   nX-  n[         R                  " US:  U* [        S5      5      n[         R                  " U5      nX4U4$ )Nr#   r   z-inf)rx   r   r*   expwherefloat)r4   r   block_size_ranger   r   r   s         r8   r   'MiniMaxLightningAttention.decay_factors  s     <<81<ii.>q$w.G GHIIzkT__?OPQSWPW?X-XYZ	)!T'25EdAg5NN'dAq(89#4^q%8>/5QW=Y>2~55rC   hidden_statesposition_embeddingsattention_maskpast_key_valuecache_positionkwargsreturnc                 	   UR                   u  pxn	XR                  -   S-
  U R                  -  n
U R                  U R                  U5      5      nUR	                  XxU R
                  SU R                  -  5      n[        R                  " XR                  SS9u  pnUR                  SS5      nUR                  SS5      nUR                  SS5      nS nUb  UR                  U R                  5      nUGc  [        R                  " XpR
                  U R                  U R                  5      R                  U5      nUbN  UR                  [        R                  S9nUR                  UR!                  S5      R!                  S5      ) S5      n/ n[#        U
5       GHh  nUU R                  -  n[%        UU R                  -   U5      nUU-
  nUS S 2S S 2UU24   nUS S 2S S 2UU24   nUS S 2S S 2UU24   nU R&                  S S 2S U24   nU R(                  S S 2U* S 24   nU R*                  S S 2S S 2S U2S U24   n[        R,                  " U R.                  * U-  5      n[        R0                  " UUR                  SS5      5      n[        R0                  " UU-  U5      n[        R0                  " UU-  U5      nUU-   nUR3                  U5        [        R0                  " UU-  R                  SS5      U5      n UU-  U -   nGMk     O[        R,                  " U R.                  * 5      n!/ n[#        U5       H  nUS S 2S S 2UUS-   24   nUS S 2S S 2UUS-   24   nUS S 2S S 2UUS-   24   n[        R0                  " UR                  SS5      U5      n"U!U-  U"-   n[        R0                  " UU5      nUR3                  U5        M     [        R4                  " USS9nUR                  SS5      nUR	                  XxU R
                  U R                  -  5      nU R7                  U5      n[8        R:                  " U R=                  U5      5      U-  nU R?                  U5      nUb  URA                  U R                  U5        UU4$ )	Nr#   r   rh   r   )dtyper   )!shaper*   r   r   reshaper   r   rx   split	transposerZ   rT   zerostor3   masked_fill	unsqueezer1   minr   r   r   r   r   matmulrS   catr   Fsigmoidr   r   rV   )#r4   r   r   r   r   r   r   
batch_sizeseq_lenr   
num_blocks
qkv_statesquery_states
key_statesvalue_statesattn_weights_interattn_outputr6   	start_idxend_idxcurrent_block_sizecurrent_query_statescurrent_key_statescurrent_value_statescurrent_query_decaycurrent_key_decaycurrent_diagonal_decayblock_decayattn_weights_intraattn_output_intraattn_output_intercurrent_attn_outputnext_attn_weights_interratiocurrent_attn_weights_inters#                                      r8   forward!MiniMaxLightningAttention.forward  s    ,9+>+>(
[/!3G
[[}!=>
''
T=U=UWX[_[h[hWhi
16Z\]1^.,#--a3))!Q/
#--a3 "%!/!@!@!P%!&Z9Q9QSWS`S`bfbobo!p!s!s"
 )!/!2!2!2!D+779Q9QRS9T9^9^_a9b8bdefK:&/	i$//97C%,y%8"'3Aq)G:K4K'L$%/1i6G0G%H"'3Aq)G:K4K'L$&*&6&6q:M;M:M7M&N#$(NN17I6I6J3J$K!)-)<)<QCVDVCVXkYkXk=k)l&#ii(8;M(MN &+\\2FHZHdHdegikHl%m"$)LL1CF\1\^r$s! %*LL1EH[1[]o$p! '8:K&K#""#67 +0,,'*;;FFr2NPd+' &8+%EH_%_"; '@ IIt./EK7^'3Aq!a!e)O'D$%/1a!a%i%@"'3Aq!a!e)O'D$-2\\:L:V:VWY[]:^`t-u*%*-?%?B\%\"&+ll3GI[&\#""#67 $ ii4 "++Aq1!))*t?W?WZ^ZgZg?ghii,ii 0 0 ?@;NmmK0 %++DNN<NO...rC   )
r   r*   r   rT   r   r   r2   r   r   r   )NN)r;   r<   r=   r>   r    rw   r(   r   r   rx   ry   tupler   r   
LongTensorr   r   r   r@   rA   rB   s   @r8   r{   r{      s    ?} ? ?,	6& +/59`/||`/ #5<<#=>`/ !.	`/
 !`/ !!1!12`/ -.`/ 
u||Xell3XeELL>Q5RR	S`/ `/rC   r{   c                       \ rS rSrSrg)MiniMaxAttentioni|  r&   NrG   r&   rC   r8   r   r   |  rH   rC   r   c                       \ rS rSrSrg)MiniMaxSparseMoeBlocki  r&   NrG   r&   rC   r8   r   r     rH   rC   r   c                     ^  \ rS rSrS\S\4U 4S jjr       SS\R                  S\	\R                  \R                  4   S\
\R                     S\
\R                     S	\
\	\R                        S
\
\   S\
\   S\
\   S\
\R                     S\\   S\	\R                  \
\	\R                  \R                  4      4   4S jjrSrU =r$ )MiniMaxDecoderLayeri  r}   rT   c                   > [         TU ]  X5        X l        UR                  U   U l        UR
                  U l        UR                  U l        U R                  S:X  a3  [        X5      U l        UR                  U l
        UR                  U l        g [        X5      U l        UR                  U l
        UR                  U l        g )Nr%   )r'   r(   rT   r)   
layer_typer/   r0   r{   	self_attnr-   attn_alpha_factorr.   attn_beta_factorr   r+   r,   )r4   r}   rT   r7   s      r8   r(   MiniMaxDecoderLayer.__init__  s    +" ,,Y7 & 7 7%55??006vIDN%+%D%DD"$*$B$BD!-f@DN%+%B%BD"$*$@$@D!rC   r   r   r   position_idsr   output_attentionsoutput_router_logits	use_cacher   r   r   c
                 &   U R                  U5      nUnU R                  " SUUUUUUUU	S.U
D6u  pXR                  -  XR                  -  -   nU R	                  U5      nUnU R                  U5      u  pXR                  -  XR                  -  -   nU$ )a  
Args:
    hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
    position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`):
        Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
        with `head_dim` being the embedding dimension of each attention head.
    attention_mask (`torch.Tensor`, *optional*): attention mask of size
        `(batch, sequence_length)` where padding elements are indicated by 0.
    past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers. See `attentions` under
        returned tensors for more detail.
    output_router_logits (`bool`, *optional*):
        Whether or not to return the logits of all the routers. They are useful for computing the router loss, and
        should not be returned during inference.
    use_cache (`bool`, *optional*):
        If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
        (see `past_key_values`).
    cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
        Indices depicting the position of the input sequence tokens in the sequence.
    kwargs (`dict`, *optional*):
        Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
        into the model
)r   r   r   r   r   r   r   r   r&   )input_layernormr   r   r   post_attention_layernormblock_sparse_moer/   r0   )r4   r   r   r   r   r   r   r   r   r   r   residualrU   s                r8   r   MiniMaxDecoderLayer.forward  s    L ,,];   >> 

' 3)%)/)

 

 !#9#99MLaLa<aa 55mD 00? #8#88=K_K_;__rC   )r   r   rT   r   r/   r0   r   )NNNFFFN)r;   r<   r=   r>   r    rw   r(   rx   ry   r   r   r   r3   r   r   FloatTensorr   r@   rA   rB   s   @r8   r   r     s&   A} A A* 26378<,1/4$)59=||= #5<<#=>= !.	=
 u//0= !u||!45= $D>= 'tn= D>= !!1!12= -.= 
u  (51B1BEDUDU1U+V"WW	X= =rC   r   c                   2    \ rS rSrSr\" \SS9\\\	/S.r
Srg)MiniMaxPreTrainedModeli  Fr#   )index)router_logitsr   
attentionsr&   N)r;   r<   r=   r>   _can_compile_fullgraphr   r   r   r   r{   _can_record_outputsr@   r&   rC   r8   r   r     s'    "'(=QG,')BCrC   r   c                       \ rS rSr        SS\R
                  S\\R                     S\\R
                     S\\   S\\R                     S\\
   S	\\
   S
\\R
                     S\\   S\4S jjrSrg)MiniMaxModeli  N	input_idsr   r   past_key_valuesinputs_embedsr   r   r   r   r   c	                    US L US L-  (       a  [        S5      eU(       a  Uc  [        5       nO4U(       a-  [        U[        5      (       d  [        S[        U5       S35      eUc  U R	                  U5      nUcD  Ub  UR                  5       OSn
[        R                  " XUR                  S   -   UR                  S9nUc  UR                  S5      nU R                  R                  c  [        O[        nU" U R                  UUUUUS9nUnU R                  X5      nU R                    H(  nUR"                  S:X  a  UnOUnU" U4UUUUUUS	.U	D6nM*     U R%                  U5      n['        UUS
9$ )Nz:You must specify exactly one of input_ids or inputs_embedszSMiniMax uses cache of its own and is not compatible with `past_key_values` of type .r   r#   )device)r}   input_embedsr   r   r   r   r$   )r   r   r   r   r   r   )last_hidden_stater   )
ValueErrorrJ   
isinstancetypeembed_tokensget_seq_lengthrx   r   r   r   r   r}   sliding_windowr
   r   
rotary_embrk   r   r   r   )r4   r   r   r   r   r   r   r   r   r   past_seen_tokensmask_functioncausal_maskr   r   decoder_layerinput_attention_masks                    r8   r   MiniMaxModel.forward  s    -t";<YZZ0*nOz/<HHefjkzf{e||}~    --i8M!CRC^==?de"\\ ]5H5H5K"KTaThThN )33A6L.2kk.H.H.P*Vw#;;&))+%
 & #oomJ![[M''+;;'2$ (6$)	$73).#-	 	M )$ 		-0%++
 	
rC   r&   )NNNNNNNN)r;   r<   r=   r>   rx   r   r   ry   rJ   r   r3   r   r   r   r   r@   r&   rC   r8   r   r     s     '+15372659$(,059G
##G
 !.G
 u//0	G

 ",/G
   1 12G
 D>G
 $D>G
 !!1!12G
 +,G
 
 G
 G
rC   r   c                   (   ^  \ rS rSrU 4S jrSrU =r$ )MiniMaxForCausalLMi*  c                 $   > [         TU ]  " S0 UD6$ )a   
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, MiniMaxForCausalLM

>>> model = MiniMaxForCausalLM.from_pretrained("MiniMaxAI/MiniMax-Text-01-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("MiniMaxAI/MiniMax-Text-01-hf")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```r&   )r'   r   )r4   r5   r7   s     r8   r   MiniMaxForCausalLM.forward+  s    . w...rC   r&   )r;   r<   r=   r>   r   r@   rA   rB   s   @r8   r  r  *  s    / /rC   r  c                       \ rS rSrSrg) MiniMaxForSequenceClassificationiE  r&   NrG   r&   rC   r8   r  r  E  rH   rC   r  c                       \ rS rSrSrg)MiniMaxForTokenClassificationiI  r&   NrG   r&   rC   r8   r  r  I  rH   rC   r  c                       \ rS rSrSrg)MiniMaxForQuestionAnsweringiM  r&   NrG   r&   rC   r8   r  r  M  rH   rC   r  )r    r   r   r  r  r  r  )>r?   typingr   rx   torch.nn.functionalr   
functionalr   activationsr   cache_utilsr   r   configuration_utilsr	   masking_utilsr
   r   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   processing_utilsr   utilsr   r   utils.genericr   mixtral.configuration_mixtralr   mixtral.modeling_mixtralr   r   r   r   r   r   r   r   r   r   
get_loggerr;   loggerr    rE   rJ   Moduler{   r   r   r   r   r   r  r  r  r  __all__r&   rC   r8   <module>r)     s          ! . 8 R B 9 6 & 0 + 9   
		H	%@0M @0F	^ 	+I< +I\P/		 P/f	' 		1 	O-/I Od3 H
< H
V/+ /6	'G 		$A 		"= 	rC   