§
    ‚Štjh¦  ã                   óà  — d Z ddlZddlmZ ddlZddlmZ ddlmZ ddl	m
Z
 ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZmZ ddlmZ ddlmZ  ej        e ¦  «        Z!dej"        de#fd„Z$d„ Z%d„ Z&d„ Z' G d„ dej        j(        ¦  «        Z) G d„ dej(        ¦  «        Z* G d„ dej(        ¦  «        Z+ G d„ de¦  «        Z,e G d„ d e¦  «        ¦   «         Z- ed!¬"¦  «        e G d#„ d$e¦  «        ¦   «         ¦   «         Z. ed%¬"¦  «        e G d&„ d'e¦  «        ¦   «         ¦   «         Z/e G d(„ d)e-¦  «        ¦   «         Z0 ed*¬"¦  «         G d+„ d,e-e¦  «        ¦   «         Z1g d-¢Z2dS ).zPyTorch MAMBA2 model.é    N)Ú	dataclass)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úlazy_load_kernel)ÚGradientCheckpointingLayer)ÚPreTrainedModel)ÚModelOutputÚauto_docstringÚis_torchdynamo_compilingÚlogging)Úresolve_internal_importé   )ÚMamba2ConfigÚinput_tensorÚpad_sizec                 ó¦   — t          | j        ¦  «        dk    r
ddddd|ddfnddd|ddf}t          j        j                             | |dd¬¦  «        S )z‚
    Padding x tensor with `pad_size` on the seq_len dim (dim=1)

    Assumes that we only have tensors of either size 4 or 3
    é   r   Úconstant)ÚmodeÚvalue)ÚlenÚshapeÚtorchr   Ú
functionalÚpad)r   r   Ú	pad_shapes      úh/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/mamba2/modeling_mamba2.pyÚpad_tensor_by_sizer#   (   sj   € õ 47°|Ô7IÑ3JÔ3JÈaÒ3OÐ3O��A�q˜!˜Q ¨!¨QÐ/Ð/ÐVWÐYZÐ\]Ð_gÐijÐlmÐUn€IåŒ8Ô×"Ò" <°ÀÐSTÐ"ÑUÔUÐUó    c                 ó"  — t          | |¦  «        } t          | j        ¦  «        dk    r.|                      | j        d         d|| j        d         ¦  «        S |                      | j        d         d|| j        d         | j        d         ¦  «        S )zÀ
    Padding input_tensor with `pad_size` on the seq_len dim (dim=1) and
    simultaneously splitting it into chunk sequences.

    Assumes that we only have tensors of either size 4 or 3
    r   r   éÿÿÿÿé   )r#   r   r   Úreshape)r   r   Ú
chunk_sizes      r"   Úreshape_into_chunksr*   3   s’   € õ & l°HÑ=Ô=€Lå
ˆ<ÔÑÔ !Ò#Ð#à×#Ò# LÔ$6°qÔ$9¸2¸zÈ<ÔK]Ð^_ÔK`ÑaÔaÐað ×#Ò#ØÔ˜qÔ! 2 z°<Ô3EÀaÔ3HÈ,ÔJ\Ð]^ÔJ_ñ
ô 
ð 	
r$   c                 ó  — |                       d¦  «        } | d         j        g |                       ¦   «         ¢|‘R Ž } t          j        t          j        ||| j        t          j        ¬¦  «        d¬¦  «        }|                      | d¦  «        } t          j        | d¬¦  «        }t          j        t          j        ||| j        t          j        ¬¦  «        d¬¦  «        }|                     | t          j	         ¦  «        }|S )zo
    More stable segment sum calculation. Uses cumulative sums and masking instead of direct subtractions.
    r&   ©.N©ÚdeviceÚdtype)Údiagonalr   éþÿÿÿ©Údim)
ÚsizeÚexpandr   ÚtrilÚonesr.   ÚboolÚmasked_fillÚcumsumÚinf)r   r)   ÚmaskÚtensor_segsums       r"   Úsegment_sumr>   G   só   € ð ×"Ò" 2Ñ&Ô&€Jð 2�< 	Ô*Ô1ÐS°<×3DÒ3DÑ3FÔ3FÐSÈ
ÐSÐSÐS€LåŒ:•e”j ¨ZÀÔ@SÕ[`Ô[eÐfÑfÔfÐqsÐtÑtÔt€DØ×+Ò+¨T¨E°1Ñ5Ô5€Lå”L °2Ð6Ñ6Ô6€Mõ Œ:•e”j ¨ZÀÔ@SÕ[`Ô[eÐfÑfÔfÐqrÐsÑsÔs€DØ!×-Ò-¨t¨eµe´i°ZÑ@Ô@€MØÐr$   c                 ó¦   — |�N|j         d         dk    r=|j         d         dk    r,| j        }| |dd…dd…df         z                       |¦  «        } | S )zm
    Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66
    Nr   r   )r   r/   Úto)Úhidden_statesÚattention_maskr/   s      r"   Úapply_mask_to_padding_statesrC   [   si   € ð
 Ð! nÔ&:¸1Ô&=ÀÒ&AÐ&AÀnÔFZÐ[\ÔF]Ð`aÒFaÐFaØÔ#ˆØ&¨¸¸¸¸1¸1¸1¸d¸
Ô)CÑC×GÒGÈÑNÔNˆàÐr$   c                   ó(   ‡ — e Zd Zdˆ fd„	Zdd„Zˆ xZS )ÚMambaRMSNormGatedç�íµ ÷Æ°>c                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        d S ©N©ÚsuperÚ__init__r   Ú	Parameterr   r7   ÚweightÚvariance_epsilon©ÚselfÚhidden_sizeÚepsÚ	__class__s      €r"   rK   zMambaRMSNormGated.__init__h   sB   ø€ Ý‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr$   Nc                 óœ  — |j         }|                     t          j        ¦  «        }|�?|t          j                             |                     t          j        ¦  «        ¦  «        z  }|                     d¦  «                             dd¬¦  «        }|t          j	        || j
        z   ¦  «        z  }| j        |                     |¦  «        z  S ©Nr'   r&   T)Úkeepdim)r/   r@   r   Úfloat32r   r   ÚsiluÚpowÚmeanÚrsqrtrN   rM   )rP   rA   ÚgateÚinput_dtypeÚvariances        r"   ÚforwardzMambaRMSNormGated.forwardm   sª   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆàÐØ)­B¬M×,>Ò,>¸t¿wºwÅuÄ}Ñ?UÔ?UÑ,VÔ,VÑVˆMØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆàŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r$   ©rF   rH   ©Ú__name__Ú
__module__Ú__qualname__rK   r_   Ú__classcell__©rS   s   @r"   rE   rE   g   sQ   ø€ € € € € ð$ð $ð $ð $ð $ð $ð
	;ð 	;ð 	;ð 	;ð 	;ð 	;ð 	;ð 	;r$   rE   c                   ó  ‡ — e Zd ZdZddededefˆ fd„Z ej	        ¦   «         d„ ¦   «         Z
	 	 dd	ej        d
edz  dej        dz  fd„Z	 	 dd	ej        d
edz  dej        dz  fd„Z	 	 dd
edz  dej        dz  fd„Zˆ xZS )ÚMamba2Mixeruƒ  
    Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)
    TÚconfigÚ	layer_idxÚinitialize_mixer_weightsc           	      ób  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        |j        | _        t          |j	        | j        z  ¦  «        | _
        t          |j        ¦  «        | _        || _        |j        | _        |j        | _        t           |j                 | _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        | j
        d| j        z  | j        z  z   | _        t9          j        | j        | j        |j        |j        | j        |j        dz
  ¬¦  «        | _        | j
        | j        z   | j        z   }t9          j        | j        ||j         ¬¦  «        | _!        t9          j"        tG          j$        | j        ¦  «        ¦  «        | _%        t9          j"        tG          j$        | j        ¦  «        ¦  «        | _&        tO          | j
        | j        ¬¦  «        | _(        t9          j"        tG          j$        | j        ¦  «        ¦  «        | _)        |r)| j%        j*        j+        dk    r|  ,                    ¦   «          t9          j        | j
        | j        |j         ¬¦  «        | _-        |j         | _         t]          d¦  «        }t_          |dd ¦  «        a0t_          |d	d ¦  «        a1t]          d
¦  «        }te          |d¬¦  «        a3te          |d¬¦  «        a4te          |d¬¦  «        a5tm          tf          th          tj          tb          t`          f¦  «        a7tn          stp           9                    d¦  «         |j:        |         | _;        d S )Nr'   r   )Úin_channelsÚout_channelsÚbiasÚkernel_sizeÚgroupsÚpadding©ro   ©rR   Úmetazcausal-conv1dÚcausal_conv1d_updateÚcausal_conv1d_fnz	mamba-ssmz8ops.triton.selective_state_update.selective_state_update)Úchained_pathz1ops.triton.ssd_combined.mamba_chunk_scan_combinedz8ops.triton.ssd_combined.mamba_split_conv1d_scan_combineda  The fast path is not available because one of `(selective_state_update, causal_conv1d_fn, causal_conv1d_update)` is None. Falling back to the naive implementation. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1d)<rJ   rK   Ú	num_headsrQ   Ú
state_sizeÚssm_state_sizeÚconv_kernelÚconv_kernel_sizeÚintr5   Úintermediate_sizeÚtime_step_rankrj   Úuse_conv_biasÚ
hidden_actÚ
activationr   ÚactÚlayer_norm_epsilonÚrms_normÚn_groupsÚhead_dimr)   Útime_step_limitÚtime_step_minÚtime_step_maxÚtime_step_floorÚconv_dimr   ÚConv1dÚconv1dÚLinearÚuse_biasÚin_projrL   r   ÚemptyÚdt_biasÚA_logrE   ÚnormÚDr.   ÚtypeÚinit_mamba2_weightsÚout_projr   Úgetattrrv   rw   r   Úselective_state_updateÚmamba_chunk_scan_combinedÚ mamba_split_conv1d_scan_combinedÚallÚis_fast_path_availableÚloggerÚwarning_onceÚlayer_typesÚ
layer_type)rP   ri   rj   rk   Úprojection_sizeÚcausal_conv1dÚ	mamba_ssmrS   s          €r"   rK   zMamba2Mixer.__init__�   sZ  ø€ Ý‰Œ×ÒÑÔÐØÔ)ˆŒØ!Ô-ˆÔØ$Ô/ˆÔØ &Ô 2ˆÔÝ!$ V¤]°TÔ5EÑ%EÑ!FÔ!FˆÔÝ! &Ô"7Ñ8Ô8ˆÔØ"ˆŒØ#Ô1ˆÔØ Ô+ˆŒÝ˜&Ô+Ô,ˆŒà"(Ô";ˆÔØœˆŒàœˆŒØœˆŒØ Ô+ˆŒà%Ô5ˆÔØ#Ô1ˆÔØ#Ô1ˆÔØ%Ô5ˆÔàÔ.°°T´]Ñ1BÀTÔEXÑ1XÑXˆŒÝ”iØœØœØÔ%ØÔ*Ø”=ØÔ&¨Ñ*ð
ñ 
ô 
ˆŒð Ô0°4´=Ñ@À4Ä>ÑQˆÝ”yØÔØØ”ð
ñ 
ô 
ˆŒõ ”|¥E¤K°´Ñ$?Ô$?Ñ@Ô@ˆŒõ ”\¥%¤+¨d¬nÑ"=Ô"=Ñ>Ô>ˆŒ
Ý% dÔ&<À$ÔBYÐZÑZÔZˆŒ	Ý”�eœk¨$¬.Ñ9Ô9Ñ:Ô:ˆŒØ#ð 	'¨¬Ô(;Ô(@ÀFÒ(JÐ(JØ×$Ò$Ñ&Ô&Ð&åœ	 $Ô"8¸$Ô:JÐQWÔQ`ÐaÑaÔaˆŒØœˆŒõ )¨Ñ9Ô9ˆÝ& }Ð6LÈdÑSÔSÐÝ" =Ð2DÀdÑKÔKÐõ % [Ñ1Ô1ˆ	Ý!8ØÐ$^ð"
ñ "
ô "
Ðõ %<ØÐ$Wð%
ñ %
ô %
Ð!õ ,CØÐ$^ð,
ñ ,
ô ,
Ð(õ
 "%å&Ý)Ý0Ý Ý$ðñ"
ô "
Ðõ &ð 	Ý×Òð>ñô ð ð !Ô,¨YÔ7ˆŒˆˆr$   c                 óÔ  — t          j        d| j        dz   | j        j        t           j        ¬¦  «        }t          j        | j        t          j        |¦  «        ¦  «         t          j	        | j
        ¦  «         t          j        t          j        | j        | j        j        t           j        ¬¦  «        t          j        | j        ¦  «        t          j        | j        ¦  «        z
  z  t          j        | j        ¦  «        z   ¦  «                             | j        ¬¦  «        }|t          j        t          j        | ¦  «         ¦  «        z   }t          j        | j        |¦  «         d S )Nr   r-   )Úmin)r   Úarangery   r•   r.   rW   ÚinitÚcopy_ÚlogÚones_r—   ÚexpÚrandr”   Úmathr‹   rŠ   ÚclamprŒ   Úexpm1)rP   ÚAÚdtÚinv_dts       r"   r™   zMamba2Mixer.init_mamba2_weightsà   s  € åŒL˜˜DœN¨QÑ.°t´zÔ7HÕPUÔP]Ð^Ñ^Ô^ˆÝŒ
�4”:�uœy¨™|œ|Ñ,Ô,Ð,ÝŒ
�4”6ÑÔÐåŒYÝŒJ�t”~¨d¬lÔ.AÍÌÐWÑWÔWÝŒx˜Ô*Ñ+Ô+­d¬h°tÔ7IÑ.JÔ.JÑJñLåŒh�tÔ)Ñ*Ô*ñ+ñ
ô 
÷ Š%�DÔ(ˆ%Ñ
)Ô
)ð	 	ð •e”i¥¤¨b¨SÑ!1Ô!1Ð 1Ñ2Ô2Ñ2ˆÝŒ
�4”< Ñ(Ô(Ð(Ð(Ð(r$   NrA   Úcache_paramsrB   c                 ó°  — t          ||¦  «        }|                      |¦  «        }|j        \  }}}| j        | j        z  }|j        d         d| j        z  z
  d| j        z  | j        z  z
  | j        z
  dz  }	|d uo|                     | j        ¦  «        }
|
r:|j	        | j                 j
        d         }|j	        | j                 j        d         }|
�r¢|dk    �r›|                     d¦  «                             |	|	| j        | j        | j        gd¬¦  «        \  }}}}}t          ||| j        j                             d¦  «        | j        j        | j        ¦  «        }t)          j        || j        ||gd¬¦  «        \  }}}t)          j        | j                             ¦   «         ¦  «         }|d d …d df         d d …d d …d f                              d| j        | j        ¦  «                             t(          j        ¬¦  «        }|d d …d d …d f                              dd| j        ¦  «        }| j        d d …d df                              d| j        ¦  «        }| j        d d …d df                              d| j        ¦  «        }|                     || j        |j        d         | j        z  ¦  «        }|                     || j        |j        d         | j        z  ¦  «        }|                     || j        | j        ¦  «        }t?          |||||||d |d¬	¦
  «
        }|                     || j        | j        z  ¦  «        }|                       ||¦  «        }|  !                    |¦  «        d d …d df         }�nat)          j        | j                             ¦   «         ¦  «         }| j"        d
t/          d¦  «        fk    ri nd| j"        i}| j#        r�|€�tI          || j        j                             d¦  «        | j        j        | j        |f| j        | j%        d | j        | j         j        | j         j&        | j!        j        | j!        j        | j        | j        dddœ|¤Ž}�nz|                     |	|	| j        | j        | j        gd¬¦  «        \  }}}}}| '                    dd¦  «        }|
rt)          j(        ||gd¬¦  «        }|�QtR          j*         +                    || j,        |j        d         z
  df¦  «        }| -                    || j        ¬¦  «         | j        dvr>|  .                    |                      |¦  «        dd |j        d         …f         ¦  «        }n?t_          || j        j                             d¦  «        | j        j        | j        ¬¦  «        }|
r|d d …d d …| d …f         }| '                    dd¦  «        }t          ||¦  «        }t)          j        || j        ||gd¬¦  «        \  }}}ta          |                     ||d| j        ¦  «        |||                     ||| j        d¦  «        |                     ||| j        d¦  «        f| j%        | j        d d d| j        d|
r|nd dœ|¤Ž\  }}|�|�| 1                    || j        ¬¦  «         |                     ||d¦  «        }|                       ||¦  «        }|  !                    |¦  «        }|S )Nr&   r'   r   r   r2   .©r/   T)Úzr”   Údt_softplusg        r;   Údt_limitF)r—   r)   Úseq_idxrƒ   Úrmsnorm_weightÚrmsnorm_epsÚoutproj_weightÚoutproj_biasÚheaddimÚngroupsÚnorm_before_gateÚreturn_final_states©rj   )rX   Úswish)ÚxrM   ro   rƒ   )r)   r—   rº   r½   rÅ   r”   r»   Úinitial_states)2rC   r’   r   r‡   r{   r   ry   Úhas_previous_staterj   ÚlayersÚconv_statesÚrecurrent_statesÚsqueezeÚsplitr�   rv   r�   rM   ro   rƒ   r   r¯   r•   Úfloatr5   rˆ   r@   rW   r”   r—   Úviewrœ   r–   rš   r‰   Útrainingrž   r)   rN   Ú	transposeÚcatr   r   r    r}   Úupdate_conv_stater„   rw   r�   Úupdate_recurrent_state)rP   rA   r·   rB   Úprojected_statesÚ
batch_sizeÚseq_lenÚ_Úgroups_time_state_sizeÚd_mlpÚuse_precomputed_statesÚ
conv_stateÚrecurrent_stater\   Úhidden_states_B_Crµ   ÚBÚCr´   r”   r—   Úhidden_states_reshapedÚoutÚdt_limit_kwargsÚnew_conv_stateÚscan_outputÚ	ssm_states                              r"   Úcuda_kernels_forwardz Mamba2Mixer.cuda_kernels_forwardð   sz  € õ 5°]ÀNÑSÔSˆØŸ<š<¨Ñ6Ô6Ðð "/Ô!4Ñˆ
�G˜QØ!%¤°Ô1DÑ!DÐàÔ" 2Ô&Ø�$Ô(Ñ(ñ)à�$”-Ñ $Ô"5Ñ5ñ6ð Œnñð ñˆð ".°TÐ!9Ð!m¸l×>]Ò>]Ð^bÔ^lÑ>mÔ>mÐð "ð 	VØ%Ô,¨T¬^Ô<ÔHÈÔKˆJØ*Ô1°$´.ÔAÔRÐSTÔUˆOð "ñ P	1 g°¢l¡lØ0@×0HÒ0HÈÑ0KÔ0K×0QÒ0QØ˜˜tÔ5°t´}ÀdÄnÐUÐ[]ð 1Rñ 1ô 1Ñ-ˆAˆq�$Ð)¨2õ
 !5Ø!ØØ”Ô"×*Ò*¨1Ñ-Ô-Ø”Ô Ø”ñ!ô !Ðõ #(¤+Ø!ØÔ'Ð)?ÐAWÐXØð#ñ #ô #ÑˆM˜1˜aõ ”˜4œ:×+Ò+Ñ-Ô-Ñ.Ô.Ð.ˆAØ�!�!�!�T˜3�,”    1 1 1 d 
Ô+×2Ò2°2°t´}ÀdÔFYÑZÔZ×]Ò]ÕdiÔdqÐ]ÑrÔrˆAØ�A�A�A�q�q�q˜$�J”×&Ò& r¨2¨t¬}Ñ=Ô=ˆBØ”l 1 1 1 d¨C <Ô0×7Ò7¸¸D¼MÑJÔJˆGØ”�q�q�q˜$ �|Ô$×+Ò+¨B°´Ñ>Ô>ˆAØ—’�z 4¤=°!´'¸!´*ÀÄÑ2MÑNÔNˆAØ—’�z 4¤=°!´'¸!´*ÀÄÑ2MÑNÔNˆAØ%2×%7Ò%7¸
ÀDÄNÐTXÔTaÑ%bÔ%bÐ"Ý2ØØ&ØØØØØØØØ ðñ ô ˆMð *×.Ò.¨z¸4¼>ÈDÌMÑ;YÑZÔZˆMØ ŸIšI m°TÑ:Ô:ˆMð —-’- Ñ.Ô.¨q¨q¨q°$¸¨|Ô<ˆC‰Cõ ”˜4œ:×+Ò+Ñ-Ô-Ñ.Ô.Ð.ˆAØ$(Ô$8¸SÅ%ÈÁ,Ä,Ð<OÒ$OÐ$O˜b˜bÐV`ÐbfÔbvÐUwˆOð Œ}ð [1 Ð!5Ý6Ø$Ø”KÔ&×.Ò.¨qÑ1Ô1Ø”KÔ$Ø”LØðð ”fØ#œØ Ø#œØ#'¤9Ô#3Ø $¤	Ô :Ø#'¤=Ô#7Ø!%¤Ô!3Ø œMØ œMØ%*Ø(-ð#ð ð$ &ð%ð �‘ð, 5E×4JÒ4JØ˜E 4Ô#9¸4¼=È$Ì.ÐYÐ_að 5Kñ 5ô 5Ñ1��1�dÐ-¨rð
 %6×$?Ò$?ÀÀ1Ñ$EÔ$EÐ!à)ð [õ ).¬	°:Ð?PÐ2QÐWYÐ(ZÑ(ZÔ(ZÐ%àÐ+Ý%'¤]×%6Ò%6Ø)¨DÔ,AÐDUÔD[Ð\^ÔD_Ñ,_ÐabÐ+cñ&ô &�Nð !×2Ò2°>ÈTÌ^Ð2Ñ\Ô\Ð\à”?Ð*;Ð;Ð;Ø(,¯ª°·²Ð=NÑ1OÔ1OÐPSÐUrÐWhÔWnÐoqÔWrÐUrÐPrÔ1sÑ(tÔ(tÐ%Ð%å(8Ø+Ø#œ{Ô1×9Ò9¸!Ñ<Ô<Ø!œ[Ô-Ø#'¤?ð	)ñ )ô )Ð%ð *ð KØ(9¸!¸!¸!¸Q¸Q¸QÀÀÀ	À	¸/Ô(JÐ%à$5×$?Ò$?ÀÀ1Ñ$EÔ$EÐ!å$@ÐARÐTbÑ$cÔ$cÐ!Ý&+¤kØ%ØÔ+Ð-CÐE[Ð\Øð'ñ 'ô 'Ñ#�˜q !õ *CØ!×&Ò& z°7¸BÀÄÑNÔNØØØ—F’F˜: w°´¸rÑBÔBØ—F’F˜: w°´¸rÑBÔBð*ð  $œØ”fØØ Ø(,Ø œLØ $Ø6LÐ#V ? ?ÐRVð*ð *ð &ð*ð *Ñ&�˜Yð$ Ð(¨\Ð-EØ ×7Ò7¸	ÈTÌ^Ð7Ñ\Ô\Ð\à)×.Ò.¨z¸7ÀBÑGÔG�à"Ÿiši¨°TÑ:Ô:�ð —m’m KÑ0Ô0�Øˆ
r$   c                 óJ  ‡ ‡2— |j         \  }}}|j        }t          ||¦  «        }‰                      |¦  «        }|j         d         d‰ j        z  z
  d‰ j        z  ‰ j        z  z
  ‰ j        z
  dz  }	|                     |	|	‰ j        ‰ j	        ‰ j        gd¬¦  «        \  }}}
}}| 
                    dd¦  «        }|d uo|                     ‰ j        ¦  «        }|r|j        ‰ j                 j        d         }|r”|dk    rŽ|                     |‰ j        ¬¦  «        d‰ j         d …f         }t#          j        |‰ j        j                             d¦  «        z  d¬¦  «        }‰ j        r|‰ j        j        z   }‰                      |¦  «        }nÒ|rt#          j        ||gd¬¦  «        }|�Qt4          j                             |‰ j        |j         d         z
  df¦  «        }|                     |‰ j        ¬¦  «         ‰                      ‰                      |¦  «        dd |j         d         …f          
                    dd¦  «        ¦  «        }|r|d d …| d …d d …f         }t          ||¦  «        }t#          j        |‰ j        ‰ j        ‰ j        z  ‰ j        ‰ j        z  gd¬¦  «        \  }}}t#          j        ‰ j                             ¦   «         ¦  «         }|�r\|dk    �rU|j        ‰ j                 j         }|d d …dd d …f         d d …d df         }| 
                    dd¦  «         !                    ||j         d         ‰ j"        ¦  «        }‰ j#        d          !                    ‰ j#        j         d         ‰ j"        ¦  «        }t"          j        j         $                    || %                    |j        ¦  «        z   ¦  «        }t#          j&        |‰ j'        d         ‰ j'        d         ¦  «        }|d	          !                    ‰ j        ‰ j"        ‰ j        ¦  «         %                    t"          j(        ¬
¦  «        }t#          j        |d         |z  ¦  «         %                    |¬¦  «        }| )                    |‰ j        d¦  «        dd d d …f         }| !                    |‰ j        ‰ j        ‰ j        z  |j         d         ¦  «         *                    ¦   «         }| )                    |d|j         d         ¦  «        }|d         |dd d d …f         z  }| )                    |d‰ j"        ¦  «        }||d         z   %                    |¬¦  «        }|j        ‰ j                 j+        d         |z  |z   }| ,                    |‰ j        ¬¦  «        }| )                    |‰ j        d¦  «        dd d d …f         }| !                    |‰ j        ‰ j        ‰ j        z  |j         d         ¦  «         *                    ¦   «         }| )                    |d|j         d         ¦  «        }| %                    |j         |j        ¬¦  «        }| -                    |‰ j        z  ‰ j"        ‰ j        ¦  «        }| -                    |‰ j        z  ‰ j        d¦  «        }t#          j.        ||¦  «        }| -                    |‰ j        ‰ j"        ¦  «        }‰ j/        d          !                    ‰ j/        j         d         ‰ j"        ¦  «        }|||z  z    %                    |j        ¦  «        }| )                    |d¦  «        d d …d df         }�n1t4          j         $                    |‰ j#        z   ¦  «        }t#          j&        |‰ j'        d         ‰ j'        d         ¦  «        }| )                    ||d‰ j"        ¦  «                             ¦   «         }| )                    ||d‰ j        ¦  «                             ¦   «         }| )                    ||d‰ j        ¦  «                             ¦   «         }| 0                    ‰ j        ‰ j        z  d‰ j        ¬¦  «        }| 0                    ‰ j        ‰ j        z  d‰ j        ¬¦  «        }‰ j1        |‰ j1        z  z
  ‰ j1        z  Š2‰ j/        d         te          |‰2¦  «        z  }||d         z  }| %                    |j        ¦  «        |z  }ˆ2ˆ fd„||||fD ¦   «         \  }}}}| 3                    dddd¦  «        }t#          j4        |d¬¦  «        }t#          j        tk          |¦  «        ¦  «        }|d d …d d …d d …d d d …d d …f         |d d …d d …d d d …d d …d d …f         z  } |                      d¬¦  «        }!|!d         | 3                    ddddd¦  «        d         z  }"|"                     d¬¦  «        }#|#d         |d d …d d …d f         z                       d¬¦  «        }$t#          j        |d d …d d …d d …dd …f         |z
  ¦  «        }%||% 3                    dddd¦  «        d         z  }&|&dd d d …f         |d         z                       d¬¦  «        }'|rF|j        ‰ j                 j+        d         d d …d f          %                    |'j        |'j         ¬¦  «        nt#          j6        |'d d …d d…f         ¦  «        }(t#          j        |(|'gd¬¦  «        }'t#          j        tk          t4          j                             |d d …d d …d d …df         d¦  «        ¦  «        ¦  «        })|) 
                    dd¦  «        })|)d	         |'d d …d d …d df         z                       d¬¦  «        }*|*d d …d d…f         |*d d …df         }+}'t#          j        |¦  «        },|dd d d …f         |'d d …d d …d df         z  }-|, 3                    dddd¦  «        }.|-                     d¦  «        |.d         z  }/|$|/z   }| )                    |d‰ j        ‰ j"        ¦  «        }||z   }‰2dk    r|d d …d |…d d …d d …f         }| )                    ||d¦  «        }|+�|�| ,                    |+‰ j        ¬¦  «         ‰  7                    ||
¦  «        }0‰  8                    |0 %                    |¦  «        ¦  «        }1|1S )Nr&   r'   r2   r   r   rÆ   .r,   ).NNr¹   )r.   r-   )r3   Úoutput_sizec                 ó<   •— g | ]}t          |‰‰j        ¦  «        ‘ŒS © )r*   r)   )Ú.0Útr   rP   s     €€r"   ú
<listcomp>z-Mamba2Mixer.torch_forward.<locals>.<listcomp>!  s)   ø€ Ð%zÐ%zÐ%zÐ\]Õ&9¸!¸XÀtÄÑ&WÔ&WÐ%zÐ%zÐ%zr$   r   r   r1   )r/   r.   )r   r   )9r   r/   rC   r’   r   r‡   r{   ry   rÏ   r�   rÓ   rÊ   rj   rË   rÌ   rÕ   r}   r   Úsumr�   rM   rÎ   r�   ro   r„   rÔ   r   r   r    r¯   r•   rÐ   r.   r5   rˆ   r”   Úsoftplusr@   r²   r‰   rW   r(   Ú
contiguousrÍ   rÖ   rÑ   Úbmmr—   Úrepeat_interleaver)   r#   Úpermuter:   r>   Ú
zeros_liker–   rš   )3rP   rA   r·   rB   rØ   rÙ   rÚ   r/   r×   rÜ   r\   rà   rµ   rÝ   rÞ   rÌ   rá   râ   r´   Úcache_devicer”   ÚdAÚdBÚdBxÚ
ssm_statesÚssm_states_reshapedÚ
C_reshapedÚyr—   Ú
D_residualÚA_cumsumÚLÚG_intermediateÚGÚM_intermediateÚMÚY_diagÚdecay_statesÚB_decayÚstatesÚprevious_statesÚdecay_chunkÚ
new_statesrè   Ústate_decay_outÚC_times_statesÚstate_decay_out_permutedÚY_offrç   Úcontextualized_statesr   s3   `                                                 @r"   Útorch_forwardzMamba2Mixer.torch_forward   sï  øø€ ð "/Ô!4Ñˆ
�G˜QØÔ#ˆõ 5°]ÀNÑSÔSˆØŸ<š<¨Ñ6Ô6ÐØ!Ô'¨Ô+¨a°$Ô2HÑ.HÑHÈ1ÈtÌ}ÑK\Ð_cÔ_rÑKrÑrÐswô  tBñ  Bð  GHñ  HˆØ,<×,BÒ,BØ˜˜tÔ5¸¼ÀtÄ~ÐVÐ\^ð -Cñ -
ô -
Ñ)ˆˆ1ˆdÐ% rð .×7Ò7¸¸!Ñ<Ô<Ðà!-°TÐ!9Ð!m¸l×>]Ò>]Ð^bÔ^lÑ>mÔ>mÐØ!ð 	LØ%Ô,¨T¬^Ô<ÔHÈÔKˆJð "ð 	G g°¢l lØ&×8Ò8Ð9JÐVZÔVdÐ8ÑeÔeÐfiÐlpô  mBð  lBð  lCð  lCð  gCô  DˆKå %¤	Ø˜dœkÔ0×8Ò8¸Ñ;Ô;Ñ;Àð!ñ !ô !Ðð Ô!ð IØ$5¸¼Ô8HÑ$HÐ!Ø $§¢Ð):Ñ ;Ô ;ÐÐà%ð WÝ$)¤I¨zÐ;LÐ.MÐSUÐ$VÑ$VÔ$VÐ!àÐ'Ý œm×/Ò/Ø%¨Ô(=Ð@QÔ@WÐXZÔ@[Ñ([Ð]^Ð'_ñô �ð ×.Ò.¨{ÀdÄnÐ.ÑUÔUÐUà $§¢¨¯ªÐ5FÑ)GÔ)GÈÐMiÐN_ÔNeÐfhÔNiÐMiÐHiÔ)j×)tÒ)tÐuvÐxyÑ)zÔ)zÑ {Ô {ÐØ%ð GØ$5°a°a°a¸'¸¸¸ÀAÀAÀA°oÔ$FÐ!å8Ð9JÈNÑ[Ô[ÐÝ#œkØØÔ# T¤]°TÔ5HÑ%HÈ$Ì-ÐZ^ÔZmÑJmÐnØð
ñ 
ô 
Ñˆ�q˜!õ ŒY�t”z×'Ò'Ñ)Ô)Ñ*Ô*Ð*ˆØ!ñ F	Y g°¢l¡là'Ô.¨t¬~Ô>ÔEˆLð �A�A�A�q˜!˜!˜!�G”˜Q˜Q˜Q  c˜\Ô*ˆBØ—’˜a Ñ#Ô#×*Ò*¨:°r´xÀ´|ÀTÄ]ÑSÔSˆBà”l 9Ô-×4Ò4°T´\Ô5GÈÔ5JÈDÌMÑZÔZˆGå”Ô$×-Ò-¨b°7·:²:¸b¼hÑ3GÔ3GÑ.GÑHÔHˆBÝ”˜R Ô!5°aÔ!8¸$Ô:NÈqÔ:QÑRÔRˆBØ�/Ô"×)Ò)¨$¬.¸$¼-ÈÔI\Ñ]Ô]×`Ò`ÕglÔgtÐ`ÑuÔuˆAå”)˜B˜yœM¨AÑ-Ñ.Ô.×2Ò2¸,Ð2ÑGÔGˆBð
 —	’	˜* d¤m°RÑ8Ô8¸¸dÀAÀAÀA¸ÔFˆAØ—’˜ T¤]°D´NÀdÄmÑ4SÐUVÔU\Ð]_ÔU`ÑaÔa×lÒlÑnÔnˆAØ—	’	˜* b¨!¬'°"¬+Ñ6Ô6ˆAà�I”  3¨¨a¨a¨a <¤Ñ0ˆBð *×1Ò1°*¸bÀ$Ä-ÑPÔPˆMØ˜ iÔ0Ñ0×4Ò4¸LÐ4ÑIÔIˆCð &Ô,¨T¬^Ô<ÔMÈaÔPÐSUÑUÐX[Ñ[ˆJØ%×<Ò<¸ZÐSWÔSaÐ<ÑbÔbˆJð —	’	˜* d¤m°RÑ8Ô8¸¸dÀAÀAÀA¸ÔFˆAØ—’˜ T¤]°D´NÀdÄmÑ4SÐUVÔU\Ð]_ÔU`ÑaÔa×lÒlÑnÔnˆAØ—	’	˜* b¨!¬'°"¬+Ñ6Ô6ˆAð $Ÿš¨a¬h¸a¼g˜ÑFÔFˆJØ",§/¢/°*¸t¼~Ñ2MÈtÌ}Ð^bÔ^qÑ"rÔ"rÐØŸš 
¨T¬^Ñ ;¸TÔ=PÐRSÑTÔTˆJÝ”	Ð-¨zÑ:Ô:ˆAØ—’�z 4¤>°4´=ÑAÔAˆAð ”�yÔ!×(Ò(¨¬¬°a¬¸$¼-ÑHÔHˆAØ�] QÑ&Ñ&×*Ò*¨1¬7Ñ3Ô3ˆAð —	’	˜* bÑ)Ô)¨!¨!¨!¨T°3¨,Ô7ˆA‰Aõ ”×'Ò'¨¨T¬\Ñ(9Ñ:Ô:ˆBÝ”˜R Ô!5°aÔ!8¸$Ô:NÈqÔ:QÑRÔRˆBØ)×1Ò1°*¸gÀrÈ4Ì=ÑYÔY×_Ò_ÑaÔaˆMØ—	’	˜* g¨r°4Ô3FÑGÔG×MÒMÑOÔOˆAØ—	’	˜* g¨r°4Ô3FÑGÔG×MÒMÑOÔOˆAØ×#Ò# D¤N°d´mÑ$CÈÐX\ÔXfÐ#ÑgÔgˆAØ×#Ò# D¤N°d´mÑ$CÈÐX\ÔXfÐ#ÑgÔgˆAØœ¨'°D´OÑ*CÑCÀtÄÑVˆHàœ 	Ô*Õ-?ÀÈxÑ-XÔ-XÑXˆJð *¨B¨y¬MÑ9ˆMØ—’�]Ô(Ñ)Ô)¨BÑ.ˆAð &{Ð%zÐ%zÐ%zÐ%zÐboÐqrÐtuÐwxÐayÐ%zÑ%zÔ%zÑ"ˆM˜1˜a ð —	’	˜!˜Q  1Ñ%Ô%ˆAÝ”| A¨2Ð.Ñ.Ô.ˆHõ ”	�+ a™.œ.Ñ)Ô)ˆAð ˜q˜q˜q ! ! ! Q Q Q¨¨a¨a¨a°°°Ð2Ô3°a¸¸¸¸1¸1¸1¸dÀAÀAÀAÀqÀqÀqÈ!È!È!Ð8KÔ6LÑLˆNØ×"Ò" rÐ"Ñ*Ô*ˆAð ˜yœ\¨A¯IªI°a¸¸A¸qÀ!Ñ,DÔ,DÀYÔ,OÑOˆNØ×"Ò" rÐ"Ñ*Ô*ˆAð ˜	”l ]°1°1°1°a°a°a¸°:Ô%>Ñ>×CÒCÈÐCÑJÔJˆFõ !œ9 X¨a¨a¨a°°°°A°A°A°r°s°s¨lÔ%;¸hÑ%FÑGÔGˆLØ˜,×.Ò.¨q°"°b¸!Ñ<Ô<¸YÔGÑGˆGØ˜c 4¨¨¨˜lÔ+¨m¸IÔ.FÑF×KÒKÐPQÐKÑRÔRˆFð *ð5�Ô# D¤NÔ3ÔDÀQÔGÈÈÈÈ4ÈÔP×SÒSÐZ`ÔZfÐouÔo|ÐSÑ}Ô}Ð}åÔ% f¨Q¨Q¨Q°°°¨U¤mÑ4Ô4ð õ
 ”Y °Ð8¸aÐ@Ñ@Ô@ˆFÝœ)¥Kµ´×0AÒ0AÀ(È1È1È1ÈaÈaÈaÐQRÐQRÐQRÐTVÈ;ÔBWÐY_Ñ0`Ô0`Ñ$aÔ$aÑbÔbˆKØ%×/Ò/°°1Ñ5Ô5ˆKØ% oÔ6¸ÀÀÀÀ1À1À1ÀdÈCÀÔ9PÑP×UÒUÐZ[ÐUÑ\Ô\ˆJØ *¨1¨1¨1¨c¨r¨c¨6Ô 2°J¸q¸q¸qÀ"¸uÔ4E�IˆFõ $œi¨Ñ1Ô1ˆOØ  T¨1¨1¨1 œo°°q°q°q¸!¸!¸!¸TÀ3°Ô0GÑGˆNØ'6×'>Ò'>¸qÀ!ÀQÈÑ'JÔ'JÐ$Ø#×'Ò'¨Ñ+Ô+Ð.FÀyÔ.QÑQˆEð ˜‘ˆAà—	’	˜* b¨$¬.¸$¼-ÑHÔHˆAà�J‘ˆAà˜!Š|ˆ|Ø�a�a�a˜˜'˜ 1 1 1 a a aÐ'Ô(�Ø—	’	˜* g¨rÑ2Ô2ˆAð Ð$¨Ð)AØ×3Ò3°IÈÌÐ3ÑXÔXÐXà—i’i  4Ñ(Ô(ˆð
 !%§¢¨k¯nªn¸UÑ.CÔ.CÑ DÔ DÐØ$Ð$r$   c                 ó¸   — t           r=d| j        j        j        j        v r%t          ¦   «         s|                      |||¦  «        S |                      |||¦  «        S )NÚcuda)r    r’   rM   r.   r˜   r   ré   r  )rP   rA   r·   rB   Úkwargss        r"   r_   zMamba2Mixer.forwardh  s^   € õ "ð 	Z f°´Ô0CÔ0JÔ0OÐ&OÐ&OÕXpÑXrÔXrÐ&OØ×,Ò,¨]¸LÈ.ÑYÔYÐYØ×!Ò! -°¸~ÑNÔNÐNr$   )T©NN)rb   rc   rd   Ú__doc__r   r~   r8   rK   r   Úno_gradr™   ÚTensorr   ré   r  r_   re   rf   s   @r"   rh   rh   y   s�  ø€ € € € € ðð ð]8ð ]8˜|ð ]8¸ð ]8ÐW[ð ]8ð ]8ð ]8ð ]8ð ]8ð ]8ð~ €U„]�_„_ð)ð )ñ „_ð)ð$ &*Ø.2ð	mð mà”|ðmð ˜d‘lðmð œ tÑ+ð	mð mð mð mðf &*Ø.2ð	E%ð E%à”|ðE%ð ˜d‘lðE%ð œ tÑ+ð	E%ð E%ð E%ð E%ðV &*Ø.2ð		Oð 	Oð ˜d‘lð	Oð œ tÑ+ð		Oð 	Oð 	Oð 	Oð 	Oð 	Oð 	Oð 	Or$   rh   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚMamba2RMSNormrF   c                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )zM
        Mamba2RMSNorm is equivalent to T5LayerNorm and LlamaRMSNorm
        NrI   rO   s      €r"   rK   zMamba2RMSNorm.__init__u  sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr$   c                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S rU   )	r/   r@   r   rW   rY   rZ   r[   rN   rM   )rP   rA   r]   r^   s       r"   r_   zMamba2RMSNorm.forward}  s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r$   r`   ra   rf   s   @r"   r  r  t  sL   ø€ € € € € ð$ð $ð $ð $ð $ð $ð;ð ;ð ;ð ;ð ;ð ;ð ;r$   r  c                   óJ   ‡ — e Zd Zˆ fd„Z	 	 ddedz  dej        dz  fd„Zˆ xZS )ÚMamba2Blockc                 óê   •— t          ¦   «                              ¦   «          || _        || _        |j        | _        t          |j        |j        ¬¦  «        | _        t          ||d¬¦  «        | _
        d S )Nrt   F)rj   rk   )rJ   rK   ri   rj   Úresidual_in_fp32r  rQ   r…   r–   rh   Úmixer)rP   ri   rj   rS   s      €r"   rK   zMamba2Block.__init__†  sg   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ &Ô 7ˆÔÝ! &Ô"4¸&Ô:SÐTÑTÔTˆŒ	Ý  °9ÐW\Ð]Ñ]Ô]ˆŒ
ˆ
ˆ
r$   Nr·   rB   c                 ó   — |}|                       |                     | j         j        j        ¬¦  «        ¦  «        }| j        r|                     t
          j        ¦  «        }|                      |||¬¦  «        }||z   }|S )Nr¹   ©r·   rB   )r–   r@   rM   r/   r"  r   rW   r#  )rP   rA   r·   rB   r  Úresiduals         r"   r_   zMamba2Block.forwardŽ  sy   € ð !ˆØŸ	š	 -×"2Ò"2¸¼Ô9IÔ9OÐ"2Ñ"PÔ"PÑQÔQˆØÔ ð 	2Ø—{’{¥5¤=Ñ1Ô1ˆHàŸ
š
 =¸|Ð\j˜
ÑkÔkˆØ  =Ñ0ˆØÐr$   r  )	rb   rc   rd   rK   r   r   r  r_   re   rf   s   @r"   r   r   …  s}   ø€ € € € € ð^ð ^ð ^ð ^ð ^ð &*Ø.2ð	ð ð ˜d‘lðð œ tÑ+ð	ð ð ð ð ð ð ð r$   r   c                   ój   ‡ — e Zd ZU eed<   dZdgZdZdZdZ	 e
j        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚMamba2PreTrainedModelri   Úbackboner   Tc                 ó2  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rÝ|                     ¦   «          t          j        |j        j        t          j
        d¦  «        ¬¦  «         |j        j        �t          j        |j        j        ¦  «         t          j        |j        j        t          j
        d¦  «        ¬¦  «         | j        j        r1|j        j        }|t          j
        | j        j        ¦  «        z  }dS dS dS )zInitialize the weights.é   )ÚaN)rJ   Ú_init_weightsÚ
isinstancerh   r™   r«   Úkaiming_uniform_r�   rM   r±   Úsqrtro   Úzeros_rš   ri   Úrescale_prenorm_residualÚnum_hidden_layers)rP   ÚmoduleÚprS   s      €r"   r-  z#Mamba2PreTrainedModel._init_weights¨  só   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�kÑ*Ô*ð 	>ð ×&Ò&Ñ(Ô(Ð(åÔ! &¤-Ô"6½$¼)ÀA¹,¼,ÐGÑGÔGÐGØŒ}Ô!Ð-Ý”˜FœMÔ.Ñ/Ô/Ð/ÝÔ! &¤/Ô"8½D¼IÀa¹L¼LÐIÑIÔIÐIàŒ{Ô3ð >ð ”OÔ*�Ø•T”Y˜tœ{Ô<Ñ=Ô=Ñ=���ð-	>ð 	>ð>ð >r$   )rb   rc   rd   r   Ú__annotations__Úbase_model_prefixÚ_no_split_modulesÚsupports_gradient_checkpointingÚ_can_compile_fullgraphÚ_is_statefulr   r  r-  re   rf   s   @r"   r(  r(  Ÿ  sw   ø€ € € € € € àÐÐÑØ"ÐØ&˜ÐØ&*Ð#Ø!ÐØ€Là€U„]�_„_ð>ð >ð >ð >ñ „_ð>ð >ð >ð >ð >r$   r(  z-
    Class for the MAMBA2 model outputs.
    )Úcustom_introc                   óp   — e Zd ZU dZdZej        dz  ed<   dZe	dz  ed<   dZ
eej                 dz  ed<   dS )ÚMamba2Outputa4  
    cache_params (`Cache`):
        The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to
        avoid providing the old `input_ids`.

        Includes both the State space model state matrices after the selective scan, and the Convolutional states
    NÚlast_hidden_stater·   rA   )rb   rc   rd   r  r?  r   ÚFloatTensorr6  r·   r   rA   Útuplerí   r$   r"   r>  r>  Å  sh   € € € € € € ðð ð 37Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø!%€L�%˜$‘,Ð%Ð%Ñ%Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ð9Ð9r$   r>  zK
    Base class for causal language model (or autoregressive) outputs.
    c                   óŽ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
dz  ed<   dZeej                 dz  ed<   dS )ÚMamba2CausalLMOutputa™  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    cache_params (`Cache`):
        The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to
        avoid providing the old `input_ids`.

        Includes both the State space model state matrices after the selective scan, and the Convolutional states
    NÚlossÚlogitsr·   rA   )rb   rc   rd   r  rD  r   r@  r6  rE  r·   r   rA   rA  rí   r$   r"   rC  rC  Ú  s   € € € € € € ð
ð 
ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø!%€L�%˜$‘,Ð%Ð%Ñ%Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ð9Ð9r$   rC  c                   óÆ   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zd„ Ze	 	 	 	 	 	 	 ddej	        dz  dej	        dz  de
dz  d	edz  d
edz  dedz  dej        dz  deez  fd„¦   «         Zˆ xZS )ÚMamba2Modelc                 ó¬  •‡— t          ¦   «                              ‰¦  «         t          j        ‰j        ‰j        ¦  «        | _        t          j        ˆfd„t          ‰j	        ¦  «        D ¦   «         ¦  «        | _
        d| _        t          ‰j        ‰j        ¬¦  «        | _        |                      | j        ¦  «         |                      ¦   «          d S )Nc                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS )rÆ   )r   )rî   Úidxri   s     €r"   rð   z(Mamba2Model.__init__.<locals>.<listcomp>ú  s&   ø€ Ð$sÐ$sÐ$sÈC¥[°À3Ð%GÑ%GÔ%GÐ$sÐ$sÐ$sr$   Frt   )rJ   rK   r   Ú	EmbeddingÚ
vocab_sizerQ   Ú
embeddingsÚ
ModuleListÚranger3  rË   Úgradient_checkpointingr  r…   Únorm_fÚ"_register_load_state_dict_pre_hookÚ	load_hookÚ	post_init©rP   ri   rS   s    `€r"   rK   zMamba2Model.__init__ö  s¶   øø€ Ý‰Œ×Ò˜Ñ Ô Ð åœ, vÔ'8¸&Ô:LÑMÔMˆŒÝ”mÐ$sÐ$sÐ$sÐ$sÕSXÐY_ÔYqÑSrÔSrÐ$sÑ$sÔ$sÑtÔtˆŒà&+ˆÔ#Ý# FÔ$6¸FÔ<UÐVÑVÔVˆŒà×/Ò/°´Ñ?Ô?Ð?Ø�ŠÑÔÐÐÐr$   c                 óv   — |D ]5}d|v r/|                      |¦  «        ||                     dd¦  «        <    d S Œ6d S )Nz
embedding.zembeddings.)ÚpopÚreplace)rP   Ú
state_dictÚprefixÚargsÚks        r"   rS  zMamba2Model.load_hook  sW   € Øð 	ð 	ˆAØ˜qÐ Ð ØEOÇ^Â^ÐTUÑEVÔEV�
˜1Ÿ9š9 \°=ÑAÔAÑBØ��ð !ð	ð 	r$   c                 ó   — | j         S rH   ©rM  ©rP   s    r"   Úget_input_embeddingsz Mamba2Model.get_input_embeddings  s
   € ØŒÐr$   c                 ó   — || _         d S rH   r^  ©rP   Únew_embeddingss     r"   Úset_input_embeddingsz Mamba2Model.set_input_embeddings  s   € Ø(ˆŒˆˆr$   NÚ	input_idsÚinputs_embedsr·   Ú	use_cacheÚoutput_hidden_statesÚreturn_dictrB   Úreturnc                 ó  — |�|n| j         j        }|�|n| j        s| j         j        nd}|�|n| j         j        }|du |duz  rt          d¦  «        ‚|€|                      |¦  «        }| j        r| j        r|rd}|r|€t          | j         ¬¦  «        }|}	|rdnd}
| j	        D ]} ||	||¬¦  «        }	|r|
|	fz   }
Œ|  
                    |	¦  «        }	|r|
|	fz   }
|st          d„ |	||
fD ¦   «         ¦  «        S t          |	|r|nd|
¬¦  «        S )	a¤  
        cache_params (`Cache`, *optional*):
            If passed along, the model uses the previous state in all the blocks (which will give the output for the
            `input_ids` provided as if the model add `state_input_ids + input_ids` as context).
        use_cache (`bool`, *optional*):
            If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits.
        NFz:You must specify exactly one of input_ids or inputs_embeds)ri   rí   r%  c              3   ó   K  — | ]}|®|V — Œ	d S rH   rí   )rî   Úvs     r"   ú	<genexpr>z&Mamba2Model.forward.<locals>.<genexpr>E  s(   è è € ÐfÐf˜qÐXYÐXe˜ÐXeÐXeÐXeÐXeÐfÐfr$   )r?  r·   rA   )ri   rh  rÒ   rg  ri  Ú
ValueErrorrM  rP  r	   rË   rQ  rA  r>  )rP   re  rf  r·   rg  rh  ri  rB   r  rA   Úall_hidden_statesÚmixer_blocks               r"   r_   zMamba2Model.forward  s¯  € ð( %9Ð$DÐ Ð È$Ì+ÔJjð 	ð "+Ð!6�I�IÐZ^ÔZgÐ=r¸T¼[Ô=RÐ=RÐmrˆ	Ø%0Ð%<�k�kÀ$Ä+ÔBYˆà˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ŸOšO¨IÑ6Ô6ˆMàÔ&ð 	¨4¬=ð 	¸Yð 	ØˆIàð 	<˜Ð-Ý'¨t¬{Ð;Ñ;Ô;ˆLà%ˆØ"6Ð@˜B˜B¸DÐØœ;ð 	Ið 	IˆKØ'˜KØØ)Ø-ðñ ô ˆMð $ð IØ$5¸Ð8HÑ$HÐ!øàŸš MÑ2Ô2ˆàð 	EØ 1°]Ð4DÑ DÐàð 	gÝÐfÐf ]°LÐBSÐ$TÐfÑfÔfÑfÔfÐfåØ+Ø)2Ð<˜˜¸Ø+ð
ñ 
ô 
ð 	
r$   )NNNNNNN)rb   rc   rd   rK   rS  r`  rd  r   r   Ú
LongTensorr   r8   r  rA  r>  r_   re   rf   s   @r"   rG  rG  ô  s  ø€ € € € € ð
ð 
ð 
ð 
ð 
ðð ð ðð ð ð)ð )ð )ð ð .2Ø15Ø%)Ø!%Ø,0Ø#'Ø.2ð<
ð <
àÔ# dÑ*ð<
ð Ô'¨$Ñ.ð<
ð ˜d‘lð	<
ð
 ˜$‘;ð<
ð # T™kð<
ð ˜D‘[ð<
ð œ tÑ+ð<
ð 
�Ñ	ð<
ð <
ð <
ñ „^ð<
ð <
ð <
ð <
ð <
r$   rG  z�
    The MAMBA2 Model transformer with a language modeling head on top (linear layer with weights not tied to the input
    embeddings).
    c                   ó4  ‡ — e Zd ZddiZˆ fd„Zd„ Zd„ Z	 	 	 	 	 ddedz  d	ej	        dz  d
e
dz  fˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dedz  dej        dz  de
dz  de
dz  de
dz  d	ej	        dz  deej	        z  deez  fd„¦   «         Zˆ xZS )ÚMamba2ForCausalLMzlm_head.weightzbackbone.embeddings.weightc                 óæ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S )NFrs   )
rJ   rK   rG  r)  r   r�   rQ   rL  Úlm_headrT  rU  s     €r"   rK   zMamba2ForCausalLM.__init__W  s^   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý# FÑ+Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒà�ŠÑÔÐÐÐr$   c                 ó4   — | j                              ¦   «         S rH   )r)  r`  r_  s    r"   r`  z&Mamba2ForCausalLM.get_input_embeddings^  s   € ØŒ}×1Ò1Ñ3Ô3Ð3r$   c                 ó6   — | j                              |¦  «        S rH   )r)  rd  rb  s     r"   rd  z&Mamba2ForCausalLM.set_input_embeddingsa  s   € ØŒ}×1Ò1°.ÑAÔAÐAr$   NFr·   rB   Úis_first_iterationc           	      óZ   •—  t          ¦   «         j        |f|||||dœ|¤Ž}|r|sd |d<   |S )N)rf  rg  r·   rB   ry  rB   )rJ   Úprepare_inputs_for_generation)
rP   re  rf  rg  r·   rB   ry  r  Úmodel_inputsrS   s
            €r"   r{  z/Mamba2ForCausalLM.prepare_inputs_for_generationd  sf   ø€ ð =•u‘w”wÔ<Øð
à'ØØ%Ø)Ø1ð
ð 
ð ð
ð 
ˆð ð 	2Ð/ð 	2Ø-1ˆLÐ)Ñ*àÐr$   r   re  rf  Úlabelsrh  ri  rg  Úlogits_to_keeprj  c
           	      ó  — |�|n| j         j        }|                      |||||||¬¦  «        }|d         }t          |	t          ¦  «        rt          |	 d¦  «        n|	}|                      |dd…|dd…f                              | j        j        j	        ¦  «        ¦  «         
                    ¦   «         }d}|� | j        d||| j         j        dœ|
¤Ž}|s|f|dd…         z   }|�|f|z   n|S t          |||j        |j        ¬¦  «        S )aN  
        cache_params (`Cache`, *optional*):
            If passed along, the model uses the previous state in all the blocks (which will give the output for the
            `input_ids` provided as if the model add `state_input_ids + input_ids` as context).
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        use_cache (`bool`, *optional*):
            If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits.
        N)r·   rf  rh  ri  rg  rB   r   )rE  r}  rL  r   )rD  rE  r·   rA   rí   )ri   ri  r)  r.  r~   Úslicerv  r@   rM   r/   rÐ   Úloss_functionrL  rC  r·   rA   )rP   re  rf  r·   r}  rh  ri  rg  rB   r~  r  Úmamba2_outputsrA   Úslice_indicesrE  rD  Úoutputs                    r"   r_   zMamba2ForCausalLM.forward}  sN  € ð2 &1Ð%<�k�kÀ$Ä+ÔBYˆàŸšØØ%Ø'Ø!5Ø#ØØ)ð 'ñ 
ô 
ˆð ' qÔ)ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@×CÒCÀDÄLÔDWÔD]Ñ^Ô^Ñ_Ô_×eÒeÑgÔgˆàˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDàð 	FØ�Y °°°Ô!3Ñ3ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå#ØØØ'Ô4Ø(Ô6ð	
ñ 
ô 
ð 	
r$   )NNNNF)	NNNNNNNNr   )rb   rc   rd   Ú_tied_weights_keysrK   r`  rd  r   r   r  r8   r{  r   rr  r@  r~   rA  rC  r_   re   rf   s   @r"   rt  rt  N  s¨  ø€ € € € € ð +Ð,HÐIÐðð ð ð ð ð4ð 4ð 4ðBð Bð Bð ØØ%)Ø.2Ø*/ðð ð
 ˜d‘lðð œ tÑ+ðð ! 4™Kðð ð ð ð ð ð2 ð .2Ø26Ø%)Ø*.Ø,0Ø#'Ø!%Ø.2Ø-.ð6
ð 6
àÔ# dÑ*ð6
ð Ô(¨4Ñ/ð6
ð ˜d‘lð	6
ð
 Ô  4Ñ'ð6
ð # T™kð6
ð ˜D‘[ð6
ð ˜$‘;ð6
ð œ tÑ+ð6
ð ˜eœlÑ*ð6
ð 
Ð%Ñ	%ð6
ð 6
ð 6
ñ „^ð6
ð 6
ð 6
ð 6
ð 6
r$   rt  )rt  rG  r(  )3r  r±   Údataclassesr   r   r   Ú r   r«   Úactivationsr   Úcache_utilsr   r	   Ú
generationr
   Úintegrationsr   Úmodeling_layersr   Úmodeling_utilsr   Úutilsr   r   r   r   Úutils.import_utilsr   Úconfiguration_mamba2r   Ú
get_loggerrb   r¡   r  r~   r#   r*   r>   rC   ÚModulerE   rh   r  r   r(  r>  rC  rG  rt  Ú__all__rí   r$   r"   ú<module>r”     sÖ  ðð Ð à €€€Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø -Ð -Ð -Ð -Ð -Ð -Ø SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø .Ð .Ð .Ð .Ð .Ð .ð 
ˆÔ	˜HÑ	%Ô	%€ðV U¤\ð V¸Sð Vð Vð Vð Vð
ð 
ð 
ð(ð ð ð(	ð 	ð 	ð;ð ;ð ;ð ;ð ;˜œœñ ;ô ;ð ;ð$xOð xOð xOð xOð xO�"”)ñ xOô xOð xOðv;ð ;ð ;ð ;ð ;�B”Iñ ;ô ;ð ;ð"ð ð ð ð Ð,ñ ô ð ð4 ð">ð ">ð ">ð ">ð ">˜Oñ ">ô ">ñ „ð">ðJ €ððñ ô ð
 ð:ð :ð :ð :ð :�;ñ :ô :ñ „ñô ð:ð €ððñ ô ð
 ð:ð :ð :ð :ð :˜;ñ :ô :ñ „ñô ð:ð& ðV
ð V
ð V
ð V
ð V
Ð'ñ V
ô V
ñ „ðV
ðr €ððñ ô ð`
ð `
ð `
ð `
ð `
Ð-¨ñ `
ô `
ñô ð`
ðF HÐ
GÐ
G€€€r$   