§
    ‚Štjª~  ã                   óò  — d Z ddlZddlmZ ddlZddlmZ ddlmZ ddlm	Z
 ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZ ddlmZm Z m!Z!m"Z" ddl#m$Z$  ej%        e&¦  «        Z' e d¦  «        rddl(m)Z) ndZ) e¦   «         rddl*m+Z+ ndZ+ G d„ dej,        ¦  «        Z- G d„ dej,        ¦  «        Z. G d„ de¦  «        Z/e G d„ de¦  «        ¦   «         Z0 ed¬¦  «        e G d „ d!e¦  «        ¦   «         ¦   «         Z1 ed"¬¦  «        e G d#„ d$e¦  «        ¦   «         ¦   «         Z2e G d%„ d&e0¦  «        ¦   «         Z3 ed'¬¦  «         G d(„ d)e0e¦  «        ¦   «         Z4g d*¢Z5dS )+zPyTorch MAMBA model.é    N)Ú	dataclass)Únn)ÚCrossEntropyLossé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úlazy_load_kernel)Úforce_accelerate_hooks)ÚGradientCheckpointingLayer)ÚPreTrainedModel)ÚModelOutputÚauto_docstringÚlogging)Úis_mambapy_availableÚis_torch_greater_or_equalÚ
is_tracingÚresolve_internal_importé   )ÚMambaConfigz2.9.0)Úassociative_scan)Úpscanc                   ó  ‡ — e Zd ZdZddededefˆ fd„Z ej	        ¦   «         d„ ¦   «         Z
d„ Z	 	 dd
ej        ded	z  dej        d	z  fd„Zdded	z  dej        d	z  fd„Z ed¦  «        	 	 dded	z  dej        d	z  fd„¦   «         Zˆ xZS )Ú
MambaMixeruƒ  
    Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)
    TÚconfigÚ	layer_idxÚinitialize_mixer_weightsc           	      ó  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        |j        | _        t          |j
        ¦  «        | _
        || _        |j        | _        t          j        | j        | j        |j        |j        | j        |j        dz
  ¬¦  «        | _        |j        | _        t$          |j                 | _        |j        | _        |j        | _        t          j        | j        | j        dz  |j        ¬¦  «        | _        t          j        | j        | j
        | j        dz  z   d¬¦  «        | _        t          j        | j
        | j        d¬¦  «        | _        t          j        t9          j        | j        | j        ¦  «        ¦  «        | _        t          j        t9          j        | j        ¦  «        ¦  «        | _        |r.| j        j         j!        j"        dk    r|  #                    ¦   «          t          j        | j        | j        |j        ¬¦  «        | _$        |j        | _        tK          d¦  «        a&tO          tL          d	d ¦  «        a(tO          tL          d
d ¦  «        a)tK          d¦  «        a*tW          tT          d¬¦  «        a,tO          tT          dd ¦  «        a-tO          tT          dd ¦  «        a.|  /                    ¦   «          |j0        |         | _1        d S )Nr   )Úin_channelsÚout_channelsÚbiasÚkernel_sizeÚgroupsÚpaddingé   ©r#   FTÚmetazcausal-conv1dÚcausal_conv1d_updateÚcausal_conv1d_fnz	mamba-ssmz8ops.triton.selective_state_update.selective_state_update)Úchained_pathÚselective_scan_fnÚmamba_inner_fn)2ÚsuperÚ__init__r   Úhidden_sizeÚ
state_sizeÚssm_state_sizeÚconv_kernelÚconv_kernel_sizeÚintermediate_sizeÚintÚtime_step_rankr   Úuse_conv_biasr   ÚConv1dÚconv1dÚ
hidden_actÚ
activationr   ÚactÚuse_mambapyÚuse_associative_scanÚLinearÚuse_biasÚin_projÚx_projÚdt_projÚ	ParameterÚtorchÚemptyÚA_logÚDÚweightÚdeviceÚtypeÚinit_mamba_weightsÚout_projr   Úcausal_conv1dÚgetattrr*   r+   Ú	mamba_ssmr   Úselective_state_updater-   r.   Úwarn_slow_implementationÚlayer_typesÚ
layer_type)Úselfr   r   r   Ú	__class__s       €úf/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/mamba/modeling_mamba.pyr0   zMambaMixer.__init__C   s   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ$Ô/ˆÔØ &Ô 2ˆÔØ!'Ô!9ˆÔÝ! &Ô"7Ñ8Ô8ˆÔØ"ˆŒØ#Ô1ˆÔÝ”iØÔ.ØÔ/ØÔ%ØÔ*ØÔ)ØÔ&¨Ñ*ð
ñ 
ô 
ˆŒð !Ô+ˆŒÝ˜&Ô+Ô,ˆŒà!Ô-ˆÔØ$*Ô$?ˆÔ!õ ”y Ô!1°4Ô3IÈAÑ3MÐTZÔTcÐdÑdÔdˆŒå”i Ô 6¸Ô8KÈdÔNaÐdeÑNeÑ8eÐlqÐrÑrÔrˆŒå”y Ô!4°dÔ6LÐSWÐXÑXÔXˆŒõ ”\¥%¤+¨dÔ.DÀdÔFYÑ"ZÔ"ZÑ[Ô[ˆŒ
Ý”�eœk¨$Ô*@ÑAÔAÑBÔBˆŒØ#ð 	&¨¬Ô(;Ô(BÔ(GÈ6Ò(QÐ(QØ×#Ò#Ñ%Ô%Ð%Ýœ	 $Ô"8¸$Ô:JÐQWÔQ`ÐaÑaÔaˆŒØœˆŒõ )¨Ñ9Ô9ˆÝ&¥}Ð6LÈdÑSÔSÐÝ"¥=Ð2DÀdÑKÔKÐõ % [Ñ1Ô1ˆ	Ý!8ÝÐ$^ð"
ñ "
ô "
Ðõ $¥IÐ/BÀDÑIÔIÐÝ ¥Ð,<¸dÑCÔCˆà×%Ò%Ñ'Ô'Ð'à Ô,¨YÔ7ˆŒˆˆó    c                 óx  — t          j        d| j        dz   t           j        | j        j        ¬¦  «        d d d …f         }|                     | j        d¦  «                             ¦   «         }t          j
        | j        t          j        |¦  «        ¦  «         t          j        | j        ¦  «         | j        j        dz  | j        j        z  }| j        j        dk    r t          j        | j        j        |¦  «         n1| j        j        dk    r!t          j        | j        j        | |¦  «         t          j        t          j        | j        | j        j        j        t           j        ¬¦  «        t3          j        | j        j        ¦  «        t3          j        | j        j        ¦  «        z
  z  t3          j        | j        j        ¦  «        z   ¦  «                             | j        j        ¬¦  «        }|t          j        t          j        | ¦  «         ¦  «        z   }t          j
        | j        j        |¦  «         d S )	Nr   )ÚdtyperL   éÿÿÿÿg      à¿ÚconstantÚrandom©rL   r\   )Úmin)rG   Úaranger3   Úfloat32rI   rL   Úexpandr6   Ú
contiguousÚinitÚcopy_ÚlogÚones_rJ   r   r8   Útime_step_scaleÚtime_step_init_schemeÚ	constant_rE   rK   Úuniform_ÚexpÚrandr#   ÚmathÚtime_step_maxÚtime_step_minÚclampÚtime_step_floorÚexpm1)rW   ÚAÚdt_init_stdÚdtÚinv_dts        rY   rN   zMambaMixer.init_mamba_weights}   sÍ  € åŒL˜˜DÔ/°!Ñ3½5¼=ÐQUÔQ[ÔQbÐcÑcÔcÐdhÐjkÐjkÐjkÐdkÔlˆØ�HŠH�TÔ+¨RÑ0Ô0×;Ò;Ñ=Ô=ˆÝŒ
�4”:�uœy¨™|œ|Ñ,Ô,Ð,ÝŒ
�4”6ÑÔÐà”kÔ0°$Ñ6¸¼Ô9TÑTˆØŒ;Ô,°
Ò:Ð:ÝŒN˜4œ<Ô.°Ñ<Ô<Ð<Ð<ØŒ[Ô.°(Ò:Ð:ÝŒM˜$œ,Ô-°¨|¸[ÑIÔIÐIåŒYÝŒJ�tÔ-°d´lÔ6GÔ6NÕV[ÔVcÐdÑdÔdÝŒx˜œÔ1Ñ2Ô2µT´X¸d¼kÔ>WÑ5XÔ5XÑXñZåŒh�t”{Ô0Ñ1Ô1ñ2ñ
ô 
÷ Š%�D”KÔ/ˆ%Ñ
0Ô
0ð	 	ð •e”i¥¤¨b¨SÑ!1Ô!1Ð 1Ñ2Ô2Ñ2ˆÝŒ
�4”<Ô$ fÑ-Ô-Ð-Ð-Ð-rZ   c                 ó  — t          t          t          t          t          t
          f¦  «        }|s\| j        r9t          ¦   «         rt           	                    d¦  «         d S t          d¦  «        ‚t           	                    d¦  «         d S d S )Na’  The fast path is not available because one of `(selective_state_update, selective_scan_fn, causal_conv1d_fn, causal_conv1d_update, mamba_inner_fn)` is None. Falling back to the mamba.py backend. To install follow https://github.com/state-spaces/mamba/#installation for mamba-ssm and install the kernels library using `pip install kernels` or https://github.com/Dao-AILab/causal-conv1d for causal-conv1dzuse_mambapy is set to True but the mambapy package is not installed. To install it follow https://github.com/alxndrTL/mamba.py.a  The fast path is not available because one of `(selective_state_update, selective_scan_fn, causal_conv1d_fn, causal_conv1d_update, mamba_inner_fn)` is None. Falling back to the sequential implementation of Mamba, as use_mambapy is set to False. To install follow https://github.com/state-spaces/mamba/#installation for mamba-ssm and install the kernels library using `pip install kernels` or https://github.com/Dao-AILab/causal-conv1d for causal-conv1d. For the mamba.py backend, follow https://github.com/alxndrTL/mamba.py.)ÚallrS   r-   r+   r*   r.   r?   r   ÚloggerÚwarning_onceÚImportError)rW   Úis_fast_path_availables     rY   rT   z#MambaMixer.warn_slow_implementation“   s½   € Ý!$Ý#Õ%6Õ8HÕJ^Õ`nÐoñ"
ô "
Ðð &ð 	ØÔð Ý'Ñ)Ô)ð 	Ý×'Ò'ðSñô ð ð ð õ &ð Zñô ð õ ×#Ò#ðWñô ð ð ð ð	ð 	rZ   NÚhidden_statesÚcache_paramsÚattention_maskc                 óH	  — |                       |¦  «                             dd¦  «        }| j        rÝ|€Ût          || j        j        | j        r| j        j        nd | j        j        | j	        j        | j
        j        | j        r| j
        j                             ¦   «         nd t          j        | j                             ¦   «         ¦  «         d d | j                             ¦   «         | j	        j                             ¦   «         d¬¦  «        }�n”|                     dd¬¦  «        \  }}|�||                     d¦  «        z  }|d uo|                     | j        ¦  «        }| j        j                             | j        j                             d¦  «        | j        j                             d¦  «        ¦  «        }|rft/          |                     d¦  «        |j        | j                 j        d         || j        j        | j        ¦  «        }|                     d¦  «        }nt|�Pt8          j                             || j        |j         d         z
  df¦  «        }	| !                    |	| j        ¦  «         tE          ||| j        j        | j        ¬¦  «        }|�||                     d¦  «        z  }|                      |                     dd¦  «        ¦  «        }
t          j#        |
| j$        | j%        | j%        gd¬¦  «        \  }}}| j	        j        |                     dd¦  «        z  }t          j        | j                             ¦   «         ¦  «         }tM          | j	        d	¦  «        r| j	        j                             ¦   «         nd }|rstO          |j        | j                 j(        d         |d
         |d
         ||d d …df         |d d …df         | j        |d
         |d¬¦
  «
                             d¦  «        }nztS          ||||                     dd¦  «        |                     dd¦  «        | j                             ¦   «         ||dd¬¦
  «
        \  }}|�|�| *                    || j        ¦  «         |  
                    |                     dd¦  «        ¦  «        }|S )Nr   r'   T)Ú
delta_biasÚdelta_softplus©Údimr   r]   )r=   r#   ).r   )Údt_softplus)r…   Úreturn_last_state)+rC   Ú	transposeÚtrainingr.   r;   rK   r9   r#   rD   rE   rO   rB   ÚfloatrG   rn   rI   rJ   ÚchunkÚ	unsqueezeÚhas_previous_stater   ÚviewÚsizer*   ÚsqueezeÚlayersÚconv_statesr=   r   Ú
functionalÚpadr5   ÚshapeÚupdate_conv_stater+   Úsplitr8   r3   ÚhasattrrS   Úrecurrent_statesr-   Úupdate_recurrent_state)rW   r€   r�   r‚   Úprojected_statesÚcontextualized_statesÚgateÚis_decodingÚconv_weightsr”   Ússm_parametersÚ	time_stepÚBÚCÚdiscrete_time_steprv   Útime_proj_biasÚscan_outputsÚ	ssm_states                      rY   Úcuda_kernels_forwardzMambaMixer.cuda_kernels_forwardª   s‡  € ð  Ÿ<š<¨Ñ6Ô6×@Ò@ÀÀAÑFÔFÐàŒ=ð Z	P˜\Ð1Ý$2Ø Ø”Ô"Ø$(Ô$6Ð@�”Ô Ð ¸DØ”Ô"Ø”Ô#Ø”Ô$Ø.2¬mÐE�”Ô"×(Ò(Ñ*Ô*Ð*ÀÝ”˜4œ:×+Ò+Ñ-Ô-Ñ.Ô.Ð.ØØØ”—’‘”Øœ<Ô,×2Ò2Ñ4Ô4Ø#ð%ñ %ô %Ð!Ñ!ð" #3×"8Ò"8¸ÀÐ"8Ñ"BÔ"BÑˆM˜4àÐ)Ø -°×0HÒ0HÈÑ0KÔ0KÑ K�à&¨dÐ2Ðf°|×7VÒ7VÐW[ÔWeÑ7fÔ7fˆKð  œ;Ô-×2Ò2°4´;Ô3E×3JÒ3JÈ1Ñ3MÔ3MÈtÌ{ÔOa×OfÒOfÐghÑOiÔOiÑjÔjˆLØð Ý 4Ø!×)Ò)¨"Ñ-Ô-Ø Ô'¨¬Ô7ÔCÀAÔFØ Ø”KÔ$Ø”Oñ!ô !�ð !.× 7Ò 7¸Ñ ;Ô ;��àÐ+Ý"$¤-×"3Ò"3Ø%¨Ô(=ÀÔ@SÐTVÔ@WÑ(WÐYZÐ'[ñ#ô #�Kð !×2Ò2°;ÀÄÑOÔOÐOÝ 0Ø! <°´Ô1AÈdÌoð!ñ !ô !�ð Ð)Ø -°×0HÒ0HÈÑ0KÔ0KÑ K�ð "Ÿ[š[¨×)@Ò)@ÀÀAÑ)FÔ)FÑGÔGˆNÝ#œkØ Ô!4°dÔ6IÈ4ÔK^Ð _Ðegðñ ô ‰OˆI�q˜!ð "&¤Ô!4°y×7JÒ7JÈ1ÈaÑ7PÔ7PÑ!PÐå”˜4œ:×+Ò+Ñ-Ô-Ñ.Ô.Ð.ˆAå:AÀ$Ä,ÐPVÑ:WÔ:WÐa˜Tœ\Ô.×4Ò4Ñ6Ô6Ð6Ð]aˆNØð SÝ5Ø Ô'¨¬Ô7ÔHÈÔKØ! &Ô)Ø& vÔ.ØØ�a�a�a˜�d”GØ�a�a�a˜�d”GØ”FØ˜”LØ"Ø $ð ñ  ô  ÷ ’)˜B‘-”-ð �õ +<Ø!Ø&ØØ—K’K  1Ñ%Ô%Ø—K’K  1Ñ%Ô%Ø”F—L’L‘N”NØØ"Ø#'Ø&*ð+ñ +ô +Ñ'�˜ið Ð(¨\Ð-EØ ×7Ò7¸	À4Ä>ÑRÔRÐRð %)§M¢M°,×2HÒ2HÈÈAÑ2NÔ2NÑ$OÔ$OÐ!Ø$Ð$rZ   c           	      óŠ  — |j         \  }}}|j        }|                      |¦  «                             dd¦  «        }|                     dd¬¦  «        \  }	}
|�|	|                     d¦  «        z  }	|�J|                     | j        ¦  «        r0|j        | j                 j	        d          
                    ¦   «         }n)t          j        || j        | j        f|	j        |¬¦  «        }|��o|                     | j        ¦  «        s„t           j                             |	| j        |	j         d         z
  df¦  «        }|                     || j        ¦  «         |                      |                      |	¦  «        dd |…f         ¦  «        }	�n|                     |	| j        ¦  «        d| j         d …f         }|                     | j        j        j        ¦  «        }t          j        || j        j        d d …dd d …f         z  d¬¦  «        }	| j        r|	| j        j        z  }	|                      |	¦  «                             |¦  «                             d¦  «        }	n2|                      |                      |	¦  «        dd |…f         ¦  «        }	|�|	|                     d¦  «        z  }	|                      |	                     dd¦  «        ¦  «        }t          j        || j        | j        | j        gd¬¦  «        \  }}}|                      |¦  «        }t           j                              |¦  «                             dd¦  «        }t          j!        | j"         #                    ¦   «         ¦  «         }t          j!        |d d d …d d d …f         |d d …d d …d d …d f         z  ¦  «        }|d d …d d …d d …d f         |d d …d d d …d d …f          #                    ¦   «         z  }||	d d …d d …d d …d f          #                    ¦   «         z  }| j$        r²| j%        r«|€©tM          |                     dd¦  «        |                     dd¦  «        ¦  «        }||                     d¦  «        z   '                    d¦  «                             dd¦  «        }||	| j(        d d d …d f         z  z   }||                      |
¦  «        z  }�ná| j)        rÑtT          �ÊtW          |	¦  «        r»|€¹d	„ }|j        j,        d
v rdnd}tU          |||fd|¬¦  «        \  }}t          j-        | .                    dddd¦  «                             |¦  «        |                     d¦  «        ¦  «         '                    d¦  «         .                    ddd¦  «        }|d d …d d …dd d …f         }n¼g }t_          |¦  «        D ]”}|d d …d d …|d d …f         |z  |d d …d d …|d d …f         z   }t          j-        |                     |¦  «        |d d …|d d …f                              d¦  «        ¦  «        }| 0                    |d d …d d …df         ¦  «         Œ•t          j1        |d¬¦  «        }||	| j(        d d d …d f         z  z   }||                      |
¦  «        z  }|�| 2                    || j        ¦  «         |  3                    |                     dd¦  «        ¦  «        }|S )Nr   r'   r†   r   r`   r]   .r   c                 ó0   — | \  }}|\  }}||z  ||z  |z   fS ©N© )ÚleftÚrightÚa_leftÚb_leftÚa_rightÚb_rights         rY   Ú
combine_fnz+MambaMixer.slow_forward.<locals>.combine_fnS  s/   € Ø%)‘N�F˜FØ',Ñ$�G˜WØ" WÑ,¨g¸Ñ.>ÀÑ.HÐIÐIrZ   )ÚcudaÚxpuÚ	pointwiseÚgeneric)r‡   Úcombine_mode)4r—   r\   rC   rŠ   r�   rŽ   r�   r   r“   r›   ÚclonerG   Úzerosr6   r3   rL   r   r•   r–   r5   r˜   r>   r;   ÚtorK   Úsumr9   r#   rD   r™   r8   rE   Úsoftplusrn   rI   rŒ   r?   r‹   r   r’   rJ   r@   r   r   rM   ÚmatmulÚpermuteÚrangeÚappendÚstackrœ   rO   )rW   Úinput_statesr�   r‚   Ú
batch_sizeÚseq_lenÚ_r\   r�   r€   rŸ   r©   Ú
conv_stater¢   r£   r¤   r¥   r¦   rv   Ú
discrete_AÚ
discrete_BÚdeltaB_uÚhsÚscan_outputrµ   rº   Úall_hr¨   Úirž   s                                 rY   Úslow_forwardzMambaMixer.slow_forward  s  € Ø!-Ô!3Ñˆ
�G˜QØÔ"ˆàŸ<š<¨Ñ5Ô5×?Ò?ÀÀ1ÑEÔEÐØ.×4Ò4°Q¸AÐ4Ñ>Ô>Ñˆ�tàÐ%Ø)¨N×,DÒ,DÀQÑ,GÔ,GÑGˆMàÐ#¨×(GÒ(GÈÌÑ(WÔ(WÐ#Ø$Ô+¨D¬NÔ;ÔLÈQÔO×UÒUÑWÔWˆIˆIåœØ˜TÔ3°TÔ5HÐIØ$Ô+°5ðñ ô ˆIð Ñ#Ø×2Ò2°4´>ÑBÔBð PÝœ]×.Ò.Ø!ØÔ*¨]Ô-@ÀÔ-DÑDÀaÐHñô �
ð
 ×.Ò.¨z¸4¼>ÑJÔJÐJØ $§¢¨¯ª°]Ñ)CÔ)CÀCÈÈ'ÈÀMÔ)RÑ SÔ S�‘à)×;Ò;¸MÈ4Ì>ÑZÔZÐ[^ÐaeÔavÐ`vÐ`wÐ`wÐ[wÔx�
Ø'Ÿ]š]¨4¬;Ô+=Ô+DÑEÔE�
Ý %¤	¨*°t´{Ô7IÈ!È!È!ÈQÐPQÐPQÐPQÈ'Ô7RÑ*RÐXZÐ [Ñ [Ô [�ØÔ%ð 6Ø! T¤[Ô%5Ñ5�MØ $§¢¨Ñ 7Ô 7× :Ò :¸5Ñ AÔ A× KÒ KÈBÑ OÔ O��à ŸHšH T§[¢[°Ñ%?Ô%?ÀÀXÀgÀXÀÔ%NÑOÔOˆMàÐ%Ø)¨N×,DÒ,DÀQÑ,GÔ,GÑGˆMð Ÿš ]×%<Ò%<¸QÀÑ%BÔ%BÑCÔCˆÝœ+Ø˜TÔ0°$Ô2EÀtÔGZÐ[Ðacð
ñ 
ô 
‰ˆ	�1�að "Ÿ\š\¨)Ñ4Ô4ÐÝœ]×3Ò3Ð4FÑGÔG×QÒQÐRSÐUVÑWÔWÐõ ŒY�t”z×'Ò'Ñ)Ô)Ñ*Ô*Ð*ˆÝ”Y˜q  q q q¨$°°°Ð!1Ô2Ð5GÈÈÈÈ1È1È1ÈaÈaÈaÐQUÈÔ5VÑVÑWÔWˆ
Ø'¨¨¨¨1¨1¨1¨a¨a¨a°¨Ô6¸¸1¸1¸1¸dÀAÀAÀAÀqÀqÀq¸=Ô9I×9OÒ9OÑ9QÔ9QÑQˆ
Ø ¨a¨a¨a°°°°A°A°A°t¨mÔ <× BÒ BÑ DÔ DÑDˆð Ôð  	O ¤ð  	O°,Ð2FÝ�z×+Ò+¨A¨qÑ1Ô1°8×3EÒ3EÀaÈÑ3KÔ3KÑLÔLˆBà §¢¨B¡¤Ñ/×8Ò8¸Ñ;Ô;×EÒEÀaÈÑKÔKˆKØ%¨¸¼¸tÀQÀQÀQÈ¸}Ô8MÑ(MÑMˆKØ%¨¯ª°©¬Ñ6ˆK‰Kð Ô(ð @Õ-=Ð-IÍjÐYfÑNgÔNgÐ-IÐlxð  mAðJð Jð Jð
 /9Ô.?Ô.DÈÐ.WÐ.W˜{˜{Ð]f�Ý+¨J¸ÀXÐ8NÐTUÐdpÐqÑqÔq‘��5å#œl¨5¯=ª=¸¸A¸qÀ!Ñ+DÔ+D×+GÒ+GÈÑ+NÔ+NÐPQ×P[ÒP[Ð\^ÑP_ÔP_Ñ`Ô`×hÒhÐikÑlÔl×tÒtÐuvÐxyÐ{|Ñ}Ô}�Ø! ! ! ! Q Q Q¨¨A¨A¨A +Ô.�	�	ð  "�Ý˜w™œð >ð >�AØ *¨1¨1¨1¨a¨a¨a°°A°A°A¨:Ô 6¸Ñ BÀXÈaÈaÈaÐQRÐQRÐQRÐTUÐWXÐWXÐWXÈjÔEYÑ Y�IÝ"'¤,¨y¯|ª|¸EÑ/BÔ/BÀAÀaÀaÀaÈÈAÈAÈAÀgÄJ×DXÒDXÐY[ÑD\ÔD\Ñ"]Ô"]�KØ ×'Ò'¨°A°A°A°q°q°q¸!°GÔ(<Ñ=Ô=Ð=Ð=Ý#œk¨,¸BÐ?Ñ?Ô?�à%¨¸¼ÀÀaÀaÀaÈÀÔ9NÑ)NÑOˆKØ&¨¯ª°$©¬Ñ7ˆKàÐ'Ø×3Ò3°I¸t¼~ÑNÔNÐNð !%§¢¨k×.CÒ.CÀAÀqÑ.IÔ.IÑ JÔ JÐØ$Ð$rZ   r;   c                 ó
  — t          t          t          t          t          t
          f¦  «        }|r>d| j        j        j        j	        v r&t          |¦  «        s|                      |||¦  «        S |                      |||¦  «        S )Nr¶   )r{   rS   r-   r+   r*   r.   rD   rK   rL   rM   r   rª   rÑ   )rW   r€   r�   r‚   Úkwargsr   s         rY   ÚforwardzMambaMixer.forwardq  s…   € õ "%Ý#Õ%6Õ8HÕJ^Õ`nÐoñ"
ô "
Ðð "ð 	Z f°´Ô0BÔ0IÔ0NÐ&NÐ&NÕWaÐboÑWpÔWpÐ&NØ×,Ò,¨]¸LÈ.ÑYÔYÐYØ× Ò  °¸nÑMÔMÐMrZ   )T©NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r7   Úboolr0   rG   Úno_gradrN   rT   ÚTensorr	   Ú
LongTensorrª   rÑ   r   rÔ   Ú__classcell__©rX   s   @rY   r   r   ;   sŠ  ø€ € € € € ðð ð88ð 88˜{ð 88°sð 88ÐVZð 88ð 88ð 88ð 88ð 88ð 88ðt €U„]�_„_ð.ð .ñ „_ð.ð*ð ð ð4 &*Ø26ð	d%ð d%à”|ðd%ð ˜d‘lðd%ð Ô(¨4Ñ/ð	d%ð d%ð d%ð d%ðN]%ð ]%°u¸t±|ð ]%ÐZ_ÔZjÐmqÑZqð ]%ð ]%ð ]%ð ]%ð@ Ð˜HÑ%Ô%ð &*Ø26ð	Nð Nð ˜d‘lðNð Ô(¨4Ñ/ð	Nð Nð Nñ &Ô%ðNð Nð Nð Nð NrZ   r   c                   ó,   ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Zˆ xZS )ÚMambaRMSNormç�íµ ÷Æ°>c                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )zL
        MambaRMSNorm is equivalent to T5LayerNorm and LlamaRMSNorm
        N)r/   r0   r   rF   rG   ÚonesrK   Úvariance_epsilon)rW   r1   ÚepsrX   s      €rY   r0   zMambaRMSNorm.__init__‚  sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐrZ   c                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )Nr'   r]   T)Úkeepdim)	r\   r½   rG   rc   ÚpowÚmeanÚrsqrtrå   rK   )rW   r€   Úinput_dtypeÚvariances       rY   rÔ   zMambaRMSNorm.forwardŠ  s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:rZ   c                 ó:   — | j         j        d         › d| j        › �S )Nr   z, eps=)rK   r—   rå   ©rW   s    rY   Ú
extra_reprzMambaRMSNorm.extra_repr‘  s#   € Ø”+Ô# AÔ&ÐEÐE¨dÔ.CÐEÐEÐErZ   )râ   )rÖ   r×   rØ   r0   rÔ   rð   rÞ   rß   s   @rY   rá   rá   �  sb   ø€ € € € € ð$ð $ð $ð $ð $ð $ð;ð ;ð ;ðFð Fð Fð Fð Fð Fð FrZ   rá   c                   óJ   ‡ — e Zd Zˆ fd„Z	 	 ddedz  dej        dz  fd„Zˆ xZS )Ú
MambaBlockc                 óê   •— t          ¦   «                              ¦   «          || _        || _        |j        | _        t          |j        |j        ¬¦  «        | _        t          ||d¬¦  «        | _
        d S )N©ræ   F)r   r   )r/   r0   r   r   Úresidual_in_fp32rá   r1   Úlayer_norm_epsilonÚnormr   Úmixer)rW   r   r   rX   s      €rY   r0   zMambaBlock.__init__–  sg   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ &Ô 7ˆÔÝ  Ô!3¸Ô9RÐSÑSÔSˆŒ	Ý °)ÐV[Ð\Ñ\Ô\ˆŒ
ˆ
ˆ
rZ   Nr�   r‚   c                 ó   — |}|                       |                     | j         j        j        ¬¦  «        ¦  «        }| j        r|                     t
          j        ¦  «        }|                      |||¬¦  «        }||z   }|S )N)r\   ©r�   r‚   )r÷   r½   rK   r\   rõ   rG   rc   rø   )rW   r€   r�   r‚   rÓ   Úresiduals         rY   rÔ   zMambaBlock.forwardž  sy   € ð !ˆØŸ	š	 -×"2Ò"2¸¼Ô9IÔ9OÐ"2Ñ"PÔ"PÑQÔQˆØÔ ð 	2Ø—{’{¥5¤=Ñ1Ô1ˆHàŸ
š
 =¸|Ð\j˜
ÑkÔkˆØ  =Ñ0ˆØÐrZ   rÕ   )	rÖ   r×   rØ   r0   r	   rG   rÝ   rÔ   rÞ   rß   s   @rY   rò   rò   •  s~   ø€ € € € € ð]ð ]ð ]ð ]ð ]ð &*Ø26ð	ð ð ˜d‘lðð Ô(¨4Ñ/ð	ð ð ð ð ð ð ð rZ   rò   c                   óh   ‡ — e Zd ZU eed<   dZddgZdZdZ e	j
        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚMambaPreTrainedModelr   Úbackbonerò   r   Tc                 ó2  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rÝ|                     ¦   «          t          j        |j        j        t          j
        d¦  «        ¬¦  «         |j        j        �t          j        |j        j        ¦  «         t          j        |j        j        t          j
        d¦  «        ¬¦  «         | j        j        r1|j        j        }|t          j
        | j        j        ¦  «        z  }dS dS dS )zInitialize the weights.é   )ÚaN)r/   Ú_init_weightsÚ
isinstancer   rN   rf   Úkaiming_uniform_r;   rK   rp   Úsqrtr#   Úzeros_rO   r   Úrescale_prenorm_residualÚnum_hidden_layers)rW   ÚmoduleÚprX   s      €rY   r  z"MambaPreTrainedModel._init_weights·  só   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�jÑ)Ô)ð 	>ð ×%Ò%Ñ'Ô'Ð'åÔ! &¤-Ô"6½$¼)ÀA¹,¼,ÐGÑGÔGÐGØŒ}Ô!Ð-Ý”˜FœMÔ.Ñ/Ô/Ð/ÝÔ! &¤/Ô"8½D¼IÀa¹L¼LÐIÑIÔIÐIàŒ{Ô3ð >ð ”OÔ*�Ø•T”Y˜tœ{Ô<Ñ=Ô=Ñ=���ð-	>ð 	>ð>ð >rZ   )rÖ   r×   rØ   r   Ú__annotations__Úbase_model_prefixÚ_no_split_modulesÚsupports_gradient_checkpointingÚ_is_statefulrG   rÛ   r  rÞ   rß   s   @rY   rý   rý   ¯  st   ø€ € € € € € àÐÐÑØ"ÐØ% |Ð4ÐØ&*Ð#Ø€Là€U„]�_„_ð>ð >ð >ð >ñ „_ð>ð >ð >ð >ð >rZ   rý   z,
    Class for the MAMBA model outputs.
    )Úcustom_introc                   óp   — e Zd ZU dZdZej        dz  ed<   dZe	dz  ed<   dZ
eej                 dz  ed<   dS )ÚMambaOutputa4  
    cache_params (`Cache`):
        The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to
        avoid providing the old `input_ids`.

        Includes both the State space model state matrices after the selective scan, and the Convolutional states
    NÚlast_hidden_stater�   r€   )rÖ   r×   rØ   rÙ   r  rG   ÚFloatTensorr  r�   r	   r€   Útupler®   rZ   rY   r  r  Ô  sh   € € € € € € ðð ð 37Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø!%€L�%˜$‘,Ð%Ð%Ñ%Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ð9Ð9rZ   r  zK
    Base class for causal language model (or autoregressive) outputs.
    c                   óŽ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
dz  ed<   dZeej                 dz  ed<   dS )ÚMambaCausalLMOutputa™  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    cache_params (`Cache`):
        The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to
        avoid providing the old `input_ids`.

        Includes both the State space model state matrices after the selective scan, and the Convolutional states
    NÚlossÚlogitsr�   r€   )rÖ   r×   rØ   rÙ   r  rG   r  r  r  r�   r	   r€   r  r®   rZ   rY   r  r  è  s   € € € € € € ð
ð 
ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø!%€L�%˜$‘,Ð%Ð%Ñ%Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ð9Ð9rZ   r  c                   óÆ   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zd„ Ze	 	 	 	 	 	 	 ddej	        dz  dej	        dz  de
dz  d	edz  d
edz  dedz  dej	        dz  deez  fd„¦   «         Zˆ xZS )Ú
MambaModelc                 ó¬  •‡— t          ¦   «                              ‰¦  «         t          j        ‰j        ‰j        ¦  «        | _        t          j        ˆfd„t          ‰j	        ¦  «        D ¦   «         ¦  «        | _
        d| _        t          ‰j        ‰j        ¬¦  «        | _        |                      | j        ¦  «         |                      ¦   «          d S )Nc                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS ))r   )rò   )Ú.0Úidxr   s     €rY   ú
<listcomp>z'MambaModel.__init__.<locals>.<listcomp>  s&   ø€ Ð$rÐ$rÐ$rÈ3¥Z°À#Ð%FÑ%FÔ%FÐ$rÐ$rÐ$rrZ   Frô   )r/   r0   r   Ú	EmbeddingÚ
vocab_sizer1   Ú
embeddingsÚ
ModuleListrÂ   r  r“   Úgradient_checkpointingrá   rö   Únorm_fÚ"_register_load_state_dict_pre_hookÚ	load_hookÚ	post_init©rW   r   rX   s    `€rY   r0   zMambaModel.__init__  s¶   øø€ Ý‰Œ×Ò˜Ñ Ô Ð åœ, vÔ'8¸&Ô:LÑMÔMˆŒÝ”mÐ$rÐ$rÐ$rÐ$rÕRWÐX^ÔXpÑRqÔRqÐ$rÑ$rÔ$rÑsÔsˆŒà&+ˆÔ#Ý" 6Ô#5¸6Ô;TÐUÑUÔUˆŒà×/Ò/°´Ñ?Ô?Ð?Ø�ŠÑÔÐÐÐrZ   c                 óv   — |D ]5}d|v r/|                      |¦  «        ||                     dd¦  «        <    d S Œ6d S )Nz
embedding.zembeddings.)ÚpopÚreplace)rW   Ú
state_dictÚprefixÚargsÚks        rY   r(  zMambaModel.load_hook  sW   € Øð 	ð 	ˆAØ˜qÐ Ð ØEOÇ^Â^ÐTUÑEVÔEV�
˜1Ÿ9š9 \°=ÑAÔAÑBØ��ð !ð	ð 	rZ   c                 ó   — | j         S r­   ©r#  rï   s    rY   Úget_input_embeddingszMambaModel.get_input_embeddings  s
   € ØŒÐrZ   c                 ó   — || _         d S r­   r3  ©rW   Únew_embeddingss     rY   Úset_input_embeddingszMambaModel.set_input_embeddings  s   € Ø(ˆŒˆˆrZ   NÚ	input_idsÚinputs_embedsr�   Ú	use_cacheÚoutput_hidden_statesÚreturn_dictr‚   Úreturnc                 ó  — |�|n| j         j        }|�|n| j        s| j         j        nd}|�|n| j         j        }|du |duz  rt          d¦  «        ‚|€|                      |¦  «        }| j        r| j        r|rd}|r|€t          | j         ¬¦  «        }|}	|rdnd}
| j	        D ]} ||	||¬¦  «        }	|r|
|	fz   }
Œ|  
                    |	¦  «        }	|r|
|	fz   }
|st          d„ |	||
fD ¦   «         ¦  «        S t          |	|r|nd|
¬¦  «        S )	a¤  
        cache_params (`Cache`, *optional*):
            If passed along, the model uses the previous state in all the blocks (which will give the output for the
            `input_ids` provided as if the model add `state_input_ids + input_ids` as context).
        use_cache (`bool`, *optional*):
            If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits.
        NFz:You must specify exactly one of input_ids or inputs_embeds)r   r®   rú   c              3   ó   K  — | ]}|®|V — Œ	d S r­   r®   )r  Úvs     rY   ú	<genexpr>z%MambaModel.forward.<locals>.<genexpr>R  s(   è è € ÐfÐf˜qÐXYÐXe˜ÐXeÐXeÐXeÐXeÐfÐfrZ   )r  r�   r€   )r   r<  r‹   r;  r=  Ú
ValueErrorr#  r%  r
   r“   r&  r  r  )rW   r9  r:  r�   r;  r<  r=  r‚   rÓ   r€   Úall_hidden_statesÚmixer_blocks               rY   rÔ   zMambaModel.forward  s¯  € ð( %9Ð$DÐ Ð È$Ì+ÔJjð 	ð "+Ð!6�I�IÐZ^ÔZgÐ=r¸T¼[Ô=RÐ=RÐmrˆ	Ø%0Ð%<�k�kÀ$Ä+ÔBYˆà˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ŸOšO¨IÑ6Ô6ˆMàÔ&ð 	¨4¬=ð 	¸Yð 	ØˆIàð 	<˜Ð-Ý'¨t¬{Ð;Ñ;Ô;ˆLà%ˆØ"6Ð@˜B˜B¸DÐØœ;ð 	Ið 	IˆKØ'˜KØØ)Ø-ðñ ô ˆMð $ð IØ$5¸Ð8HÑ$HÐ!øàŸš MÑ2Ô2ˆàð 	EØ 1°]Ð4DÑ DÐàð 	gÝÐfÐf ]°LÐBSÐ$TÐfÑfÔfÑfÔfÐfåØ+Ø)2Ð<˜˜¸Ø+ð
ñ 
ô 
ð 	
rZ   )NNNNNNN)rÖ   r×   rØ   r0   r(  r4  r8  r   rG   rÝ   r	   rÚ   r  r  rÔ   rÞ   rß   s   @rY   r  r    s  ø€ € € € € ð
ð 
ð 
ð 
ð 
ðð ð ðð ð ð)ð )ð )ð ð .2Ø15Ø%)Ø!%Ø,0Ø#'Ø26ð<
ð <
àÔ# dÑ*ð<
ð Ô'¨$Ñ.ð<
ð ˜d‘lð	<
ð
 ˜$‘;ð<
ð # T™kð<
ð ˜D‘[ð<
ð Ô(¨4Ñ/ð<
ð 
�Ñ	ð<
ð <
ð <
ñ „^ð<
ð <
ð <
ð <
ð <
rZ   r  zˆ
    The MAMBA Model transformer with a language modeling head on top (linear layer with weights tied to the input
    embeddings).
    c                   ó4  ‡ — e Zd ZddiZˆ fd„Zd„ Zd„ Z	 	 	 	 	 ddedz  d	ej	        dz  d
e
dz  fˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej	        dz  d	ej	        dz  dej        dz  dedz  dej	        dz  de
dz  de
dz  de
dz  deej        z  deez  fd„¦   «         Zˆ xZS )ÚMambaForCausalLMzlm_head.weightzbackbone.embeddings.weightc                 óæ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S )NFr(   )
r/   r0   r  rþ   r   rA   r1   r"  Úlm_headr)  r*  s     €rY   r0   zMambaForCausalLM.__init__d  s^   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý" 6Ñ*Ô*ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒà�ŠÑÔÐÐÐrZ   c                 ó4   — | j                              ¦   «         S r­   )rþ   r4  rï   s    rY   r4  z%MambaForCausalLM.get_input_embeddingsk  s   € ØŒ}×1Ò1Ñ3Ô3Ð3rZ   c                 ó6   — | j                              |¦  «        S r­   )rþ   r8  r6  s     rY   r8  z%MambaForCausalLM.set_input_embeddingsn  s   € ØŒ}×1Ò1°.ÑAÔAÐArZ   NFr�   r‚   Úis_first_iterationc           	      óZ   •—  t          ¦   «         j        |f|||||dœ|¤Ž}|r|sd |d<   |S )N)r:  r;  r�   r‚   rL  r‚   )r/   Úprepare_inputs_for_generation)
rW   r9  r:  r;  r�   r‚   rL  rÓ   Úmodel_inputsrX   s
            €rY   rN  z.MambaForCausalLM.prepare_inputs_for_generationq  sf   ø€ ð =•u‘w”wÔ<Øð
à'ØØ%Ø)Ø1ð
ð 
ð ð
ð 
ˆð ð 	2Ð/ð 	2Ø-1ˆLÐ)Ñ*àÐrZ   r   r9  r:  Úlabelsr<  r=  r;  Úlogits_to_keepr>  c
           	      ó6  — |�|n| j         j        }|                      |||||||¬¦  «        }|d         }t          |	t          ¦  «        rt          |	 d¦  «        n|	}|                      |dd…|dd…f                              | j        j        j	        ¦  «        ¦  «         
                    ¦   «         }d}|�­|                     |j        ¦  «        }|ddd…dd…f                              ¦   «         }|ddd…f                              ¦   «         }t          ¦   «         } ||                     d|                     d¦  «        ¦  «        |                     d¦  «        ¦  «        }|s|f|dd…         z   }|�|f|z   n|S t!          |||j        |j        ¬¦  «        S )aN  
        cache_params (`Cache`, *optional*):
            If passed along, the model uses the previous state in all the blocks (which will give the output for the
            `input_ids` provided as if the model add `state_input_ids + input_ids` as context).
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        use_cache (`bool`, *optional*):
            If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits.
        N)r�   r:  r<  r=  r;  r‚   r   .r]   r   )r  r  r�   r€   )r   r=  rþ   r  r7   ÚslicerI  r½   rK   r\   rŒ   rL   re   r   r�   r‘   r  r�   r€   )rW   r9  r‚   r:  r�   rP  r<  r=  r;  rQ  rÓ   Úmamba_outputsr€   Úslice_indicesr  r  Úshift_logitsÚshift_labelsÚloss_fctÚoutputs                       rY   rÔ   zMambaForCausalLM.forwardŠ  sÄ  € ð2 &1Ð%<�k�kÀ$Ä+ÔBYˆàŸšØØ%Ø'Ø!5Ø#ØØ)ð &ñ 
ô 
ˆð & aÔ(ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@×CÒCÀDÄLÔDWÔD]Ñ^Ô^Ñ_Ô_×eÒeÑgÔgˆàˆØÐà—Y’Y˜vœ}Ñ-Ô-ˆFà! # s¨ s¨A¨A¨A +Ô.×9Ò9Ñ;Ô;ˆLØ! # q r r 'œ?×5Ò5Ñ7Ô7ˆLå'Ñ)Ô)ˆHØ�8˜L×-Ò-¨b°,×2CÒ2CÀBÑ2GÔ2GÑHÔHÈ,×J[ÒJ[Ð\^ÑJ_ÔJ_Ñ`Ô`ˆDàð 	FØ�Y ¨q¨r¨rÔ!2Ñ2ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå"ØØØ&Ô3Ø'Ô5ð	
ñ 
ô 
ð 	
rZ   )NNNNF)	NNNNNNNNr   )rÖ   r×   rØ   Ú_tied_weights_keysr0   r4  r8  r	   rG   rÝ   rÚ   rN  r   r  r7   rÜ   r  r  rÔ   rÞ   rß   s   @rY   rG  rG  [  sª  ø€ € € € € ð +Ð,HÐIÐðð ð ð ð ð4ð 4ð 4ðBð Bð Bð ØØ%)Ø26Ø*/ðð ð
 ˜d‘lðð Ô(¨4Ñ/ðð ! 4™Kðð ð ð ð ð ð2 ð .2Ø26Ø26Ø%)Ø*.Ø,0Ø#'Ø!%Ø-.ð=
ð =
àÔ# dÑ*ð=
ð Ô(¨4Ñ/ð=
ð Ô(¨4Ñ/ð	=
ð
 ˜d‘lð=
ð Ô  4Ñ'ð=
ð # T™kð=
ð ˜D‘[ð=
ð ˜$‘;ð=
ð ˜eœlÑ*ð=
ð 
Ð$Ñ	$ð=
ð =
ð =
ñ „^ð=
ð =
ð =
ð =
ð =
rZ   rG  )rG  r  rý   )6rÙ   rp   Údataclassesr   rG   r   Útorch.nnr   Ú r   rf   Úactivationsr   Úcache_utilsr	   r
   Ú
generationr   Úintegrationsr   Úintegrations.accelerater   Úmodeling_layersr   Úmodeling_utilsr   Úutilsr   r   r   Úutils.import_utilsr   r   r   r   Úconfiguration_mambar   Ú
get_loggerrÖ   r|   Ú(torch._higher_order_ops.associative_scanr   Úmambapy.pscanr   ÚModuler   rá   rò   rý   r  r  r  rG  Ú__all__r®   rZ   rY   ú<module>rm     s  ðð Ð à €€€Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ø %Ð %Ð %Ð %Ð %Ð %à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø =Ð =Ð =Ð =Ð =Ð =Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø -Ð -Ð -Ð -Ð -Ð -ðð ð ð ð ð ð ð ð ð ð
ð ð ð ð ð ð ð ð ð ð ð ð -Ð ,Ð ,Ð ,Ð ,Ð ,ð 
ˆÔ	˜HÑ	%Ô	%€àÐ˜WÑ%Ô%ð ØIÐIÐIÐIÐIÐIÐIàÐð ÐÑÔð Ø#Ð#Ð#Ð#Ð#Ð#Ð#à€EðCNð CNð CNð CNð CN�”ñ CNô CNð CNðL
Fð Fð Fð Fð F�2”9ñ Fô Fð Fð(ð ð ð ð Ð+ñ ô ð ð4 ð!>ð !>ð !>ð !>ð !>˜?ñ !>ô !>ñ „ð!>ðH €ððñ ô ð
 ð:ð :ð :ð :ð :�+ñ :ô :ñ „ñô ð:ð €ððñ ô ð
 ð:ð :ð :ð :ð :˜+ñ :ô :ñ „ñô ð:ð& ðV
ð V
ð V
ð V
ð V
Ð%ñ V
ô V
ñ „ðV
ðr €ððñ ô ðg
ð g
ð g
ð g
ð g
Ð+¨_ñ g
ô g
ñô ðg
ðT EÐ
DÐ
D€€€rZ   