§
    ‚ŠtjÕ¦  ã                   óŠ  — d dl mZ d dlZd dlmZ ddlmZ ddlmZ ddl	m
Z
mZ ddlmZ dd	lmZmZmZmZmZ dd
lmZ ddlmZmZ ddlmZmZ ddlmZmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1  e(j2        e3¦  «        Z4 ed¦  «         G d„ dej5        ¦  «        ¦   «         Z6d„ Z7 ed¦  «        dFd„¦   «         Z8dej9        de:dej9        fd„Z;	 dGd!ej5        d"ej9        d#ej9        d$ej9        d%ej9        dz  d&e<d'e<d(e#e%         fd)„Z= ee8¦  «         G d*„ d+ej5        ¦  «        ¦   «         Z> G d,„ d-ej5        ¦  «        Z? G d.„ d/ej5        ¦  «        Z@e G d0„ d1ej5        ¦  «        ¦   «         ZA G d2„ d3ej5        ¦  «        ZB G d4„ d5e¦  «        ZC G d6„ d7e¦  «        ZD G d8„ d9e!¦  «        ZEeCeDd:œZFe& G d;„ d<eE¦  «        ¦   «         ZG	 	 	 dHd>ej9        eHej9                 z  dz  d?e:dz  d%ej9        dz  dej9        e:z  fd@„ZIe& G dA„ dBeEe¦  «        ¦   «         ZJ G dC„ dDeeE¦  «        ZKg dE¢ZLdS )Ié    )ÚCallableN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úlazy_load_kernelÚuse_experts_implementationÚuse_kernel_forward_from_hubÚuse_kernel_func_from_hubÚuse_kernelized_func)Úforce_accelerate_hooks)Úcreate_causal_maskÚcreate_recurrent_attention_mask)Ú GenericForSequenceClassificationÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)Úresolve_internal_import)ÚOutputRecorderÚcapture_outputsé   )ÚJambaConfigÚRMSNormc                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
ÚJambaRMSNormç�íµ ÷Æ°>ÚepsÚreturnNc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z;
        JambaRMSNorm is equivalent to T5LayerNorm
        N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer(   Ú	__class__s      €úf/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/jamba/modeling_jamba.pyr,   zJambaRMSNorm.__init__;   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐó    Úhidden_statesc                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )Né   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtor.   Úfloat32ÚpowÚmeanÚrsqrtr1   r0   )r2   r7   Úinput_dtypeÚvariances       r5   ÚforwardzJambaRMSNorm.forwardC   s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r6   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)Útupler0   Úshaper1   )r2   s    r5   Ú
extra_reprzJambaRMSNorm.extra_reprJ   s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr6   )r'   )
Ú__name__Ú
__module__Ú__qualname__Úfloatr,   r.   ÚTensorrD   rH   Ú__classcell__©r4   s   @r5   r&   r&   9   sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð Jr6   r&   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr:   r9   ©Údim)rG   r.   Úcat)ÚxÚx1Úx2s      r5   Úrotate_halfrW   N   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r6   Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezerW   )ÚqÚkÚcosÚsinÚunsqueeze_dimÚq_embedÚk_embeds          r5   Úapply_rotary_pos_embrb   U   sc   € ð& �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr6   r7   Ún_repr)   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r"   N)rG   ÚexpandÚreshape)r7   rc   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         r5   Ú	repeat_kvrk   o   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr6   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr9   r   r:   ©rR   r<   )ÚpÚtrainingr"   )rk   Únum_key_value_groupsr.   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr>   r=   r<   rs   rx   Ú
contiguous)rm   rn   ro   rp   rq   rr   rs   rt   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r5   Úeager_attention_forwardrƒ   {   sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r6   c                   ó¤   ‡ — e Zd ZdZdedefˆ fd„Z	 	 ddej        dej        dz  de	dz  d	e
e         d
eej        ej        dz  f         f
d„Zˆ xZS )ÚJambaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperÚconfigÚ	layer_idxc                 ó†  •— t          ¦   «                              ¦   «          || _        || _        t	          |d|j        |j        z  ¦  «        | _        |j        |j        z  | _	        | j        dz  | _
        |j        | _        d| _        t          j        |j        |j        | j        z  d¬¦  «        | _        t          j        |j        |j        | j        z  d¬¦  «        | _        t          j        |j        |j        | j        z  d¬¦  «        | _        t          j        |j        | j        z  |j        d¬¦  «        | _        d S )Nrj   g      à¿TF©Úbias)r+   r,   r†   r‡   Úgetattrr3   Únum_attention_headsrj   rh   ry   rr   Úattention_dropoutÚ	is_causalr   ÚLinearÚq_projÚk_projÚv_projÚo_proj)r2   r†   r‡   r4   s      €r5   r,   zJambaAttention.__init__˜   s  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!Ø”} dÑ*ˆŒØ!'Ô!9ˆÔØˆŒÝ”i Ô 2°FÔ4NÐQUÔQ^Ñ4^ÐejÐkÑkÔkˆŒÝ”i Ô 2°FÔ4NÐQUÔQ^Ñ4^ÐejÐkÑkÔkˆŒÝ”i Ô 2°FÔ4NÐQUÔQ^Ñ4^ÐejÐkÑkÔkˆŒÝ”i Ô :¸T¼]Ñ JÈFÔL^ÐejÐkÑkÔkˆŒˆˆr6   Nr7   rq   Úpast_key_valuesrt   r)   c                 óî  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|�|                     ||	| j        ¦  «        \  }}	t          j
        | j        j        t          ¦  «        }
 |
| |||	|f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr:   r"   r9   rl   )rs   rr   )rG   rj   r�   Úviewr{   r‘   r’   Úupdater‡   r   Úget_interfacer†   Ú_attn_implementationrƒ   rx   r�   rr   rf   r~   r“   )r2   r7   rq   r”   rt   Úinput_shapeÚhidden_shapeÚquery_statesr   r€   Úattention_interfacer‚   r�   s                r5   rD   zJambaAttention.forward¦   s�  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r6   ©NN)rI   rJ   rK   Ú__doc__r#   Úintr,   r.   rM   r   r   r   rF   rD   rN   rO   s   @r5   r…   r…   ”   sÐ   ø€ € € € € àGÐGðl˜{ð l°sð lð lð lð lð lð lð" /3Ø(,ð	")ð ")à”|ð")ð œ tÑ+ð")ð  ™ð	")ð
 Ð+Ô,ð")ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð")ð ")ð ")ð ")ð ")ð ")ð ")ð ")r6   r…   c                   óØ   ‡ — e Zd ZdZdefˆ fd„Z	 	 ddej        dedz  dej	        dz  fd„Z
ddedz  dej	        dz  fd	„Z ed
¦  «        	 	 ddedz  dej	        dz  fd„¦   «         Zˆ xZS )ÚJambaMambaMixeruƒ  
    Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)
    r†   c           	      ó  •— t          ¦   «                              ¦   «          || _        || _        |j        | _        |j        | _        |j        | _        |j	        |j        z  | _
        |j        | _        |j        | _        |j        | _        t#          j        | j
        | j
        | j        | j        | j
        | j        dz
  ¬¦  «        | _        |j        | _        t,          |j                 | _        t#          j        | j        | j
        dz  | j        ¬¦  «        | _        t#          j        | j
        | j        | j        dz  z   d¬¦  «        | _        t#          j        | j        | j
        d¬¦  «        | _        t9          j        d| j        dz   ¦  «        d d d …f         }|                     | j
        d¦  «                             ¦   «         }t#          j         t9          j!        |¦  «        ¦  «        | _"        t#          j         t9          j#        | j
        ¦  «        ¦  «        | _$        t#          j        | j
        | j        | j        ¬¦  «        | _%        tM          | j        |j'        ¬¦  «        | _(        tM          | j        |j'        ¬¦  «        | _)        tM          | j        |j'        ¬¦  «        | _*        tW          d	¦  «        }tY          |d
d ¦  «        a-tY          |dd ¦  «        a.tW          d¦  «        }t_          |d¬¦  «        a0tY          |dd ¦  «        a1tY          |dd ¦  «        a2tg          t`          tb          t\          tZ          td          f¦  «        a4th          stj           6                    d¦  «         |j7        |         | _8        d S )Nr"   )Úin_channelsÚout_channelsrŠ   Úkernel_sizeÚgroupsÚpaddingr9   r‰   FTr:   ©r(   zcausal-conv1dÚcausal_conv1d_updateÚcausal_conv1d_fnz	mamba-ssmz8ops.triton.selective_state_update.selective_state_update)Úchained_pathÚselective_scan_fnÚmamba_inner_fna  The fast path is not available because on of `(selective_state_update, selective_scan_fn, causal_conv1d_fn, causal_conv1d_update, mamba_inner_fn)` is None. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1d.)9r+   r,   r†   r‡   r3   Úmamba_d_stateÚssm_state_sizeÚmamba_d_convÚconv_kernel_sizeÚmamba_expandÚintermediate_sizeÚmamba_dt_rankÚtime_step_rankÚmamba_conv_biasÚuse_conv_biasÚmamba_proj_biasÚuse_biasr   ÚConv1dÚconv1dÚ
hidden_actÚ
activationr   Úactr�   Úin_projÚx_projÚdt_projr.   Úarangere   r~   r-   ÚlogÚA_logr/   ÚDÚout_projr&   Úrms_norm_epsÚdt_layernormÚb_layernormÚc_layernormr   r‹   rª   r«   r   Úselective_state_updater­   r®   ÚallÚis_fast_path_availableÚloggerÚwarning_onceÚlayer_typesÚ
layer_type)r2   r†   r‡   ÚAÚcausal_conv1dÚ	mamba_ssmr4   s         €r5   r,   zJambaMambaMixer.__init__Ó   s'  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ!Ô-ˆÔØ$Ô2ˆÔØ &Ô 3ˆÔØ!'Ô!4°vÔ7IÑ!IˆÔØ$Ô2ˆÔØ#Ô3ˆÔØÔ.ˆŒÝ”iØÔ.ØÔ/ØÔ#ØÔ-ØÔ)ØÔ)¨AÑ-ð
ñ 
ô 
ˆŒð !Ô+ˆŒÝ˜&Ô+Ô,ˆŒõ ”y Ô!1°4Ô3IÈAÑ3MÐTXÔTaÐbÑbÔbˆŒå”i Ô 6¸Ô8KÈdÔNaÐdeÑNeÑ8eÐlqÐrÑrÔrˆŒå”y Ô!4°dÔ6LÐSWÐXÑXÔXˆŒõ ŒL˜˜DÔ/°!Ñ3Ñ4Ô4°T¸1¸1¸1°WÔ=ˆØ�HŠH�TÔ+¨RÑ0Ô0×;Ò;Ñ=Ô=ˆå”\¥%¤)¨A¡,¤,Ñ/Ô/ˆŒ
Ý”�eœj¨Ô)?Ñ@Ô@ÑAÔAˆŒÝœ	 $Ô"8¸$Ô:JÐQUÔQ^Ð_Ñ_Ô_ˆŒå(¨Ô)<À&ÔBUÐVÑVÔVˆÔÝ'¨Ô(;ÀÔATÐUÑUÔUˆÔÝ'¨Ô(;ÀÔATÐUÑUÔUˆÔõ )¨Ñ9Ô9ˆÝ& }Ð6LÈdÑSÔSÐÝ" =Ð2DÀdÑKÔKÐõ % [Ñ1Ô1ˆ	Ý!8ØÐ$^ð"
ñ "
ô "
Ðõ $ IÐ/BÀDÑIÔIÐÝ  Ð,<¸dÑCÔCˆõ "%Ý#Õ%6Õ8HÕJ^Õ`nÐoñ"
ô "
Ðõ &ð 	Ý×ÒðRñô ð ð
 !Ô,¨YÔ7ˆŒˆˆr6   Nr7   Úcache_paramsrq   c                 óF	  — |j         \  }}}|d uo|                     | j        ¦  «        o|dk    }|                      |¦  «                             dd¦  «        }|                     dd¬¦  «        \  }}	|�||                     d¦  «        z  }| j        j         	                    | j        j         
                    d¦  «        | j        j         
                    d¦  «        ¦  «        }
|rft          |                     d¦  «        |j        | j                 j        d         |
| j        j        | j        ¦  «        }|                     d¦  «        }nt|�Pt"          j                             || j        |j         d         z
  df¦  «        }|                     || j        ¦  «         t-          ||
| j        j        | j        ¬¦  «        }|�||                     d¦  «        z  }|                      |                     dd¦  «        ¦  «        }t1          j        || j        | j        | j        gd¬¦  «        \  }}}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }| j        j        j         }t1          j!        ¦   «         5  t1          j"        | j        j        j         ¦  «        | j        j        _         d d d ¦  «         n# 1 swxY w Y   |                      |¦  «                             dd¦  «        }t1          j!        ¦   «         5  || j        j        _         d d d ¦  «         n# 1 swxY w Y   t1          j#        | j$         %                    ¦   «         ¦  «         }|�| %                    ¦   «         nd }|rstM          |j        | j                 j'        d         |d         |d         ||d d …df         |d d …df         | j(        |	d         |d¬	¦
  «
                             d¦  «        }nztS          ||||                     dd¦  «        |                     dd¦  «        | j(         %                    ¦   «         |	|dd¬
¦
  «
        \  }}|�|�| *                    || j        ¦  «         |  +                    |                     dd¦  «        ¦  «        }|S )Nr"   r9   rQ   r   r:   )r¾   ).r   T)Údt_softplus)Údelta_softplusÚreturn_last_state),rG   Úhas_previous_stater‡   rÀ   r{   ÚchunkrZ   r¼   r0   r–   Úsizerª   ÚsqueezeÚlayersÚconv_statesrŠ   r¾   r   r|   Úpadr²   Úupdate_conv_stater«   rÁ   r.   Úsplitr¶   r°   rÉ   rÊ   rË   rÂ   ÚdataÚno_gradÚ
zeros_likeÚexprÅ   rL   rÌ   Úrecurrent_statesrÆ   r­   Úupdate_recurrent_staterÇ   )r2   r7   rÖ   rq   Ú
batch_sizeÚseq_lenÚ_Úuse_precomputed_statesÚprojected_statesÚgateÚconv_weightsrà   Ússm_parametersÚ	time_stepÚBÚCÚtime_proj_biasÚdiscrete_time_steprÓ   Úscan_outputsÚ	ssm_stateÚcontextualized_statess                         r5   Úcuda_kernels_forwardz$JambaMambaMixer.cuda_kernels_forward  s
  € ð "/Ô!4Ñˆ
�G˜Qà Ð$Ði¨×)HÒ)HÈÌÑ)XÔ)XÐiÐ]dÐhiÒ]ið 	ð  Ÿ<š<¨Ñ6Ô6×@Ò@ÀÀAÑFÔFÐð /×4Ò4°Q¸AÐ4Ñ>Ô>Ñˆ�tàÐ%Ø)¨N×,DÒ,DÀQÑ,GÔ,GÑGˆMð ”{Ô)×.Ò.¨t¬{Ô/A×/FÒ/FÀqÑ/IÔ/IÈ4Ì;ÔK]×KbÒKbÐcdÑKeÔKeÑfÔfˆØ!ð 	xÝ0Ø×%Ò% bÑ)Ô)ØÔ# D¤NÔ3Ô?ÀÔBØØ”Ô Ø”ñô ˆMð *×3Ò3°BÑ7Ô7ˆMˆMàÐ'Ý œm×/Ò/°ÀÔ@UÐXeÔXkÐlnÔXoÑ@oÐqrÐ?sÑtÔt�Ø×.Ò.¨{¸D¼NÑKÔKÐKÝ,¨]¸LÈ$Ì+ÔJZÐgkÔgvÐwÑwÔwˆMàÐ%Ø)¨N×,DÒ,DÀQÑ,GÔ,GÑGˆMð Ÿš ]×%<Ò%<¸QÀÑ%BÔ%BÑCÔCˆÝœ+Ø˜TÔ0°$Ô2EÀtÔGZÐ[Ðacð
ñ 
ô 
‰ˆ	�1�að ×%Ò% iÑ0Ô0ˆ	Ø×Ò˜QÑÔˆØ×Ò˜QÑÔˆð œÔ*Ô/ˆÝŒ]‰_Œ_ð 	Nð 	NÝ%*Ô%5°d´lÔ6GÔ6LÑ%MÔ%MˆDŒLÔÔ"ð	Nð 	Nð 	Nñ 	Nô 	Nð 	Nð 	Nð 	Nð 	Nð 	Nð 	Nøøøð 	Nð 	Nð 	Nð 	Nà!Ÿ\š\¨)Ñ4Ô4×>Ò>¸qÀ!ÑDÔDÐÝŒ]‰_Œ_ð 	4ð 	4Ø%3ˆDŒLÔÔ"ð	4ð 	4ð 	4ñ 	4ô 	4ð 	4ð 	4ð 	4ð 	4ð 	4ð 	4øøøð 	4ð 	4ð 	4ð 	4õ ŒY�t”z×'Ò'Ñ)Ô)Ñ*Ô*Ð*ˆà3AÐ3M˜×-Ò-Ñ/Ô/Ð/ÐSWˆØ!ð 	OÝ1ØÔ# D¤NÔ3ÔDÀQÔGØ˜fÔ%Ø" 6Ô*ØØ�!�!�!�Q�$”Ø�!�!�!�Q�$”Ø”Ø�V”ØØ ðñ ô ÷ Ši˜‰mŒmð ˆLõ '8ØØ"ØØ—’˜A˜qÑ!Ô!Ø—’˜A˜qÑ!Ô!Ø”—’‘”ØØØ#Ø"&ð'ñ 'ô 'Ñ#ˆL˜)ð Ð$¨Ð)AØ×3Ò3°I¸t¼~ÑNÔNÐNð !%§¢¨l×.DÒ.DÀQÈÑ.JÔ.JÑ KÔ KÐà$Ð$s$   Ê3KËKËKÌL:Ì:L>ÍL>c           	      óÈ
  — |j         \  }}}|j        }|                      |¦  «                             dd¦  «        }|                     dd¬¦  «        \  }	}
|�|	|                     d¦  «        z  }	|�J|                     | j        ¦  «        r0|j        | j                 j	        d          
                    ¦   «         }n)t          j        || j        | j        f|	j        |¬¦  «        }|��`|                     | j        ¦  «        r³|dk    r­|                     |	| j        ¦  «        d| j         d …f         }t          j        || j        j        d d …dd d …f         z  d¬¦  «        }	| j        r|	| j        j        z  }	|                      |	¦  «                             |¦  «                             d¦  «        }	nÅt2          j                             |	| j        |	j         d         z
  df¦  «        }|                     || j        ¦  «        d| j         d …f         }|                      |                      |	¦  «        dd |…f         ¦  «        }	n2|                      |                      |	¦  «        dd |…f         ¦  «        }	|�|	|                     d¦  «        z  }	|                      |	                     dd¦  «        ¦  «        }t          j        || j        | j        | j        gd¬¦  «        \  }}}|                      |¦  «        }|                       |¦  «        }|  !                    |¦  «        }|  "                    |¦  «        }t2          j         #                    |¦  «                             dd¦  «        }t          j$        | j%         &                    ¦   «         ¦  «         }t          j$        |d d d …d d d …f         |d d …d d …d d …d f         z  ¦  «        }|d d …d d …d d …d f         |d d …d d d …d d …f          &                    ¦   «         z  }||	d d …d d …d d …d f          &                    ¦   «         z  }g }tO          |¦  «        D ]”}|d d …d d …|d d …f         |z  |d d …d d …|d d …f         z   }t          j(        |                     |¦  «        |d d …|d d …f                              d¦  «        ¦  «        }| )                    |d d …d d …df         ¦  «         Œ•t          j*        |d¬¦  «        }||	| j+        d d d …d f         z  z   }||                      |
¦  «        z  }|�| ,                    || j        ¦  «         |  -                    |                     dd¦  «        ¦  «        }|S )Nr"   r9   rQ   r   )Údevicer<   .r:   ).rG   r<   rÀ   r{   rÜ   rZ   rÛ   r‡   rß   rè   Úcloner.   Úzerosr´   r°   rü   râ   r²   Úsumr¼   r0   r¸   rŠ   r¿   r=   r   r|   rá   rÁ   rã   r¶   rÉ   rÊ   rË   rÂ   Úsoftplusrç   rÅ   rL   Úrangerz   ÚappendÚstackrÆ   ré   rÇ   )r2   Úinput_statesrÖ   rq   rê   rë   rì   r<   rî   r7   rï   rø   Ú
conv_staterñ   rò   ró   rô   rö   rÓ   Ú
discrete_AÚ
discrete_BÚdeltaB_ur÷   ÚiÚscan_outputrù   s                             r5   Úslow_forwardzJambaMambaMixer.slow_forward~  s�  € Ø!-Ô!3Ñˆ
�G˜QØÔ"ˆàŸ<š<¨Ñ5Ô5×?Ò?ÀÀ1ÑEÔEÐØ.×4Ò4°Q¸AÐ4Ñ>Ô>Ñˆ�tàÐ%Ø)¨N×,DÒ,DÀQÑ,GÔ,GÑGˆMàÐ#¨×(GÒ(GÈÌÑ(WÔ(WÐ#à$Ô+¨D¬NÔ;ÔLÈQÔO×UÒUÑWÔWˆIˆIåœØ˜TÔ3°TÔ5HÐIØ$Ô+°5ðñ ô ˆIð Ñ#Ø×.Ò.¨t¬~Ñ>Ô>ð TÀ7ÈaÂ<À<Ø)×;Ò;¸MÈ4Ì>ÑZÔZÐ[^ÐaeÔavÐ`vÐ`wÐ`wÐ[wÔx�
Ý %¤	¨*°t´{Ô7IÈ!È!È!ÈQÐPQÐPQÐPQÈ'Ô7RÑ*RÐXZÐ [Ñ [Ô [�ØÔ%ð 6Ø! T¤[Ô%5Ñ5�MØ $§¢¨Ñ 7Ô 7× :Ò :¸5Ñ AÔ A× KÒ KÈBÑ OÔ O��åœ]×.Ò.Ø!ØÔ*¨]Ô-@ÀÔ-DÑDÀaÐHñô �
ð *×;Ò;¸JÈÌÑWÔWÐX[Ð^bÔ^sÐ]sÐ]tÐ]tÐXtÔu�
Ø $§¢¨¯ª°]Ñ)CÔ)CÀCÈÈ'ÈÀMÔ)RÑ SÔ S��à ŸHšH T§[¢[°Ñ%?Ô%?ÀÀXÀgÀXÀÔ%NÑOÔOˆMàÐ%Ø)¨N×,DÒ,DÀQÑ,GÔ,GÑGˆMð Ÿš ]×%<Ò%<¸QÀÑ%BÔ%BÑCÔCˆÝœ+Ø˜TÔ0°$Ô2EÀtÔGZÐ[Ðacð
ñ 
ô 
‰ˆ	�1�að ×%Ò% iÑ0Ô0ˆ	Ø×Ò˜QÑÔˆØ×Ò˜QÑÔˆà!Ÿ\š\¨)Ñ4Ô4ÐÝœ]×3Ò3Ð4FÑGÔG×QÒQÐRSÐUVÑWÔWÐõ ŒY�t”z×'Ò'Ñ)Ô)Ñ*Ô*Ð*ˆÝ”Y˜q  q q q¨$°°°Ð!1Ô2Ð5GÈÈÈÈ1È1È1ÈaÈaÈaÐQUÈÔ5VÑVÑWÔWˆ
Ø'¨¨¨¨1¨1¨1¨a¨a¨a°¨Ô6¸¸1¸1¸1¸dÀAÀAÀAÀqÀqÀq¸=Ô9I×9OÒ9OÑ9QÔ9QÑQˆ
Ø ¨a¨a¨a°°°°A°A°A°t¨mÔ <× BÒ BÑ DÔ DÑDˆàˆÝ�w‘”ð 	6ð 	6ˆAØ" 1 1 1 a a a¨¨A¨A¨A :Ô.°Ñ:¸XÀaÀaÀaÈÈÈÈAÈqÈqÈqÀjÔ=QÑQˆIÝœ, y§|¢|°EÑ':Ô':¸A¸a¸a¸aÀÀAÀAÀA¸g¼J×<PÒ<PÐQSÑ<TÔ<TÑUÔUˆKØ×Ò ¨A¨A¨A¨q¨q¨q°!¨GÔ 4Ñ5Ô5Ð5Ð5Ý”k ,°BÐ7Ñ7Ô7ˆØ! ]°T´V¸DÀ!À!À!ÀT¸MÔ5JÑ%JÑKˆØ" T§X¢X¨d¡^¤^Ñ3ˆàÐ#Ø×/Ò/°	¸4¼>ÑJÔJÐJð !%§¢¨k×.CÒ.CÀAÀqÑ.IÔ.IÑ JÔ JÐØ$Ð$r6   r¼   c                 ó  — | j         j        rEt          rd| j        j        j        j        vr&t                               d¦  «         d| j         _        | j         j        r|  	                    |||¦  «        S |  
                    |||¦  «        S )NÚcudazÔFast Mamba kernels are not available. Make sure that they are installed and that the mamba module is on a CUDA device. Turning off the fast path `config.use_mamba_kernels=False` and falling back to the slow path.F)r†   Úuse_mamba_kernelsrÎ   rÁ   r0   rü   ÚtyperÏ   rÐ   rú   r  )r2   r7   rÖ   rq   s       r5   rD   zJambaMambaMixer.forwardË  s—   € ð Œ;Ô(ð 	2Ý&ð	2Ø*0¸¼Ô8JÔ8QÔ8VÐ*VÐ*Vå×ÒðVñô ð ð
 -2ˆDŒKÔ)àŒ;Ô(ð 	ZØ×,Ò,¨]¸LÈ.ÑYÔYÐYØ× Ò  °¸nÑMÔMÐMr6   rž   )rI   rJ   rK   rŸ   r#   r,   r.   rM   r   Ú
LongTensorrú   r  r   rD   rN   rO   s   @r5   r¢   r¢   Ë   sK  ø€ € € € € ðð ðC8˜{ð C8ð C8ð C8ð C8ð C8ð C8ðP &*Ø26ð	c%ð c%à”|ðc%ð ˜d‘lðc%ð Ô(¨4Ñ/ð	c%ð c%ð c%ð c%ðLJ%ð J%°u¸t±|ð J%Ð\aÔ\lÐosÑ\sð J%ð J%ð J%ð J%ðZ Ð˜HÑ%Ô%ð &*Ø26ð	Nð Nð ˜d‘lðNð Ô(¨4Ñ/ð	Nð Nð Nñ &Ô%ðNð Nð Nð Nð Nr6   r¢   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚJambaMLPc                 ó˜  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _	        t          |j                 | _        d S ©NFr‰   )r+   r,   r†   r3   r´   r   r�   Ú	gate_projÚup_projÚ	down_projr   r½   Úact_fn©r2   r†   r4   s     €r5   r,   zJambaMLP.__init__â  s¦   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔÝœ 4Ô#3°TÔ5KÐRWÐXÑXÔXˆŒÝ”y Ô!1°4Ô3IÐPUÐVÑVÔVˆŒÝœ 4Ô#9¸4Ô;KÐRWÐXÑXÔXˆŒÝ˜VÔ.Ô/ˆŒˆˆr6   c                 ó¨   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        }|S )N)r  r  r  r  )r2   rT   r  s      r5   rD   zJambaMLP.forwardì  sA   € Ø—N’N 4§;¢;¨t¯~ª~¸aÑ/@Ô/@Ñ#AÔ#AÀDÇLÂLÐQRÁOÄOÑ#SÑTÔTˆ	ØÐr6   )rI   rJ   rK   r,   rD   rN   rO   s   @r5   r  r  á  sG   ø€ € € € € ð0ð 0ð 0ð 0ð 0ðð ð ð ð ð ð r6   r  c                   óh   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        dej        dej        fd„Zˆ xZ	S )	ÚJambaExpertsz2Collection of expert weights stored as 3D tensors.r†   c                 ó´  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          j	        t          j        | j        d| j        z  | j        ¦  «        ¦  «        | _        t          j	        t          j        | j        | j        | j        ¦  «        ¦  «        | _        t          |j                 | _        d S )Nr9   )r+   r,   Únum_local_expertsÚnum_expertsr3   Ú
hidden_dimr´   Úintermediate_dimr   r-   r.   ÚemptyÚgate_up_projr  r   r½   r  r  s     €r5   r,   zJambaExperts.__init__õ  s£   ø€ Ý‰Œ×ÒÑÔÐØ!Ô3ˆÔØ Ô,ˆŒØ &Ô 8ˆÔÝœL­¬°TÔ5EÀqÈ4ÔK`ÑG`ÐbfÔbqÑ)rÔ)rÑsÔsˆÔÝœ¥e¤k°$Ô2BÀDÄOÐUYÔUjÑ&kÔ&kÑlÔlˆŒÝ˜VÔ.Ô/ˆŒˆˆr6   r7   Útop_k_indexÚtop_k_weightsr)   c                 ó€  — t          j        |¦  «        }t          j        ¦   «         5  t           j        j                             || j        ¬¦  «        }|                     ddd¦  «        }t          j        | 	                    d¬¦  «        d¦  «         
                    ¦   «         }d d d ¦  «         n# 1 swxY w Y   |D ]þ}|d         }|| j        k    rŒt          j        ||         ¦  «        \  }}	||	         }
t          j                             |
| j        |         ¦  «                             dd¬¦  «        \  }}|                      |¦  «        |z  }t          j                             || j        |         ¦  «        }|||	|d f         z  }|                     d|	|                     |j        ¦  «        ¦  «         Œÿ|S )N)Únum_classesr9   r"   r   )r:   éþÿÿÿrQ   r:   )r.   ræ   rå   r   r|   Úone_hotr  ÚpermuteÚgreaterrÿ   ÚnonzeroÚwhereÚlinearr#  rÜ   r  r  Ú
index_add_r=   r<   )r2   r7   r$  r%  Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_staterï   ÚupÚcurrent_hidden_statess                 r5   rD   zJambaExperts.forwardþ  sø  € õ $Ô.¨}Ñ=Ô=ÐÝŒ]‰_Œ_ð 	Sð 	SÝœ(Ô-×5Ò5°kÈtÔO_Ð5Ñ`Ô`ˆKØ%×-Ò-¨a°°AÑ6Ô6ˆKÝœ {§¢¸8 Ñ'DÔ'DÀaÑHÔH×PÒPÑRÔRˆJð	Sð 	Sð 	Sñ 	Sô 	Sð 	Sð 	Sð 	Sð 	Sð 	Sð 	Søøøð 	Sð 	Sð 	Sð 	Sð
 %ð 
	nð 
	nˆJØ# AœˆJØ˜TÔ-Ò-Ð-ØÝ#(¤;¨{¸:Ô/FÑ#GÔ#GÑ ˆI�yØ)¨)Ô4ˆMÝ”}×+Ò+¨M¸4Ô;LÈZÔ;XÑYÔY×_Ò_Ð`aÐgiÐ_ÑjÔj‰HˆD�"Ø$(§K¢K°Ñ$5Ô$5¸Ñ$:Ð!Ý$&¤M×$8Ò$8Ð9NÐPTÔP^Ð_iÔPjÑ$kÔ$kÐ!Ø$9¸MÈ)ÐU^Ð`dÐJdÔ<eÑ$eÐ!Ø×*Ò*¨1¨iÐ9N×9QÒ9QÐReÔRkÑ9lÔ9lÑmÔmÐmÐmà"Ð"s   ¨A>B2Â2B6Â9B6)
rI   rJ   rK   rŸ   r#   r,   r.   rM   rD   rN   rO   s   @r5   r  r  ñ  sŒ   ø€ € € € € à<Ð<ð0˜{ð 0ð 0ð 0ð 0ð 0ð 0ð#à”|ð#ð ”\ð#ð ”|ð	#ð
 
Œð#ð #ð #ð #ð #ð #ð #ð #r6   r  c                   óR   ‡ — e Zd ZdZdefˆ fd„Zd„ Zdej        dej        fd„Z	ˆ xZ
S )ÚJambaSparseMoeBlockaÈ  
    This implementation is
    strictly equivalent to standard MoE with full capacity (no
    dropped tokens). It's faster since it formulates MoE operations
    in terms of block-sparse operations to accommodate imbalanced
    assignments of tokens to experts, whereas standard MoE either
    (1) drop tokens at the cost of reduced performance or (2) set
    capacity factor to number of experts and thus waste computation
    and memory on padding.
    r†   c                 ó  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        |j        | _        t          j
        | j        | j        d¬¦  «        | _        t          |¦  «        | _        d S r  )r+   r,   r3   r   r´   Úffn_dimr  Únum_experts_per_tokÚtop_kr   r�   Úrouterr  Úexpertsr  s     €r5   r,   zJambaSparseMoeBlock.__init__%  sr   ø€ Ý‰Œ×ÒÑÔÐØ Ô,ˆŒØÔ/ˆŒØ!Ô-ˆÔØÔ/ˆŒ
å”i ¤°Ô1AÈÐNÑNÔNˆŒÝ# FÑ+Ô+ˆŒˆˆr6   c                 óÚ   — t           j        j                             |dt           j        ¬¦  «        }t          j        || j        d¬¦  «        \  }}||                     |j        ¦  «        fS )Nr:   rv   rQ   )	r.   r   r|   r}   rL   Útopkr>  r=   r<   )r2   r7   Úrouter_logitsÚrouting_weightsr%  r$  s         r5   Úroute_tokens_to_expertsz+JambaSparseMoeBlock.route_tokens_to_experts/  s_   € Ýœ(Ô-×5Ò5°mÈÕSXÔS^Ð5Ñ_Ô_ˆÝ%*¤Z°ÀÄÐQSÐ%TÑ%TÔ%TÑ"ˆ�{Ø˜M×,Ò,¨]Ô-@ÑAÔAÐAÐAr6   r7   r)   c                 ó   — |j         \  }}}|                     d|¦  «        }|                      |¦  «        }|                      ||¦  «        \  }}|                      |||¦  «        }|                     |||¦  «        }|S )Nr:   )rG   r–   r?  rE  r@  rf   )r2   r7   rê   Úsequence_lengthr   rC  r$  r%  s           r5   rD   zJambaSparseMoeBlock.forward4  s„   € Ø2?Ô2EÑ/ˆ
�O ZØ%×*Ò*¨2¨zÑ:Ô:ˆØŸš MÑ2Ô2ˆØ%)×%AÒ%AÀ-ÐQ^Ñ%_Ô%_Ñ"ˆ�]ØŸš ]°KÀÑOÔOˆØ%×-Ò-¨j¸/È:ÑVÔVˆØÐr6   )rI   rJ   rK   rŸ   r#   r,   rE  r.   rM   rD   rN   rO   s   @r5   r:  r:    s†   ø€ € € € € ð	ð 	ð,˜{ð ,ð ,ð ,ð ,ð ,ð ,ðBð Bð Bð
 U¤\ð °e´lð ð ð ð ð ð ð ð r6   r:  c                   ó¢   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 ddej        dej        dz  dej        dz  d	e	dz  d
e
dz  dee         dej        fd„Zˆ xZS )ÚJambaAttentionDecoderLayerr†   r‡   c                 óf  •— t          ¦   «                              ¦   «          |j        r|j        |         nd}t          ||¦  «        | _        |dk    rt
          nt          } ||¦  «        | _        t          |j	        |j
        ¬¦  «        | _        t          |j	        |j
        ¬¦  «        | _        d S )Nr"   r©   )r+   r,   Úlayers_num_expertsr…   Ú	self_attnr:  r  Úfeed_forwardr&   r3   rÈ   Úinput_layernormÚpre_ff_layernorm©r2   r†   r‡   r  Úffn_layer_classr4   s        €r5   r,   z#JambaAttentionDecoderLayer.__init__?  s¦   ø€ Ý‰Œ×ÒÑÔÐØ>DÔ>WÐ^�fÔ/°	Ô:Ð:Ð]^ˆÝ'¨°	Ñ:Ô:ˆŒà1<¸q²°Õ-Ð-ÅhˆØ+˜O¨FÑ3Ô3ˆÔÝ+¨FÔ,>ÀFÔDWÐXÑXÔXˆÔÝ ,¨VÔ-?ÀVÔEXÐ YÑ YÔ YˆÔÐÐr6   NFr7   rq   Úposition_idsr”   Ú	use_cachert   r)   c           	      óÌ   — |}|                       |¦  «        } | j        d|||||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N)r7   rq   rR  r”   rS  © )rN  rL  rO  rM  )	r2   r7   rq   rR  r”   rS  rt   Úresidualrì   s	            r5   rD   z"JambaAttentionDecoderLayer.forwardI  sž   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆØ)˜4œ>ð 
Ø'Ø)Ø%Ø+Øð
ð 
ð ð
ð 
Ñˆ�qð ! =Ñ0ˆØ ˆØ×-Ò-¨mÑ<Ô<ˆØ×)Ò)¨-Ñ8Ô8ˆØ  =Ñ0ˆØÐr6   )NNNF)rI   rJ   rK   r#   r    r,   r.   rM   r  r   Úboolr   r   ÚFloatTensorrD   rN   rO   s   @r5   rI  rI  >  sà   ø€ € € € € ðZ˜{ð Z°sð Zð Zð Zð Zð Zð Zð /3Ø04Ø(,Ø!&ðð à”|ðð œ tÑ+ðð Ô&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð Ð+Ô,ðð 
Ô	ðð ð ð ð ð ð ð r6   rI  c                   ó–   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej        dej        dz  dej        dz  de	dz  d	e
e         d
ej        fd„Zˆ xZS )ÚJambaMambaDecoderLayerr†   r‡   c                 óh  •— t          ¦   «                              ¦   «          |j        r|j        |         nd}t          ||¬¦  «        | _        |dk    rt
          nt          } ||¦  «        | _        t          |j	        |j
        ¬¦  «        | _        t          |j	        |j
        ¬¦  «        | _        d S )Nr"   )r†   r‡   r©   )r+   r,   rK  r¢   Úmambar:  r  rM  r&   r3   rÈ   rN  rO  rP  s        €r5   r,   zJambaMambaDecoderLayer.__init__e  s©   ø€ Ý‰Œ×ÒÑÔÐØ>DÔ>WÐ^�fÔ/°	Ô:Ð:Ð]^ˆÝ$¨F¸iÐHÑHÔHˆŒ
Ø1<¸q²°Õ-Ð-ÅhˆØ+˜O¨FÑ3Ô3ˆÔÝ+¨FÔ,>ÀFÔDWÐXÑXÔXˆÔÝ ,¨VÔ-?ÀVÔEXÐ YÑ YÔ YˆÔÐÐr6   Nr7   rq   rR  r”   rt   r)   c                 óÐ   — |}|                       |¦  «        }|                      |||¬¦  «        }||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N)r7   rÖ   rq   )rN  r\  rO  rM  )r2   r7   rq   rR  r”   rt   rV  s          r5   rD   zJambaMambaDecoderLayer.forwardn  s„   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆØŸ
š
Ø'Ø(Ø)ð #ñ 
ô 
ˆð
 ! =Ñ0ˆØ ˆØ×-Ò-¨mÑ<Ô<ˆØ×)Ò)¨-Ñ8Ô8ˆØ  =Ñ0ˆØÐr6   )NNN)rI   rJ   rK   r#   r    r,   r.   rM   r  r   r   r   rX  rD   rN   rO   s   @r5   rZ  rZ  d  sÏ   ø€ € € € € ðZ˜{ð Z°sð Zð Zð Zð Zð Zð Zð /3Ø04Ø(,ðð à”|ðð œ tÑ+ðð Ô&¨Ñ-ð	ð
  ™ðð Ð+Ô,ðð 
Ô	ðð ð ð ð ð ð ð r6   rZ  c                   óª   ‡ — e Zd ZU eed<   dZdZddgZdgZdZ	dZ
dZdZeege eej        d¬¦  «        d	œZ ej        ¦   «         ˆ fd
„¦   «         Zˆ xZS )ÚJambaPreTrainedModelr†   ÚmodelTrI  rZ  r”   r?  )Ú
layer_name)r7   Ú
attentionsrC  c                 óp  •— t          ¦   «                              |¦  «         t          |t          ¦  «        r›t	          j        d|j        dz   ¦  «        d d d …f         }|                     |j        d¦  «         	                    ¦   «         }t          j        |j        t	          j        |¦  «        ¦  «         t          j        |j        ¦  «         d S t          |t           ¦  «        rNt          j        |j        d| j        j        ¬¦  «         t          j        |j        d| j        j        ¬¦  «         d S d S )Nr"   r:   rl   )r@   Ústd)r+   Ú_init_weightsÚ
isinstancer¢   r.   rÃ   r°   re   r´   r~   ÚinitÚcopy_rÅ   rÄ   Úones_rÆ   r  Únormal_r#  r†   Úinitializer_ranger  )r2   rm   rÓ   r4   s      €r5   re  z"JambaPreTrainedModel._init_weights•  s  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�oÑ.Ô.ð 	XÝ”˜Q Ô 5¸Ñ 9Ñ:Ô:¸4ÀÀÀ¸7ÔCˆAØ—’˜Ô1°2Ñ6Ô6×AÒAÑCÔCˆAÝŒJ�v”|¥U¤Y¨q¡\¤\Ñ2Ô2Ð2ÝŒJ�v”xÑ Ô Ð Ð Ð Ý˜¥Ñ-Ô-ð 	XÝŒL˜Ô,°3¸D¼KÔ<YÐZÑZÔZÐZÝŒL˜Ô)°¸¼Ô9VÐWÑWÔWÐWÐWÐWð	Xð 	Xr6   )rI   rJ   rK   r#   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_is_statefulÚ_can_compile_fullgraphrI  rZ  r…   r    r   r�   Ú_can_record_outputsr.   rå   re  rN   rO   s   @r5   r_  r_  …  sÇ   ø€ € € € € € ØÐÐÑØÐØ&*Ð#Ø5Ð7OÐPÐØ#4Ð"5ÐØÐØ€NØ€LØ!Ðà4Ð6LÐMØ$Ø'˜¨¬	¸hÐGÑGÔGðð Ðð €U„]�_„_ð	Xð 	Xð 	Xð 	Xñ „_ð	Xð 	Xð 	Xð 	Xð 	Xr6   r_  )Ú	attentionr\  c                   óâ   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej	        dz  dej
        dz  dej	        dz  dedz  dej        dz  d	edz  d
ee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú
JambaModelr†   c                 ó  •— t          ¦   «                              |¦  «         |j        | _        |j        | _        t          j        |j        |j        | j        ¦  «        | _        g }t          |j
        ¦  «        D ]:}t          |j        |                  }|                      |||¬¦  «        ¦  «         Œ;t          j        |¦  «        | _        t!          |j        |j        ¬¦  «        | _        d| _        |                      ¦   «          d S )N)r‡   r©   F)r+   r,   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr3   Úembed_tokensr  Únum_hidden_layersÚALL_DECODER_LAYER_TYPESÚlayers_block_typer  Ú
ModuleListrß   r&   rÈ   Úfinal_layernormÚgradient_checkpointingÚ	post_init)r2   r†   Údecoder_layersr	  Úlayer_classr4   s        €r5   r,   zJambaModel.__init__§  só   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔØˆÝ�vÔ/Ñ0Ô0ð 	Dð 	DˆAÝ1°&Ô2JÈ1Ô2MÔNˆKØ×!Ò! + +¨fÀÐ"BÑ"BÔ"BÑCÔCÐCÐCÝ”m NÑ3Ô3ˆŒå+¨FÔ,>ÀFÔDWÐXÑXÔXˆÔà&+ˆÔ#à�ŠÑÔÐÐÐr6   NÚ	input_idsrq   rR  r”   Úinputs_embedsrS  rt   r)   c           	      óˆ  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          |x}	t          ¦  «        s&| j        ||||dœ}
t          d
i |
¤Žt          d
i |
¤Ždœ}	|}t          | j        ¦  «        D ])\  }} ||f|	| j        j        |                  |||dœ|¤Ž}Œ*|                      |¦  «        }t%          ||¬	¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embeds)r†   r   r"   )rü   )r†   r‰  rq   r”   rR  )Úfull_attentionÚlinear_attention)rq   rR  r”   rS  )Úlast_hidden_stater”   rU  )Ú
ValueErrorr~  r	   r†   Úget_seq_lengthr.   rÃ   rG   rü   rZ   rf  Údictr   r   Ú	enumeraterß   rÑ   rƒ  r   )r2   rˆ  rq   rR  r”   r‰  rS  rt   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr7   r	  Údecoder_layers                 r5   rD   zJambaModel.forward¹  sÂ  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå°Ð?Ð-ÅÑFÔFð 	ð œ+Ø!.Ø"0Ø#2Ø ,ðð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ$CÐ$RÐ$RÀkÐ$RÐ$Rð#ð #Ðð &ˆÝ )¨$¬+Ñ 6Ô 6ð 	ð 	ÑˆAˆ}Ø)˜MØðà2°4´;Ô3JÈ1Ô3MÔNØ)Ø /Ø#ðð ð ðð ˆMˆMð ×,Ò,¨]Ñ;Ô;ˆå%Ø+Ø+ð
ñ 
ô 
ð 	
r6   )NNNNNN)rI   rJ   rK   r#   r,   r   r!   r   r.   r  rM   r   rX  rW  r   r   r   rD   rN   rO   s   @r5   rx  rx  ¥  s  ø€ € € € € ð˜{ð ð ð ð ð ð ð$  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ð6
ð 6
àÔ# dÑ*ð6
ð œ tÑ+ð6
ð Ô&¨Ñ-ð	6
ð
  ™ð6
ð Ô(¨4Ñ/ð6
ð ˜$‘;ð6
ð Ð+Ô,ð6
ð 
 ð6
ð 6
ð 6
ñ „^ñ „_ñ  Ôð6
ð 6
ð 6
ð 6
ð 6
r6   rx  r9   Úgate_logitsr  c                 óÆ  ‡— | �t          | t          ¦  «        sdS t          | t          ¦  «        r/| d         j        Št          j        ˆfd„| D ¦   «         d¬¦  «        }t          j        j                             |d¬¦  «        }t          j        ||d¬¦  «        \  }}t          j        j         	                    ||¦  «        }|€@t          j
        |                     ¦   «         d¬¦  «        }	t          j
        |d¬¦  «        }
�n.|j        \  }}|j        d         ||z  z  }|ddd…dd…ddf                              |||||f¦  «                             d||¦  «                             ‰¦  «        }t          j        |                     ¦   «         |z  d¬¦  «        t          j        |d¬¦  «        z  }	|ddd…dd…df                              ||||f¦  «                             d|¦  «                             ‰¦  «        }t          j        ||z  d¬¦  «        t          j        |d¬¦  «        z  }
t          j        |	|
                     d¦  «        z  ¦  «        }||z  S )aÄ  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   c                 ó:   •— g | ]}|                      ‰¦  «        ‘ŒS rU  )r=   )Ú.0Ú
layer_gateÚcompute_devices     €r5   ú
<listcomp>z,load_balancing_loss_func.<locals>.<listcomp>  s&   ø€ Ð-jÐ-jÐ-jÐPZ¨j¯mªm¸NÑ.KÔ.KÐ-jÐ-jÐ-jr6   rQ   r:   )rf  rF   rü   r.   rS   r   r|   r}   rB  r)  r@   rL   rG   re   rf   r=   rÿ   rZ   )r–  r  r>  rq   Úconcatenated_gate_logitsrD  rì   Úselected_expertsr1  Útokens_per_expertÚrouter_prob_per_expertrê   rG  r  Úexpert_attention_maskÚ router_per_expert_attention_maskÚoverall_lossr›  s                    @r5   Úload_balancing_loss_funcr¤  õ  s�  ø€ ð: Ð¥*¨[½%Ñ"@Ô"@ÐØˆqå�+�uÑ%Ô%ð sØ$ QœÔ.ˆÝ#(¤9Ð-jÐ-jÐ-jÐ-jÐ^iÐ-jÑ-jÔ-jÐpqÐ#rÑ#rÔ#rÐ å”hÔ)×1Ò1Ð2JÐPRÐ1ÑSÔS€Oåœ* _°eÀÐDÑDÔDÑ€AÐå”(Ô%×-Ò-Ð.>ÀÑLÔL€KàÐå!œJ {×'8Ò'8Ñ':Ô':ÀÐBÑBÔBÐõ "'¤¨OÀÐ!CÑ!CÔ!CÐÑà&4Ô&:Ñ#ˆ
�OØ4Ô:¸1Ô=À*ÈÑB^Ñ_Ðð ˜4    A A A t¨TÐ1Ô2ßŠVÐ&¨
°OÀUÈKÐXÑYÔYßŠW�R˜ Ñ,Ô,ßŠR�ÑÔð	 	õ "œI k×&7Ò&7Ñ&9Ô&9Ð<QÑ&QÐWXÐYÑYÔYÕ\aÔ\eØ! qð]
ñ ]
ô ]
ñ 
Ðð ˜4    A A A tÐ+Ô,ßŠVÐ&¨
°OÀ[ÐQÑRÔRßŠW�R˜Ñ%Ô%ßŠR�ÑÔð	 	)õ "'¤¨?Ð=]Ñ+]ÐcdÐ!eÑ!eÔ!eÕhmÔhqØ,°!ði
ñ i
ô i
ñ "
Ðõ ”9Ð.Ð1G×1QÒ1QÐRSÑ1TÔ1TÑTÑUÔU€LØ˜+Ñ%Ð%r6   c                   ó*  ‡ — e Zd ZddiZddiZddgdgfiZdefˆ fd„Zee		 	 	 	 	 	 	 	 	 dde
j        d	z  de
j        d	z  de
j        d	z  ded	z  de
j        d	z  de
j        d	z  ded	z  ded	z  dee
j        z  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚJambaForCausalLMzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr7   Úlogitsr†   c                 óF  •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _        |j	        | _	        |j
        | _
        |j        | _        |                      ¦   «          d S r  )r+   r,   rx  r`  r|  r   r�   r3   r§  Úrouter_aux_loss_coefr  r=  r…  r  s     €r5   r,   zJambaForCausalLM.__init__M  s‹   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý Ñ'Ô'ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒØ$*Ô$?ˆÔ!Ø!Ô-ˆÔØ#)Ô#=ˆÔ ð 	�ŠÑÔÐÐÐr6   Nr   rˆ  rq   rR  r”   r‰  ÚlabelsrS  Úoutput_router_logitsÚlogits_to_keeprt   r)   c
                 ó  — |�|n| j         j        } | j        d|||||||dœ|
¤Ž}|j        }t	          |	t
          ¦  «        rt          |	 d¦  «        n|	}|                      |dd…|dd…f         ¦  «        }d}|� | j        ||| j	        fi |
¤Ž}d}|rHt          |j        | j        | j        |¦  «        }|�%|| j        |                     |j        ¦  «        z  z  }t#          ||||j        |j        |j        |j        ¬¦  «        S )aj  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, JambaForCausalLM

        >>> model = JambaForCausalLM.from_pretrained("ai21labs/Jamba-v0.1")
        >>> tokenizer = AutoTokenizer.from_pretrained("ai21labs/Jamba-v0.1")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)rˆ  rq   rR  r”   r‰  rS  r­  )ÚlossÚaux_lossr©  r”   r7   rb  rC  rU  )r†   r­  r`  r�  rf  r    Úslicer§  Úloss_functionr|  r¤  rC  r  r=  r«  r=   rü   r   r”   r7   rb  )r2   rˆ  rq   rR  r”   r‰  r¬  rS  r­  r®  rt   Úoutputsr7   Úslice_indicesr©  r°  r±  s                    r5   rD   zJambaForCausalLM.forwardY  sn  € ðN %9Ð$DÐ Ð È$Ì+ÔJjð 	ð
 +5¨$¬*ð 	+
ØØ)Ø%Ø+Ø'ØØ!5ð	+
ð 	+
ð ð	+
ð 	+
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDàˆØð 	MÝ/ØÔ%ØÔ ØÔ(Øñ	ô ˆHð Ð!Ø˜Ô1°H·K²KÀÄÑ4LÔ4LÑLÑL�å(ØØØØ#Ô3Ø!Ô/ØÔ)Ø!Ô/ð
ñ 
ô 
ð 	
r6   )	NNNNNNNNr   )rI   rJ   rK   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr#   r,   r   r   r.   r  rM   r   rX  rW  r    r   r   r   rD   rN   rO   s   @r5   r¦  r¦  G  s|  ø€ € € € € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€Hð
˜{ð 
ð 
ð 
ð 
ð 
ð 
ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø,0Ø-.ðP
ð P
àÔ# dÑ*ðP
ð œ tÑ+ðP
ð Ô&¨Ñ-ð	P
ð
  ™ðP
ð Ô(¨4Ñ/ðP
ð Ô  4Ñ'ðP
ð ˜$‘;ðP
ð # T™kðP
ð ˜eœlÑ*ðP
ð Ð+Ô,ðP
ð 
#ðP
ð P
ð P
ñ „^ñ ÔðP
ð P
ð P
ð P
ð P
r6   r¦  c                   ó   — e Zd ZdS )ÚJambaForSequenceClassificationN)rI   rJ   rK   rU  r6   r5   rº  rº  ®  s   € € € € € Ø€Dr6   rº  )r¦  rº  rx  r_  )r"   )rl   )Nr9   N)MÚcollections.abcr   r.   r   Ú r   rg  Úactivationsr   Úcache_utilsr   r	   Ú
generationr
   Úintegrationsr   r   r   r   r   Úintegrations.accelerater   Úmasking_utilsr   r   Úmodeling_layersr   r   Úmodeling_outputsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úutils.import_utilsr   Úutils.output_capturingr    r!   Úconfiguration_jambar#   Ú
get_loggerrI   rÏ   ÚModuler&   rW   rb   rM   r    rk   rL   rƒ   r…   r¢   r  r  r:  rI  rZ  r_  r€  rx  rF   r¤  r¦  rº  Ú__all__rU  r6   r5   ú<module>rÏ     sú  ðð2 %Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )ðð ð ð ð ð ð ð ð ð ð ð ð ð ð >Ð =Ð =Ð =Ð =Ð =Ø PÐ PÐ PÐ PÐ PÐ PÐ PÐ PØ [Ð [Ð [Ð [Ð [Ð [Ð [Ð [Ø QÐ QÐ QÐ QÐ QÐ QÐ QÐ QØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ ,Ð ,Ð ,Ð ,Ð ,Ð ,ð 
ˆÔ	˜HÑ	%Ô	%€ð Ð˜YÑ'Ô'ðJð Jð Jð Jð J�2”9ñ Jô Jñ (Ô'ðJð((ð (ð (ð ÐÐ*Ñ+Ô+ðð ð ñ ,Ô+ðð2	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð2 ÐÐ)Ñ*Ô*ð3)ð 3)ð 3)ð 3)ð 3)�R”Yñ 3)ô 3)ñ +Ô*ð3)ðlSNð SNð SNð SNð SN�b”iñ SNô SNð SNðlð ð ð ð ˆrŒyñ ô ð ð  ð$#ð $#ð $#ð $#ð $#�2”9ñ $#ô $#ñ Ôð$#ðN"ð "ð "ð "ð "˜"œ)ñ "ô "ð "ðJ#ð #ð #ð #ð #Ð!;ñ #ô #ð #ðLð ð ð ð Ð7ñ ô ð ðBXð Xð Xð Xð X˜?ñ Xô Xð Xð: )CÐMcÐdÐdÐ ð ðL
ð L
ð L
ð L
ð L
Ð%ñ L
ô L
ñ „ðL
ðb #Ø
Ø*.ð	O&ð O&Ø”  e¤lÔ 3Ñ3°dÑ:ðO&à�t‘ðO&ð ”L 4Ñ'ð	O&ð
 „\�CÑðO&ð O&ð O&ð O&ðd ðc
ð c
ð c
ð c
ð c
Ð+¨_ñ c
ô c
ñ „ðc
ðL	ð 	ð 	ð 	ð 	Ð%EÐG[ñ 	ô 	ð 	ð gÐ
fÐ
f€€€r6   