§
    ‚ŠtjÍ  ã                   ó   — d dl Z d dlmZ d dlmZ d dlZd dlmZ ddlmZ	 ddl
mZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZ ddlmZmZ ddlmZ ddlmZmZm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z*m+Z+ ddl,m-Z-m.Z.m/Z/ ddl0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9 ddl:m;Z; dZ< e"j=        e>¦  «        Z? G d„ dej        j@        ¦  «        ZA G d„ de8¦  «        ZB G d„ de*¦  «        ZC G d „ d!e1¦  «        ZD G d"„ d#ej@        ¦  «        ZE G d$„ d%ej@        ¦  «        ZF G d&„ d'e2¦  «        ZG G d(„ d)e6¦  «        ZH G d*„ d+e5¦  «        ZIe G d,„ d-e¦  «        ¦   «         ZJ G d.„ d/e7eJ¦  «        ZK G d0„ d1e3¦  «        ZL G d2„ d3e4¦  «        ZMg d4¢ZNdS )5é    N)ÚCallable)Úcycle)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)Úforce_accelerate_hooks)Úlazy_load_kernel)Úcreate_causal_mask)ÚBaseModelOutputWithPastÚ SequenceClassifierOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_torchdynamo_compilingÚlogging)Úmerge_with_config_defaults)Úresolve_internal_import)Úcapture_outputsé   )ÚLlamaRotaryEmbeddingÚapply_rotary_pos_emb)Úpad_tensor_by_sizeÚreshape_into_chunksÚsegment_sum)	ÚZambaAttentionÚZambaAttentionDecoderLayerÚZambaForCausalLMÚZambaForSequenceClassificationÚZambaHybridLayerÚZambaMambaDecoderLayerÚ
ZambaModelÚZambaRMSNormÚeager_attention_forwardé   )ÚZamba2ConfigzZyphra/Zamba2-2.7Bc                   ó(   ‡ — e Zd Zdˆ fd„	Zdd„Zˆ xZS )ÚZamba2RMSNormGatedç�íµ ÷Æ°>c                 óº   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        || _        d S ©N)	ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilonÚ
group_size)ÚselfÚhidden_sizer8   ÚepsÚ	__class__s       €úg/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/zamba2/modular_zamba2.pyr2   zZamba2RMSNormGated.__init__9   sG   ø€ Ý‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔØ$ˆŒˆˆó    Nc                 ó  — |j         }|                     t          j        ¦  «        }|�?|t          j                             |                     t          j        ¦  «        ¦  «        z  }|j        �^ }}|| j        z  } |j	        g |¢|‘| j        ‘R Ž }| 
                    d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  } |j	        g |¢|| j        z  ‘R Ž }| j        |                     |¦  «        z  S )Nr   éÿÿÿÿT)Úkeepdim)ÚdtypeÚtor4   Úfloat32r   Ú
functionalÚsiluÚshaper8   ÚviewÚpowÚmeanÚrsqrtr7   r6   )	r9   Úhidden_statesÚgateÚinput_dtypeÚprefix_dimsÚlast_dimÚgroup_countÚhidden_states_groupÚvariances	            r=   ÚforwardzZamba2RMSNormGated.forward?   s  € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØÐØ)­B¬M×,>Ò,>¸t¿wºwÅuÄ}Ñ?UÔ?UÑ,VÔ,VÑVˆMØ!.Ô!4Ñˆ�hØ $¤/Ñ1ˆØ0˜mÔ0Ð\°+Ð\¸{Ð\ÈDÌOÐ\Ð\Ð\ÐØ&×*Ò*¨1Ñ-Ô-×2Ò2°2¸tÐ2ÑDÔDˆØ1µE´KÀÈ4ÔK`Ñ@`Ñ4aÔ4aÑaÐØ0Ð+Ô0Ð]°+Ð]¸{ÈTÌ_Ñ?\Ð]Ð]Ð]ˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r>   )r.   r0   )Ú__name__Ú
__module__Ú__qualname__r2   rT   Ú__classcell__©r<   s   @r=   r-   r-   8   sQ   ø€ € € € € ð%ð %ð %ð %ð %ð %ð;ð ;ð ;ð ;ð ;ð ;ð ;ð ;r>   r-   c                   ó   — e Zd ZdS )ÚZamba2RMSNormN©rU   rV   rW   © r>   r=   r[   r[   M   ó   € € € € € Ø€Dr>   r[   c                   ó   — e Zd ZdS )ÚZamba2RotaryEmbeddingNr\   r]   r>   r=   r`   r`   Q   r^   r>   r`   c                   ó  ‡ — e Zd ZdZ	 	 	 ddededz  dedz  dedz  fˆ fd„Z	 	 	 ddej        ded	ej        dz  d
e	dz  de
ej        ej        f         dz  dee         de
ej        ej        dz  e
ej                 dz  f         fd„Zˆ xZS )ÚZamba2AttentionaZ  
    Multi-headed attention from 'Attention Is All You Need' paper.

    Adapted from transformers.models.mistral.modeling_mistral.MistralAttention:
    The input dimension here is attention_hidden_size = 2 * hidden_size, and head_dim = attention_hidden_size // num_heads.
    The extra factor of 2 comes from the input being the concatenation of original_hidden_states with the output of the previous (mamba) layer
    (see fig. 2 in https://huggingface.co/papers/2405.16712).
    Additionally, replaced
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) with
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim/2)
    Finally, this attention layer contributes to tied transformer blocks aimed to increasing compute without increasing model size. Because this
    layer is tied, un-tied adapters (formally the same as LoRA but used in the base model) modules are added to the q, k, v projectors to increase
    expressivity with a small memory overhead (see Fig. 2 of https://huggingface.co/papers/2411.15242).
    NÚconfigÚ	layer_idxÚnum_fwd_mem_blocksÚblock_idc           	      óú  •— t          ¦   «                              ||¦  «         || _        |j        | _        || _        |j        �rt          j        g ¦  «        | _	        t          j        g ¦  «        | _
        t          j        g ¦  «        | _        t          | j        ¦  «        D �]±}||j        z  |k    �rt          j        t          j        | j        | j        j        d¬¦  «        t          j        | j        j        | j        d¬¦  «        ¦  «        }t          j        t          j        | j        | j        j        d¬¦  «        t          j        | j        j        | j        d¬¦  «        ¦  «        }t          j        t          j        | j        | j        j        d¬¦  «        t          j        | j        j        | j        d¬¦  «        ¦  «        }n9t          j        ¦   «         }t          j        ¦   «         }t          j        ¦   «         }| j	                             |¦  «         | j
                             |¦  «         | j                             |¦  «         �Œ³d„ t+          | j        ¦  «        D ¦   «         | _        d S )NF©Úbiasc                 ó   — i | ]\  }}||“Œ	S r]   r]   ©Ú.0ÚindexÚvalues      r=   ú
<dictcomp>z,Zamba2Attention.__init__.<locals>.<dictcomp>Œ   s   € Ð[Ð[Ð[©<¨5°%˜% Ð[Ð[Ð[r>   )r1   r2   re   Úhybrid_layer_idsÚlayer_block_maprf   Úuse_shared_attention_adapterr   Ú
ModuleListÚlinear_q_adapter_listÚlinear_k_adapter_listÚlinear_v_adapter_listÚrangeÚnum_mem_blocksÚ
SequentialÚLinearÚattention_hidden_sizerc   Úadapter_rankÚIdentityÚappendÚ	enumerateÚ	layer_dic)
r9   rc   rd   re   rf   ÚiÚlinear_q_adapterÚlinear_k_adapterÚlinear_v_adapterr<   s
            €r=   r2   zZamba2Attention.__init__e   s*  ø€ õ 	‰Œ×Ò˜ Ñ+Ô+Ð+Ø"4ˆÔØ%Ô6ˆÔØ ˆŒàÔ.ñ 	DÝ)+¬°rÑ):Ô):ˆDÔ&Ý)+¬°rÑ):Ô):ˆDÔ&Ý)+¬°rÑ):Ô):ˆDÔ&å˜4Ô2Ñ3Ô3ð Dñ D�Ø�vÔ,Ñ,°Ò8Ñ8Ý')¤}Ýœ	 $Ô"<¸d¼kÔ>VÐ]bÐcÑcÔcÝœ	 $¤+Ô":¸DÔ<VÐ]bÐcÑcÔcñ(ô (Ð$õ (*¤}Ýœ	 $Ô"<¸d¼kÔ>VÐ]bÐcÑcÔcÝœ	 $¤+Ô":¸DÔ<VÐ]bÐcÑcÔcñ(ô (Ð$õ (*¤}Ýœ	 $Ô"<¸d¼kÔ>VÐ]bÐcÑcÔcÝœ	 $¤+Ô":¸DÔ<VÐ]bÐcÑcÔcñ(ô (Ð$Ð$õ
 (*¤{¡}¤}Ð$Ý')¤{¡}¤}Ð$Ý')¤{¡}¤}Ð$ØÔ*×1Ò1Ð2BÑCÔCÐCØÔ*×1Ò1Ð2BÑCÔCÐCØÔ*×1Ò1Ð2BÑCÔCÐCÑCà[Ð[½9ÀTÔEYÑ;ZÔ;ZÐ[Ñ[Ô[ˆŒˆˆr>   rL   Úattention_maskÚpast_key_valuesÚposition_embeddingsÚkwargsÚreturnc                 ó  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «        }	|                      |¦  «        }
|                      |¦  «        }| j        j        rX| j        |         }|	 | j        |         |¦  «        z   }	|
 | j	        |         |¦  «        z   }
| | j
        |         |¦  «        z   }|	                     |¦  «                             dd¦  «        }	|
                     |¦  «                             dd¦  «        }
|                     |¦  «                             dd¦  «        }| j        j        r|\  }}t          |	|
||¦  «        \  }	}
|�|                     |
||¦  «        \  }
}t!          j        | j        j        t&          ¦  «        } || |	|
||f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr@   r*   r   g        )ÚdropoutÚscaling)rG   Úhead_dimÚq_projÚk_projÚv_projrc   rr   r€   rt   ru   rv   rH   Ú	transposeÚuse_mem_roper   Úupdater   Úget_interfaceÚ_attn_implementationr)   ÚtrainingÚattention_dropoutrŒ   ÚreshapeÚ
contiguousÚo_proj)r9   rL   rd   r…   r†   r‡   rˆ   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚadapter_layer_idxÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                     r=   rT   zZamba2Attention.forwardŽ   sM  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1ˆØ—[’[ Ñ/Ô/ˆ
Ø—{’{ =Ñ1Ô1ˆØŒ;Ô3ð 	gØ $¤¨yÔ 9ÐØ'Ð*W¨$Ô*DÐEVÔ*WÐXeÑ*fÔ*fÑfˆLØ#Ð&S dÔ&@ÐARÔ&SÐTaÑ&bÔ&bÑbˆJØ'Ð*W¨$Ô*DÐEVÔ*WÐXeÑ*fÔ*fÑfˆLà#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆØ—_’_ \Ñ2Ô2×<Ò<¸QÀÑBÔBˆ
Ø#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆàŒ;Ô#ð 	`Ø*‰HˆC�Ý';¸LÈ*ÐVYÐ[^Ñ'_Ô'_Ñ$ˆL˜*àÐ&Ø'6×'=Ò'=¸jÈ,ÐXaÑ'bÔ'bÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r>   ©NNN)rU   rV   rW   Ú__doc__r+   Úintr2   r4   ÚTensorr	   Útupler   r   rT   rX   rY   s   @r=   rb   rb   U   sD  ø€ € € € € ðð ð$ !%Ø)-Ø#ð'\ð '\àð'\ð ˜‘:ð'\ð   $™Jð	'\ð
 ˜‘*ð'\ð '\ð '\ð '\ð '\ð '\ðZ /3Ø(,ØHLð1)ð 1)à”|ð1)ð ð1)ð œ tÑ+ð	1)ð
  ™ð1)ð # 5¤<°´Ð#=Ô>ÀÑEð1)ð Ð+Ô,ð1)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)r>   rb   c                   óä   ‡ — e Zd ZdZddededz  fˆ fd„Z	 	 ddej        de	dz  dej        dz  fd	„Z
dde	dz  dej        dz  fd
„Z ed¦  «        	 	 dde	dz  dej        dz  fd„¦   «         Zˆ xZS )ÚZamba2MambaMixeruƒ  
    Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)
    Nrc   rd   c           	      ó$  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        t          |j	        | j        z  ¦  «        | _
        || _        |j        | _        d| _        t          j        ¦   «         | _        |j        | _        |j        | _        |j        | _        | j        j        | _        |j        | _        |j        | _        |j        | _        |j        | _        | j
        d| j        z  | j        z  z   | _        t          j        | j        | j        d|j        | j        |j        dz
  ¬¦  «        | _        | j
        | j        z   | j        z   }t          j        | j        ||j         ¬¦  «        | _!        t          j"        tG          j$        | j        ¦  «        ¦  «        | _%        tG          j&        d| j        dz   ¦  «        }t          j"        tG          j'        |¦  «        ¦  «        | _(        tS          | j
        | j
        | j        z  d¬¦  «        | _*        t          j"        tG          j$        | j        ¦  «        ¦  «        | _+        t          j        | j
        | j        |j         ¬¦  «        | _,        |j-        r¡t]          d	¦  «        }t_          |d
d ¦  «        a0t_          |dd ¦  «        a1t]          d¦  «        }te          |d¬¦  «        a3te          |d¬¦  «        a4te          |d¬¦  «        a5tm          tf          th          tj          tb          t`          f¦  «        a7nd a0d a1d a3d a4d a5da7t_          |dd¦  «        r!tn          stp           9                    d¦  «         |j:        |         | _;        d S )NrF   r   Tr*   )Úin_channelsÚout_channelsri   Úkernel_sizeÚgroupsÚpaddingrh   gñhãˆµøä>)r8   r;   zcausal-conv1dÚcausal_conv1d_updateÚcausal_conv1d_fnz	mamba-ssmz8ops.triton.selective_state_update.selective_state_update)Úchained_pathz1ops.triton.ssd_combined.mamba_chunk_scan_combinedz8ops.triton.ssd_combined.mamba_split_conv1d_scan_combinedFÚuse_mamba_kernelsa  The fast path is not available because one of `(selective_state_update, causal_conv1d_fn, causal_conv1d_update)` is None. Falling back to the naive implementation. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1d)<r1   r2   rc   r:   Úmamba_d_stateÚssm_state_sizeÚmamba_d_convÚconv_kernel_sizer¨   Úmamba_expandÚintermediate_sizerd   Úuse_conv_biasÚ
activationr   ÚSiLUÚactÚuse_mem_eff_pathÚmamba_ngroupsÚn_groupsÚmamba_headdimr�   Ún_mamba_headsÚ	num_headsÚ
chunk_sizeÚtime_step_limitÚtime_step_minÚtime_step_maxÚconv_dimÚConv1dÚconv1drz   Úadd_bias_linearÚin_projr3   r4   r5   Údt_biasÚarangeÚlogÚA_logr-   ÚnormÚDÚout_projr¶   r   Úgetattrr³   r´   r   Úselective_state_updateÚmamba_chunk_scan_combinedÚ mamba_split_conv1d_scan_combinedÚallÚis_fast_path_availableÚloggerÚwarning_onceÚlayer_typesÚ
layer_type)r9   rc   rd   Úprojection_sizeÚAÚcausal_conv1dÚ	mamba_ssmr<   s          €r=   r2   zZamba2MambaMixer.__init__Ê   sq  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ$Ô2ˆÔØ &Ô 3ˆÔÝ!$ VÔ%8¸4Ô;KÑ%KÑ!LÔ!LˆÔØ"ˆŒØ#Ô1ˆÔØ ˆŒÝ”7‘9”9ˆŒØ &Ô 7ˆÔàÔ,ˆŒØÔ,ˆŒØœÔ2ˆŒØ Ô+ˆŒà%Ô5ˆÔØ#Ô1ˆÔØ#Ô1ˆÔàÔ.°°T´]Ñ1BÀTÔEXÑ1XÑXˆŒÝ”iØœØœØØÔ+Ø”=ØÔ'¨!Ñ+ð
ñ 
ô 
ˆŒð Ô0°4´=Ñ@À4Ä>ÑQˆÝ”yØÔØØÔ'ð
ñ 
ô 
ˆŒõ ”|¥E¤J¨t¬~Ñ$>Ô$>Ñ?Ô?ˆŒõ ŒL˜˜DœN¨QÑ.Ñ/Ô/ˆÝ”\¥%¤)¨A¡,¤,Ñ/Ô/ˆŒ
Ý&ØÔ"¨tÔ/EÈÌÑ/VÐ\`ð
ñ 
ô 
ˆŒ	õ ”�eœj¨¬Ñ8Ô8Ñ9Ô9ˆŒåœ	 $Ô"8¸$Ô:JÐQWÔQgÐhÑhÔhˆŒð Ô#ð 	+Ý,¨_Ñ=Ô=ˆMÝ#*¨=Ð:PÐRVÑ#WÔ#WÐ Ý& }Ð6HÈ$ÑOÔOÐå(¨Ñ5Ô5ˆIÝ%<ØÐ(bð&ñ &ô &Ð"õ )@ØÐ([ð)ñ )ô )Ð%õ 0GØÐ(bð0ñ 0ô 0Ð,õ &)å*Ý-Ý4Ý$Ý(ðñ&ô &Ð"Ð"ð $(Ð Ø#ÐØ%)Ð"Ø(,Ð%Ø/3Ð,Ø%*Ð"å�6Ð.°Ñ5Ô5ð 	Õ>Tð 	Ý×Òð>ñô ð ð !Ô,¨YÔ7ˆŒˆˆr>   rL   Úcache_paramsr…   c                 óp  — |j         \  }}}| j        | j        z  }d| j        z  d| j        z  | j        z  z   | j        z   }|d uo|                     | j        ¦  «        }	|	r:|j        | j                 j        d         }
|j        | j                 j	        d         }|	�rî|dk    �rç|  
                    |                     d¦  «        ¦  «        }|j         d         |z
  dz  }||| j        | j        | j        g}t          j        ||d¬¦  «        \  }}}}}t          ||
| j        j                             d¦  «        | j        j        | j        ¦  «        }t          j        || j        ||gd¬¦  «        \  }}}t          j        | j                             ¦   «         ¦  «         }|d d …d df         d d …d d …d f                              d| j        | j        ¦  «                             t          j        ¬¦  «        }|d d …d d …d f                              dd| j        ¦  «        }| j        d d …d df                              d| j        ¦  «        }| j        d d …d df                              d| j        ¦  «        }|                     || j        |j         d         | j        z  ¦  «        }|                     || j        |j         d         | j        z  ¦  «        }|                     || j        | j        ¦  «        }t=          |||||||d |d¬	¦
  «
        }|                     || j        | j        z  ¦  «        }|                      ||¦  «        }|                       |                     | j         j        j!        ¦  «        ¦  «        d d …d df         }�n+|�Dt          j"        |dk    ¦  «        s,|j!        }||d d …d d …d f         z                       |¦  «        }|  
                    |¦  «        }t          j        | j                             ¦   «         ¦  «         }| j#        €i nd
| j#        i}|�t          j"        |dk    ¦  «        }nd}| j$        r›| j%        r”|€’|r�tM          || j        j                             d¦  «        | j        j        | j        |f| j        | j'        d | j        | j        j        | j        j(        | j         j        | j         j        | j        | j        dddœ|¤Ž\  }}�nÓt          j        || j        | j        | j        gd¬¦  «        \  }}}| )                    dd¦  «        }|	rt          j*        |
|gd¬¦  «        }|�PtV          j,         -                    || j.        |j         d         z
  df¦  «        }| /                    || j        ¦  «         t`          �	| j        dvr>|  1                    |                      |¦  «        dd |j         d         …f         ¦  «        }n?ta          || j        j                             d¦  «        | j        j        | j        ¬¦  «        }|	r|d d …d d …| d …f         }| )                    dd¦  «        }t          j        || j        ||gd¬¦  «        \  }}}|�Dt          j"        |dk    ¦  «        s,|j!        }||d d …d d …d f         z                       |¦  «        }te          |                     ||d| j        ¦  «        |||                     ||| j        d¦  «        |                     ||| j        d¦  «        f| j'        | j        d d d| j        d|	r|nd dœ|¤Ž\  } }|�|�| 3                    || j        ¦  «         |                      ||d¦  «        } |                      | |¦  «        } |                       |                      | j         j        j!        ¦  «        ¦  «        }|S )Nr   r   r*   r@   ©Údim.©rB   T)ÚzrÐ   Údt_softplusÚdt_limitF)rÕ   rÇ   Úseq_idxr¾   Úrmsnorm_weightÚrmsnorm_epsÚoutproj_weightÚoutproj_biasÚheaddimÚngroupsÚnorm_before_gateÚreturn_final_states)rF   Úswish)Úxr6   ri   r¾   )rÇ   rÕ   rê   rí   rõ   rÐ   rë   Úinitial_states)4rG   rÃ   r¸   r¼   rÆ   Úhas_previous_staterd   ÚlayersÚconv_statesÚrecurrent_statesrÏ   ÚsqueezerË   r4   Úsplitr³   rÍ   r6   ri   r¾   ÚexprÓ   ÚfloatÚexpandr�   rC   rD   rÐ   rÕ   rH   rØ   rÔ   rÖ   rB   rÛ   rÈ   rÁ   r–   rÚ   rÇ   r7   r‘   Úcatr   rE   Úpadrº   Úupdate_conv_stater´   rÀ   rÙ   Úupdate_recurrent_state)!r9   rL   rå   r…   Ú
batch_sizeÚseq_lenÚ_Úgroups_time_state_sizeÚd_to_removeÚuse_precomputed_statesÚ
conv_stateÚrecurrent_stateÚin_projected_statesÚd_mlpÚsplit_projection_dimrM   Úhidden_states_B_CÚdtÚBÚCrâ   rÐ   rÕ   Úhidden_states_reshapedÚoutrB   Úprojected_statesÚdt_limit_kwargsÚinput_not_maskedÚ	ssm_stateÚ	time_stepÚnew_conv_stateÚscan_outputs!                                    r=   Úcuda_kernels_forwardz%Zamba2MambaMixer.cuda_kernels_forward0  s?  € ð "/Ô!4Ñˆ
�G˜QØ!%¤°Ô1DÑ!DÐØ˜$Ô0Ñ0°1°t´}Ñ3DÀtÔGZÑ3ZÑZÐ]aÔ]kÑkˆà!-°TÐ!9Ð!m¸l×>]Ò>]Ð^bÔ^lÑ>mÔ>mÐØ!ð 	VØ%Ô,¨T¬^Ô<ÔHÈÔKˆJØ*Ô1°$´.ÔAÔRÐSTÔUˆOð "ñ R	P g°¢l¡lØ"&§,¢,¨}×/DÒ/DÀQÑ/GÔ/GÑ"HÔ"HÐØ(Ô.¨rÔ2°[Ñ@ÀQÑFˆEØ$)¨5°$Ô2HÈ$Ì-ÐY]ÔYgÐ#hÐ Ý05´Ð<OÐQeÐkmÐ0nÑ0nÔ0nÑ-ˆAˆq�$Ð)¨2å 4Ø!ØØ”Ô"×*Ò*¨1Ñ-Ô-Ø”Ô Ø”ñ!ô !Ðõ #(¤+Ø!ØÔ'Ð)?ÐAWÐXØð#ñ #ô #ÑˆM˜1˜aõ
 ”˜4œ:×+Ò+Ñ-Ô-Ñ.Ô.Ð.ˆAà�!�!�!�T˜3�,”    1 1 1 d 
Ô+×2Ò2°2°t´}ÀdÔFYÑZÔZ×]Ò]ÕdiÔdqÐ]ÑrÔrˆAØ�A�A�A�q�q�q˜$�J”×&Ò& r¨2¨t¬}Ñ=Ô=ˆBØ”l 1 1 1 d¨C <Ô0×7Ò7¸¸D¼MÑJÔJˆGØ”�q�q�q˜$ �|Ô$×+Ò+¨B°´Ñ>Ô>ˆAØ—’�z 4¤=°!´'¸!´*ÀÄÑ2MÑNÔNˆAØ—’�z 4¤=°!´'¸!´*ÀÄÑ2MÑNÔNˆAØ%2×%7Ò%7¸
ÀDÄNÐTXÔTaÑ%bÔ%bÐ"Ý2ØØ&ØØØØØØØØ ðñ ô ˆMð *×.Ò.¨z¸4¼>ÈDÌMÑ;YÑZÔZˆMØ ŸIšI m°TÑ:Ô:ˆMð —-’- × 0Ò 0°´Ô1EÔ1KÑ LÔ LÑMÔMÈaÈaÈaÐQUÐWZÈlÔ[ˆC‰Cð Ð)µ%´)¸NÈaÒ<OÑ2PÔ2PÐ)à%Ô+�Ø!.°ÀÀÀÀ1À1À1ÀdÀ
Ô1KÑ!K× OÒ OÐPUÑ VÔ V�à#Ÿ|š|¨MÑ:Ô:ÐÝ”˜4œ:×+Ò+Ñ-Ô-Ñ.Ô.Ð.ˆAØ$(Ô$8Ð$@˜b˜bÀzÐSWÔSgÐFhˆOØÐ)Ý#(¤9¨^¸qÒ-@Ñ#AÔ#AÐ Ð à#'Ð àÔ$ð UP¨¬ð UP¸<Ð;OÐTdÐ;OÝ!AØ$Ø”KÔ&×.Ò.¨qÑ1Ô1Ø”KÔ$Ø”LØð"ð ”fØ#œØ Ø#œØ#'¤9Ô#3Ø $¤	Ô :Ø#'¤=Ô#7Ø!%¤Ô!3Ø œMØ œMØ%*Ø(,ð#"ð "ð$ &ð%"ð "‘��Y‘Yõ, 6;´[Ø$ØÔ+¨T¬]¸D¼NÐKØð6ñ 6ô 6Ñ2�Ð'¨ð %6×$?Ò$?ÀÀ1Ñ$EÔ$EÐ!Ø)ð [õ ).¬	°:Ð?PÐ2QÐWYÐ(ZÑ(ZÔ(ZÐ%ØÐ+Ý%'¤]×%6Ò%6Ø)¨DÔ,AÐDUÔD[Ð\^ÔD_Ñ,_ÐabÐ+cñ&ô &�Nð !×2Ò2°>À4Ä>ÑRÔRÐRÝ#Ð+¨t¬ÐFWÐ/WÐ/WØ(,¯ª°·²Ð=NÑ1OÔ1OÐPSÐUrÐWhÔWnÐoqÔWrÐUrÐPrÔ1sÑ(tÔ(tÐ%Ð%å(8Ø+Ø#œ{Ô1×9Ò9¸!Ñ<Ô<Ø!œ[Ô-Ø#'¤?ð	)ñ )ô )Ð%ð *ð KØ(9¸!¸!¸!¸Q¸Q¸QÀÀÀ	À	¸/Ô(JÐ%Ø$5×$?Ò$?ÀÀ1Ñ$EÔ$EÐ!Ý&+¤kØ%ØÔ+Ð-CÐE[Ð\Øð'ñ 'ô 'Ñ#�˜q !ð
 "Ð-µe´iÀÐRSÒ@SÑ6TÔ6TÐ-à)Ô/�EØ%2°^ÀAÀAÀAÀqÀqÀqÈ$ÀJÔ5OÑ%O×$SÒ$SÐTYÑ$ZÔ$Z�MÝ)BØ!×&Ò& z°7¸BÀÄÑNÔNØØØ—F’F˜: w°´¸rÑBÔBØ—F’F˜: w°´¸rÑBÔBð*ð  $œØ”fØØ Ø(,Ø œLØ $Ø6LÐ#V ? ?ÐRVð*ð *ð &ð*ð *Ñ&�˜Yð  Ð(¨\Ð-EØ ×7Ò7¸	À4Ä>ÑRÔRÐRØ)×.Ò.¨z¸7ÀBÑGÔG�à"Ÿiši¨°TÑ:Ô:�ð —m’m K§N¢N°4´=Ô3GÔ3MÑ$NÔ$NÑOÔO�Øˆ
r>   c                 ó¦  ‡ ‡3— |j         \  }}}|j        }|�0|                     ‰ j        ¦  «        r‰                      |¦  «        }n<|�%||d d …d d …d f         z                       |¦  «        }‰                      |¦  «        }|j         d         d‰ j        z  z
  d‰ j        z  ‰ j        z  z
  ‰ j	        z
  dz  }	| 
                    |	|	‰ j        ‰ j        ‰ j	        gd¬¦  «        \  }}}
}}|                     dd¦  «        }|d uo|                     ‰ j        ¦  «        }|r|j        ‰ j                 j        d         }|r¬|dk    r¦|                     |‰ j        ¦  «        d‰ j         d …f         }t#          j        |‰ j        j        d d …dd d …f         z  d¬¦  «        }‰ j        r|‰ j        j        z  }‰                      |¦  «                             |¦  «        d d …d df         }�n|rt#          j        ||gd¬¦  «        }|�`t2          j                             |‰ j        |j         d         z
  df¦  «        }|                     |‰ j        ¦  «        d‰ j         d …f         }‰                      ‰                      |¦  «        dd |j         d         …f                              dd¦  «        ¦  «        }|r|d d …| d …d d …f         }|�,|j        }||d d …d d …d f         z                       |¦  «        }t#          j
        |‰ j        ‰ j        ‰ j        z  ‰ j        ‰ j        z  gd¬¦  «        \  }}}t#          j        ‰ j                             ¦   «         ¦  «         }|�r/|dk    �r(|j        dk    r|d d …d df         n|d d …dd d …f         d d …d df         }|                     dd¦  «                              ||j         d         ‰ j!        ¦  «        }‰ j"        d                               ‰ j"        j         d         ‰ j!        ¦  «        }t"          j        j         #                    ||                     |j        ¦  «        z   ¦  «        }t#          j$        |‰ j%        ¦  «        }|d                               ‰ j	        ‰ j!        ‰ j        ¦  «                             t"          j&        ¬	¦  «        }t#          j        |d         |z  ¦  «        }| '                    |‰ j        d¦  «        dd d d …f         }|                      |‰ j        ‰ j	        ‰ j        z  |j         d         ¦  «         (                    ¦   «         }| '                    |d|j         d         ¦  «        }|d         |dd d d …f         z  }| '                    |d‰ j!        ¦  «        }||d         z  }|j        ‰ j                 j)        d          *                    ¦   «         }||z  |z   }| +                    |‰ j        ¦  «        }| '                    |‰ j        d¦  «        dd d d …f         }|                      |‰ j        ‰ j	        ‰ j        z  |j         d         ¦  «         (                    ¦   «         }| '                    |d|j         d         ¦  «        }|                     |j        ¦  «        }| ,                    |‰ j	        z  ‰ j!        ‰ j        ¦  «        }| ,                    |‰ j	        z  ‰ j        d¦  «        }t#          j-        ||¦  «        }| ,                    |‰ j	        ‰ j!        ¦  «        }‰ j.        d                               ‰ j.        j         d         ‰ j!        ¦  «        }|||z  z                        |j        ¦  «        }| '                    |d¦  «        d d …d df         }�n~t2          j         #                    |‰ j"        z   ¦  «        }t#          j$        |‰ j%        ¦  «        }| '                    ||d‰ j!        ¦  «                             ¦   «         }| '                    ||d‰ j        ¦  «                             ¦   «         }| '                    ||d‰ j        ¦  «                             ¦   «         }| /                    ‰ j	        ‰ j        z  d‰ j	        ¬
¦  «        }| /                    ‰ j	        ‰ j        z  d‰ j	        ¬
¦  «        }‰ j0        |‰ j0        z  z
  ‰ j0        z  Š3‰ j.        d         tc          |‰3¦  «        z  }||d         z  }|                     |j        ¦  «        |z  }ˆ3ˆ fd„||||fD ¦   «         \  }}}}| 2                    dddd¦  «        }t#          j3        |d¬¦  «        }t#          j        ti          |¦  «        ¦  «        }|d d …d d …d d …d d d …d d …f         |d d …d d …d d d …d d …d d …f         z  }|                     d¬¦  «        } | d         | 2                    ddddd¦  «        d         z  }!|!                     d¬¦  «        }"|"d         |d d …d d …d f         z                       d¦  «        }#t#          j        |d d …d d …d d …dd …f         |z
  ¦  «        }$||$ 2                    dddd¦  «        d         z  }%|% 2                    ddddd¦  «        d         | 2                    ddddd¦  «        dd d d …f         z                       d¬¦  «         2                    ddddd¦  «        }&|rF|j        ‰ j                 j)        d         d d …d f                              |&j        |&j5        ¬¦  «        nt#          j6        |&d d …d d…f         ¦  «        }'t#          j        |'|&gd¬¦  «        }&t#          j        ti          t2          j                             |d d …d d …d d …df         d¦  «        ¦  «        ¦  «        }(|& 2                    ddddd¦  «        })|(d         |)d d …d d …d df         z                       d¬¦  «        }*|* 2                    ddddd¦  «        }+|+d d …d d…f         |+d d …df         },}&t#          j        |¦  «        }-|dd d d …f         |&d d …d d …d df         z  }.|- 2                    dddd¦  «        }/|.                     d¦  «        |/d         z  }0|#|0z   }| '                    |d‰ j	        ‰ j!        ¦  «        }||z   }‰3dk    r|d d …d |…d d …d d …f         }| '                    ||d¦  «        }|,�|�| +                    |,‰ j        ¦  «         ‰  7                    ||
¦  «        }1‰  8                    |1                     |¦  «        ¦  «        }2|2S )Nr@   r   rç   r*   r   .).N).NNré   )rè   Úoutput_sizec                 ó<   •— g | ]}t          |‰‰j        ¦  «        ‘ŒS r]   )r   rÇ   )rl   ÚtÚpad_sizer9   s     €€r=   ú
<listcomp>z2Zamba2MambaMixer.torch_forward.<locals>.<listcomp>Q  s)   ø€ Ð%zÐ%zÐ%zÐ\]Õ&9¸!¸XÀtÄÑ&WÔ&WÐ%zÐ%zÐ%zr>   r   é   )rB   Údevice)r*   r   )9rG   rB   rù   rd   rÏ   rC   r¼   rÃ   r¸   rÆ   rþ   rË   r‘   rú   rû   r  rº   r4   ÚsumrÍ   r6   r½   ri   rÀ   r  r   rE   r  rÿ   rÓ   r   Úndimr  r�   rÐ   ÚsoftplusÚclamprÉ   rD   r˜   r™   rü   Úcloner  rH   ÚbmmrÕ   Úrepeat_interleaverÇ   r   ÚpermuteÚcumsumr    r&  Ú
zeros_likerÔ   rÖ   )4r9   Úinput_statesrå   r…   r  r  r  rB   r  r  rM   rL   r  Úuse_precomputed_stater  rû   r  r  râ   rÐ   ÚdAÚdBÚdBxÚ
ssm_statesÚssm_states_reshapedÚ
C_reshapedÚyrÕ   Ú
D_residualÚA_cumsumÚLÚG_intermediateÚGÚM_intermediateÚMÚY_diagÚdecay_statesÚB_decay_contractionÚstatesÚprevious_statesÚdecay_chunkÚstates_permutedÚresultÚ
new_statesr  Ústate_decay_outÚC_times_statesÚstate_decay_out_permutedÚY_offr  Úcontextualized_statesr#  s4   `                                                  @r=   Útorch_forwardzZamba2MambaMixer.torch_forwardØ  s  øø€ Ø!-Ô!3Ñˆ
�G˜QØÔ"ˆàÐ#¨×(GÒ(GÈÌÑ(WÔ(WÐ#Ø#Ÿ|š|¨LÑ9Ô9ÐÐàÐ)à ,¨~¸a¸a¸aÀÀÀÀD¸jÔ/IÑ I×MÒMÈeÑTÔT�Ø#Ÿ|š|¨LÑ9Ô9ÐØ!Ô'¨Ô+¨a°$Ô2HÑ.HÑHÈ1ÈtÌ}ÑK\Ð_cÔ_rÑKrÑrÐtxô  uCñ  Cð  HIñ  IˆØ(8×(>Ò(>Ø˜˜tÔ5¸¼ÀtÄ~ÐVÐ\^ð )?ñ )
ô )
Ñ%ˆˆ1ˆd�M 2ð &×/Ò/°°1Ñ5Ô5ˆà ,°DÐ 8Ð l¸\×=\Ò=\Ð]aÔ]kÑ=lÔ=lÐØ ð 	LØ%Ô,¨T¬^Ô<ÔHÈÔKˆJð !ð 	W W°¢\ \Ø&×8Ò8¸ÈÌÑWÔWÐX[Ð^bÔ^sÐ]sÐ]tÐ]tÐXtÔuˆKÝ!œI k°D´KÔ4FÀqÀqÀqÈ!ÈQÈQÈQÀwÔ4OÑ&OÐUWÐXÑXÔXˆMØÔ!ð 2Ø ¤Ô!1Ñ1�Ø ŸHšH ]Ñ3Ô3×6Ò6°uÑ=Ô=¸a¸a¸aÀÀs¸lÔKˆM‰Mà$ð Oå %¤	¨:°}Ð*EÈ2Ð NÑ NÔ N�ØÐ'Ý œm×/Ò/Ø!ØÔ*¨]Ô-@ÀÔ-DÑDÀaÐHñô �ð +×<Ò<¸[È$Ì.ÑYÔYÐZ]Ð`dÔ`uÐ_uÐ_vÐ_vÐZvÔw�à ŸHšH T§[¢[°Ñ%?Ô%?ÀÐE]ÀmÔFYÐZ\ÔF]ÐE]Ð@]Ô%^×%hÒ%hÐijÐlmÑ%nÔ%nÑoÔoˆMØ$ð ?Ø -¨a¨a¨a°'°°°¸A¸A¸A¨oÔ >�ØÐ)Ø%Ô+�à!.°ÀÀÀÀ1À1À1ÀdÀ
Ô1KÑ!K× OÒ OÐPUÑ VÔ V�å#œk¨-¸$Ô:PÐRVÔR_ÐbfÔbuÑRuÐw{ô  xEð  HLô  H[ñ  x[ð  :\ð  bdð  eñ  eô  eÑˆ�q˜!ÝŒY�t”z×'Ò'Ñ)Ô)Ñ*Ô*Ð*ˆØ ñ E	O W°¢\¡\ð &(¤W°¢\ \��A�A�A�t˜S�LÔ!Ð!°r¸!¸!¸!¸QÀÀÀ¸'´{À1À1À1ÀdÈCÀ<Ô7PˆBØ—’˜a Ñ#Ô#×*Ò*¨:°r´xÀ´|ÀTÄ]ÑSÔSˆBà”l 9Ô-×4Ò4°T´\Ô5GÈÔ5JÈDÌMÑZÔZˆGå”Ô$×-Ò-¨b°7·:²:¸b¼hÑ3GÔ3GÑ.GÑHÔHˆBÝ”˜R Ô!3Ñ4Ô4ˆBØ�/Ô"×)Ò)¨$¬.¸$¼-ÈÔI\Ñ]Ô]×`Ò`ÕglÔgtÐ`ÑuÔuˆAå”˜2˜iœ=¨1Ñ,Ñ-Ô-ˆBð
 —	’	˜* d¤m°RÑ8Ô8¸¸dÀAÀAÀA¸ÔFˆAØ—’˜ T¤]°D´NÀdÄmÑ4SÐUVÔU\Ð]_ÔU`ÑaÔa×lÒlÑnÔnˆAØ—	’	˜* b¨!¬'°"¬+Ñ6Ô6ˆAà�I”  3¨¨a¨a¨a <¤Ñ0ˆBð *×1Ò1°*¸bÀ$Ä-ÑPÔPˆMØ�} YÔ/Ñ/ˆCð &Ô,¨T¬^Ô<ÔMÈaÔP×VÒVÑXÔXˆJØ# b™¨3Ñ.ˆJØ%×<Ò<¸ZÈÌÑXÔXˆJð —	’	˜* d¤m°RÑ8Ô8¸¸dÀAÀAÀA¸ÔFˆAØ—’˜ T¤]°D´NÀdÄmÑ4SÐUVÔU\Ð]_ÔU`ÑaÔa×lÒlÑnÔnˆAØ—	’	˜* b¨!¬'°"¬+Ñ6Ô6ˆAð $Ÿš q¤wÑ/Ô/ˆJà",§/¢/°*¸t¼~Ñ2MÈtÌ}Ð^bÔ^qÑ"rÔ"rÐØŸš 
¨T¬^Ñ ;¸TÔ=PÐRSÑTÔTˆJÝ”	Ð-¨zÑ:Ô:ˆAØ—’�z 4¤>°4´=ÑAÔAˆAð ”�yÔ!×(Ò(¨¬¬°a¬¸$¼-ÑHÔHˆAØ�] QÑ&Ñ&×*Ò*¨1¬7Ñ3Ô3ˆAð —	’	˜* bÑ)Ô)¨!¨!¨!¨T°3¨,Ô7ˆA‰Aõ ”×'Ò'¨¨T¬\Ñ(9Ñ:Ô:ˆBÝ”˜R Ô!3Ñ4Ô4ˆBØ)×1Ò1°*¸gÀrÈ4Ì=ÑYÔY×_Ò_ÑaÔaˆMØ—	’	˜* g°°DÔ4GÑHÔH×NÒNÑPÔPˆAØ—	’	˜* g¨r°4Ô3FÑGÔG×MÒMÑOÔOˆAØ×#Ò# D¤N°d´mÑ$CÈÐX\ÔXfÐ#ÑgÔgˆAØ×#Ò# D¤N°d´mÑ$CÈÐX\ÔXfÐ#ÑgÔgˆAØœ¨'°D´OÑ*CÑCÀtÄÑVˆHàœ 	Ô*Õ-?ÀÈxÑ-XÔ-XÑXˆJð *¨B¨y¬MÑ9ˆMØ—’�]Ô(Ñ)Ô)¨BÑ.ˆAð &{Ð%zÐ%zÐ%zÐ%zÐboÐqrÐtuÐwxÐayÐ%zÑ%zÔ%zÑ"ˆM˜1˜a ð —	’	˜!˜Q  1Ñ%Ô%ˆAÝ”| A¨2Ð.Ñ.Ô.ˆHõ ”	�+ a™.œ.Ñ)Ô)ˆAð ˜q˜q˜q ! ! ! Q Q Q¨¨a¨a¨a°°°Ð2Ô3°a¸¸¸¸1¸1¸1¸dÀAÀAÀAÀqÀqÀqÈ!È!È!Ð8KÔ6LÑLˆNØ×"Ò" rÐ"Ñ*Ô*ˆAð ˜yœ\¨A¯IªI°a¸¸A¸qÀ!Ñ,DÔ,DÀYÔ,OÑOˆNØ×"Ò" rÐ"Ñ*Ô*ˆAð ˜	”l ]°1°1°1°a°a°a¸°:Ô%>Ñ>×CÒCÀAÑFÔFˆFõ !œ9 X¨a¨a¨a°°°°A°A°A°r°s°s¨lÔ%;¸hÑ%FÑGÔGˆLØ"# l×&:Ò&:¸1¸aÀÀAÑ&FÔ&FÀyÔ&QÑ"QÐà)×1Ò1°!°Q¸¸1¸aÑ@Ô@ÀÔKÈ}×OdÒOdÐefÐhiÐklÐnoÐqrÑOsÔOsÐtwÐy}ð  @Að  @Að  @Að  uAô  PBñ  B÷  Gò  Gð  LMð  Gñ  Nô  N÷  Vò  Vð  WXð  Z[ð  ]^ð  `að  cdñ  eô  eˆFð )ð5�Ô# D¤NÔ3ÔDÀQÔGÈÈÈÈ4ÈÔP×SÒSÐZ`ÔZfÐouÔo|ÐSÑ}Ô}Ð}åÔ% f¨Q¨Q¨Q°°°¨U¤mÑ4Ô4ð õ
 ”Y °Ð8¸aÐ@Ñ@Ô@ˆFÝœ)¥Kµ´×0AÒ0AÀ(È1È1È1ÈaÈaÈaÐQRÐQRÐQRÐTVÈ;ÔBWÐY_Ñ0`Ô0`Ñ$aÔ$aÑbÔbˆKà$Ÿnšn¨Q°°1°a¸Ñ;Ô;ˆOØ! /Ô2°_ÀQÀQÀQÈÈÈÈ4ÐQTÀ_Ô5UÑU×ZÒZÐ_`ÐZÑaÔaˆFØŸš¨¨1¨a°°AÑ6Ô6ˆJØ *¨1¨1¨1¨c¨r¨c¨6Ô 2°J¸q¸q¸qÀ"¸uÔ4E�IˆFõ $œi¨Ñ1Ô1ˆOà  T¨1¨1¨1 œo°°q°q°q¸!¸!¸!¸TÀ3°Ô0GÑGˆNØ'6×'>Ò'>¸qÀ!ÀQÈÑ'JÔ'JÐ$Ø#×'Ò'¨Ñ+Ô+Ð.FÀyÔ.QÑQˆEð ˜‘ˆAà—	’	˜* b¨$¬.¸$¼-ÑHÔHˆAà�J‘ˆAà˜!Š|ˆ|Ø�a�a�a˜˜'˜ 1 1 1 a a aÐ'Ô(�Ø—	’	˜* g¨rÑ2Ô2ˆAØÐ$¨Ð)AØ×3Ò3°I¸t¼~ÑNÔNÐNà—i’i  4Ñ(Ô(ˆð
 !%§¢¨k¯nªn¸UÑ.CÔ.CÑ DÔ DÐØ$Ð$r>   rÍ   c                 ó¸   — t           r=d| j        j        j        j        v r%t          ¦   «         s|                      |||¦  «        S |                      |||¦  «        S )NÚcuda)rÜ   rÏ   r6   r&  Útyper   r  rO  )r9   rL   rå   r…   rˆ   s        r=   rT   zZamba2MambaMixer.forward™  s^   € õ "ð 	Z f°´Ô0CÔ0JÔ0OÐ&OÐ&OÕXpÑXrÔXrÐ&OØ×,Ò,¨]¸LÈ.ÑYÔYÐYà×!Ò! -°¸~ÑNÔNÐNr>   r0   ©NN)rU   rV   rW   r§   r+   r¨   r2   r4   r©   r	   r  rO  r   rT   rX   rY   s   @r=   r¬   r¬   Â   s[  ø€ € € € € ðð ðd8ð d8˜|ð d8¸¸d¹
ð d8ð d8ð d8ð d8ð d8ð d8ðR &*Ø.2ð	eð eà”|ðeð ˜d‘lðeð œ tÑ+ð	eð eð eð eðP~%ð ~%¸À¹ð ~%Ð[`Ô[gÐjnÑ[nð ~%ð ~%ð ~%ð ~%ðB Ð˜HÑ%Ô%ð &*Ø.2ð	
Oð 
Oð ˜d‘lð
Oð œ tÑ+ð	
Oð 
Oð 
Oñ &Ô%ð
Oð 
Oð 
Oð 
Oð 
Or>   r¬   c                   ó8   ‡ — e Zd Zddededz  fˆ fd„Zdd„Zˆ xZS )Ú	Zamba2MLPNrc   rf   c           	      ón  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        || _        || _        t          j        | j        d| j        z  |j	        ¬¦  «        | _
        t          j        | j        | j        |j	        ¬¦  «        | _        t          |j                 | _        t          j        g ¦  «        | _        t#          | j        ¦  «        D ]£}||j        z  |k    rft          j        t          j        | j        j        | j        j        d¬¦  «        t          j        | j        j        d| j        z  d¬¦  «        ¦  «        }nt          j        ¦   «         }| j                             |¦  «         Œ¤|j        }d„ t1          |¦  «        D ¦   «         | _        dS )aQ  
        This MLP layer contributes to tied transformer blocks aimed to increasing compute without increasing model size. Because this layer
        is tied, un-tied adapter modules (formally same as LoRA, but used in the base model) are added to the up and gate projectors to increase expressivity with a small memory overhead.
        r   rh   Fc                 ó   — i | ]\  }}||“Œ	S r]   r]   rk   s      r=   ro   z&Zamba2MLP.__init__.<locals>.<dictcomp>Ä  s   € ÐVÐVÐV©<¨5°%˜% ÐVÐVÐVr>   N)r1   r2   rc   r:   r¼   re   rf   r   rz   rÎ   Úgate_up_projÚ	down_projr   Ú
hidden_actÚact_fnrs   Úgate_up_proj_adapter_listrw   rx   ry   r|   r}   r~   rp   r   r€   )r9   rc   re   rf   r�   Úgate_up_proj_adapterrq   r<   s          €r=   r2   zZamba2MLP.__init__¨  s‰  ø€ õ
 	‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔØ"4ˆÔØ ˆŒåœI dÔ&6¸¸DÔ<RÑ8RÐY_ÔYoÐpÑpÔpˆÔÝœ 4Ô#9¸4Ô;KÐRXÔRhÐiÑiÔiˆŒÝ˜VÔ.Ô/ˆŒå)+¬°rÑ):Ô):ˆÔ&Ý�tÔ.Ñ/Ô/ð 	Hð 	HˆAØ�6Ô(Ñ(¨HÒ4Ð4Ý')¤}Ý”I˜dœkÔ5°t´{Ô7OÐV[Ð\Ñ\Ô\Ý”I˜dœkÔ6¸¸DÔ<RÑ8RÐY^Ð_Ñ_Ô_ñ(ô (Ð$Ð$õ
 (*¤{¡}¤}Ð$ØÔ*×1Ò1Ð2FÑGÔGÐGÐGà Ô1ˆØVÐV½9À_Ñ;UÔ;UÐVÑVÔVˆŒˆˆr>   c                 ó  — |                       |¦  «        }| j        |         }| | j        |         |¦  «        z   }t          j        |dd¬¦  «        }|                      |d         ¦  «        |d         z  }|                      |¦  «        }|S )Nr   r@   rç   r   r*   )rX  r€   r\  r4   Úchunkr[  rY  )r9   Úhidden_staterd   Úgate_up_stateÚoutputs        r=   rT   zZamba2MLP.forwardÆ  s‹   € Ø×)Ò)¨,Ñ7Ô7ˆØ”N 9Ô-ˆ	Ø%Ð(Q¨Ô(FÀyÔ(QÐR^Ñ(_Ô(_Ñ_ˆåœ M°1¸"Ð=Ñ=Ô=ˆØ—{’{ =°Ô#3Ñ4Ô4°}ÀQÔ7GÑGˆØ—’ Ñ-Ô-ˆØˆr>   rS  r0   )rU   rV   rW   r+   r¨   r2   rT   rX   rY   s   @r=   rU  rU  §  st   ø€ € € € € ðWð W˜|ð WÐPSÐVZÑPZð Wð Wð Wð Wð Wð Wð<ð ð ð ð ð ð ð r>   rU  c                   óÆ   ‡ — e Zd Zddededz  dedz  fˆ fd„Z	 	 	 ddej        dej        dedej        dz  d	edz  d
ej	        dz  de
e         deej                 fd„Zˆ xZS )ÚZamba2AttentionDecoderLayerNrc   rf   rd   c                 óà   •— || _         t          |j        ¦  «        }t          ¦   «                              ||¦  «         t          |d||¬¦  «        | _        t          |||¬¦  «        | _        d S )Nr@   )rd   re   rf   )re   rf   )	rf   Úlenrp   r1   r2   rb   Ú	self_attnrU  Úfeed_forward)r9   rc   rf   rd   Únum_gsr<   s        €r=   r2   z$Zamba2AttentionDecoderLayer.__init__Ò  sl   ø€ Ø ˆŒÝ�VÔ,Ñ-Ô-ˆÝ‰Œ×Ò˜ Ñ+Ô+Ð+Ý(¨¸2ÐRXÐckÐlÑlÔlˆŒÝ% fÀÐRZÐ[Ñ[Ô[ˆÔÐÐr>   rL   Úoriginal_hidden_statesr…   r†   r‡   rˆ   r‰   c           	      óâ   — t          j        ||gd¬¦  «        }|                      |¦  «        } | j        d|||||dœ|¤Ž\  }}|                      |¦  «        }|                      ||¦  «        }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`): output of previous Mamba layer of shape `(batch, seq_len, embed_dim)`
            original_hidden_states (`torch.FloatTensor`): word embedding output of shape `(batch, seq_len, embed_dim)`.
                This is concatenated with `hidden_states` (which is the output of the previous (mamba) layer). The
                concatenated tensor is then used as input of the pre-attention RMSNorm
                (see fig. 2 in https://huggingface.co/papers/2405.16712).
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
        r@   rç   )rL   rd   r…   r†   r‡   r]   )r4   ÚconcatenateÚinput_layernormrg  Úpre_ff_layernormrh  )	r9   rL   rj  rd   r…   r†   r‡   rˆ   r  s	            r=   rT   z#Zamba2AttentionDecoderLayer.forwardÙ  sž   € õ6 Ô)¨=Ð:PÐ*QÐWYÐZÑZÔZˆØ×,Ò,¨]Ñ;Ô;ˆØ)˜4œ>ð 
Ø'ØØ)Ø+Ø 3ð
ð 
ð ð
ð 
Ñˆ�qð ×-Ò-¨mÑ<Ô<ˆØ×)Ò)¨-¸ÑCÔCˆàÐr>   rS  r¦   )rU   rV   rW   r+   r¨   r2   r4   r©   r	   Ú
LongTensorr   r   rª   ÚFloatTensorrT   rX   rY   s   @r=   rd  rd  Ñ  s  ø€ € € € € ð\ð \˜|ð \°s¸T±zð \ÐUXÐ[_ÑU_ð \ð \ð \ð \ð \ð \ð /3Ø(,Ø7;ð)ð )à”|ð)ð !&¤ð)ð ð	)ð
 œ tÑ+ð)ð  ™ð)ð #Ô-°Ñ4ð)ð Ð+Ô,ð)ð 
ˆuÔ Ô	!ð)ð )ð )ð )ð )ð )ð )ð )r>   rd  c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚZamba2MambaDecoderLayerrc   rd   c                 ó¸   •— t          ¦   «                              ||¦  «         t          ||¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        d S )N)rc   rd   ©r;   )r1   r2   r¬   Úmambar[   r:   Úrms_norm_epsrm  )r9   rc   rd   r<   s      €r=   r2   z Zamba2MambaDecoderLayer.__init__  sR   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+Ý%¨V¸yÐIÑIÔIˆŒ
Ý,¨VÔ-?ÀVÔEXÐYÑYÔYˆÔÐÐr>   )rU   rV   rW   r+   r¨   r2   rX   rY   s   @r=   rr  rr    sW   ø€ € € € € ðZ˜|ð Z¸ð Zð Zð Zð Zð Zð Zð Zð Zð Zð Zr>   rr  c                   ó8  ‡ — e Zd Zdedej        defˆ fd„Z	 	 	 	 	 	 	 	 ddej	        dej	        dz  d	e
dz  d
ej	        dz  dej	        dz  dedz  dedz  dej        dz  dej        dz  dee         deej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚZamba2HybridLayerÚshared_transformerÚlinearru  c                 ó`   •— t          ¦   «                              |||¦  «         | `|| _        d S r0   )r1   r2   Úshared_transfry  )r9   ry  rz  ru  r<   s       €r=   r2   zZamba2HybridLayer.__init__  s6   ø€ õ 	‰Œ×ÒÐ+¨V°UÑ;Ô;Ð;ØÐØ"4ˆÔÐÐr>   NFrL   rj  rd   r…   Úcausal_maskr†   Ú	use_cacher‡   Úposition_idsrˆ   r‰   c
           
      ó‚   —  | j         |f||||||	dœ|
¤Ž}|                      |¦  «        } | j        |f|||||dœ|
¤Ž}|S )ap  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            original_hidden_states (`torch.FloatTensor`): word embedding output that will be concatenated with
            hidden activations to form the input of the shared transformer layer.
            layer_idx (`int`): layer number.
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
        )rj  rd   r…   r†   r‡   r  )Útransformer_hidden_statesr…   r†   r~  r‡   )ry  rz  Úmamba_decoder)r9   rL   rj  rd   r…   r}  r†   r~  r‡   r  rˆ   r�  s               r=   rT   zZamba2HybridLayer.forward  s™   € ð< %< DÔ$;Øð	%
à#9ØØ&Ø+Ø 3Ø%ð	%
ð 	%
ð ð	%
ð 	%
Ð!ð %)§K¢KÐ0IÑ$JÔ$JÐ!à*˜Ô*Øð
à&?Ø)Ø+ØØ 3ð
ð 
ð ð
ð 
ˆð Ðr>   )NNNNNFNN)rU   rV   rW   rd  r   rz   rr  r2   r4   r©   r¨   r	   Úboolro  r   r   rª   rp  rT   rX   rY   s   @r=   rx  rx    sR  ø€ € € € € ð5Ø"=ð5ØGIÄyð5ØYpð5ð 5ð 5ð 5ð 5ð 5ð 7;Ø $Ø.2Ø+/Ø(,Ø!&Ø7;Ø04ð4ð 4à”|ð4ð !&¤¨tÑ 3ð4ð ˜‘:ð	4ð
 œ tÑ+ð4ð ”\ DÑ(ð4ð  ™ð4ð ˜$‘;ð4ð #Ô-°Ñ4ð4ð Ô&¨Ñ-ð4ð Ð+Ô,ð4ð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uð4ð 4ð 4ð 4ð 4ð 4ð 4ð 4r>   rx  c                   ó„   ‡ — e Zd ZU eed<   dZdZddgZdgZdZ	dZ
dZdZeedœZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )	ÚZamba2PreTrainedModelrc   ÚmodelTrx  rr  r†   )rL   Ú
attentionsc                 ó  •— t          ¦   «                              |¦  «         t          |t          ¦  «        �rTt	          j        t	          j        | j        j        ¦  «        t          j
        | j        j        ¦  «        t          j
        | j        j        ¦  «        z
  z  t          j
        | j        j        ¦  «        z   ¦  «                             | j        j        ¬¦  «        }|t	          j
        t	          j        | ¦  «         ¦  «        z   }t!          j        |j        |¦  «         t	          j        d|j        dz   ¦  «        }t!          j        |j        t	          j
        |¦  «        ¦  «         t!          j        |j        ¦  «         d S d S )N)Úminr*   )r1   Ú_init_weightsÚ
isinstancer¬   r4   rÿ   Úrandrc   rÅ   ÚmathrÒ   rÊ   rÉ   r*  Útime_step_floorÚexpm1ÚinitÚcopy_rÐ   rÑ   rÆ   rÓ   Úones_rÕ   )r9   Úmoduler  Úinv_dtrâ   r<   s        €r=   rŠ  z#Zamba2PreTrainedModel._init_weights[  s=  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ.Ñ/Ô/ñ 	!Ý”Ý”
˜4œ;Ô4Ñ5Ô5Ý”8˜DœKÔ5Ñ6Ô6½¼À$Ä+ÔB[Ñ9\Ô9\Ñ\ñ^å”(˜4œ;Ô4Ñ5Ô5ñ6ñô ÷ Še˜œÔ3ˆeÑ4Ô4ð	 ð �%œ)¥U¤[°"°Ñ%5Ô%5Ð$5Ñ6Ô6Ñ6ˆFÝŒJ�v”~ vÑ.Ô.Ð.å”˜Q Ô 0°1Ñ 4Ñ5Ô5ˆAÝŒJ�v”|¥U¤Y¨q¡\¤\Ñ2Ô2Ð2ÝŒJ�v”xÑ Ô Ð Ð Ð ð	!ð 	!r>   )rU   rV   rW   r+   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_sdpaÚ_is_statefulrr  rb   Ú_can_record_outputsr4   Úno_gradrŠ  rX   rY   s   @r=   r…  r…  K  s¤   ø€ € € € € € àÐÐÑØÐØ&*Ð#Ø,Ð.GÐHÐØ#4Ð"5ÐØÐØÐØ€NØ€Là0Ø%ðð Ðð
 €U„]�_„_ð!ð !ð !ð !ñ „_ð!ð !ð !ð !ð !r>   r…  c                   óè   — e Zd ZdZdefd„Zd„ Zeee		 	 	 	 	 	 dde
j        dz  de
j        dz  de
j        dz  d	edz  d
e
j        dz  dedz  dee         deez  fd„¦   «         ¦   «         ¦   «         ZdS )ÚZamba2Modelzh
    Model consisting of *config.num_hidden_layers* layers.

    Args:
        config: Zamba2Config
    rc   c                 ó  — t                                | |¦  «         || _        |j        | _        |j        | _        t          j        |j        |j        | j        ¦  «        | _	        |j
        | _
        |                      ¦   «         | _        |j        | _        t          |j        |j        ¬¦  «        | _        |j        r5|j        rt&                               d¦  «         t+          |¦  «        | _        d| _        |                      ¦   «          d S )Nrt  ze`use_long_context` set to `True`: using rescaled `rope_theta` and extended `max_position_embeddings`.F)r…  r2   rc   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr:   Úembed_tokensÚlayers_block_typeÚ
get_layersrú   r•   r[   rv  Úfinal_layernormr’   Úuse_long_contextrÝ   rÞ   r`   Ú
rotary_embÚgradient_checkpointingÚ	post_init)r9   rc   s     r=   r2   zZamba2Model.__init__u  sø   € Ý×&Ò& t¨VÑ4Ô4Ð4ØˆŒØ!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔØ!'Ô!9ˆÔØ—o’oÑ'Ô'ˆŒà$*Ô$?ˆÔ!Ý,¨VÔ-?ÀVÔEXÐYÑYÔYˆÔØÔð 	<ØÔ&ð Ý×#Ò#Ø{ñô ð õ 4°FÑ;Ô;ˆDŒOØ&+ˆÔ#ð 	�ŠÑÔÐÐÐr>   c                 ó  — g }i | _         d| _        g }t          | j        ¦  «        D �]K\  }}t	          | j        |¬¦  «        }|dk    �rd|› d�}t          |t          ¦  «        rt          |¦  «        | j        j	        k    rPt          |t          ¦  «        rt          |¦  «        }t          |¦  «        }| j                              ||i¦  «         n|                     |¦  «         || j        j	        z  }t          | j        |¬¦  «        }	t          j        | j        j        | j        j        d¬¦  «        }
|                     t%          |	|
|¦  «        ¦  «         �Œ6|                     |¦  «         �ŒMt          j        |¦  «        S )	Nr   )rd   Úhybridzlayers.z.shared_transformer)rf   Frh   )Ú_tied_weights_keysÚfirst_transformer_layer_idr   r¨  rr  rc   r‹  Úlistrf  rx   r   Únextr“   r~   rd  r   rz   r:   rx  rs   )r9   rú   Úunique_hybrid_blocksÚlayer_idrà   Úmamba_layerÚprefix_patternÚtarget_patternrf   Ú
attn_blockÚlinear_layers              r=   r©  zZamba2Model.get_layersŒ  s‹  € ØˆØ"$ˆÔØ*+ˆÔ'Ø!Ðå$-¨dÔ.DÑ$EÔ$Eð 	+ñ 	+Ñ ˆH�jÝ1°$´+ÈÐRÑRÔRˆKØ˜XÒ%Ñ%Ø!H¨8Ð!HÐ!HÐ!H�õ #Ð#7½Ñ>Ô>ð
@åÐ/Ñ0Ô0°D´KÔ4NÒNÐNå!Ð"6½Ñ=Ô=ð KÝ/4Ð5IÑ/JÔ/JÐ,Ý%)Ð*>Ñ%?Ô%?�NØÔ+×2Ò2°NÀNÐ3SÑTÔTÐTÐTð )×/Ò/°Ñ?Ô?Ð?à# d¤kÔ&@Ñ@�Ý8¸¼ÈxÐXÑXÔX�
Ý!œy¨¬Ô)@À$Ä+ÔBYÐ`eÐfÑfÔf�Ø—’Õ/°
¸LÈ+ÑVÔVÑWÔWÐWÑWà—’˜kÑ*Ô*Ð*Ñ*ÝŒ}˜VÑ$Ô$Ð$r>   NÚ	input_idsr…   r  r†   Úinputs_embedsr~  rˆ   r‰   c           	      ó˜  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|}t          j        |¦  «        }	|r|€t	          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}
t          j        |j        d         |j	        ¬¦  «        |
z   }| 
                    d¦  «        }t          | j        ||||¬¦  «        }| j        j        r|                      ||¬¦  «        }nd }t          | j        ¦  «        D ]\  }} |||	|||f||||dœ|¤Ž}Œ|                      |¦  «        }t#          ||r|nd ¬	¦  «        S )
NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either one)rc   r   r*   ©r&  )rc   r½  r…   r†   r  )r  )r†   r~  r‡   r  )Úlast_hidden_stater†   )Ú
ValueErrorr§  r4   r+  r
   rc   Úget_seq_lengthrÑ   rG   r&  Ú	unsqueezer   r’   r¬  r   rú   rª  r   )r9   r¼  r…   r  r†   r½  r~  rˆ   rL   rj  Úpast_seen_tokensr}  r‡   rd   Úlayers                  r=   rT   zZamba2Model.forward®  sÌ  € ð ˜Ð -°tÐ";Ñ<ð 	ÝØsñô ð ð Ð Ø ×-Ò-¨iÑ8Ô8ˆMà%ˆå!&¤¨]Ñ!;Ô!;Ðð ð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå(Ø”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆð Œ;Ô#ð 	'Ø"&§/¢/°-Èl /Ñ"[Ô"[ÐÐà"&Ðå )¨$¬+Ñ 6Ô 6ð 	ð 	ÑˆI�uØ!˜EØØ&ØØØðð !0Ø#Ø$7Ø)ðð ð ðð ˆMˆMð ×,Ò,¨]Ñ;Ô;ˆå&Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
r>   )NNNNNN)rU   rV   rW   r§   r+   r2   r©  r   r   r   r4   ro  r©   r	   rp  rƒ  r   r   rª   r   rT   r]   r>   r=   r¡  r¡  m  s'  € € € € € ðð ð˜|ð ð ð ð ð. %ð  %ð  %ðD  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ð@
ð @
àÔ# dÑ*ð@
ð œ tÑ+ð@
ð Ô&¨Ñ-ð	@
ð
  ™ð@
ð Ô(¨4Ñ/ð@
ð ˜$‘;ð@
ð Ð+Ô,ð@
ð 
Ð(Ñ	(ð@
ð @
ð @
ñ „^ñ „_ñ  Ôð@
ð @
ð @
r>   r¡  c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚZamba2ForCausalLMrc   c                 óš   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |                      ¦   «          d S r0   ©r1   r2   r¡  r†  r®  ©r9   rc   r<   s     €r=   r2   zZamba2ForCausalLM.__init__õ  ó@   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý  Ñ(Ô(ˆŒ
Ø�ŠÑÔÐÐÐr>   )rU   rV   rW   r+   r2   rX   rY   s   @r=   rÇ  rÇ  ô  sD   ø€ € € € € ð˜|ð ð ð ð ð ð ð ð ð ð r>   rÇ  c                   ó  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 	 	 	 ddej        dz  dej	        dz  dej        dz  de
dz  d	ej        dz  d
ej        dz  dedz  deej	        z  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚZamba2ForSequenceClassificationrc   c                 óš   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |                      ¦   «          d S r0   rÉ  rÊ  s     €r=   r2   z(Zamba2ForSequenceClassification.__init__ü  rË  r>   Nr   r¼  r…   r  r†   r½  Úlabelsr~  Úlogits_to_keeprˆ   r‰   c	           	      ó  —  | j         |f|||||dœ|	¤Ž}
|
d         }|                      |¦  «        }|�|j        d         }n|j        d         }| j        j        €|dk    rt          d¦  «        ‚| j        j        €d}n¨|�}|| j        j        k                         |j        t          j	        ¦  «        }t          j
        |j        d         |j        t          j	        ¬¦  «        }||z                       d¦  «        }n)d}t                               | j        j        › d�¦  «         |t          j
        ||j        ¬	¦  «        |f         }d}|� | j        d|||| j        d
œ|	¤Ž}t#          |||
j        |
j        |
j        ¬¦  «        S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        )r…   r  r†   r½  r~  r   Nr*   z=Cannot handle batch sizes > 1 if no padding token is defined.r@   )r&  rB   zŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r¿  )ÚlogitsrÏ  Úpooled_logitsrc   )ÚlossrÒ  r†   rL   r‡  r]   )r†  ÚscorerG   rc   r£  rÁ  rC   r&  r4   Úint32rÑ   ÚargmaxrÝ   rÞ   r<   rU   Úloss_functionr   r†   rL   r‡  )r9   r¼  r…   r  r†   r½  rÏ  r~  rÐ  rˆ   Útransformer_outputsrL   rÒ  r  Úlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesrÓ  rÔ  s                      r=   rT   z'Zamba2ForSequenceClassification.forward  sç  € ð( 8B°t´zØð8
à)Ø%Ø+Ø'Øð8
ð 8
ð ð8
ð 8
Ðð ,¨AÔ.ˆØ—’˜MÑ*Ô*ˆàÐ Ø"œ¨Ô+ˆJˆJà&Ô,¨QÔ/ˆJàŒ;Ô#Ð+°
¸a²°ÝÐ\Ñ]Ô]Ð]ØŒ;Ô#Ð+Ø!#ÐÐØÐ"Ø%¨¬Ô)AÒA×EÒEÀfÄmÕUZÔU`ÑaÔaˆLÝ!œL¨¬¸Ô)<ÀVÄ]ÕZ_ÔZeÐfÑfÔfˆMØ"/°,Ñ">×!FÒ!FÀrÑ!JÔ!JÐÐà!#ÐÝ×ÒØ”>Ô*ð Zð Zð Zñô ð ð
 �uœ|¨J¸v¼}ÐMÑMÔMÐOaÐaÔbˆàˆØÐØ%�4Ô%ð Ø$¨VÀ=ÐY]ÔYdðð Øhnðð ˆDõ 0ØØ Ø/Ô?Ø-Ô;Ø*Ô5ð
ñ 
ô 
ð 	
r>   )NNNNNNNr   )rU   rV   rW   r+   r2   r   r   r4   ro  r©   r	   rp  rƒ  r¨   r   r   rª   r   rT   rX   rY   s   @r=   rÍ  rÍ  û  sE  ø€ € € € € ð˜|ð ð ð ð ð ð ð
 Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð@
ð @
àÔ# dÑ*ð@
ð œ tÑ+ð@
ð Ô&¨Ñ-ð	@
ð
  ™ð@
ð Ô(¨4Ñ/ð@
ð Ô  4Ñ'ð@
ð ˜$‘;ð@
ð ˜eœlÑ*ð@
ð Ð+Ô,ð@
ð 
Ð1Ñ	1ð@
ð @
ð @
ñ „^ñ Ôð@
ð @
ð @
ð @
ð @
r>   rÍ  )rÇ  rÍ  r¡  r…  )Or�  Úcollections.abcr   Ú	itertoolsr   r4   r   Ú r   r�  Úactivationsr   Úcache_utilsr	   r
   Úintegrations.accelerater   Úintegrations.hub_kernelsr   Úmasking_utilsr   Úmodeling_outputsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.genericr   Úutils.import_utilsr   Úutils.output_capturingr   Úllama.modeling_llamar   r   Úmamba2.modeling_mamba2r   r   r    Úzamba.modeling_zambar!   r"   r#   r$   r%   r&   r'   r(   r)   Úconfiguration_zamba2r+   Ú_CONFIG_FOR_DOCÚ
get_loggerrU   rÝ   ÚModuler-   r[   r`   rb   r¬   rU  rd  rr  rx  r…  r¡  rÇ  rÍ  Ú__all__r]   r>   r=   ú<module>rô     sÀ  ðð €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø =Ð =Ð =Ð =Ð =Ð =Ø 8Ð 8Ð 8Ð 8Ð 8Ð 8Ø /Ð /Ð /Ð /Ð /Ð /Ø YÐ YÐ YÐ YÐ YÐ YÐ YÐ YØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lØ 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø MÐ MÐ MÐ MÐ MÐ MÐ MÐ MØ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ Yð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð /Ð .Ð .Ð .Ð .Ð .ð '€à	ˆÔ	˜HÑ	%Ô	%€ð;ð ;ð ;ð ;ð ;˜œœñ ;ô ;ð ;ð*	ð 	ð 	ð 	ð 	�Lñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð0ñ 	ô 	ð 	ðj)ð j)ð j)ð j)ð j)�nñ j)ô j)ð j)ðZbOð bOð bOð bOð bO�r”yñ bOô bOð bOðJ'ð 'ð 'ð 'ð '�”	ñ 'ô 'ð 'ðT1ð 1ð 1ð 1ð 1Ð"<ñ 1ô 1ð 1ðhZð Zð Zð Zð ZÐ4ñ Zô Zð Zð<ð <ð <ð <ð <Ð(ñ <ô <ð <ð~ ð!ð !ð !ð !ð !˜Oñ !ô !ñ „ð!ðBD
ð D
ð D
ð D
ð D
�*Ð3ñ D
ô D
ð D
ðNð ð ð ð Ð(ñ ô ð ðH
ð H
ð H
ð H
ð H
Ð&Dñ H
ô H
ð H
ðVð ð €€€r>   