§
    ‚Štjq ã                   óN  — d dl Z d dlmZ d dlmZ d dlmZ d dlZd dlmZ ddl	m
Z ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZmZm Z  ddl!m"Z"m#Z# ddl$m%Z%m&Z& ddl'm(Z( ddl)m*Z*m+Z+m,Z,m-Z-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5 ddl6m7Z7  e.j8        e9¦  «        Z: G d„ dej        j;        ¦  «        Z< G d„ dej;        ¦  «        Z= G d„ dej;        ¦  «        Z>d ej?        d!e@d"ej?        fd#„ZA	 dLd%ej;        d&ej?        d'ej?        d(ej?        d)ej?        dz  d*eBd+eBfd,„ZCd-„ ZD ed.¦  «        dMd/„¦   «         ZE G d0„ d1ej;        ¦  «        ZFd2ej?        d3e@fd4„ZGd5„ ZHd6„ ZI G d7„ d8ej;        ¦  «        ZJ G d9„ d:ej;        ¦  «        ZK G d;„ d<ej;        ¦  «        ZL G d=„ d>e¦  «        ZM G d?„ d@e¦  «        ZNe+ G dA„ dBe&¦  «        ¦   «         ZOe+ G dC„ dDeO¦  «        ¦   «         ZP G dE„ dFeOe¦  «        ZQ e+dG¬H¦  «         G dI„ dJeO¦  «        ¦   «         ZRg dK¢ZSdS )Né    N)ÚCallable)Úcycle)ÚOptional)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_func_from_hub)Úforce_accelerate_hooks)Úlazy_load_kernel)Úcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPastÚ SequenceClassifierOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_torchdynamo_compilingÚlogging)Úmaybe_autocastÚmerge_with_config_defaults)Úresolve_internal_import)Úcapture_outputsé   )ÚZamba2Configc                   ó(   ‡ — e Zd Zdˆ fd„	Zdd„Zˆ xZS )ÚZamba2RMSNormGatedç�íµ ÷Æ°>c                 óº   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        || _        d S ©N)	ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilonÚ
group_size)ÚselfÚhidden_sizer1   ÚepsÚ	__class__s       €úh/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/zamba2/modeling_zamba2.pyr+   zZamba2RMSNormGated.__init__5   sG   ø€ Ý‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔØ$ˆŒˆˆó    Nc                 ó  — |j         }|                     t          j        ¦  «        }|�?|t          j                             |                     t          j        ¦  «        ¦  «        z  }|j        �^ }}|| j        z  } |j	        g |¢|‘| j        ‘R Ž }| 
                    d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  } |j	        g |¢|| j        z  ‘R Ž }| j        |                     |¦  «        z  S ©Né   éÿÿÿÿT)Úkeepdim)ÚdtypeÚtor-   Úfloat32r   Ú
functionalÚsiluÚshaper1   ÚviewÚpowÚmeanÚrsqrtr0   r/   )	r2   Úhidden_statesÚgateÚinput_dtypeÚprefix_dimsÚlast_dimÚgroup_countÚhidden_states_groupÚvariances	            r6   ÚforwardzZamba2RMSNormGated.forward;   s  € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØÐØ)­B¬M×,>Ò,>¸t¿wºwÅuÄ}Ñ?UÔ?UÑ,VÔ,VÑVˆMØ!.Ô!4Ñˆ�hØ $¤/Ñ1ˆØ0˜mÔ0Ð\°+Ð\¸{Ð\ÈDÌOÐ\Ð\Ð\ÐØ&×*Ò*¨1Ñ-Ô-×2Ò2°2¸tÐ2ÑDÔDˆØ1µE´KÀÈ4ÔK`Ñ@`Ñ4aÔ4aÑaÐØ0Ð+Ô0Ð]°+Ð]¸{ÈTÌ_Ñ?\Ð]Ð]Ð]ˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r7   ©r'   r)   )Ú__name__Ú
__module__Ú__qualname__r+   rO   Ú__classcell__©r5   s   @r6   r&   r&   4   sQ   ø€ € € € € ð%ð %ð %ð %ð %ð %ð;ð ;ð ;ð ;ð ;ð ;ð ;ð ;r7   r&   c                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
ÚZamba2RMSNormr'   r4   ÚreturnNc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z<
        Zamba2RMSNorm is equivalent to T5LayerNorm
        N)r*   r+   r   r,   r-   r.   r/   r0   )r2   r3   r4   r5   s      €r6   r+   zZamba2RMSNorm.__init__J   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr7   rG   c                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S r9   )	r=   r>   r-   r?   rD   rE   rF   r0   r/   )r2   rG   rI   rN   s       r6   rO   zZamba2RMSNorm.forwardR   s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r7   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)Útupler/   rB   r0   )r2   s    r6   Ú
extra_reprzZamba2RMSNorm.extra_reprY   s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr7   rP   )
rQ   rR   rS   Úfloatr+   r-   ÚTensorrO   r]   rT   rU   s   @r6   rW   rW   I   sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð Jr7   rW   c                   óÔ   ‡ — e Zd ZU ej        ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d         de
dz  ded	ef         fd
„¦   «         Z ej        ¦   «         ed„ ¦   «         ¦   «         Zˆ xZS )ÚZamba2RotaryEmbeddingÚinv_freqNÚconfigc                 ó²  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        |                      d|d¬¦  «         |                      d|                     ¦   «         d¬¦  «         d S )NÚ	rope_typeÚdefaultrb   F)Ú
persistentÚoriginal_inv_freq)r*   r+   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrc   Úrope_parametersre   Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)r2   rc   ÚdeviceÚrope_init_fnrb   r5   s        €r6   r+   zZamba2RotaryEmbedding.__init__`   sÊ   ø€ Ý‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(à×Ò˜Z¨¸eÐÑDÔDÐDØ×ÒÐ0°(·.².Ñ2BÔ2BÈuÐÑUÔUÐUÐUÐUr7   rq   ztorch.deviceÚseq_lenrX   ztorch.Tensorc                 óü   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «                             |t          j        ¬¦  «        |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNg      ð?r   r:   ©r=   ©rq   r=   )	rl   Úgetattrr3   Únum_attention_headsr-   ÚarangeÚint64r>   r^   )rc   rq   rs   ÚbaseÚdimÚattention_factorrb   s          r6   rm   z5Zamba2RotaryEmbedding.compute_default_rope_parametersp   sŒ   € ð& Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐð Ø•U”\ ! S¨!µ5´;Ð?Ñ?Ô?×BÒBÈ&ÕX]ÔXcÐBÑdÔdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r7   c                 óN  — | j         d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  |                     ¦   «         |                     ¦   «         z   
                    dd¦  «        }t          j        ||fd¬	¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬
¦  «        |	                     |j        ¬
¦  «        fS )Nr   r;   r#   ÚmpsÚcpuF)Údevice_typeÚenabledr:   ©r~   rw   )rb   r^   ÚexpandrB   r>   rq   Ú
isinstanceÚtypeÚstrr   Ú	transposer-   ÚcatÚcosrn   Úsinr=   )
r2   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrƒ   ÚfreqsÚembrŒ   r�   s
             r6   rO   zZamba2RotaryEmbedding.forwardŽ   s·  € ð !œM¨$°°°°4¨-Ô8×>Ò>Ñ@Ô@×GÒGÈÔHZÐ[\ÔH]Ð_aÐcdÑeÔe×hÒhÐijÔiqÑrÔrÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	5ð 	5Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃBE&Å&E*Å-E*r)   ©NNN)rQ   rR   rS   r-   r_   Ú__annotations__r$   r+   Ústaticmethodr   Úintr\   r^   rm   Úno_gradr   rO   rT   rU   s   @r6   ra   ra   ]   sù   ø€ € € € € € ØŒlÐÐÑðVð V˜|ð Vð Vð Vð Vð Vð Vð  à&*Ø+/Ø"ð*ð *Ø˜tÑ#ð*à˜Ô(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ô	%ð	*ð *ð *ñ „\ð*ð: €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r7   ra   rG   Ún_reprX   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r#   N)rB   r†   Úreshape)rG   r™   ÚbatchÚnum_key_value_headsÚslenrv   s         r6   Ú	repeat_kvrŸ   ž   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr7   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutc                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr:   r   r;   )r~   r=   )ÚpÚtrainingr#   )rŸ   Únum_key_value_groupsr-   ÚmatmulrŠ   r   r@   Úsoftmaxr?   r>   r=   r§   rª   Ú
contiguous)r¡   r¢   r£   r¤   r¥   r¦   r§   ÚkwargsÚ
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r6   Úeager_attention_forwardr´   ª   sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r7   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr;   r:   r…   )rB   r-   r‹   )rŽ   Úx1Úx2s      r6   Úrotate_halfr¸   Ã   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r7   Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer¸   )ÚqÚkrŒ   r�   Úunsqueeze_dimÚq_embedÚk_embeds          r6   Úapply_rotary_pos_embrÁ   Ê   sc   € ð& �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr7   c                   ó  ‡ — e Zd ZdZ	 	 	 ddededz  dedz  dedz  fˆ fd„Z	 	 	 ddej        ded	ej        dz  d
e	dz  de
ej        ej        f         dz  dee         de
ej        ej        dz  e
ej                 dz  f         fd„Zˆ xZS )ÚZamba2AttentionaZ  
    Multi-headed attention from 'Attention Is All You Need' paper.

    Adapted from transformers.models.mistral.modeling_mistral.MistralAttention:
    The input dimension here is attention_hidden_size = 2 * hidden_size, and head_dim = attention_hidden_size // num_heads.
    The extra factor of 2 comes from the input being the concatenation of original_hidden_states with the output of the previous (mamba) layer
    (see fig. 2 in https://huggingface.co/papers/2405.16712).
    Additionally, replaced
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) with
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim/2)
    Finally, this attention layer contributes to tied transformer blocks aimed to increasing compute without increasing model size. Because this
    layer is tied, un-tied adapters (formally the same as LoRA but used in the base model) modules are added to the q, k, v projectors to increase
    expressivity with a small memory overhead (see Fig. 2 of https://huggingface.co/papers/2411.15242).
    Nrc   Ú	layer_idxÚnum_fwd_mem_blocksÚblock_idc           	      ó<  •— t          ¦   «                              ¦   «          || _        || _        |j        | _        |j        | _        |j        |j        z  | _	        |j
        | _
        | j        dz  dz  | _        d| _        |j        | _        t          j        |j        |j        | j        z  d¬¦  «        | _        t          j        |j        |j        | j        z  d¬¦  «        | _        t          j        |j        |j        | j        z  d¬¦  «        | _        t          j        |j        | j        z  |j        d¬¦  «        | _        || _        |j        | _        || _        |j        �rt          j        g ¦  «        | _        t          j        g ¦  «        | _        t          j        g ¦  «        | _        t=          | j        ¦  «        D �]±}||j        z  |k    �rt          j         t          j        | j        | j        j!        d¬¦  «        t          j        | j        j!        | j        d¬¦  «        ¦  «        }t          j         t          j        | j        | j        j!        d¬¦  «        t          j        | j        j!        | j        d¬¦  «        ¦  «        }t          j         t          j        | j        | j        j!        d¬¦  «        t          j        | j        j!        | j        d¬¦  «        ¦  «        }n9t          j"        ¦   «         }t          j"        ¦   «         }t          j"        ¦   «         }| j         #                    |¦  «         | j         #                    |¦  «         | j         #                    |¦  «         �Œ³d„ tI          | j        ¦  «        D ¦   «         | _%        d S )Nr:   g      à¿TF©Úbiasc                 ó   — i | ]\  }}||“Œ	S © rË   ©Ú.0Úindexr¤   s      r6   ú
<dictcomp>z,Zamba2Attention.__init__.<locals>.<dictcomp>*  s   € Ð[Ð[Ð[©<¨5°%˜% Ð[Ð[Ð[r7   )&r*   r+   rc   rÄ   Úattention_hidden_sizeÚattention_head_dimrv   rz   r�   r«   ri   r¦   Ú	is_causalÚattention_dropoutr   ÚLinearÚq_projÚk_projÚv_projr3   Úo_projrÅ   Úhybrid_layer_idsÚlayer_block_maprÆ   Úuse_shared_attention_adapterÚ
ModuleListÚlinear_q_adapter_listÚlinear_k_adapter_listÚlinear_v_adapter_listÚrangeÚnum_mem_blocksÚ
SequentialÚadapter_rankÚIdentityÚappendÚ	enumerateÚ	layer_dic)
r2   rc   rÄ   rÅ   rÆ   ÚiÚlinear_q_adapterÚlinear_k_adapterÚlinear_v_adapterr5   s
            €r6   r+   zZamba2Attention.__init__ô   s*  ø€ õ 	‰Œ×ÒÑÔÐØˆŒØ"ˆŒà%+Ô%AˆÔ"ØÔ1ˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!Ø'-Ô'EˆÔ$Øœ¨Ñ)¨dÑ2ˆŒØˆŒØ!'Ô!9ˆÔå”i Ô <¸fÔ>XÐ[_Ô[hÑ>hÐotÐuÑuÔuˆŒÝ”i Ô <¸fÔ>XÐ[_Ô[hÑ>hÐotÐuÑuÔuˆŒÝ”i Ô <¸fÔ>XÐ[_Ô[hÑ>hÐotÐuÑuÔuˆŒÝ”i Ô :¸T¼]Ñ JÈFÔL^ÐejÐkÑkÔkˆŒØ"4ˆÔØ%Ô6ˆÔØ ˆŒàÔ.ñ 	DÝ)+¬°rÑ):Ô):ˆDÔ&Ý)+¬°rÑ):Ô):ˆDÔ&Ý)+¬°rÑ):Ô):ˆDÔ&å˜4Ô2Ñ3Ô3ð Dñ D�Ø�vÔ,Ñ,°Ò8Ñ8Ý')¤}Ýœ	 $Ô"<¸d¼kÔ>VÐ]bÐcÑcÔcÝœ	 $¤+Ô":¸DÔ<VÐ]bÐcÑcÔcñ(ô (Ð$õ (*¤}Ýœ	 $Ô"<¸d¼kÔ>VÐ]bÐcÑcÔcÝœ	 $¤+Ô":¸DÔ<VÐ]bÐcÑcÔcñ(ô (Ð$õ (*¤}Ýœ	 $Ô"<¸d¼kÔ>VÐ]bÐcÑcÔcÝœ	 $¤+Ô":¸DÔ<VÐ]bÐcÑcÔcñ(ô (Ð$Ð$õ
 (*¤{¡}¤}Ð$Ý')¤{¡}¤}Ð$Ý')¤{¡}¤}Ð$ØÔ*×1Ò1Ð2BÑCÔCÐCØÔ*×1Ò1Ð2BÑCÔCÐCØÔ*×1Ò1Ð2BÑCÔCÐCÑCà[Ð[½9ÀTÔEYÑ;ZÔ;ZÐ[Ñ[Ô[ˆŒˆˆr7   rG   r¥   Úpast_key_valuesÚposition_embeddingsr¯   rX   c                 ó  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «        }	|                      |¦  «        }
|                      |¦  «        }| j        j        rX| j        |         }|	 | j        |         |¦  «        z   }	|
 | j	        |         |¦  «        z   }
| | j
        |         |¦  «        z   }|	                     |¦  «                             dd¦  «        }	|
                     |¦  «                             dd¦  «        }
|                     |¦  «                             dd¦  «        }| j        j        r|\  }}t          |	|
||¦  «        \  }	}
|�|                     |
||¦  «        \  }
}t!          j        | j        j        t&          ¦  «        } || |	|
||f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr;   r#   r:   r    )r§   r¦   )rB   rv   rÕ   rÖ   r×   rc   rÛ   rç   rÝ   rÞ   rß   rC   rŠ   Úuse_mem_roperÁ   Úupdater   Úget_interfaceÚ_attn_implementationr´   rª   rÓ   r¦   r›   r®   rØ   )r2   rG   rÄ   r¥   rì   rí   r¯   Úinput_shapeÚhidden_shapeÚquery_statesr°   r±   Úadapter_layer_idxrŒ   r�   Úattention_interfacer³   r²   s                     r6   rO   zZamba2Attention.forward,  sM  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1ˆØ—[’[ Ñ/Ô/ˆ
Ø—{’{ =Ñ1Ô1ˆØŒ;Ô3ð 	gØ $¤¨yÔ 9ÐØ'Ð*W¨$Ô*DÐEVÔ*WÐXeÑ*fÔ*fÑfˆLØ#Ð&S dÔ&@ÐARÔ&SÐTaÑ&bÔ&bÑbˆJØ'Ð*W¨$Ô*DÐEVÔ*WÐXeÑ*fÔ*fÑfˆLà#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆØ—_’_ \Ñ2Ô2×<Ò<¸QÀÑBÔBˆ
Ø#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆàŒ;Ô#ð 	`Ø*‰HˆC�Ý';¸LÈ*ÐVYÐ[^Ñ'_Ô'_Ñ$ˆL˜*àÐ&Ø'6×'=Ò'=¸jÈ,ÐXaÑ'bÔ'bÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r7   r”   )rQ   rR   rS   Ú__doc__r$   r—   r+   r-   r_   r
   r\   r   r   rO   rT   rU   s   @r6   rÃ   rÃ   ä   sD  ø€ € € € € ðð ð$ !%Ø)-Ø#ð6\ð 6\àð6\ð ˜‘:ð6\ð   $™Jð	6\ð
 ˜‘*ð6\ð 6\ð 6\ð 6\ð 6\ð 6\ðx /3Ø(,ØHLð1)ð 1)à”|ð1)ð ð1)ð œ tÑ+ð	1)ð
  ™ð1)ð # 5¤<°´Ð#=Ô>ÀÑEð1)ð Ð+Ô,ð1)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)r7   rÃ   Úinput_tensorÚpad_sizec                 ó¦   — t          | j        ¦  «        dk    r
ddddd|ddfnddd|ddf}t          j        j                             | |dd¬¦  «        S )z‚
    Padding x tensor with `pad_size` on the seq_len dim (dim=1)

    Assumes that we only have tensors of either size 4 or 3
    é   r   Úconstant)Úmoder¤   )ÚlenrB   r-   r   r@   Úpad)rù   rú   Ú	pad_shapes      r6   Úpad_tensor_by_sizer  c  sj   € õ 47°|Ô7IÑ3JÔ3JÈaÒ3OÐ3O��A�q˜!˜Q ¨!¨QÐ/Ð/ÐVWÐYZÐ\]Ð_gÐijÐlmÐUn€IåŒ8Ô×"Ò" <°ÀÐSTÐ"ÑUÔUÐUr7   c                 ó"  — t          | |¦  «        } t          | j        ¦  «        dk    r.|                      | j        d         d|| j        d         ¦  «        S |                      | j        d         d|| j        d         | j        d         ¦  «        S )zÀ
    Padding input_tensor with `pad_size` on the seq_len dim (dim=1) and
    simultaneously splitting it into chunk sequences.

    Assumes that we only have tensors of either size 4 or 3
    r   r   r;   r:   )r  rÿ   rB   r›   )rù   rú   Ú
chunk_sizes      r6   Úreshape_into_chunksr  n  s’   € õ & l°HÑ=Ô=€Lå
ˆ<ÔÑÔ !Ò#Ð#à×#Ò# LÔ$6°qÔ$9¸2¸zÈ<ÔK]Ð^_ÔK`ÑaÔaÐað ×#Ò#ØÔ˜qÔ! 2 z°<Ô3EÀaÔ3HÈ,ÔJ\Ð]^ÔJ_ñ
ô 
ð 	
r7   c                 ó  — |                       d¦  «        } | d         j        g |                       ¦   «         ¢|‘R Ž } t          j        t          j        ||| j        t          j        ¬¦  «        d¬¦  «        }|                      | d¦  «        } t          j        | d¬¦  «        }t          j        t          j        ||| j        t          j        ¬¦  «        d¬¦  «        }|                     | t          j	         ¦  «        }|S )zo
    More stable segment sum calculation. Uses cumulative sums and masking instead of direct subtractions.
    r;   ©.Nrx   )Údiagonalr   éþÿÿÿr…   )
Úsizer†   r-   Útrilr.   rq   ÚboolÚmasked_fillÚcumsumÚinf)rù   r  ÚmaskÚtensor_segsums       r6   Úsegment_sumr  ‚  só   € ð ×"Ò" 2Ñ&Ô&€Jð 2�< 	Ô*Ô1ÐS°<×3DÒ3DÑ3FÔ3FÐSÈ
ÐSÐSÐS€LåŒ:•e”j ¨ZÀÔ@SÕ[`Ô[eÐfÑfÔfÐqsÐtÑtÔt€DØ×+Ò+¨T¨E°1Ñ5Ô5€Lå”L °2Ð6Ñ6Ô6€Mõ Œ:•e”j ¨ZÀÔ@SÕ[`Ô[eÐfÑfÔfÐqrÐsÑsÔs€DØ!×-Ò-¨t¨eµe´i°ZÑ@Ô@€MØÐr7   c                   óä   ‡ — e Zd ZdZddededz  fˆ fd„Z	 	 ddej        de	dz  dej        dz  fd	„Z
dde	dz  dej        dz  fd
„Z ed¦  «        	 	 dde	dz  dej        dz  fd„¦   «         Zˆ xZS )ÚZamba2MambaMixeruƒ  
    Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)
    Nrc   rÄ   c           	      ó$  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        t          |j	        | j        z  ¦  «        | _
        || _        |j        | _        d| _        t          j        ¦   «         | _        |j        | _        |j        | _        |j        | _        | j        j        | _        |j        | _        |j        | _        |j        | _        |j        | _        | j
        d| j        z  | j        z  z   | _        t          j        | j        | j        d|j        | j        |j        dz
  ¬¦  «        | _        | j
        | j        z   | j        z   }t          j        | j        ||j         ¬¦  «        | _!        t          j"        tG          j$        | j        ¦  «        ¦  «        | _%        tG          j&        d| j        dz   ¦  «        }t          j"        tG          j'        |¦  «        ¦  «        | _(        tS          | j
        | j
        | j        z  d¬¦  «        | _*        t          j"        tG          j$        | j        ¦  «        ¦  «        | _+        t          j        | j
        | j        |j         ¬¦  «        | _,        |j-        r¡t]          d	¦  «        }t_          |d
d ¦  «        a0t_          |dd ¦  «        a1t]          d¦  «        }te          |d¬¦  «        a3te          |d¬¦  «        a4te          |d¬¦  «        a5tm          tf          th          tj          tb          t`          f¦  «        a7nd a0d a1d a3d a4d a5da7t_          |dd¦  «        r!tn          stp           9                    d¦  «         |j:        |         | _;        d S )NrA   r:   Tr#   )Úin_channelsÚout_channelsrÉ   Úkernel_sizeÚgroupsÚpaddingrÈ   gñhãˆµøä>)r1   r4   zcausal-conv1dÚcausal_conv1d_updateÚcausal_conv1d_fnz	mamba-ssmz8ops.triton.selective_state_update.selective_state_update)Úchained_pathz1ops.triton.ssd_combined.mamba_chunk_scan_combinedz8ops.triton.ssd_combined.mamba_split_conv1d_scan_combinedFÚuse_mamba_kernelsa  The fast path is not available because one of `(selective_state_update, causal_conv1d_fn, causal_conv1d_update)` is None. Falling back to the naive implementation. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1d)<r*   r+   rc   r3   Úmamba_d_stateÚssm_state_sizeÚmamba_d_convÚconv_kernel_sizer—   Úmamba_expandÚintermediate_sizerÄ   Úuse_conv_biasÚ
activationr   ÚSiLUÚactÚuse_mem_eff_pathÚmamba_ngroupsÚn_groupsÚmamba_headdimrv   Ún_mamba_headsÚ	num_headsr  Útime_step_limitÚtime_step_minÚtime_step_maxÚconv_dimÚConv1dÚconv1drÔ   Úadd_bias_linearÚin_projr,   r-   r.   Údt_biasr{   ÚlogÚA_logr&   ÚnormÚDÚout_projr  r   ry   r  r  r!   Úselective_state_updateÚmamba_chunk_scan_combinedÚ mamba_split_conv1d_scan_combinedÚallÚis_fast_path_availableÚloggerÚwarning_onceÚlayer_typesÚ
layer_type)r2   rc   rÄ   Úprojection_sizeÚAÚcausal_conv1dÚ	mamba_ssmr5   s          €r6   r+   zZamba2MambaMixer.__init__ž  sq  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ$Ô2ˆÔØ &Ô 3ˆÔÝ!$ VÔ%8¸4Ô;KÑ%KÑ!LÔ!LˆÔØ"ˆŒØ#Ô1ˆÔØ ˆŒÝ”7‘9”9ˆŒØ &Ô 7ˆÔàÔ,ˆŒØÔ,ˆŒØœÔ2ˆŒØ Ô+ˆŒà%Ô5ˆÔØ#Ô1ˆÔØ#Ô1ˆÔàÔ.°°T´]Ñ1BÀTÔEXÑ1XÑXˆŒÝ”iØœØœØØÔ+Ø”=ØÔ'¨!Ñ+ð
ñ 
ô 
ˆŒð Ô0°4´=Ñ@À4Ä>ÑQˆÝ”yØÔØØÔ'ð
ñ 
ô 
ˆŒõ ”|¥E¤J¨t¬~Ñ$>Ô$>Ñ?Ô?ˆŒõ ŒL˜˜DœN¨QÑ.Ñ/Ô/ˆÝ”\¥%¤)¨A¡,¤,Ñ/Ô/ˆŒ
Ý&ØÔ"¨tÔ/EÈÌÑ/VÐ\`ð
ñ 
ô 
ˆŒ	õ ”�eœj¨¬Ñ8Ô8Ñ9Ô9ˆŒåœ	 $Ô"8¸$Ô:JÐQWÔQgÐhÑhÔhˆŒð Ô#ð 	+Ý,¨_Ñ=Ô=ˆMÝ#*¨=Ð:PÐRVÑ#WÔ#WÐ Ý& }Ð6HÈ$ÑOÔOÐå(¨Ñ5Ô5ˆIÝ%<ØÐ(bð&ñ &ô &Ð"õ )@ØÐ([ð)ñ )ô )Ð%õ 0GØÐ(bð0ñ 0ô 0Ð,õ &)å*Ý-Ý4Ý$Ý(ðñ&ô &Ð"Ð"ð $(Ð Ø#ÐØ%)Ð"Ø(,Ð%Ø/3Ð,Ø%*Ð"å�6Ð.°Ñ5Ô5ð 	Õ>Tð 	Ý×Òð>ñô ð ð !Ô,¨YÔ7ˆŒˆˆr7   rG   Úcache_paramsr¥   c                 óp  — |j         \  }}}| j        | j        z  }d| j        z  d| j        z  | j        z  z   | j        z   }|d uo|                     | j        ¦  «        }	|	r:|j        | j                 j        d         }
|j        | j                 j	        d         }|	�rî|dk    �rç|  
                    |                     d¦  «        ¦  «        }|j         d         |z
  dz  }||| j        | j        | j        g}t          j        ||d¬¦  «        \  }}}}}t          ||
| j        j                             d¦  «        | j        j        | j        ¦  «        }t          j        || j        ||gd¬¦  «        \  }}}t          j        | j                             ¦   «         ¦  «         }|d d …d df         d d …d d …d f                              d| j        | j        ¦  «                             t          j        ¬¦  «        }|d d …d d …d f                              dd| j        ¦  «        }| j        d d …d df                              d| j        ¦  «        }| j        d d …d df                              d| j        ¦  «        }|                     || j        |j         d         | j        z  ¦  «        }|                     || j        |j         d         | j        z  ¦  «        }|                     || j        | j        ¦  «        }t=          |||||||d |d¬	¦
  «
        }|                     || j        | j        z  ¦  «        }|                      ||¦  «        }|                       |                     | j         j        j!        ¦  «        ¦  «        d d …d df         }�n+|�Dt          j"        |dk    ¦  «        s,|j!        }||d d …d d …d f         z                       |¦  «        }|  
                    |¦  «        }t          j        | j                             ¦   «         ¦  «         }| j#        €i nd
| j#        i}|�t          j"        |dk    ¦  «        }nd}| j$        r›| j%        r”|€’|r�tM          || j        j                             d¦  «        | j        j        | j        |f| j        | j'        d | j        | j        j        | j        j(        | j         j        | j         j        | j        | j        dddœ|¤Ž\  }}�nÓt          j        || j        | j        | j        gd¬¦  «        \  }}}| )                    dd¦  «        }|	rt          j*        |
|gd¬¦  «        }|�PtV          j,         -                    || j.        |j         d         z
  df¦  «        }| /                    || j        ¦  «         t`          �	| j        dvr>|  1                    |                      |¦  «        dd |j         d         …f         ¦  «        }n?ta          || j        j                             d¦  «        | j        j        | j        ¬¦  «        }|	r|d d …d d …| d …f         }| )                    dd¦  «        }t          j        || j        ||gd¬¦  «        \  }}}|�Dt          j"        |dk    ¦  «        s,|j!        }||d d …d d …d f         z                       |¦  «        }te          |                     ||d| j        ¦  «        |||                     ||| j        d¦  «        |                     ||| j        d¦  «        f| j'        | j        d d d| j        d|	r|nd dœ|¤Ž\  } }|�|�| 3                    || j        ¦  «         |                      ||d¦  «        } |                      | |¦  «        } |                       |                      | j         j        j!        ¦  «        ¦  «        }|S )Nr:   r   r#   r;   r…   .rw   T)Úzr7  Údt_softplusÚdt_limitF)r;  r  Úseq_idxr&  Úrmsnorm_weightÚrmsnorm_epsÚoutproj_weightÚoutproj_biasÚheaddimÚngroupsÚnorm_before_gateÚreturn_final_states)rA   Úswish)rŽ   r/   rÉ   r&  )r  r;  rL  rO  rW  r7  rM  Úinitial_states)4rB   r+  r   r$  r.  Úhas_previous_staterÄ   ÚlayersÚconv_statesÚrecurrent_statesr6  Úsqueezer2  r-   Úsplitr  r4  r/   rÉ   r&  Úexpr9  r^   r†   rv   r>   r?   r7  r;  rC   r=  r:  r<  r=   r@  r/  r)  rª   r?  r  r0   rŠ   r‹   r   r@   r   r"  Úupdate_conv_stater  r(  r>  Úupdate_recurrent_state)!r2   rG   rJ  r¥   Ú
batch_sizers   Ú_Úgroups_time_state_sizeÚd_to_removeÚuse_precomputed_statesÚ
conv_stateÚrecurrent_stateÚin_projected_statesÚd_mlpÚsplit_projection_dimrH   Úhidden_states_B_CÚdtÚBÚCrG  r7  r;  Úhidden_states_reshapedÚoutr=   Úprojected_statesÚdt_limit_kwargsÚinput_not_maskedÚ	ssm_stateÚ	time_stepÚnew_conv_stateÚscan_outputs!                                    r6   Úcuda_kernels_forwardz%Zamba2MambaMixer.cuda_kernels_forward  s?  € ð "/Ô!4Ñˆ
�G˜QØ!%¤°Ô1DÑ!DÐØ˜$Ô0Ñ0°1°t´}Ñ3DÀtÔGZÑ3ZÑZÐ]aÔ]kÑkˆà!-°TÐ!9Ð!m¸l×>]Ò>]Ð^bÔ^lÑ>mÔ>mÐØ!ð 	VØ%Ô,¨T¬^Ô<ÔHÈÔKˆJØ*Ô1°$´.ÔAÔRÐSTÔUˆOð "ñ R	P g°¢l¡lØ"&§,¢,¨}×/DÒ/DÀQÑ/GÔ/GÑ"HÔ"HÐØ(Ô.¨rÔ2°[Ñ@ÀQÑFˆEØ$)¨5°$Ô2HÈ$Ì-ÐY]ÔYgÐ#hÐ Ý05´Ð<OÐQeÐkmÐ0nÑ0nÔ0nÑ-ˆAˆq�$Ð)¨2å 4Ø!ØØ”Ô"×*Ò*¨1Ñ-Ô-Ø”Ô Ø”ñ!ô !Ðõ #(¤+Ø!ØÔ'Ð)?ÐAWÐXØð#ñ #ô #ÑˆM˜1˜aõ
 ”˜4œ:×+Ò+Ñ-Ô-Ñ.Ô.Ð.ˆAà�!�!�!�T˜3�,”    1 1 1 d 
Ô+×2Ò2°2°t´}ÀdÔFYÑZÔZ×]Ò]ÕdiÔdqÐ]ÑrÔrˆAØ�A�A�A�q�q�q˜$�J”×&Ò& r¨2¨t¬}Ñ=Ô=ˆBØ”l 1 1 1 d¨C <Ô0×7Ò7¸¸D¼MÑJÔJˆGØ”�q�q�q˜$ �|Ô$×+Ò+¨B°´Ñ>Ô>ˆAØ—’�z 4¤=°!´'¸!´*ÀÄÑ2MÑNÔNˆAØ—’�z 4¤=°!´'¸!´*ÀÄÑ2MÑNÔNˆAØ%2×%7Ò%7¸
ÀDÄNÐTXÔTaÑ%bÔ%bÐ"Ý2ØØ&ØØØØØØØØ ðñ ô ˆMð *×.Ò.¨z¸4¼>ÈDÌMÑ;YÑZÔZˆMØ ŸIšI m°TÑ:Ô:ˆMð —-’- × 0Ò 0°´Ô1EÔ1KÑ LÔ LÑMÔMÈaÈaÈaÐQUÐWZÈlÔ[ˆC‰Cð Ð)µ%´)¸NÈaÒ<OÑ2PÔ2PÐ)à%Ô+�Ø!.°ÀÀÀÀ1À1À1ÀdÀ
Ô1KÑ!K× OÒ OÐPUÑ VÔ V�à#Ÿ|š|¨MÑ:Ô:ÐÝ”˜4œ:×+Ò+Ñ-Ô-Ñ.Ô.Ð.ˆAØ$(Ô$8Ð$@˜b˜bÀzÐSWÔSgÐFhˆOØÐ)Ý#(¤9¨^¸qÒ-@Ñ#AÔ#AÐ Ð à#'Ð àÔ$ð UP¨¬ð UP¸<Ð;OÐTdÐ;OÝ!AØ$Ø”KÔ&×.Ò.¨qÑ1Ô1Ø”KÔ$Ø”LØð"ð ”fØ#œØ Ø#œØ#'¤9Ô#3Ø $¤	Ô :Ø#'¤=Ô#7Ø!%¤Ô!3Ø œMØ œMØ%*Ø(,ð#"ð "ð$ &ð%"ð "‘��Y‘Yõ, 6;´[Ø$ØÔ+¨T¬]¸D¼NÐKØð6ñ 6ô 6Ñ2�Ð'¨ð %6×$?Ò$?ÀÀ1Ñ$EÔ$EÐ!Ø)ð [õ ).¬	°:Ð?PÐ2QÐWYÐ(ZÑ(ZÔ(ZÐ%ØÐ+Ý%'¤]×%6Ò%6Ø)¨DÔ,AÐDUÔD[Ð\^ÔD_Ñ,_ÐabÐ+cñ&ô &�Nð !×2Ò2°>À4Ä>ÑRÔRÐRÝ#Ð+¨t¬ÐFWÐ/WÐ/WØ(,¯ª°·²Ð=NÑ1OÔ1OÐPSÐUrÐWhÔWnÐoqÔWrÐUrÐPrÔ1sÑ(tÔ(tÐ%Ð%å(8Ø+Ø#œ{Ô1×9Ò9¸!Ñ<Ô<Ø!œ[Ô-Ø#'¤?ð	)ñ )ô )Ð%ð *ð KØ(9¸!¸!¸!¸Q¸Q¸QÀÀÀ	À	¸/Ô(JÐ%Ø$5×$?Ò$?ÀÀ1Ñ$EÔ$EÐ!Ý&+¤kØ%ØÔ+Ð-CÐE[Ð\Øð'ñ 'ô 'Ñ#�˜q !ð
 "Ð-µe´iÀÐRSÒ@SÑ6TÔ6TÐ-à)Ô/�EØ%2°^ÀAÀAÀAÀqÀqÀqÈ$ÀJÔ5OÑ%O×$SÒ$SÐTYÑ$ZÔ$Z�MÝ)BØ!×&Ò& z°7¸BÀÄÑNÔNØØØ—F’F˜: w°´¸rÑBÔBØ—F’F˜: w°´¸rÑBÔBð*ð  $œØ”fØØ Ø(,Ø œLØ $Ø6LÐ#V ? ?ÐRVð*ð *ð &ð*ð *Ñ&�˜Yð  Ð(¨\Ð-EØ ×7Ò7¸	À4Ä>ÑRÔRÐRØ)×.Ò.¨z¸7ÀBÑGÔG�à"Ÿiši¨°TÑ:Ô:�ð —m’m K§N¢N°4´=Ô3GÔ3MÑ$NÔ$NÑOÔO�Øˆ
r7   c                 ó¦  ‡ ‡3— |j         \  }}}|j        }|�0|                     ‰ j        ¦  «        r‰                      |¦  «        }n<|�%||d d …d d …d f         z                       |¦  «        }‰                      |¦  «        }|j         d         d‰ j        z  z
  d‰ j        z  ‰ j        z  z
  ‰ j	        z
  dz  }	| 
                    |	|	‰ j        ‰ j        ‰ j	        gd¬¦  «        \  }}}
}}|                     dd¦  «        }|d uo|                     ‰ j        ¦  «        }|r|j        ‰ j                 j        d         }|r¬|dk    r¦|                     |‰ j        ¦  «        d‰ j         d …f         }t#          j        |‰ j        j        d d …dd d …f         z  d¬¦  «        }‰ j        r|‰ j        j        z  }‰                      |¦  «                             |¦  «        d d …d df         }�n|rt#          j        ||gd¬¦  «        }|�`t2          j                             |‰ j        |j         d         z
  df¦  «        }|                     |‰ j        ¦  «        d‰ j         d …f         }‰                      ‰                      |¦  «        dd |j         d         …f                              dd¦  «        ¦  «        }|r|d d …| d …d d …f         }|�,|j        }||d d …d d …d f         z                       |¦  «        }t#          j
        |‰ j        ‰ j        ‰ j        z  ‰ j        ‰ j        z  gd¬¦  «        \  }}}t#          j        ‰ j                             ¦   «         ¦  «         }|�r/|dk    �r(|j        dk    r|d d …d df         n|d d …dd d …f         d d …d df         }|                     dd¦  «                              ||j         d         ‰ j!        ¦  «        }‰ j"        d                               ‰ j"        j         d         ‰ j!        ¦  «        }t"          j        j         #                    ||                     |j        ¦  «        z   ¦  «        }t#          j$        |‰ j%        ¦  «        }|d                               ‰ j	        ‰ j!        ‰ j        ¦  «                             t"          j&        ¬	¦  «        }t#          j        |d         |z  ¦  «        }| '                    |‰ j        d¦  «        dd d d …f         }|                      |‰ j        ‰ j	        ‰ j        z  |j         d         ¦  «         (                    ¦   «         }| '                    |d|j         d         ¦  «        }|d         |dd d d …f         z  }| '                    |d‰ j!        ¦  «        }||d         z  }|j        ‰ j                 j)        d          *                    ¦   «         }||z  |z   }| +                    |‰ j        ¦  «        }| '                    |‰ j        d¦  «        dd d d …f         }|                      |‰ j        ‰ j	        ‰ j        z  |j         d         ¦  «         (                    ¦   «         }| '                    |d|j         d         ¦  «        }|                     |j        ¦  «        }| ,                    |‰ j	        z  ‰ j!        ‰ j        ¦  «        }| ,                    |‰ j	        z  ‰ j        d¦  «        }t#          j-        ||¦  «        }| ,                    |‰ j	        ‰ j!        ¦  «        }‰ j.        d                               ‰ j.        j         d         ‰ j!        ¦  «        }|||z  z                        |j        ¦  «        }| '                    |d¦  «        d d …d df         }�n~t2          j         #                    |‰ j"        z   ¦  «        }t#          j$        |‰ j%        ¦  «        }| '                    ||d‰ j!        ¦  «                             ¦   «         }| '                    ||d‰ j        ¦  «                             ¦   «         }| '                    ||d‰ j        ¦  «                             ¦   «         }| /                    ‰ j	        ‰ j        z  d‰ j	        ¬
¦  «        }| /                    ‰ j	        ‰ j        z  d‰ j	        ¬
¦  «        }‰ j0        |‰ j0        z  z
  ‰ j0        z  Š3‰ j.        d         tc          |‰3¦  «        z  }||d         z  }|                     |j        ¦  «        |z  }ˆ3ˆ fd„||||fD ¦   «         \  }}}}| 2                    dddd¦  «        }t#          j3        |d¬¦  «        }t#          j        ti          |¦  «        ¦  «        }|d d …d d …d d …d d d …d d …f         |d d …d d …d d d …d d …d d …f         z  }|                     d¬¦  «        } | d         | 2                    ddddd¦  «        d         z  }!|!                     d¬¦  «        }"|"d         |d d …d d …d f         z                       d¦  «        }#t#          j        |d d …d d …d d …dd …f         |z
  ¦  «        }$||$ 2                    dddd¦  «        d         z  }%|% 2                    ddddd¦  «        d         | 2                    ddddd¦  «        dd d d …f         z                       d¬¦  «         2                    ddddd¦  «        }&|rF|j        ‰ j                 j)        d         d d …d f                              |&j        |&j5        ¬¦  «        nt#          j6        |&d d …d d…f         ¦  «        }'t#          j        |'|&gd¬¦  «        }&t#          j        ti          t2          j                             |d d …d d …d d …df         d¦  «        ¦  «        ¦  «        }(|& 2                    ddddd¦  «        })|(d         |)d d …d d …d df         z                       d¬¦  «        }*|* 2                    ddddd¦  «        }+|+d d …d d…f         |+d d …df         },}&t#          j        |¦  «        }-|dd d d …f         |&d d …d d …d df         z  }.|- 2                    dddd¦  «        }/|.                     d¦  «        |/d         z  }0|#|0z   }| '                    |d‰ j	        ‰ j!        ¦  «        }||z   }‰3dk    r|d d …d |…d d …d d …f         }| '                    ||d¦  «        }|,�|�| +                    |,‰ j        ¦  «         ‰  7                    ||
¦  «        }1‰  8                    |1                     |¦  «        ¦  «        }2|2S )Nr;   r:   r…   r#   r   .r  ).NNrw   )r~   Úoutput_sizec                 ó<   •— g | ]}t          |‰‰j        ¦  «        ‘ŒS rË   )r  r  )rÍ   Útrú   r2   s     €€r6   ú
<listcomp>z2Zamba2MambaMixer.torch_forward.<locals>.<listcomp>%  s)   ø€ Ð%zÐ%zÐ%zÐ\]Õ&9¸!¸XÀtÄÑ&WÔ&WÐ%zÐ%zÐ%zr7   r   rü   )r=   rq   )r#   r   )9rB   r=   rZ  rÄ   r6  r>   r$  r+  r   r.  r_  r2  rŠ   r[  r\  ra  r"  r-   Úsumr4  r/   r%  rÉ   r(  r‹   r   r@   r   r`  r9  r^   Úndimr†   rv   r7  ÚsoftplusÚclampr0  r?   r›   r®   r]  rp   rb  rC   Úbmmr;  Úrepeat_interleaver  r  Úpermuter  r  rq   Ú
zeros_liker:  r<  )4r2   Úinput_statesrJ  r¥   rc  rs   rd  r=   rs  rk  rH   rG   rn  Úuse_precomputed_staterh  r\  ro  rp  rG  r7  ÚdAÚdBÚdBxÚ
ssm_statesÚssm_states_reshapedÚ
C_reshapedÚyr;  Ú
D_residualÚA_cumsumÚLÚG_intermediateÚGÚM_intermediateÚMÚY_diagÚdecay_statesÚB_decay_contractionÚstatesÚprevious_statesÚdecay_chunkÚstates_permutedÚresultÚ
new_statesrv  Ústate_decay_outÚC_times_statesÚstate_decay_out_permutedÚY_offry  Úcontextualized_statesrú   s4   `                                                  @r6   Útorch_forwardzZamba2MambaMixer.torch_forward¬  s  øø€ Ø!-Ô!3Ñˆ
�G˜QØÔ"ˆàÐ#¨×(GÒ(GÈÌÑ(WÔ(WÐ#Ø#Ÿ|š|¨LÑ9Ô9ÐÐàÐ)à ,¨~¸a¸a¸aÀÀÀÀD¸jÔ/IÑ I×MÒMÈeÑTÔT�Ø#Ÿ|š|¨LÑ9Ô9ÐØ!Ô'¨Ô+¨a°$Ô2HÑ.HÑHÈ1ÈtÌ}ÑK\Ð_cÔ_rÑKrÑrÐtxô  uCñ  Cð  HIñ  IˆØ(8×(>Ò(>Ø˜˜tÔ5¸¼ÀtÄ~ÐVÐ\^ð )?ñ )
ô )
Ñ%ˆˆ1ˆd�M 2ð &×/Ò/°°1Ñ5Ô5ˆà ,°DÐ 8Ð l¸\×=\Ò=\Ð]aÔ]kÑ=lÔ=lÐØ ð 	LØ%Ô,¨T¬^Ô<ÔHÈÔKˆJð !ð 	W W°¢\ \Ø&×8Ò8¸ÈÌÑWÔWÐX[Ð^bÔ^sÐ]sÐ]tÐ]tÐXtÔuˆKÝ!œI k°D´KÔ4FÀqÀqÀqÈ!ÈQÈQÈQÀwÔ4OÑ&OÐUWÐXÑXÔXˆMØÔ!ð 2Ø ¤Ô!1Ñ1�Ø ŸHšH ]Ñ3Ô3×6Ò6°uÑ=Ô=¸a¸a¸aÀÀs¸lÔKˆM‰Mà$ð Oå %¤	¨:°}Ð*EÈ2Ð NÑ NÔ N�ØÐ'Ý œm×/Ò/Ø!ØÔ*¨]Ô-@ÀÔ-DÑDÀaÐHñô �ð +×<Ò<¸[È$Ì.ÑYÔYÐZ]Ð`dÔ`uÐ_uÐ_vÐ_vÐZvÔw�à ŸHšH T§[¢[°Ñ%?Ô%?ÀÐE]ÀmÔFYÐZ\ÔF]ÐE]Ð@]Ô%^×%hÒ%hÐijÐlmÑ%nÔ%nÑoÔoˆMØ$ð ?Ø -¨a¨a¨a°'°°°¸A¸A¸A¨oÔ >�ØÐ)Ø%Ô+�à!.°ÀÀÀÀ1À1À1ÀdÀ
Ô1KÑ!K× OÒ OÐPUÑ VÔ V�å#œk¨-¸$Ô:PÐRVÔR_ÐbfÔbuÑRuÐw{ô  xEð  HLô  H[ñ  x[ð  :\ð  bdð  eñ  eô  eÑˆ�q˜!ÝŒY�t”z×'Ò'Ñ)Ô)Ñ*Ô*Ð*ˆØ ñ E	O W°¢\¡\ð &(¤W°¢\ \��A�A�A�t˜S�LÔ!Ð!°r¸!¸!¸!¸QÀÀÀ¸'´{À1À1À1ÀdÈCÀ<Ô7PˆBØ—’˜a Ñ#Ô#×*Ò*¨:°r´xÀ´|ÀTÄ]ÑSÔSˆBà”l 9Ô-×4Ò4°T´\Ô5GÈÔ5JÈDÌMÑZÔZˆGå”Ô$×-Ò-¨b°7·:²:¸b¼hÑ3GÔ3GÑ.GÑHÔHˆBÝ”˜R Ô!3Ñ4Ô4ˆBØ�/Ô"×)Ò)¨$¬.¸$¼-ÈÔI\Ñ]Ô]×`Ò`ÕglÔgtÐ`ÑuÔuˆAå”˜2˜iœ=¨1Ñ,Ñ-Ô-ˆBð
 —	’	˜* d¤m°RÑ8Ô8¸¸dÀAÀAÀA¸ÔFˆAØ—’˜ T¤]°D´NÀdÄmÑ4SÐUVÔU\Ð]_ÔU`ÑaÔa×lÒlÑnÔnˆAØ—	’	˜* b¨!¬'°"¬+Ñ6Ô6ˆAà�I”  3¨¨a¨a¨a <¤Ñ0ˆBð *×1Ò1°*¸bÀ$Ä-ÑPÔPˆMØ�} YÔ/Ñ/ˆCð &Ô,¨T¬^Ô<ÔMÈaÔP×VÒVÑXÔXˆJØ# b™¨3Ñ.ˆJØ%×<Ò<¸ZÈÌÑXÔXˆJð —	’	˜* d¤m°RÑ8Ô8¸¸dÀAÀAÀA¸ÔFˆAØ—’˜ T¤]°D´NÀdÄmÑ4SÐUVÔU\Ð]_ÔU`ÑaÔa×lÒlÑnÔnˆAØ—	’	˜* b¨!¬'°"¬+Ñ6Ô6ˆAð $Ÿš q¤wÑ/Ô/ˆJà",§/¢/°*¸t¼~Ñ2MÈtÌ}Ð^bÔ^qÑ"rÔ"rÐØŸš 
¨T¬^Ñ ;¸TÔ=PÐRSÑTÔTˆJÝ”	Ð-¨zÑ:Ô:ˆAØ—’�z 4¤>°4´=ÑAÔAˆAð ”�yÔ!×(Ò(¨¬¬°a¬¸$¼-ÑHÔHˆAØ�] QÑ&Ñ&×*Ò*¨1¬7Ñ3Ô3ˆAð —	’	˜* bÑ)Ô)¨!¨!¨!¨T°3¨,Ô7ˆA‰Aõ ”×'Ò'¨¨T¬\Ñ(9Ñ:Ô:ˆBÝ”˜R Ô!3Ñ4Ô4ˆBØ)×1Ò1°*¸gÀrÈ4Ì=ÑYÔY×_Ò_ÑaÔaˆMØ—	’	˜* g°°DÔ4GÑHÔH×NÒNÑPÔPˆAØ—	’	˜* g¨r°4Ô3FÑGÔG×MÒMÑOÔOˆAØ×#Ò# D¤N°d´mÑ$CÈÐX\ÔXfÐ#ÑgÔgˆAØ×#Ò# D¤N°d´mÑ$CÈÐX\ÔXfÐ#ÑgÔgˆAØœ¨'°D´OÑ*CÑCÀtÄÑVˆHàœ 	Ô*Õ-?ÀÈxÑ-XÔ-XÑXˆJð *¨B¨y¬MÑ9ˆMØ—’�]Ô(Ñ)Ô)¨BÑ.ˆAð &{Ð%zÐ%zÐ%zÐ%zÐboÐqrÐtuÐwxÐayÐ%zÑ%zÔ%zÑ"ˆM˜1˜a ð —	’	˜!˜Q  1Ñ%Ô%ˆAÝ”| A¨2Ð.Ñ.Ô.ˆHõ ”	�+ a™.œ.Ñ)Ô)ˆAð ˜q˜q˜q ! ! ! Q Q Q¨¨a¨a¨a°°°Ð2Ô3°a¸¸¸¸1¸1¸1¸dÀAÀAÀAÀqÀqÀqÈ!È!È!Ð8KÔ6LÑLˆNØ×"Ò" rÐ"Ñ*Ô*ˆAð ˜yœ\¨A¯IªI°a¸¸A¸qÀ!Ñ,DÔ,DÀYÔ,OÑOˆNØ×"Ò" rÐ"Ñ*Ô*ˆAð ˜	”l ]°1°1°1°a°a°a¸°:Ô%>Ñ>×CÒCÀAÑFÔFˆFõ !œ9 X¨a¨a¨a°°°°A°A°A°r°s°s¨lÔ%;¸hÑ%FÑGÔGˆLØ"# l×&:Ò&:¸1¸aÀÀAÑ&FÔ&FÀyÔ&QÑ"QÐà)×1Ò1°!°Q¸¸1¸aÑ@Ô@ÀÔKÈ}×OdÒOdÐefÐhiÐklÐnoÐqrÑOsÔOsÐtwÐy}ð  @Að  @Að  @Að  uAô  PBñ  B÷  Gò  Gð  LMð  Gñ  Nô  N÷  Vò  Vð  WXð  Z[ð  ]^ð  `að  cdñ  eô  eˆFð )ð5�Ô# D¤NÔ3ÔDÀQÔGÈÈÈÈ4ÈÔP×SÒSÐZ`ÔZfÐouÔo|ÐSÑ}Ô}Ð}åÔ% f¨Q¨Q¨Q°°°¨U¤mÑ4Ô4ð õ
 ”Y °Ð8¸aÐ@Ñ@Ô@ˆFÝœ)¥Kµ´×0AÒ0AÀ(È1È1È1ÈaÈaÈaÐQRÐQRÐQRÐTVÈ;ÔBWÐY_Ñ0`Ô0`Ñ$aÔ$aÑbÔbˆKà$Ÿnšn¨Q°°1°a¸Ñ;Ô;ˆOØ! /Ô2°_ÀQÀQÀQÈÈÈÈ4ÐQTÀ_Ô5UÑU×ZÒZÐ_`ÐZÑaÔaˆFØŸš¨¨1¨a°°AÑ6Ô6ˆJØ *¨1¨1¨1¨c¨r¨c¨6Ô 2°J¸q¸q¸qÀ"¸uÔ4E�IˆFõ $œi¨Ñ1Ô1ˆOà  T¨1¨1¨1 œo°°q°q°q¸!¸!¸!¸TÀ3°Ô0GÑGˆNØ'6×'>Ò'>¸qÀ!ÀQÈÑ'JÔ'JÐ$Ø#×'Ò'¨Ñ+Ô+Ð.FÀyÔ.QÑQˆEð ˜‘ˆAà—	’	˜* b¨$¬.¸$¼-ÑHÔHˆAà�J‘ˆAà˜!Š|ˆ|Ø�a�a�a˜˜'˜ 1 1 1 a a aÐ'Ô(�Ø—	’	˜* g¨rÑ2Ô2ˆAØÐ$¨Ð)AØ×3Ò3°I¸t¼~ÑNÔNÐNà—i’i  4Ñ(Ô(ˆð
 !%§¢¨k¯nªn¸UÑ.CÔ.CÑ DÔ DÐØ$Ð$r7   r4  c                 ó¸   — t           r=d| j        j        j        j        v r%t          ¦   «         s|                      |||¦  «        S |                      |||¦  «        S )NÚcuda)rA  r6  r/   rq   rˆ   r   rz  r¦  )r2   rG   rJ  r¥   r¯   s        r6   rO   zZamba2MambaMixer.forwardm  s^   € õ "ð 	Z f°´Ô0CÔ0JÔ0OÐ&OÐ&OÕXpÑXrÔXrÐ&OØ×,Ò,¨]¸LÈ.ÑYÔYÐYà×!Ò! -°¸~ÑNÔNÐNr7   r)   ©NN)rQ   rR   rS   rø   r$   r—   r+   r-   r_   r
   rz  r¦  r   rO   rT   rU   s   @r6   r  r  –  s[  ø€ € € € € ðð ðd8ð d8˜|ð d8¸¸d¹
ð d8ð d8ð d8ð d8ð d8ð d8ðR &*Ø.2ð	eð eà”|ðeð ˜d‘lðeð œ tÑ+ð	eð eð eð eðP~%ð ~%¸À¹ð ~%Ð[`Ô[gÐjnÑ[nð ~%ð ~%ð ~%ð ~%ðB Ð˜HÑ%Ô%ð &*Ø.2ð	
Oð 
Oð ˜d‘lð
Oð œ tÑ+ð	
Oð 
Oð 
Oñ &Ô%ð
Oð 
Oð 
Oð 
Oð 
Or7   r  c                   ó8   ‡ — e Zd Zddededz  fˆ fd„Zdd„Zˆ xZS )Ú	Zamba2MLPNrc   rÆ   c           	      ón  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        || _        || _        t          j        | j        d| j        z  |j	        ¬¦  «        | _
        t          j        | j        | j        |j	        ¬¦  «        | _        t          |j                 | _        t          j        g ¦  «        | _        t#          | j        ¦  «        D ]£}||j        z  |k    rft          j        t          j        | j        j        | j        j        d¬¦  «        t          j        | j        j        d| j        z  d¬¦  «        ¦  «        }nt          j        ¦   «         }| j                             |¦  «         Œ¤|j        }d„ t1          |¦  «        D ¦   «         | _        dS )aQ  
        This MLP layer contributes to tied transformer blocks aimed to increasing compute without increasing model size. Because this layer
        is tied, un-tied adapter modules (formally same as LoRA, but used in the base model) are added to the up and gate projectors to increase expressivity with a small memory overhead.
        r:   rÈ   Fc                 ó   — i | ]\  }}||“Œ	S rË   rË   rÌ   s      r6   rÏ   z&Zamba2MLP.__init__.<locals>.<dictcomp>˜  s   € ÐVÐVÐV©<¨5°%˜% ÐVÐVÐVr7   N)r*   r+   rc   r3   r$  rÅ   rÆ   r   rÔ   r5  Úgate_up_projÚ	down_projr	   Ú
hidden_actÚact_fnrÜ   Úgate_up_proj_adapter_listrà   rá   râ   rã   rä   rå   rÙ   ræ   rç   )r2   rc   rÅ   rÆ   rè   Úgate_up_proj_adapterrÚ   r5   s          €r6   r+   zZamba2MLP.__init__|  s‰  ø€ õ
 	‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔØ"4ˆÔØ ˆŒåœI dÔ&6¸¸DÔ<RÑ8RÐY_ÔYoÐpÑpÔpˆÔÝœ 4Ô#9¸4Ô;KÐRXÔRhÐiÑiÔiˆŒÝ˜VÔ.Ô/ˆŒå)+¬°rÑ):Ô):ˆÔ&Ý�tÔ.Ñ/Ô/ð 	Hð 	HˆAØ�6Ô(Ñ(¨HÒ4Ð4Ý')¤}Ý”I˜dœkÔ5°t´{Ô7OÐV[Ð\Ñ\Ô\Ý”I˜dœkÔ6¸¸DÔ<RÑ8RÐY^Ð_Ñ_Ô_ñ(ô (Ð$Ð$õ
 (*¤{¡}¤}Ð$ØÔ*×1Ò1Ð2FÑGÔGÐGÐGà Ô1ˆØVÐV½9À_Ñ;UÔ;UÐVÑVÔVˆŒˆˆr7   c                 ó  — |                       |¦  «        }| j        |         }| | j        |         |¦  «        z   }t          j        |dd¬¦  «        }|                      |d         ¦  «        |d         z  }|                      |¦  «        }|S )Nr:   r;   r…   r   r#   )r®  rç   r²  r-   Úchunkr±  r¯  )r2   Úhidden_staterÄ   Úgate_up_stateÚoutputs        r6   rO   zZamba2MLP.forwardš  s‹   € Ø×)Ò)¨,Ñ7Ô7ˆØ”N 9Ô-ˆ	Ø%Ð(Q¨Ô(FÀyÔ(QÐR^Ñ(_Ô(_Ñ_ˆåœ M°1¸"Ð=Ñ=Ô=ˆØ—{’{ =°Ô#3Ñ4Ô4°}ÀQÔ7GÑGˆØ—’ Ñ-Ô-ˆØˆr7   r©  r)   )rQ   rR   rS   r$   r—   r+   rO   rT   rU   s   @r6   r«  r«  {  st   ø€ € € € € ðWð W˜|ð WÐPSÐVZÑPZð Wð Wð Wð Wð Wð Wð<ð ð ð ð ð ð ð r7   r«  c                   óÆ   ‡ — e Zd Zddededz  dedz  fˆ fd„Z	 	 	 ddej        dej        dedej        dz  d	edz  d
ej	        dz  de
e         deej                 fd„Zˆ xZS )ÚZamba2AttentionDecoderLayerNrc   rÆ   rÄ   c                 ó\  •— t          ¦   «                              ¦   «          || _        t          |j        ¦  «        }t          |d||¬¦  «        | _        t          |||¬¦  «        | _        t          |j
        |j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        d S )Nr;   )rÄ   rÅ   rÆ   )rÅ   rÆ   ©r4   )r*   r+   rÆ   rÿ   rÙ   rÃ   Ú	self_attnr«  Úfeed_forwardrW   rÐ   Úrms_norm_epsÚinput_layernormr3   Úpre_ff_layernorm)r2   rc   rÆ   rÄ   Únum_gsr5   s        €r6   r+   z$Zamba2AttentionDecoderLayer.__init__¦  sž   ø€ Ý‰Œ×ÒÑÔÐØ ˆŒÝ�VÔ,Ñ-Ô-ˆÝ(¨¸2ÐRXÐckÐlÑlÔlˆŒÝ% fÀÐRZÐ[Ñ[Ô[ˆÔÝ,¨VÔ-IÈvÔObÐcÑcÔcˆÔÝ -¨fÔ.@ÀfÔFYÐ ZÑ ZÔ ZˆÔÐÐr7   rG   Úoriginal_hidden_statesr¥   rì   rí   r¯   rX   c           	      óâ   — t          j        ||gd¬¦  «        }|                      |¦  «        } | j        d|||||dœ|¤Ž\  }}|                      |¦  «        }|                      ||¦  «        }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`): output of previous Mamba layer of shape `(batch, seq_len, embed_dim)`
            original_hidden_states (`torch.FloatTensor`): word embedding output of shape `(batch, seq_len, embed_dim)`.
                This is concatenated with `hidden_states` (which is the output of the previous (mamba) layer). The
                concatenated tensor is then used as input of the pre-attention RMSNorm
                (see fig. 2 in https://huggingface.co/papers/2405.16712).
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
        r;   r…   )rG   rÄ   r¥   rì   rí   rË   )r-   ÚconcatenaterÀ  r½  rÁ  r¾  )	r2   rG   rÃ  rÄ   r¥   rì   rí   r¯   rd  s	            r6   rO   z#Zamba2AttentionDecoderLayer.forward¯  sž   € õ6 Ô)¨=Ð:PÐ*QÐWYÐZÑZÔZˆØ×,Ò,¨]Ñ;Ô;ˆØ)˜4œ>ð 
Ø'ØØ)Ø+Ø 3ð
ð 
ð ð
ð 
Ñˆ�qð ×-Ò-¨mÑ<Ô<ˆØ×)Ò)¨-¸ÑCÔCˆàÐr7   r©  r”   )rQ   rR   rS   r$   r—   r+   r-   r_   r
   Ú
LongTensorr   r   r\   ÚFloatTensorrO   rT   rU   s   @r6   rº  rº  ¥  s  ø€ € € € € ð[ð [˜|ð [°s¸T±zð [ÐUXÐ[_ÑU_ð [ð [ð [ð [ð [ð [ð /3Ø(,Ø7;ð)ð )à”|ð)ð !&¤ð)ð ð	)ð
 œ tÑ+ð)ð  ™ð)ð #Ô-°Ñ4ð)ð Ð+Ô,ð)ð 
ˆuÔ Ô	!ð)ð )ð )ð )ð )ð )ð )ð )r7   rº  c                   ó*  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 	 	 ddej        dej        dz  dedz  dej        dz  d	ej        dz  d
edz  de	dz  dej
        dz  dej        dz  dee         deej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚZamba2MambaDecoderLayerrc   rÄ   c                 óÂ   •— t          ¦   «                              ¦   «          t          ||¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        || _        d S )N)rc   rÄ   r¼  )	r*   r+   r  ÚmambarW   r3   r¿  rÀ  rÄ   )r2   rc   rÄ   r5   s      €r6   r+   z Zamba2MambaDecoderLayer.__init__Ü  sS   ø€ Ý‰Œ×ÒÑÔÐÝ%¨V¸yÐIÑIÔIˆŒ
Ý,¨VÔ-?ÀVÔEXÐYÑYÔYˆÔØ"ˆŒˆˆr7   NFrG   rÃ  r¥   Úcausal_maskrì   Ú	use_cacher�   Útransformer_hidden_statesr¯   rX   c
                 ór   — |}|	�||	z   n|}|                       |¦  «        } | j        d|||dœ|
¤Ž}||z   }|S )aX  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
        N)rG   rJ  r¥   rË   )rÀ  rË  )r2   rG   rÃ  rÄ   r¥   rÌ  rì   rÍ  r�   rÎ  r¯   Úresiduals               r6   rO   zZamba2MambaDecoderLayer.forwardâ  s|   € ð0 !ˆð
 :SÐ9^ˆMÐ5Ñ5Ð5Ðdqð 	ð ×,Ò,¨]Ñ;Ô;ˆà"˜œ
ð 
Ø'Ø(Ø)ð
ð 
ð ð	
ð 
ˆð ! =Ñ0ˆàÐr7   ©NNNNNFNN)rQ   rR   rS   r$   r—   r+   r-   r_   r
   r  rÆ  r   r   r\   rÇ  rO   rT   rU   s   @r6   rÉ  rÉ  Û  sE  ø€ € € € € ð#˜|ð #¸ð #ð #ð #ð #ð #ð #ð 7;Ø $Ø.2Ø+/Ø(,Ø!&Ø04Ø9=ð*ð *à”|ð*ð !&¤¨tÑ 3ð*ð ˜‘:ð	*ð
 œ tÑ+ð*ð ”\ DÑ(ð*ð  ™ð*ð ˜$‘;ð*ð Ô&¨Ñ-ð*ð $)¤<°$Ñ#6ð*ð Ð+Ô,ð*ð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uð*ð *ð *ð *ð *ð *ð *ð *r7   rÉ  c                   ó8  ‡ — e Zd Zdedej        defˆ fd„Z	 	 	 	 	 	 	 	 ddej	        dej	        dz  d	e
dz  d
ej	        dz  dej	        dz  dedz  dedz  dej        dz  dej        dz  dee         deej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚZamba2HybridLayerÚshared_transformerÚlinearrË  c                 ór   •— t          ¦   «                              ¦   «          || _        || _        || _        d S r)   )r*   r+   rÕ  Úmamba_decoderrÔ  )r2   rÔ  rÕ  rË  r5   s       €r6   r+   zZamba2HybridLayer.__init__  s8   ø€ õ 	‰Œ×ÒÑÔÐØˆŒØ"ˆÔØ"4ˆÔÐÐr7   NFrG   rÃ  rÄ   r¥   rÌ  rì   rÍ  rí   r�   r¯   rX   c
           
      ó‚   —  | j         |f||||||	dœ|
¤Ž}|                      |¦  «        } | j        |f|||||dœ|
¤Ž}|S )ap  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            original_hidden_states (`torch.FloatTensor`): word embedding output that will be concatenated with
            hidden activations to form the input of the shared transformer layer.
            layer_idx (`int`): layer number.
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
        )rÃ  rÄ   r¥   rì   rí   r�   )rÎ  r¥   rì   rÍ  rí   )rÔ  rÕ  r×  )r2   rG   rÃ  rÄ   r¥   rÌ  rì   rÍ  rí   r�   r¯   rÎ  s               r6   rO   zZamba2HybridLayer.forward  s™   € ð< %< DÔ$;Øð	%
à#9ØØ&Ø+Ø 3Ø%ð	%
ð 	%
ð ð	%
ð 	%
Ð!ð %)§K¢KÐ0IÑ$JÔ$JÐ!à*˜Ô*Øð
à&?Ø)Ø+ØØ 3ð
ð 
ð ð
ð 
ˆð Ðr7   rÑ  )rQ   rR   rS   rº  r   rÔ   rÉ  r+   r-   r_   r—   r
   r  rÆ  r   r   r\   rÇ  rO   rT   rU   s   @r6   rÓ  rÓ    sR  ø€ € € € € ð5Ø"=ð5ØGIÄyð5ØYpð5ð 5ð 5ð 5ð 5ð 5ð 7;Ø $Ø.2Ø+/Ø(,Ø!&Ø7;Ø04ð4ð 4à”|ð4ð !&¤¨tÑ 3ð4ð ˜‘:ð	4ð
 œ tÑ+ð4ð ”\ DÑ(ð4ð  ™ð4ð ˜$‘;ð4ð #Ô-°Ñ4ð4ð Ô&¨Ñ-ð4ð Ð+Ô,ð4ð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uð4ð 4ð 4ð 4ð 4ð 4ð 4ð 4r7   rÓ  c                   ó„   ‡ — e Zd ZU eed<   dZdZddgZdgZdZ	dZ
dZdZeedœZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )	ÚZamba2PreTrainedModelrc   ÚmodelTrÓ  rÉ  rì   )rG   Ú
attentionsc                 ó  •— t          ¦   «                              |¦  «         t          |t          ¦  «        �rTt	          j        t	          j        | j        j        ¦  «        t          j
        | j        j        ¦  «        t          j
        | j        j        ¦  «        z
  z  t          j
        | j        j        ¦  «        z   ¦  «                             | j        j        ¬¦  «        }|t	          j
        t	          j        | ¦  «         ¦  «        z   }t!          j        |j        |¦  «         t	          j        d|j        dz   ¦  «        }t!          j        |j        t	          j
        |¦  «        ¦  «         t!          j        |j        ¦  «         d S d S )N)Úminr#   )r*   Ú_init_weightsr‡   r  r-   r`  Úrandrc   r-  Úmathr8  r1  r0  rƒ  Útime_step_floorÚexpm1ÚinitÚcopy_r7  r{   r.  r9  Úones_r;  )r2   r¡   rn  Úinv_dtrG  r5   s        €r6   rß  z#Zamba2PreTrainedModel._init_weights_  s=  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ.Ñ/Ô/ñ 	!Ý”Ý”
˜4œ;Ô4Ñ5Ô5Ý”8˜DœKÔ5Ñ6Ô6½¼À$Ä+ÔB[Ñ9\Ô9\Ñ\ñ^å”(˜4œ;Ô4Ñ5Ô5ñ6ñô ÷ Še˜œÔ3ˆeÑ4Ô4ð	 ð �%œ)¥U¤[°"°Ñ%5Ô%5Ð$5Ñ6Ô6Ñ6ˆFÝŒJ�v”~ vÑ.Ô.Ð.å”˜Q Ô 0°1Ñ 4Ñ5Ô5ˆAÝŒJ�v”|¥U¤Y¨q¡\¤\Ñ2Ô2Ð2ÝŒJ�v”xÑ Ô Ð Ð Ð ð	!ð 	!r7   )rQ   rR   rS   r$   r•   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_sdpaÚ_is_statefulrÉ  rÃ   Ú_can_record_outputsr-   r˜   rß  rT   rU   s   @r6   rÚ  rÚ  O  s¤   ø€ € € € € € àÐÐÑØÐØ&*Ð#Ø,Ð.GÐHÐØ#4Ð"5ÐØÐØÐØ€NØ€Là0Ø%ðð Ðð
 €U„]�_„_ð!ð !ð !ð !ñ „_ð!ð !ð !ð !ð !r7   rÚ  c                   óò   ‡ — e Zd ZdZdefˆ fd„Zeee	 	 	 	 	 	 dde	j
        dz  de	j        dz  de	j
        dz  dedz  d	e	j        dz  d
edz  dee         deez  fd„¦   «         ¦   «         ¦   «         Zd„ Zˆ xZS )ÚZamba2Modelzh
    Model consisting of *config.num_hidden_layers* layers.

    Args:
        config: Zamba2Config
    rc   c                 ó,  •— t          ¦   «                              |¦  «         || _        |j        | _        |j        | _        t          j        |j        |j        | j        ¦  «        | _	        |j
        | _
        |                      ¦   «         | _        |j        | _        t          |j        |j        ¬¦  «        | _        |j        r5|j        rt&                               d¦  «         t+          |¦  «        | _        d| _        |                      ¦   «          d S )Nr¼  ze`use_long_context` set to `True`: using rescaled `rope_theta` and extended `max_position_embeddings`.F)r*   r+   rc   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr3   Úembed_tokensÚlayers_block_typeÚ
get_layersr[  rò   rW   r¿  Úfinal_layernormrï   Úuse_long_contextrB  rC  ra   Ú
rotary_embÚgradient_checkpointingÚ	post_init©r2   rc   r5   s     €r6   r+   zZamba2Model.__init__z  sû   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒØ!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔØ!'Ô!9ˆÔØ—o’oÑ'Ô'ˆŒà$*Ô$?ˆÔ!Ý,¨VÔ-?ÀVÔEXÐYÑYÔYˆÔØÔð 	<ØÔ&ð Ý×#Ò#Ø{ñô ð õ 4°FÑ;Ô;ˆDŒOØ&+ˆÔ#ð 	�ŠÑÔÐÐÐr7   NÚ	input_idsr¥   r�   rì   Úinputs_embedsrÍ  r¯   rX   c           	      ó˜  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|}t          j        |¦  «        }	|r|€t	          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}
t          j        |j        d         |j	        ¬¦  «        |
z   }| 
                    d¦  «        }t          | j        ||||¬¦  «        }| j        j        r|                      ||¬¦  «        }nd }t          | j        ¦  «        D ]\  }} |||	|||f||||dœ|¤Ž}Œ|                      |¦  «        }t#          ||r|nd ¬	¦  «        S )
NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either one)rc   r   r#   ©rq   )rc   r  r¥   rì   r�   )r�   )rì   rÍ  rí   r�   )Úlast_hidden_staterì   )Ú
ValueErrorrø  r-   rp   r   rc   Úget_seq_lengthr{   rB   rq   r»   r   rï   rý  ræ   r[  rû  r   )r2   r  r¥   r�   rì   r  rÍ  r¯   rG   rÃ  Úpast_seen_tokensrÌ  rí   rÄ   Úlayers                  r6   rO   zZamba2Model.forward‘  sÌ  € ð ˜Ð -°tÐ";Ñ<ð 	ÝØsñô ð ð Ð Ø ×-Ò-¨iÑ8Ô8ˆMà%ˆå!&¤¨]Ñ!;Ô!;Ðð ð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå(Ø”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆð Œ;Ô#ð 	'Ø"&§/¢/°-Èl /Ñ"[Ô"[ÐÐà"&Ðå )¨$¬+Ñ 6Ô 6ð 	ð 	ÑˆI�uØ!˜EØØ&ØØØðð !0Ø#Ø$7Ø)ðð ð ðð ˆMˆMð ×,Ò,¨]Ñ;Ô;ˆå&Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
r7   c                 ó  — g }i | _         d| _        g }t          | j        ¦  «        D �]K\  }}t	          | j        |¬¦  «        }|dk    �rd|› d�}t          |t          ¦  «        rt          |¦  «        | j        j	        k    rPt          |t          ¦  «        rt          |¦  «        }t          |¦  «        }| j                              ||i¦  «         n|                     |¦  «         || j        j	        z  }t          | j        |¬¦  «        }	t          j        | j        j        | j        j        d¬¦  «        }
|                     t%          |	|
|¦  «        ¦  «         �Œ6|                     |¦  «         �ŒMt          j        |¦  «        S )	Nr   )rÄ   Úhybridzlayers.z.shared_transformer)rÆ   FrÈ   )Ú_tied_weights_keysÚfirst_transformer_layer_idræ   rù  rÉ  rc   r‡   Úlistrÿ   rá   r   Únextrð   rå   rº  r   rÔ   r3   rÓ  rÜ   )r2   r[  Úunique_hybrid_blocksÚlayer_idrE  Úmamba_layerÚprefix_patternÚtarget_patternrÆ   Ú
attn_blockÚlinear_layers              r6   rú  zZamba2Model.get_layersÖ  s‹  € ØˆØ"$ˆÔØ*+ˆÔ'Ø!Ðå$-¨dÔ.DÑ$EÔ$Eð 	+ñ 	+Ñ ˆH�jÝ1°$´+ÈÐRÑRÔRˆKØ˜XÒ%Ñ%Ø!H¨8Ð!HÐ!HÐ!H�õ #Ð#7½Ñ>Ô>ð
@åÐ/Ñ0Ô0°D´KÔ4NÒNÐNå!Ð"6½Ñ=Ô=ð KÝ/4Ð5IÑ/JÔ/JÐ,Ý%)Ð*>Ñ%?Ô%?�NØÔ+×2Ò2°NÀNÐ3SÑTÔTÐTÐTð )×/Ò/°Ñ?Ô?Ð?à# d¤kÔ&@Ñ@�Ý8¸¼ÈxÐXÑXÔX�
Ý!œy¨¬Ô)@À$Ä+ÔBYÐ`eÐfÑfÔf�Ø—’Õ/°
¸LÈ+ÑVÔVÑWÔWÐWÑWà—’˜kÑ*Ô*Ð*Ñ*ÝŒ}˜VÑ$Ô$Ð$r7   )NNNNNN)rQ   rR   rS   rø   r$   r+   r    r"   r   r-   rÆ  r_   r
   rÇ  r  r   r   r\   r   rO   rú  rT   rU   s   @r6   rò  rò  q  s:  ø€ € € € € ðð ð˜|ð ð ð ð ð ð ð.  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ð@
ð @
àÔ# dÑ*ð@
ð œ tÑ+ð@
ð Ô&¨Ñ-ð	@
ð
  ™ð@
ð Ô(¨4Ñ/ð@
ð ˜$‘;ð@
ð Ð+Ô,ð@
ð 
Ð(Ñ	(ð@
ð @
ð @
ñ „^ñ „_ñ  Ôð@
ðD %ð  %ð  %ð  %ð  %ð  %ð  %r7   rò  c                   ó$  ‡ — e Zd ZddiZdefˆ fd„Zee	 	 	 	 	 	 	 	 ddej	        dz  dej
        dz  d	ej	        dz  d
edz  dej        dz  dej	        dz  dedz  deej
        z  dee         deez  fd„¦   «         ¦   «         Z	 	 	 	 	 	 dˆ fd„	Zˆ xZS )ÚZamba2ForCausalLMzlm_head.weightzmodel.embed_tokens.weightrc   c                 óþ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S ©NFrÈ   )
r*   r+   rò  rÛ  rö  r   rÔ   r3   Úlm_headrÿ  r   s     €r6   r+   zZamba2ForCausalLM.__init__ý  sj   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý  Ñ(Ô(ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒð 	�ŠÑÔÐÐÐr7   Nr   r  r¥   r�   rì   r  ÚlabelsrÍ  Úlogits_to_keepr¯   rX   c	           
      óD  —  | j         d||||||dœ|	¤Ž}
|
j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        ||| j        fi |	¤Ž}t          |||
j	        |
j
        |
j        ¬¦  «        S )al  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, Zamba2ForCausalLM

        >>> model = Zamba2ForCausalLM.from_pretrained("Zyphra/Zamba2-7B-v1")
        >>> tokenizer = AutoTokenizer.from_pretrained("Zyphra/Zamba2-7B-v1")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```)r  r¥   r�   rì   r  rÍ  N©ÚlossÚlogitsrì   rG   rÜ  rË   )rÛ  r  r‡   r—   Úslicer  Úloss_functionrö  r   rì   rG   rÜ  )r2   r  r¥   r�   rì   r  r  rÍ  r  r¯   ÚoutputsrG   Úslice_indicesr!  r   s                  r6   rO   zZamba2ForCausalLM.forward  sþ   € ðH ,6¨4¬:ð ,
ØØ)Ø%Ø+Ø'Øð,
ð ,
ð ð,
ð ,
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%ØØØ”ðð ð ð	ð ˆDõ &ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r7   TFc           
      óh   •— | j         j        |d<    t          ¦   «         j        |f||||||dœ|¤Ž}	|	S )Nr  )rì   r¥   r  r�   rÍ  Úis_first_iteration)rc   Únum_logits_to_keepr*   Úprepare_inputs_for_generation)r2   r  rì   r¥   r  r�   rÍ  r'  r¯   Úmodel_inputsr5   s             €r6   r)  z/Zamba2ForCausalLM.prepare_inputs_for_generationJ  s^   ø€ ð $(¤;Ô#AˆÐÑ Ø<•u‘w”wÔ<Øð	
à+Ø)Ø'Ø%ØØ1ð	
ð 	
ð ð	
ð 	
ˆð Ðr7   ©NNNNNNNr   )NNNNTF)rQ   rR   rS   r  r$   r+   r   r   r-   rÆ  r_   r
   rÇ  r  r—   r   r   r\   r   rO   r)  rT   rU   s   @r6   r  r  ú  s€  ø€ € € € € Ø*Ð,GÐHÐð˜|ð ð ð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð@
ð @
àÔ# dÑ*ð@
ð œ tÑ+ð@
ð Ô&¨Ñ-ð	@
ð
  ™ð@
ð Ô(¨4Ñ/ð@
ð Ô  4Ñ'ð@
ð ˜$‘;ð@
ð ˜eœlÑ*ð@
ð Ð+Ô,ð@
ð 
Ð'Ñ	'ð@
ð @
ð @
ñ „^ñ Ôð@
ðJ ØØØØØ ðð ð ð ð ð ð ð ð ð r7   r  aÌ  
    The Zamba2 Model with a sequence classification head on top (linear layer).

    [`Zamba2ForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-2) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    )Úcustom_introc                   ó  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 	 	 	 ddej        dz  dej	        dz  dej        dz  de
dz  d	ej        dz  d
ej        dz  dedz  deej	        z  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚZamba2ForSequenceClassificationrc   c                 óþ   •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        | j        d¬¦  «        | _        |  	                    ¦   «          d S r  )
r*   r+   Ú
num_labelsrò  rÛ  r   rÔ   r3   Úscorerÿ  r   s     €r6   r+   z(Zamba2ForSequenceClassification.__init__s  si   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒÝ  Ñ(Ô(ˆŒ
Ý”Y˜vÔ1°4´?ÈÐOÑOÔOˆŒ
ð 	�ŠÑÔÐÐÐr7   Nr   r  r¥   r�   rì   r  r  rÍ  r  r¯   rX   c	           	      ó  —  | j         |f|||||dœ|	¤Ž}
|
d         }|                      |¦  «        }|�|j        d         }n|j        d         }| j        j        €|dk    rt          d¦  «        ‚| j        j        €d}n¨|�}|| j        j        k                         |j        t          j	        ¦  «        }t          j
        |j        d         |j        t          j	        ¬¦  «        }||z                       d¦  «        }n)d}t                               | j        j        › d�¦  «         |t          j
        ||j        ¬	¦  «        |f         }d}|� | j        d|||| j        d
œ|	¤Ž}t#          |||
j        |
j        |
j        ¬¦  «        S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        )r¥   r�   rì   r  rÍ  r   Nr#   z=Cannot handle batch sizes > 1 if no padding token is defined.r;   rx   zŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r  )r!  r  Úpooled_logitsrc   r  rË   )rÛ  r1  rB   rc   rô  r  r>   rq   r-   Úint32r{   ÚargmaxrB  rC  r5   rQ   r#  r   rì   rG   rÜ  )r2   r  r¥   r�   rì   r  r  rÍ  r  r¯   Útransformer_outputsrG   r!  rc  Úlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesr3  r   s                      r6   rO   z'Zamba2ForSequenceClassification.forward|  sç  € ð( 8B°t´zØð8
à)Ø%Ø+Ø'Øð8
ð 8
ð ð8
ð 8
Ðð ,¨AÔ.ˆØ—’˜MÑ*Ô*ˆàÐ Ø"œ¨Ô+ˆJˆJà&Ô,¨QÔ/ˆJàŒ;Ô#Ð+°
¸a²°ÝÐ\Ñ]Ô]Ð]ØŒ;Ô#Ð+Ø!#ÐÐØÐ"Ø%¨¬Ô)AÒA×EÒEÀfÄmÕUZÔU`ÑaÔaˆLÝ!œL¨¬¸Ô)<ÀVÄ]ÕZ_ÔZeÐfÑfÔfˆMØ"/°,Ñ">×!FÒ!FÀrÑ!JÔ!JÐÐà!#ÐÝ×ÒØ”>Ô*ð Zð Zð Zñô ð ð
 �uœ|¨J¸v¼}ÐMÑMÔMÐOaÐaÔbˆàˆØÐØ%�4Ô%ð Ø$¨VÀ=ÐY]ÔYdðð Øhnðð ˆDõ 0ØØ Ø/Ô?Ø-Ô;Ø*Ô5ð
ñ 
ô 
ð 	
r7   r+  )rQ   rR   rS   r$   r+   r   r   r-   rÆ  r_   r
   rÇ  r  r—   r   r   r\   r   rO   rT   rU   s   @r6   r.  r.  d  sE  ø€ € € € € ð˜|ð ð ð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð@
ð @
àÔ# dÑ*ð@
ð œ tÑ+ð@
ð Ô&¨Ñ-ð	@
ð
  ™ð@
ð Ô(¨4Ñ/ð@
ð Ô  4Ñ'ð@
ð ˜$‘;ð@
ð ˜eœlÑ*ð@
ð Ð+Ô,ð@
ð 
Ð1Ñ	1ð@
ð @
ð @
ñ „^ñ Ôð@
ð @
ð @
ð @
ð @
r7   r.  )r  r.  rò  rÚ  )r    )r#   )Trá  Úcollections.abcr   Ú	itertoolsr   Útypingr   r-   r   Ú r   rä  Úactivationsr	   Úcache_utilsr
   r   Ú
generationr   Úintegrationsr   Úintegrations.accelerater   Úintegrations.hub_kernelsr   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.genericr   r    Úutils.import_utilsr!   Úutils.output_capturingr"   Úconfiguration_zamba2r$   Ú
get_loggerrQ   rB  ÚModuler&   rW   ra   r_   r—   rŸ   r^   r´   r¸   rÁ   rÃ   r  r  r  r  r«  rº  rÉ  rÓ  rÚ  rò  r  r.  Ú__all__rË   r7   r6   ú<module>rR     sç  ðð* €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4Ø =Ð =Ð =Ð =Ð =Ð =Ø 8Ð 8Ð 8Ð 8Ð 8Ð 8Ø /Ð /Ð /Ð /Ð /Ð /Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qÐ qØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lØ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø .Ð .Ð .Ð .Ð .Ð .ð 
ˆÔ	˜HÑ	%Ô	%€ð;ð ;ð ;ð ;ð ;˜œœñ ;ô ;ð ;ð*Jð Jð Jð Jð J�B”Iñ Jô Jð Jð(><ð ><ð ><ð ><ð ><˜BœIñ ><ô ><ð ><ðB	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð %ð %ð %ð2(ð (ð (ð ÐÐ*Ñ+Ô+ðð ð ñ ,Ô+ðð2y)ð y)ð y)ð y)ð y)�b”iñ y)ô y)ð y)ð~V U¤\ð V¸Sð Vð Vð Vð Vð
ð 
ð 
ð(ð ð ð(bOð bOð bOð bOð bO�r”yñ bOô bOð bOðJ'ð 'ð 'ð 'ð '�”	ñ 'ô 'ð 'ðT3ð 3ð 3ð 3ð 3 "¤)ñ 3ô 3ð 3ðl1ð 1ð 1ð 1ð 1Ð8ñ 1ô 1ð 1ðh=ð =ð =ð =ð =Ð2ñ =ô =ð =ð@ ð!ð !ð !ð !ð !˜Oñ !ô !ñ „ð!ðB ðD%ð D%ð D%ð D%ð D%Ð'ñ D%ô D%ñ „ðD%ðPgð gð gð gð gÐ-¨ñ gô gð gðT €ððñ ô ðL
ð L
ð L
ð L
ð L
Ð&;ñ L
ô L
ñô ðL
ð^ kÐ
jÐ
j€€€r7   