§
    ‚Štj'¤  ã                   óô  — d dl mZ d dlmZ d dlmZ d dlZd dlmZ d dlm	Z	 ddl
mZ ddlmZmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZmZmZmZmZ ddlm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z* ddl+m,Z,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2  e)d¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z3 G d„ dej4        ¦  «        Z5 G d„ dej4        ¦  «        Z6 G d„ dej4        ¦  «        Z7d ej8        d!e9d"ej8        fd#„Z:	 dFd%ej4        d&ej8        d'ej8        d(ej8        d)ej8        dz  d*e;d+e;d,e&e(         fd-„Z<d.„ Z=dGd/„Z> G d0„ d1ej4        ¦  «        Z? G d2„ d3e¦  «        Z@ G d4„ d5e¦  «        ZAe) G d6„ d7e$¦  «        ¦   «         ZB G d8„ d9eB¦  «        ZCe) G d:„ d;eB¦  «        ¦   «         ZDe) G d<„ d=eB¦  «        ¦   «         ZEd>ej8        d?e9d@e9fdA„ZF e)dB¬¦  «         G dC„ dDeBe¦  «        ¦   «         ZGg dE¢ZHdS )Hé    )ÚCallable)Ú	dataclass)ÚOptionalN)ÚCrossEntropyLossé   )ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPastÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚMoonshineConfigz™
    Extends [~modeling_outputs.BaseModelOutput] to include the output attention mask since sequence length is not preserved in the model's forward.
    )Úcustom_introc                   ó2   — e Zd ZU dZdZej        dz  ed<   dS )ÚMoonshineEncoderModelOutputa–  
    attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
        Mask to avoid performing attention on padding token indices after sequence compression. Returned because the
        sequence length may differ from the input sequence length. Mask values selected in `[0, 1]`:

        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.
    NÚattention_mask)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r'   ÚtorchÚTensorÚ__annotations__© ó    ún/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/moonshine/modeling_moonshine.pyr&   r&   3   s5   € € € € € € ðð ð +/€N�E”L 4Ñ'Ð.Ð.Ñ.Ð.Ð.r0   r&   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚMoonshineEncoderMLPc                 ó
  •— t          ¦   «                              ¦   «          || _        t          |         | _        t          j        |j        |j        ¦  «        | _	        t          j        |j        |j        ¦  «        | _
        d S ©N©ÚsuperÚ__init__Úconfigr   Úactivation_fnÚnnÚLinearÚhidden_sizeÚintermediate_sizeÚfc1Úfc2©Úselfr9   Ú
hidden_actÚ	__class__s      €r1   r8   zMoonshineEncoderMLP.__init__G   sc   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ# JÔ/ˆÔÝ”9˜VÔ/°Ô1IÑJÔJˆŒÝ”9˜VÔ5°vÔ7IÑJÔJˆŒˆˆr0   Úhidden_statesÚreturnc                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r5   )r?   r:   r@   )rB   rE   s     r1   ÚforwardzMoonshineEncoderMLP.forwardN   s=   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆØÐr0   ©r(   r)   r*   r8   r,   r-   rH   Ú__classcell__©rD   s   @r1   r3   r3   F   sc   ø€ € € € € ðKð Kð Kð Kð Kð U¤\ð °e´lð ð ð ð ð ð ð ð r0   r3   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚMoonshineDecoderMLPc                 ó  •— t          ¦   «                              ¦   «          || _        t          |         | _        t          j        |j        |j        dz  ¦  «        | _	        t          j        |j        |j        ¦  «        | _
        d S )Né   r6   rA   s      €r1   r8   zMoonshineDecoderMLP.__init__V   sh   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ# JÔ/ˆÔÝ”9˜VÔ/°Ô1IÈAÑ1MÑNÔNˆŒÝ”9˜VÔ5°vÔ7IÑJÔJˆŒˆˆr0   rE   rF   c                 ó¾   — |                       |¦  «        }|                     dd¬¦  «        \  }}|                      |¦  «        |z  }|                      |¦  «        }|S )NrO   éÿÿÿÿ©Údim)r?   Úchunkr:   r@   )rB   rE   Úgates      r1   rH   zMoonshineDecoderMLP.forward]   s_   € ØŸš Ñ/Ô/ˆØ+×1Ò1°!¸Ð1Ñ<Ô<Ñˆ�tØ×*Ò*¨4Ñ0Ô0°=Ñ@ˆØŸš Ñ/Ô/ˆØÐr0   rI   rK   s   @r1   rM   rM   U   sc   ø€ € € € € ðKð Kð Kð Kð Kð U¤\ð °e´lð ð ð ð ð ð ð ð r0   rM   c                   óÔ   ‡ — e Zd ZU ej        ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d         de
dz  ded	ef         fd
„¦   «         Z ej        ¦   «         ed„ ¦   «         ¦   «         Zˆ xZS )ÚMoonshineRotaryEmbeddingÚinv_freqNr9   c                 ó²  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        |                      d|d¬¦  «         |                      d|                     ¦   «         d¬¦  «         d S )NÚ	rope_typeÚdefaultrX   F)Ú
persistentÚoriginal_inv_freq)r7   r8   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr9   Úrope_parametersrZ   Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)rB   r9   ÚdeviceÚrope_init_fnrX   rD   s        €r1   r8   z!MoonshineRotaryEmbedding.__init__h   sÊ   ø€ Ý‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(à×Ò˜Z¨¸eÐÑDÔDÐDØ×ÒÐ0°(·.².Ñ2BÔ2BÈuÐÑUÔUÐUÐUÐUr0   rf   ztorch.deviceÚseq_lenrF   ztorch.Tensorc                 óV  — | j         d         }| j                              dd¦  «        }t          | dd¦  «        p| j        | j        z  }t          ||z  ¦  «        }d}d|t          j        d|dt          j        ¬¦  «         	                    |t          j
        ¬	¦  «        |z  z  z  }||fS )
a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚpartial_rotary_factorg      ð?Úhead_dimNr   rO   ©Údtype)rf   rn   )ra   ÚgetÚgetattrr=   Únum_attention_headsÚintr,   ÚarangeÚint64ÚtoÚfloat)	r9   rf   rh   Úbaserk   rl   rS   Úattention_factorrX   s	            r1   rb   z8MoonshineRotaryEmbedding.compute_default_rope_parametersx   sº   € ð& Ô% lÔ3ˆØ &Ô 6× :Ò :Ð;RÐTWÑ XÔ XÐÝ˜6 :¨tÑ4Ô4Ðh¸Ô8JÈfÔNhÑ8hˆÝ�(Ð2Ñ2Ñ3Ô3ˆàÐð Ø•U”\ ! S¨!µ5´;Ð?Ñ?Ô?×BÒBÈ&ÕX]ÔXcÐBÑdÔdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r0   c                 óN  — | j         d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  |                     ¦   «         |                     ¦   «         z   
                    dd¦  «        }t          j        ||fd¬	¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬
¦  «        |	                     |j        ¬
¦  «        fS )Nr   rQ   r"   ÚmpsÚcpuF)Údevice_typeÚenabledrO   rR   rm   )rX   rv   ÚexpandÚshaperu   rf   Ú
isinstanceÚtypeÚstrr   Ú	transposer,   ÚcatÚcosrc   Úsinrn   )
rB   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr|   ÚfreqsÚembr…   r†   s
             r1   rH   z MoonshineRotaryEmbedding.forward˜   s·  € ð !œM¨$°°°°4¨-Ô8×>Ò>Ñ@Ô@×GÒGÈÔHZÐ[\ÔH]Ð_aÐcdÑeÔe×hÒhÐijÔiqÑrÔrÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	5ð 	5Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃBE&Å&E*Å-E*r5   )NNN)r(   r)   r*   r,   r-   r.   r#   r8   Ústaticmethodr   rr   Útuplerv   rb   Úno_gradr   rH   rJ   rK   s   @r1   rW   rW   e   sù   ø€ € € € € € ØŒlÐÐÑðVð V˜ð Vð Vð Vð Vð Vð Vð  à)-Ø+/Ø"ð*ð *Ø $Ñ&ð*à˜Ô(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ô	%ð	*ð *ð *ñ „\ð*ð> €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r0   rW   rE   Ún_reprF   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r"   N)r   r~   Úreshape)rE   r�   ÚbatchÚnum_key_value_headsÚslenrl   s         r1   Ú	repeat_kvr–   ¨   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr0   ç        ÚmoduleÚqueryÚkeyÚvaluer'   ÚscalingÚdropoutÚkwargsc                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )NrO   r   rQ   )rS   rn   )ÚpÚtrainingr"   )r–   Únum_key_value_groupsr,   Úmatmulrƒ   r;   Ú
functionalÚsoftmaxÚfloat32ru   rn   r�   r¡   Ú
contiguous)r˜   r™   rš   r›   r'   rœ   r�   rž   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r1   Úeager_attention_forwardr¬   ´   sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r0   c                 óŽ   — | dddd…f         }| dddd…f         }t          j        | |fd¬¦  «                             d¦  «        S )	z*Rotates half the hidden dims of the input..r   NrO   r"   rQ   rR   éþÿÿÿ)r,   ÚstackÚflatten)r‡   Úx1Úx2s      r1   Úrotate_halfr³   Í   sQ   € à	
ˆ3���1�ˆ9Œ€BØ	
ˆ3���1�ˆ9Œ€BÝŒ;˜˜˜R�y bÐ)Ñ)Ô)×1Ò1°"Ñ5Ô5Ð5r0   c                 óT  — |                      |¦  «        }|                      |¦  «        }|dd|j        d         dz  …f                              dd¬¦  «        }|dd|j        d         dz  …f                              dd¬¦  «        }|j        d         }| dd|…f         | d|d…f         }}|dd|…f         |d|d…f         }	}||z  t          |¦  «        |z  z   }
||z  t          |¦  «        |z  z   }t	          j        |
|gd¬¦  «        }
t	          j        ||	gd¬¦  «        }|
|fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    .NrQ   rO   rR   )Ú	unsqueezer   Úrepeat_interleaver³   r,   r„   )ÚqÚkr…   r†   Úunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds               r1   Úapply_rotary_pos_embrÁ   Ô   s\  € ð$ �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€Cð ˆcÐ'�S”Y˜r”] aÑ'Ð'Ð'Ô
(×
:Ò
:¸1À"Ð
:Ñ
EÔ
E€CØ
ˆcÐ'�S”Y˜r”] aÑ'Ð'Ð'Ô
(×
:Ò
:¸1À"Ð
:Ñ
EÔ
E€Cð ”˜2”€JØ�c˜;˜J˜;Ð&Ô'¨¨3°
°°Ð+;Ô)<ˆ6€EØ�c˜;˜J˜;Ð&Ô'¨¨3°
°°Ð+;Ô)<ˆ6€Eð �s‰{�{¨5Ñ1Ô1°CÑ7Ñ8€GØ�s‰{�{¨5Ñ1Ô1°CÑ7Ñ8€Gõ Œi˜ &Ð)¨rÐ2Ñ2Ô2€GÝŒi˜ &Ð)¨rÐ2Ñ2Ô2€GØ�GÐÐr0   c                   ó  ‡ — e Zd ZdZdededededef
ˆ fd„Z	 	 	 	 dd	ej	        d
e
ej	        ej	        f         dz  dej	        dz  dedz  dej	        dz  dee         de
ej	        ej	        dz  e
ej	                 dz  f         fd„Zˆ xZS )ÚMoonshineAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr9   Ú	layer_idxÚ	is_causalrq   r”   c                 óZ  •— t          ¦   «                              ¦   «          |                     ||dœ¦  «         || _        || _        t          |d|j        |j        z  ¦  «        | _        |j        |j	        z  | _
        | j        dz  | _        |j        | _        || _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j	        | j        z  |j        ¬¦  «        | _        t          j        |j        |j	        | j        z  |j        ¬¦  «        | _        t          j        |j        | j        z  |j        d¬¦  «        | _        | j        j        �0| j        j        }|| j        |z   dz
  |z  z  }|| j        z
  | _        d S d| _        d S )N)rq   r”   rl   g      à¿©ÚbiasFr"   r   )r7   r8   Úupdater9   rÄ   rp   r=   rq   rl   r”   r¢   rœ   Úattention_dropoutrÅ   r;   r<   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚpad_head_dim_to_multiple_ofÚhead_dim_padding)	rB   r9   rÄ   rÅ   rq   r”   Útarget_multipleÚtarget_head_dimrD   s	           €r1   r8   zMoonshineAttention.__init__ÿ   s©  ø€ õ 	‰Œ×ÒÑÔÐØ�ŠÐ.AÐZmÐnÐnÑoÔoÐoØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!Ø”} dÑ*ˆŒØ!'Ô!9ˆÔØ"ˆŒå”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”i Ô :¸T¼]Ñ JÈFÔL^ÐejÐkÑkÔkˆŒð Œ;Ô2Ð>Ø"œkÔEˆOØ-°$´-À/Ñ2QÐTUÑ2UÐZiÑ1iÑjˆOØ$3°d´mÑ$CˆDÔ!Ð!Ð!à$%ˆDÔ!Ð!Ð!r0   NrE   Úposition_embeddingsr'   Úpast_key_valuesÚkey_value_statesrž   rF   c                 óN  — |j         d d…         \  }}|                      |¦  «                             ||| j        j        | j        ¦  «                             dd¦  «        }	|d u}
|�?|j                             | j	        ¦  «        }|
rd|j        | j	        <   |j
        }n|j        }|�|n|}|
r3|r1|r/|j        | j	                 j        }|j        | j	                 j        }n¿|                      |¦  «                             |d| j        j        | j        ¦  «                             dd¦  «        }|                      |¦  «                             |d| j        j        | j        ¦  «                             dd¦  «        }|
r!|�|                     ||| j	        ¦  «        \  }}|
s;|\  }}t%          |	|||¦  «        \  }	}|�|                     ||| j	        ¦  «        \  }}t'          j        | j        j        t,          ¦  «        }| j        o	|d u o|dk    }| j        dk    r„t2          j        j                             |	d| j        f¦  «        }	t2          j        j                             |d| j        f¦  «        }t2          j        j                             |d| j        f¦  «        } || |	|||f| j        sdn| j        | j        |dœ|¤Ž\  }}| j        dk    r|dd | j         …f         }|                      ||d¦  «         !                    ¦   «         }|  "                    |¦  «        }||fS )	NrQ   r"   rO   Tr   r—   )r�   rœ   rÅ   .)#r   rÌ   Úviewr9   r”   rl   rƒ   Ú
is_updatedro   rÄ   Úcross_attention_cacheÚself_attention_cacheÚlayersÚkeysÚvaluesrÍ   rÎ   rÉ   rÁ   r   Úget_interfaceÚ_attn_implementationr¬   rÅ   rÑ   r,   r;   r¤   Úpadr¡   rÊ   rœ   r’   r§   rÏ   )rB   rE   rÔ   r'   rÕ   rÖ   rž   ÚbszÚq_lenÚquery_statesÚis_cross_attentionrÙ   Úcurrent_statesr¨   r©   r…   r†   Úattention_interfacerÅ   r«   rª   s                        r1   rH   zMoonshineAttention.forward$  sm  € ð #Ô(¨¨"¨Ô-‰
ˆˆUð �KŠK˜Ñ&Ô&×+Ò+¨C°¸¼Ô8WÐY]ÔYfÑgÔg×qÒqÐrsÐuvÑwÔwð 	ð .°TÐ9ÐØÐ&Ø(Ô3×7Ò7¸¼ÑGÔGˆJØ!ð Gà=A�Ô*¨4¬>Ñ:Ø"1Ô"G��à"1Ô"F�ð .>Ð-IÐ)Ð)È}ˆØð 	l /ð 	l°jð 	lØ(Ô/°´Ô?ÔDˆJØ*Ô1°$´.ÔAÔHˆLˆLð —’˜NÑ+Ô+ß’�c˜2˜tœ{Ô>ÀÄÑNÔNß’˜1˜a‘”ð ð —’˜NÑ+Ô+ß’�c˜2˜tœ{Ô>ÀÄÑNÔNß’˜1˜a‘”ð ð
 "ð l oÐ&AØ+:×+AÒ+AÀ*ÈlÐ\`Ô\jÑ+kÔ+kÑ(�
˜Là!ð 	lØ*‰HˆC�Ý';¸LÈ*ÐVYÐ[^Ñ'_Ô'_Ñ$ˆL˜*àÐ*Ø+:×+AÒ+AÀ*ÈlÐ\`Ô\jÑ+kÔ+kÑ(�
˜Lå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð ”NÐK ~¸Ð'=ÐKÀ%È!Â)ˆ	àÔ  1Ò$Ð$Ý œ8Ô.×2Ò2°<À!ÀTÔEZÐA[Ñ\Ô\ˆLÝœÔ,×0Ò0°¸aÀÔAVÐ=WÑXÔXˆJÝ œ8Ô.×2Ò2°<À!ÀTÔEZÐA[Ñ\Ô\ˆLà$7Ð$7ØØØØØð
%
ð  $œ}ÐH�C�C°$Ô2HØ”LØð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð Ô  1Ò$Ð$Ø% cÐ+C¨dÔ.CÐ-CÐ+CÐ&CÔDˆKà!×)Ò)¨#¨u°bÑ9Ô9×DÒDÑFÔFˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r0   )NNNN)r(   r)   r*   r+   r#   rr   Úboolr8   r,   r-   rŽ   r	   r   r   rH   rJ   rK   s   @r1   rÃ   rÃ   ü   s?  ø€ € € € € ØGÐGð#&àð#&ð ð#&ð ð	#&ð
 !ð#&ð !ð#&ð #&ð #&ð #&ð #&ð #&ðP IMØ.2Ø(,Ø04ðO)ð O)à”|ðO)ð # 5¤<°´Ð#=Ô>ÀÑEðO)ð œ tÑ+ð	O)ð
  ™ðO)ð  œ,¨Ñ-ðO)ð Ð-Ô.ðO)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	MðO)ð O)ð O)ð O)ð O)ð O)ð O)ð O)r0   rÃ   c                   óÒ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej        dej        dz  dej        dz  d	e	dz  d
e
dz  deej        ej        f         dz  dee         dej        fd„Zˆ xZS )ÚMoonshineEncoderLayerr9   rÄ   c                 óZ  •— t          ¦   «                              ¦   «          |j        | _        t          ||d|j        |j        ¬¦  «        | _        t          ||j        ¦  «        | _	        t          j        |j        d¬¦  «        | _        t          j        |j        d¬¦  «        | _        d S )NF©r9   rÄ   rÅ   rq   r”   rÇ   )r7   r8   r=   rÃ   Úencoder_num_attention_headsÚencoder_num_key_value_headsÚ	self_attnr3   Úencoder_hidden_actÚmlpr;   Ú	LayerNormÚinput_layernormÚpost_attention_layernorm©rB   r9   rÄ   rD   s      €r1   r8   zMoonshineEncoderLayer.__init__w  sŸ   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔå+ØØØØ &Ô BØ &Ô Bð
ñ 
ô 
ˆŒõ ' v¨vÔ/HÑIÔIˆŒÝ!œ|¨FÔ,>ÀUÐKÑKÔKˆÔÝ(*¬°VÔ5GÈeÐ(TÑ(TÔ(TˆÔ%Ð%Ð%r0   NFrE   r'   rˆ   rÕ   Ú	use_cacherÔ   rž   rF   c           
      óÎ   — |}|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}	||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N©rE   r'   rˆ   rÕ   rö   rÔ   r/   )ró   rï   rô   rñ   )
rB   rE   r'   rˆ   rÕ   rö   rÔ   rž   ÚresidualÚ_s
             r1   rH   zMoonshineEncoderLayer.forward‡  s¡   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆà)˜4œ>ð 
Ø'Ø)Ø%Ø+ØØ 3ð
ð 
ð ð
ð 
Ñˆ�qð ! =Ñ0ˆð !ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆØÐr0   )NNNFN)r(   r)   r*   r#   rr   r8   r,   r-   Ú
LongTensorr	   rè   rŽ   r   r   rH   rJ   rK   s   @r1   rê   rê   v  sÿ   ø€ € € € € ðU˜ð U¸3ð Uð Uð Uð Uð Uð Uð& /3Ø04Ø(,Ø!&ØHLðð à”|ðð œ tÑ+ðð Ô&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5¤<°´Ð#=Ô>ÀÑEðð Ð+Ô,ðð 
Œðð ð ð ð ð ð ð r0   rê   c                   ó†  ‡ — e Zd Zddededz  fˆ fd„Z	 	 	 	 	 	 	 	 	 ddej        dej        dz  dej        dz  d	ej        dz  d
ej        dz  dej        dz  de	dz  de
dz  deej        ej        f         dz  deej        ej        f         dz  dee         deej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚMoonshineDecoderLayerNr9   rÄ   c                 óà  •— t          ¦   «                              ¦   «          |j        | _        t          ||d|j        |j        ¬¦  «        | _        t          ||d|j        |j        ¬¦  «        | _        t          ||j	        ¦  «        | _
        t          j        |j        d¬¦  «        | _        t          j        |j        d¬¦  «        | _        t          j        |j        d¬¦  «        | _        d S )NTrì   FrÇ   )r7   r8   r=   rÃ   rq   r”   rï   Úencoder_attnrM   rC   rñ   r;   rò   ró   rô   Úfinal_layernormrõ   s      €r1   r8   zMoonshineDecoderLayer.__init__¨  sç   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔå+ØØØØ &Ô :Ø &Ô :ð
ñ 
ô 
ˆŒõ /ØØØØ &Ô :Ø &Ô :ð
ñ 
ô 
ˆÔõ ' v¨vÔ/@ÑAÔAˆŒÝ!œ|¨FÔ,>ÀUÐKÑKÔKˆÔÝ(*¬°VÔ5GÈeÐ(TÑ(TÔ(TˆÔ%Ý!œ|¨FÔ,>ÀUÐKÑKÔKˆÔÐÐr0   FrE   r'   Úencoder_hidden_statesÚencoder_attention_maskrˆ   Úencoder_position_idsrÕ   rö   rÔ   Úencoder_position_embeddingsrž   rF   c           
      óD  — |}|                       |¦  «        } | j        d||||||	dœ|¤Ž\  }}||z   }|�9|}|                      |¦  «        }|                      |||||¬¦  «        \  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )Nrø   )rE   rÖ   r'   rÕ   rö   r/   )ró   rï   rô   rÿ   r   rñ   )rB   rE   r'   r  r  rˆ   r  rÕ   rö   rÔ   r  rž   rù   rú   s                 r1   rH   zMoonshineDecoderLayer.forwardÀ  s÷   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆà)˜4œ>ð 
Ø'Ø)Ø%Ø+ØØ 3ð
ð 
ð ð
ð 
Ñˆ�qð ! =Ñ0ˆà Ð,Ø$ˆHØ ×9Ò9¸-ÑHÔHˆMØ#×0Ò0Ø+Ø!6Ø5Ø /Ø#ð  1ñ  ô  ÑˆM˜1ð % }Ñ4ˆMà ˆØ×,Ò,¨]Ñ;Ô;ˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆØÐr0   r5   )	NNNNNNFNN)r(   r)   r*   r#   rr   r8   r,   r-   rû   r	   rè   rŽ   r   r   ÚFloatTensorrH   rJ   rK   s   @r1   rý   rý   §  s‡  ø€ € € € € ðLð L˜ð L¸3À¹:ð Lð Lð Lð Lð Lð Lð6 /3Ø59Ø6:Ø04Ø8<Ø(,Ø!&ØHLØPTð,ð ,à”|ð,ð œ tÑ+ð,ð  %œ|¨dÑ2ð	,ð
 !&¤¨tÑ 3ð,ð Ô&¨Ñ-ð,ð $Ô.°Ñ5ð,ð  ™ð,ð ˜$‘;ð,ð # 5¤<°´Ð#=Ô>ÀÑEð,ð &+¨5¬<¸¼Ð+EÔ%FÈÑ%Mð,ð Ð+Ô,ð,ð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uð,ð ,ð ,ð ,ð ,ð ,ð ,ð ,r0   rý   c                   óT   — e Zd ZU eed<   dZdZdZdZddgZ	dZ
dZdZdej        fd	„Zd
S )ÚMoonshinePreTrainedModelr9   ÚmodelÚinput_valuesÚaudioTrê   rý   Úinput_lengthsc                 ó–   — t          |dz
  dz  dz   ¦  «        }t          |dz
  dz  dz   ¦  «        }t          |dz
  dz  dz   ¦  «        }|S )zH
        Computes the output length of the convolutional layers
        é   é@   r"   é   r   rO   )rr   )rB   r  Úoutput_conv1_lengthÚoutput_conv2_lengthÚoutput_conv3_lengths        r1   Ú _get_feat_extract_output_lengthsz9MoonshinePreTrainedModel._get_feat_extract_output_lengthsý  sc   € õ " =°3Ñ#6¸"Ñ"<¸qÑ"@ÑAÔAÐÝ!Ð#6¸Ñ#:¸aÑ"?À!Ñ"CÑDÔDÐÝ!Ð#6¸Ñ#:¸aÑ"?À!Ñ"CÑDÔDÐà"Ð"r0   N)r(   r)   r*   r#   r.   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_can_compile_fullgraphr,   rû   r  r/   r0   r1   r  r  ï  st   € € € € € € àÐÐÑØÐØ$€OØÐØ&*Ð#Ø0Ð2IÐJÐØÐØ€Nà!Ðð#¸eÔ>Nð #ð #ð #ð #ð #ð #r0   r  c                   óÊ   ‡ — e Zd ZdZdZeedœZdefˆ fd„Z	de
j        fd„Zde
j        fd	„Zee	 ddej        dej        d
z  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚMoonshineEncoderz£
    Transformer encoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MoonshineEncoderLayer`]

    Args:
        config: MoonshineConfig
    r
  )Ú
attentionsrE   r9   c                 ó`  •‡— t          ¦   «                              ‰¦  «         ‰| _        ‰j        }t	          j        d|ddd¬¦  «        | _        t	          j        |d|z  dd¬	¦  «        | _        t	          j        d|z  |dd¬	¦  «        | _        t	          j	        d|d
¬¦  «        | _
        t	          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t	          j        |d¬¦  «        | _        t#          ‰¬¦  «        | _        d| _        |                      ¦   «          d S )Nr"   r  r  F)Úkernel_sizeÚstriderÈ   rO   r  r   )r!  r"  gñhãˆµøä>)Ú
num_groupsÚnum_channelsÚepsc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r/   )rê   ©Ú.0Úidxr9   s     €r1   ú
<listcomp>z-MoonshineEncoder.__init__.<locals>.<listcomp>!  s$   ø€ ÐcÐcÐc°CÕ" 6¨3Ñ/Ô/ÐcÐcÐcr0   rÇ   ©r9   )r7   r8   r9   r=   r;   ÚConv1dÚconv1Úconv2Úconv3Ú	GroupNormÚ	groupnormÚ
ModuleListÚrangeÚencoder_num_hidden_layersrÜ   rò   Ú
layer_normrW   Ú
rotary_embÚgradient_checkpointingÚ	post_init)rB   r9   Ú	embed_dimrD   s    ` €r1   r8   zMoonshineEncoder.__init__  s  øø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒØÔ&ˆ	å”Y˜q )¸ÀRÈeÐTÑTÔTˆŒ
Ý”Y˜y¨!¨i©-ÀQÈqÐQÑQÔQˆŒ
Ý”Y˜q 9™}¨iÀQÈqÐQÑQÔQˆŒ
Ýœ°ÀÐPTÐUÑUÔUˆŒå”mØcÐcÐcÐc½5ÀÔAaÑ;bÔ;bÐcÑcÔcñ
ô 
ˆŒõ œ, y°uÐ=Ñ=Ô=ˆŒÝ2¸&ÐAÑAÔAˆŒØ&+ˆÔ#Ø�ŠÑÔÐÐÐr0   rF   c                 ó   — | j         S r5   ©r-  ©rB   s    r1   Úget_input_embeddingsz%MoonshineEncoder.get_input_embeddings(  s
   € ØŒzÐr0   r›   c                 ó   — || _         d S r5   r;  ©rB   r›   s     r1   Úset_input_embeddingsz%MoonshineEncoder.set_input_embeddings+  s   € ØˆŒ
ˆ
ˆ
r0   Nr'   rž   c                 ó¦  — |                      d¦  «        }t          j                             |                      |¦  «        ¦  «        }|                      |¦  «        }t          j                             |                      |¦  «        ¦  «        }t          j                             |                      |¦  «        ¦  «        }| 	                    ddd¦  «        }d}|�;|  
                    |j        d         ¦  «        }d}|ddd|…f         dd|…f         }|}t          | j        |||¬¦  «        }t          j        d|j        d         |j        ¬	¦  «                              d¦  «        }|                      ||¬
¦  «        }	| j        D ]}
 |
|f|||	dœ|¤Ž}Œ|                      |¦  «        }t)          ||�|                     ¦   «         nd¬¦  «        S )a.  
        Args:
            input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
                Float values of the raw speech waveform. Raw speech waveform can be
                obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
                `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
                the soundfile library (`pip install soundfile`). To prepare the array into
                `input_values`, the [`AutoFeatureExtractor`] should be used for padding
                and conversion into a tensor of type `torch.FloatTensor`.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding indices in `input_values`. Mask values selected in `[0, 1]`:
                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.
                [What are attention masks?](../glossary#attention-mask)
        r"   r   rO   NrQ   i€  .©r9   Úinputs_embedsr'   r  ©rf   ©rˆ   )r'   rˆ   rÔ   )Úlast_hidden_stater'   )rµ   r;   r¤   Útanhr-  r1  Úgelur.  r/  Úpermuter  r   r   r9   r,   rs   rf   r6  rÜ   r5  r&   rr   )rB   r
  r'   rž   rE   Úoutput_attention_maskÚmask_lenÚdownsample_striderˆ   rÔ   Úencoder_layers              r1   rH   zMoonshineEncoder.forward.  sô  € ð. $×-Ò-¨aÑ0Ô0ˆÝœ×*Ò*¨4¯:ª:°lÑ+CÔ+CÑDÔDˆØŸš }Ñ5Ô5ˆÝœ×*Ò*¨4¯:ª:°mÑ+DÔ+DÑEÔEˆÝœ×*Ò*¨4¯:ª:°mÑ+DÔ+DÑEÔEˆØ%×-Ò-¨a°°AÑ6Ô6ˆð !%ÐØÐ%Ø×<Ò<¸^Ô=QÐRTÔ=UÑVÔVˆHØ *ÐØ+¨CÐ1DÐ1DÐ3DÐ1DÐ,DÔEÀcÈ9ÈHÈ9ÀnÔUˆNØ$2Ð!å2Ø”;Ø'Ø)Ø"/ð	
ñ 
ô 
ˆõ ”| A }Ô':¸1Ô'=ÀmÔFZÐ[Ñ[Ô[×eÒeÐfgÑhÔhˆØ"Ÿošo¨mÈ,˜oÑWÔWÐà!œ[ð 	ð 	ˆMØ)˜MØðà-Ø)Ø$7ð	ð ð
 ðð ˆMˆMð Ÿš¨Ñ6Ô6ˆå*Ø+Ø:OÐ:[Ð0×4Ò4Ñ6Ô6Ð6Ðaeð
ñ 
ô 
ð 	
r0   r5   )r(   r)   r*   r+   r  rÃ   rê   Ú_can_record_outputsr#   r8   r;   ÚModuler=  r@  r   r!   r,   r  r-   r   r   rŽ   r   rH   rJ   rK   s   @r1   r  r    s  ø€ € € € € ðð ð %€Oà(Ø.ðð Ðð
˜ð ð ð ð ð ð ð$ b¤ið ð ð ð ð¨"¬)ð ð ð ð ð  Øð /3ð<
ð <
àÔ'ð<
ð œ tÑ+ð<
ð Ð+Ô,ð	<
ð
 
Ð(Ñ	(ð<
ð <
ð <
ñ „_ñ  Ôð<
ð <
ð <
ð <
ð <
r0   r  c                   óD  ‡ — e Zd ZdZ eedd¬¦  «        e eedd¬¦  «        dœZdefˆ fd„Z	e
e	 	 	 	 	 	 	 	 ddej        d	z  d
ej        d	z  dej        d	z  ded	z  dej        d	z  ded	z  dej        d	z  dej        d	z  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚMoonshineDecoderÚ	input_idsr"   rï   )ÚindexÚ
layer_namerÿ   )r  rE   Úcross_attentionsr9   c                 óÞ  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j        d¬¦  «        | _        t          ‰¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r/   )rý   r'  s     €r1   r*  z-MoonshineDecoder.__init__.<locals>.<listcomp>~  s$   ø€ Ð$sÐ$sÐ$sÈCÕ%:¸6À3Ñ%GÔ%GÐ$sÐ$sÐ$sr0   FrÇ   r+  )r7   r8   Úpad_token_idÚpadding_idxÚ
vocab_sizer;   Ú	Embeddingr=   Úembed_tokensr2  r3  Únum_hidden_layersrÜ   rò   ÚnormrW   r6  r7  r8  ©rB   r9   rD   s    `€r1   r8   zMoonshineDecoder.__init__x  sÏ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔÝ”mÐ$sÐ$sÐ$sÐ$sÕSXÐY_ÔYqÑSrÔSrÐ$sÑ$sÔ$sÑtÔtˆŒÝ”L Ô!3¸%Ð@Ñ@Ô@ˆŒ	Ý2¸&ÐAÑAÔAˆŒØ&+ˆÔ#ð 	�ŠÑÔÐÐÐr0   Nr'   rˆ   rÕ   rC  rö   r  r  rž   rF   c	           
      ó¢  — |du |duz  rt          d¦  «        ‚|€|                      |¦  «        }|r8|€6t          t          | j        ¬¦  «        t          | j        ¬¦  «        ¦  «        }|€V|�|                     ¦   «         nd}
t          j        |j        d         |j	        ¬¦  «        |
z   }| 
                    d¦  «        }t          | j        ||||¬¦  «        }t          | j        |||¬¦  «        }|}|                      ||¬	¦  «        }| j        D ]} ||||f|||||d
œ|	¤Ž}Œ|                      |¦  «        }t!          ||r|nd¬¦  «        S )a¤  
        encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
            of the decoder.
        encoder_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding indices in `encoder_hidden_states`. Mask values selected in `[0, 1]`:
            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.
            [What are attention masks?](../glossary#attention-mask)
        Nz:You must specify exactly one of input_ids or inputs_embedsr+  r   r"   rD  )r9   rC  r'   rÕ   rˆ   rB  rE  )r  rˆ   rÕ   rö   rÔ   )rF  rÕ   )Ú
ValueErrorr\  r   r
   r9   Úget_seq_lengthr,   rs   r   rf   rµ   r   r   r6  rÜ   r^  r   )rB   rR  r'   rˆ   rÕ   rC  rö   r  r  rž   Úpast_seen_tokensÚcausal_maskrE   rÔ   Údecoder_layers                  r1   rH   zMoonshineDecoder.forward†  sË  € ð0 ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	v˜Ð0Ý1µ,ÀdÄkÐ2RÑ2RÔ2RÕT`ÐhlÔhsÐTtÑTtÔTtÑuÔuˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå(Ø”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆõ ";Ø”;Ø'Ø1Ø"7ð	"
ñ "
ô "
Ðð &ˆØ"Ÿošo¨mÈ,˜oÑWÔWÐà!œ[ð 	ð 	ˆMØ)˜MØØØ%ð
ð (>Ø)Ø /Ø#Ø$7ð
ð 
ð ð
ð 
ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå8Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
r0   )NNNNNNNN)r(   r)   r*   r  r    rÃ   rý   rN  r#   r8   r   r!   r,   rû   r-   r	   r  rè   r   r   rŽ   r   rH   rJ   rK   s   @r1   rQ  rQ  o  s…  ø€ € € € € à!€Oà$�nÐ%7¸qÈ[ÐYÑYÔYØ.Ø*˜NÐ+=ÀQÐSaÐbÑbÔbðð Ðð˜ð ð ð ð ð ð ð  Øð .2Ø.2Ø04Ø(,Ø26Ø!%Ø:>Ø6:ðG
ð G
àÔ# dÑ*ðG
ð œ tÑ+ðG
ð Ô&¨Ñ-ð	G
ð
  ™ðG
ð Ô(¨4Ñ/ðG
ð ˜$‘;ðG
ð  %Ô0°4Ñ7ðG
ð !&¤¨tÑ 3ðG
ð Ð+Ô,ðG
ð 
Ð(Ñ	(ðG
ð G
ð G
ñ „_ñ  ÔðG
ð G
ð G
ð G
ð G
r0   rQ  c                   ó\  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zd„ Zd„ Ze	e
	 	 	 	 	 	 	 	 	 ddej        dz  d	ej        dz  d
ej        dz  dej        dz  deeej                          dz  dedz  deej                 dz  deej                 dz  dedz  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚMoonshineModelr9   c                 óÂ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¦  «        | _        |                      ¦   «          d S r5   )r7   r8   r  ÚencoderrQ  Údecoderr8  r_  s     €r1   r8   zMoonshineModel.__init__Ô  sO   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å'¨Ñ/Ô/ˆŒÝ'¨Ñ/Ô/ˆŒà�ŠÑÔÐÐÐr0   c                 ó   — | j         j        S r5   ©rj  r\  r<  s    r1   r=  z#MoonshineModel.get_input_embeddingsÜ  s   € ØŒ|Ô(Ð(r0   c                 ó   — || j         _        d S r5   rl  r?  s     r1   r@  z#MoonshineModel.set_input_embeddingsß  s   € Ø$)ˆŒÔ!Ð!Ð!r0   c                 ó8   — | j                              ¦   «          dS )z«
        Calling this function will disable the gradient computation for the Moonshine encoder so that its parameters will
        not be updated during training.
        N)ri  Ú_freeze_parametersr<  s    r1   Úfreeze_encoderzMoonshineModel.freeze_encoderâ  s   € ð
 	Œ×'Ò'Ñ)Ô)Ð)Ð)Ð)r0   c                 ó    — t          d¦  «        ‚)z¢
        Masks extracted features along time axis and/or along feature axis according to
        [SpecAugment](https://huggingface.co/papers/1904.08779).
        zNot needed for Moonshine)ÚAttributeErrorr<  s    r1   Ú_mask_input_featuresz#MoonshineModel._mask_input_featuresé  s   € õ
 Ð7Ñ8Ô8Ð8r0   Nr
  r'   Údecoder_input_idsÚdecoder_attention_maskÚencoder_outputsrÕ   Údecoder_inputs_embedsÚdecoder_position_idsrö   rž   rF   c
                 óä   — |€ | j         |fd|i|
¤Ž} | j        d|||j        |j        ||||	dœ|
¤Ž}t	          |j        |j        |j        |j        |j        |j        |j        |j        ¬¦  «        S )a
  
        input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
            Float values of the raw speech waveform. Raw speech waveform can be
            obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
            `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
            the soundfile library (`pip install soundfile`). To prepare the array into
            `input_values`, the [`AutoFeatureExtractor`] should be used for padding
            and conversion into a tensor of type `torch.FloatTensor`.
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`

        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoFeatureExtractor, MoonshineModel
        >>> from datasets import load_dataset

        >>> model = MoonshineModel.from_pretrained("UsefulSensors/moonshine-tiny")
        >>> feature_extractor = AutoFeatureExtractor.from_pretrained("UsefulSensors/moonshine-tiny")
        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> inputs = feature_extractor(ds[0]["audio"]["array"], return_tensors="pt")
        >>> input_values = inputs.input_values
        >>> decoder_input_ids = torch.tensor([[1, 1]]) * model.config.decoder_start_token_id
        >>> last_hidden_state = model(input_values, decoder_input_ids=decoder_input_ids).last_hidden_state
        >>> list(last_hidden_state.shape)
        [1, 2, 288]
        ```
        Nr'   )rR  r'   r  r  rÕ   rC  rˆ   rö   )rF  rÕ   Údecoder_hidden_statesÚdecoder_attentionsrU  Úencoder_last_hidden_stater  Úencoder_attentionsr/   )	ri  rj  rF  r'   r   rÕ   rE   r  rU  )rB   r
  r'   rt  ru  rv  rÕ   rw  rx  rö   rž   Údecoder_outputss               r1   rH   zMoonshineModel.forwardð  s¾   € ðZ Ð"Ø/;¨t¬|¸LÐ/rÐ/rÐYgÐ/rÐkqÐ/rÐ/rˆOàEQÀTÄ\ð 
F
Ø'Ø1Ø"1Ô"CØ#2Ô#AØ+Ø/Ø-Øð
F
ð 
F
ð ð
F
ð 
F
ˆõ "Ø-Ô?Ø+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð	
ñ 	
ô 	
ð 		
r0   )	NNNNNNNNN)r(   r)   r*   r#   r8   r=  r@  rp  rs  r   r   r,   r  rû   rŽ   r   rè   r   r   r   rH   rJ   rK   s   @r1   rg  rg  Ò  s¨  ø€ € € € € ð˜ð ð ð ð ð ð ð)ð )ð )ð*ð *ð *ð*ð *ð *ð9ð 9ð 9ð Øð 26Ø26Ø59Ø:>ØBFØ6:ØAEØ?CØ!%ðC
ð C
àÔ'¨$Ñ.ðC
ð Ô(¨4Ñ/ðC
ð !Ô+¨dÑ2ð	C
ð
 !&Ô 0°4Ñ 7ðC
ð ˜u UÔ%6Ô7Ô8¸4Ñ?ðC
ð -¨tÑ3ðC
ð  % UÔ%6Ô7¸$Ñ>ðC
ð $ EÔ$4Ô5¸Ñ<ðC
ð ˜$‘;ðC
ð Ð+Ô,ðC
ð 
ðC
ð C
ð C
ñ „^ñ ÔðC
ð C
ð C
ð C
ð C
r0   rg  rR  rX  Údecoder_start_token_idc                 óô   — |                       | j        ¦  «        }| dd…dd…f                              ¦   «         |dd…dd…f<   ||dd…df<   |€t          d¦  «        ‚|                     |dk    |¦  «         |S )z1
    Shift input ids one token to the right.
    NrQ   r"   r   z1self.model.config.pad_token_id has to be defined.iœÿÿÿ)Ú	new_zerosr   re   ra  Úmasked_fill_)rR  rX  r  Úshifted_input_idss       r1   Úshift_tokens_rightr„  8  s˜   € ð "×+Ò+¨I¬OÑ<Ô<ÐØ(¨¨¨¨C¨R¨C¨Ô0×6Ò6Ñ8Ô8Ð�a�a�a˜˜˜�eÑØ4Ð�a�a�a˜�dÑàÐÝÐLÑMÔMÐMà×"Ò"Ð#4¸Ò#<¸lÑKÔKÐKàÐr0   zj
    The Moonshine Model with a language modeling head. Can be used for automatic speech recognition.
    c                   ó„  ‡ — e Zd ZddiZdefˆ fd„Zd„ Zd„ Zdej	        fd„Z
ee	 	 	 	 	 	 	 	 	 	 dd
ej        d	z  dej        d	z  dej        d	z  dej        d	z  deeej                          d	z  ded	z  deej                 d	z  deej                 d	z  ded	z  dej        d	z  dee         defd„¦   «         ¦   «         Zˆ xZS )Ú!MoonshineForConditionalGenerationzproj_out.weightz!model.decoder.embed_tokens.weightr9   c                 óæ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S )NFrÇ   )
r7   r8   rg  r	  r;   r<   r=   rZ  Úproj_outr8  r_  s     €r1   r8   z*MoonshineForConditionalGeneration.__init__P  s`   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý# FÑ+Ô+ˆŒ
Ýœ	 &Ô"4°fÔ6GÈeÐTÑTÔTˆŒð 	�ŠÑÔÐÐÐr0   c                 ó   — | j         S r5   ©rˆ  r<  s    r1   Úget_output_embeddingsz7MoonshineForConditionalGeneration.get_output_embeddingsX  s
   € ØŒ}Ðr0   c                 ó   — || _         d S r5   rŠ  )rB   Únew_embeddingss     r1   Úset_output_embeddingsz7MoonshineForConditionalGeneration.set_output_embeddings[  s   € Ø&ˆŒˆˆr0   rF   c                 ó4   — | j                              ¦   «         S r5   )r	  r=  r<  s    r1   r=  z6MoonshineForConditionalGeneration.get_input_embeddings^  s   € ØŒz×.Ò.Ñ0Ô0Ð0r0   Nr
  r'   rt  ru  rv  rÕ   rw  rx  rö   Úlabelsrž   c                 óÎ  — |
�)|€'|€%t          |
| j        j        | j        j        ¦  «        } | j        |f||||||||	dœ|¤Ž}|                      |j        ¦  «        }d}|
�Kt          ¦   «         } ||                     d| j        j	        ¦  «        |
                     d¦  «        ¦  «        }t          |||j        |j        |j        |j        |j        |j        |j        ¬¦	  «	        S )a0  
        input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
            Float values of the raw speech waveform. Raw speech waveform can be
            obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
            `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
            the soundfile library (`pip install soundfile`). To prepare the array into
            `input_values`, the [`AutoFeatureExtractor`] should be used for padding
            and conversion into a tensor of type `torch.FloatTensor`.
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`

        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoProcessor, MoonshineForConditionalGeneration
        >>> from datasets import load_dataset

        >>> processor = AutoProcessor.from_pretrained("UsefulSensors/moonshine-tiny")
        >>> model = MoonshineForConditionalGeneration.from_pretrained("UsefulSensors/moonshine-tiny")

        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")

        >>> inputs = processor(ds[0]["audio"]["array"], return_tensors="pt")
        >>> input_values = inputs.input_values

        >>> generated_ids = model.generate(input_values, max_new_tokens=100)

        >>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        >>> transcription
        'Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.'
        ```N)r'   rt  rv  ru  rÕ   rw  rx  rö   rQ   )	ÚlossÚlogitsrÕ   rz  r{  rU  r|  r  r}  )r„  r9   rX  r  r	  rˆ  rF  r   r’   rZ  r   rÕ   rz  r{  rU  r|  r  r}  )rB   r
  r'   rt  ru  rv  rÕ   rw  rx  rö   r�  rž   Úoutputsr“  r’  Úloss_fcts                   r1   rH   z)MoonshineForConditionalGeneration.forwarda  s   € ðd ÐØ Ð(Ð-BÐ-JÝ$6Ø˜DœKÔ4°d´kÔ6Xñ%ô %Ð!ð '1 d¤jØð'
à)Ø/Ø+Ø#9Ø+Ø"7Ø!5Øð'
ð '
ð ð'
ð '
ˆð —’˜wÔ8Ñ9Ô9ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸNšN¨2¨t¬{Ô/EÑFÔFÈÏÊÐWYÑHZÔHZÑ[Ô[ˆDåØØØ#Ô3Ø")Ô"?Ø&Ô9Ø$Ô5Ø&-Ô&GØ")Ô"?Ø&Ô9ð

ñ 

ô 

ð 
	
r0   )
NNNNNNNNNN)r(   r)   r*   Ú_tied_weights_keysr#   r8   r‹  rŽ  r;   rO  r=  r   r   r,   r  rû   rŽ   r   rè   r   r   r   rH   rJ   rK   s   @r1   r†  r†  H  sË  ø€ € € € € ð ,Ð-PÐQÐð˜ð ð ð ð ð ð ðð ð ð'ð 'ð 'ð1 b¤ið 1ð 1ð 1ð 1ð Øð 26Ø26Ø59Ø:>ØBFØ6:ØAEØ?CØ!%Ø*.ðS
ð S
àÔ'¨$Ñ.ðS
ð Ô(¨4Ñ/ðS
ð !Ô+¨dÑ2ð	S
ð
 !&Ô 0°4Ñ 7ðS
ð ˜u UÔ%6Ô7Ô8¸4Ñ?ðS
ð -¨tÑ3ðS
ð  % UÔ%6Ô7¸$Ñ>ðS
ð $ EÔ$4Ô5¸Ñ<ðS
ð ˜$‘;ðS
ð Ô  4Ñ'ðS
ð Ð+Ô,ðS
ð 
ðS
ð S
ð S
ñ „^ñ ÔðS
ð S
ð S
ð S
ð S
r0   r†  )rg  r  r†  )r—   )r"   )IÚcollections.abcr   Údataclassesr   Útypingr   r,   Útorch.nnr;   r   Úactivationsr   Úcache_utilsr	   r
   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   r   Úutils.output_capturingr    r!   Úconfiguration_moonshiner#   r&   rO  r3   rM   rW   r-   rr   r–   rv   r¬   r³   rÁ   rÃ   rê   rý   r  r  rQ  rg  r„  r†  Ú__all__r/   r0   r1   ú<module>rª     sª  ðð* %Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ø %Ð %Ð %Ð %Ð %Ð %à !Ð !Ð !Ð !Ð !Ð !Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ )Ð )Ð )Ð )Ð )Ð )Ø JÐ JÐ JÐ JÐ JÐ JÐ JÐ JØ BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð LÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ 4Ð 4Ð 4Ð 4Ð 4Ð 4ð €ððñ ô ð
 ð
/ð 
/ð 
/ð 
/ð 
/ /ñ 
/ô 
/ñ „ñô ð
/ðð ð ð ð ˜"œ)ñ ô ð ðð ð ð ð ˜"œ)ñ ô ð ð @<ð @<ð @<ð @<ð @<˜rœyñ @<ô @<ð @<ðF	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð26ð 6ð 6ð%ð %ð %ð %ðPw)ð w)ð w)ð w)ð w)˜œñ w)ô w)ð w)ðt.ð .ð .ð .ð .Ð6ñ .ô .ð .ðbEð Eð Eð Eð EÐ6ñ Eô Eð EðP ð#ð #ð #ð #ð #˜ñ #ô #ñ „ð#ð0d
ð d
ð d
ð d
ð d
Ð/ñ d
ô d
ð d
ðN ð_
ð _
ð _
ð _
ð _
Ð/ñ _
ô _
ñ „ð_
ðD ðb
ð b
ð b
ð b
ð b
Ð-ñ b
ô b
ñ „ðb
ðJ %¤,ð ¸cð Ð[^ð ð ð ð ð  €ððñ ô ð
i
ð i
ð i
ð i
ð i
Ð(@À/ñ i
ô i
ñô ð
i
ðX ^Ð
]Ð
]€€€r0   