§
    ‚Štj! ã                   óÌ  — d Z ddlZddlZddlmZ ddlmZ ddlZddlmZ ddl	m
Z
 ddlmZ dd	lmZmZ dd
lmZmZ ddlmZmZ ddlmZmZ ddlmZmZmZ ddlmZ ddl m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0m1Z1  e)j2        e3¦  «        Z4dej5        dej5        fd„Z6dej5        dej5        fd„Z7d„ Z8dId„Z9	 	 	 	 	 	 dJd„Z: e'd ¬!¦  «        e G d"„ d#e%¦  «        ¦   «         ¦   «         Z;e'e G d$„ d%e%¦  «        ¦   «         ¦   «         Z< G d&„ d'ej=        ¦  «        Z> G d(„ d)ej=        ¦  «        Z? G d*„ d+ej=        ¦  «        Z@ G d,„ d-ej=        ¦  «        ZA G d.„ d/ej=        ¦  «        ZB G d0„ d1ej=        ¦  «        ZC G d2„ d3ej=        ¦  «        ZD G d4„ d5ej=        ¦  «        ZE G d6„ d7ej=        ¦  «        ZF G d8„ d9ej=        ¦  «        ZGe' G d:„ d;e¦  «        ¦   «         ZH G d<„ d=eH¦  «        ZI G d>„ d?eH¦  «        ZJe' G d@„ dAeH¦  «        ¦   «         ZK e'dB¬!¦  «         G dC„ dDeHe¦  «        ¦   «         ZL e'dE¬!¦  «         G dF„ dGeHe¦  «        ¦   «         ZMg dH¢ZNdS )KzPyTorch CLVP model.é    N)ÚCallable)Ú	dataclass)Únn)ÚCrossEntropyLossé   )Úinitialization)ÚACT2FNÚget_activation)ÚCacheÚDynamicCache)ÚGenerationConfigÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)Ú)BaseModelOutputWithPastAndCrossAttentionsÚBaseModelOutputWithPoolingÚ!CausalLMOutputWithCrossAttentions)ÚPreTrainedModel)ÚUnpack)ÚConv1D)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)Úcapture_outputsé   )Ú
ClvpConfigÚClvpDecoderConfigÚClvpEncoderConfigÚlogitsÚreturnc                 óŽ   — t           j                             | t          j        t          | ¦  «        | j        ¬¦  «        ¦  «        S )N©Údevice)r   Ú
functionalÚcross_entropyÚtorchÚarangeÚlenr&   )r"   s    úd/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/clvp/modeling_clvp.pyÚcontrastive_lossr-   ;   s3   € ÝŒ=×&Ò& v­u¬|½CÀ¹K¼KÐPVÔP]Ð/^Ñ/^Ô/^Ñ_Ô_Ð_ó    Ú
similarityc                 óX   — t          | ¦  «        }t          | j        ¦  «        }||z   dz  S )Ng       @)r-   ÚT)r/   Úcaption_lossÚspeech_losss      r,   Úspeech_text_contrastive_lossr4   @   s.   € Ý# JÑ/Ô/€LÝ" :¤<Ñ0Ô0€KØ˜;Ñ&¨#Ñ-Ð-r.   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Néÿÿÿÿé   ©Údim)Úshaper)   Úcat)ÚxÚx1Úx2s      r,   Úrotate_halfr?   G   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r.   c                 ó  — ||                               |¦  «        }||                               |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||z  t          |¦  «        |z  z   }	|||	fS )an  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`):
            The position indices of the tokens corresponding to the query and key tensors. For example, this can be
            used to pass offsetted position ids when working with a KV-cache.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer?   )
ÚqÚkÚvÚcosÚsinÚposition_idsÚunsqueeze_dimÚq_embedÚk_embedÚv_embeds
             r,   Úapply_rotary_pos_embrL   N   sŒ   € ð* ˆlÔ
×
%Ò
% mÑ
4Ô
4€CØ
ˆlÔ
×
%Ò
% mÑ
4Ô
4€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�G˜WÐ$Ð$r.   éÿ   Tc                 ó  — |rRt           j        j                             | d|¬¦  «        } |�'t           j        j                             |dd¬¦  «        n|}| }|�r&| j        d         }| |k    }	t          j        |	                     dd¬¦  «        |	                     ¦   «                              dd¬¦  «        t          j	        || j
        ¬¦  «        ¦  «        }
t          j        |dz   | j
        ¬¦  «        }t          j        ||
k     ||dz
                       d¬	¦  «        ¦  «        }|                      d|¦  «                             ||
k    |¦  «        }|�'t           j        j                             |dd¬¦  «        n|}||fS )
zß
    This method adds extra bos and eos tokens to input_ids and accordingly modifies the attention_mask which is used in
    `ClvpConditioningEncoder` and the generation loop of the `ClvpModelForConditionalGeneration`.
    )r   r   ©ÚvalueNr   T©Úkeepdimr%   r   )Úmin)r)   r   r'   Úpadr:   ÚwhereÚanyÚintÚargmaxÚtensorr&   r*   ÚclampÚgatherÚmasked_fill)Ú	input_idsÚattention_maskÚpad_token_idÚbos_token_idÚeos_token_idÚadd_bos_tokenÚadd_eos_tokenÚmodified_input_idsÚseq_lenÚis_padÚ	first_padÚcolÚsrcs                r,   Ú_pad_extra_bos_eos_tokensrj   k   s‡  € ð ð 
Ý”HÔ'×+Ò+¨I°vÀ\Ð+ÑRÔRˆ	àHVÐHb�EŒHÔ×#Ò# N°FÀ!Ð#ÑDÔDÐDÐhvð 	ð #ÐØñ 
ð ”/ !Ô$ˆØ˜lÒ*ˆÝ”KØ�JŠJ�q $ˆJÑ'Ô'Ø�JŠJ‰LŒL×Ò ¨4ÐÑ0Ô0ÝŒL˜¨Ô)9Ð:Ñ:Ô:ñ
ô 
ˆ	õ
 Œl˜7 Q™;¨yÔ/?Ð@Ñ@Ô@ˆÝŒk˜# 	š/¨3°°q±·²ÀA°Ñ0FÔ0FÑGÔGˆØ&×-Ò-¨a°Ñ5Ô5×AÒAÀ#ÈÒBRÐT`ÑaÔaÐð IWÐHb�EŒHÔ×#Ò# N°FÀ!Ð#ÑDÔDÐDÐhvð 	ð ˜~Ð-Ð-r.   z²
    Base class for CLVP encoder's outputs that contains a pooling of the last hidden states as well as a projection
    output (a linear layer on top of the pooled output).
    ©Úcustom_introc                   óÂ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dZ
eej                 dz  ed<   dZeej                 dz  ed<   dS )ÚClvpEncoderOutputa
  
    embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)`, *optional*, returned when model is initialized with `with_projection=True`):
        The embeddings obtained by applying the projection layer to the pooler_output.
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        The hidden state of the last layer of the model.
    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`):
        Pooled output of the `last_hidden_state`.
    NÚembedsÚlast_hidden_stateÚpooler_outputÚhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ro   r)   ÚFloatTensorÚ__annotations__rp   rq   rr   Útuplers   © r.   r,   rn   rn   —   s¡   € € € € € € ðð ð (,€FˆEÔ Ñ$Ð+Ð+Ñ+Ø26Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø.2€M�5Ô$ tÑ+Ð2Ð2Ñ2Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ð6Ð6r.   rn   c                   ó>  — e Zd ZU dZdZej        dz  ed<   dZej	        dz  ed<   dZ
ej        dz  ed<   dZej        dz  ed<   dZej        dz  ed<   dZej        dz  ed<   dZeed	<   dZeed
<   dZej        dz  ed<   dZej        dz  ed<   dZej        dz  ed<   dS )Ú
ClvpOutputa÷  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
        Contrastive loss for speech-text similarity.
    speech_ids (`torch.LongTensor`, *optional*):
        speech_ids (or speech candidates) generated by the `ClvpForCausalLM` model.
    logits_per_speech (`torch.FloatTensor` of shape `(speech_batch_size, text_batch_size)`):
        The scaled dot product scores between `speech_embeds` and `text_embeds`. This represents the speech-text
        similarity scores.
    logits_per_text (`torch.FloatTensor` of shape `(text_batch_size, speech_batch_size)`):
        The scaled dot product scores between `text_embeds` and `speech_embeds`. This represents the text-speech
        similarity scores.
    text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):
        The text embeddings obtained by applying the projection layer to the pooled output of the text encoder
        model.
    speech_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):
        The speech embeddings obtained by applying the projection layer to the pooled output of the speech encoder
        model.
    text_model_output (`BaseModelOutputWithPooling`):
        The pooled output of the `last_hidden_state` of the text encoder Model.
    speech_model_output (`BaseModelOutputWithPooling`):
        The pooled output of the `last_hidden_state` of the speech encoder Model.
    decoder_hidden_states (`torch.FloatTensor`, *optional*):
        The hidden states of the decoder model.
    text_encoder_hidden_states (`torch.FloatTensor`, *optional*):
        The hidden states of the text encoder model.
    speech_encoder_hidden_states (`torch.FloatTensor`, *optional*):
        The hidden states of the speech encoder model.
    NÚlossÚ
speech_idsÚlogits_per_speechÚlogits_per_textÚtext_embedsÚspeech_embedsÚtext_model_outputÚspeech_model_outputÚdecoder_hidden_statesÚtext_encoder_hidden_statesÚspeech_encoder_hidden_states)rt   ru   rv   rw   r~   r)   rx   ry   r   Ú
LongTensorr€   r�   r‚   rƒ   r„   r   r…   r†   r‡   rˆ   r{   r.   r,   r}   r}   ¯   s  € € € € € € ðð ð: &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø*.€J�Ô  4Ñ'Ð.Ð.Ñ.Ø26Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø04€O�UÔ&¨Ñ-Ð4Ð4Ñ4Ø,0€K�Ô" TÑ)Ð0Ð0Ñ0Ø.2€M�5Ô$ tÑ+Ð2Ð2Ñ2Ø48ÐÐ1Ð8Ð8Ñ8Ø6:ÐÐ3Ð:Ð:Ñ:Ø6:Ð˜5Ô,¨tÑ3Ð:Ð:Ñ:Ø;?Ð Ô 1°DÑ 8Ð?Ð?Ñ?Ø=AÐ  %Ô"3°dÑ":ÐAÐAÑAÐAÐAr.   r}   c                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
ÚClvpRMSNormç�íµ ÷Æ°>Úepsr#   Nc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z:
        ClvpRMSNorm is equivalent to T5LayerNorm
        N)ÚsuperÚ__init__r   Ú	Parameterr)   ÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer�   Ú	__class__s      €r,   r�   zClvpRMSNorm.__init__Þ   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr.   rr   c                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )Nr7   r6   TrQ   )	ÚdtypeÚtor)   Úfloat32ÚpowÚmeanÚrsqrtr”   r“   )r•   rr   Úinput_dtypeÚvariances       r,   ÚforwardzClvpRMSNorm.forwardæ   s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r.   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)rz   r“   r:   r”   ©r•   s    r,   Ú
extra_reprzClvpRMSNorm.extra_reprí   s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr.   )rŒ   )
rt   ru   rv   Úfloatr�   r)   ÚTensorr¡   r¤   Ú__classcell__©r—   s   @r,   r‹   r‹   Ý   sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð Jr.   r‹   c                   óF   ‡ — e Zd ZdZˆ fd„Zdej        dej        fd„Zˆ xZS )ÚClvpRotaryPositionalEmbeddingzÊ
    Rotary Position Embedding Class for CLVP. It was proposed in the paper 'ROFORMER: ENHANCED TRANSFORMER WITH ROTARY
    POSITION EMBEDDING', Please see https://huggingface.co/papers/2104.09864.
    c                 óJ  •— t          ¦   «                              ¦   «          t          |j        |j        dz  z  d¦  «        }ddt          j        d|dt
          j        ¬¦  «                             ¦   «         |z  z  z  }|  	                    d|¦  «         d | _
        d | _        d S )Nr7   é    ç      ð?é'  r   ©r™   Úinv_freq)r�   r�   ÚmaxÚprojection_dimÚnum_attention_headsr)   r*   Úint64r¥   Úregister_bufferÚcached_sequence_lengthÚ"cached_rotary_positional_embedding)r•   Úconfigr9   r°   r—   s       €r,   r�   z&ClvpRotaryPositionalEmbedding.__init__÷   sš   ø€ Ý‰Œ×ÒÑÔÐÝ�&Ô'¨FÔ,FÈÑ,JÑKÈRÑPÔPˆØ˜%¥E¤L°°C¸Å%Ä+Ð$NÑ$NÔ$N×$TÒ$TÑ$VÔ$VÐY\Ñ$\Ñ]Ñ^ˆà×Ò˜Z¨Ñ2Ô2Ð2Ø&*ˆÔ#Ø26ˆÔ/Ð/Ð/r.   rr   r#   c                 ój  — |j         d         }|| j        k    r| j        �| j        S || _        t          j        ||j        ¬¦  «                             | j        ¦  «        }t          j        d|| j        ¦  «        }t          j	        ||fd¬¦  «        }| 
                    d¦  «        | _        | j        S )Nr   r%   zi,j->ijr6   r8   r   )r:   r¶   r·   r)   r*   r&   Útype_asr°   Úeinsumr;   rA   )r•   rr   Úsequence_lengthÚtime_stampsÚfreqsÚ
embeddingss         r,   r¡   z%ClvpRotaryPositionalEmbedding.forward   s«   € Ø'Ô-¨aÔ0ˆà˜dÔ9Ò9Ð9¸dÔ>eÐ>qØÔ:Ð:à&5ˆÔ#Ý”l ?¸=Ô;OÐPÑPÔP×XÒXÐY]ÔYfÑgÔgˆÝ”˜Y¨°T´]ÑCÔCˆÝ”Y  u˜~°2Ð6Ñ6Ô6ˆ
à2<×2FÒ2FÀqÑ2IÔ2IˆÔ/ØÔ6Ð6r.   ©	rt   ru   rv   rw   r�   r)   rx   r¡   r§   r¨   s   @r,   rª   rª   ñ   sj   ø€ € € € € ðð ð
7ð 7ð 7ð 7ð 7ð7 UÔ%6ð 7¸5Ô;Lð 7ð 7ð 7ð 7ð 7ð 7ð 7ð 7r.   rª   c                   óò   ‡ — e Zd ZdZdˆ fd„	Zdej        dedefd„Z	 	 	 	 	 dd	ej	        d
ej	        dz  dej
        dz  dej
        dz  dedz  dedz  dee         deej	        ej	        dz  f         fd„Zˆ xZS )ÚClvpSelfAttentionzu
    Multi-headed attention to combine Absolute and Rotary Positional Embeddings into a single Attention module.
    Nc                 ó¢  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        | j        | j        z  | _        | j        | j        z  | j        k    r t          d| j        › d| j        › d�¦  «        ‚| j        dz  | _	        |j
        | _        || _        t          |d¦  «        rk|j        }t          j        t          j        ||ft          j        ¬¦  «        ¦  «        }|                     dd||¦  «        }|                      d|d	¬
¦  «         t+          j        | j        | j        |j        ¬¦  «        | _        t+          j        | j        | j        |j        ¬¦  «        | _        t+          j        | j        | j        |j        ¬¦  «        | _        t+          j        | j        | j        ¦  «        | _        d S )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).ç      à¿Úmax_position_embeddingsr¯   r   ÚbiasF)Ú
persistent©rÆ   )r�   r�   r¸   r–   Ú	embed_dimr³   Ú	num_headsÚhead_dimÚ
ValueErrorÚscaleÚattention_dropoutÚdropoutÚ	layer_idxÚhasattrrÅ   r)   Útrilr’   ÚboolÚviewrµ   r   ÚLinearÚuse_attention_biasÚk_projÚv_projÚq_projÚout_proj)r•   r¸   rÐ   Úmax_positionsrÆ   r—   s        €r,   r�   zClvpSelfAttention.__init__  sŸ  ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØÔ3ˆŒØœ¨$¬.Ñ8ˆŒØŒ=˜4œ>Ñ)¨T¬^Ò;Ð;Ýð'ÈdÌnð 'ð 'Ø”Nð'ð 'ð 'ñô ð ð ”] DÑ(ˆŒ
ØÔ/ˆŒØ"ˆŒå�6Ð4Ñ5Ô5ð 	AØ"Ô:ˆMÝ”:�eœj¨-¸Ð)GÍuÌzÐZÑZÔZÑ[Ô[ˆDØ—9’9˜Q  =°-Ñ@Ô@ˆDØ× Ò  ¨¸%Ð Ñ@Ô@Ð@å”i ¤°´ÀVÔE^Ð_Ñ_Ô_ˆŒÝ”i ¤°´ÀVÔE^Ð_Ñ_Ô_ˆŒÝ”i ¤°´ÀVÔE^Ð_Ñ_Ô_ˆŒÝœ	 $¤.°$´.ÑAÔAˆŒˆˆr.   rY   re   Úbszc                 ó’   — |                      ||| j        | j        ¦  «                             dd¦  «                             ¦   «         S )Nr   r7   )rÔ   rÊ   rË   Ú	transposeÚ
contiguous)r•   rY   re   rÜ   s       r,   Ú_shapezClvpSelfAttention._shape.  s<   € Ø�{Š{˜3 ¨¬¸¼ÑGÔG×QÒQÐRSÐUVÑWÔW×bÒbÑdÔdÐdr.   Frr   Úrotary_pos_embr^   rG   Úpast_key_valuesÚ	use_cacheÚkwargsr#   c                 ó  — |�|€t          d¦  «        ‚|                     ¦   «         \  }}	}
|                      |                      |¦  «        d|¦  «        | j        z  }|                      |                      |¦  «        d|¦  «        }|                      |                      |¦  «        d|¦  «        }|�|                     ||| j        ¦  «        \  }}|��|j	        d         }|dd |…f         |d|d …f         }}|dd |…f         |d|d …f         }}|dd |…f         |d|d …f         }}| 
                    ¦   «                              d¦  «        |                     ¦   «                              d¦  «        }}t          ||||||¦  «        \  }}}t          j        ||fd¬¦  «        }t          j        ||fd¬¦  «        }t          j        ||fd¬¦  «        }|j	        d         }|j	        d         }t          j        ||                     dd¦  «        ¦  «        }|�L|                     ¦   «         |d||fk    r+t          d	|d||f› d
|                     ¦   «         › �¦  «        ‚||z   }t$          j                             |d¬¦  «        }t$          j                             || j        | j        ¬¦  «        }t          j        ||¦  «        }|                     ¦   «         || j        || j        fk    r5t          d|| j        || j        f› d
|                     ¦   «         › �¦  «        ‚|                     dd¦  «                             ¦   «         }|                     ||| j        ¦  «        }|                      |¦  «        }||fS )NzB`position_ids` must be provided when `rotary_pos_emb` is not None.r6   .r   r8   r7   r   r   z!Attention mask should be of size z	, but is )ÚpÚtrainingz `attn_output` should be of size )rÌ   Úsizerà   rÙ   rÍ   r×   rØ   ÚupdaterÐ   r:   rE   ÚsqueezerF   rL   r)   r;   ÚmatmulrÞ   r   r'   ÚsoftmaxrÏ   rç   rÊ   rË   rß   ÚreshaperÉ   rÚ   )r•   rr   rá   r^   rG   râ   rã   rä   rÜ   Ú_rÉ   Úquery_statesÚ
key_statesÚvalue_statesÚrotary_emb_dimÚ	query_rotÚ
query_passÚkey_rotÚkey_passÚ	value_rotÚ
value_passrE   rF   Útgt_lenÚsrc_lenÚattn_weightsÚ
attn_probsÚattn_outputs                               r,   r¡   zClvpSelfAttention.forward1  sÇ  € ð Ð%¨,Ð*>ÝÐaÑbÔbÐbà)×.Ò.Ñ0Ô0ÑˆˆQ�	ð —{’{ 4§;¢;¨}Ñ#=Ô#=¸rÀ3ÑGÔGÈ$Ì*ÑTˆØ—[’[ §¢¨]Ñ!;Ô!;¸RÀÑEÔEˆ
Ø—{’{ 4§;¢;¨}Ñ#=Ô#=¸rÀ3ÑGÔGˆàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜àÑ%Ø+Ô1°"Ô5ˆNð ˜S / > /Ð1Ô2Ø˜S . / /Ð1Ô2ð "ˆIð
 ˜3   Ð/Ô0Ø˜3   Ð/Ô0ð ˆGð
 ˜S / > /Ð1Ô2Ø˜S . / /Ð1Ô2ð "ˆIð
 &×)Ò)Ñ+Ô+×3Ò3°AÑ6Ô6¸×8JÒ8JÑ8LÔ8L×8TÒ8TÐUVÑ8WÔ8W�ˆCÝ,@ÀÈGÐU^Ð`cÐehÐjvÑ,wÔ,wÑ)ˆI�w 	õ !œ9 i°Ð%<À"ÐEÑEÔEˆLÝœ G¨XÐ#6¸BÐ?Ñ?Ô?ˆJÝ œ9 i°Ð%<À"ÐEÑEÔEˆLàÔ$ QÔ'ˆØÔ" 1Ô%ˆÝ”| L°*×2FÒ2FÀqÈ!Ñ2LÔ2LÑMÔMˆàÐ%Ø×"Ò"Ñ$Ô$¨¨a°¸'Ð(BÒBÐBÝ Øt¸¸aÀÈ'Ð8RÐtÐtÐ]k×]pÒ]pÑ]rÔ]rÐtÐtñô ð ð (¨.Ñ8ˆLå”}×,Ò,¨\¸rÐ,ÑBÔBˆå”]×*Ò*¨<¸4¼<ÐRVÔR_Ð*Ñ`Ô`ˆ
Ý”l :¨|Ñ<Ô<ˆà×ÒÑÔ # t¤~°wÀÄÐ!NÒNÐNÝð)°C¸¼ÈÐRVÔR_Ð3`ð )ð )Ø×$Ò$Ñ&Ô&ð)ð )ñô ð ð
 "×+Ò+¨A¨qÑ1Ô1×<Ò<Ñ>Ô>ˆØ!×)Ò)¨#¨w¸¼ÑGÔGˆà—m’m KÑ0Ô0ˆà˜LÐ(Ð(r.   ©N)NNNNF)rt   ru   rv   rw   r�   r)   r¦   rW   rà   rx   r‰   r   rÓ   r   r   rz   r¡   r§   r¨   s   @r,   rÂ   rÂ     sF  ø€ € € € € ðð ðBð Bð Bð Bð Bð Bð4e˜Uœ\ð e°Cð e¸cð eð eð eð eð 48Ø26Ø04Ø(,Ø!&ðM)ð M)àÔ(ðM)ð Ô)¨DÑ0ðM)ð Ô(¨4Ñ/ð	M)ð
 Ô&¨Ñ-ðM)ð  ™ðM)ð ˜$‘;ðM)ð Ð+Ô,ðM)ð 
ˆuÔ  %Ô"3°dÑ":Ð:Ô	;ðM)ð M)ð M)ð M)ð M)ð M)ð M)ð M)r.   rÂ   c                   óF   ‡ — e Zd ZdZˆ fd„Zdej        dej        fd„Zˆ xZS )ÚClvpGatedLinearUnitzÈ
    `ClvpGatedLinearUnit` uses the second half of the `hidden_states` to act as a gate for the first half of the
    `hidden_states` which controls the flow of data from the first of the tensor.
    c                 óÄ   •— t          ¦   «                              ¦   «          t          |j                 | _        t          j        |j        |j        dz  ¦  «        | _	        d S )Nr7   )
r�   r�   r	   Ú
hidden_actÚactivation_fnr   rÕ   r–   Úintermediate_sizeÚproj©r•   r¸   r—   s     €r,   r�   zClvpGatedLinearUnit.__init__‡  sK   ø€ Ý‰Œ×ÒÑÔÐÝ# FÔ$5Ô6ˆÔÝ”I˜fÔ0°&Ô2JÈQÑ2NÑOÔOˆŒ	ˆ	ˆ	r.   rr   r#   c                 óŒ   — |                       |¦  «                             dd¬¦  «        \  }}||                      |¦  «        z  S )Nr7   r6   r8   )r  Úchunkr  )r•   rr   Úgates      r,   r¡   zClvpGatedLinearUnit.forwardŒ  sD   € Ø"Ÿiši¨Ñ6Ô6×<Ò<¸QÀBÐ<ÑGÔGÑˆ�tØ˜t×1Ò1°$Ñ7Ô7Ñ7Ð7r.   rÀ   r¨   s   @r,   r   r   �  so   ø€ € € € € ðð ð
Pð Pð Pð Pð Pð
8 UÔ%6ð 8¸5Ô;Lð 8ð 8ð 8ð 8ð 8ð 8ð 8ð 8r.   r   c                   óF   ‡ — e Zd ZdZˆ fd„Zdej        dej        fd„Zˆ xZS )ÚClvpEncoderMLPzA
    This MLP is used in CLVP speech or text encoder models.
    c                 ó  •— t          ¦   «                              ¦   «          || _        t          |¦  «        | _        t          j        |j        |j        ¦  «        | _	        t          j
        |j        ¦  «        | _        d S rþ   )r�   r�   r¸   r   Úfc1r   rÕ   r  r–   Úfc2ÚDropoutrÏ   Údropout_layerr  s     €r,   r�   zClvpEncoderMLP.__init__–  sb   ø€ Ý‰Œ×ÒÑÔÐØˆŒå& vÑ.Ô.ˆŒÝ”9˜VÔ5°vÔ7IÑJÔJˆŒÝœZ¨¬Ñ7Ô7ˆÔÐÐr.   rr   r#   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rþ   )r  r  r  ©r•   rr   s     r,   r¡   zClvpEncoderMLP.forwardž  s=   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆØÐr.   rÀ   r¨   s   @r,   r  r  ‘  sj   ø€ € € € € ðð ð8ð 8ð 8ð 8ð 8ð UÔ%6ð ¸5Ô;Lð ð ð ð ð ð ð ð r.   r  c                   ó‚   ‡ — e Zd Zdefˆ fd„Zdej        dej        dej        dej        dee	         dej        fd	„Z
ˆ xZS )
ÚClvpEncoderLayerr¸   c                 ó>  •— t          ¦   «                              ¦   «          || _        |j        | _        t          |¦  «        | _        t          |¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        d S )N©r�   )r�   r�   r¸   r–   rÉ   rÂ   Ú	self_attnr  Úmlpr‹   Úlayer_norm_epsÚinput_rmsnormÚpost_attention_rmsnormr  s     €r,   r�   zClvpEncoderLayer.__init__¦  s‚   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒÝ*¨6Ñ2Ô2ˆŒÝ! &Ñ)Ô)ˆŒå(¨¬¸VÔ=RÐSÑSÔSˆÔÝ&1°$´.ÀfÔF[Ð&\Ñ&\Ô&\ˆÔ#Ð#Ð#r.   rr   rá   r^   rG   rä   r#   c                 óÊ   — |}|                       |¦  «        } | j        |f|||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N)rá   r^   rG   )r  r  r  r  )r•   rr   rá   r^   rG   rä   Úresidualrî   s           r,   r¡   zClvpEncoderLayer.forward°  s™   € ð !ˆà×*Ò*¨=Ñ9Ô9ˆà)˜4œ>Øð
à)Ø)Ø%ð	
ð 
ð
 ð
ð 
Ñˆ�qð ! =Ñ0ˆà ˆØ×3Ò3°MÑBÔBˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆàÐr.   )rt   ru   rv   r   r�   r)   rx   r‰   r   r   r¡   r§   r¨   s   @r,   r  r  ¥  s®   ø€ € € € € ð]˜zð ]ð ]ð ]ð ]ð ]ð ]ðàÔ(ðð Ô)ðð Ô(ð	ð
 Ô&ðð Ð+Ô,ðð 
Ô	ðð ð ð ð ð ð ð r.   r  c                   ód   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej        dej        dz  dej        fd„Z	ˆ xZ
S )
ÚClvpSequenceSummaryaÌ  
    Compute a single vector summary of a sequence hidden states.

    Args:
        config ([`ClvpConfig`]):
            The config used by the model. Relevant arguments in the config class of the model are (refer to the actual
            config class of your model for the default values it uses):

            - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are:

                - `"last"` -- Take the last token hidden state (like XLNet)
                - `"first"` -- Take the first token hidden state (like Bert)
                - `"mean"` -- Take the mean of all tokens hidden states
                - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2)
                - `"attn"` -- Not implemented now, use multi-head attention

            - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction.
            - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes
              (otherwise to `config.hidden_size`).
            - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output,
              another string or `None` will add no activation.
            - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation.
            - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation.
    r¸   c                 óV  •— t          ¦   «                              ¦   «          t          |dd¦  «        | _        | j        dk    rt          ‚t          j        ¦   «         | _        t          |d¦  «        rW|j	        rPt          |d¦  «        r|j
        r|j        dk    r|j        }n|j        }t          j        |j        |¦  «        | _        t          |dd ¦  «        }|rt          |¦  «        nt          j        ¦   «         | _        t          j        ¦   «         | _        t          |d¦  «        r)|j        dk    rt          j        |j        ¦  «        | _        t          j        ¦   «         | _        t          |d	¦  «        r+|j        dk    r"t          j        |j        ¦  «        | _        d S d S d S )
NÚsummary_typeÚlastÚattnÚsummary_use_projÚsummary_proj_to_labelsr   Úsummary_activationÚsummary_first_dropoutÚsummary_last_dropout)r�   r�   Úgetattrr!  ÚNotImplementedErrorr   ÚIdentityÚsummaryrÑ   r$  r%  Ú
num_labelsr–   rÕ   r
   Ú
activationÚfirst_dropoutr'  r  Úlast_dropoutr(  )r•   r¸   Únum_classesÚactivation_stringr—   s       €r,   r�   zClvpSequenceSummary.__init__é  sœ  ø€ Ý‰Œ×ÒÑÔÐå# F¨N¸FÑCÔCˆÔØÔ Ò&Ð&õ &Ð%å”{‘}”}ˆŒÝ�6Ð-Ñ.Ô.ð 	F°6Ô3Jð 	FÝ�vÐ7Ñ8Ô8ð 1¸VÔ=Zð 1Ð_eÔ_pÐstÒ_tÐ_tØ$Ô/��à$Ô0�Ýœ9 VÔ%7¸ÑEÔEˆDŒLå# FÐ,@À$ÑGÔGÐØIZÐ$m¥NÐ3DÑ$EÔ$EÐ$EÕ`bÔ`kÑ`mÔ`mˆŒåœ[™]œ]ˆÔÝ�6Ð2Ñ3Ô3ð 	J¸Ô8TÐWXÒ8XÐ8XÝ!#¤¨FÔ,HÑ!IÔ!IˆDÔåœK™MœMˆÔÝ�6Ð1Ñ2Ô2ð 	H°vÔ7RÐUVÒ7VÐ7VÝ "¤
¨6Ô+FÑ GÔ GˆDÔÐÐð	Hð 	HÐ7VÐ7Vr.   Nrr   Ú	cls_indexr#   c                 ó:  — | j         dk    r|dd…df         }�n-| j         dk    r|dd…df         }�n| j         dk    r|                     d¬¦  «        }nò| j         d	k    rÕ|€=t          j        |d
dd…dd…f         |j        d         dz
  t          j        ¬¦  «        }nl|                     d¦  «                             d¦  «        }|                     d|                     ¦   «         dz
  z  | 	                    d¦  «        fz   ¦  «        }| 
                    d|¦  «                             d¦  «        }n| j         dk    rt          ‚|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S )ak  
        Compute a single vector summary of a sequence hidden states.

        Args:
            hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`):
                The hidden states of the last layer.
            cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*):
                Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token.

        Returns:
            `torch.FloatTensor`: The summary of the sequence hidden states.
        r"  Nr6   Úfirstr   r�   r   r8   r3  .éþÿÿÿr¯   )r6   r#  )r!  r�   r)   Ú	full_liker:   ÚlongrA   Úexpandr9   rè   r[   rê   r*  r/  r,  r.  r0  )r•   rr   r3  Úoutputs       r,   r¡   zClvpSequenceSummary.forward  sª  € ð Ô Ò&Ð&Ø" 1 1 1 b 5Ô)ˆF‰FØÔ 'Ò)Ð)Ø" 1 1 1 a 4Ô(ˆF‰FØÔ &Ò(Ð(Ø"×'Ò'¨AÐ'Ñ.Ô.ˆFˆFØÔ +Ò-Ð-ØÐ Ý!œOØ! # r¨ r¨1¨1¨1 *Ô-Ø!Ô'¨Ô+¨aÑ/Ýœ*ðñ ô �	�	ð &×/Ò/°Ñ3Ô3×=Ò=¸bÑAÔA�	Ø%×,Ò,¨U°i·m²m±o´oÈÑ6IÑ-JÈm×N`ÒN`ÐacÑNdÔNdÐMfÑ-fÑgÔg�	à"×)Ò)¨"¨iÑ8Ô8×@Ò@ÀÑDÔDˆFˆFØÔ &Ò(Ð(Ý%Ð%à×#Ò# FÑ+Ô+ˆØ—’˜fÑ%Ô%ˆØ—’ Ñ(Ô(ˆØ×"Ò" 6Ñ*Ô*ˆàˆr.   rþ   )rt   ru   rv   rw   r   r�   r)   rx   r‰   r¡   r§   r¨   s   @r,   r  r  Ï  s›   ø€ € € € € ðð ð2H˜zð Hð Hð Hð Hð Hð Hð< VZð)ð )Ø"Ô.ð)Ø;@Ô;KÈdÑ;Rð)à	Ô	ð)ð )ð )ð )ð )ð )ð )ð )r.   r  c                   óT   ‡ — e Zd Zˆ fd„Zdeej                 dz  dej        fd„Zˆ xZS )ÚClvpDecoderMLPc                 ó  •— t          ¦   «                              ¦   «          |j        }t          ||¦  «        | _        t          ||¦  «        | _        t          |j                 | _        t          j
        |j        ¦  «        | _        d S rþ   )r�   r�   r–   r   Úc_fcÚc_projr	   Úactivation_functionÚactr   r  Úresid_pdroprÏ   )r•   r  r¸   rÉ   r—   s       €r,   r�   zClvpDecoderMLP.__init__4  sl   ø€ Ý‰Œ×ÒÑÔÐØÔ&ˆ	ÝÐ,¨iÑ8Ô8ˆŒ	Ý˜YÐ(9Ñ:Ô:ˆŒÝ˜&Ô4Ô5ˆŒÝ”z &Ô"4Ñ5Ô5ˆŒˆˆr.   rr   Nr#   c                 ó®   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rþ   )r>  rA  r?  rÏ   r  s     r,   r¡   zClvpDecoderMLP.forward<  sL   € ØŸ	š	 -Ñ0Ô0ˆØŸš Ñ/Ô/ˆØŸš MÑ2Ô2ˆØŸš ]Ñ3Ô3ˆØÐr.   )	rt   ru   rv   r�   rz   r)   rx   r¡   r§   r¨   s   @r,   r<  r<  3  sj   ø€ € € € € ð6ð 6ð 6ð 6ð 6ð U¨5Ô+<Ô%=ÀÑ%Dð ÈÔIZð ð ð ð ð ð ð ð r.   r<  c                   óš   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej        dedz  dej        dz  dej        dz  dedz  d	e	e
         d
ej        fd„Zˆ xZS )ÚClvpDecoderLayerNc                 óR  •— t          ¦   «                              ¦   «          |j        }|j        �|j        nd|z  }t	          j        ||j        ¬¦  «        | _        t          ||¬¦  «        | _	        t	          j        ||j        ¬¦  «        | _
        t          ||¦  «        | _        d S )Né   r  ©rÐ   )r�   r�   r–   Ún_innerr   Ú	LayerNormÚlayer_norm_epsilonÚinput_layernormrÂ   r#  Úpost_attention_layernormr<  r  )r•   r¸   rÐ   r–   Ú	inner_dimr—   s        €r,   r�   zClvpDecoderLayer.__init__E  s”   ø€ Ý‰Œ×ÒÑÔÐØÔ(ˆØ&,¤nÐ&@�F”N�NÀaÈ+Áoˆ	å!œ|¨K¸VÔ=VÐWÑWÔWˆÔÝ% f¸	ÐBÑBÔBˆŒ	Ý(*¬°[ÀfÔF_Ð(`Ñ(`Ô(`ˆÔ%å! )¨VÑ4Ô4ˆŒˆˆr.   Frr   râ   r^   rG   rã   rä   r#   c                 óÌ   — |}|                       |¦  «        } | j        |f||||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }	||	z   }|S )N©râ   r^   rG   rã   )rL  r#  rM  r  )
r•   rr   râ   r^   rG   rã   rä   r  rî   Úfeed_forward_hidden_statess
             r,   r¡   zClvpDecoderLayer.forwardP  sž   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆØ$˜4œ9Øð
à+Ø)Ø%Øð
ð 
ð ð
ð 
Ñˆ�qð &¨Ñ0ˆà ˆØ×5Ò5°mÑDÔDˆØ%)§X¢X¨mÑ%<Ô%<Ð"à Ð#=Ñ=ˆàÐr.   rþ   ©NNNF)rt   ru   rv   r�   r)   rx   r   r‰   rÓ   r   r   r¦   r¡   r§   r¨   s   @r,   rE  rE  D  sË   ø€ € € € € ð	5ð 	5ð 	5ð 	5ð 	5ð 	5ð )-Ø26Ø04Ø!&ðð àÔ(ðð  ™ðð Ô(¨4Ñ/ð	ð
 Ô&¨Ñ-ðð ˜$‘;ðð Ð+Ô,ðð 
Œðð ð ð ð ð ð ð r.   rE  c            
       ó”   ‡ — e Zd ZdZdefˆ fd„Zddedefd„Z	 	 	 dd	ej	        d
ej
        dz  dej	        dz  dej
        dz  fd„Zˆ xZS )ÚClvpConditioningEncodera—  
    This class processes the log-mel spectrograms(extracted by the Feature Extractor) and text tokens(produced by the
    tokenizer) as inputs for the decoder model.

    First each log-mel spectrogram is processed into a single vector which captures valuable characteristics from each
    of them, then the text tokens are converted into token embeddings and position embeddings are added afterwards.
    Both of these vectors are concatenated and then passed to the decoder model.

    The text tokens helps to incorporate the "text information" and the log-mel spectrogram is used to specify the
    "voice characteristics" into the generated mel tokens.
    r¸   c                 óÒ  •‡ ‡— t          ¦   «                              ¦   «          |j        ‰ _        |j        ‰ _        t	          j        ‰ j        j        ‰ j        j        ¦  «        ‰ _        t	          j        ‰ j        j	        ‰ j        j        ¦  «        ‰ _
        t	          j        ‰ j        j        ‰ j        j        d¬¦  «        ‰ _        ‰                      ‰ j        j        ¦  «        Št	          j        ˆˆ fd„t!          ‰ j        j        ¦  «        D ¦   «         ¦  «        ‰ _        t	          j        ˆ fd„t!          ‰ j        j        ¦  «        D ¦   «         ¦  «        ‰ _        d‰ _        d S )Nr   )Úkernel_sizec                 óT   •— g | ]$}t          j        ‰‰j        j        d d¬¦  «        ‘Œ%S )gñhãˆµøä>T)r�   Úaffine)r   Ú	GroupNormÚdecoder_configr–   )Ú.0rî   Ú
num_groupsr•   s     €€r,   ú
<listcomp>z4ClvpConditioningEncoder.__init__.<locals>.<listcomp>Œ  sC   ø€ ð ð ð àõ ”˜Z¨Ô)<Ô)HÈdÐ[_Ð`Ñ`Ô`ðð ð r.   c                 ó8   •— g | ]}t          ‰j        ¦  «        ‘ŒS r{   )rÂ   rZ  )r[  rî   r•   s     €r,   r]  z4ClvpConditioningEncoder.__init__.<locals>.<listcomp>”  s%   ø€ ÐlÐlÐl¸Õ˜tÔ2Ñ3Ô3ÐlÐlÐlr.   F)r�   r�   Útext_configrZ  r   Ú	EmbeddingÚ
vocab_sizer–   Útext_token_embeddingÚmax_text_tokensÚtext_position_embeddingÚConv1dÚfeature_sizeÚmel_convÚcompute_groupnorm_groupsÚ
ModuleListÚrangeÚnum_mel_attn_blocksÚgroup_normsÚmel_attn_blocksÚgradient_checkpointing)r•   r¸   r\  r—   s   ` @€r,   r�   z ClvpConditioningEncoder.__init__|  sP  øøø€ Ý‰Œ×ÒÑÔÐà!Ô-ˆÔØ$Ô3ˆÔå$&¤L°Ô1AÔ1LÈdÔNaÔNmÑ$nÔ$nˆÔ!Ý')¤|ØÔÔ/°Ô1DÔ1Pñ(
ô (
ˆÔ$õ œ	 $Ô"5Ô"BÀDÔDWÔDcÐqrÐsÑsÔsˆŒð ×2Ò2°4Ô3FÔ3RÑSÔSˆ
Ýœ=ðð ð ð ð å˜tÔ2ÔFÑGÔGðñ ô ñ
ô 
ˆÔõ  "œ}ØlÐlÐlÐl½UÀ4ÔCVÔCjÑ=kÔ=kÐlÑlÔlñ 
ô  
ˆÔð ',ˆÔ#Ð#Ð#r.   r¬   ÚchannelsÚgroupsc                 ó¢   — |dk    rd}n|dk    rd}||z  dk    rt          |dz  ¦  «        }||z  dk    °|dk    rt          d|› d�¦  «        ‚|S )a  
        Calculates the value of `num_groups` for nn.GroupNorm. This logic is taken from the official tortoise
        repository. link :
        https://github.com/neonbjb/tortoise-tts/blob/4003544b6ff4b68c09856e04d3eff9da26d023c2/tortoise/models/arch_util.py#L26
        é   é   é@   r   r7   zENumber of groups for the GroupNorm must be greater than 2, but it is z0.Please consider using a different `hidden_size`)rW   rÌ   )r•   ro  rp  s      r,   rh  z0ClvpConditioningEncoder.compute_groupnorm_groups™  s’   € ð �rŠ>ˆ>ØˆFˆFØ˜Š^ˆ^ØˆFØ˜Ñ 1Ò$Ð$Ý˜ !™‘_”_ˆFð ˜Ñ 1Ò$Ð$ð �QŠ;ˆ;ÝðCÐX^ð Cð Cð Cñô ð ð
 ˆr.   NÚinput_featuresr]   Úinputs_embedsr^   c                 ó$  — |�|�t          d¦  «        ‚|�|                     ¦   «         \  }}n1|� |                     ¦   «         d d…         \  }}nt          d¦  «        ‚|€(t          j        ||gt          j        |j        ¬¦  «        }t          ||| j        j        | j        j	        ¬¦  «        \  }}|  
                    |¦  «        }|                     d¦  «        dz
  }|                      |¦  «        }||z   }	| j        rê| j        rãt          j        j                             | j        |¦  «        }
t%          | j        ¦  «        D ]£\  }}|
                     dd¦  «        }t          j        j                             | j        |         |
¦  «                             dd¦  «        }
t          j        j                             ||
¦  «        d         |z   }
|
                     dd¦  «        }
Œ¤n™|                      |¦  «        }
t%          | j        ¦  «        D ]o\  }}|
                     dd¦  «        } | j        |         |
¦  «                             dd¦  «        }
 ||
¦  «        d         |z   }
|
                     dd¦  «        }
Œp|
d d …d d …df         }
|
                     d¦  «        }
|	j        d         dk    r4|
j        d         dk    r#|	                     |
j        d         dd¦  «        }	n�|	j        d         dk    r4|
j        d         dk    r#|
                     |	j        d         dd¦  «        }
nH|	j        d         |
j        d         k    r,t          d	|	j        d         › d
|
j        d         › d�¦  «        ‚t          j        |
|	gd¬¦  «        S )NzDYou cannot specify both input_ids and inputs_embeds at the same timer6   z5You have to specify either input_ids or inputs_embeds©r™   r&   )r`   ra   r   r7   r   z=The number of texts and number of audios must be same. Found z
 texts vs z audiosr8   )rÌ   rè   r)   r’   r8  r&   rj   r_  r`   ra   rb  Úcumsumrd  rn  rç   ÚutilsÚ
checkpointrg  Ú	enumeraterm  rÞ   rl  rA   r:   ÚrepeatÚconcat)r•   ru  r]   rv  r^   Ú
batch_sizeÚ
seq_lengthrG   Úposition_embedsr‚   Úmel_specÚiÚmel_attn_blockÚresidual_mel_specs                 r,   r¡   zClvpConditioningEncoder.forward®  sÇ  € ð Ð  ]Ð%>ÝÐcÑdÔdÐdØÐ"Ø%.§^¢^Ñ%5Ô%5Ñ"ˆJ˜
˜
ØÐ&Ø%2×%7Ò%7Ñ%9Ô%9¸#¸2¸#Ô%>Ñ"ˆJ˜
˜
åÐTÑUÔUÐUð Ð!Ý"œZ¨°ZÐ(@ÍÌ
Ð[dÔ[kÐlÑlÔlˆNõ %>ØØØÔ)Ô6ØÔ)Ô6ð	%
ñ %
ô %
Ñ!ˆ	�>ð ×1Ò1°)Ñ<Ô<ˆØ%×,Ò,¨RÑ0Ô0°1Ñ4ˆØ×6Ò6°|ÑDÔDˆØ# oÑ5ˆàÔ&ð 	4¨4¬=ð 	4å”{Ô-×8Ò8¸¼ÈÑWÔWˆHå%.¨tÔ/CÑ%DÔ%Dð 4ð 4Ñ!��>Ø$,×$6Ò$6°q¸!Ñ$<Ô$<Ð!å œ;Ô1×<Ò<¸TÔ=MÈaÔ=PÐRZÑ[Ô[×eÒeÐfgÐijÑkÔk�Ý œ;Ô1×<Ò<¸^ÈXÑVÔVÐWXÔYÐ\mÑm�Ø#×-Ò-¨a°Ñ3Ô3��ð4ð —}’} ^Ñ4Ô4ˆHå%.¨tÔ/CÑ%DÔ%Dð 4ð 4Ñ!��>Ø$,×$6Ò$6°q¸!Ñ$<Ô$<Ð!à.˜4Ô+¨AÔ.¨xÑ8Ô8×BÒBÀ1ÀaÑHÔH�Ø)˜>¨(Ñ3Ô3°AÔ6Ð9JÑJ�Ø#×-Ò-¨a°Ñ3Ô3��à˜A˜A˜A˜q˜q˜q !˜GÔ$ˆØ×%Ò% aÑ(Ô(ˆð Ô˜QÔ 1Ò$Ð$¨¬¸Ô):¸aÒ)?Ð)?Ø%×,Ò,¨X¬^¸AÔ->ÀÀ1ÑEÔEˆKˆKØÔ˜qÔ! QÒ&Ð&¨8¬>¸!Ô+<ÀÒ+AÐ+AØ—’ {Ô'8¸Ô';¸QÀÑBÔBˆHˆHàÔ˜qÔ! X¤^°AÔ%6Ò6Ð6ÝðTØ$Ô*¨1Ô-ðTð TØ9A¼ÈÔ9JðTð Tð Tñô ð õ
 Œ|˜X {Ð3¸Ð;Ñ;Ô;Ð;r.   )r¬   ©NNN)rt   ru   rv   rw   r   r�   rW   rh  r)   rx   r‰   r¡   r§   r¨   s   @r,   rT  rT  o  sç   ø€ € € € € ð
ð 
ð,˜zð ,ð ,ð ,ð ,ð ,ð ,ð:ð °ð ¸cð ð ð ð ð0 .2Ø26Ø26ðH<ð H<àÔ)ðH<ð Ô# dÑ*ðH<ð Ô(¨4Ñ/ð	H<ð
 Ô(¨4Ñ/ðH<ð H<ð H<ð H<ð H<ð H<ð H<ð H<r.   rT  c                   ó€   ‡ — e Zd ZU eed<   dZdZdgZee	ge
dœZ ej        ¦   «         dej        fˆ fd„¦   «         Zˆ xZS )ÚClvpPreTrainedModelr¸   ÚmodelTrâ   )rr   rs   Úmodulec           
      óp  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          j        ¦  «        r!t          j        |j	        d|dz  ¬¦  «         dS t	          |t
          j
        t          t
          j        f¦  «        rCt          j        |j	        d|dz  ¬¦  «         |j        �t          j        |j        ¦  «         dS dS t	          |t          ¦  «        r¢|j        j        dz  d|j        j        z  dz  z  |z  }d|j        j        z  dz  |z  }t          j        t%          |j        d¦  «        r|j        j        j	        n|j        j	        |¬¦  «         t          j        |j        j	        |¬¦  «         dS t	          |t,          ¦  «        rN| j                             ¦   «         }|j        }t          j        |j        j	        d||j        dz  z  ¬¦  «         dS t	          |t2          ¦  «        rAt          j        |j        j	        d|¬¦  «         t          j        |j        j        ¦  «         dS t	          |t6          ¦  «        re|                     ¦   «         D ]N\  }}|d	k    rCt          j        |d| j        j        t=          j        d| j        j        z  ¦  «        z  ¬¦  «         ŒOdS t	          |t@          ¦  «        r&t          j!        |j"        | j        j#        ¦  «         dS t	          |tH          ¦  «        r‹tK          |j        d
¦  «        rt|j        j&        }tO          j(        tO          j)        ||ftN          j*        ¬¦  «        ¦  «        }	|	 +                    dd||¦  «        }	t          j,        |j        |	¦  «         dS dS t	          |tZ          ¦  «        rƒt]          | j        j/        | j        j0        dz  z  d¦  «        }
ddtO          j1        d|
dtN          j2        ¬¦  «         3                    ¦   «         |
z  z  z  }t          j,        |j4        |¦  «         dS dS )zInitialize the weightsg        g{®Gáz”?)r�   ÚstdNrÄ   r7   r  )rŒ  zc_proj.weightrÅ   r¯   r   r¬   r­   r®   r   )5r�   Ú_init_weightsr¸   Úinitializer_factorÚ
isinstancer   r`  ÚinitÚnormal_r“   rÕ   r   re  rÆ   Úzeros_r  r–   Únum_hidden_layersr)  r  r  r  ÚClvpEncoderÚget_text_configÚ
projectionrT  rg  ÚClvpForCausalLMÚnamed_parametersÚinitializer_rangeÚmathÚsqrtÚ!ClvpModelForConditionalGenerationÚ	constant_Úlogit_scaleÚlogit_scale_init_valuerÂ   rÑ   rÅ   r)   rÒ   r’   rÓ   rÔ   Úcopy_rª   r±   r²   r³   r*   r´   r¥   r°   )r•   rŠ  ÚfactorÚin_proj_stdÚfc_stdr¸   Únameræ   rÛ   rÆ   r9   r°   r—   s               €r,   r�  z!ClvpPreTrainedModel._init_weights  sü  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ø”Ô/ˆÝ�f�bœlÑ+Ô+ð #	2ÝŒL˜œ¨S°f¸t±mÐDÑDÔDÐDÐDÐDÝ˜¥¤­FµB´IÐ >Ñ?Ô?ð !	2ÝŒL˜œ¨S°f¸t±mÐDÑDÔDÐDØŒ{Ð&Ý”˜FœKÑ(Ô(Ð(Ð(Ð(ð 'Ð&å˜¥Ñ/Ô/ð 	2Ø!œ=Ô4°dÑ:ÀÀFÄMÔDcÑ@cÐhlÑ?lÑmÐpvÑvˆKØ˜&œ-Ô3Ñ3¸Ñ<¸vÑEˆFÝŒLµ7¸6¼:ÀvÑ3NÔ3NÐe˜œœÔ/Ð/ÐTZÔT^ÔTeÐkqÐrÑrÔrÐrÝŒL˜œÔ*°Ð<Ñ<Ô<Ð<Ð<Ð<Ý˜¥Ñ,Ô,ð 	2Ø”[×0Ò0Ñ2Ô2ˆFØÔ.ˆFÝŒL˜Ô*Ô1¸ÀÈ6ÔK]Ð_cÑKcÑAdÐeÑeÔeÐeÐeÐeÝ˜Õ 7Ñ8Ô8ð 	2ÝŒL˜œÔ/°c¸vÐFÑFÔFÐFÝŒK˜œÔ,Ñ-Ô-Ð-Ð-Ð-Ý˜¥Ñ0Ô0ð 	2Ø!×2Ò2Ñ4Ô4ð ð ‘��aØ˜?Ò*Ð*Ý”LØ ¨¬Ô)FÍÌÐSTÐW[ÔWbÔWtÑStÑIuÔIuÑ)uðñ ô ð øðð õ
 ˜Õ AÑBÔBð 	2ÝŒN˜6Ô-¨t¬{Ô/QÑRÔRÐRÐRÐRÝ˜Õ 1Ñ2Ô2ð 		2Ý�v”}Ð&?Ñ@Ô@ð .Ø &¤Ô E�Ý”z¥%¤*¨m¸]Ð-KÕSXÔS]Ð"^Ñ"^Ô"^Ñ_Ô_�Ø—y’y  A }°mÑDÔD�Ý”
˜6œ;¨Ñ-Ô-Ð-Ð-Ð-ð	.ð .õ
 ˜Õ =Ñ>Ô>ð 	2Ý�d”kÔ0°T´[Ô5TÐWXÑ5XÑYÐ[]Ñ^Ô^ˆCØ˜e­¬°Q¸¸QÅeÄkÐ(RÑ(RÔ(R×(XÒ(XÑ(ZÔ(ZÐ]`Ñ(`ÑaÑbˆHÝŒJ�v”¨Ñ1Ô1Ð1Ð1Ð1ð	2ð 	2r.   )rt   ru   rv   r   ry   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_skip_keys_device_placementr  rE  rÂ   Ú_can_record_outputsr)   Úno_gradr   ÚModuler�  r§   r¨   s   @r,   rˆ  rˆ  ù  s–   ø€ € € € € € àÐÐÑØÐØ&*Ð#Ø#4Ð"5Ðà*Ð,<Ð=Ø'ðð Ðð
 €U„]�_„_ð'2 B¤Ið '2ð '2ð '2ð '2ð '2ñ „_ð'2ð '2ð '2ð '2ð '2r.   rˆ  c                   óæ   ‡ — e Zd ZU dZeed<   defˆ fd„Zd„ Zd„ Z	e
ee	 	 	 	 ddej        dz  dej        dz  d	ej        dz  d
ej        dz  dee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )r”  z­
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`ClvpEncoderLayer`].

    Args:
        config: ClvpConfig
    r¸   c                 ó>  •‡— t          ¦   «                              ‰¦  «         ‰| _        t          j        ‰j        ‰j        ¦  «        | _        ‰j        rt          ‰¦  «        nd | _
        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰¦  «        | _        t          j        ‰j        ‰j        ¬¦  «        | _        t          j        ‰j        ‰j        d¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r{   )r  )r[  rî   r¸   s     €r,   r]  z(ClvpEncoder.__init__.<locals>.<listcomp>@  s"   ø€ Ð$gÐ$gÐ$gÀ!Õ%5°fÑ%=Ô%=Ð$gÐ$gÐ$gr.   r  FrÈ   )r�   r�   r¸   r   r`  ra  r–   Útoken_embeddingÚuse_rotary_embeddingrª   rá   ri  rj  r“  Úlayersr  Úsequence_summaryrJ  r  Úfinal_layer_normrÕ   r²   r–  rn  Ú	post_initr  s    `€r,   r�   zClvpEncoder.__init__:  s÷   øø€ Ý‰Œ×Ò˜Ñ Ô Ð àˆŒÝ!œ|¨FÔ,=¸vÔ?QÑRÔRˆÔØGMÔGbÐlÕ;¸FÑCÔCÐCÐhlˆÔÝ”mÐ$gÐ$gÐ$gÐ$gÅuÈVÔMeÑGfÔGfÐ$gÑ$gÔ$gÑhÔhˆŒå 3°FÑ ;Ô ;ˆÔÝ "¤¨VÔ-?ÀVÔEZÐ [Ñ [Ô [ˆÔåœ) FÔ$6¸Ô8MÐTYÐZÑZÔZˆŒà&+ˆÔ#à�ŠÑÔÐÐÐr.   c                 ó   — | j         S rþ   ©r®  r£   s    r,   Úget_input_embeddingsz ClvpEncoder.get_input_embeddingsK  s   € ØÔ#Ð#r.   c                 ó   — || _         d S rþ   rµ  ©r•   rP   s     r,   Úset_input_embeddingsz ClvpEncoder.set_input_embeddingsN  s   € Ø$ˆÔÐÐr.   Nr]   rv  r^   rG   rä   r#   c                 óB  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }t          | j        ||¬¦  «        }|€Q|�|j        n|j        }t          j        |j        d         t
          j        |¬¦  «        }| 	                    d¦  «        }| j
        �|  
                    |¦  «        nd }|}| j        D ]}	 |	||||fi |¤Ž}Œ|                      |¦  «        }
|                      |
¦  «        }|                      |¦  «        }t          ||
|¬¦  «        S )Nú:You must specify exactly one of input_ids or inputs_embeds)r¸   rv  r^   r   rx  r   )ro   rp   rq   )rÌ   r®  r   r¸   r&   r)   r*   r:   r8  rA   rá   r°  r²  r±  r–  rn   )r•   r]   rv  r^   rG   rä   r&   rá   rr   Úencoder_layerrp   Úpooled_outputro   s                r,   r¡   zClvpEncoder.forwardQ  sz  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×0Ò0°Ñ;Ô;ˆMõ 3Ø”;Ø'Ø)ð
ñ 
ô 
ˆð ÐØ)2Ð)>�YÔ%Ð%ÀMÔDXˆFÝ œ<¨Ô(;¸AÔ(>ÅeÄjÐY_Ð`Ñ`Ô`ˆLØ'×1Ò1°!Ñ4Ô4ˆLà?CÔ?RÐ?^˜×,Ò,¨]Ñ;Ô;Ð;Ðdhˆà%ˆØ!œ[ð 	ð 	ˆMØ)˜MØØØØð	ð ð
 ðð ˆMˆMð !×1Ò1°-Ñ@Ô@Ðð ×-Ò-Ð.?Ñ@Ô@ˆð —’ Ñ/Ô/ˆå ØØ/Ø'ð
ñ 
ô 
ð 	
r.   )NNNN)rt   ru   rv   rw   r!   ry   r   r�   r¶  r¹  r   r   r   r)   r‰   r   r   rn   r¡   r§   r¨   s   @r,   r”  r”  /  s$  ø€ € € € € € ðð ð ÐÐÑð˜zð ð ð ð ð ð ð"$ð $ð $ð%ð %ð %ð  ØØð .2Ø15Ø26Ø04ð2
ð 2
àÔ# dÑ*ð2
ð Ô'¨$Ñ.ð2
ð Ô(¨4Ñ/ð	2
ð
 Ô&¨Ñ-ð2
ð Ð+Ô,ð2
ð 
ð2
ð 2
ð 2
ñ „^ñ „_ñ  Ôð2
ð 2
ð 2
ð 2
ð 2
r.   r”  c                   ó  ‡ — e Zd ZU dZeed<   ˆ fd„Zd„ Zd„ Ze	e
e	 	 	 	 	 	 	 ddej        dz  dej        dz  d	ej        dz  d
ej        dz  dedz  dej        dz  dedz  dee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚClvpDecoderzs
    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`ClvpDecoderLayer`]
    r¸   c                 ód  •‡ — t          ¦   «                              |¦  «         |‰ _        t          j        ‰ j        j        ‰ j        j        ¦  «        ‰ _        t          j        ‰ j        j        ‰ j        j        ¦  «        ‰ _	        t          j
        ‰ j        j        ¦  «        ‰ _        t          j        ˆ fd„t          ‰ j        j        ¦  «        D ¦   «         ¦  «        ‰ _        t          j        ‰ j        j        ‰ j        j        ¬¦  «        ‰ _        d‰ _        ‰                      ¦   «          d S )Nc                 ó<   •— g | ]}t          ‰j        |¬ ¦  «        ‘ŒS )rH  )rE  r¸   )r[  rƒ  r•   s     €r,   r]  z(ClvpDecoder.__init__.<locals>.<listcomp>š  s)   ø€ ÐfÐfÐf¸AÕ˜dœk°QÐ7Ñ7Ô7ÐfÐfÐfr.   r  F)r�   r�   r¸   r   r`  ra  r–   Úinput_embeds_layerrÅ   Úposition_embeds_layerr  Ú
embd_pdropÚdropri  rj  r“  r°  rJ  rK  Ú
layer_normrn  r³  r  s   ` €r,   r�   zClvpDecoder.__init__�  sð   øø€ Ý‰Œ×Ò˜Ñ Ô Ð àˆŒå"$¤,¨t¬{Ô/EÀtÄ{ÔG^Ñ"_Ô"_ˆÔÝ%'¤\°$´+Ô2UÐW[ÔWbÔWnÑ%oÔ%oˆÔ"å”J˜tœ{Ô5Ñ6Ô6ˆŒ	Ý”mØfÐfÐfÐfÅÀtÄ{ÔGdÑAeÔAeÐfÑfÔfñ
ô 
ˆŒõ œ, t¤{Ô'>ÀDÄKÔDbÐcÑcÔcˆŒà&+ˆÔ#ð 	�ŠÑÔÐÐÐr.   c                 ó   — | j         S rþ   ©rÂ  r£   s    r,   r¶  z ClvpDecoder.get_input_embeddings£  s   € ØÔ&Ð&r.   c                 ó   — || _         d S rþ   rÈ  ©r•   Únew_embeddingss     r,   r¹  z ClvpDecoder.set_input_embeddings¦  s   € Ø"0ˆÔÐÐr.   Nr]   r^   Útoken_type_idsrG   râ   rv  rã   rä   r#   c           	      ó2  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|j        d         }	|�|                     d|	¦  «        }|r|€t	          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}
t          j        |j        d         |j	        ¬¦  «        |
z   }| 
                    d¦  «        }|                      |¦  «        }||z   }t          | j        |||¬¦  «        }|}|�|                      |¦  «        }||z   }|                      |¦  «        }d|	f|                     d¦  «        fz   }| j        D ]} ||f||||dœ|¤Ž}Œ|                      |¦  «        }|                     |¦  «        }t#          ||¬	¦  «        S )
Nr»  r   r6   )r¸   r   r%   )r¸   rv  r^   râ   rP  )rp   râ   )rÌ   rÂ  r:   rÔ   r   r¸   Úget_seq_lengthr)   r*   r&   rA   rÃ  r   rÅ  rè   r°  rÆ  r   )r•   r]   r^   rÌ  rG   râ   rv  rã   rä   re   Úpast_seen_tokensr�  rr   Útoken_type_embedsÚoutput_shapeÚblocks                   r,   r¡   zClvpDecoder.forward©  s  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×3Ò3°IÑ>Ô>ˆMàÔ% aÔ(ˆØÐ%Ø+×0Ò0°°WÑ=Ô=ˆNàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLà×4Ò4°\ÑBÔBˆØ%¨Ñ7ˆå+Ø”;Ø'Ø)Ø+ð	
ñ 
ô 
ˆð &ˆàÐ%Ø $× 7Ò 7¸Ñ GÔ GÐØ)Ð,=Ñ=ˆMàŸ	š	 -Ñ0Ô0ˆð Øð
ð ×Ò Ñ#Ô#Ð%ñ&ˆð
 ”[ð 	ð 	ˆEØ!˜EØðà /Ø-Ø)Ø#ðð ð ðð ˆMˆMð Ÿš¨Ñ6Ô6ˆà%×*Ò*¨<Ñ8Ô8ˆå8Ø+Ø+ð
ñ 
ô 
ð 	
r.   ©NNNNNNN)rt   ru   rv   rw   r    ry   r�   r¶  r¹  r   r   r   r)   r‰   rx   r   rÓ   r   r   r   r¡   r§   r¨   s   @r,   r¿  r¿  ‰  sa  ø€ € € € € € ðð ð ÐÐÑðð ð ð ð ð&'ð 'ð 'ð1ð 1ð 1ð  ØØð .2Ø37Ø26Ø04Ø(,Ø26Ø!%ðE
ð E
àÔ# dÑ*ðE
ð Ô)¨DÑ0ðE
ð Ô(¨4Ñ/ð	E
ð
 Ô&¨Ñ-ðE
ð  ™ðE
ð Ô(¨4Ñ/ðE
ð ˜$‘;ðE
ð Ð+Ô,ðE
ð 
3ðE
ð E
ð E
ñ „^ñ „_ñ  ÔðE
ð E
ð E
ð E
ð E
r.   r¿  c                   ó   ‡ — e Zd ZU eed<   defˆ fd„Zd„ Zd„ Zee		 	 	 	 	 	 	 dde
j        dz  de
j        dz  de
j        dz  d	e
j        dz  d
edz  de
j        dz  dedz  dee         defd„¦   «         ¦   «         Zˆ xZS )Ú	ClvpModelr¸   c                 ó²   •— t          ¦   «                              |¦  «         || _        t          | j        ¦  «        | _        |                      ¦   «          d S rþ   )r�   r�   r¸   r¿  Údecoderr³  r  s     €r,   r�   zClvpModel.__init__ø  sK   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ" 4¤;Ñ/Ô/ˆŒð 	�ŠÑÔÐÐÐr.   c                 ó   — | j         j        S rþ   ©r×  rÂ  r£   s    r,   r¶  zClvpModel.get_input_embeddings   s   € ØŒ|Ô.Ð.r.   c                 ó   — || j         _        d S rþ   rÙ  r¸  s     r,   r¹  zClvpModel.set_input_embeddings  s   € Ø*/ˆŒÔ'Ð'Ð'r.   Nr]   r^   rÌ  rG   râ   rv  rã   rä   r#   c                 ó†   —  | j         d|||||||dœ|¤Ž}	t          |	j        |	j        |	j        |	j        |	j        ¬¦  «        S )N)r]   r^   rÌ  rG   râ   rv  rã   )rp   râ   rr   rs   Úcross_attentionsr{   )r×  r   rp   râ   rr   rs   rÜ  )
r•   r]   r^   rÌ  rG   râ   rv  rã   rä   Údecoder_outputss
             r,   r¡   zClvpModel.forward  s~   € ð FRÀTÄ\ð 	F
ØØ)Ø)Ø%Ø+Ø'Øð	F
ð 	F
ð ð	F
ð 	F
ˆõ 9Ø-Ô?Ø+Ô;Ø)Ô7Ø&Ô1Ø,Ô=ð
ñ 
ô 
ð 	
r.   rÓ  )rt   ru   rv   r    ry   r�   r¶  r¹  r   r   r)   r‰   rx   r   rÓ   r   r   r   r¡   r§   r¨   s   @r,   rÕ  rÕ  ô  sF  ø€ € € € € € àÐÐÑðÐ0ð ð ð ð ð ð ð/ð /ð /ð0ð 0ð 0ð Øð .2Ø37Ø26Ø04Ø(,Ø26Ø!%ð
ð 
àÔ# dÑ*ð
ð Ô)¨DÑ0ð
ð Ô(¨4Ñ/ð	
ð
 Ô&¨Ñ-ð
ð  ™ð
ð Ô(¨4Ñ/ð
ð ˜$‘;ð
ð Ð+Ô,ð
ð 
3ð
ð 
ð 
ñ „^ñ Ôð
ð 
ð 
ð 
ð 
r.   rÕ  zG
    The CLVP decoder model with a language modelling head on top.
    c                   ó´  ‡ — e Zd ZU eed<   ˆ fd„Zd„ Zd„ Zd„ Zde	j
        dz  dedz  d	eee	j
        f         d
ee	j
        edz  eee	j
        f         f         fd„Z	 	 	 	 dˆ fd„	Zee	 	 	 	 	 	 	 	 dde	j        dz  dedz  de	j        dz  de	j        dz  de	j        dz  de	j        dz  de	j        dz  dedz  dee         d
eez  fd„¦   «         ¦   «         Zˆ xZS )r—  r¸   c                 óX  •— t          ¦   «                              |¦  «         || _        t          | j        ¦  «        | _        t          j        | j        j        ¦  «        | _        t          j	        | j        j        | j        j
        d¬¦  «        | _        |                      ¦   «          d S )NTrÈ   )r�   r�   r¸   rÕ  r‰  r   rJ  r–   Ú
final_normrÕ   ra  Úlm_headr³  r  s     €r,   r�   zClvpForCausalLM.__init__0  s„   ø€ Ý‰Œ×Ò˜Ñ Ô Ð àˆŒÝ˜tœ{Ñ+Ô+ˆŒ
åœ, t¤{Ô'>Ñ?Ô?ˆŒÝ”y ¤Ô!8¸$¼+Ô:PÐW[Ð\Ñ\Ô\ˆŒð 	�ŠÑÔÐÐÐr.   c                 ó   — d S rþ   r{   r£   s    r,   Úget_output_embeddingsz%ClvpForCausalLM.get_output_embeddings<  s   € Øˆtr.   c                 ó$   — | j         j        j        S rþ   ©r‰  r×  rÂ  r£   s    r,   r¶  z$ClvpForCausalLM.get_input_embeddings?  s   € ØŒzÔ!Ô4Ð4r.   c                 ó(   — || j         j        _        d S rþ   rå  rÊ  s     r,   r¹  z$ClvpForCausalLM.set_input_embeddingsB  s   € Ø0>ˆŒ
ÔÔ-Ð-Ð-r.   ÚinputsNr`   Úmodel_kwargsr#   c                 ó(  — | j         }d„ |                     ¦   «         D ¦   «         }|                     |d¦  «        }|�|�t          d|› d|› d|› d|› d�	¦  «        ‚|�|}|dk    r)d	|v r%|                      |||¬
¦  «        |d<   |d	         d	}}|                     d¦  «        }|��Ñ| j        j                             t          j
        |j        d         df| j        j        |j        ¬¦  «        ¦  «        }|| j        j                             t          j
        |j        d         dfd|j        ¬¦  «        ¦  «        z  }t          j        ||gd¬¦  «        }t#          |d¦  «        r1|d                              ¦   «                              d¦  «        dz
  }n2t          j        d|j        d         t          j        |j        ¬¦  «        }|                     d¦  «                             |j        d         d¦  «        }|| j        j                             |¦  «        z
  |d	<   t          j        |d	         j        d         dft          j        | j        ¬¦  «        | j        j        z  |d<   |d	         d	|fS |                      |||¦  «        }|||fS )zT
        This function extracts the model-specific `inputs` for generation.
        c                 ó   — i | ]
\  }}|®||“ŒS rþ   r{   )r[  rC   rD   s      r,   ú
<dictcomp>z9ClvpForCausalLM._prepare_model_inputs.<locals>.<dictcomp>P  s   € ÐOÐOÐO¡  AÀÀ˜˜1ÀÀÀr.   Nz
`inputs`: z` were passed alongside z/ which is not allowed.Make sure to either pass z or z=...r]   rv  )rè  Úconditioning_embedsr   r   )Ú
fill_valuer&   r8   r^   r6   rx  )Úmain_input_nameÚitemsÚpoprÌ   Ú*_maybe_initialize_input_ids_for_generationÚgetr‰  r×  rÂ  r)   Úfullr:   r¸   r`   r&   rÃ  r~  rÑ   r8  ry  r*   rA   r}  r’   )	r•   rç  r`   rè  Ú
input_nameÚinputs_kwargrì  Úmel_start_token_embeddingrG   s	            r,   Ú_prepare_model_inputsz%ClvpForCausalLM._prepare_model_inputsE  s  € ð Ô)ˆ
àOÐO¨×);Ò);Ñ)=Ô)=ÐOÑOÔOˆà#×'Ò'¨
°DÑ9Ô9ˆØÐ#¨Ð(:ÝðI˜Vð Ið I¸Zð Ið IØ,2ðIð IØ8BðIð Ið Iñô ð ð Ð%Ø!ˆFà˜Ò$Ð$¨¸LÐ)HÐ)HØ(,×(WÒ(WØ˜°<ð )Xñ )ô )ˆL˜Ñ%ð ".¨oÔ!>À�JˆFð +×.Ò.Ð/DÑEÔEÐàÑ*Ø(,¬
Ô(:×(MÒ(MÝ”
Ø(Ô.¨qÔ1°1Ð5Ø#œ{Ô7Ø.Ô5ðñ ô ñ)ô )Ð%ð &¨¬Ô);×)QÒ)QÝ”
Ð/Ô5°aÔ8¸!Ð<ÈÐSfÔSmÐnÑnÔnñ*ô *ñ Ð%õ #(¤,Ð0CÐE^Ð/_ÐefÐ"gÑ"gÔ"gÐõ �|Ð%5Ñ6Ô6ð Ø+Ð,<Ô=×BÒBÑDÔD×KÒKÈBÑOÔOÐRSÑS��å$œ|ØÐ*Ô0°Ô3½5¼:ÐNaÔNhð ñ  ô  �ð (×1Ò1°!Ñ4Ô4×;Ò;Ð<OÔ<UÐVWÔ<XÐZ[Ñ\Ô\ˆLà,?À$Ä*ÔBT×BjÒBjØñCô Cñ -ˆL˜Ñ)õ ”
˜L¨Ô9Ô?ÀÔBÀAÐFÍeÌjÐaeÔalÐmÑmÔmØ”+Ô*ñ+ð ˜Ñ%ð
   Ô0°/À<ÐOÐOà×@Ò@ÀÈÐWcÑdÔdˆØ�z <Ð/Ð/r.   Fc                 óº   •— |j         d         } t          ¦   «         j        |f|||dœ|¤Ž}|�,|s*t          j        |gt          j        |j        ¬¦  «        |d<   |S )Nr6   )râ   rv  Úis_first_iterationrx  rG   )r:   r�   Úprepare_inputs_for_generationr)   rY   r8  r&   )
r•   r]   râ   rv  rì  rù  rä   Úinput_ids_lengthÚmodel_inputsr—   s
            €r,   rú  z-ClvpForCausalLM.prepare_inputs_for_generationˆ  s…   ø€ ð %œ?¨2Ô.Ðà<•u‘w”wÔ<Øð
à+Ø'Ø1ð	
ð 
ð
 ð
ð 
ˆð Ð*Ð3EÐ*Ý+0¬<Ð9IÐ8JÕRWÔR\ÐenÔeuÐ+vÑ+vÔ+vˆL˜Ñ(àÐr.   r]   râ   r^   rÌ  rG   rv  Úlabelsrã   rä   c	                 óB  —  | j         d|||||||dœ|	¤Ž}
|
j        }|                      |¦  «        }|                      |¦  «        }d}|�­|                     |j        ¦  «        }|ddd…dd…f                              ¦   «         }|ddd…f                              ¦   «         }t          ¦   «         } ||                     d| 	                    d¦  «        ¦  «        |                     d¦  «        ¦  «        }t          |||
j        |
j        |
j        |
j        ¬¦  «        S )a³  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        )r]   râ   r^   rÌ  rG   rv  rã   N.r6   r   )r~   r"   râ   rr   rs   rÜ  r{   )r‰  rp   rà  rá  rš   r&   rß   r   rÔ   rè   r   râ   rr   rs   rÜ  )r•   r]   râ   r^   rÌ  rG   rv  rý  rã   rä   Úoutputsrr   Ú	lm_logitsr~   Úshift_logitsÚshift_labelsÚloss_fcts                    r,   r¡   zClvpForCausalLM.forward¡  sK  € ð* >H¸T¼Zð 	>
ØØ+Ø)Ø)Ø%Ø'Øð	>
ð 	>
ð ð	>
ð 	>
ˆð  Ô1ˆà—O’O MÑ2Ô2ˆ	Ø—L’L Ñ+Ô+ˆ	àˆØÐØ—Y’Y˜yÔ/Ñ0Ô0ˆFà$ S¨#¨2¨#¨q¨q¨q [Ô1×<Ò<Ñ>Ô>ˆLØ! # q r r 'œ?×5Ò5Ñ7Ô7ˆLå'Ñ)Ô)ˆHØ�8˜L×-Ò-¨b°,×2CÒ2CÀBÑ2GÔ2GÑHÔHÈ,×J[ÒJ[Ð\^ÑJ_ÔJ_Ñ`Ô`ˆDå0ØØØ#Ô3Ø!Ô/ØÔ)Ø$Ô5ð
ñ 
ô 
ð 	
r.   rR  )NNNNNNNN)rt   ru   rv   r    ry   r�   rã  r¶  r¹  r)   r¦   rW   ÚdictÚstrrz   r÷  rú  r   r   r‰   r   rx   rÓ   r   r   r   r¡   r§   r¨   s   @r,   r—  r—  (  s	  ø€ € € € € € ð ÐÐÑð
ð 
ð 
ð 
ð 
ðð ð ð5ð 5ð 5ð?ð ?ð ?ðA0à”˜tÑ#ðA0ð ˜D‘jðA0ð ˜3 ¤Ð,Ô-ð	A0ð
 
ˆuŒ|˜S 4™Z¨¨c°5´<Ð.?Ô)@Ð@Ô	AðA0ð A0ð A0ð A0ðL ØØ Ø ðð ð ð ð ð ð2 Øð .2Ø(,Ø37Ø26Ø04Ø26Ø*.Ø!%ð4
ð 4
àÔ# dÑ*ð4
ð  ™ð4
ð Ô)¨DÑ0ð	4
ð
 Ô(¨4Ñ/ð4
ð Ô&¨Ñ-ð4
ð Ô(¨4Ñ/ð4
ð Ô  4Ñ'ð4
ð ˜$‘;ð4
ð Ð+Ô,ð4
ð 
Ð2Ñ	2ð4
ð 4
ð 4
ñ „^ñ Ôð4
ð 4
ð 4
ð 4
ð 4
r.   r—  z`
    The composite CLVP model with a text encoder, speech encoder and speech decoder model.
    c                   óÌ  ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Ze e	d¬¦  «        	 	 	 dd	ej        dz  d
ej
        dz  dej        dz  dee         deez  f
d„¦   «         ¦   «         Z	 	 	 	 	 	 ddej        dz  d	ej        dz  dej
        dz  dej
        dz  dej        dz  dedz  dej
        fd„Zee		 	 	 	 	 	 dd	ej        dz  dej
        dz  dej
        dz  d
ej
        dz  dej        dz  dedz  dee         deez  fd„¦   «         ¦   «         Z ej        ¦   «         	 	 	 	 	 	 dd	ej        dz  dej
        dz  dej        dz  dedz  dedz  dedz  fd„¦   «         Zˆ xZS )rœ  r¸   c                 ó  •— t          ¦   «                              |¦  «         t          |j        t          ¦  «        s%t          dt          |j        ¦  «        › d�¦  «        ‚t          |j        t          ¦  «        s%t          dt          |j        ¦  «        › d�¦  «        ‚t          |j        t          ¦  «        s%t          dt          |j        ¦  «        › d�¦  «        ‚t          |¦  «        | _        t          |j        ¦  «        | _        t          |j        ¦  «        | _        t          |j        ¦  «        | _        t#          j        t'          j        | j        j        ¦  «        ¦  «        | _        |                      ¦   «          d S )NzPconfig.text_config is expected to be of type `ClvpEncoderConfig` but is of type ú.zRconfig.speech_config is expected to be of type `ClvpEncoderConfig` but is of type zSconfig.decoder_config is expected to be of type `ClvpDecoderConfig` but is of type )r�   r�   r�  r_  r!   Ú	TypeErrorÚtypeÚspeech_configrZ  r    rT  Úconditioning_encoderr—  Úspeech_decoder_modelr”  Útext_encoder_modelÚspeech_encoder_modelr   r‘   r)   rY   r¸   rŸ  rž  r³  r  s     €r,   r�   z*ClvpModelForConditionalGeneration.__init__à  s�  ø€ Ý‰Œ×Ò˜Ñ Ô Ð å˜&Ô,Õ.?Ñ@Ô@ð 	Ýð0Ý˜Ô+Ñ,Ô,ð0ð 0ð 0ñô ð õ
 ˜&Ô.Õ0AÑBÔBð 	Ýð2Ý˜Ô-Ñ.Ô.ð2ð 2ð 2ñô ð õ
 ˜&Ô/Õ1BÑCÔCð 	Ýð3Ý˜Ô.Ñ/Ô/ð3ð 3ð 3ñô ð õ
 %<¸FÑ$CÔ$CˆÔ!å$3°FÔ4IÑ$JÔ$JˆÔ!å"-¨fÔ.@Ñ"AÔ"AˆÔÝ$/°Ô0DÑ$EÔ$EˆÔ!åœ<­¬°T´[Ô5WÑ(XÔ(XÑYÔYˆÔð 	�ŠÑÔÐÐÐr.   r   r#   c                 óÆ  — | j         j        j        }|dd…dd…f         }|j        d         }|| j        j         j        k    }|                     ||d         ¦  «        }t          j        |f|d         |j	        |j
        ¬¦  «        }t          j        |dd…         |j	        |j
        ¬¦  «        |dd…<   t          j        ||j	        ¬¦  «        }|                     ¦   «                              d¦  «                             t!          d|dz
  ¦  «        ¬¦  «                             d¦  «        }|                     dd	¬
¦  «        ||k    z  }t          j        |||¦  «        S )a#  
        This method modifies the output of the decoder model, such as replacing the `eos_token_id` and changing the
        last few tokens of each sequence.

        Args:
            speech_ids (`torch.LongTensor`):
                This refers to the output of the decoder model.
        Nr   r   )r&   r™   éýÿÿÿr%   r   )r±   TrQ   )r¸   rZ  Údecoder_fixing_codesr:   r  ra   r\   r)   ró  r&   r™   rY   r*   rW   rX   rZ   r±   rA   rV   rU   )	r•   r   r  re   Úis_stopÚfillerrh   ÚboundaryÚmasks	            r,   Úfix_speech_decoder_outputz;ClvpModelForConditionalGeneration.fix_speech_decoder_output  sM  € ð  $œ{Ô9ÔNÐØ    1 2 2 Ô&ˆ
ð Ô" 1Ô%ˆØ Ô 9Ô @Ô MÒMˆØ×+Ò+¨GÐ5IÈ!Ô5LÑMÔMˆ
å”˜W˜JÐ(<¸QÔ(?È
ÔHYÐakÔaqÐrÑrÔrˆÝ”lÐ#7¸¸¸Ô#;ÀJÔDUÐ]gÔ]mÐnÑnÔnˆˆrˆsˆs‰åŒl˜7¨:Ô+<Ð=Ñ=Ô=ˆØ—;’;‘=”=×'Ò'¨Ñ*Ô*×0Ò0µS¸¸GÀa¹KÑ5HÔ5HÐ0ÑIÔI×SÒSÐTUÑVÔVˆØ�{Š{˜1 dˆ{Ñ+Ô+¨s°hªÑ?ˆÝŒ{˜4 ¨Ñ4Ô4Ð4r.   zÈ
        This method can be used to extract text_embeds from a text. The text embeddings obtained by applying the
        projection layer to the pooled output of the CLVP text encoder model.
        rk   Nr]   Útext_encoder_inputs_embedsr^   rä   c                 ó$   —  | j         d|||dœ|¤ŽS )a$  
        text_encoder_inputs_embeds (`torch.FloatTensor`, *optional*):
            inputs_embeds for the text encoder model passed in place of `input_ids`.

        Examples:

        ```python
        >>> from transformers import ClvpProcessor, ClvpModelForConditionalGeneration

        >>> # Define the Text
        >>> text = "This is an example text."

        >>> # Define processor and model
        >>> processor = ClvpProcessor.from_pretrained("susnato/clvp_dev")
        >>> model = ClvpModelForConditionalGeneration.from_pretrained("susnato/clvp_dev")

        >>> # Generate processor output and text embeds
        >>> processor_output = processor(text=text, return_tensors="pt")
        >>> text_embeds = model.get_text_features(input_ids=processor_output["input_ids"])
        ```
        ©r]   rv  r^   r{   )r  )r•   r]   r  r^   rä   s        r,   Úget_text_featuresz3ClvpModelForConditionalGeneration.get_text_features  s9   € ðF 'ˆtÔ&ð 
ØØ4Ø)ð
ð 
ð ð	
ð 
ð 	
r.   ru  Ú"conditioning_encoder_inputs_embedsÚgeneration_configc                 ó:  — |€{|€|�|€t          d¦  «        ‚|€| j        } |j        di |¤Ž |                      ||||¬¦  «        }| j                             ||¬¦  «        }|                      |d         ¦  «        }|                      ||¬¦  «        }	|	d         S )a  
        This method can be used to extract speech_embeds. The speech embeddings are obtained by applying the speech
        model on speech_ids. If speech_ids is not present but both input_ids and input_features are given then the
        decoder model will be used to first generate the speech_ids and then applying the speech model.

        Args:
            speech_ids (`torch.LongTensor` of shape `(batch_size, num_speech_ids)`, *optional*):
                Speech Tokens. Padding will be ignored by default should you provide it. If speech_ids are provided
                then input_ids and input_features will be automatically ignored.
            input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
                Input text Tokens. Processed from the [`ClvpTokenizer`]. If speech_ids is not provided, then input_ids
                and input_features will be used.
            conditioning_encoder_inputs_embeds (`torch.FloatTensor`, *optional*):
                inputs_embeds for `ClvpConditioningEncoder`. Can be used in place of `input_ids`.
            attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding speech token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            generation_config (`GenerationConfig`, *optional*):
                generation config to control the generation of speech_ids if they are not provided.

        Returns:
            `torch.FloatTensor` of shape `(batch_size, output_dim)`:
                The speech embeddings obtained by applying the projection layer to the pooled output of the CLVP Speech
                Model.

        Examples:

        ```python
        >>> import datasets
        >>> from transformers import ClvpProcessor, ClvpModelForConditionalGeneration

        >>> # Define the Text and Load the Audio (We are taking an audio example from HuggingFace Hub using `datasets` library)
        >>> text = "This is an example text."
        >>> ds = datasets.load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> ds = ds.cast_column("audio", datasets.Audio(sampling_rate=22050))
        >>> audio = ds.sort("id")["audio"][0]
        >>> audio_sample, sr = audio["array"], audio["sampling_rate"]

        >>> # Define processor and model
        >>> processor = ClvpProcessor.from_pretrained("susnato/clvp_dev")
        >>> model = ClvpModelForConditionalGeneration.from_pretrained("susnato/clvp_dev")

        >>> # Generate processor output and model output
        >>> processor_output = processor(raw_speech=audio_sample, sampling_rate=sr, text=text, return_tensors="pt")
        >>> speech_embeds = model.get_speech_features(
        ...     input_ids=processor_output["input_ids"], input_features=processor_output["input_features"]
        ... )
        ```
        NzfEither speech_ids or input_ids/conditioning_encoder_inputs_embeds and input_features must be provided.©ru  r]   rv  r^   )rì  r  r   )r]   r^   r{   )rÌ   r  ré   r  r  Úgenerater  r  )
r•   r   r]   ru  r  r^   r  rä   rì  rÿ  s
             r,   Úget_speech_featuresz5ClvpModelForConditionalGeneration.get_speech_featuresI  sî   € ð@ ÐØÐ!Ð&HÐ&PÐUcÐUkÝ Ø|ñô ð ð !Ð(Ø$(Ô$:Ð!Ø$ÐÔ$Ð.Ð. vÐ.Ð.Ð.à"&×";Ò";Ø-Ø#Ø@Ø-ð	 #<ñ #ô #Ðð Ô2×;Ò;Ø$7Ø"3ð <ñ ô ˆJð
 ×7Ò7¸
À1¼ÑFÔFˆJà×+Ò+Ø Ø)ð ,ñ 
ô 
ˆð
 �qŒzÐr.   Úreturn_lossc                 óÊ  — |                       ||||¬¦  «        } | j        dd|i|¤Ž}	|	j        }
|
j        dk    r|
                     d¦  «        }
|                      |
¦  «        }
 | j        dd|
i|¤Ž} | j        d|||dœ|¤Ž}|j        }|j        }|| 	                    ddd¬	¦  «        z  }|| 	                    ddd¬	¦  «        z  }| j
                             ¦   «         }t          j        ||                     ¦   «         ¦  «        |z  }|                     ¦   «         }d
}|rt          |¦  «        }t!          ||||||j        |j        |	j        |j        |j        ¬¦
  «
        S )az  
        conditioning_encoder_inputs_embeds (`torch.FloatTensor`, *optional*):
            inputs_embeds for `ClvpConditioningEncoder`. Can be used in place of `input_ids`.
        text_encoder_inputs_embeds (`torch.FloatTensor`, *optional*):
            inputs_embeds for the text encoder model passed in place of `input_ids`.
        return_loss (`bool`, *optional*):
            Whether or not to return the contrastive loss.

        Examples:

        ```python
        >>> import datasets
        >>> from transformers import ClvpProcessor, ClvpModelForConditionalGeneration

        >>> # Define the Text and Load the Audio (We are taking an audio example from HuggingFace Hub using `datasets` library)
        >>> text = "This is an example text."

        >>> ds = datasets.load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> ds = ds.cast_column("audio", datasets.Audio(sampling_rate=22050))
        >>> audio = ds.sort("id")["audio"][0]
        >>> audio_sample, sr = audio["array"], audio["sampling_rate"]

        >>> # Define processor and model
        >>> processor = ClvpProcessor.from_pretrained("susnato/clvp_dev")
        >>> model = ClvpModelForConditionalGeneration.from_pretrained("susnato/clvp_dev")

        >>> # processor outputs and model outputs
        >>> processor_output = processor(raw_speech=audio_sample, sampling_rate=sr, text=text, return_tensors="pt")
        >>> outputs = model(
        ...     input_ids=processor_output["input_ids"],
        ...     input_features=processor_output["input_features"],
        ...     return_dict=True,
        ... )
        ```
        r  rv  r   r7   r]   r  r6   T©ræ   r9   rR   N)
r~   r€   r�   r‚   rƒ   r„   r…   r†   r‡   rˆ   r{   )r  r  r"   ÚndimrX   r  r  r  ro   Únormrž  Úexpr)   rë   Útr4   r}   rq   rr   )r•   r]   ru  r  r  r^   r"  rä   rì  rÝ  r   Úspeech_outputsÚtext_outputsrƒ   r‚   rž  r�   r€   r~   s                      r,   r¡   z)ClvpModelForConditionalGeneration.forward¨  sî  € ð` #×7Ò7Ø)ØØ<Ø)ð	 8ñ 
ô 
Ðð >W¸TÔ=Vð >
ð >
Ø-ð>
àð>
ð >
ˆð
 %Ô+ˆ
ð Œ?˜aÒÐØ#×*Ò*¨1Ñ-Ô-ˆJØ×3Ò3°JÑ?Ô?ˆ
à,E¨DÔ,Eð -
ð -
Ø ð-
àð-
ð -
ˆð
 +B¨$Ô*Að +
ØØ4Ø)ð+
ð +
ð ð	+
ð +
ˆð 'Ô-ˆØ"Ô)ˆð &¨×(:Ò(:¸QÀBÐPTÐ(:Ñ(UÔ(UÑUˆØ! K×$4Ò$4°q¸bÈ$Ð$4Ñ$OÔ$OÑOˆð Ô&×*Ò*Ñ,Ô,ˆÝœ, {°M·O²OÑ4EÔ4EÑFÔFÈÑTˆØ+×-Ò-Ñ/Ô/ÐàˆØð 	AÝ/°Ñ@Ô@ˆDåØØ/Ø+Ø#Ø'Ø*Ô8Ø .Ô <Ø"1Ô"?Ø'3Ô'AØ)7Ô)Eð
ñ 
ô 
ð 	
r.   Úpad_to_max_mel_tokensÚoutput_hidden_statesc                 ó   — |j         d         }|| j        j        j        dz
  k    r't	          d|› d| j        j        j        dz
  › �¦  «        ‚|€| j        }t          j        |¦  «        } |j        di |¤Ž}	| 	                    ¦   «          |  
                    |	                     ¦   «         ¦  «         t          ||d| j        j        j        | j        j        j        ¬¦  «        \  }}|                      |||¬¦  «        }
| j                             |
||¬	¦  «        }t%          |t&          ¦  «        r|j        }|�C||j         d         z
  }t*          j        j                             |d
|f| j        j        ¬¦  «        }|                      |¦  «        }|                      |||j        ¬¦  «        }|                      ||||j        ¬¦  «        }|j        }|j        }||                     ddd¬¦  «        z  }||                     ddd¬¦  «        z  }| j                              ¦   «         }t+          j!        || "                    ¦   «         ¦  «        |z  }| "                    ¦   «         }|j        s/||||||j#        |j#        f}|r||d         |j$        |j$        fz  }|S tK          ||||||j#        |j#        |j$        |j$        |j$        ¬¦
  «
        S )ab	  
        Generate method for `ClvpModelForConditionalGeneration`, this method calls the `generate` method of
        `ClvpForCausalLM` and then uses those generated `speech_ids` to process `text_embeds` and `speech_embeds` using
        `ClvpEncoder`.

        Args:
            input_ids (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
                Input text Tokens. Processed from the [`ClvpTokenizer`].
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding text token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            generation_config (`~generation.GenerationConfig`, *optional*):
                The generation configuration to be used as base parametrization for the generation call. `**kwargs`
                passed to generate matching the attributes of `generation_config` will override them. If
                `generation_config` is not provided, the default will be used, which had the following loading
                priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model
                configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s
                default values, whose documentation should be checked to parameterize generation.
            pad_to_max_mel_tokens (`int`, *optional*):
                Pads generated speech_ids to the specified value. This is to implement the same logic from the official
                repo, link: https://github.com/neonbjb/tortoise-tts/blob/80f89987a5abda5e2b082618cd74f9c7411141dc/tortoise/api.py#L430
                and to make sure the logits are same.
                This does not affect generation quality so please don't consider using it since it is less efficient.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of decoder model, text encoder and speech encoder models.

        Returns:
            `ClvpOutput` or tuple: A `ClvpOutput` (if `return_dict_in_generate=True` or when
            `config.return_dict_in_generate=True`) or a tuple.
        r6   r   z;Maximum sequence length reached! Found input_ids of length z:.Please make sure that the maximum length of input_ids is NF)rb   r`   ra   )ru  r]   r^   )rì  r  r,  r   rO   )r]   r,  Úreturn_dict)r]   r^   r,  r.  r7   Tr$  )
r   r€   r�   r‚   rƒ   r„   r…   r†   r‡   rˆ   r{   )&r:   r¸   rZ  rc  rÌ   r  ÚcopyÚdeepcopyré   ÚvalidateÚ_validate_model_kwargsrj   r_  r`   ra   r  r  r   r�  r   Ú	sequencesr)   r   r'   rT   r  r  Úreturn_dict_in_generater  ro   r&  rž  r'  rë   r(  rq   rr   r}   )r•   r]   ru  r^   r  r+  r,  rä   r¼   rè  rì  rÝ  r   Úpadding_neededr)  r*  rƒ   r‚   rž  r�   r€   r:  s                         r,   r   z*ClvpModelForConditionalGeneration.generate  sp  € ðb $œ/¨"Ô-ˆØ˜dœkÔ8ÔHÈ1ÑLÒMÐMÝð}Èoð }ð }ØLPÌKÔLfÔLvÐyzÑLzð}ð }ñô ð ð
 Ð$Ø $Ô 6Ðå œMÐ*;Ñ<Ô<ÐØ/Ð(Ô/Ð9Ð9°&Ð9Ð9ˆØ×"Ò"Ñ$Ô$Ð$Ø×#Ò# L×$5Ò$5Ñ$7Ô$7Ñ8Ô8Ð8õ %>ØØØØœÔ0Ô=ØœÔ0Ô=ð%
ñ %
ô %
Ñ!ˆ	�>ð #×7Ò7Ø)ØØ)ð 8ñ 
ô 
Ðð Ô3×<Ò<Ø 3Ø/Ø!5ð =ñ 
ô 
ˆõ
 �o¥{Ñ3Ô3ð 	3Ø(Ô2ˆJð !Ð,Ø2°ZÔ5EÀbÔ5IÑIˆNÝœÔ,×0Ò0Ø˜Q Ð/°tÔ7MÔ7Zð 1ñ ô ˆJð ×3Ò3°JÑ?Ô?ˆ
à,0×,EÒ,EØ Ø!5Ø)ÔAð -Fñ -
ô -
ˆð
 +/×*AÒ*AØØ)Ø!5Ø)ÔAð	 +Bñ +
ô +
ˆð 'Ô-ˆØ"Ô)ˆð &¨×(:Ò(:¸QÀBÐPTÐ(:Ñ(UÔ(UÑUˆØ! K×$4Ò$4°q¸bÈ$Ð$4Ñ$OÔ$OÑOˆð Ô&×*Ò*Ñ,Ô,ˆÝœ, {°M·O²OÑ4EÔ4EÑFÔFÈÑTˆØ+×-Ò-Ñ/Ô/Ðà Ô8ð 	àØ!ØØØØÔ*ØÔ,ðˆFð $ð ØØ# BÔ'Ø Ô.Ø"Ô0ðñ �ð ˆMåØ!Ø/Ø+Ø#Ø'Ø*Ô8Ø .Ô <Ø"1Ô"?Ø'3Ô'AØ)7Ô)Eð
ñ 
ô 
ð 	
r.   r†  )NNNNNN)rt   ru   rv   r   r�   r)   r‰   r  r   r   rx   r   r   rz   rn   r  r¦   r   r!  rÓ   r}   r¡   r©  rW   r   r§   r¨   s   @r,   rœ  rœ  Ú  sI  ø€ € € € € ð˜zð ð ð ð ð ð ðF5°EÔ4Dð 5ÈÔIYð 5ð 5ð 5ð 5ð8 Ø€^ððñ ô ð .2Ø?CØ26ð	!
ð !
àÔ# dÑ*ð!
ð %*Ô$5¸Ñ$<ð!
ð Ô(¨4Ñ/ð	!
ð
 Ð+Ô,ð!
ð 
Ð"Ñ	"ð!
ð !
ð !
ñô ñ Ôð!
ðJ /3Ø-1Ø37ØGKØ.2Ø59ð]ð ]àÔ$ tÑ+ð]ð Ô# dÑ*ð]ð Ô)¨DÑ0ð	]ð
 -2Ô,=ÀÑ,Dð]ð œ tÑ+ð]ð ,¨dÑ2ð]ð 
Ô	ð]ð ]ð ]ð ]ð~ Øð .2Ø37ØGKØ?CØ26Ø#'ði
ð i
àÔ# dÑ*ði
ð Ô)¨DÑ0ði
ð -2Ô,=ÀÑ,Dð	i
ð
 %*Ô$5¸Ñ$<ði
ð Ô(¨4Ñ/ði
ð ˜D‘[ði
ð Ð+Ô,ði
ð 
�Ñ	ði
ð i
ð i
ñ „^ñ Ôði
ðV €U„]�_„_ð .2Ø37Ø26Ø59Ø,0Ø,0ðW
ð W
àÔ# dÑ*ðW
ð Ô)¨DÑ0ðW
ð Ô(¨4Ñ/ð	W
ð
 ,¨dÑ2ðW
ð  # T™zðW
ð # T™kðW
ð W
ð W
ñ „_ðW
ð W
ð W
ð W
ð W
r.   rœ  )rœ  r—  rÕ  rˆ  r”  r¿  )r   )Nr   rM   r   TT)Orw   r/  rš  Úcollections.abcr   Údataclassesr   r)   r   Útorch.nnr   Ú r   r�  Úactivationsr	   r
   Úcache_utilsr   r   Ú
generationr   r   Úmasking_utilsr   r   Úmodeling_outputsr   r   r   Úmodeling_utilsr   Úprocessing_utilsr   Úpytorch_utilsr   rz  r   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úconfiguration_clvpr   r    r!   Ú
get_loggerrt   Úloggerr¦   r-   r4   r?   rL   rj   rn   r}   rª  r‹   rª   rÂ   r   r  r  r  r<  rE  rT  rˆ  r”  r¿  rÕ  r—  rœ  Ú__all__r{   r.   r,   ú<module>rH     s¿  ðð Ð à €€€Ø €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ø %Ð %Ð %Ð %Ð %Ð %à &Ð &Ð &Ð &Ð &Ð &Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ø JÐ JÐ JÐ JÐ JÐ JÐ JÐ Jðð ð ð ð ð ð ð ð ð ð
 .Ð -Ð -Ð -Ð -Ð -Ø &Ð &Ð &Ð &Ð &Ð &Ø #Ð #Ð #Ð #Ð #Ð #ðð ð ð ð ð ð ð ð ð ð ð ð ð ð 8Ð 7Ð 7Ð 7Ð 7Ð 7Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5ðð ð ð ð ð ð ð ð ð ð 
ˆÔ	˜HÑ	%Ô	%€ð`˜Uœ\ð `¨e¬lð `ð `ð `ð `ð
.¨U¬\ð .¸e¼lð .ð .ð .ð .ð(ð (ð (ð%ð %ð %ð %ð> ØØØØØð).ð ).ð ).ð ).ðX €ððñ ô ð ð7ð 7ð 7ð 7ð 7˜ñ 7ô 7ñ „ñô ð7ð" Ø
ð(Bð (Bð (Bð (Bð (B�ñ (Bô (Bñ „ñ „ð(BðXJð Jð Jð Jð J�"”)ñ Jô Jð Jð(7ð 7ð 7ð 7ð 7 B¤Iñ 7ô 7ð 7ð<o)ð o)ð o)ð o)ð o)˜œ	ñ o)ô o)ð o)ðd8ð 8ð 8ð 8ð 8˜"œ)ñ 8ô 8ð 8ð ð ð ð ð �R”Yñ ô ð ð(&ð &ð &ð &ð &�r”yñ &ô &ð &ðT`ð `ð `ð `ð `˜"œ)ñ `ô `ð `ðHð ð ð ð �R”Yñ ô ð ð"(ð (ð (ð (ð (�r”yñ (ô (ð (ðVG<ð G<ð G<ð G<ð G<˜bœiñ G<ô G<ð G<ðT ð22ð 22ð 22ð 22ð 22˜/ñ 22ô 22ñ „ð22ðjW
ð W
ð W
ð W
ð W
Ð%ñ W
ô W
ð W
ðth
ð h
ð h
ð h
ð h
Ð%ñ h
ô h
ð h
ðV ð0
ð 0
ð 0
ð 0
ð 0
Ð#ñ 0
ô 0
ñ „ð0
ðf €ððñ ô ð
j
ð j
ð j
ð j
ð j
Ð)¨?ñ j
ô j
ñô ð
j
ðZ €ððñ ô ð
N
ð N
ð N
ð N
ð N
Ð(;¸_ñ N
ô N
ñô ð
N
ðbð ð €€€r.   