§
    ‚ŠtjO“  ã                   ól  — d dl mZ d dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZmZmZ dd	lmZmZmZ dd
lmZmZ ddlmZ ddlmZ ddlmZmZmZmZ ddlm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3m4Z4m5Z5 ddl6m7Z7  e,j8        e9¦  «        Z:e) G d„ de$¦  «        ¦   «         Z; G d„ dej<        ¦  «        Z= G d„ dej<        ¦  «        Z> ed¦  «         G d„ dej<        ¦  «        ¦   «         Z? G d „ d!ej<        ¦  «        Z@d"„ ZA ed#¦  «        dGd$„¦   «         ZBd%ejC        d&eDd'ejC        fd(„ZE	 dHd*ej<        d+ejC        d,ejC        d-ejC        d.ejC        dz  d/eFd0eFd1e&e(         fd2„ZG eeB¦  «         G d3„ d4ej<        ¦  «        ¦   «         ZH G d5„ d6ej<        ¦  «        ZI G d7„ d8e¦  «        ZJ G d9„ d:e;¦  «        ZK G d;„ d<e¦  «        ZL G d=„ d>e;¦  «        ZM e)d?¬@¦  «         G dA„ dBe;¦  «        ¦   «         ZN e)dC¬@¦  «         G dD„ dEe;e7¦  «        ¦   «         ZOg dF¢ZPdS )Ié    )ÚCallable)ÚOptionalN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)Úuse_kernel_forward_from_hubÚuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_torchdynamo_compilingÚlogging)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )Ú	DiaConfigÚDiaDecoderConfigÚDiaEncoderConfig)ÚDiaGenerationMixinc                   óN   ‡ — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZddgZˆ fd„Zˆ xZS )ÚDiaPreTrainedModelÚconfigÚmodelTÚ	input_idsÚDiaEncoderLayerÚDiaDecoderLayerc                 ó  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rSt	          j        | j        j        t          j        ¬¦  «        | j        j	        z  }t          j        |j        |¦  «         d S d S )N©Údtype)ÚsuperÚ_init_weightsÚ
isinstanceÚDiaMultiChannelEmbeddingÚtorchÚaranger+   Únum_channelsÚlongÚ
vocab_sizeÚinitÚcopy_Úoffsets)ÚselfÚmoduler>   Ú	__class__s      €úb/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/dia/modeling_dia.pyr4   z DiaPreTrainedModel._init_weightsA   sy   ø€ Ý‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ6Ñ7Ô7ð 	0Ý”l 4¤;Ô#;Å5Ä:ÐNÑNÔNÐQUÔQ\ÔQgÑgˆGÝŒJ�v”~ wÑ/Ô/Ð/Ð/Ð/ð	0ð 	0ó    )Ú__name__Ú
__module__Ú__qualname__r%   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚmain_input_nameÚ_no_split_modulesr4   Ú__classcell__©rA   s   @rB   r*   r*   5   sz   ø€ € € € € € àÐÐÑØÐØ&*Ð#ØÐØ€NØÐØ!ÐØ!€OØ*Ð,=Ð>Ðð0ð 0ð 0ð 0ð 0ð 0ð 0ð 0ð 0rC   r*   c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )r6   a  In order to efficiently compute the audio embedding from the 9 different channels,
    we vectorize the embedding process by using a single embedding layer and an offset.
    Example:
    - num_embeds = 4
    - vocab_size = 8
    - num_channels = 3
    We would have offsets = [0, 8, 16]
    If audio_codes = [0, 1, 2, 3], [1, 3, 4, 7], [5, 6, 7, 8],
    then tokens = audio_codes + offsets
                = [0, 1, 2, 3, 9, 11, 12, 15, 21, 22, 23, 24]
    This allows us to use a single embedding layer for all channels.
    r+   c                 óZ  •— t          ¦   «                              ¦   «          t          j        |j        |j        z  |j        ¦  «        | _        |j        | _        |j        | _        t          j	        |j        t          j
        ¬¦  «        |j        z  }|                      d|d¬¦  «         d S )Nr1   r>   F©Ú
persistent)r3   Ú__init__r   Ú	Embeddingr;   r9   Úhidden_sizeÚembedr7   r8   r:   Úregister_buffer)r?   r+   r>   rA   s      €rB   rV   z!DiaMultiChannelEmbedding.__init__V   s’   ø€ Ý‰Œ×ÒÑÔÐÝ”\ &Ô"3°fÔ6IÑ"IÈ6ÔK]Ñ^Ô^ˆŒ
Ø!Ô-ˆÔØ"Ô/ˆÔÝ”,˜vÔ2½%¼*ÐEÑEÔEÈÔHYÑYˆØ×Ò˜Y¨¸EÐÑBÔBÐBÐBÐBrC   Úaudio_codesÚreturnc                 óX  — || j                              |j        ¦  «        z                        d|j        d         |j        d         z  ¦  «        }|                      |¦  «                             |j        d         |j        d         d| j        ¦  «        }|                     d¬¦  «        S )Néÿÿÿÿr$   é   r   ©Údim)r>   ÚtoÚdeviceÚviewÚshaperY   rX   Úsum)r?   r[   ÚtokensÚembedss       rB   Úforwardz DiaMultiChannelEmbedding.forward^   s•   € Ø ¤§¢°Ô0BÑ CÔ CÑC×IÒIØ�Ô! !Ô$ {Ô'8¸Ô';Ñ;ñ
ô 
ˆð —’˜FÑ#Ô#×(Ò(¨¬°a¬¸+Ô:KÈAÔ:NÐPRÐTXÔTdÑeÔeˆØ�zŠz˜aˆzÑ Ô Ð rC   )
rD   rE   rF   Ú__doc__r&   rV   r7   ÚTensorri   rP   rQ   s   @rB   r6   r6   H   s|   ø€ € € € € ðð ðCÐ/ð Cð Cð Cð Cð Cð Cð! 5¤<ð !°E´Lð !ð !ð !ð !ð !ð !ð !ð !rC   r6   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚDiaMLPc                 ó"  •— t          ¦   «                              ¦   «          || _        t          j        |j        d|j        z  d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          |j
                 | _        d S )Nr_   F©Úbias)r3   rV   r+   r   ÚLinearrX   Úintermediate_sizeÚgate_up_projÚ	down_projr   Ú
hidden_actÚactivation_fn©r?   r+   rA   s     €rB   rV   zDiaMLP.__init__g   sz   ø€ Ý‰Œ×ÒÑÔÐàˆŒÝœI fÔ&8¸!¸fÔ>VÑ:VÐ]bÐcÑcÔcˆÔÝœ 6Ô#;¸VÔ=OÐV[Ð\Ñ\Ô\ˆŒÝ# FÔ$5Ô6ˆÔÐÐrC   Úhidden_statesr\   c                 óº   — |                       |¦  «        }|                     dd¬¦  «        \  }}||                      |¦  «        z  }|                      |¦  «        S )Nr_   r^   r`   )rs   Úchunkrv   rt   )r?   rx   Ú	up_statesÚgates       rB   ri   zDiaMLP.forwardo   sX   € Ø×%Ò% mÑ4Ô4ˆ	à#Ÿ/š/¨!°˜/Ñ4Ô4‰ˆˆiØ × 2Ò 2°4Ñ 8Ô 8Ñ8ˆ	à�~Š~˜iÑ(Ô(Ð(rC   )rD   rE   rF   rV   r7   ÚFloatTensorri   rP   rQ   s   @rB   rm   rm   f   s`   ø€ € € € € ð7ð 7ð 7ð 7ð 7ð) UÔ%6ð )¸5Ô;Lð )ð )ð )ð )ð )ð )ð )ð )rC   rm   ÚRMSNormc                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
Ú
DiaRMSNormç�íµ ÷Æ°>Úepsr\   Nc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z9
        DiaRMSNorm is equivalent to T5LayerNorm
        N)r3   rV   r   Ú	Parameterr7   ÚonesÚweightÚvariance_epsilon)r?   rX   r‚   rA   s      €rB   rV   zDiaRMSNorm.__init__z   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐrC   rx   c                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )Nr_   r^   T)Úkeepdim)	r2   rb   r7   Úfloat32ÚpowÚmeanÚrsqrtr‡   r†   )r?   rx   Úinput_dtypeÚvariances       rB   ri   zDiaRMSNorm.forward‚   s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:rC   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)Útupler†   re   r‡   )r?   s    rB   Ú
extra_reprzDiaRMSNorm.extra_repr‰   s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIrC   )r�   )
rD   rE   rF   ÚfloatrV   r7   rk   ri   r’   rP   rQ   s   @rB   r€   r€   x   sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð JrC   r€   c                   óÔ   ‡ — e Zd ZU ej        ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d         de
dz  ded	ef         fd
„¦   «         Z ej        ¦   «         ed„ ¦   «         ¦   «         Zˆ xZS )ÚDiaRotaryEmbeddingÚinv_freqNr+   c                 ó²  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        |                      d|d¬¦  «         |                      d|                     ¦   «         d¬¦  «         d S )NÚ	rope_typeÚdefaultr–   FrT   Úoriginal_inv_freq)r3   rV   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr+   Úrope_parametersr˜   Úcompute_default_rope_parametersr   Úattention_scalingrZ   Úclone)r?   r+   rc   Úrope_init_fnr–   rA   s        €rB   rV   zDiaRotaryEmbedding.__init__�   sÊ   ø€ Ý‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(à×Ò˜Z¨¸eÐÑDÔDÐDØ×ÒÐ0°(·.².Ñ2BÔ2BÈuÐÑUÔUÐUÐUÐUrC   rc   ztorch.deviceÚseq_lenr\   ztorch.Tensorc                 óü   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «                             |t          j        ¬¦  «        |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNg      ð?r   r_   r1   )rc   r2   )	rž   ÚgetattrrX   Únum_attention_headsr7   r8   Úint64rb   r“   )r+   rc   r£   Úbasera   Úattention_factorr–   s          rB   rŸ   z2DiaRotaryEmbedding.compute_default_rope_parameters    sŒ   € ð& Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐð Ø•U”\ ! S¨!µ5´;Ð?Ñ?Ô?×BÒBÈ&ÕX]ÔXcÐBÑdÔdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)rC   c                 óN  — | j         d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  |                     ¦   «         |                     ¦   «         z   
                    dd¦  «        }t          j        ||fd¬	¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬
¦  «        |	                     |j        ¬
¦  «        fS )Nr   r^   r$   ÚmpsÚcpuF)Údevice_typeÚenabledr_   r`   r1   )r–   r“   Úexpandre   rb   rc   r5   ÚtypeÚstrr!   Ú	transposer7   ÚcatÚcosr    Úsinr2   )
r?   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr¯   ÚfreqsÚembr¶   r·   s
             rB   ri   zDiaRotaryEmbedding.forward¾   s·  € ð !œM¨$°°°°4¨-Ô8×>Ò>Ñ@Ô@×GÒGÈÔHZÐ[\ÔH]Ð_aÐcdÑeÔe×hÒhÐijÔiqÑrÔrÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	5ð 	5Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃBE&Å&E*Å-E*©N©NNN)rD   rE   rF   r7   rk   rG   r%   rV   Ústaticmethodr   Úintr‘   r“   rŸ   Úno_gradr   ri   rP   rQ   s   @rB   r•   r•   �   sù   ø€ € € € € € ØŒlÐÐÑðVð V˜yð Vð Vð Vð Vð Vð Vð  à#'Ø+/Ø"ð*ð *Ø˜DÑ ð*à˜Ô(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ô	%ð	*ð *ð *ñ „\ð*ð: €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <rC   r•   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr^   r_   r`   )re   r7   rµ   )r¸   Úx1Úx2s      rB   Úrotate_halfrÆ   Î   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'rC   Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezerÆ   )ÚqÚkr¶   r·   Úunsqueeze_dimÚq_embedÚk_embeds          rB   Úapply_rotary_pos_embrÏ   Õ   sc   € ð& �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐrC   rx   Ún_repr\   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r$   N)re   r±   Úreshape)rx   rÐ   ÚbatchÚnum_key_value_headsÚslenr¦   s         rB   Ú	repeat_kvrÖ   ï   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTrC   ç        r@   ÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr_   r   r^   )ra   r2   )ÚpÚtrainingr$   )rÖ   Únum_key_value_groupsr7   Úmatmulr´   r   Ú
functionalÚsoftmaxrŠ   rb   r2   rÝ   rá   Ú
contiguous)r@   rØ   rÙ   rÚ   rÛ   rÜ   rÝ   rÞ   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               rB   Úeager_attention_forwardrë   û   sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$rC   c                   óÚ   ‡ — e Zd ZdZddeez  dedefˆ fd„Z	 	 	 dde	j
        d	ee	j
        e	j
        f         dz  d
e	j
        dz  dedz  dee         dee	j
        e	j
        f         fd„Zˆ xZS )ÚDiaSelfAttentionú=Multi-headed attention from 'Attention Is All You Need' paperFr+   Ú	layer_idxÚ	is_causalc                 óÖ  •— t          ¦   «                              ¦   «          || _        || _        |j        | _        | j        j        | _        | j        j        p| j        | _        | j        | j        z  | _        t          |d|j        | j        z  ¦  «        | _
        d| _        d| _        || _        t          j        | j        | j        | j
        z  d¬¦  «        | _        t          j        | j        | j        | j
        z  d¬¦  «        | _        t          j        | j        | j        | j
        z  d¬¦  «        | _        t          j        | j        | j
        z  | j        d¬¦  «        | _        d S )Nr¦   r$   r×   Fro   )r3   rV   r+   rï   rX   r¨   Ú	num_headsrÔ   râ   r§   r¦   rÜ   Úattention_dropoutrð   r   rq   Úq_projÚk_projÚv_projÚo_proj)r?   r+   rï   rð   rA   s       €rB   rV   zDiaSelfAttention.__init__  s1  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ!Ô-ˆÔØœÔ8ˆŒØ#'¤;Ô#BÐ#TÀdÄnˆÔ Ø$(¤N°dÔ6NÑ$NˆÔ!Ý ¨
°FÔ4FÈ$Ì.Ñ4XÑYÔYˆŒØˆŒØ!$ˆÔØ"ˆŒå”i Ô 0°$´.À4Ä=Ñ2PÐW\Ð]Ñ]Ô]ˆŒÝ”i Ô 0°$Ô2JÈTÌ]Ñ2ZÐafÐgÑgÔgˆŒÝ”i Ô 0°$Ô2JÈTÌ]Ñ2ZÐafÐgÑgÔgˆŒÝ”i ¤°´Ñ >ÀÔ@PÐW\Ð]Ñ]Ô]ˆŒˆˆrC   Nrx   Úposition_embeddingsrÛ   Úpast_key_valuesrÞ   r\   c                 ó"  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|\  }}t          ||	||¦  «        \  }}	|�|                     |	|
| j	        ¦  «        \  }	}
t          j        | j        j        t          ¦  «        } || ||	|
|f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr^   r$   r_   r×   )rÝ   rÜ   )re   r¦   rô   rd   r´   rõ   rö   rÏ   Úupdaterï   r   Úget_interfacer+   Ú_attn_implementationrë   rá   ró   rÜ   rÒ   ræ   r÷   )r?   rx   rø   rÛ   rù   rÞ   Úinput_shapeÚhidden_shapeÚquery_statesrç   rè   r¶   r·   Úattention_interfacerê   ré   s                   rB   ri   zDiaSelfAttention.forward*  sÀ  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(rC   )Fr¿   )rD   rE   rF   rj   r'   r&   rÁ   ÚboolrV   r7   rk   r‘   r	   r   r   ri   rP   rQ   s   @rB   rí   rí     s  ø€ € € € € àGÐGð^ð ^Ð/Ð2BÑBð ^Èsð ^Ð_cð ^ð ^ð ^ð ^ð ^ð ^ð* IMØ.2Ø(,ð&)ð &)à”|ð&)ð # 5¤<°´Ð#=Ô>ÀÑEð&)ð œ tÑ+ð	&)ð
  ™ð&)ð Ð+Ô,ð&)ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð&)ð &)ð &)ð &)ð &)ð &)ð &)ð &)rC   rí   c                   ó²   ‡ — e Zd ZdZdedefˆ fd„Z	 	 ddej        dej        dej        dz  d	e	dz  d
e
e         deej        ej        dz  f         fd„Zˆ xZS )ÚDiaCrossAttentionrî   r+   rï   c                 ó²  •— t          ¦   «                              ¦   «          || _        || _        |j        | _        |j        | _        | j        j        | _        | j        j        | _	        | j        | j	        z  | _
        |j        | _        d| _        d| _        d| _        t!          j        | j        | j        | j        z  d¬¦  «        | _        t!          j        | j        | j	        | j        z  d¬¦  «        | _        t!          j        | j        | j	        | j        z  d¬¦  «        | _        t!          j        | j        | j        z  | j        d¬¦  «        | _        d S )Nr$   r×   Fro   )r3   rV   r+   rï   rX   Úcross_hidden_sizeÚcross_num_attention_headsrò   Úcross_num_key_value_headsrÔ   râ   Úcross_head_dimr¦   rÜ   ró   rð   r   rq   rô   rõ   rö   r÷   ©r?   r+   rï   rA   s      €rB   rV   zDiaCrossAttention.__init__V  s&  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔØœÔ>ˆŒØ#'¤;Ô#HˆÔ Ø$(¤N°dÔ6NÑ$NˆÔ!ØÔ-ˆŒØˆŒØ!$ˆÔØˆŒå”i Ô 0°$´.À4Ä=Ñ2PÐW\Ð]Ñ]Ô]ˆŒÝ”i Ô 6¸Ô8PÐSWÔS`Ñ8`ÐglÐmÑmÔmˆŒÝ”i Ô 6¸Ô8PÐSWÔS`Ñ8`ÐglÐmÑmÔmˆŒÝ”i ¤°´Ñ >ÀÔ@PÐW\Ð]Ñ]Ô]ˆŒˆˆrC   Nrx   Úcross_attention_statesrÛ   rù   rÞ   r\   c                 ó  — |j         d d…         }g |¢d‘| j        ‘R }g |j         d d…         ¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }	|�|j                             | j        ¦  «        nd}
|�;|
r9|j        j	        | j                 j
        }|j        j	        | j                 j        }n­|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|�3|j                             ||| j        ¦  «        \  }}d|j        | j        <   t          j        | j        j        t&          ¦  «        } || |	|||fd| j        i|¤Ž\  }}|                     g |¢d‘R ¦  «                             ¦   «         }|                      |¦  «        }||fS )Nr^   r$   r_   FTrÜ   )re   r¦   rô   rd   r´   Ú
is_updatedÚgetrï   Úcross_attention_cacheÚlayersÚkeysÚvaluesrõ   rö   rû   r   rü   r+   rý   rë   rÜ   rÒ   ræ   r÷   )r?   rx   r  rÛ   rù   rÞ   rþ   rÿ   Úcross_shaper   r  rç   rè   r  rê   ré   s                   rB   ri   zDiaCrossAttention.forwardi  s.  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆØMÐ.Ô4°S°b°SÔ9ÐM¸2ÐM¸t¼}ÐMÐMˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆàGVÐGb�_Ô/×3Ò3°D´NÑCÔCÐCÐhmˆ
ØÐ&¨:Ð&à(Ô>ÔEÀdÄnÔUÔZˆJØ*Ô@ÔGÈÌÔWÔ^ˆLˆLàŸšÐ%;Ñ<Ô<×AÒAÀ+ÑNÔN×XÒXÐYZÐ\]Ñ^Ô^ˆJØŸ;š;Ð'=Ñ>Ô>×CÒCÀKÑPÔP×ZÒZÐ[\Ð^_Ñ`Ô`ˆLàÐ*à+:Ô+P×+WÒ+WØØ Ø”Nñ,ô ,Ñ(�
˜Lð >B�Ô*¨4¬>Ñ:å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð%
ð %
ð ”Lð%
ð ð%
ð %
Ñ!ˆ�\ð "×)Ò)Ð*<¨KÐ*<¸Ð*<Ð*<Ñ=Ô=×HÒHÑJÔJˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(rC   ©NN)rD   rE   rF   rj   r&   rÁ   rV   r7   rk   r   r   r   r‘   ri   rP   rQ   s   @rB   r  r  S  sÞ   ø€ € € € € ØGÐGð^Ð/ð ^¸Cð ^ð ^ð ^ð ^ð ^ð ^ð. /3Ø6:ð1)ð 1)à”|ð1)ð !&¤ð1)ð œ tÑ+ð	1)ð
 -¨tÑ3ð1)ð Ð-Ô.ð1)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)rC   r  c                   óÄ   ‡ — e Zd Zdedefˆ fd„Z	 	 ddej        deej        ej        f         dz  dej        dz  de	e
         d	eej        ej        dz  f         f
d
„Zˆ xZS )r.   r+   rï   c                 ó  •— t          ¦   «                              ¦   «          t          |j        |j        ¬¦  «        | _        t          ||d¬¦  «        | _        t          |j        |j        ¬¦  «        | _        t          |¦  «        | _
        d S )N©r‚   F©rð   )r3   rV   r€   rX   Únorm_epsÚpre_sa_normrí   Úself_attentionÚpost_sa_normrm   Úmlpr
  s      €rB   rV   zDiaEncoderLayer.__init__ž  su   ø€ Ý‰Œ×ÒÑÔÐÝ% fÔ&8¸f¼oÐNÑNÔNˆÔÝ.¨v°yÈEÐRÑRÔRˆÔÝ& vÔ'9¸v¼ÐOÑOÔOˆÔÝ˜&‘>”>ˆŒˆˆrC   Nrx   rø   rÛ   rÞ   r\   c                 óÈ   — |}|                       |¦  «        } | j        |f||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }	||	z   }|S )N)rø   rÛ   )r  r  r  r  )
r?   rx   rø   rÛ   rÞ   ÚresidualÚnormed_statesÚself_attn_outputÚ_Úmlp_outs
             rB   ri   zDiaEncoderLayer.forward¥  s™   € ð !ˆØ×(Ò(¨Ñ7Ô7ˆØ1˜dÔ1Øð
à 3Ø)ð
ð 
ð ð	
ð 
ÑÐ˜!ð !Ð#3Ñ3ˆà ˆØ×)Ò)¨-Ñ8Ô8ˆØ—(’(˜=Ñ)Ô)ˆØ  7Ñ*ˆàÐrC   r  )rD   rE   rF   r'   rÁ   rV   r7   rk   r‘   r   r   ri   rP   rQ   s   @rB   r.   r.   �  sÔ   ø€ € € € € ð"Ð/ð "¸Cð "ð "ð "ð "ð "ð "ð IMØ.2ð	ð à”|ðð # 5¤<°´Ð#=Ô>ÀÑEðð œ tÑ+ð	ð
 Ð-Ô.ðð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ðð ð ð ð ð ð ð rC   r.   c                   ó    ‡ — e Zd ZeedœZdefˆ fd„Zee	e
	 d
dej        dej        dz  dee         defd	„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú
DiaEncoder)rx   Ú
attentionsr+   c                 ó¢  •‡— t          ¦   «                              ‰¦  «         ‰| _        t          j        ‰j        ‰j        ¦  «        | _        t          j        ˆfd„t          ‰j
        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t          ‰¬¦  «        | _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS © )r.   ©Ú.0rï   r+   s     €rB   ú
<listcomp>z'DiaEncoder.__init__.<locals>.<listcomp>Ê  ó#   ø€ ÐaÐaÐa°I�_˜V YÑ/Ô/ÐaÐaÐarC   r  ©r+   )r3   rV   r+   r   rW   r;   rX   Ú	embeddingÚ
ModuleListÚrangeÚnum_hidden_layersr  r€   r  Únormr•   Ú
rotary_embÚ	post_initrw   s    `€rB   rV   zDiaEncoder.__init__Ä  s´   øø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒåœ fÔ&7¸Ô9KÑLÔLˆŒÝ”mØaÐaÐaÐaÅÀvÔG_ÑA`ÔA`ÐaÑaÔañ
ô 
ˆŒõ ˜vÔ1°v´ÐGÑGÔGˆŒ	Ý,°FÐ;Ñ;Ô;ˆŒà�ŠÑÔÐÐÐrC   Nr-   rÛ   rÞ   r\   c                 ó`  — |                       |¦  «        }t          j        |j        d         |j        ¬¦  «        d d d …f         }t          | j        ||¬¦  «        }|                      ||¬¦  «        }| j        D ]} ||f|||dœ|¤Ž}Œ|  	                    |¦  «        }t          |¬¦  «        S )Nr^   ©rc   )r+   Úinputs_embedsrÛ   ©r¹   )rÛ   r¹   rø   )Úlast_hidden_state)r/  r7   r8   re   rc   r   r+   r4  r  r3  r   )r?   r-   rÛ   rÞ   rx   r¹   rø   Úencoder_layers           rB   ri   zDiaEncoder.forwardÑ  sæ   € ð Ÿš yÑ1Ô1ˆõ
 ”| I¤O°BÔ$7À	Ô@PÐQÑQÔQÐRVÐXYÐXYÐXYÐRYÔZˆå2Ø”;Ø'Ø)ð
ñ 
ô 
ˆð
 #Ÿošo¨mÈ,˜oÑWÔWÐà!œ[ð 	ð 	ˆMØ)˜MØðà-Ø)Ø$7ð	ð ð
 ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå°Ð?Ñ?Ô?Ð?rC   r¾   )rD   rE   rF   r.   rí   Ú_can_record_outputsr'   rV   r"   r#   r   r7   rk   r   r   r   ri   rP   rQ   s   @rB   r%  r%  ¾  sÙ   ø€ € € € € à(Ø&ðð Ðð
Ð/ð ð ð ð ð ð ð  ØØð /3ð@ð @à”<ð@ð œ tÑ+ð@ð Ð+Ô,ð	@ð
 
ð@ð @ð @ñ „^ñ „_ñ  Ôð@ð @ð @ð @ð @rC   r%  c                   óþ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej        deej        ej        f         dz  dej        dz  dej        dz  d	ej        dz  d
e	dz  deej        ej        dz  ej        dz  f         fd„Z
ˆ xZS )r/   r+   rï   c                 ó   •— t          ¦   «                              ¦   «          |j        | _        t	          ||d¬¦  «        | _        t          ||¦  «        | _        t          |j        |j	        ¬¦  «        | _
        t          |j        |j	        ¬¦  «        | _        t          |j        |j	        ¬¦  «        | _        t          |¦  «        | _        d S )NTr  r  )r3   rV   rX   Ú	embed_dimrí   r  r  Úcross_attentionr€   r  r  Úpre_ca_normÚpre_mlp_normrm   r  r
  s      €rB   rV   zDiaDecoderLayer.__init__÷  s«   ø€ Ý‰Œ×ÒÑÔÐØÔ+ˆŒÝ.¨v°yÈDÐQÑQÔQˆÔÝ0°¸ÑCÔCˆÔÝ% fÔ&8¸f¼oÐNÑNÔNˆÔÝ% fÔ&8¸f¼oÐNÑNÔNˆÔÝ& vÔ'9¸v¼ÐOÑOÔOˆÔÝ˜&‘>”>ˆŒˆˆrC   Nrx   rø   rÛ   Úencoder_hidden_statesÚencoder_attention_maskrù   r\   c                 óf  — |}t          |t          ¦  «        r|j        }|}	|                      |¦  «        }
 | j        |
|||fi |¤Ž\  }}|	|z   }|}	|                      |¦  «        }
 | j        |
|f||dœ|¤Ž\  }}|	|z   }|}	|                      |¦  «        }
|                      |
¦  «        }|	|z   }|S )N)rÛ   rù   )	r5   r   Úself_attention_cacher  r  rA  r@  rB  r  )r?   rx   rø   rÛ   rC  rD  rù   rÞ   Úself_attn_cacher  r   r!  r"  Úcross_statesr#  s                  rB   ri   zDiaDecoderLayer.forward  s  € ð *ˆÝ�oÕ':Ñ;Ô;ð 	CØ-ÔBˆOà ˆØ×(Ò(¨Ñ7Ô7ˆØ1˜dÔ1ØØØð ð
ð 
ð ð
ð 
ÑÐ˜!ð !Ð#3Ñ3ˆà ˆØ×(Ò(¨Ñ7Ô7ˆØ.˜$Ô.ØØ!ð
ð 2Ø+ð	
ð 
ð
 ð
ð 
‰ˆ�að ! <Ñ/ˆà ˆØ×)Ò)¨-Ñ8Ô8ˆØ—(’(˜=Ñ)Ô)ˆØ  7Ñ*ˆàÐrC   ©NNNNN)rD   rE   rF   r&   rÁ   rV   r7   rk   r‘   r   ri   rP   rQ   s   @rB   r/   r/   ö  s  ø€ € € € € ð"Ð/ð "¸Cð "ð "ð "ð "ð "ð "ð IMØ.2Ø59Ø6:Ø6:ð+ð +à”|ð+ð # 5¤<°´Ð#=Ô>ÀÑEð+ð œ tÑ+ð	+ð
  %œ|¨dÑ2ð+ð !&¤¨tÑ 3ð+ð -¨tÑ3ð+ð 
ˆuŒ|˜Uœ\¨DÑ0°%´,ÀÑ2EÐEÔ	Fð+ð +ð +ð +ð +ð +ð +ð +rC   r/   c                   óú   ‡ — e Zd ZdZeeedœZdefˆ fd„Z	e
ee	 	 	 	 	 ddej        dej        dz  dej        dz  d	ej        dz  d
ej        dz  dedz  dee         deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú
DiaDecoderz-Transformer Decoder Stack using DenseGeneral.)rx   r&  Úcross_attentionsr+   c                 ó¤  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t	          ‰¦  «        | _        t          j        ˆfd„t          ‰j	        ¦  «        D ¦   «         ¦  «        | _
        t          ‰j        ‰j        ¬¦  «        | _        t          ‰¬¦  «        | _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r)  )r/   r*  s     €rB   r,  z'DiaDecoder.__init__.<locals>.<listcomp>>  r-  rC   r  r.  )r3   rV   r9   r;   r6   Ú
embeddingsr   r0  r1  r2  r  r€   rX   r  r3  r•   r4  r5  rw   s    `€rB   rV   zDiaDecoder.__init__8  s¸   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø"Ô/ˆÔØ Ô+ˆŒÝ2°6Ñ:Ô:ˆŒÝ”mØaÐaÐaÐaÅÀvÔG_ÑA`ÔA`ÐaÑaÔañ
ô 
ˆŒõ ˜vÔ1°v´ÐGÑGÔGˆŒ	Ý,°FÐ;Ñ;Ô;ˆŒà�ŠÑÔÐÐÐrC   Nr-   r¹   rÛ   rC  rD  rù   rÞ   r\   c                 ót  — |                      ¦   «         dd…         \  }}	|�|                     ¦   «         nd}
|€3t          j        |	|j        ¬¦  «        |
z   }|                     d¦  «        }|                      |¦  «        }|€/t          ¦   «         s!|
|	z   }t          j        |||j        ¬¦  «        }t          | j
        |||¬¦  «        }t          | j
        |||¬¦  «        }|                      ||¬¦  «        }| j        D ]} |||||f|||dœ|¤Ž}Œ|                      |¦  «        }t          ||¬	¦  «        S )
a  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks)`):
            The original `decoder_input_ids` in 3D shape to facilitate more efficient computations.

            [What are input IDs?](../glossary#input-ids)
        Nr^   r   r7  )r+   r8  rÛ   rù   )r+   r8  rÛ   rC  r9  )rD  rù   r¹   )r:  rù   )ÚsizeÚget_seq_lengthr7   r8   rc   rÉ   rO  r   r…   r   r+   r   r4  r  r3  r   )r?   r-   r¹   rÛ   rC  rD  rù   rÞ   Ú
batch_sizeÚ
seq_lengthÚpast_key_values_lengthrx   Úmask_seq_lengthrø   Úlayers                  rB   ri   zDiaDecoder.forwardE  sœ  € ð( "+§¢Ñ!1Ô!1°#°2°#Ô!6Ñˆ
�JØETÐE` ×!?Ò!?Ñ!AÔ!AÐ!AÐfgÐàÐÝ œ<¨
¸9Ô;KÐLÑLÔLÐOeÑeˆLØ'×1Ò1°!Ñ4Ô4ˆLð Ÿš¨	Ñ2Ô2ˆàÐ!Õ*BÑ*DÔ*DÐ!à4°zÑAˆOÝ"œZ¨
°OÈIÔL\Ð]Ñ]Ô]ˆNå+Ø”;Ø'Ø)Ø+ð	
ñ 
ô 
ˆõ ";Ø”;Ø'Ø1Ø"7ð	"
ñ "
ô "
Ðð #Ÿošo¨mÈ,˜oÑWÔWÐà”[ð 	ð 	ˆEØ!˜EØð $ØØ%ðð (>Ø /Ø)ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå8Ø+Ø+ð
ñ 
ô 
ð 	
rC   rI  )rD   rE   rF   rj   r/   rí   r  r<  r&   rV   r"   r#   r   r7   rk   Ú
LongTensorr}   r   r   r   r   r‘   ri   rP   rQ   s   @rB   rK  rK  /  s>  ø€ € € € € Ø7Ð7ð )Ø&Ø-ðð ÐðÐ/ð ð ð ð ð ð ð  ØØð 15Ø.2Ø:>Ø:>Ø6:ðA
ð A
à”<ðA
ð Ô&¨Ñ-ðA
ð œ tÑ+ð	A
ð
  %Ô0°4Ñ7ðA
ð !&Ô 0°4Ñ 7ðA
ð -¨tÑ3ðA
ð Ð+Ô,ðA
ð 
3°UÑ	:ðA
ð A
ð A
ñ „^ñ „_ñ  ÔðA
ð A
ð A
ð A
ð A
rC   rK  z[
    The bare Dia model outputting raw hidden-states without any specific head on top.
    )Úcustom_introc                   óð   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	e	e
z  dz  d
edz  dedz  de
ez  fd„¦   «         ¦   «         Zˆ xZS )ÚDiaModelr+   c                 óä   •— t          ¦   «                              |¦  «         || _        t          |j        ¦  «        | _        t          |j        ¦  «        | _        |  	                    ¦   «          d S r¾   )
r3   rV   r+   r%  Úencoder_configÚencoderrK  Údecoder_configÚdecoderr5  rw   s     €rB   rV   zDiaModel.__init__’  s\   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ! &Ô"7Ñ8Ô8ˆŒÝ! &Ô"7Ñ8Ô8ˆŒØ�ŠÑÔÐÐÐrC   Nr-   rÛ   Údecoder_input_idsÚdecoder_position_idsÚdecoder_attention_maskÚencoder_outputsrù   Ú	use_cacher\   c	                 ó®  — |€|€t          d¦  «        ‚| j        r%| j        r|rt                               d¦  «         d}|r8|€6t          t          | j        ¬¦  «        t          | j        ¬¦  «        ¦  «        }|€ | j        d||dœ|	¤Ž}nct          |t          ¦  «        sNt          |d         t          |¦  «        dk    r|d         ndt          |¦  «        d	k    r|d	         nd¬
¦  «        }|d         j        d         d| j        j        j        }}}
|€.t          j        |
d|f| j        j        j        | j        ¬¦  «        }|j        d	k    r+|                     |
||¦  «                             dd	¦  «        } | j        d||||d         |||dœ|	¤Ž}t/          |j        |j        |j        |j        |j        |d         |j        |j        ¬¦  «        S )a\  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
        or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
            1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
            the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
            tened audio logits which are used to calculate the loss.

            2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
            Dia to calculate embeddings and subsequent steps more efficiently.

            If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
            `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
            [`DiaProcessor.__call__`] for more details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

            [What are position IDs?](../glossary#position-ids)
        NzXYou should either provide text ids or the cached text encodings. Neither has been found.zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr.  )r-   rÛ   r   r$   r_   )r:  rx   r&  r^   )rQ  Ú
fill_valuerc   )r-   r¹   rÛ   rC  rD  rù   re  )r:  rù   Údecoder_hidden_statesÚdecoder_attentionsrL  Úencoder_last_hidden_staterC  Úencoder_attentionsr)  )Ú
ValueErrorÚis_gradient_checkpointingrá   ÚloggerÚwarning_oncer   r
   r+   r^  r5   r   Úlenre   r_  r9   r7   ÚfullÚbos_token_idrc   ÚndimrÒ   r´   r`  r   r:  rù   rx   r&  rL  )r?   r-   rÛ   ra  rb  rc  rd  rù   re  rÞ   Úbszr£   ÚchannelsÚdecoder_outputss                 rB   ri   zDiaModel.forward™  sd  € ðH Ð Ð!8ÝØjñô ð ð Ô)ð 	"¨d¬mð 	"Øð "Ý×#Ò#Øpñô ð ð "�	àð 	v˜Ð0Ý1µ,ÀdÄkÐ2RÑ2RÔ2RÕT`ÐhlÔhsÐTtÑTtÔTtÑuÔuˆOàÐ"Ø*˜dœlð Ø#Ø-ðð ð ðð ˆOˆOõ ˜O­_Ñ=Ô=ð 	Ý-Ø"1°!Ô"4Ý47¸Ñ4HÔ4HÈ1Ò4LÐ4L˜o¨aÔ0Ð0ÐRVÝ14°_Ñ1EÔ1EÈÒ1IÐ1I˜?¨1Ô-Ð-Ètðñ ô ˆOð #2°!Ô"4Ô":¸1Ô"=¸rÀ4Ä;ÔC]ÔCj�hˆWˆØÐ$Ý %¤
Ø˜1˜hÐ'°D´KÔ4NÔ4[ÐdhÔdoð!ñ !ô !Ðð Ô! QÒ&Ð&Ø 1× 9Ò 9¸#¸xÈÑ QÔ Q× [Ò [Ð\]Ð_`Ñ aÔ aÐà&˜$œ,ð 	
Ø'Ø-Ø1Ø"1°!Ô"4Ø#1Ø+Øð	
ð 	
ð ð	
ð 	
ˆõ "Ø-Ô?Ø+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5°aÔ&8Ø"1Ô"?Ø.Ô9ð	
ñ 	
ô 	
ð 		
rC   )NNNNNNNN)rD   rE   rF   r%   rV   r   r   r7   rX  r   r‘   r   r  r   ri   rP   rQ   s   @rB   r[  r[  Œ  s9  ø€ € € € € ð˜yð ð ð ð ð ð ð Øð .2Ø26Ø59Ø8<Ø:>Ø:>Ø6:Ø!%ð]
ð ]
àÔ# dÑ*ð]
ð Ô(¨4Ñ/ð]
ð !Ô+¨dÑ2ð	]
ð
 $Ô.°Ñ5ð]
ð !&Ô 0°4Ñ 7ð]
ð )¨5Ñ0°4Ñ7ð]
ð -¨tÑ3ð]
ð ˜$‘;ð]
ð 
Ð#Ñ	#ð]
ð ]
ð ]
ñ Ôñ „^ð]
ð ]
ð ]
ð ]
ð ]
rC   r[  zl
    The Dia model consisting of a (byte) text encoder and audio decoder with a prediction head on top.
    c                   ó  ‡ — e Zd ZdZdZdefˆ fd„Zee	 	 	 	 	 	 	 	 	 dde	j
        dz  de	j
        dz  de	j
        dz  d	e	j
        dz  d
e	j
        dz  deez  dz  dedz  dedz  de	j
        dz  deez  fd„¦   «         ¦   «         Zˆ xZS )ÚDiaForConditionalGenerationr,   )Úaudior+   c                 ó`  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        |j        j        | _        |j        j        | _        t          j	        |j        j
        | j        | j        z  d¬¦  «        | _        d| _        |                      ¦   «          d S )NFro   ÚForMaskedLM)r3   rV   r+   r[  r,   r_  r9   r;   r   rq   rX   Úlogits_denseÚ	loss_typer5  rw   s     €rB   rV   z$DiaForConditionalGeneration.__init__  sž   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ˜fÑ%Ô%ˆŒ
à"Ô1Ô>ˆÔØ Ô/Ô:ˆŒÝœIØÔ!Ô-°Ô0AÀDÄOÑ0SÐ[`ð
ñ 
ô 
ˆÔð 'ˆŒð 	�ŠÑÔÐÐÐrC   Nr-   rÛ   ra  rb  rc  rd  rù   re  Úlabelsr\   c
                 óþ  —  | j         d	||||||||dœ|
¤Ž}|d         }|j        d         }|                      |¦  «                             |d| j        | j        f¦  «                             dd¦  «                             ¦   «                              || j        z  d| j        ¦  «        }d}|	� | j        d	||	| j        dœ|
¤Ž}t          |||j
        |j        |j        |j        |j        |j        |j        ¬¦	  «	        S )
a   
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
        or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
            1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
            the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
            tened audio logits which are used to calculate the loss.

            2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
            Dia to calculate embeddings and subsequent steps more efficiently.

            If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
            `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
            [`DiaProcessor.__call__`] for more details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

            [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size * num_codebooks,)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in
            `[0, ..., config.decoder_config.vocab_size - 1]` or -100. Tokens with indices set to `-100`
            are ignored (masked).
        )r-   rÛ   ra  rb  rc  rd  rù   re  r   r^   r$   r_   N)Úlogitsr~  r;   )	Úlossr€  rù   rh  ri  rL  rj  rC  rk  r)  )r,   re   r|  rd   r9   r;   r´   ræ   Úloss_functionr   rù   rh  ri  rL  rj  rC  rk  )r?   r-   rÛ   ra  rb  rc  rd  rù   re  r~  rÞ   Úoutputsr:  rS  Úaudio_logitsr�  s                   rB   ri   z#DiaForConditionalGeneration.forward  s<  € ðR �$”*ð 

ØØ)Ø/Ø!5Ø#9Ø+Ø+Øð

ð 

ð ð

ð 

ˆð $ AœJÐØ&Ô,¨QÔ/ˆ
ð ×ÒÐ/Ñ0Ô0ßŠT�:˜r 4Ô#4°d´oÐFÑGÔGßŠY�q˜!‰_Œ_ßŠZ‰\Œ\ßŠT�*˜tÔ0Ñ0°"°d´oÑFÔFð 	ð ˆØÐØ%�4Ô%Ðo¨\À&ÐUYÔUdÐoÐoÐhnÐoÐoˆDåØØØ#Ô3Ø")Ô"?Ø&Ô9Ø$Ô5Ø&-Ô&GØ")Ô"?Ø&Ô9ð

ñ 

ô 

ð 
	
rC   )	NNNNNNNNN)rD   rE   rF   rH   Úoutput_modalitiesr%   rV   r   r   r7   rX  r   r‘   r   r  r   ri   rP   rQ   s   @rB   rx  rx  û  s\  ø€ € € € € ð  ÐØ"Ðð˜yð ð ð ð ð ð ð Øð .2Ø26Ø59Ø8<Ø:>Ø:>Ø6:Ø!%Ø*.ðL
ð L
àÔ# dÑ*ðL
ð Ô(¨4Ñ/ðL
ð !Ô+¨dÑ2ð	L
ð
 $Ô.°Ñ5ðL
ð !&Ô 0°4Ñ 7ðL
ð )¨5Ñ0°4Ñ7ðL
ð -¨tÑ3ðL
ð ˜$‘;ðL
ð Ô  4Ñ'ðL
ð 
�Ñ	 ðL
ð L
ð L
ñ Ôñ „^ðL
ð L
ð L
ð L
ð L
rC   rx  )r[  r*   rx  )r$   )r×   )QÚcollections.abcr   Útypingr   r7   r   Ú r   r<   Úactivationsr   Úcache_utilsr	   r
   r   Úintegrationsr   r   r   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r    Úutils.genericr!   r"   Úutils.output_capturingr#   Úconfiguration_diar%   r&   r'   Úgeneration_diar(   Ú
get_loggerrD   rn  r*   ÚModuler6   rm   r€   r•   rÆ   rÏ   rk   rÁ   rÖ   r“   rë   rí   r  r.   r%  r/   rK  r[  rx  Ú__all__r)  rC   rB   ú<module>r›     só  ðð* %Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ fÐ fÐ fÐ fÐ fÐ fÐ fÐ fÐ fÐ fØ JÐ JÐ JÐ JÐ JÐ JÐ JÐ JØ BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð LÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lØ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø LÐ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LØ .Ð .Ð .Ð .Ð .Ð .ð 
ˆÔ	˜HÑ	%Ô	%€ð ð0ð 0ð 0ð 0ð 0˜ñ 0ô 0ñ „ð0ð$!ð !ð !ð !ð !˜rœyñ !ô !ð !ð<)ð )ð )ð )ð )ˆRŒYñ )ô )ð )ð$ Ð˜YÑ'Ô'ðJð Jð Jð Jð J�”ñ Jô Jñ (Ô'ðJð(><ð ><ð ><ð ><ð ><˜œñ ><ô ><ð ><ðB(ð (ð (ð ÐÐ*Ñ+Ô+ðð ð ñ ,Ô+ðð2	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð2 ÐÐ)Ñ*Ô*ð;)ð ;)ð ;)ð ;)ð ;)�r”yñ ;)ô ;)ñ +Ô*ð;)ð|G)ð G)ð G)ð G)ð G)˜œ	ñ G)ô G)ð G)ðTð ð ð ð Ð0ñ ô ð ðB5@ð 5@ð 5@ð 5@ð 5@Ð#ñ 5@ô 5@ð 5@ðp6ð 6ð 6ð 6ð 6Ð0ñ 6ô 6ð 6ðrZ
ð Z
ð Z
ð Z
ð Z
Ð#ñ Z
ô Z
ð Z
ðz €ððñ ô ð
g
ð g
ð g
ð g
ð g
Ð!ñ g
ô g
ñô ð
g
ðT €ððñ ô ð
a
ð a
ð a
ð a
ð a
Ð"4Ð6Hñ a
ô a
ñô ð
a
ðH LÐ
KÐ
K€€€rC   