§
    ‚ŠtjÑÁ  ã                   óØ  — d Z ddlZddlZddlZddlmZ ddlmZ ddlmZ ddl	m
Z ddlmZ dd	lmZmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZmZ ddlmZ ddlm Z m!Z!m"Z" ddl#m$Z$  e"j%        e&¦  «        Z' G d„ dej(        ¦  «        Z) G d„ dej(        ¦  «        Z* G d„ dej(        ¦  «        Z+ G d„ dej(        ¦  «        Z, G d„ dej(        ¦  «        Z- G d„ dej(        ¦  «        Z. G d„ dej(        ¦  «        Z/ G d „ d!e¦  «        Z0e  G d"„ d#e¦  «        ¦   «         Z1 G d$„ d%e1¦  «        Z2 G d&„ d'ej(        ¦  «        Z3 e d(¬)¦  «         G d*„ d+e1e¦  «        ¦   «         Z4d+d#gZ5dS ),zPyTorch Pop2Piano model.é    N)Únn)ÚCrossEntropyLoss)ÚGenerationConfigé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutput)ÚPreTrainedModel)Úauto_docstringÚis_torchdynamo_compilingÚloggingé   )ÚPop2PianoConfigc                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚPop2PianoLayerNormç�íµ ÷Æ°>c                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )zj
        Construct a layernorm module in the Pop2Piano style. No bias and no subtraction of mean.
        N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚ	__class__s      €ún/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/pop2piano/modeling_pop2piano.pyr   zPop2PianoLayerNorm.__init__*   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐó    c                 óh  — |                      t          j        ¦  «                             d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        j        t          j	        t          j
        fv r|                      | j        j        ¦  «        }| j        |z  S )Né   éÿÿÿÿT)Úkeepdim)Útor    Úfloat32ÚpowÚmeanÚrsqrtr#   r"   ÚdtypeÚfloat16Úbfloat16)r$   Úhidden_statesÚvariances      r(   ÚforwardzPop2PianoLayerNorm.forward2   s–   € ð !×#Ò#¥E¤MÑ2Ô2×6Ò6°qÑ9Ô9×>Ò>¸rÈ4Ð>ÑPÔPˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆð Œ;Ô¥¤µ´Ð ?Ð?Ð?Ø)×,Ò,¨T¬[Ô->Ñ?Ô?ˆMàŒ{˜]Ñ*Ð*r)   )r   )Ú__name__Ú
__module__Ú__qualname__r   r8   Ú__classcell__©r'   s   @r(   r   r   )   sL   ø€ € € € € ð$ð $ð $ð $ð $ð $ð+ð +ð +ð +ð +ð +ð +r)   r   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚPop2PianoDenseActDenseÚconfigc                 óJ  •— t          ¦   «                              ¦   «          t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j        |j	        ¦  «        | _
        t          |j                 | _        d S ©NF©Úbias)r   r   r   ÚLinearÚd_modelÚd_ffÚwiÚwoÚDropoutÚdropout_rateÚdropoutr   Údense_act_fnÚact©r$   r@   r'   s     €r(   r   zPop2PianoDenseActDense.__init__D   sx   ø€ Ý‰Œ×ÒÑÔÐÝ”)˜FœN¨F¬K¸eÐDÑDÔDˆŒÝ”)˜FœK¨¬¸eÐDÑDÔDˆŒÝ”z &Ô"5Ñ6Ô6ˆŒÝ˜&Ô-Ô.ˆŒˆˆr)   c                 ó°  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }t          | j        j        t          j        ¦  «        r]|j        | j        j        j        k    rC| j        j        j        t          j	        k    r$| 
                    | j        j        j        ¦  «        }|                      |¦  «        }|S ©N)rH   rN   rL   Ú
isinstancerI   r"   r    ÚTensorr3   Úint8r.   )r$   r6   s     r(   r8   zPop2PianoDenseActDense.forwardK   s¨   € ØŸš Ñ.Ô.ˆØŸš Ñ/Ô/ˆØŸš ]Ñ3Ô3ˆå�t”w”~¥u¤|Ñ4Ô4ð	CàÔ# t¤w¤~Ô';Ò;Ð;Ø””Ô$­¬
Ò2Ð2à)×,Ò,¨T¬W¬^Ô-AÑBÔBˆMØŸš Ñ.Ô.ˆØÐr)   ©r9   r:   r;   r   r   r8   r<   r=   s   @r(   r?   r?   C   sS   ø€ € € € € ð/˜ð /ð /ð /ð /ð /ð /ðð ð ð ð ð ð r)   r?   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚPop2PianoDenseGatedActDenser@   c                 ó–  •— t          ¦   «                              ¦   «          t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j	        |j
        ¦  «        | _        t          |j                 | _        d S rB   )r   r   r   rE   rF   rG   Úwi_0Úwi_1rI   rJ   rK   rL   r   rM   rN   rO   s     €r(   r   z$Pop2PianoDenseGatedActDense.__init__[   s”   ø€ Ý‰Œ×ÒÑÔÐÝ”I˜fœn¨f¬kÀÐFÑFÔFˆŒ	Ý”I˜fœn¨f¬kÀÐFÑFÔFˆŒ	Ý”)˜FœK¨¬¸eÐDÑDÔDˆŒÝ”z &Ô"5Ñ6Ô6ˆŒÝ˜&Ô-Ô.ˆŒˆˆr)   c                 óà  — |                       |                      |¦  «        ¦  «        }|                      |¦  «        }||z  }|                      |¦  «        }t	          | j        j        t          j        ¦  «        r]|j	        | j        j        j	        k    rC| j        j        j	        t          j
        k    r$|                     | j        j        j	        ¦  «        }|                      |¦  «        }|S rQ   )rN   rY   rZ   rL   rR   rI   r"   r    rS   r3   rT   r.   )r$   r6   Úhidden_geluÚhidden_linears       r(   r8   z#Pop2PianoDenseGatedActDense.forwardc   sÀ   € Ø—h’h˜tŸyšy¨Ñ7Ô7Ñ8Ô8ˆØŸ	š	 -Ñ0Ô0ˆØ# mÑ3ˆØŸš ]Ñ3Ô3ˆõ �t”w”~¥u¤|Ñ4Ô4ð	CàÔ# t¤w¤~Ô';Ò;Ð;Ø””Ô$­¬
Ò2Ð2à)×,Ò,¨T¬W¬^Ô-AÑBÔBˆMàŸš Ñ.Ô.ˆØÐr)   rU   r=   s   @r(   rW   rW   Z   sS   ø€ € € € € ð/˜ð /ð /ð /ð /ð /ð /ðð ð ð ð ð ð r)   rW   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚPop2PianoLayerFFr@   c                 ó$  •— t          ¦   «                              ¦   «          |j        rt          |¦  «        | _        nt          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          j        |j        ¦  «        | _        d S )N©r&   )r   r   Úis_gated_actrW   ÚDenseReluDenser?   r   rF   Úlayer_norm_epsilonÚ
layer_normr   rJ   rK   rL   rO   s     €r(   r   zPop2PianoLayerFF.__init__y   sy   ø€ Ý‰Œ×ÒÑÔÐØÔð 	AÝ"=¸fÑ"EÔ"EˆDÔÐå"8¸Ñ"@Ô"@ˆDÔå,¨V¬^ÀÔAZÐ[Ñ[Ô[ˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr)   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }||                      |¦  «        z   }|S rQ   )re   rc   rL   )r$   r6   Úforwarded_statess      r(   r8   zPop2PianoLayerFF.forwardƒ   sF   € ØŸ?š?¨=Ñ9Ô9ÐØ×.Ò.Ð/?Ñ@Ô@ÐØ%¨¯ªÐ5EÑ(FÔ(FÑFˆØÐr)   rU   r=   s   @r(   r_   r_   x   sS   ø€ € € € € ð7˜ð 7ð 7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð r)   r_   c                   óf   ‡ — e Zd Z	 	 ddededz  fˆ fd„Zedd	„¦   «         Zdd„Z	 	 	 	 	 dd„Z	ˆ xZ
S )ÚPop2PianoAttentionFNr@   Ú	layer_idxc                 ó*  •— t          ¦   «                              ¦   «          |j        | _        || _        |j        | _        |j        | _        |j        | _        |j        | _        |j	        | _
        |j        | _        | j
        | j        z  | _        || _        |€/| j        r(t                               d| j        j        › d�¦  «         t'          j        | j        | j        d¬¦  «        | _        t'          j        | j        | j        d¬¦  «        | _        t'          j        | j        | j        d¬¦  «        | _        t'          j        | j        | j        d¬¦  «        | _        | j        r$t'          j        | j        | j
        ¦  «        | _        d| _        d S )NzInstantiating a decoder z³ without passing `layer_idx` is not recommended and will to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.FrC   )r   r   Ú
is_decoderÚhas_relative_attention_biasÚrelative_attention_num_bucketsÚrelative_attention_max_distancerF   Úd_kvÚkey_value_proj_dimÚ	num_headsÚn_headsrK   rL   Ú	inner_dimrj   ÚloggerÚwarning_oncer'   r9   r   rE   ÚqÚkÚvÚoÚ	EmbeddingÚrelative_attention_biasÚgradient_checkpointing©r$   r@   rm   rj   r'   s       €r(   r   zPop2PianoAttention.__init__Œ   si  ø€ õ 	‰Œ×ÒÑÔÐØ Ô+ˆŒØ+FˆÔ(Ø.4Ô.SˆÔ+Ø/5Ô/UˆÔ,Ø”~ˆŒØ"(¤+ˆÔØÔ'ˆŒØÔ*ˆŒØœ¨Ô(?Ñ?ˆŒØ"ˆŒØÐ ¤ÐÝ×Òð,¨4¬>Ô+Bð ,ð ,ð ,ñô ð õ ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ>¨4¬<¸eÐDÑDÔDˆŒàÔ+ð 	kÝ+-¬<¸Ô8[Ð]aÔ]iÑ+jÔ+jˆDÔ(à&+ˆÔ#Ð#Ð#r)   Té    é€   c                 óP  — d}|rC|dz  }|| dk                          t          j        ¦  «        |z  z  }t          j        | ¦  «        } n(t          j        | t          j        | ¦  «        ¦  «         } |dz  }| |k     }|t          j        |                      ¦   «         |z  ¦  «        t          j        ||z  ¦  «        z  ||z
  z                        t          j        ¦  «        z   }t          j        |t          j	        ||dz
  ¦  «        ¦  «        }|t          j
        || |¦  «        z  }|S )aÒ  
        Adapted from Mesh Tensorflow:
        https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593

        Translate relative position to a bucket number for relative attention. The relative position is defined as
        memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to
        position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for
        small absolute relative_position and larger buckets for larger absolute relative_positions. All relative
        positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket.
        This should allow for more graceful generalization to longer sequences than the model has been trained on

        Args:
            relative_position: an int32 Tensor
            bidirectional: a boolean - whether the attention is bidirectional
            num_buckets: an integer
            max_distance: an integer

        Returns:
            a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets)
        r   r+   r   )r.   r    ÚlongÚabsÚminÚ
zeros_likeÚlogÚfloatÚmathÚ	full_likeÚwhere)Úrelative_positionÚbidirectionalÚnum_bucketsÚmax_distanceÚrelative_bucketsÚ	max_exactÚis_smallÚrelative_position_if_larges           r(   Ú_relative_position_bucketz,Pop2PianoAttention._relative_position_bucket®   s>  € ð, ÐØð 	cØ˜AÑˆKØÐ!2°QÒ!6× :Ò :½5¼:Ñ FÔ FÈÑ TÑTÐÝ %¤	Ð*;Ñ <Ô <ÐÐå!&¤Ð+<½eÔ>NÐO`Ñ>aÔ>aÑ!bÔ!bÐ bÐð   1Ñ$ˆ	Ø$ yÒ0ˆð &/ÝŒIÐ'×-Ò-Ñ/Ô/°)Ñ;Ñ<Ô<ÝŒh�| iÑ/Ñ0Ô0ñ1à˜YÑ&ñ(÷ Š"�UŒZ‰.Œ.ñ	&Ð"õ
 &+¤YØ&­¬Ð8RÐT_ÐbcÑTcÑ(dÔ(dñ&
ô &
Ð"ð 	�EœK¨Ð2CÐE_Ñ`Ô`Ñ`ÐØÐr)   r   c                 ó¸  — |€| j         j        j        }t          j        |t          j        |¬¦  «        dd…df         |z   }t          j        |t          j        |¬¦  «        ddd…f         }||z
  }|                      || j         | j        | j	        ¬¦  «        }|                       |¦  «        }	|	 
                    g d¢¦  «                             d¦  «        }	|	S )z%Compute binned relative position biasN)r3   Údevice)rŒ   r�   rŽ   )r+   r   r   r   )r|   r"   r•   r    Úaranger‚   r“   rl   rn   ro   ÚpermuteÚ	unsqueeze)
r$   Úquery_lengthÚ
key_lengthr•   Úpast_seen_tokensÚcontext_positionÚmemory_positionr‹   Úrelative_position_bucketÚvaluess
             r(   Úcompute_biaszPop2PianoAttention.compute_biasÞ   sí   € àˆ>ØÔ1Ô8Ô?ˆFÝ œ<¨½E¼JÈvÐVÑVÔVÐWXÐWXÐWXÐZ^ÐW^Ô_ÐbrÑrÐÝœ, z½¼ÈFÐSÑSÔSÐTXÐZ[ÐZ[ÐZ[ÐT[Ô\ˆØ+Ð.>Ñ>ÐØ#'×#AÒ#AØØ#œÐ.ØÔ;ØÔ=ð	 $Bñ $
ô $
Ð ð ×-Ò-Ð.FÑGÔGˆØ—’ 	 	 	Ñ*Ô*×4Ò4°QÑ7Ô7ˆØˆr)   c                 ó�  — |j         dd…         }g |¢d‘| j        ‘R }	|�|                     | j        ¦  «        nd}
t	          |
t
          j        ¦  «        r|
                     ¦   «         n|
}
|du}|                      |¦  «         	                    |	¦  «         
                    dd¦  «        }d}t	          |t          ¦  «        r1|j                             | j        ¦  «        }|r|j        }n
|j        }n|}|r|n|}|r3|�1|r/|j        | j                 j        }|j        | j                 j        }nÚg |j         dd…         ¢d‘| j        ‘R }|                      |¦  «         	                    |¦  «         
                    dd¦  «        }|                      |¦  «         	                    |¦  «         
                    dd¦  «        }|�E|                     ||| j        ¦  «        \  }}|r$t	          |t          ¦  «        rd|j        | j        <   t          j        || 
                    dd¦  «        ¦  «        }|€ª|j         d	         }| j        sLt          j        d|j         d         |d         |f|j        |j        ¬
¦  «        }| j        r| j        rd|_        n$|                      |d         ||j        |
¬¦  «        }|�$|dd…dd…dd…d|j         d	         …f         }||z   }|}||z  }t>          j          !                    | "                    ¦   «         d¬¦  «         #                    |¦  «        }t>          j          $                    || j$        | j        ¬¦  «        }t          j        ||¦  «        }| 
                    dd¦  «         %                    ¦   «         } |j&        g |¢d‘R Ž }|  '                    |¦  «        }||f}|r||fz   }|S )z€
        Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states).
        Nr,   r   r   r+   FTr   éþÿÿÿ)r•   r3   )r•   r›   ©Údim)ÚpÚtraining)(Úshaperq   Úget_seq_lengthrj   rR   r    rS   Úclonerw   ÚviewÚ	transposer   Ú
is_updatedÚgetÚcross_attention_cacheÚself_attention_cacheÚlayersÚkeysrŸ   rx   ry   ÚupdateÚmatmulrm   Úzerosr•   r3   r}   r¦   Úrequires_gradr    r   Ú
functionalÚsoftmaxr‡   Útype_asrL   Ú
contiguousÚreshaperz   )r$   r6   ÚmaskÚkey_value_statesÚposition_biasÚpast_key_valuesÚoutput_attentionsÚkwargsÚinput_shapeÚhidden_shaper›   Úis_cross_attentionÚquery_statesr¬   Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚkv_shapeÚscoresrš   Úcausal_maskÚposition_bias_maskedÚattn_weightsÚattn_outputÚoutputss                             r(   r8   zPop2PianoAttention.forwardï   s  € ð $Ô)¨#¨2¨#Ô.ˆØB˜ÐB bÐB¨$Ô*AÐBÐBˆØM\ÐMh˜?×9Ò9¸$¼.ÑIÔIÐIÐnoÐå7AÐBRÕTYÔT`Ñ7aÔ7aÐwÐ+×1Ò1Ñ3Ô3Ð3ÐgwÐð .°TÐ9Ðà—v’v˜mÑ,Ô,×1Ò1°,Ñ?Ô?×IÒIÈ!ÈQÑOÔOˆð ˆ
Ý�oÕ':Ñ;Ô;ð 	3Ø(Ô3×7Ò7¸¼ÑGÔGˆJØ!ð Là'6Ô'LÐ$Ð$à'6Ô'KÐ$Ð$à#2Ð à-?ÐRÐ)Ð)À]ˆØð 	F /Ð"=À*Ð"=à-Ô4°T´^ÔDÔIˆJØ/Ô6°t´~ÔFÔMˆLˆLàP˜Ô-¨c¨r¨cÔ2ÐP°BÐP¸Ô8OÐPÐPˆHØŸš Ñ/Ô/×4Ò4°XÑ>Ô>×HÒHÈÈAÑNÔNˆJØŸ6š6 .Ñ1Ô1×6Ò6°xÑ@Ô@×JÒJÈ1ÈaÑPÔPˆLàÐ*Ø+?×+FÒ+FÀzÐS_ÐaeÔaoÑ+pÔ+pÑ(�
˜Là%ð F­*°_ÕFYÑ*ZÔ*Zð FØAE�OÔ.¨t¬~Ñ>õ ”˜l¨J×,@Ò,@ÀÀAÑ,FÔ,FÑGÔGˆàÐ Ø#Ô)¨"Ô-ˆJØÔ3ð 	Ý %¤Ø˜Ô*¨1Ô-¨{¸1¬~¸zÐJÐSYÔS`ÐhnÔhtð!ñ !ô !�ð Ô.ð 7°4´=ð 7Ø26�MÔ/øà $× 1Ò 1Ø ”N J°v´}ÐWgð !2ñ !ô !�ð ÐØ" 1 1 1 a a a¨¨¨Ð,B¨jÔ.>¸rÔ.BÐ,BÐ#BÔC�Ø -°Ñ ;�à,ÐØÐ&Ñ&ˆõ ”}×,Ò,¨V¯\ª\©^¬^ÀÐ,ÑDÔD×LÒLÈVÑTÔTˆÝ”}×,Ò,¨\¸T¼\ÐTXÔTaÐ,ÑbÔbˆå”l <°Ñ>Ô>ˆà!×+Ò+¨A¨qÑ1Ô1×<Ò<Ñ>Ô>ˆØ)�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;ˆØ—f’f˜[Ñ)Ô)ˆà Ð.ˆàð 	0Ø  Ñ/ˆGØˆr)   ©FN)Tr   r€   )Nr   )NNNNF)r9   r:   r;   r   Úintr   Ústaticmethodr“   r    r8   r<   r=   s   @r(   ri   ri   ‹   sÃ   ø€ € € € € ð %*Ø $ð	 ,ð  ,àð ,ð ˜‘:ð	 ,ð  ,ð  ,ð  ,ð  ,ð  ,ðD ð- ð - ð - ñ „\ð- ð^ð ð ð ð( ØØØØð[ð [ð [ð [ð [ð [ð [ð [r)   ri   c                   ó>   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 	 	 dd„Zˆ xZS )ÚPop2PianoLayerSelfAttentionFNrj   c                 óò   •— t          ¦   «                              ¦   «          t          |||¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S )N©rm   rj   ra   )r   r   ri   ÚSelfAttentionr   rF   rd   re   r   rJ   rK   rL   r~   s       €r(   r   z$Pop2PianoLayerSelfAttention.__init__O  sl   ø€ Ý‰Œ×ÒÑÔÐÝ/ØÐ0KÐW`ð
ñ 
ô 
ˆÔõ -¨V¬^ÀÔAZÐ[Ñ[Ô[ˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr)   c                 ó¾   — |                       |¦  «        }|                      ||||||¬¦  «        }	||                      |	d         ¦  «        z   }|f|	dd …         z   }
|
S )N)r»   r½   r¾   Ú	use_cacher¿   r   r   )re   r×   rL   )r$   r6   Úattention_maskr½   r¾   rÙ   r¿   rÀ   Únormed_hidden_statesÚattention_outputrÏ   s              r(   r8   z#Pop2PianoLayerSelfAttention.forwardW  s}   € ð  $Ÿš¨}Ñ=Ô=ÐØ×-Ò-Ø ØØ'Ø+ØØ/ð .ñ 
ô 
Ðð &¨¯ªÐ5EÀaÔ5HÑ(IÔ(IÑIˆØ Ð"Ð%5°a°b°bÔ%9Ñ9ˆØˆr)   rÐ   )NNNFF©r9   r:   r;   rÑ   r   r8   r<   r=   s   @r(   rÔ   rÔ   N  ss   ø€ € € € € ð7ð 7ÈSÐSWÉZð 7ð 7ð 7ð 7ð 7ð 7ð ØØØØðð ð ð ð ð ð ð r)   rÔ   c                   ó<   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 	 dd„Zˆ xZS )ÚPop2PianoLayerCrossAttentionNrj   c                 óò   •— t          ¦   «                              ¦   «          t          |d|¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S )NFrÖ   ra   )r   r   ri   ÚEncDecAttentionr   rF   rd   re   r   rJ   rK   rL   )r$   r@   rj   r'   s      €r(   r   z%Pop2PianoLayerCrossAttention.__init__q  sc   ø€ Ý‰Œ×ÒÑÔÐÝ1°&ÐV[ÐgpÐqÑqÔqˆÔÝ,¨V¬^ÀÔAZÐ[Ñ[Ô[ˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr)   Fc                 ó¾   — |                       |¦  «        }|                      ||||||¬¦  «        }	||                      |	d         ¦  «        z   }
|
f|	dd …         z   }|S )N)r»   r¼   r½   r¾   r¿   r   r   )re   rá   rL   )r$   r6   r¼   rÚ   r½   r¾   r¿   rÀ   rÛ   rÜ   Úlayer_outputrÏ   s               r(   r8   z$Pop2PianoLayerCrossAttention.forwardw  s|   € ð  $Ÿš¨}Ñ=Ô=ÐØ×/Ò/Ø ØØ-Ø'Ø+Ø/ð 0ñ 
ô 
Ðð % t§|¢|Ð4DÀQÔ4GÑ'HÔ'HÑHˆØ�/Ð$4°Q°R°RÔ$8Ñ8ˆØˆr)   rQ   )NNNFrÝ   r=   s   @r(   rß   rß   p  so   ø€ € € € € ð7ð 7¨#°©*ð 7ð 7ð 7ð 7ð 7ð 7ð ØØØðð ð ð ð ð ð ð r)   rß   c                   óF   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 	 	 	 	 	 	 dd„Zˆ xZS )	ÚPop2PianoBlockFNrj   c                 ó’  •— t          ¦   «                              ¦   «          |j        | _        t          j        ¦   «         | _        | j                             t          |||¬¦  «        ¦  «         | j        r)| j                             t          ||¬¦  «        ¦  «         | j                             t          |¦  «        ¦  «         d S )NrÖ   )rj   )
r   r   rl   r   Ú
ModuleListÚlayerÚappendrÔ   rß   r_   r~   s       €r(   r   zPop2PianoBlock.__init__‘  s»   ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒÝ”]‘_”_ˆŒ
ØŒ
×ÒÝ'ØÐ4OÐ[dðñ ô ñ	
ô 	
ð 	
ð
 Œ?ð 	YØŒJ×ÒÕ:¸6ÈYÐWÑWÔWÑXÔXÐXàŒ
×ÒÕ*¨6Ñ2Ô2Ñ3Ô3Ð3Ð3Ð3r)   Tc                 óâ  —  | j         d         ||||||	¬¦  «        }|d         }|dd …         }|j        t          j        k    r�t          j        t          j        |¦  «                             ¦   «         t          j        |j        ¦  «        j        dz
  t          j        |j        ¦  «        j        ¦  «        }t          j	        || |¬¦  «        }| j
        o|d u}|rÓ | j         d         ||||||	¬¦  «        }|d         }|j        t          j        k    r�t          j        t          j        |¦  «                             ¦   «         t          j        |j        ¦  «        j        dz
  t          j        |j        ¦  «        j        ¦  «        }t          j	        || |¬¦  «        }||dd …         z   } | j         d         |¦  «        }|j        t          j        k    r�t          j        t          j        |¦  «                             ¦   «         t          j        |j        ¦  «        j        dz
  t          j        |j        ¦  «        j        ¦  «        }t          j	        || |¬¦  «        }|f}||z   S )Nr   )rÚ   r½   r¾   rÙ   r¿   r   iè  )r„   Úmax)r¼   rÚ   r½   r¾   r¿   r,   )rè   r3   r    r4   rŠ   ÚisinfÚanyÚfinforë   Úclamprl   )r$   r6   rÚ   r½   Úencoder_hidden_statesÚencoder_attention_maskÚencoder_decoder_position_biasr¾   rÙ   r¿   Úreturn_dictrÀ   Úself_attention_outputsÚattention_outputsÚclamp_valueÚdo_cross_attentionÚcross_attention_outputsrÏ   s                     r(   r8   zPop2PianoBlock.forwardŸ  so  € ð "/ ¤¨A¤ØØ)Ø'Ø+ØØ/ð"
ñ "
ô "
Ðð /¨qÔ1ˆØ2°1°2°2Ô6Ðð Ô¥%¤-Ò/Ð/Ýœ+Ý”˜MÑ*Ô*×.Ò.Ñ0Ô0Ý”˜MÔ/Ñ0Ô0Ô4°tÑ;Ý”˜MÔ/Ñ0Ô0Ô4ñô ˆKõ
 "œK¨¸K¸<È[ÐYÑYÔYˆMà!œ_ÐRÐ1FÈdÐ1RÐØð 	PØ&3 d¤j°¤mØØ!6Ø5Ø;Ø /Ø"3ð'ñ 'ô 'Ð#ð 4°AÔ6ˆMð Ô"¥e¤mÒ3Ð3Ý#œkÝ”K Ñ.Ô.×2Ò2Ñ4Ô4Ý”K Ô 3Ñ4Ô4Ô8¸4Ñ?Ý”K Ô 3Ñ4Ô4Ô8ñô �õ
 !&¤¨MÀ¸|ÐQ\Ð ]Ñ ]Ô ]�ð !2Ð4KÈAÈBÈBÔ4OÑ OÐð '˜œ
 2œ }Ñ5Ô5ˆð Ô¥%¤-Ò/Ð/Ýœ+Ý”˜MÑ*Ô*×.Ò.Ñ0Ô0Ý”˜MÔ/Ñ0Ô0Ô4°tÑ;Ý”˜MÔ/Ñ0Ô0Ô4ñô ˆKõ
 "œK¨¸K¸<È[ÐYÑYÔYˆMà Ð"ˆð Ð'Ñ'ð	
r)   rÐ   )	NNNNNNFFTrÝ   r=   s   @r(   rå   rå   �  s‡   ø€ € € € € ð4ð 4ÈSÐSWÉZð 4ð 4ð 4ð 4ð 4ð 4ð" ØØ"Ø#Ø&*ØØØØðJ
ð J
ð J
ð J
ð J
ð J
ð J
ð J
r)   rå   c                   óv   ‡ — e Zd ZU eed<   dZdZdZdZdgZ	dgZ
 ej        ¦   «         ˆ fd„¦   «         Zd	„ Zˆ xZS )
ÚPop2PianoPreTrainedModelr@   Útransformer)ÚaudioTFrå   rI   c                 ó
	  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          ¦  «        rt          j        |j        |dz  ¦  «         dS t	          |t          ¦  «        r&t          j
        |j        j        d|dz  ¬¦  «         dS t	          |t          ¦  «        r\t          j
        |j        j        d|dz  ¬¦  «         t          |d¦  «        r&t          j
        |j        j        d|dz  ¬¦  «         dS dS t	          |t           ¦  «        ræt          j
        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j
        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         dS dS dS t	          |t.          ¦  «        �rVt          j
        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j
        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j
        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         dS dS dS t	          |t4          ¦  «        rö| j        j        }| j        j        }| j        j        }t          j
        |j        j        d|||z  dz  z  ¬¦  «         t          j
        |j        j        d||dz  z  ¬¦  «         t          j
        |j        j        d||dz  z  ¬¦  «         t          j
        |j         j        d|||z  dz  z  ¬¦  «         |j!        r+t          j
        |j"        j        d||dz  z  ¬¦  «         dS dS dS )zInitialize the weightsç      ð?ç        )r1   ÚstdÚlm_headç      à¿rD   N)#r   Ú_init_weightsr@   Úinitializer_factorrR   r   ÚinitÚ	constant_r"   ÚPop2PianoConcatEmbeddingToMelÚnormal_Ú	embeddingÚ!Pop2PianoForConditionalGenerationÚsharedÚhasattrr  r?   rH   rF   rD   Úzeros_rI   rG   rW   rY   rZ   ri   rp   rr   rw   rx   ry   rz   rm   r|   )r$   ÚmoduleÚfactorrF   rq   rs   r'   s         €r(   r  z&Pop2PianoPreTrainedModel._init_weights÷  s]  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ø”Ô/ˆÝ�fÕ0Ñ1Ô1ð "	pÝŒN˜6œ=¨&°3©,Ñ7Ô7Ð7Ð7Ð7Ý˜Õ =Ñ>Ô>ð  	pÝŒL˜Ô)Ô0°sÀÈÁÐMÑMÔMÐMÐMÐMÝ˜Õ AÑBÔBð 	pÝŒL˜œÔ-°C¸VÀc¹\ÐJÑJÔJÐJÝ�v˜yÑ)Ô)ð PÝ”˜Vœ^Ô2¸À&È3Á,ÐOÑOÔOÐOÐOÐOðPð På˜Õ 6Ñ7Ô7ð 	pÝŒL˜œÔ)°¸ÀDÄKÔDWÐ\`ÑC`Ñ9aÐbÑbÔbÐbÝ�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+ÝŒL˜œÔ)°¸ÀDÄKÔDTÐY]ÑC]Ñ9^Ð_Ñ_Ô_Ð_Ý�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+Ð+Ð+ð,ð ,Ð.HÐ.Hå˜Õ ;Ñ<Ô<ñ 	pÝŒL˜œÔ+°#¸6ÀdÄkÔFYÐ^bÑEbÑ;cÐdÑdÔdÐdÝ�v”{ FÑ+Ô+ð .°´Ô0@Ð0LÝ”˜FœKÔ,Ñ-Ô-Ð-ÝŒL˜œÔ+°#¸6ÀdÄkÔFYÐ^bÑEbÑ;cÐdÑdÔdÐdÝ�v”{ FÑ+Ô+ð .°´Ô0@Ð0LÝ”˜FœKÔ,Ñ-Ô-Ð-ÝŒL˜œÔ)°¸ÀDÄKÔDTÐY]ÑC]Ñ9^Ð_Ñ_Ô_Ð_Ý�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+Ð+Ð+ð,ð ,Ð.HÐ.Hå˜Õ 2Ñ3Ô3ð 		pØ”kÔ)ˆGØ!%¤Ô!1ÐØ”kÔ+ˆGÝŒL˜œœ¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÐjÑjÔjÐjÝŒL˜œœ¨s¸À'È4Á-Ñ8PÐQÑQÔQÐQÝŒL˜œœ¨s¸À'È4Á-Ñ8PÐQÑQÔQÐQÝŒL˜œœ¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÐjÑjÔjÐjØÔ1ð pÝ”˜VÔ;ÔBÈÐRXÐ]dÐimÑ\mÑRnÐoÑoÔoÐoÐoÐoð		pð 		pðpð pr)   c                 ó6  — | j         j        }| j         j        }|€t          d¦  «        ‚|                     |j        ¦  «        }|dd d…f                              ¦   «         |ddd …f<   ||d<   |€t          d¦  «        ‚|                     |dk    |¦  «         |S )Nzoself.model.config.decoder_start_token_id has to be defined. In Pop2Piano it is usually set to the pad_token_id..r,   r   ).r   z1self.model.config.pad_token_id has to be defined.éœÿÿÿ)r@   Údecoder_start_token_idÚpad_token_idÚ
ValueErrorÚ	new_zerosr§   r©   Úmasked_fill_)r$   Ú	input_idsr  r  Úshifted_input_idss        r(   Ú_shift_rightz%Pop2PianoPreTrainedModel._shift_right   s»   € Ø!%¤Ô!CÐØ”{Ô/ˆà!Ð)Ýð Bñô ð ð &×/Ò/°	´Ñ@Ô@ÐØ%.¨s°C°R°C¨xÔ%8×%>Ò%>Ñ%@Ô%@Ð˜#˜q˜r˜r˜'Ñ"Ø$:Ð˜&Ñ!àÐÝÐPÑQÔQÐQà×&Ò&Ð'8¸DÒ'@À,ÑOÔOÐOà Ð r)   )r9   r:   r;   r   Ú__annotations__Úbase_model_prefixÚoutput_modalitiesÚsupports_gradient_checkpointingÚ_can_compile_fullgraphÚ_no_split_modulesÚ_keep_in_fp32_modulesr    Úno_gradr  r  r<   r=   s   @r(   rú   rú   ì  s–   ø€ € € € € € àÐÐÑØ%ÐØ"ÐØ&*Ð#à"ÐØ)Ð*ÐØ!˜FÐà€U„]�_„_ð&pð &pð &pð &pñ „_ð&pðP!ð !ð !ð !ð !ð !ð !r)   rú   c                   ó@   ‡ — e Zd Zˆ fd„Zd„ Z	 	 	 	 	 	 	 	 	 	 dd„Zˆ xZS )ÚPop2PianoStackc                 óÌ  •‡— t          ¦   «                              ‰¦  «         t          j        ‰j        ‰j        ¦  «        | _        ‰j        | _        t          j        ˆfd„t          ‰j
        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t          j        ‰j        ¦  «        | _        |                      ¦   «          d| _        d S )Nc           	      óV   •— g | ]%}t          ‰t          |d k    ¦  «        |¬¦  «        ‘Œ&S )r   rÖ   )rå   Úbool)Ú.0Úir@   s     €r(   ú
<listcomp>z+Pop2PianoStack.__init__.<locals>.<listcomp>>  sC   ø€ ð ð ð àõ ˜vÅ4ÈÈQÊÁ<Ä<Ð[\Ð]Ñ]Ô]ðð ð r)   ra   F)r   r   r   r{   Ú
vocab_sizerF   Úembed_tokensrl   rç   ÚrangeÚ
num_layersÚblockr   rd   Úfinal_layer_normrJ   rK   rL   Ú	post_initr}   rO   s    `€r(   r   zPop2PianoStack.__init__7  sÓ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð åœL¨Ô):¸F¼NÑKÔKˆÔØ Ô+ˆŒå”]ðð ð ð å˜vÔ0Ñ1Ô1ðñ ô ñ
ô 
ˆŒ
õ !3°6´>ÀvÔG`Ð aÑ aÔ aˆÔÝ”z &Ô"5Ñ6Ô6ˆŒð 	�ŠÑÔÐØ&+ˆÔ#Ð#Ð#r)   c                 ó   — || _         d S rQ   )r+  ©r$   Únew_embeddingss     r(   Úset_input_embeddingsz#Pop2PianoStack.set_input_embeddingsK  s   € Ø*ˆÔÐÐr)   Nc                 óZ  — |�|n| j         j        }|�|n| j         j        }|	�|	n| j         j        }	|
�|
n| j         j        }
|�#|�!| j        rdnd}t          d|› d|› d�¦  «        ‚|�1|                     ¦   «         }|                     d|d         ¦  «        }n@|�|                     ¦   «         d d…         }n!| j        rdnd}t          d|› d|› d	�¦  «        ‚| j	        r%| j
        r|rt                               d
¦  «         d}|€+| j        €t          d¦  «        ‚|                      |¦  «        }|\  }}|du r| j        st          d| › d�¦  «        ‚| j        r]|rZ|€X| j         j        r7t          t!          | j         ¬¦  «        t!          | j         ¬¦  «        ¦  «        }nt!          | j         ¬¦  «        }n	| j        sd }|�|                     ¦   «         nd}|€/t%          ¦   «         s!||z   }t'          j        |||j        ¬¦  «        }| j         j        rt-          | j         |||¬¦  «        }nO|d d …d d d d …f         }|                     |j        ¬¦  «        }d|z
  t'          j        |j        ¦  «        j        z  }|�t7          | j         |||¬¦  «        }|	rdnd }|rdnd }|r	| j        rdnd }d }d }|                      |¦  «        }t;          | j        ¦  «        D ]g\  }}|	r||fz   } ||||||||||¬¦	  «	        }|d         }|d         }| j        r|�||rdnd         }|r||d         fz   }| j        r||d         fz   }Œh|                      |¦  «        }|                      |¦  «        }|	r||fz   }|
stA          d„ |||||fD ¦   «         ¦  «        S tC          |||||¬¦  «        S )NÚdecoder_Ú zYou cannot specify both zinput_ids and zinputs_embeds at the same timer,   zYou have to specify either zinput_ids or Úinputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fz<You have to initialize the model with valid token embeddingsTz)`use_cache` can only be set to `True` if z is used as a decoder)r@   r   ©r•   )r@   r8  rÚ   r¾   )r3   rþ   )r@   r8  rÚ   rð   © )r¾   rÙ   r¿   r   r   r+   é   c              3   ó   K  — | ]}|®|V — Œ	d S rQ   r:  )r'  ry   s     r(   ú	<genexpr>z)Pop2PianoStack.forward.<locals>.<genexpr>Ø  s4   è è € ð 
ð 
àð �=ð ð !�=�=�=ð
ð 
r)   )Úlast_hidden_stater¾   r6   Ú
attentionsÚcross_attentions)"r@   rÙ   r¿   Úoutput_hidden_statesró   rl   r  Úsizerª   r}   r¦   ru   rv   r+  Úis_encoder_decoderr   r
   r¨   r   r    r!   r•   r   r.   r3   rî   r„   r   rL   Ú	enumerater.  r/  Útupler   )r$   r  rÚ   rð   rñ   r8  r¾   rÙ   r¿   rA  ró   rÀ   Úerr_msg_prefixrÁ   Ú
batch_sizeÚ
seq_lengthÚpast_key_values_lengthÚmask_seq_lengthrË   Úall_hidden_statesÚall_attentionsÚall_cross_attentionsr½   rò   r6   r(  Úlayer_moduleÚlayer_outputss                               r(   r8   zPop2PianoStack.forwardN  s)  € ð "+Ð!6�I�I¸D¼KÔ<Qˆ	Ø1BÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàÐ  ]Ð%>Ø+/¬?ÐB˜Z˜ZÀˆNÝØw¨>ÐwÐwÈÐwÐwÐwñô ð ð Ð"Ø#Ÿ.š.Ñ*Ô*ˆKØ!Ÿš r¨;°r¬?Ñ;Ô;ˆIˆIØÐ&Ø'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKˆKà+/¬?ÐB˜Z˜ZÀˆNÝÐu¸>ÐuÐuÐXfÐuÐuÐuÑvÔvÐvàÔ&ð 	"¨4¬=ð 	"Øð "Ý×#Ò#Øpñô ð ð "�	àÐ ØÔ Ð(Ý Ð!_Ñ`Ô`Ð`Ø ×-Ò-¨iÑ8Ô8ˆMà!,Ñˆ
�Jà˜ÐÐØ”?ð jÝ Ð!hÈTÐ!hÐ!hÐ!hÑiÔiÐiàŒ?ð 	#Øð G˜_Ð4Ø”;Ô1ð GÝ&9Ý$¨D¬KÐ8Ñ8Ô8½,ÈdÌkÐ:ZÑ:ZÔ:Zñ'ô '�O�Oõ '3¸$¼+Ð&FÑ&FÔ&F�OøØ”ð 	#ð #ˆOàETÐE` ×!?Ò!?Ñ!AÔ!AÐ!AÐfgÐØÐ!Õ*BÑ*DÔ*DÐ!à4°zÑAˆOÝ"œZ¨
°OÈMÔL`ÐaÑaÔaˆNàŒ;Ô!ð 
	UÝ,Ø”{Ø+Ø-Ø /ð	ñ ô ˆKˆKð )¨¨¨¨D°$¸¸¸Ð)9Ô:ˆKØ%Ÿ.š.¨}Ô/B˜.ÑCÔCˆKØ Ñ,µ´¸MÔ<OÑ0PÔ0PÔ0TÑTˆKà!Ð-Ý%>Ø”{Ø+Ø5Ø&;ð	&ñ &ô &Ð"ð #7Ð@˜B˜B¸DÐØ0Ð:˜˜°dˆØ&7ÐV¸D¼OÐV˜r˜rÐRVÐØˆØ(,Ð%àŸš ]Ñ3Ô3ˆå(¨¬Ñ4Ô4ð 	Vð 	V‰OˆAˆ|Ø#ð IØ$5¸Ð8HÑ$HÐ!à(˜LØØØØ%Ø&Ø-Ø /Ø#Ø"3ð
ñ 
ô 
ˆMð *¨!Ô,ˆMð
 *¨!Ô,ˆMØŒð ]Ð#8Ð#DØ0=ÐCTÐ>[¸a¸aÐZ[Ô0\Ð-à ð VØ!/°=ÀÔ3CÐ2EÑ!E�Ø”?ð VØ+?À=ÐQRÔCSÐBUÑ+UÐ(øà×-Ò-¨mÑ<Ô<ˆØŸš ]Ñ3Ô3ˆð  ð 	EØ 1°]Ð4DÑ DÐàð 	Ýð 
ð 
ð "Ø#Ø%Ø"Ø(ðð
ñ 
ô 
ñ 
ô 
ð 
õ 9Ø+Ø+Ø+Ø%Ø1ð
ñ 
ô 
ð 	
r)   )
NNNNNNNNNN)r9   r:   r;   r   r4  r8   r<   r=   s   @r(   r#  r#  5  sƒ   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð(+ð +ð +ð
 ØØ"Ø#ØØØØØ!Øð[
ð [
ð [
ð [
ð [
ð [
ð [
ð [
r)   r#  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )r  z'Embedding Matrix for `composer` tokens.c                 ó’   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¬¦  «        | _        d S )N)Únum_embeddingsÚembedding_dim)r   r   r   r{   Úcomposer_vocab_sizerF   r	  rO   s     €r(   r   z&Pop2PianoConcatEmbeddingToMel.__init__ï  s:   ø€ Ý‰Œ×ÒÑÔÐÝœ°VÔ5OÐ_eÔ_mÐnÑnÔnˆŒˆˆr)   c                 ó�   — ||z
  }|                       |¦  «                             d¦  «        }t          j        ||gd¬¦  «        }|S )Nr   r£   )r	  r˜   r    Úcat)r$   ÚfeatureÚindex_valueÚembedding_offsetÚindex_shiftedÚcomposer_embeddingr8  s          r(   r8   z%Pop2PianoConcatEmbeddingToMel.forwardó  sM   € Ø#Ð&6Ñ6ˆØ!Ÿ^š^¨MÑ:Ô:×DÒDÀQÑGÔGÐÝœ	Ð#5°wÐ"?ÀQÐGÑGÔGˆØÐr)   )r9   r:   r;   Ú__doc__r   r8   r<   r=   s   @r(   r  r  ì  sR   ø€ € € € € Ø1Ð1ðoð oð oð oð oðð ð ð ð ð ð r)   r  zA
    Pop2Piano Model with a `language modeling` head on top.
    )Úcustom_introc            !       ó  ‡ — e Zd ZdddœZdefˆ fd„Zd„ Zd„ Z	 ddej	        d	e
d
edej	        dz  fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej	        dz  dej        dz  dej        dz  deeej                          dz  dedz  dej	        dz  dej	        dz  dej	        dz  dej        dz  dedz  dedz  dedz  dedz  deej	                 ez  fd„¦   «         Z ej        ¦   «         	 	 	 d ˆ fd„	¦   «         Zdej        fd„Zˆ xZS )!r
  zshared.weight)zencoder.embed_tokens.weightzdecoder.embed_tokens.weightr@   c                 ó6  •— t          ¦   «                              |¦  «         || _        |j        | _        t          j        |j        |j        ¦  «        | _        t          |¦  «        | _
        t          j        |¦  «        }d|_        d|_        t          |¦  «        | _        t          j        |¦  «        }d|_        |j        |_        t          |¦  «        | _        t          j        |j        |j        d¬¦  «        | _        |                      ¦   «          d S )NFTrC   )r   r   r@   rF   Ú	model_dimr   r{   r*  r  r  Úmel_conditionerÚcopyÚdeepcopyrl   rÙ   r#  ÚencoderÚnum_decoder_layersr-  ÚdecoderrE   r  r0  )r$   r@   Úencoder_configÚdecoder_configr'   s       €r(   r   z*Pop2PianoForConditionalGeneration.__init__  sç   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒØœˆŒå”l 6Ô#4°f´nÑEÔEˆŒå<¸VÑDÔDˆÔåœ vÑ.Ô.ˆØ$)ˆÔ!Ø#(ˆÔ å% nÑ5Ô5ˆŒåœ vÑ.Ô.ˆØ$(ˆÔ!Ø$*Ô$=ˆÔ!Ý% nÑ5Ô5ˆŒå”y ¤°Ô1BÈÐOÑOÔOˆŒð 	�ŠÑÔÐÐÐr)   c                 ó   — | j         S rQ   )r  )r$   s    r(   Úget_input_embeddingsz6Pop2PianoForConditionalGeneration.get_input_embeddings  s
   € ØŒ{Ðr)   c                 ó|   — || _         | j                             |¦  «         | j                             |¦  «         d S rQ   )r  rd  r4  rf  r2  s     r(   r4  z6Pop2PianoForConditionalGeneration.set_input_embeddings!  s;   € Ø$ˆŒØŒ×)Ò)¨.Ñ9Ô9Ð9ØŒ×)Ò)¨.Ñ9Ô9Ð9Ð9Ð9r)   NÚinput_featuresÚcomposerÚgeneration_configrÚ   c                 ó<  — |j         }||vr4t          dt          |                     ¦   «         ¦  «        › d|› �¦  «        ‚||         }t	          j        || j        ¬¦  «        }|                     |j        d         ¦  «        }t          | 
                    ¦   «         ¦  «        }|                      |||¬¦  «        }|�\d||dd…df                              ¦   «          <   t	          j        |dd…df                              dd	¦  «        |gd	¬
¦  «        }||fS |dfS )aã  
        This method is used to concatenate mel conditioner tokens at the front of the input_features in order to
        control the type of MIDI token generated by the model.

        Args:
            input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                input features extracted from the feature extractor.
            composer (`str`):
                composer token which determines the type of MIDI tokens to be generated.
            generation_config (`~generation.GenerationConfig`):
                The generation is used to get the composer-feature_token pair.
            attention_mask (``, *optional*):
                For batched generation `input_features` are padded to have the same shape across all examples.
                `attention_mask` helps to determine which areas were padded and which were not.
                - 1 for tokens that are **not padded**,
                - 0 for tokens that are **padded**.
        zPlease choose a composer from z. Composer received - r9  r   )rW  rX  rY  Nrÿ   r,   r   )Úaxis)Úcomposer_to_feature_tokenr  Úlistr±   r    Útensorr•   Úrepeatr§   r„   rŸ   ra  r&  Úconcatenaterª   )r$   rl  rm  rn  rÚ   rq  Úcomposer_valuerY  s           r(   Úget_mel_conditioner_outputsz=Pop2PianoForConditionalGeneration.get_mel_conditioner_outputs&  sN  € ð0 %6Ô$OÐ!ØÐ4Ð4Ð4ÝØyµÐ6O×6TÒ6TÑ6VÔ6VÑ1WÔ1WÐyÐyÐowÐyÐyñô ð ð 3°8Ô<ˆÝœ n¸T¼[ÐIÑIÔIˆØ'×.Ò.¨~Ô/CÀAÔ/FÑGÔGˆåÐ8×?Ò?ÑAÔAÑBÔBÐà×-Ò-Ø"Ø&Ø-ð .ñ 
ô 
ˆð
 Ð%Ø;>ˆN˜N¨1¨1¨1¨a¨4Ô0×5Ò5Ñ7Ô7Ð7Ñ8õ #Ô.°¸q¸q¸qÀ!¸tÔ0D×0IÒ0IÈ"ÈaÑ0PÔ0PÐR`Ð/aÐhiÐjÑjÔjˆNØ! >Ð1Ð1à˜tÐ#Ð#r)   r  Údecoder_input_idsÚdecoder_attention_maskÚencoder_outputsr¾   r8  Údecoder_inputs_embedsÚlabelsrÙ   r¿   rA  ró   Úreturnc                 óÎ  — |�|n| j         j        }|�|n| j         j        }|�|�t          d¦  «        ‚|�|€|}|€|                      ||||||¬¦  «        }ne|rct          |t          ¦  «        sNt          |d         t          |¦  «        dk    r|d         ndt          |¦  «        dk    r|d         nd¬¦  «        }|d         }|
�|€|	€|                      |
¦  «        }|  	                    |||	|||||||¬¦
  «
        }|d         }| j         j
        r|| j        d	z  z  }|                      |¦  «        }d}|
�Vt          d
¬¦  «        } ||                     d|                     d¦  «        ¦  «        |
                     d¦  «        ¦  «        }|s|f|dd…         z   |z   }|�|f|z   n|S t!          |||j        |j        |j        |j        |j        |j        |j        ¬¦	  «	        S )aq  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Pop2Piano is a model with relative position embeddings
            so you should be able to pad the inputs on both the right and the left. Indices can be obtained using
            [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail.
            [What are input IDs?](../glossary#input-ids) To know more on how to prepare `input_ids` for pretraining
            take a look a [Pop2Piano Training](./Pop2Piano#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using
            [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.
            [What are decoder input IDs?](../glossary#decoder-input-ids) Pop2Piano uses the `pad_token_id` as the
            starting token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last
            `decoder_input_ids` have to be input (see `past_key_values`). To know more on how to prepare
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
            config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
            labels in `[0, ..., config.vocab_size]`
        NzSBoth `inputs_embeds` and `input_features` received! Please provide only one of them)r  rÚ   r8  r¿   rA  ró   r   r   r+   )r>  r6   r?  )
r  rÚ   r8  r¾   rð   rñ   rÙ   r¿   rA  ró   r  r  )Úignore_indexr,   )	ÚlossÚlogitsr¾   Údecoder_hidden_statesÚdecoder_attentionsr@  Úencoder_last_hidden_staterð   Úencoder_attentions)r@   rÙ   ró   r  rd  rR   r   Úlenr  rf  Útie_word_embeddingsr`  r  r   rª   rB  r   r¾   r6   r?  r@  r>  )r$   r  rÚ   rx  ry  rz  r¾   r8  rl  r{  r|  rÙ   r¿   rA  ró   rÀ   r6   Údecoder_outputsÚsequence_outputÚ	lm_logitsr€  Úloss_fctÚoutputs                          r(   r8   z)Pop2PianoForConditionalGeneration.forwardW  sw  € ðP "+Ð!6�I�I¸D¼KÔ<Qˆ	Ø%0Ð%<�k�kÀ$Ä+ÔBYˆàÐ$¨Ð)CÝÐrÑsÔsÐsØÐ'¨MÐ,AØ*ˆMð Ð"à"ŸlšlØ#Ø-Ø+Ø"3Ø%9Ø'ð +ñ ô ˆOˆOð ð 	¥¨O½_Ñ!MÔ!Mð 	Ý-Ø"1°!Ô"4Ý47¸Ñ4HÔ4HÈ1Ò4LÐ4L˜o¨aÔ0Ð0ÐRVÝ14°_Ñ1EÔ1EÈÒ1IÐ1I˜?¨1Ô-Ð-Ètðñ ô ˆOð (¨Ô*ˆàÐÐ"3Ð";Ð@UÐ@]à $× 1Ò 1°&Ñ 9Ô 9Ðð Ÿ,š,Ø'Ø1Ø/Ø+Ø"/Ø#1ØØ/Ø!5Ø#ð 'ñ 
ô 
ˆð *¨!Ô,ˆàŒ;Ô*ð 	GØ-°´ÀÑ1EÑFˆOà—L’L Ñ1Ô1ˆ	àˆØÐÝ'°TÐ:Ñ:Ô:ˆHØ�8˜IŸNšN¨2¨y¯~ª~¸bÑ/AÔ/AÑBÔBÀFÇKÂKÐPRÁOÄOÑTÔTˆDàð 	FØ�\ O°A°B°BÔ$7Ñ7¸/ÑIˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEåØØØ+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð

ñ 

ô 

ð 
	
r)   Ú	composer1c                 óŠ  •— |€| j         } |j        d	i |¤Ž t          |d¦  «        st          d¦  «        ‚t	          |j        ¦  «        | j        j        k    r2t          d| j        j        › dt	          |j        ¦  «        › d�¦  «        ‚|                      ||||¬¦  «        \  }} t          ¦   «         j
        d	d|||dœ|¤ŽS )
aŽ  
        Generates token ids for midi outputs.

        <Tip warning={true}>

        Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the
        model's default generation configuration. You can override any `generation_config` by passing the corresponding
        parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`. For an overview of generation
        strategies and code examples, check out the [following guide](./generation_strategies).

        </Tip>

        Parameters:
            input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
                This is the featurized version of audio generated by `Pop2PianoFeatureExtractor`.
            attention_mask:
                For batched generation `input_features` are padded to have the same shape across all examples.
                `attention_mask` helps to determine which areas were padded and which were not.
                - 1 for tokens that are **not padded**,
                - 0 for tokens that are **padded**.
            composer (`str`, *optional*, defaults to `"composer1"`):
                This value is passed to `Pop2PianoConcatEmbeddingToMel` to generate different embeddings for each
                `"composer"`. Please make sure that the composer value is present in `composer_to_feature_token` in
                `generation_config`. For an example please see
                https://huggingface.co/sweetcocoa/pop2piano/blob/main/generation_config.json .
            generation_config (`~generation.GenerationConfig`, *optional*):
                The generation configuration to be used as base parametrization for the generation call. `**kwargs`
                passed to generate matching the attributes of `generation_config` will override them. If
                `generation_config` is not provided, the default will be used, which had the following loading
                priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model
                configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s
                default values, whose documentation should be checked to parameterize generation.
            kwargs:
                Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be
                forwarded to the `forward` function of the model. If the model is an encoder-decoder model, encoder
                specific kwargs should not be prefixed and decoder specific kwargs should be prefixed with *decoder_*.
        Return:
            [`~utils.ModelOutput`] or `torch.LongTensor`: A [`~utils.ModelOutput`] (if `return_dict_in_generate=True`
            or when `config.return_dict_in_generate=True`) or a `torch.FloatTensor`.
                Since Pop2Piano is an encoder-decoder model (`model.config.is_encoder_decoder=True`), the possible
                [`~utils.ModelOutput`] types are:
                    - [`~generation.GenerateEncoderDecoderOutput`],
                    - [`~generation.GenerateBeamEncoderDecoderOutput`]
        Nrq  z¢`composer_to_feature_token` was not found! Please refer to https://huggingface.co/sweetcocoa/pop2piano/blob/main/generation_config.jsonand parse a dict like that.ztconfig.composer_vocab_size must be same as the number of keys in generation_config.composer_to_feature_token! Found z vs ú.)rl  rÚ   rm  rn  )Úinputsr8  rÚ   rn  r:  )rn  r²   r  r  r†  rq  r@   rT  rw  r   Úgenerate)r$   rl  rÚ   rm  rn  rÀ   r'   s         €r(   r‘  z*Pop2PianoForConditionalGeneration.generateÉ  s2  ø€ ðl Ð$Ø $Ô 6ÐØ ÐÔ Ð*Ð* 6Ð*Ð*Ð*õ Ð(Ð*EÑFÔFð 	Ýð.ñô ð õ Ð Ô:Ñ;Ô;¸t¼{Ô?^Ò^Ð^ÝðràœÔ8ðrð rå>AÐBSÔBmÑ>nÔ>nðrð rð rñô ð ð *.×)IÒ)IØ)Ø)ØØ/ð	 *Jñ *
ô *
Ñ&ˆ˜ð  �u‰wŒwÔð 
ØØ(Ø)Ø/ð	
ð 
ð
 ð
ð 
ð 	
r)   c                 ó,   — |                       |¦  «        S rQ   )r  )r$   r|  s     r(   Ú%prepare_decoder_input_ids_from_labelszGPop2PianoForConditionalGeneration.prepare_decoder_input_ids_from_labels#  s   € Ø× Ò  Ñ(Ô(Ð(r)   rQ   )NNNNNNNNNNNNNN)Nr�  N)r9   r:   r;   Ú_tied_weights_keysr   r   rj  r4  r    ÚFloatTensorÚstrr   rw  r   Ú
LongTensorÚ
BoolTensorrE  rS   r	   r&  r   r8   r!  r‘  r“  r<   r=   s   @r(   r
  r
  ú  s”  ø€ € € € € ð (7Ø'6ðð Ðð
˜ð ð ð ð ð ð ð2ð ð ð:ð :ð :ð 48ð/$ð /$àÔ)ð/$ð ð/$ð ,ð	/$ð
 Ô)¨DÑ0ð/$ð /$ð /$ð /$ðb ð .2Ø37Ø59Ø:>Ø=AØ(,Ø26Ø37Ø:>Ø*.Ø!%Ø)-Ø,0Ø#'ðo
ð o
àÔ# dÑ*ðo
ð Ô)¨DÑ0ðo
ð !Ô+¨dÑ2ð	o
ð
 !&Ô 0°4Ñ 7ðo
ð ˜u U¤\Ô2Ô3°dÑ:ðo
ð  ™ðo
ð Ô(¨4Ñ/ðo
ð Ô)¨DÑ0ðo
ð  %Ô0°4Ñ7ðo
ð Ô  4Ñ'ðo
ð ˜$‘;ðo
ð   $™;ðo
ð # T™kðo
ð ˜D‘[ðo
ð" 
ˆuÔ Ô	! OÑ	3ð#o
ð o
ð o
ñ „^ðo
ðb €U„]�_„_ð ØØðW
ð W
ð W
ð W
ð W
ñ „_ðW
ðr)¸E¼Lð )ð )ð )ð )ð )ð )ð )ð )r)   r
  )6r\  rb  rˆ   r    r   Útorch.nnr   Útransformers.generationr   r7  r   r  Úactivationsr   Úcache_utilsr	   r
   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_utilsr   Úutilsr   r   r   Úconfiguration_pop2pianor   Ú
get_loggerr9   ru   ÚModuler   r?   rW   r_   ri   rÔ   rß   rå   rú   r#  r  r
  Ú__all__r:  r)   r(   ú<module>r§     sÎ  ðð Ð à €€€Ø €€€à €€€Ø Ð Ð Ð Ð Ð Ø %Ð %Ð %Ð %Ð %Ð %à 4Ð 4Ð 4Ð 4Ð 4Ð 4à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ )Ð )Ð )Ð )Ð )Ð )Ø JÐ JÐ JÐ JÐ JÐ JÐ JÐ JØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø kÐ kÐ kÐ kÐ kÐ kÐ kÐ kÐ kÐ kØ -Ð -Ð -Ð -Ð -Ð -Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ 4Ð 4Ð 4Ð 4Ð 4Ð 4ð 
ˆÔ	˜HÑ	%Ô	%€ð+ð +ð +ð +ð +˜œñ +ô +ð +ð4ð ð ð ð ˜RœYñ ô ð ð.ð ð ð ð  "¤)ñ ô ð ð<ð ð ð ð �r”yñ ô ð ð&ð ð ð ð ˜œñ ô ð ðFð ð ð ð  "¤)ñ ô ð ðDð ð ð ð  2¤9ñ ô ð ð@Y
ð Y
ð Y
ð Y
ð Y
Ð/ñ Y
ô Y
ð Y
ðx ðE!ð E!ð E!ð E!ð E!˜ñ E!ô E!ñ „ðE!ðPt
ð t
ð t
ð t
ð t
Ð-ñ t
ô t
ð t
ðnð ð ð ð  B¤Iñ ô ð ð €ððñ ô ð
e)ð e)ð e)ð e)ð e)Ð(@À/ñ e)ô e)ñô ð
e)ðP	 /Ð0JÐ
K€€€r)   