§
    ‚ŠtjÔ²  ã                   ó¸  — d Z ddlZddlmZ ddlZddlmZmZ ddlmZm	Z	 ddl
mZ ddlmZ dd	lmZmZmZ dd
lmZ ddlmZmZmZmZ ddlmZ ddlmZmZ ddlm Z   ej!        e"¦  «        Z#	 d„ Z$d/d„Z%ddej&        fd„Z'e G d„ de¦  «        ¦   «         Z(d„ Z)d„ Z*d„ Z+d0d„Z, G d„ dej-        ¦  «        Z. G d„ dej-        ¦  «        Z/ G d„ dej-        ¦  «        Z0 G d„ d ej-        ¦  «        Z1d!„ Z2 G d"„ d#ej-        ¦  «        Z3d$„ Z4d%„ Z5e G d&„ d'e(¦  «        ¦   «         Z6 ed(¬)¦  «         G d*„ d+e(e¦  «        ¦   «         Z7 G d,„ d-ej8        ¦  «        Z9g d.¢Z:dS )1z`PyTorch Fairseq model, ported from https://github.com/pytorch/fairseq/tree/master/examples/wmt19é    N)ÚAny)ÚTensorÚnn)ÚCrossEntropyLossÚ	LayerNormé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚPreTrainedModel)Úauto_docstringÚloggingé   )Ú
FSMTConfigc                 ó`   — |                       ¦   «         dk    sJ ‚|                      d¦  «        S )z+Turns 1->0, 0->1, False->True, True-> Falseé   r   )ÚdimÚeq)Úattention_masks    úd/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/fsmt/modeling_fsmt.pyÚinvert_maskr   ®   s3   € à×ÒÑÔ 1Ò$Ð$Ð$Ð$Ø×Ò˜QÑÔÐó    c                 óö   — | j         d         }t          j        || j        ¬¦  «        }|                     ||¦  «        }|                     d¦  «        }|r||z   }||k    }|                      |dk    d¦  «        S )Nr   ©Údeviceéÿÿÿÿ)ÚshapeÚtorchÚaranger"   ÚexpandÚ	unsqueezeÚmasked_fill)ÚxÚdiagonalÚlr&   Úmasks        r   Ú	triu_onnxr.   ´   s|   € Ø	Œ�Œ
€AÝŒ\˜! A¤HÐ-Ñ-Ô-€FØ�=Š=˜˜AÑÔ€DØ×Ò˜bÑ!Ô!€FØð #Ø˜(Ñ"ˆØ�6Š>€DØ�=Š=˜ š AÑ&Ô&Ð&r   c           	      óF  — | j         }|€t          ||¦  «        }|                     ¦   «         \  }}|€t          ||¦  «        }nt	          |¦  «        }t          t          t          j        |||¬¦  «        ¦  «        d¦  «         	                    |j
        ¬¦  «        }|||fS )z÷
    Prepare masks that ignore padding tokens in the decoder and a causal mask for the decoder if none are provided.
    This mimics the default behavior in fairseq. To override it pass in masks. Note: this is not called during
    generation
    N©Údtyper   r!   )Úpad_token_idÚshift_tokens_rightÚsizeÚmake_padding_maskr   r.   Úfill_with_neg_infr%   ÚzerosÚtor"   )	ÚconfigÚ	input_idsÚdecoder_input_idsÚdecoder_padding_maskÚcausal_mask_dtyper2   ÚbszÚtgt_lenÚcausal_masks	            r   Ú_prepare_fsmt_decoder_inputsrA   ¿   s·   € ð Ô&€LØÐ Ý.¨y¸,ÑGÔGÐØ$×)Ò)Ñ+Ô+�L€CˆØÐ#Ý0Ð1BÀLÑQÔQÐÐå*Ð+?Ñ@Ô@ÐÝÕ-­e¬k¸'À7ÐRcÐ.dÑ.dÔ.dÑeÔeÐghÑiÔi×lÒlØ Ô'ð mñ ô €Kð Ð2°KÐ?Ð?r   c                   ón   ‡ — e Zd ZU eed<   dZ ej        ¦   «         ˆ fd„¦   «         Ze	d„ ¦   «         Z
ˆ xZS )ÚPretrainedFSMTModelr9   Úmodelc                 óè   •— t          ¦   «                              |¦  «         t          |t          ¦  «        r: |j        g |j        j        ¢|j        ‘R Ž }t          j	        |j        |¦  «         d S d S ©N)
ÚsuperÚ_init_weightsÚ
isinstanceÚSinusoidalPositionalEmbeddingÚget_embeddingÚweightr$   Úpadding_idxÚinitÚcopy_)ÚselfÚmodulerL   Ú	__class__s      €r   rH   z!PretrainedFSMTModel._init_weightsÞ   sw   ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ;Ñ<Ô<ð 	.Ø)�VÔ)ÐS¨6¬=Ô+>ÐSÀÔ@RÐSÐSÐSˆFÝŒJ�v”} fÑ-Ô-Ð-Ð-Ð-ð	.ð 	.r   c                 ó–   — | j         j        }t          j        g d¢dddd|gg| j        ¬¦  «        }|                     |¦  «        |dœ}|S )N)r   é   é
   é   r   r   é   é   r   r!   )r   r:   )r9   r2   r%   Útensorr"   Úne)rP   Ú	pad_tokenr:   Údummy_inputss       r   r\   z PretrainedFSMTModel.dummy_inputså   sa   € à”KÔ,ˆ	Ý”LÐ"2Ð"2Ð"2°Q¸¸2¸qÀ)Ð4LÐ!MÐVZÔVaÐbÑbÔbˆ	à'Ÿlšl¨9Ñ5Ô5Ø"ð
ð 
ˆð Ðr   )Ú__name__Ú
__module__Ú__qualname__r   Ú__annotations__Úbase_model_prefixr%   Úno_gradrH   Úpropertyr\   Ú__classcell__©rR   s   @r   rC   rC   Ù   sy   ø€ € € € € € àÐÐÑØÐà€U„]�_„_ð.ð .ð .ð .ñ „_ð.ð ðð ñ „Xðð ð ð ð r   rC   c                 ó~   — | j         j        \  }}t          j        ||d¬¦  «        }| j         j        |j         _        |S )NF©Úbias)rL   r$   r   ÚLinearÚdata)ÚembÚ
vocab_sizeÚemb_sizeÚ	lin_layers       r   Ú_make_linear_from_embro   ð   s<   € Øœ:Ô+Ñ€J�Ý”	˜* h°UÐ;Ñ;Ô;€IØœJœO€IÔÔØÐr   c                 ó<   — | |k    rt          d| › d|› �¦  «        ‚d S )Nzshape mismatch: z != )ÚAssertionError)Úshape_1Úshape2s     r   Ú_check_shapesrt   ø   s4   € Ø�&ÒÐÝÐE°ÐEÐE¸VÐEÐEÑFÔFÐFð Ðr   c                 ól  — |                       | dk    |¦  «         |                      ¦   «         }|                      |¦  «                             d¬¦  «        dz
                       d¦  «        }|                      d|¦  «                             ¦   «         |dd…df<   | dd…dd…f         |dd…dd…f<   |S )zXShift input ids one token to the right, and wrap the last non pad token (usually <eos>).iœÿÿÿr   ©r   r#   Nr   )Úmasked_fill_ÚclonerZ   Úsumr(   ÚgatherÚsqueeze)r:   r2   Úprev_output_tokensÚindex_of_eoss       r   r3   r3   ý   sÁ   € ð ×Ò˜9¨Ò,¨lÑ;Ô;Ð;à"ŸšÑ*Ô*ÐØ—L’L Ñ.Ô.×2Ò2°qÐ2Ñ9Ô9¸AÑ=×HÒHÈÑLÔL€LØ(×/Ò/°°<Ñ@Ô@×HÒHÑJÔJÐ�q�q�q˜!�tÑØ )¨!¨!¨!¨S¨b¨S¨&Ô 1Ð�q�q�q˜!˜"˜"�uÑØÐr   c                 ó\   — |                       |¦  «        }|                     ¦   «         sd}|S )zTrue for pad tokensN)r   Úany)r:   rM   Úpadding_masks      r   r5   r5   
  s2   € à—<’< Ñ,Ô,€LØ×ÒÑÔð ØˆØÐr   c                   ó,   ‡ — e Zd Zdefˆ fd„Zdd„Zˆ xZS )ÚEncoderLayerr9   c                 óþ  •— t          ¦   «                              ¦   «          |j        | _        t	          | j        |j        |j        ¬¦  «        | _        t          | j        ¦  «        | _	        |j
        | _
        t          |j                 | _        |j        | _        t          j        | j        |j        ¦  «        | _        t          j        |j        | j        ¦  «        | _        t          | j        ¦  «        | _        d S )N)Údropout)rG   Ú__init__Úd_modelÚ	embed_dimÚ	AttentionÚencoder_attention_headsÚattention_dropoutÚ	self_attnr   Úself_attn_layer_normr„   r
   Úactivation_functionÚactivation_fnÚactivation_dropoutr   ri   Úencoder_ffn_dimÚfc1Úfc2Úfinal_layer_norm©rP   r9   rR   s     €r   r…   zEncoderLayer.__init__  s¿   ø€ Ý‰Œ×ÒÑÔÐØœˆŒÝ" 4¤>°6Ô3QÐ[aÔ[sÐtÑtÔtˆŒÝ$-¨d¬nÑ$=Ô$=ˆÔ!Ø”~ˆŒÝ# FÔ$>Ô?ˆÔØ"(Ô";ˆÔÝ”9˜Tœ^¨VÔ-CÑDÔDˆŒÝ”9˜VÔ3°T´^ÑDÔDˆŒÝ )¨$¬.Ñ 9Ô 9ˆÔÐÐr   Fc                 ó4  — |}|                       ||||¬¦  «        \  }}t          j                             || j        | j        ¬¦  «        }||z   }|                      |¦  «        }|}|                      |                      |¦  «        ¦  «        }t          j                             || j        | j        ¬¦  «        }|  	                    |¦  «        }t          j                             || j        | j        ¬¦  «        }||z   }|  
                    |¦  «        }||fS )aÒ  
        Args:
            x (`torch.Tensor`): input to the layer of shape *(seq_len, batch, embed_dim)*
            encoder_padding_mask (`torch.ByteTensor`): binary ByteTensor of shape
                *(batch, src_len)* where padding elements are indicated by `1`.
            for t_tgt, t_src is excluded (or masked out), =0 means it is
            included in attention

        Returns:
            encoded output of shape *(seq_len, batch, embed_dim)*
        )ÚqueryÚkeyÚkey_padding_maskÚoutput_attentions©ÚpÚtraining)r‹   r   Ú
functionalr„   rœ   rŒ   rŽ   r‘   r�   r’   r“   )rP   r*   Úencoder_padding_maskr™   ÚresidualÚattn_weightss         r   ÚforwardzEncoderLayer.forward"  s  € ð ˆØŸ.š.ØØØ1Ø/ð	 )ñ 
ô 
‰ˆˆ<õ ŒM×!Ò! ! t¤|¸d¼mÐ!ÑLÔLˆØ�q‰LˆØ×%Ò% aÑ(Ô(ˆàˆØ×Ò˜tŸxšx¨™{œ{Ñ+Ô+ˆÝŒM×!Ò! ! tÔ'>ÈÌÐ!ÑWÔWˆØ�HŠH�Q‰KŒKˆÝŒM×!Ò! ! t¤|¸d¼mÐ!ÑLÔLˆØ�q‰LˆØ×!Ò! !Ñ$Ô$ˆØ�,ˆÐr   )F©r]   r^   r_   r   r…   r¡   rd   re   s   @r   r‚   r‚     sX   ø€ € € € € ð
:˜zð 
:ð 
:ð 
:ð 
:ð 
:ð 
:ðð ð ð ð ð ð ð r   r‚   c                   ó~   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 	 	 ddej        dej        dz  d	ej        dz  d
ededefd„Z	ˆ xZ
S )ÚFSMTEncoderz¢
    Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a [`EncoderLayer`].

    Args:
        config: FSMTConfig
    r9   c                 ó  •‡— t          ¦   «                              ¦   «          ‰j        | _        ‰j        | _        ‰j        | _        t          j        ‰j	        ‰j
        ‰j        ¦  «        | _        | j        j        }‰j        rt          j        |¦  «        nd| _        t#          ‰j        | j        z   dz   || j        ¦  «        | _        t          j        ˆfd„t+          ‰j        ¦  «        D ¦   «         ¦  «        | _        d S )Nç      ð?r   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS © )r‚   )Ú.0Ú_r9   s     €r   ú
<listcomp>z(FSMTEncoder.__init__.<locals>.<listcomp>V  s!   ø€ Ð$`Ð$`Ð$`¸a¥\°&Ñ%9Ô%9Ð$`Ð$`Ð$`r   )rG   r…   r„   Úencoder_layerdropÚ	layerdropr2   rM   r   Ú	EmbeddingÚsrc_vocab_sizer†   Úembed_tokensÚembedding_dimÚscale_embeddingÚmathÚsqrtÚembed_scalerJ   Úmax_position_embeddingsÚembed_positionsÚ
ModuleListÚrangeÚencoder_layersÚlayers©rP   r9   r‡   rR   s    ` €r   r…   zFSMTEncoder.__init__K  sä   øø€ Ý‰Œ×ÒÑÔÐØ”~ˆŒØÔ1ˆŒØ!Ô.ˆÔÝœL¨Ô)>ÀÄÐPVÔPcÑdÔdˆÔØÔ%Ô3ˆ	Ø39Ô3IÐR�4œ9 YÑ/Ô/Ð/ÈsˆÔÝ<ØÔ*¨TÔ-=Ñ=ÀÑAÀ9ÈdÔN^ñ 
ô  
ˆÔõ ”mÐ$`Ð$`Ð$`Ð$`Å5ÈÔI^ÑC_ÔC_Ð$`Ñ$`Ô$`ÑaÔaˆŒˆˆr   NFTr:   r   Úinputs_embedsr™   Úoutput_hidden_statesÚreturn_dictc                 ó  — |�t          |¦  «        }|�|�t          d¦  «        ‚|�3|                      |¦  «        | j        z  }|                      |¦  «        }n~|�m|| j        z  }|dd…dd…df                              |dd…dd…df                              d¦  «        | j        j        ¦  «        }|                      |¦  «        }nt          d¦  «        ‚||z   }	t          j	         
                    |	| j
        | j        ¬¦  «        }	|	                     dd¦  «        }	|rdnd}
|rdnd}t          | j        ¦  «        D ]{\  }}|r2|	                     dd¦  «        }	|
|	fz  }
|	                     dd¦  «        }	t          j        g ¦  «        }| j        r|| j        k     rd}n ||	||¬¦  «        \  }	}|r||fz   }Œ||	                     dd¦  «        }	|r|
|	fz  }
|st%          d	„ |	|
|fD ¦   «         ¦  «        S t'          |	|
|¬
¦  «        S )a©  
        Args:
            input_ids (`torch.LongTensor`): tokens in the source language of shape
                *(batch, src_len)*
            attention_mask (`torch.LongTensor`): indicating which indices are padding tokens
            inputs_embeds (`torch.FloatTensor`):
                embedding vectors of shape *(batch, src_len, embed_dim)*

        Returns:
            BaseModelOutput or Tuple comprised of:

                - **x** (`torch.Tensor`): the last encoder layer's output of shape *(src_len, batch, embed_dim)*
                - **encoder_states** (`Tuple(torch.FloatTensor)`): all intermediate hidden states of shape *(src_len,
                  batch, embed_dim)*. Only populated if *output_hidden_states:* is True.
                - **all_attentions** (`Tuple(torch.FloatTensor)`): Attention weights for each layer.
                During training might not be of length n_layers because of layer dropout.
        NzDYou cannot specify both input_ids and inputs_embeds at the same timer   z5You have to specify either input_ids or inputs_embedsrš   r   r¨   )r™   c              3   ó   K  — | ]}|®|V — Œ	d S rF   r¨   ©r©   Úvs     r   ú	<genexpr>z&FSMTEncoder.forward.<locals>.<genexpr>ª  s"   è è € ÐYÐY˜qÈ1È=˜È=È=È=È=ÐYÐYr   ©Úlast_hidden_stateÚhidden_statesÚ
attentions)r   Ú
ValueErrorr°   rµ   r·   r)   r   rM   r   r�   r„   rœ   Ú	transposeÚ	enumerater»   r%   Úrandr­   Útupler   )rP   r:   r   r½   r™   r¾   r¿   Ú	embed_posÚposition_idsr*   Úencoder_statesÚall_attentionsÚidxÚencoder_layerÚdropout_probabilityÚattns                   r   r¡   zFSMTEncoder.forwardX  s{  € ð6 Ð%Ý(¨Ñ8Ô8ˆNàÐ  ]Ð%>ÝÐcÑdÔdÐdØÐ"Ø ×-Ò-¨iÑ8Ô8¸4Ô;KÑKˆMØ×,Ò,¨YÑ7Ô7ˆIˆIØÐ&Ø)¨DÔ,<Ñ<ˆMð )¨¨¨¨A¨A¨A¨q¨Ô1×=Ò=Ø˜a˜a˜a    A˜gÔ&×)Ò)¨!Ñ,Ô,¨dÔ.BÔ.Nñô ˆLð ×,Ò,¨\Ñ:Ô:ˆIˆIåÐTÑUÔUÐUà˜IÑ%ˆÝŒM×!Ò! ! t¤|¸d¼mÐ!ÑLÔLˆð �KŠK˜˜1ÑÔˆà3Ð=˜˜¸ˆØ0Ð:˜˜°dˆÝ"+¨D¬KÑ"8Ô"8ð 	:ð 	:ÑˆC�Ø#ð &Ø—K’K  1Ñ%Ô%�Ø 1 $Ñ&�Ø—K’K  1Ñ%Ô%�å"'¤*¨R¡.¤.ÐØŒ}ð Ð"5¸¼Ò"FÐ"FØ��à'˜-ØØ"Ø&7ðñ ô ‘��4ð !ð :Ø!/°4°'Ñ!9�øð �KŠK˜˜1ÑÔˆàð 	#Ø˜q˜dÑ"ˆNàð 	ZÝÐYÐY Q¨¸Ð$GÐYÑYÔYÑYÔYÐYÝ°À.Ð]kÐlÑlÔlÐlr   )NNFFT)r]   r^   r_   Ú__doc__r   r…   r%   r   Úboolr¡   rd   re   s   @r   r¤   r¤   C  sç   ø€ € € € € ðð ðb˜zð bð bð bð bð bð bð  /3Ø-1Ø"'Ø%*Ø ðSmð Smà”<ðSmð œ tÑ+ðSmð ”| dÑ*ð	Smð
  ðSmð #ðSmð ðSmð Smð Smð Smð Smð Smð Smð Smr   r¤   c                   ó8   ‡ — e Zd Zddefˆ fd„Z	 	 	 	 	 dd„Zˆ xZS )ÚDecoderLayerNr9   c                 ó‚  •— t          ¦   «                              ¦   «          |j        | _        t	          | j        |j        |j        |¬¦  «        | _        |j        | _        t          |j
                 | _        |j        | _        t          | j        ¦  «        | _        t	          | j        |j        |j        d|¬¦  «        | _        t          | j        ¦  «        | _        t#          j        | j        |j        ¦  «        | _        t#          j        |j        | j        ¦  «        | _        t          | j        ¦  «        | _        d S )N)r‡   Ú	num_headsr„   Ú	layer_idxT)r„   Úencoder_decoder_attentionrÜ   )rG   r…   r†   r‡   rˆ   Údecoder_attention_headsrŠ   r‹   r„   r
   r�   rŽ   r�   r   rŒ   Úencoder_attnÚencoder_attn_layer_normr   ri   Údecoder_ffn_dimr‘   r’   r“   )rP   r9   rÜ   rR   s      €r   r…   zDecoderLayer.__init__¯  s  ø€ Ý‰Œ×ÒÑÔÐØœˆŒå"Ø”nØÔ4ØÔ,Øð	
ñ 
ô 
ˆŒð ”~ˆŒÝ# FÔ$>Ô?ˆÔØ"(Ô";ˆÔå$-¨d¬nÑ$=Ô$=ˆÔ!Ý%ØŒNØÔ*ØÔ,Ø&*Øð
ñ 
ô 
ˆÔõ (1°´Ñ'@Ô'@ˆÔ$Ý”9˜Tœ^¨VÔ-CÑDÔDˆŒÝ”9˜VÔ3°T´^ÑDÔDˆŒÝ )¨$¬.Ñ 9Ô 9ˆÔÐÐr   Fc                 ó<  — |}	|                       ||||||¬¦  «        \  }}
t          j                             || j        | j        ¬¦  «        }|	|z   }|                      |¦  «        }|}	| j        j        | j         j        k    sJ ‚|                      |||||¬¦  «        \  }}t          j                             || j        | j        ¬¦  «        }|	|z   }|                      |¦  «        }|}	|  	                    |  
                    |¦  «        ¦  «        }t          j                             || j        | j        ¬¦  «        }|                      |¦  «        }t          j                             || j        | j        ¬¦  «        }|	|z   }|                      |¦  «        }||
|fS )N)r–   r—   Úlayer_stater˜   Ú	attn_maskr™   rš   )r–   r—   r˜   rã   r™   )r‹   r   r�   r„   rœ   rŒ   rß   Ú	cache_keyrà   rŽ   r‘   r�   r’   r“   )rP   r*   Úencoder_hidden_statesÚencoder_attn_maskrã   r@   r<   r™   ÚkwargsrŸ   Úself_attn_weightsÚcross_attn_weightss               r   r¡   zDecoderLayer.forwardÊ  s¬  € ð ˆð  $Ÿ~š~ØØØ#Ø1Ø!Ø/ð  .ñ  
ô  
ÑˆÐõ ŒM×!Ò! ! t¤|¸d¼mÐ!ÑLÔLˆØ�q‰LˆØ×%Ò% aÑ(Ô(ˆð ˆØÔ Ô*¨d¬nÔ.FÒFÐFÐFÐFØ $× 1Ò 1ØØ%Ø.Ø#Ø/ð !2ñ !
ô !
ÑˆÐõ ŒM×!Ò! ! t¤|¸d¼mÐ!ÑLÔLˆØ�q‰LˆØ×(Ò(¨Ñ+Ô+ˆð ˆØ×Ò˜tŸxšx¨™{œ{Ñ+Ô+ˆÝŒM×!Ò! ! tÔ'>ÈÌÐ!ÑWÔWˆØ�HŠH�Q‰KŒKˆÝŒM×!Ò! ! t¤|¸d¼mÐ!ÑLÔLˆØ�q‰LˆØ×!Ò! !Ñ$Ô$ˆàØØð
ð 	
r   rF   )NNNNFr¢   re   s   @r   rÙ   rÙ   ®  sn   ø€ € € € € ð:ð :˜zð :ð :ð :ð :ð :ð :ð> ØØØ!Øð4
ð 4
ð 4
ð 4
ð 4
ð 4
ð 4
ð 4
r   rÙ   c                   óÊ   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 	 	 	 ddej        dej        d	ej        d
ej        dej        dej        dz  dedz  de	dz  de	dz  de	dz  de	dz  fd„Z
ˆ xZS )ÚFSMTDecoderzÈ
    Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`DecoderLayer`]

    Args:
        config: FSMTConfig
        embed_tokens (nn.Embedding): output embedding
    r9   c                 ó\  •‡— t          ¦   «                              ¦   «          ‰j        | _        ‰j        | _        ‰j        | _        ‰j        rt          j	        ‰j
        ¦  «        nd| _        t          j        ‰j        ‰j
        | j        ¦  «        | _        | j        j        }t#          ‰j        | j        z   dz   || j        ¦  «        | _        t          j        ˆfd„t+          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j
        ‰j        d¬¦  «        | _        d S )Nr¦   r   c                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS ))rÜ   )rÙ   )r©   Úir9   s     €r   r«   z(FSMTDecoder.__init__.<locals>.<listcomp>  s&   ø€ Ð$mÐ$mÐ$mÈ1¥\°&ÀAÐ%FÑ%FÔ%FÐ$mÐ$mÐ$mr   Frg   )rG   r…   r„   Údecoder_layerdropr­   r2   rM   r²   r³   r´   r†   rµ   r   r®   Útgt_vocab_sizer°   r±   rJ   r¶   r·   r¸   r¹   Údecoder_layersr»   ri   Úoutput_projectionr¼   s    ` €r   r…   zFSMTDecoder.__init__
  s
  øø€ Ý‰Œ×ÒÑÔÐØ”~ˆŒØÔ1ˆŒØ!Ô.ˆÔØ8>Ô8NÐW�4œ9 V¤^Ñ4Ô4Ð4ÐTWˆÔÝœL¨Ô)>ÀÄÐPTÔP`ÑaÔaˆÔØÔ%Ô3ˆ	Ý<ØÔ*¨TÔ-=Ñ=ÀÑAÀ9ÈdÔN^ñ 
ô  
ˆÔõ ”mÐ$mÐ$mÐ$mÐ$mÕPUÐV\ÔVkÑPlÔPlÐ$mÑ$mÔ$mÑnÔnˆŒÝ!#¤¨6¬>¸6Ô;PÐW\Ð!]Ñ!]Ô!]ˆÔÐÐr   NFTr:   ræ   rž   r<   Údecoder_causal_maskr½   Úpast_key_valuesÚ	use_cacher™   r¾   r¿   c           
      óN  — |�t          |¦  «        }|�|�t          d¦  «        ‚|�Q|                      |¦  «        }|r|dd…dd…f         }|dd…dd…f         }|                      |¦  «        | j        z  }n~|�m|dd…dd…df                              |dd…dd…df                              d¦  «        | j        j        ¦  «        }|                      |¦  «        }|| j        z  }nt          d¦  «        ‚||z  }t          j	         
                    || j
        | j        ¬¦  «        }|                     dd¦  «        }|                     dd¦  «        }|
rdnd}|	rdnd}|	rdnd}t          | j        ¦  «        D ]„\  }}|
r2|                     dd¦  «        }||fz  }|                     dd¦  «        }| j        r t          j        g ¦  «        }|| j        k     rŒ` ||||||||	¬	¦  «        \  }}}|	r||fz  }||fz  }Œ…|
r2|                     dd¦  «        }||fz  }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                      |¦  «        }|st'          d
„ |||||fD ¦   «         ¦  «        S t)          |||||¬¦  «        S )a  
        Includes several features from "Jointly Learning to Align and Translate with Transformer Models" (Garg et al.,
        EMNLP 2019).

        Args:
            input_ids (`torch.LongTensor` of shape `(batch, tgt_len)`):
                previous decoder outputs for teacher forcing
            encoder_hidden_states: output from the encoder, used for
                encoder-side attention
            encoder_padding_mask: for ignoring pad tokens
            past_key_values (dict or None): dictionary used for storing state during generation

        Returns:
            BaseModelOutputWithPast or tuple:

                - the decoder's features of shape *(batch, tgt_len, embed_dim)*
                - the cache
                - hidden states
                - attentions
        NzTYou cannot specify both decoder_input_ids and decoder_inputs_embeds at the same timer#   r   zEYou have to specify either decoder_input_ids or decoder_inputs_embedsrš   r   r¨   )rç   r<   rã   r@   r™   c              3   ó   K  — | ]}|®|V — Œ	d S rF   r¨   rÂ   s     r   rÄ   z&FSMTDecoder.forward.<locals>.<genexpr>…  s1   è è € ð ð ØÐghÐgt�ÐgtÐgtÐgtÐgtðð r   )rÆ   rõ   rÇ   rÈ   Úcross_attentions)r   rÉ   r·   r°   rµ   r)   r   rM   r   r�   r„   rœ   rÊ   rË   r»   r%   rÌ   r­   ró   rÍ   r   )rP   r:   ræ   rž   r<   rô   r½   rõ   rö   r™   r¾   r¿   rè   Ú	positionsr*   rÏ   Úall_hidden_statesÚall_self_attnsÚall_cross_attnsrÒ   Údecoder_layerrÔ   Úlayer_self_attnÚlayer_cross_attns                           r   r¡   zFSMTDecoder.forward  sn  € ðH  Ð+Ý#.Ð/CÑ#DÔ#DÐ àÐ  ]Ð%>ÝÐsÑtÔtÐtØÐ"à×,Ò,¨YÑ7Ô7ˆIØð .Ø% a a a¨¨¨ fÔ-�	Ø% a a a¨¨¨ fÔ-�	Ø×!Ò! )Ñ,Ô,¨tÔ/?Ñ?ˆAˆAØÐ&ð )¨¨¨¨A¨A¨A¨q¨Ô1×=Ò=Ø˜a˜a˜a    A˜gÔ&×)Ò)¨!Ñ,Ô,¨dÔ.BÔ.Nñô ˆLð ×,Ò,¨\Ñ:Ô:ˆIØ Ô 0Ñ0ˆAˆAåÐdÑeÔeÐeà	ˆY‰ˆÝŒM×!Ò! ! t¤|¸d¼mÐ!ÑLÔLˆð �KŠK˜˜1ÑÔˆØ 5× ?Ò ?ÀÀ1Ñ EÔ EÐð #7Ð@˜B˜B¸DÐØ0Ð:˜˜°dˆØ 1Ð;˜"˜"°tˆå"+¨D¬KÑ"8Ô"8ð 	7ð 	7ÑˆC�à#ð &Ø—K’K  1Ñ%Ô%�Ø! a TÑ)Ð!Ø—K’K  1Ñ%Ô%�ØŒ}ð Ý&+¤j°¡n¤nÐ#Ø&¨¬Ò7Ð7Øà3@°=ØØ%Ø"6Ø%9Ø+Ø/Ø"3ð4ñ 4ô 4Ñ0ˆAˆÐ 0ð !ð 7Ø ?Ð"4Ñ4�ØÐ$4Ð#6Ñ6�øð  ð 	"Ø—’˜A˜qÑ!Ô!ˆAØ ! Ñ%ÐØ—’˜A˜qÑ!Ô!ˆAð �KŠK˜˜1ÑÔˆØ 5× ?Ò ?ÀÀ1Ñ EÔ EÐà×"Ò" 1Ñ%Ô%ˆàð 	Ýð ð Ø˜Ð0AÀ>ÐSbÐcðñ ô ñ ô ð õ 9ØØ+Ø+Ø%Ø,ð
ñ 
ô 
ð 	
r   )NNFFFT)r]   r^   r_   rÖ   r   r…   r%   r   r   r×   r¡   rd   re   s   @r   rì   rì     s'  ø€ € € € € ðð ð^˜zð ^ð ^ð ^ð ^ð ^ð ^ð* .2Ø(,Ø!&Ø).Ø,1Ø#'ðv
ð v
à”<ðv
ð  %œ|ðv
ð $œlð	v
ð
 $œlðv
ð #œ\ðv
ð ”| dÑ*ðv
ð  ™ðv
ð ˜$‘;ðv
ð   $™;ðv
ð # T™kðv
ð ˜D‘[ðv
ð v
ð v
ð v
ð v
ð v
ð v
ð v
r   rì   c                 óp   — |                       ¦   «         D ] \  }}|�|                     d|¦  «        | |<   Œ!| S )Nr   )ÚitemsÚindex_select)Ú
attn_cacheÚ	new_orderÚkÚinput_buffer_ks       r   Ú_reorder_bufferr  ‘  sJ   € Ø'×-Ò-Ñ/Ô/ð Fð FÑˆˆ>ØÐ%Ø*×7Ò7¸¸9ÑEÔEˆJ�q‰MøØÐr   c                   óŠ   ‡ — e Zd ZdZ	 	 	 	 dˆ fd„	Z	 	 	 	 ddedz  dedz  d	edz  d
edz  dedz  deeedz  f         fd„Z	ˆ xZ
S )rˆ   z=Multi-headed attention from 'Attention Is All You Need' paperç        TFNc                 óú  •— t          ¦   «                              ¦   «          || _        || _        || _        ||z  | _        | j        |z  | j        k    s
J d¦   «         ‚| j        dz  | _        || _        || _        t          j
        |||¬¦  «        | _        t          j
        |||¬¦  «        | _        t          j
        |||¬¦  «        | _        t          j
        |||¬¦  «        | _        | j        rdnd| _        d S )Nz(embed_dim must be divisible by num_headsg      à¿rg   Úencoder_decoderrP   )rG   r…   r‡   rÛ   r„   Úhead_dimÚscalingrÜ   rÝ   r   ri   Úk_projÚv_projÚq_projÚout_projrå   )rP   r‡   rÛ   r„   rh   rÝ   rÜ   rR   s          €r   r…   zAttention.__init__›  sö   ø€ õ 	‰Œ×ÒÑÔÐØ"ˆŒØ"ˆŒØˆŒØ! YÑ.ˆŒØŒ}˜yÑ(¨D¬NÒ:Ð:Ð:Ð<fÑ:Ô:Ð:Ø”} dÑ*ˆŒØ"ˆŒà)BˆÔ&Ý”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝ”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝ”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝœ	 )¨Y¸TÐBÑBÔBˆŒØ.2Ô.LÐXÐ*Ð*ÐRXˆŒˆˆr   r—   r˜   rã   rä   r™   Úreturnc                 óö	  — |                      ¦   «         \  }}	}
|
| j        k    sJ ‚t          |                      ¦   «         ¦  «        ||	|
gk    sJ ‚|�Mt          |t          ¦  «        r6|j                             | j        ¦  «        }| j        r|j	        }n
|j
        }n|}| j        r|n|}| j        r3|�1|r/|j        | j                 j        }|j        | j                 j        }nÑ|                      |¦  «        }|                      |¦  «        }|                     d|	| j        | j        ¦  «                             dddd¦  «        }|                     d|	| j        | j        ¦  «                             dddd¦  «        }|�5|                     ||| j        ¦  «        \  }}| j        rd|j        | j        <   |                      |¦  «        | j        z  }|                     d|	| j        z  | j        ¦  «                             dd¦  «        }|                     |	| j        z  d| j        ¦  «        }|                     |	| j        z  d| j        ¦  «        }|€J ‚|                      d¦  «        }t3          j        ||                     dd¦  «        ¦  «        }|                      ¦   «         |	| j        z  ||fk    sJ ‚|�?|                     |	| j        ||¦  «        |z   }|                     |	| j        z  ||¦  «        }|�|                     ¦   «         dk    rd}|�$|                      ¦   «         dd…         |	|fk    sJ ‚|�–|                     |	| j        ||¦  «        }|                     d¦  «                             d¦  «        }|                     |t3          j        |j        ¦  «        j         ¦  «        }|                     |	| j        z  ||¦  «        }tB          j"         #                    |d¬¦  «        }|r=|                     |	| j        ||¦  «        }|                     |	| j        z  ||¦  «        }nd}tB          j"         $                    || j$        | j%        ¬	¦  «        }|€J ‚t3          j        ||¦  «        }|                      ¦   «         |	| j        z  || j        fk    sJ ‚|                     dd¦  «         &                    ¦   «                              ||	|
¦  «        }|  '                    |¦  «        }||fS )
z+Input shape: Time(SeqLen) x Batch x ChannelNr#   r   r   r   r   Trv   rš   )(r4   r‡   ÚlistrI   r   Ú
is_updatedÚgetrÜ   rÝ   Úcross_attention_cacheÚself_attention_cacher»   ÚkeysÚvaluesr  r  ÚviewrÛ   r  ÚpermuteÚupdater  r  rÊ   Úreshaper%   Úbmmr   r(   r)   Úfinfor1   Úminr   r�   Úsoftmaxr„   rœ   Ú
contiguousr  )rP   r–   r—   r˜   rã   rä   r™   rè   r?   r>   r‡   r  Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚquery_statesÚsrc_lenr    ÚreshapedÚattn_weights_reshapedÚ
attn_probsÚattn_outputs                          r   r¡   zAttention.forward´  s  € ð #(§*¢*¡,¤,Ñˆ��iØ˜DœNÒ*Ð*Ð*Ð*Ý�E—J’J‘L”LÑ!Ô! g¨s°IÐ%>Ò>Ð>Ð>Ð>àÐ"Ý˜+Õ':Ñ;Ô;ð 3Ø(Ô3×7Ò7¸¼ÑGÔG�
ØÔ1ð Là+6Ô+LÐ(Ð(à+6Ô+KÐ(Ð(à'2Ð$ð !%Ô >ÐI˜˜ÀEˆØÔ)ð 	B¨kÐ.EÈ*Ð.Eà-Ô4°T´^ÔDÔIˆJØ/Ô6°t´~ÔFÔMˆLˆLàŸš ^Ñ4Ô4ˆJØŸ;š; ~Ñ6Ô6ˆLØ#Ÿš¨¨S°$´.À$Ä-ÑPÔP×XÒXÐYZÐ\]Ð_`ÐbcÑdÔdˆJØ'×,Ò,¨R°°d´nÀdÄmÑTÔT×\Ò\Ð]^Ð`aÐcdÐfgÑhÔhˆLàÐ&à+?×+FÒ+FÀzÐS_ÐaeÔaoÑ+pÔ+pÑ(�
˜LàÔ1ð BØ=A�KÔ*¨4¬>Ñ:à—{’{ 5Ñ)Ô)¨D¬LÑ8ˆð $×(Ò(¨¨S°4´>Ñ-AÀ4Ä=ÑQÔQ×[Ò[Ð\]Ð_`ÑaÔaˆØ×'Ò'¨¨d¬nÑ(<¸bÀ$Ä-ÑPÔPˆ
Ø#×+Ò+¨C°$´.Ñ,@À"ÀdÄmÑTÔTˆàÐ%Ð%Ð%Ø—/’/ !Ñ$Ô$ˆÝ”y ¨z×/CÒ/CÀAÀqÑ/IÔ/IÑJÔJˆØ× Ò Ñ"Ô" s¨T¬^Ñ';¸WÀgÐ&NÒNÐNÐNÐNàÐ Ø'×,Ò,¨S°$´.À'È7ÑSÔSÐV_Ñ_ˆLØ'×,Ò,¨S°4´>Ñ-AÀ7ÈGÑTÔTˆLð Ð'Ð,<×,@Ò,@Ñ,BÔ,BÀaÒ,GÐ,GØ#ÐØÐ'Ð+;×+@Ò+@Ñ+BÔ+BÀ2ÀAÀ2Ô+FØØðK
ò ,
ð ,
ð ,
ð 
ð
 Ð'Ø'×,Ò,¨S°$´.À'È7ÑSÔSˆLØ'×1Ò1°!Ñ4Ô4×>Ò>¸qÑAÔAˆHØ'×3Ò3°H½e¼kÈ,ÔJ\Ñ>]Ô>]Ô>aÑbÔbˆLØ'×,Ò,¨S°4´>Ñ-AÀ7ÈGÑTÔTˆLå”}×,Ò,¨\¸rÐ,ÑBÔBˆàð 	)à$0×$5Ò$5°c¸4¼>È7ÐT[Ñ$\Ô$\Ð!Ø0×5Ò5°c¸D¼NÑ6JÈGÐU\Ñ]Ô]ˆLˆLà$(Ð!å”]×*Ò*ØØŒlØ”]ð +ñ 
ô 
ˆ
ð Ð'Ð'Ð'Ý”i 
¨LÑ9Ô9ˆØ×ÒÑ!Ô! c¨D¬NÑ&:¸GÀTÄ]Ð%SÒSÐSÐSÐSØ!×+Ò+¨A¨qÑ1Ô1×<Ò<Ñ>Ô>×CÒCÀGÈSÐR[Ñ\Ô\ˆØ—m’m KÑ0Ô0ˆàÐ1Ð1Ð1r   )r
  TFN)NNNF)r]   r^   r_   rÖ   r…   r   r   r×   rÍ   r¡   rd   re   s   @r   rˆ   rˆ   ˜  së   ø€ € € € € ØGÐGð ØØ"'ØðYð Yð Yð Yð Yð Yð: +/Ø$(Ø#'Ø).ð`2ð `2ð �d‰]ð`2ð ! 4™-ð	`2ð
 ˜T‘\ð`2ð ˜D‘=ð`2ð   $™;ð`2ð 
ˆv�v ‘}Ð$Ô	%ð`2ð `2ð `2ð `2ð `2ð `2ð `2ð `2r   rˆ   c                 ó®   — |                       ¦   «                              t          j        | j        ¦  «        j        ¦  «                             | ¦  «        S )z:FP16-compatible function that fills a input_ids with -inf.)ÚfloatÚfill_r%   r!  r1   r"  Útype_as©Úts    r   r6   r6     s9   € à�7Š7‰9Œ9�?Š?�5œ; q¤wÑ/Ô/Ô3Ñ4Ô4×<Ò<¸QÑ?Ô?Ð?r   c                 ó$   — t          | dd ¦  «        S )Nr$   )Úgetattrr3  s    r   Ú
_get_shaper7    s   € Ý�1�g˜tÑ$Ô$Ð$r   c                   óZ  ‡ — e Zd ZdddœZdefˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 ddej        dej	        dz  dej        dz  d	ej
        dz  d
eej                 dz  dedz  dedz  dedz  dedz  dej        dz  dej        dz  dedz  deej	                 ez  fd„¦   «         Zd„ Zd„ Zd„ Zd„ Zˆ xZS )Ú	FSMTModelzdecoder.embed_tokens.weight)zencoder.embed_tokens.weightz decoder.output_projection.weightr9   c                 óÂ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¦  «        | _        |                      ¦   «          d S rF   )rG   r…   r¤   Úencoderrì   ÚdecoderÚ	post_initr”   s     €r   r…   zFSMTModel.__init__(  sO   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý" 6Ñ*Ô*ˆŒÝ" 6Ñ*Ô*ˆŒØ�ŠÑÔÐÐÐr   Nr:   r   r;   Údecoder_attention_maskÚencoder_outputsrõ   rö   r™   r¾   r½   Údecoder_inputs_embedsr¿   r  c                 óx  — |€d}|�|n| j         j        }|	�|	n| j         j        }	|�|n| j         j        }|�|n| j         j        }|s4|�2t          | j         |||| j        j        j        j	        ¬¦  «        \  }}}nd\  }}|€|€t          d¦  «        ‚|r8|€6t          t          | j         ¬¦  «        t          | j         ¬¦  «        ¦  «        }|€|                      |||
||	|¬¦  «        }ne|rct          |t          ¦  «        sNt          |d         t!          |¦  «        d	k    r|d	         ndt!          |¦  «        d
k    r|d
         nd¬¦  «        }|                      ||d         ||||||||	|¬¦  «        }|s||z   S t#          |j        |j        |j        |j        |j        |j        |j        |j        ¬¦  «        S )a’  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            FSMT uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        NF)r;   r<   r=   ©NNzIMake sure that `decoder_input_ids` or `decoder_inputs_embeds` are passed.)r9   )r:   r   r½   r™   r¾   r¿   r   r   r   rÅ   )rô   r½   rõ   rö   r™   r¾   r¿   )rÆ   rõ   Údecoder_hidden_statesÚdecoder_attentionsrù   Úencoder_last_hidden_stateræ   Úencoder_attentions)r9   r™   r¾   rö   r¿   rA   r<  r°   rL   r1   rÉ   r   r   r;  rI   r   Úlenr   rÆ   rõ   rÇ   rÈ   rù   )rP   r:   r   r;   r>  r?  rõ   rö   r™   r¾   r½   r@  r¿   rè   r<   r@   Údecoder_outputss                    r   r¡   zFSMTModel.forward.  s[  € ð@ Ð$ØˆIà1BÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð "+Ð!6�I�I¸D¼KÔ<Qˆ	Ø%0Ð%<�k�kÀ$Ä+ÔBYˆð ð 		;˜YÐ2ÝC_Ø”ØØ"3Ø%;Ø"&¤,Ô";Ô"BÔ"HðDñ Dô DÑ@ÐÐ3°[°[ð 1;Ñ-Ð  +àÐ$Ð)>Ð)FÝÐhÑiÔiÐiàð 	v˜Ð0Ý1µ,ÀdÄkÐ2RÑ2RÔ2RÕT`ÐhlÔhsÐTtÑTtÔTtÑuÔuˆOàÐ"Ø"ŸlšlØ#Ø-Ø+Ø"3Ø%9Ø'ð +ñ ô ˆOˆOð ð 	¥¨O½_Ñ!MÔ!Mð 	Ý-Ø"1°!Ô"4Ý47¸Ñ4HÔ4HÈ1Ò4LÐ4L˜o¨aÔ0Ð0ÐRVÝ14°_Ñ1EÔ1EÈÒ1IÐ1I˜?¨1Ô-Ð-Ètðñ ô ˆOð Ÿ,š,ØØ˜AÔØØ Ø +Ø/Ø+ØØ/Ø!5Ø#ð 'ñ 
ô 
ˆð ð 	5Ø" _Ñ4Ð4å!Ø-Ô?Ø+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð	
ñ 	
ô 	
ð 		
r   c                 ó   — | j         j        S rF   ©r;  r°   ©rP   s    r   Úget_input_embeddingszFSMTModel.get_input_embeddings˜  ó   € ØŒ|Ô(Ð(r   c                 ó   — || j         _        d S rF   rJ  ©rP   Úvalues     r   Úset_input_embeddingszFSMTModel.set_input_embeddings›  ó   € Ø$)ˆŒÔ!Ð!Ð!r   c                 ó   — | j         j        S rF   ©r<  r°   rK  s    r   Úget_output_embeddingszFSMTModel.get_output_embeddingsž  rM  r   c                 ó   — || j         _        d S rF   rT  rO  s     r   Úset_output_embeddingszFSMTModel.set_output_embeddings¡  rR  r   )NNNNNNNNNNN)r]   r^   r_   Ú_tied_weights_keysr   r…   r   r%   Ú
LongTensorr   Ú
BoolTensorrÍ   ÚFloatTensorr   r×   r   r¡   rL  rQ  rU  rW  rd   re   s   @r   r9  r9  !  sÉ  ø€ € € € € ð (EØ,Iðð Ðð
˜zð ð ð ð ð ð ð ð /3Ø59Ø:>Ø;?Ø(,Ø!%Ø)-Ø,0Ø26Ø:>Ø#'ðg
ð g
àÔ#ðg
ð œ tÑ+ðg
ð !Ô+¨dÑ2ð	g
ð
 !&Ô 0°4Ñ 7ðg
ð ˜uÔ0Ô1°DÑ8ðg
ð  ™ðg
ð ˜$‘;ðg
ð   $™;ðg
ð # T™kðg
ð Ô(¨4Ñ/ðg
ð  %Ô0°4Ñ7ðg
ð ˜D‘[ðg
ð 
ˆuŒ|Ô	Ð1Ñ	1ðg
ð g
ð g
ñ „^ðg
ðR)ð )ð )ð*ð *ð *ð)ð )ð )ð*ð *ð *ð *ð *ð *ð *r   r9  zV
    The FSMT Model with a language modeling head. Can be used for summarization.
    )Úcustom_introc                   ó|  ‡ — e Zd ZdZdefˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej	        dz  dej        dz  dej
        dz  d	eej                 dz  d
edz  dej	        dz  dej	        dz  dej        dz  dedz  dedz  dedz  dedz  deej	                 ez  fd„¦   «         Zdej	        fd„Zd„ Zd„ Zˆ xZS )ÚFSMTForConditionalGenerationrD   r9   c                 óž   •— t          ¦   «                              |¦  «         t          |¦  «        }|| _        |                      ¦   «          d S rF   )rG   r…   r9  rD   r=  )rP   r9   Ú
base_modelrR   s      €r   r…   z%FSMTForConditionalGeneration.__init__­  sG   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý˜vÑ&Ô&ˆ
ØˆŒ
ð 	�ŠÑÔÐÐÐr   Nr:   r   r;   r>  r?  rõ   r½   r@  Úlabelsrö   r™   r¾   r¿   r  c                 óÄ  — |�|n| j         j        }|	�d}
|                      |||||||||
|||¬¦  «        }|d         }d}|	�Kt          ¦   «         } ||                     d| j         j        ¦  «        |	                     d¦  «        ¦  «        }|s|f|dd…         z   }|�|f|z   n|S t          |||j        |j        |j	        |j
        |j        |j        |j        ¬¦	  «	        S )uî  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            FSMT uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example Translation:

        ```python
        >>> from transformers import AutoTokenizer, FSMTForConditionalGeneration

        >>> mname = "facebook/wmt19-ru-en"
        >>> model = FSMTForConditionalGeneration.from_pretrained(mname)
        >>> tokenizer = AutoTokenizer.from_pretrained(mname)

        >>> src_text = "ÐœÐ°ÑˆÐ¸Ð½Ð½Ð¾Ðµ Ð¾Ð±ÑƒÑ‡ÐµÐ½Ð¸Ðµ - Ñ�Ñ‚Ð¾ Ð·Ð´Ð¾Ñ€Ð¾Ð²Ð¾, Ð½Ðµ Ñ‚Ð°Ðº Ð»Ð¸?"
        >>> input_ids = tokenizer(src_text, return_tensors="pt").input_ids
        >>> outputs = model.generate(input_ids, num_beams=5, num_return_sequences=3)
        >>> tokenizer.decode(outputs[0], skip_special_tokens=True)
        "Machine learning is great, isn't it?"
        ```
        NF)r½   r   r;   r@  r?  r>  rõ   rö   r™   r¾   r¿   r   r#   r   )	ÚlossÚlogitsrõ   rC  rD  rù   rE  ræ   rF  )r9   r¿   rD   r   r  rñ   r   rõ   rC  rD  rù   rE  ræ   rF  )rP   r:   r   r;   r>  r?  rõ   r½   r@  ra  rö   r™   r¾   r¿   rè   ÚoutputsÚ	lm_logitsÚmasked_lm_lossÚloss_fctÚoutputs                       r   r¡   z$FSMTForConditionalGeneration.forwardµ  s1  € ðj &1Ð%<�k�kÀ$Ä+ÔBYˆàÐØˆIà—*’*ØØ'Ø)Ø/Ø"7Ø+Ø#9Ø+ØØ/Ø!5Ø#ð ñ 
ô 
ˆð ˜A”Jˆ	àˆØÐÝ'Ñ)Ô)ˆHà%˜X i§n¢n°R¸¼Ô9SÑ&TÔ&TÐV\×VaÒVaÐbdÑVeÔVeÑfÔfˆNàð 	ZØ�\ G¨A¨B¨B¤KÑ/ˆFØ3AÐ3M�^Ð%¨Ñ.Ð.ÐSYÐYåØØØ#Ô3Ø")Ô"?Ø&Ô9Ø$Ô5Ø&-Ô&GØ")Ô"?Ø&Ô9ð

ñ 

ô 

ð 
	
r   c                 ó6   — t          || j        j        ¦  «        S rF   )r3   r9   r2   )rP   ra  s     r   Ú%prepare_decoder_input_ids_from_labelszBFSMTForConditionalGeneration.prepare_decoder_input_ids_from_labels  s   € Ý! &¨$¬+Ô*BÑCÔCÐCr   c                 ó$   — | j         j        j        S rF   ©rD   r<  r°   rK  s    r   rU  z2FSMTForConditionalGeneration.get_output_embeddings  s   € ØŒzÔ!Ô.Ð.r   c                 ó(   — || j         j        _        d S rF   rm  rO  s     r   rW  z2FSMTForConditionalGeneration.set_output_embeddings  s   € Ø*/ˆŒ
ÔÔ'Ð'Ð'r   )NNNNNNNNNNNNN)r]   r^   r_   ra   r   r…   r   r%   rY  r   rZ  rÍ   r[  r   r×   r   r¡   rk  rU  rW  rd   re   s   @r   r^  r^  ¥  sÚ  ø€ € € € € ð  Ðð˜zð ð ð ð ð ð ð ð .2Ø.2Ø59Ø:>Ø;?Ø(,Ø-1Ø59Ø*.Ø!%Ø)-Ø,0Ø#'ð]
ð ]
àÔ# dÑ*ð]
ð œ tÑ+ð]
ð !Ô+¨dÑ2ð	]
ð
 !&Ô 0°4Ñ 7ð]
ð ˜uÔ0Ô1°DÑ8ð]
ð  ™ð]
ð ”| dÑ*ð]
ð  %œ|¨dÑ2ð]
ð Ô  4Ñ'ð]
ð ˜$‘;ð]
ð   $™;ð]
ð # T™kð]
ð ˜D‘[ð]
ð  
ˆuŒ|Ô	˜Ñ	.ð!]
ð ]
ð ]
ñ „^ð]
ð~D¸E¼Lð Dð Dð Dð Dð/ð /ð /ð0ð 0ð 0ð 0ð 0ð 0ð 0r   r^  c                   ó€   ‡ — e Zd ZdZˆ fd„Zd„ Zed„ ¦   «         Zedefd„¦   «         Z		 	 dde
dz  d	edz  fˆ fd
„Zˆ xZS )rJ   a<  
    This module produces sinusoidal positional embeddings of any length.

    We don't want to save the weight of this embedding since it's not trained (deterministic) and it can be huge.

    Padding symbols are ignored.

    These embeddings get automatically extended in forward if more positions is needed.
    c                 óN   •— t          ¦   «                              |||¦  «         d S rF   )rG   r…   )rP   Únum_positionsr±   rM   rR   s       €r   r…   z&SinusoidalPositionalEmbedding.__init__*  s%   ø€ Ý‰Œ×Ò˜¨°{ÑCÔCÐCÐCÐCr   c                 ó  — |                       |||¦  «        }|                     | j        j        | j        j        ¬¦  «        }t          j        |¦  «        | _        | j                             ¦   «          d| j        _        d S )N)r1   r"   F)	rK   r8   rL   r1   r"   r   Ú	ParameterÚdetach_Úrequires_grad)rP   rq  r±   rM   rL   s        r   Úmake_weightz)SinusoidalPositionalEmbedding.make_weight-  sm   € Ø×#Ò# M°=À+ÑNÔNˆà—’ ¤Ô!2¸4¼;Ô;M�ÑNÔNˆÝ”l 6Ñ*Ô*ˆŒØŒ×ÒÑÔÐØ$)ˆŒÔ!Ð!Ð!r   c                 ó¨  — |dz  }t          j        d¦  «        |dz
  z  }t          j        t          j        |t          j        ¬¦  «                             ¦   «         | z  ¦  «        }t          j        | t          j        ¬¦  «                             ¦   «                              d¦  «        |                     d¦  «        z  }t          j        t          j	        |¦  «        t          j
        |¦  «        gd¬¦  «                             | d¦  «        }|dz  dk    r+t          j        |t          j        | d¦  «        gd¬¦  «        }|�	d||dd…f<   |S )	zÊ
        Build sinusoidal embeddings.

        This matches the implementation in tensor2tensor, but differs slightly from the description in Section 3.5 of
        "Attention Is All You Need".
        r   i'  r   r0   r   rv   r#   N)r³   Úlogr%   Úexpr&   Úint64r0  r(   ÚcatÚsinÚcosr  r7   )Únum_embeddingsr±   rM   Úhalf_dimrk   s        r   rK   z+SinusoidalPositionalEmbedding.get_embedding5  s)  € ð ! AÑ%ˆÝŒh�u‰oŒo ¨A¡Ñ.ˆÝŒi�œ XµU´[ÐAÑAÔA×GÒGÑIÔIÈSÈDÑPÑQÔQˆÝŒl˜>µ´Ð=Ñ=Ô=×CÒCÑEÔE×OÒOÐPQÑRÔRÐUX×UbÒUbÐcdÑUeÔUeÑeˆÝŒi�œ 3™œ­¬°3©¬Ð8¸aÐ@Ñ@Ô@×EÒEÀnÐVXÑYÔYˆØ˜1Ñ Ò!Ð!å”)˜S¥%¤+¨n¸aÑ"@Ô"@ÐAÀqÐIÑIÔIˆCØÐ"Ø"#ˆC�˜Q˜Q˜Q�ÑØˆ
r   rM   c                 óÒ   — |                       |¦  «                             ¦   «         }t          j        |d¬¦  «                             |¦  «        |z                       ¦   «         |z   S )z™
        Replace non-padding symbols with their position numbers.

        Position numbers begin at padding_idx+1. Padding symbols are ignored.
        r   rv   )rZ   Úintr%   Úcumsumr2  Úlong)rY   rM   r-   s      r   Úmake_positionsz,SinusoidalPositionalEmbedding.make_positionsI  sZ   € ð �yŠy˜Ñ%Ô%×)Ò)Ñ+Ô+ˆÝ”˜T qÐ)Ñ)Ô)×1Ò1°$Ñ7Ô7¸$Ñ>×DÒDÑFÔFÈÑTÐTr   NÚincremental_stateÚtimestepc                 ó8  •— |j         dd…         \  }}| j        dz   |z   }|| j                             d¦  «        k    r!|                      || j        | j        ¦  «         |                      || j        ¦  «        }t          ¦   «                              |¦  «        S )z/Input is expected to be of size [bsz x seqlen].Nr   r   r   )	r$   rM   rL   r4   rv  r±   r„  rG   r¡   )	rP   Úinputr…  r†  r>   Úseq_lenÚmax_posrú   rR   s	           €r   r¡   z%SinusoidalPositionalEmbedding.forwardW  s‘   ø€ ð ”{ 2 A 2”‰ˆˆWØÔ" QÑ&¨Ñ0ˆØ�T”[×%Ò% aÑ(Ô(Ò(Ð(à×Ò˜W dÔ&8¸$Ô:JÑKÔKÐKØ×'Ò'¨¨tÔ/?Ñ@Ô@ˆ	Ý‰wŒw�Š˜yÑ)Ô)Ð)r   rB  )r]   r^   r_   rÖ   r…   rv  ÚstaticmethodrK   r�  r„  r   r   r¡   rd   re   s   @r   rJ   rJ     sä   ø€ € € € € ðð ðDð Dð Dð Dð Dð*ð *ð *ð ðð ñ „\ðð& ðU¨Cð Uð Uð Uñ „\ðUð  )-Ø"&ð	*ð *ð  ™:ð*ð ˜4‘-ð	*ð *ð *ð *ð *ð *ð *ð *ð *ð *r   rJ   )r^  r9  rC   )r   )r   );rÖ   r³   Útypingr   r%   r   r   Útorch.nnr   r   Ú r	   rN   Úactivationsr
   Úcache_utilsr   r   r   Ú
generationr   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_fsmtr   Ú
get_loggerr]   Úloggerr   r.   Úfloat32rA   rC   ro   rt   r3   r5   ÚModuler‚   r¤   rÙ   rì   r  rˆ   r6   r7  r9  r^  r®   rJ   Ú__all__r¨   r   r   ú<module>r›     s)  ðð6 gÐ fà €€€Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ )Ð )Ð )Ð )Ð )Ð )ðð ð ð ð ð ð ð ð ð ð ð ð .Ð -Ð -Ð -Ð -Ð -Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø *Ð *Ð *Ð *Ð *Ð *ð 
ˆÔ	˜HÑ	%Ô	%€ðhDðN ð  ð  ð'ð 'ð 'ð 'ð ØØ”mð@ð @ð @ð @ð4 ðð ð ð ð ˜/ñ ô ñ „ðð,ð ð ðGð Gð Gð

ð 
ð 
ðð ð ð ð+ð +ð +ð +ð +�2”9ñ +ô +ð +ð\hmð hmð hmð hmð hm�"”)ñ hmô hmð hmðVP
ð P
ð P
ð P
ð P
�2”9ñ P
ô P
ð P
ðfM
ð M
ð M
ð M
ð M
�"”)ñ M
ô M
ð M
ð`ð ð ð|2ð |2ð |2ð |2ð |2�”	ñ |2ô |2ð |2ð~@ð @ð @ð%ð %ð %ð ð@*ð @*ð @*ð @*ð @*Ð#ñ @*ô @*ñ „ð@*ðF €ððñ ô ð
r0ð r0ð r0ð r0ð r0Ð#6¸ñ r0ô r0ñô ð
r0ðjE*ð E*ð E*ð E*ð E* B¤Lñ E*ô E*ð E*ðP OÐ
NÐ
N€€€r   