§
    ‚ŠtjeS ã                   óŠ  — d Z ddlZddlZddlmZ ddlZddlmZmZ ddlm	Z	 ddl
mZ ddlmZmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZmZ ddlmZ  ej        e ¦  «        Z!d@d„Z"d„ Z#d@d„Z$d„ Z% ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z& ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z' ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z( ed¬¦  «        e G d„ d e¦  «        ¦   «         ¦   «         Z)e G d!„ d"e¦  «        ¦   «         Z* G d#„ d$ej+        ¦  «        Z, G d%„ d&ej-        ¦  «        Z. G d'„ d(ej-        ¦  «        Z/ G d)„ d*ej-        ¦  «        Z0 G d+„ d,e¦  «        Z1 G d-„ d.e¦  «        Z2 ed/¬¦  «         G d0„ d1e*¦  «        ¦   «         Z3 ed2¬¦  «         G d3„ d4e*¦  «        ¦   «         Z4e G d5„ d6e*¦  «        ¦   «         Z5 ed7¬¦  «         G d8„ d9e*e¦  «        ¦   «         Z6 ed:¬¦  «         G d;„ d<e*e¦  «        ¦   «         Z7 G d=„ d>e*¦  «        Z8g d?¢Z9dS )AzRPyTorch ProphetNet model, ported from ProphetNet repo(fairsequery_states version).é    N)Ú	dataclass)ÚTensorÚnn)Ú	LayerNormé   )ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)ÚGradientCheckpointingLayer)ÚBaseModelOutput)ÚPreTrainedModel)ÚModelOutputÚauto_docstringÚloggingÚtorch_compilable_checké   )ÚProphetNetConfigFc                 óÄ   — |r3t           j                             |                      ¦   «         |¬¦  «        S t           j                             | |t          j        ¬¦  «        S )N©Údim©r   Údtype)r   Ú
functionalÚsoftmaxÚfloatÚtorchÚfloat32)Úhidden_stater   Ú
onnx_traces      úp/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/prophetnet/modeling_prophetnet.pyr   r   %   sQ   € Øð QÝŒ}×$Ò$ \×%7Ò%7Ñ%9Ô%9¸sÐ$ÑCÔCÐCåŒ}×$Ò$ \°sÅ%Ä-Ð$ÑPÔPÐPó    c                 óx  — t          j        |¦  «        j        }t          j        | |¬¦  «                             d| d¦  «        }t          j        | |¬¦  «                             dd| ¦  «        }t          j        ||¬¦  «         dz                        |dd¦  «        }||z
  |k    |dk    z  }||k                         || | ¦  «        }	t          j        |||¬¦  «        }
t          j        d||¬¦  «        }t          j        ||
|¦  «        }t          j        |	|
|¦  «        }t          j	        ||gd¬¦  «        S )z@
    This function computes the bias for the predict stream
    )Údevicer   r   ©r   r%   © é   r   )
r   ÚfinfoÚminÚarangeÚviewÚexpandÚtensorÚzerosÚwhereÚcat)Úsequence_lengthÚngramr%   r   Úneg_infÚrowsÚcolsÚstream_offsetsÚ	left_maskÚ
right_maskÚ	neg_inf_tÚzero_tÚ
left_blockÚright_blocks                 r"   Úngram_attention_biasr>   ,   s1  € õ Œk˜%Ñ Ô Ô$€GÝŒ<˜°Ð7Ñ7Ô7×<Ò<¸QÀÐQRÑSÔS€DÝŒ<˜°Ð7Ñ7Ô7×<Ò<¸QÀÀ?ÑSÔS€DÝ”| E°&Ð9Ñ9Ô9Ð9¸AÑ=×CÒCÀEÈ1ÈaÑPÔP€Nð ˜‘ Ò.°4¸1²9Ñ=€Ià˜$’,×&Ò& u¨o¸ÑOÔO€Jå”˜W¨E¸&ÐAÑAÔA€IÝŒ[˜ 5°Ð8Ñ8Ô8€FÝ”˜Y¨	°6Ñ:Ô:€JÝ”+˜j¨)°VÑ<Ô<€KÝŒ9�j +Ð.°AÐ6Ñ6Ô6Ð6r#   c                 ó°  — | }d}|rY| dz  } |t          j        |t          j        |¦  «        ¦  «                             ¦   «         | z  z   }t          j        |¦  «        }n't          j        |t          j        |¦  «        ¦  «        }| dz  }t          j        ||¦  «        }|t          j        |                     ¦   «         |z  ¦  «        t          j        ||z  ¦  «        z  | |z
  z  z   }t          j	        |t          j
        |¦  «        | dz
  z  ¦  «                             ¦   «         }|t          j        ||                     ¦   «         |¦  «        z   }|S )zo
    This function computes individual parts of the relative position buckets. For more detail, see paper.
    r   r(   r   )r   ÚltÚ
zeros_likeÚintÚabsÚmaxÚlogr   Úmathr*   Ú	ones_liker0   )	Únum_bucketsÚmax_distanceÚrelative_positionsÚis_bidirectionalÚinv_relative_positionsÚrel_positions_bucketÚ	max_exactÚis_smallÚval_if_larges	            r"   Úcompute_relative_bucketsrQ   B   sg  € ð 1Ð0ÐØÐàð mØ! QÑ&ˆà ÝŒhÐ-­uÔ/?Ð@VÑ/WÔ/WÑXÔX×\Ò\Ñ^Ô^ÐalÑlñmð 	õ "'¤Ð+AÑ!BÔ!BÐÐå!&¤Ð+AÅ5ÔCSÐTjÑCkÔCkÑ!lÔ!lÐà˜qÑ €IÝŒxÐ.°	Ñ:Ô:€HØ�uœyÐ)?×)EÒ)EÑ)GÔ)GÈ)Ñ)SÑTÔTÕW[ÔW_Ø�yÑ ñXô Xñ  à	�yÑ	 ñ "ñ "€Lõ ”9˜\­5¬?¸<Ñ+HÔ+HÈKÐZ[ÉOÑ+\Ñ]Ô]×aÒaÑcÔc€LØ/µ%´+¸hÐH^×HbÒHbÑHdÔHdÐfrÑ2sÔ2sÑsÐØÐr#   c                 óà  — |                      d¦  «                             d|                     d¦  «        d¦  «        }||                      d¦  «        z
  }t          j        |dz
  |fd¬¦  «                              d¦  «        }|                     d|                     d¦  «        d¦  «        }||                      d¦  «        z
  }t          | ||d¬¦  «        }t          | ||d¬¦  «        }||fS )zm
    This function computes both main and predict relative position buckets. For more detail, see paper.
    r   éÿÿÿÿr   F)rK   )Ú	unsqueezeÚrepeatÚsizer   r1   rQ   )rH   rI   Úposition_idsÚmain_stream_relative_positionsÚ$predicting_stream_relative_positionsÚmain_relative_position_bucketsÚ!predict_relative_position_bucketss          r"   Ú#compute_all_stream_relative_bucketsr\   ]   s  € ð
 &2×%;Ò%;¸AÑ%>Ô%>×%EÒ%EÀaÈ×IZÒIZÐ[]ÑI^ÔI^Ð`aÑ%bÔ%bÐ"Ø%CÀl×F\ÒF\Ð]_ÑF`ÔF`Ñ%`Ð"õ ,1¬9°lÀQÑ6FÈÐ5UÐ[]Ð+^Ñ+^Ô+^×+hÒ+hÐijÑ+kÔ+kÐ(Ø+O×+VÒ+VÐWXÐZf×ZkÒZkÐlnÑZoÔZoÐqrÑ+sÔ+sÐ(Ø+OÐR^×RhÒRhÐikÑRlÔRlÑ+lÐ(õ &>Ø�\Ð#AÐTYð&ñ &ô &Ð"õ )AØ�\Ð#GÐZ_ð)ñ )ô )Ð%ð *Ð+LÐLÐLr#   zF
    Base class for sequence-to-sequence language models outputs.
    )Úcustom_introc                   óÆ  — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dZ
edz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed	<   dZeej                 dz  ed
<   dZeej                 dz  ed<   dZej        dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dS )ÚProphetNetSeq2SeqLMOutputaÖ  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss.
    logits (`torch.FloatTensor` of shape `(batch_size, decoder_sequence_length, config.vocab_size)`):
        Prediction scores of the main stream language modeling head (scores for each vocabulary token before
        SoftMax).
    logits_ngram (`torch.FloatTensor` of shape `(batch_size, ngram * decoder_sequence_length, config.vocab_size)`):
        Prediction scores of the predict stream language modeling head (scores for each vocabulary token before
        SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
        used (see `past_key_values` input) to speed up sequential decoding.
    decoder_ngram_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.

        Hidden-states of the predict stream of the decoder at the output of each layer plus the initial embedding
        outputs.
    decoder_ngram_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
        decoder_sequence_length, decoder_sequence_length)`.

        Attentions weights of the predict stream of the decoder, after the attention softmax, used to compute the
        weighted average in the self-attention heads.
    encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
        Sequence of hidden-states at the output of the last layer of the encoder of the model.
    NÚlossÚlogitsÚlogits_ngramÚpast_key_valuesÚdecoder_hidden_statesÚdecoder_ngram_hidden_statesÚdecoder_attentionsÚdecoder_ngram_attentionsÚcross_attentionsÚencoder_last_hidden_stateÚencoder_hidden_statesÚencoder_attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r`   r   ÚFloatTensorÚ__annotations__ra   rb   rc   r	   rd   Útuplere   rf   rg   rh   ri   rj   rk   r'   r#   r"   r_   r_   t   se  € € € € € € ðð ð< &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø-1€L�%Ô# dÑ*Ð1Ð1Ñ1Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø=AÐ˜5 Ô!2Ô3°dÑ:ÐAÐAÑAØCGÐ  uÔ'8Ô!9¸DÑ!@ÐGÐGÑGØ:>Ð˜˜eÔ/Ô0°4Ñ7Ð>Ð>Ñ>Ø@DÐ˜e EÔ$5Ô6¸Ñ=ÐDÐDÑDØ8<Ð�e˜EÔ-Ô.°Ñ5Ð<Ð<Ñ<Ø:>Ð˜uÔ0°4Ñ7Ð>Ð>Ñ>Ø=AÐ˜5 Ô!2Ô3°dÑ:ÐAÐAÑAØ:>Ð˜˜eÔ/Ô0°4Ñ7Ð>Ð>Ñ>Ð>Ð>r#   r_   z‹
    Base class for model encoder's outputs that also contains : pre-computed hidden states that can speed up sequential
    decoding.
    c                   óž  — e Zd ZU dZej        ed<   dZej        dz  ed<   dZe	dz  ed<   dZ
eej                 dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed	<   dZeej                 dz  ed
<   dZej        dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dS )ÚProphetNetSeq2SeqModelOutputaÀ  
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, decoder_sequence_length, hidden_size)`):
        Sequence of main stream hidden-states at the output of the last layer of the decoder of the model.

        If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
        hidden_size)` is output.
    last_hidden_state_ngram (`torch.FloatTensor` of shape `(batch_size,ngram * decoder_sequence_length, config.vocab_size)`, *optional*):
        Sequence of predict stream hidden-states at the output of the last layer of the decoder of the model.
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
        used (see `past_key_values` input) to speed up sequential decoding.
    decoder_ngram_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.

        Hidden-states of the predict stream of the decoder at the output of each layer plus the initial embedding
        outputs.
    decoder_ngram_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
        decoder_sequence_length, decoder_sequence_length)`.

        Attentions weights of the predict stream of the decoder, after the attention softmax, used to compute the
        weighted average in the
    encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
        Sequence of hidden-states at the output of the last layer of the encoder of the model.
    Úlast_hidden_stateNÚlast_hidden_state_ngramrc   rd   re   rf   rg   rh   ri   rj   rk   )rl   rm   rn   ro   r   rp   rq   rv   rc   r	   rd   rr   re   rf   rg   rh   ri   rj   rk   r'   r#   r"   rt   rt   §   sE  € € € € € € ðð ð: Ô(Ð(Ð(Ñ(Ø8<Ð˜UÔ.°Ñ5Ð<Ð<Ñ<Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø=AÐ˜5 Ô!2Ô3°dÑ:ÐAÐAÑAØCGÐ  uÔ'8Ô!9¸DÑ!@ÐGÐGÑGØ:>Ð˜˜eÔ/Ô0°4Ñ7Ð>Ð>Ñ>Ø@DÐ˜e EÔ$5Ô6¸Ñ=ÐDÐDÑDØ8<Ð�e˜EÔ-Ô.°Ñ5Ð<Ð<Ñ<Ø:>Ð˜uÔ0°4Ñ7Ð>Ð>Ñ>Ø=AÐ˜5 Ô!2Ô3°dÑ:ÐAÐAÑAØ:>Ð˜˜eÔ/Ô0°4Ñ7Ð>Ð>Ñ>Ð>Ð>r#   rt   zs
    Base class for model's outputs that may also contain a past key/values (to speed up sequential decoding).
    c                   ó,  — e Zd ZU dZej        ed<   dZej        dz  ed<   dZe	dz  ed<   dZ
eej                 dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed	<   dZeej                 dz  ed
<   dS )ÚProphetNetDecoderModelOutputaÇ  
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, decoder_sequence_length, hidden_size)`):
        Sequence of main stream hidden-states at the output of the last layer of the decoder of the model.

        If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
        hidden_size)` is output.
    last_hidden_state_ngram (`torch.FloatTensor` of shape `(batch_size, ngram * decoder_sequence_length, config.vocab_size)`):
        Sequence of predict stream hidden-states at the output of the last layer of the decoder of the model.
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
        used (see `past_key_values` input) to speed up sequential decoding.
    hidden_states_ngram (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.

        Hidden-states of the predict stream of the decoder at the output of each layer plus the initial embedding
        outputs.
    ngram_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
        decoder_sequence_length, decoder_sequence_length)`.

        Attentions weights of the predict stream of the decoder, after the attention softmax, used to compute the
        weighted average in the
    ru   Nrv   rc   Úhidden_statesÚhidden_states_ngramÚ
attentionsÚngram_attentionsrh   )rl   rm   rn   ro   r   rp   rq   rv   rc   r	   ry   rr   rz   r{   r|   rh   r'   r#   r"   rx   rx   Ù   sî   € € € € € € ðð ð6 Ô(Ð(Ð(Ñ(Ø8<Ð˜UÔ.°Ñ5Ð<Ð<Ñ<Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø;?Ð˜˜uÔ0Ô1°DÑ8Ð?Ð?Ñ?Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø8<Ð�e˜EÔ-Ô.°Ñ5Ð<Ð<Ñ<Ø8<Ð�e˜EÔ-Ô.°Ñ5Ð<Ð<Ñ<Ð<Ð<r#   rx   c                   óT  — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dZ
edz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed	<   dZeej                 dz  ed
<   dZeej                 dz  ed<   dS )ÚProphetNetDecoderLMOutputa¶	  
    ngram_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.

        Hidden-states of the predict stream of the decoder at the output of each layer plus the initial embedding
        outputs.
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss.
    logits (`torch.FloatTensor` of shape `(batch_size, decoder_sequence_length, config.vocab_size)`):
        Prediction scores of the main stream language modeling head (scores for each vocabulary token before
        SoftMax).
    logits_ngram (`torch.FloatTensor` of shape `(batch_size, ngram * decoder_sequence_length, config.vocab_size)`):
        Prediction scores of the predict stream language modeling head (scores for each vocabulary token before
        SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
        used (see `past_key_values` input) to speed up sequential decoding.
    hidden_states_ngram (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.

        Hidden-states of the predict stream of the decoder at the output of each layer plus the initial embedding
        outputs.
    ngram_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
        decoder_sequence_length, decoder_sequence_length)`.

        Attentions weights of the predict stream of the decoder, after the attention softmax, used to compute the
        weighted average in the
    Nr`   ra   rb   rc   ry   rz   r{   r|   rh   )rl   rm   rn   ro   r`   r   rp   rq   ra   rb   rc   r	   ry   rr   rz   r{   r|   rh   r'   r#   r"   r~   r~     s  € € € € € € ð ð  ðD &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø-1€L�%Ô# dÑ*Ð1Ð1Ñ1Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø;?Ð˜˜uÔ0Ô1°DÑ8Ð?Ð?Ñ?Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø8<Ð�e˜EÔ-Ô.°Ñ5Ð<Ð<Ñ<Ø8<Ð�e˜EÔ-Ô.°Ñ5Ð<Ð<Ñ<Ð<Ð<r#   r~   c                   ó(   — e Zd ZU eed<   dZdZd„ ZdS )ÚProphetNetPreTrainedModelÚconfigÚ
prophetnetTc                 óŠ  — | j         j        }| j         j        }|€
J d¦   «         ‚|                     |j        ¦  «        }|dd d…f                              ¦   «         |ddd …f<   ||d<   |€
J d¦   «         ‚|                     |dk    |¦  «         t          j        |dk    ¦  «         	                    ¦   «         s
J d	¦   «         ‚|S )
Nz™self.model.config.decoder_start_token_id has to be defined. In ProphetNet it is usually set to the pad_token_id. See ProphetNet docs for more information.rS   r   ).r   z1self.model.config.pad_token_id has to be defined.éœÿÿÿr   z8Verify that `shifted_input_ids` has only positive values)
r�   Údecoder_start_token_idÚpad_token_idÚ	new_zerosÚshapeÚcloneÚmasked_fill_r   ÚallÚitem)ÚselfÚ	input_idsr…   r†   Úshifted_input_idss        r"   Ú_shift_rightz&ProphetNetPreTrainedModel._shift_right?  sè   € Ø!%¤Ô!CÐØ”{Ô/ˆà%Ð1Ð1ðFñ 2Ô1Ð1ð &×/Ò/°	´Ñ@Ô@ÐØ%.¨s°C°R°C¨xÔ%8×%>Ò%>Ñ%@Ô%@Ð˜#˜q˜r˜r˜'Ñ"Ø$:Ð˜&Ñ!àÐ'Ð'Ð)\Ñ'Ô'Ð'à×&Ò&Ð'8¸DÒ'@À,ÑOÔOÐOåŒyÐ*¨aÒ/Ñ0Ô0×5Ò5Ñ7Ô7ÐsÐsÐ9sÑsÔsÐ7à Ð r#   N)rl   rm   rn   r   rq   Úbase_model_prefixÚsupports_gradient_checkpointingr�   r'   r#   r"   r€   r€   9  s=   € € € € € € àÐÐÑØ$ÐØ&*Ð#ð!ð !ð !ð !ð !r#   r€   c                   óB   ‡ — e Zd ZdZdeddfˆ fd„Zdˆ fd„	Zˆ fd„Zˆ xZS )	ÚProphetNetPositionalEmbeddingsa  
    This module learns positional embeddings up to a fixed maximum size. Padding ids are ignored by either offsetting
    based on padding_idx or by setting padding_idx to None and ensuring that the appropriate position ids are passed to
    the forward function.
    r�   ÚreturnNc                 ó„   •— |j         | _        t          ¦   «                              |j         |j        |j        ¦  «         d S ©N)Úmax_position_embeddingsÚ
max_lengthÚsuperÚ__init__Úhidden_sizer†   ©r�   r�   Ú	__class__s     €r"   r›   z'ProphetNetPositionalEmbeddings.__init__]  s8   ø€ Ø Ô8ˆŒÝ‰Œ×Ò˜Ô7¸Ô9KÈVÔM`ÑaÔaÐaÐaÐar#   c                 óh  •— |�| j         �
J d¦   «         ‚|€ú|�q|                     ¦   «         dk    rY|                     ¦   «         }|d         |z   }t          j        dt          j        |¬¦  «        t          | j         |z   ¦  «        z  }n‡|€!t          j        |t          j        |¬¦  «        }t          j        |d¬¦  «                             |¦  «        |z                       ¦   «         | j         z   }|                     d| j	        dz
  ¦  «        }t          ¦   «                              |¦  «        |fS )NzCIf position_ids is pre-computed then padding_idx should not be set.r   r   )r   r   r&   r   )Úpadding_idxÚget_seq_lengthr   ÚonesÚlongrB   ÚcumsumÚtype_asÚclampr™   rš   Úforward)	r�   Úinputs_shaper%   Úattention_maskrc   rW   Úprev_num_input_idsÚnum_input_idsrž   s	           €r"   r§   z&ProphetNetPositionalEmbeddings.forwarda  s4  ø€ ØÐ$¨$Ô*:Ð*BÐ*BØQñ +CÔ*BÐCð ÐØÐ*¨×/MÒ/MÑ/OÔ/OÐSTÒ/TÐ/Tð &5×%CÒ%CÑ%EÔ%EÐ"Ø ,¨Q¤Ð2DÑ D�Ý$œz¨&½¼
È6ÐRÑRÔRÝ˜Ô(¨=Ñ8Ñ9Ô9ñ ��ð "Ð)Ý%*¤Z°ÅEÄJÐW]Ð%^Ñ%^Ô%^�Nõ ”L °QÐ7Ñ7Ô7×?Ò?ÀÑOÔOÐR`Ñ`ß’$‘&”&˜4Ô+ñ ,�ð
  ,×1Ò1°!°T´_ÀqÑ5HÑIÔI�å‰wŒw�Š˜|Ñ,Ô,¨lÐ:Ð:r#   c                 óF   •— t          ¦   «                              |¦  «        S r—   )rš   r§   )r�   rW   rž   s     €r"   Ú_forwardz'ProphetNetPositionalEmbeddings._forward}  s   ø€ Ý‰wŒw�Š˜|Ñ,Ô,Ð,r#   )NNN)	rl   rm   rn   ro   r   r›   r§   r­   Ú__classcell__©rž   s   @r"   r”   r”   V  s•   ø€ € € € € ðð ðbÐ/ð b°Dð bð bð bð bð bð bð;ð ;ð ;ð ;ð ;ð ;ð8-ð -ð -ð -ð -ð -ð -ð -ð -r#   r”   c                   óŒ   ‡ — e Zd ZdZddedededz  fˆ fd„Z	 	 	 	 ddedz  d	edz  d
edz  de	dz  de
eedz  f         f
d„Zˆ xZS )ÚProphetNetAttentionz=Multi-headed attention from 'Attention Is All You Need' paperNr�   Únum_attn_headsÚ	layer_idxc                 ó¶  •— t          ¦   «                              ¦   «          |j        }|j        | _        |j        | _        || _        ||z  | _        || _        | j        |z  |k    s
J d¦   «         ‚t          j	        ||¦  «        | _
        t          j	        ||¦  «        | _        t          j	        ||¦  «        | _        t          j	        ||¦  «        | _        d S )Nzw`config.hidden_size` must be divisible by `config.num_encoder_attention_heads` and `config.num_decoder_attention_heads`)rš   r›   rœ   Úattention_dropoutÚdropoutr²   Úhead_dimr³   r   ÚLinearÚkey_projÚ
value_projÚ
query_projÚout_proj)r�   r�   r²   r³   rœ   rž   s        €r"   r›   zProphetNetAttention.__init__„  sÇ   ø€ Ý‰Œ×ÒÑÔÐØÔ(ˆà!'Ô!9ˆÔØ”~ˆŒØ,ˆÔØ# ~Ñ5ˆŒØ"ˆŒàŒ}˜~Ñ-°Ò<Ð<Ð<ð4ñ =Ô<Ð<õ
 œ	 +¨{Ñ;Ô;ˆŒÝœ) K°Ñ=Ô=ˆŒÝœ) K°Ñ=Ô=ˆŒåœ	 +¨{Ñ;Ô;ˆŒˆˆr#   FÚkey_value_statesr©   rc   Úoutput_attentionsr•   c                 óŠ  — |                      ¦   «         \  }}}	|d u}
t          |                      ¦   «         ¦  «        |||	gk    s%J d|||	f› d|                      ¦   «         › �¦   «         ‚|                      |¦  «        | j        dz  z  }d}|�Ht	          |t
          ¦  «        r1|j                             | j        ¦  «        }|
r|j	        }n
|j
        }n|}|
r|n|}|
r3|�1|r/|j        | j                 j        }|j        | j                 j        }nÝ|                      |¦  «        }|                      |¦  «        }|                     |d| j        | j        ¦  «                             dd¦  «        }|                     |d| j        | j        ¦  «                             dd¦  «        }|�E|                     ||| j        ¦  «        \  }}|
r$t	          |t
          ¦  «        rd|j        | j        <   |                     ||| j        | j        ¦  «                             dd¦  «        }|                      d¦  «        }t)          j        d	||                     dd
¦  «        ¦  «        }|| j        ||f}|                      ¦   «         |k    r't-          d|› d|                      ¦   «         › �¦  «        ‚|�|                     ¦   «         dk    rd }|| j        d|f}|�?|                      ¦   «         |k    r't-          d|› d|                      ¦   «         › �¦  «        ‚|�||z   }|r|}nd }t0          j                             |d¬¦  «        }t0          j                             || j        | j        ¬¦  «        }t)          j        d	||¦  «        }|| j        || j        f}|                      ¦   «         |k    r't-          d|› d|                      ¦   «         › �¦  «        ‚|                     dd¦  «                             |||	¦  «        }|                      |¦  «        }t0          j                             || j        | j        ¬¦  «        }||fS )Nz Size of hidden states should be z	, but is ç      à?FrS   r   r(   Tzbsij,bsjk->bsikr   z#Attention weights should have size r   z Attention mask should have size r   ©ÚpÚtrainingz `attn_output` should have shape ú, but is of shape ) rV   Úlistr»   r·   Ú
isinstancer   Ú
is_updatedÚgetr³   Úcross_attention_cacheÚself_attention_cacheÚlayersÚkeysÚvaluesr¹   rº   r,   r²   Ú	transposeÚupdater   ÚeinsumÚ
ValueErrorr   r   r   r   r¶   rµ   rÃ   Úreshaper¼   )r�   ry   r½   r©   rc   r¾   ÚkwargsÚ
batch_sizeÚtgt_lenrœ   Úis_cross_attentionÚquery_statesrÇ   Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚsrc_lenÚattn_weightsÚexpected_shapeÚattn_weights_reshapedÚ
attn_probsÚattn_outputs                          r"   r§   zProphetNetAttention.forward™  s¥  € ð ,9×+=Ò+=Ñ+?Ô+?Ñ(ˆ
�G˜[ð .°TÐ9ÐÝ�M×&Ò&Ñ(Ô(Ñ)Ô)ØØØð.
ò 
ð 
ð 
ð p¨j¸'À;Ð-OÐoÐoÐYf×YkÒYkÑYmÔYmÐoÐoñ	
ô 
ð 
ð —’ }Ñ5Ô5¸¼ÈÑ9KÑLˆàˆ
ØÐ&Ý˜/Õ+>Ñ?Ô?ð 7Ø,Ô7×;Ò;¸D¼NÑKÔK�
Ø%ð Pà+:Ô+PÐ(Ð(à+:Ô+OÐ(Ð(à'6Ð$à-?ÐRÐ)Ð)À]ˆØð 	F /Ð"=À*Ð"=à-Ô4°T´^ÔDÔIˆJØ/Ô6°t´~ÔFÔMˆLˆLàŸš ~Ñ6Ô6ˆJØŸ?š?¨>Ñ:Ô:ˆLØ#Ÿš¨°R¸Ô9LÈdÌmÑ\Ô\×fÒfÐghÐjkÑlÔlˆJØ'×,Ò,¨Z¸¸TÔ=PÐRVÔR_Ñ`Ô`×jÒjÐklÐnoÑpÔpˆLàÐ*à+?×+FÒ+FÀzÐS_ÐaeÔaoÑ+pÔ+pÑ(�
˜Là%ð F­*°_ÕFYÑ*ZÔ*Zð FØAE�OÔ.¨t¬~Ñ>à#×(Ò(¨°W¸dÔ>QÐSWÔS`ÑaÔa×kÒkÐlmÐopÑqÔqˆØ—/’/ !Ñ$Ô$ˆå”|Ð$5°|ÀZ×EYÒEYÐZ[Ð]^ÑE_ÔE_Ñ`Ô`ˆØ$ dÔ&9¸7ÀGÐLˆØ×ÒÑÔ .Ò0Ð0ÝÐqÀ>ÐqÐqÐ\h×\mÒ\mÑ\oÔ\oÐqÐqÑrÔrÐrð Ð%¨.×*<Ò*<Ñ*>Ô*>À!Ò*CÐ*CØ!ˆNà$ dÔ&9¸1¸gÐFˆØÐ%¨.×*=Ò*=Ñ*?Ô*?À>Ò*QÐ*QÝÐpÀÐpÐpÐYg×YlÒYlÑYnÔYnÐpÐpÑqÔqÐqØÐ%Ø'¨.Ñ8ˆLØð 	)Ø$0Ð!Ð!à$(Ð!å”}×,Ò,¨\¸rÐ,ÑBÔBˆå”]×*Ò*ØØÔ$Ø”]ð +ñ 
ô 
ˆ
õ
 ”lÐ#4°jÀ,ÑOÔOˆØ$ dÔ&9¸7ÀDÄMÐRˆØ×ÒÑÔ Ò/Ð/ÝÐvÀÐvÐvÐbm×brÒbrÑbtÔbtÐvÐvÑwÔwÐwà!×+Ò+¨A¨qÑ1Ô1×9Ò9¸*ÀgÈ{Ñ[Ô[ˆØ—m’m KÑ0Ô0ˆå”m×+Ò+¨K¸4¼<ÐRVÔR_Ð+Ñ`Ô`ˆØÐ1Ð1Ð1r#   r—   )NNNF)rl   rm   rn   ro   r   rB   r›   r   r	   Úboolrr   r§   r®   r¯   s   @r"   r±   r±   �  sê   ø€ € € € € ØGÐGð<ð <Ð/ð <Àð <ÐQTÐW[ÑQ[ð <ð <ð <ð <ð <ð <ð0 +/Ø(,Ø(,Ø).ð[2ð [2ð ! 4™-ð[2ð  ™ð	[2ð
  ™ð[2ð   $™;ð[2ð 
ˆv�v ‘}Ð$Ô	%ð[2ð [2ð [2ð [2ð [2ð [2ð [2ð [2r#   r±   c                   ó2   ‡ — e Zd ZdZdedefˆ fd„Zd„ Zˆ xZS )ÚProphetNetFeedForwardzm
    This is the residual two feed-forward layer block based on the original Transformer implementation.
    r�   Úffn_dimc                 ó"  •— t          ¦   «                              ¦   «          t          |j                 | _        t          j        |j        |¦  «        | _        t          j        ||j        ¦  «        | _	        |j
        | _
        |j        | _        d S r—   )rš   r›   r   Úactivation_functionÚactivation_fnr   r¸   rœ   ÚintermediateÚoutputÚactivation_dropoutr¶   )r�   r�   rå   rž   s      €r"   r›   zProphetNetFeedForward.__init__ü  sn   ø€ Ý‰Œ×ÒÑÔÐÝ# FÔ$>Ô?ˆÔÝœI fÔ&8¸'ÑBÔBˆÔÝ”i ¨Ô);Ñ<Ô<ˆŒØ"(Ô";ˆÔØ”~ˆŒˆˆr#   c                 ó4  — |                       |¦  «        }|                      |¦  «        }t          j                             || j        | j        ¬¦  «        }|                      |¦  «        }t          j                             || j        | j        ¬¦  «        }|S )NrÁ   )ré   rè   r   r   r¶   rë   rÃ   rê   )r�   ry   s     r"   r§   zProphetNetFeedForward.forward  s„   € Ø×)Ò)¨-Ñ8Ô8ˆØ×*Ò*¨=Ñ9Ô9ˆåœ×-Ò-¨m¸tÔ?VÐaeÔanÐ-ÑoÔoˆØŸš MÑ2Ô2ˆÝœ×-Ò-¨m¸t¼|ÐVZÔVcÐ-ÑdÔdˆØÐr#   )	rl   rm   rn   ro   r   rB   r›   r§   r®   r¯   s   @r"   rä   rä   ÷  se   ø€ € € € € ðð ð&Ð/ð &¸#ð &ð &ð &ð &ð &ð &ðð ð ð ð ð ð r#   rä   c                   ó^   ‡ — e Zd Zd
defˆ fd„Zd„ Zd„ Z	 	 	 	 	 	 ddedz  fd„Zd„ Z	d	„ Z
ˆ xZS )ÚProphetNetNgramSelfAttentionNr�   c                 óð  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | j        z  | _	        |j
        | _
        || _        | j	        | j        z  |j        k    s
J d¦   «         ‚t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¦  «        | _        t          j        |j        | j        | j        z  ¦  «        | _        d| _        d S )Nz6config.hidden_size must be divisible by num_attn_headsF)rš   r›   rœ   rH   Úrelative_max_distanceÚnum_decoder_attention_headsr²   r¶   rµ   r·   r3   r³   r   r¸   r¹   rº   r»   r¼   Úrelative_pos_embeddingsr!   ©r�   r�   r³   rž   s      €r"   r›   z%ProphetNetNgramSelfAttention.__init__  s?  ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔà!Ô-ˆÔØ%+Ô%AˆÔ"Ø$Ô@ˆÔØ”~ˆŒØ!'Ô!9ˆÔØÔ*¨dÔ.AÑAˆŒØ”\ˆŒ
Ø"ˆŒàŒ}˜tÔ2Ñ2°fÔ6HÒHÐHÐHØDñ IÔHÐHõ œ	 &Ô"4°fÔ6HÑIÔIˆŒÝœ) FÔ$6¸Ô8JÑKÔKˆŒÝœ) FÔ$6¸Ô8JÑKÔKˆŒõ œ	 &Ô"4°fÔ6HÑIÔIˆŒõ (*¤y°Ô1CÀTÔEUÐX\ÔXkÑEkÑ'lÔ'lˆÔ$ð  ˆŒˆˆr#   c                 ó’   — |                      ||| j        | j        ¦  «                             dd¦  «                             ¦   «         S ©Nr   r(   )r,   r²   r·   rÎ   Ú
contiguous)r�   r.   Úseq_lenrÔ   s       r"   Ú_shapez#ProphetNetNgramSelfAttention._shape-  s=   € Ø�{Š{˜: w°Ô0CÀTÄ]ÑSÔS×]Ò]Ð^_ÐabÑcÔc×nÒnÑpÔpÐpr#   c                 ó   — d| _         d S )NT)r!   ©r�   s    r"   Úprepare_for_onnx_export_z5ProphetNetNgramSelfAttention.prepare_for_onnx_export_0  s   € ØˆŒˆˆr#   rc   c                 óØ
  ‡)‡*— |                      ¦   «         \  }	}
}t          |                      ¦   «         ¦  «        |	|
|gk    sJ d|	|
|f› d|j        › �¦   «         ‚|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|| j        dz  z  }|                      ||
|	¦  «        }|                      |d|	¦  «        }|                      |d|	¦  «        }|	| j        d| j        f} |j	        |Ž } |j	        |Ž } |j	        |Ž }| 
                    d| j        z   d¬¦  «        }| 
                    d| j        z   d¬¦  «        }| 
                    d| j        z   d¬¦  «        }| 
                    d| j        z   d¬¦  «        }|d         |dd …         }}|d         |dd …         }}|d         |dd …         cŠ)}|d         |dd …         cŠ*}|�>t          |t          ¦  «        r|j        }n|}|                     ‰)‰*| j        ¦  «        \  Š)Š*|
d| j        z   z  }t#          j        d	|‰)                     dd
¦  «        ¦  «        }|                      ||||¦  «        }||z   }|�||z   }t+          |d| j        ¬¦  «                             |¦  «        }t0          j                             || j        | j        ¬¦  «        }t#          j        d	|‰*¦  «        }|                     dd¦  «         	                    |	d||¦  «        }|                      |¦  «        }t#          j        |d¦  «                             |	| j        | j        || j        ¦  «        } t#          j        ˆ)fd„|D ¦   «         d¦  «        }!t#          j        |d¬¦  «        }"t#          j         ˆ*fd„|D ¦   «         d¦  «        }#t#          j        d| |!f¦  «        }$|  !                    |"|$||¦  «        }%|$|%z   }$|�8| "                    dddd
d¦  «        }| #                    |$j$        ¦  «        }|$|z   }$t+          |$d| j        ¬¦  «                             |$¦  «        }&t0          j                             |&| j        | j        ¬¦  «        }&t#          j        d|&|#                     dd¦  «        f¦  «        }'|'                     dd
¦  «        }'|' 	                    |	| j        ||¦  «        }'|                      |'¦  «        }'t#          j         ||'gd¦  «                             |	d|¦  «        }(|                     |	| j        |d¦  «        }t0          j                             |(| j        | j        ¬¦  «        }(|(||&fS )Nz#`hidden_states` should be of shape rÄ   rÀ   rS   r   r   r(   r   zbntc,bncs->bntsr   )r   r!   rÁ   c                 ó>   •— g | ]}t          j        ‰|gd ¦  «        ‘ŒS ©r(   )r   r1   )Ú.0ÚkeyÚmain_key_statess     €r"   ú
<listcomp>z8ProphetNetNgramSelfAttention.forward.<locals>.<listcomp>—  s+   ø€ Ð)rÐ)rÐ)rÐSV­%¬)°_ÀcÐ4JÈAÑ*NÔ*NÐ)rÐ)rÐ)rr#   c                 ód   •— g | ],}t          j        ‰|gd ¦  «                             d ¦  «        ‘Œ-S rþ   )r   r1   rT   )rÿ   Úv_pÚmain_value_statess     €r"   r  z8ProphetNetNgramSelfAttention.forward.<locals>.<listcomp>ž  s9   ø€ ÐfÐfÐfÀS�UŒYÐ)¨3Ð/°Ñ3Ô3×=Ò=¸aÑ@Ô@ÐfÐfÐfr#   zbnhtc,bnhsc->bnhtsé   zbnhts,bnhsc->bnhtc)%rV   rÅ   rˆ   r»   r¹   rº   r·   rø   r²   rÒ   Úchunkr3   rÆ   r   rÊ   rÏ   r³   r   rÐ   rÎ   Ú get_main_relative_pos_embeddingsr   r!   r¥   r   r   r¶   rµ   rÃ   r¼   Ústackr,   r1   Ú#get_predict_relative_pos_embeddingsÚpermuteÚtor   )+r�   ry   rc   r©   Úextended_predict_attention_maskrZ   r[   rW   rÓ   rÔ   Úngram_sequence_lengthrœ   r×   rÚ   rÛ   Ú
proj_shapeÚhidden_states_listÚquery_states_listÚkey_states_listÚvalue_states_listÚmain_hidden_statesÚhidden_states_predict_listÚmain_query_statesÚpredict_query_states_listÚpredict_key_states_listÚpredict_value_states_listrØ   r2   Úmain_attn_weightsÚmain_relative_pos_embeddingsÚmain_attn_probsÚmain_attn_outputÚpredict_query_statesÚpredict_key_statesÚpredict_hidden_statesÚpredict_value_statesÚpredict_attn_weightsÚpredict_relative_pos_embeddingsÚpredict_attn_probsÚpredict_attn_outputrá   r  r  s+                                            @@r"   r§   z$ProphetNetNgramSelfAttention.forward3  sU  øø€ ð :G×9KÒ9KÑ9MÔ9MÑ6ˆ
Ð)¨;Ý�M×&Ò&Ñ(Ô(Ñ)Ô)¨jÐ:OÐQ\Ð-]Ò]Ð]Ð]ð&°*Ð>SÐU`Ð1að &ð &ØÔ#ð&ð &ñ ^Ô]Ð]ð —’ }Ñ5Ô5ˆØ—]’] =Ñ1Ô1ˆ
Ø—’ }Ñ5Ô5ˆð $ t¤}°cÑ'9Ñ:ˆð —{’{ <Ð1FÈ
ÑSÔSˆØ—[’[ ¨R°Ñ<Ô<ˆ
Ø—{’{ <°°ZÑ@Ô@ˆØ  $Ô"5°r¸4¼=ÐIˆ
à+�|Ô+¨ZÐ8ˆØ'�ZÔ'¨Ð4ˆ
Ø+�|Ô+¨ZÐ8ˆð +×0Ò0°°T´Z±ÀQÐ0ÑGÔGÐØ(×.Ò.¨q°4´:©~À1Ð.ÑEÔEÐØ$×*Ò*¨1¨t¬z©>¸qÐ*ÑAÔAˆØ(×.Ò.¨q°4´:©~À1Ð.ÑEÔEÐà9KÈAÔ9NÐPbÐcdÐceÐceÔPfÐ6ÐØ7HÈÔ7KÐM^Ð_`Ð_aÐ_aÔMbÐ4ÐØ3BÀ1Ô3EÀÐWXÐWYÐWYÔGZÐ0ˆÐ0Ø7HÈÔ7KÐM^Ð_`Ð_aÐ_aÔMbÐ4ÐÐ4ð Ð&Ý˜/Õ+>Ñ?Ô?ð 7Ø'6Ô'KÐ$Ð$à'6Ð$Ø1E×1LÒ1LØÐ!2°D´Nñ2ô 2Ñ.ˆOÐ.ð
 0°A¸¼
±NÑCˆõ "œLÐ):Ð<MÈ×OhÒOhÐijÐlmÑOnÔOnÑoÔoÐð (,×'LÒ'LØÐ 1°<ÐA_ñ(
ô (
Ð$ð .Ð0LÑLÐàÐ%Ø 1°NÑ BÐå!ØØØ”ð
ñ 
ô 
÷ Š'Ð#Ñ
$Ô
$ð	 	õ œ-×/Ò/°À4ÔCYÐdhÔdqÐ/ÑrÔrˆõ
 !œ<Ð(9¸?ÐL]Ñ^Ô^Ðà+×5Ò5°a¸Ñ;Ô;×CÒCÀJÐPQÐSbÐdoÑpÔpÐØŸ=š=Ð)9Ñ:Ô:Ðõ  %œ{Ð+DÀaÑHÔH×MÒMØ˜œ
 DÔ$7¸È$Ì-ñ 
ô  
Ðõ
 #œ[Ð)rÐ)rÐ)rÐ)rÐZqÐ)rÑ)rÔ)rÐtuÑvÔvÐõ !&¤Ð,FÈAÐ NÑ NÔ NÐõ  %œyØfÐfÐfÐfÐLeÐfÑfÔfÐhiñ 
ô  
Ðõ  %œ|Ð,@ÐCWÐYkÐBlÑmÔmÐð +/×*RÒ*RØ!Ð#7¸ÐGhñ+
ô +
Ð'ð
  4Ð6UÑUÐà*Ð6à.M×.UÒ.UÐVWÐYZÐ\]Ð_`ÐbcÑ.dÔ.dÐ+Ø.M×.PÒ.PÐQeÔQkÑ.lÔ.lÐ+Ø#7Ð:YÑ#YÐ å$Ø ØØ”ð
ñ 
ô 
÷ Š'Ð&Ñ
'Ô
'ð	 	õ  œ]×2Ò2Ø $Ô"8À4Ä=ð 3ñ 
ô 
Ðõ $œlØ Ð#5Ð7K×7UÒ7UÐVWÐYZÑ7[Ô7[Ð"\ñ
ô 
Ðð 2×;Ò;¸A¸qÑAÔAÐØ1×9Ò9¸*ÀdÄjÐRaÐcnÑoÔoÐØ"ŸmšmÐ,?Ñ@Ô@Ðõ ”iÐ!1Ð3FÐ GÈÑKÔK×PÒPÐQ[Ð]_ÐalÑmÔmˆà)×.Ò.¨z¸4Ô;NÐP_ÐacÑdÔdˆå”m×+Ò+¨K¸4¼<ÐRVÔR_Ð+Ñ`Ô`ˆà˜OÐ-?Ð?Ð?r#   c                 óL  — |j         \  }}}}|                     ||||¦  «        }|€Ñ|j         d d…         \  }}	t          j        d|j         d         dz   ¦  «                             d¦  «                             d¦  «                             ||	d¦  «                             |j        ¦  «        }
|
|                     d¦  «                             ||	d¦  «        z
  }
t          | j	        | j
        |
d¦  «        }|                      |¦  «        }|                     |j         d d…         | j	        | j        fz   ¦  «        }|                     dddd¦  «        }|                     |j         d d…         dz   ¦  «        }|                     d| j        d¦  «        }|                     d|j         d         ¦  «        }|                     ¦   «         }|                     d|                     d¦  «        ¦  «        }t          j        |d|¬¦  «        }|                     |||d¦  «        }|S )	Nr(   r   rS   r   Fr   )rS   ©r   Úindex)rˆ   r,   r   r+   rT   rU   r  r%   rQ   rH   rð   rò   r²   r  rÒ   r£   rV   Úgather)r�   ry   rÝ   rW   rZ   rÔ   r²   rÕ   rÜ   r2   rJ   Úrel_pos_embeddingsr  s                r"   r  z=ProphetNetNgramSelfAttention.get_main_relative_pos_embeddingsÖ  s9  € ð 8DÔ7IÑ4ˆ
�N G¨WØ#×(Ò(¨°^ÀWÈgÑVÔVˆØ)Ð1Ø*7Ô*=¸b¸q¸bÔ*AÑ'ˆJ˜å”˜Q Ô 2°2Ô 6¸Ñ :Ñ;Ô;ß’˜1‘”ß’˜1‘”ß’˜
 O°QÑ7Ô7ß’�LÔ'Ñ(Ô(ð ð "4°l×6LÒ6LÈQÑ6OÔ6O×6VÒ6VÐWaÐcrÐtuÑ6vÔ6vÑ!vÐÝ-EØÔ  $Ô"<Ð>PÐRWñ.ô .Ð*ð
 "×9Ò9¸-ÑHÔHÐØ/×4Ò4ØÔ$ R a RÔ(¨DÔ,<¸dÔ>QÐ+RÑRñ
ô 
Ðð 0×7Ò7¸¸1¸aÀÑCÔCÐà/×7Ò7¸Ô8JÈ2ÈAÈ2Ô8NÐQVÑ8VÑWÔWÐà)G×)NÒ)NÈqÐRVÔReÐghÑ)iÔ)iÐ&à)G×)LÒ)LØÐ.Ô4°RÔ8ñ*
ô *
Ð&ð *H×)LÒ)LÑ)NÔ)NÐ&à/×7Ò7¸Ð<N×<SÒ<SÐTVÑ<WÔ<WÑXÔXÐå',¤|Ð4FÈAÐUsÐ'tÑ'tÔ'tÐ$Ø'C×'HÒ'HÈÐUcÐelÐnpÑ'qÔ'qÐ$Ø+Ð+r#   c                 óh  — |j         dd…         \  }}|€á|j         d         }t          |d         d         |dz
  k    d¦  «         t          j        d|¦  «                             d¦  «                             d¦  «                             ||d¦  «                             |j        ¦  «        }||                     d¦  «                             ||d¦  «        z
  }t          | j	        | j
        |d¦  «        }|                     dd¦  «        }|                      |¦  «        }	|	                     |j         d d…         | j	        | j        fz   ¦  «        }	|	                     ddddd¦  «        }	|	                     d| j	        ¦  «        }	|                     d¦  «        }|                     | j        d| j        d¦  «        }|                     d|                     d¦  «        ¦  «                             ¦   «         }t          j        |	d|¬	¦  «        }
|
                     || j        | j        |d¦  «        }
|
S )
Nr   r(   rS   r   zb`position_ids` are incorrect. They should be of the format 1 2 3 4 5 ... (key_sequence_length - 1)Fr  r   r'  )rˆ   r   r   r+   rT   rU   r  r%   rQ   rH   rð   rÎ   rò   r,   r²   r  rÒ   r3   rV   r£   r)  )r�   ry   rÝ   rW   r[   rÔ   r2   Úkey_sequence_lengthrJ   r*  r#  s              r"   r
  z@ProphetNetNgramSelfAttention.get_predict_relative_pos_embeddings  sU  € ð '4Ô&9¸!¸A¸#Ô&>Ñ#ˆ
�Oà,Ð4Ø".Ô"4°RÔ"8ÐÝ"Ø˜Q” Ô"Ð&9¸AÑ&=Ò=Øtñô ð õ
 ”˜QÐ 3Ñ4Ô4ß’˜1‘”ß’˜1‘”ß’˜
 O°QÑ7Ô7ß’�LÔ'Ñ(Ô(ð ð "4°l×6LÒ6LÈQÑ6OÔ6O×6VÒ6VÐWaÐcrÐtuÑ6vÔ6vÑ!vÐÝ0HØÔ  $Ô"<Ð>PÐRWñ1ô 1Ð-ð
 &×/Ò/°°1Ñ5Ô5ˆØ!×9Ò9¸-ÑHÔHÐð 0×4Ò4ØÔ   Ô$¨Ô(8¸$Ô:MÐ'NÑNñ
ô 
Ðð 0×7Ò7¸¸1¸aÀÀAÑFÔFÐà/×7Ò7¸¸DÔ<LÑMÔMÐà,M×,WÒ,WÐXYÑ,ZÔ,ZÐ)Ø,M×,TÒ,TØŒJ˜˜4Ô.°ñ-
ô -
Ð)ð -N×,RÒ,RØÐ1×6Ò6°rÑ:Ô:ñ-
ô -
ç
Š$‰&Œ&ð 	*õ +0¬,Ø AÐ-Nð+
ñ +
ô +
Ð'ð
 +J×*NÒ*NØ˜œ
 DÔ$7¸È"ñ+
ô +
Ð'ð /Ð.r#   r—   ©NNNNNN)rl   rm   rn   r   r›   rø   rû   r	   r§   r  r
  r®   r¯   s   @r"   rî   rî     sÒ   ø€ € € € € ð ð  Ð/ð  ð  ð  ð  ð  ð  ð<qð qð qðð ð ð )-ØØ(,Ø'+Ø*.Øða@ð a@ð  ™ða@ð a@ð a@ð a@ðF+,ð +,ð +,ðZ:/ð :/ð :/ð :/ð :/ð :/ð :/r#   rî   c                   ó8   ‡ — e Zd ZdZdefˆ fd„Z	 ddefd„Zˆ xZS )ÚProphetNetEncoderLayerz&
    Encoder block for Prophetnet
    r�   c                 ó  •— t          ¦   «                              ¦   «          t          ||j        ¦  «        | _        t          |j        ¦  «        | _        t          ||j	        ¦  «        | _
        t          |j        ¦  «        | _        d S r—   )rš   r›   r±   Únum_encoder_attention_headsÚ	self_attnr   rœ   Úself_attn_layer_normrä   Úencoder_ffn_dimÚfeed_forwardÚfeed_forward_layer_normr�   s     €r"   r›   zProphetNetEncoderLayer.__init__E  sp   ø€ Ý‰Œ×ÒÑÔÐå,¨V°VÔ5WÑXÔXˆŒÝ$-¨fÔ.@Ñ$AÔ$AˆÔ!õ 2°&¸&Ô:PÑQÔQˆÔÝ'0°Ô1CÑ'DÔ'DˆÔ$Ð$Ð$r#   Fr¾   c                 óÜ   — |                       |||¬¦  «        \  }}|                      ||z   ¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|f}|r||fz  }|S )N)ry   r©   r¾   )r2  r3  r5  r6  )r�   ry   r©   r¾   Úattention_outputrÝ   Úfeed_forward_outputÚoutputss           r"   r§   zProphetNetEncoderLayer.forwardO  s–   € ð *.¯ªØ'Ø)Ø/ð *8ñ *
ô *
Ñ&Ð˜,ð
 ×1Ò1Ð2BÀ]Ñ2RÑSÔSˆð #×/Ò/°Ñ>Ô>ÐØ×4Ò4Ð5HÈ=Ñ5XÑYÔYˆà Ð"ˆàð 	'Ø˜�Ñ&ˆGàˆr#   ©F©	rl   rm   rn   ro   r   r›   râ   r§   r®   r¯   s   @r"   r/  r/  @  s~   ø€ € € € € ðð ðEÐ/ð Eð Eð Eð Eð Eð Eð #(ð	ð ð  ð	ð ð ð ð ð ð ð r#   r/  c                   ó\   ‡ — e Zd ZdZd
defˆ fd„Z	 	 	 	 	 	 	 	 	 	 ddedz  dedz  fd	„Zˆ xZS )ÚProphetNetDecoderLayerz&
    Decoder block for Prophetnet
    Nr�   c                 ó„  •— t          ¦   «                              ¦   «          t          ||¬¦  «        | _        t	          |j        ¦  «        | _        |j        r5t          ||j	        |¬¦  «        | _
        t	          |j        ¦  «        | _        t          ||j        ¦  «        | _        t	          |j        ¦  «        | _        d S )N©r³   )rš   r›   rî   r2  r   rœ   r3  Úadd_cross_attentionr±   rñ   Ú
cross_attnÚcross_attn_layer_normrä   Údecoder_ffn_dimr5  r6  ró   s      €r"   r›   zProphetNetDecoderLayer.__init__n  s«   ø€ Ý‰Œ×ÒÑÔÐå5°fÈ	ÐRÑRÔRˆŒÝ$-¨fÔ.@Ñ$AÔ$AˆÔ!ð Ô%ð 	GÝ1°&¸&Ô:\ÐhqÐrÑrÔrˆDŒOÝ)2°6Ô3EÑ)FÔ)FˆDÔ&õ 2°&¸&Ô:PÑQÔQˆÔÝ'0°Ô1CÑ'DÔ'DˆÔ$Ð$Ð$r#   TFÚ	use_cacher¾   c           	      ó\  — |                       ||	|||||¬¦  «        \  }}}|                      ||z   ¦  «        }d }|�5|                      ||||	|¬¦  «        \  }}|                      ||z   ¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|f}|r||||fz  }|S )N)ry   rc   r©   r  rZ   r[   rW   )ry   r½   r©   rc   r¾   )r2  r3  rB  rC  r5  r6  )r�   ry   r©   rj   Úencoder_attn_maskr  rZ   r[   rW   rc   rE  r¾   rÓ   Úngram_attention_outputÚself_attn_weightsÚself_attn_weights_ngramÚcross_attn_weightsr8  r9  r:  s                       r"   r§   zProphetNetDecoderLayer.forward}  s  € ð  NRÏ^Ê^Ø'Ø+Ø)Ø,KØ+IØ.OØ%ð N\ñ N
ô N
ÑJÐÐ 1Ð3Jð ×1Ò1°-ÐBXÑ2XÑYÔYˆà!ÐØ Ð,à37·?²?Ø+Ø!6Ø0Ø /Ø"3ð 4Cñ 4ô 4Ñ0ÐÐ0ð !×6Ò6Ð7GÈ-Ñ7WÑXÔXˆMð #×/Ò/°Ñ>Ô>ÐØ×4Ò4Ð5HÈ=Ñ5XÑYÔYˆà Ð"ˆàð 	XØÐ)Ð+BÐDVÐWÑWˆGàˆr#   r—   )
NNNNNNNNTFr<  r¯   s   @r"   r>  r>  i  s±   ø€ € € € € ðð ðEð EÐ/ð Eð Eð Eð Eð Eð Eð$ Ø"ØØ(,Ø'+Ø*.ØØØ!%Ø).ð0ð 0ð ˜$‘;ð0ð   $™;ð0ð 0ð 0ð 0ð 0ð 0ð 0ð 0r#   r>  z=
    The standalone encoder part of the ProphetNetModel.
    c                   óº   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 ddej	        dz  dej	        dz  dej	        dz  d	e
dz  d
e
dz  de
dz  deez  fd„¦   «         Zˆ xZS )ÚProphetNetEncoderr�   c                 ó   •‡— t          ¦   «                              ‰¦  «         t          j        ‰j        ‰j        ‰j        ¬¦  «        | _        t          ‰¦  «        | _	        t          ‰j        ¦  «        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        |                      ¦   «          d S )N©r    c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r'   )r/  )rÿ   Ú_r�   s     €r"   r  z.ProphetNetEncoder.__init__.<locals>.<listcomp>½  s"   ø€ Ð$nÐ$nÐ$nÈÕ%;¸FÑ%CÔ%CÐ$nÐ$nÐ$nr#   F)rš   r›   r   Ú	EmbeddingÚ
vocab_sizerœ   r†   Úword_embeddingsr”   Úposition_embeddingsr   Úembeddings_layer_normÚ
ModuleListÚrangeÚnum_encoder_layersrË   Úgradient_checkpointingÚ	post_initr�   s    `€r"   r›   zProphetNetEncoder.__init__¶  s²   øø€ Ý‰Œ×Ò˜Ñ Ô Ð å!œ|¨FÔ,=¸vÔ?QÐ_eÔ_rÐsÑsÔsˆÔÝ#AÀ&Ñ#IÔ#IˆÔ Ý%.¨vÔ/AÑ%BÔ%BˆÔ"å”mÐ$nÐ$nÐ$nÐ$nÍUÐSYÔSlÑMmÔMmÐ$nÑ$nÔ$nÑoÔoˆŒà&+ˆÔ#à�ŠÑÔÐÐÐr#   c                 ó   — | j         S r—   ©rT  rú   s    r"   Úget_input_embeddingsz&ProphetNetEncoder.get_input_embeddingsÃ  ó   € ØÔ#Ð#r#   c                 ó   — || _         d S r—   r]  ©r�   Úvalues     r"   Úset_input_embeddingsz&ProphetNetEncoder.set_input_embeddingsÆ  ó   € Ø$ˆÔÐÐr#   NrŽ   r©   Úinputs_embedsr¾   Úoutput_hidden_statesÚreturn_dictr•   c                 óÂ  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|€|€t	          d¦  «        ‚|�|�t	          d¦  «        ‚|�|€|                      |¦  «        }|�md|dd…dddd…f                              d| j         j        dd¦  «        z
  t          j	        | j
        ¦  «        j        z  }|                     |j
        ¦  «        }nd}|                      |j        dd…         |j        ¦  «        \  }	}
||	z   }|                      |¦  «        }t"          j                             || j         j        | j        ¬¦  «        }|rdnd}|rdnd}t+          | j        ¦  «        D ]1\  }}|r||fz   } ||||¬	¦  «        }|d
         }|r||d         fz   }Œ2|r||fz   }|st/          d„ |||fD ¦   «         ¦  «        S t1          |||¬¦  «        S )a	  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, ProphetNetEncoder
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased")
        >>> model = ProphetNetEncoder.from_pretrained("patrickvonplaten/prophetnet-large-uncased-standalone")
        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
        >>> outputs = model(**inputs)

        >>> last_hidden_states = outputs.last_hidden_state
        ```Nz3Either input_ids or inputs_embeds has to be passed.z2Make sure to only pass input_ids or inputs_embeds.ç      ð?r   r(   rÁ   r'   )r©   r¾   r   c              3   ó   K  — | ]}|®|V — Œ	d S r—   r'   ©rÿ   Úvs     r"   ú	<genexpr>z,ProphetNetEncoder.forward.<locals>.<genexpr>  s(   è è € ÐlÐl˜qÐ^_Ð^k˜Ð^kÐ^kÐ^kÐ^kÐlÐlr#   )ru   ry   r{   )r�   r¾   rf  rg  rÑ   rT  rU   r1  r   r)   r   r*   r  rU  rˆ   r%   rV  r   r   r¶   rÃ   Ú	enumeraterË   rr   r   )r�   rŽ   r©   re  r¾   rf  rg  rÓ   Úextended_attention_maskrU  rW   ry   rj   Úall_attentionsÚidxÚencoder_layerÚlayer_outputss                    r"   r§   zProphetNetEncoder.forwardÉ  sŠ  € ð4 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàÐ Ð!6ÝÐRÑSÔSÐSØÐ" }Ð'@ÝÐQÑRÔRÐRØÐ" }Ð'<Ø ×0Ò0°Ñ;Ô;ˆMð Ð%à�n Q Q Q¨¨d°A°A°AÐ%5Ô6×=Ò=¸aÀÄÔAhÐjkÐmnÑoÔoÑoÝ”˜DœJÑ'Ô'Ô+ñ',Ð#ð '>×&@Ò&@ÀÔATÑ&UÔ&UÐ#Ð#à&*Ð#à,0×,DÒ,DÀ]ÔEXÐY[ÐZ[ÐY[ÔE\Ð^kÔ^rÑ,sÔ,sÑ)Ð˜\à%Ð(;Ñ;ˆØ×2Ò2°=ÑAÔAˆÝœ×-Ò-¨m¸t¼{Ô?RÐ]aÔ]jÐ-ÑkÔkˆà&:Ð D  ÀÐØ0Ð:˜˜°dˆå"+¨D¬KÑ"8Ô"8ð 	Fð 	FÑˆC�Ø#ð QØ(=ÀÐ@PÑ(PÐ%à)˜MØØ6Ø"3ðñ ô ˆMð *¨!Ô,ˆMà ð FØ!/°=ÀÔ3CÐ2EÑ!E�øàð 	MØ$9¸]Ð<LÑ$LÐ!àð 	mÝÐlÐl ]Ð4IÈ>Ð$ZÐlÑlÔlÑlÔlÐlÝØ+Ð;PÐ]kð
ñ 
ô 
ð 	
r#   r-  )rl   rm   rn   r   r›   r^  rc  r   r   r   râ   rr   r   r§   r®   r¯   s   @r"   rM  rM  °  s  ø€ € € € € ðÐ/ð ð ð ð ð ð ð$ð $ð $ð%ð %ð %ð ð *.Ø.2Ø-1Ø)-Ø,0Ø#'ðN
ð N
à”< $Ñ&ðN
ð œ tÑ+ðN
ð ”| dÑ*ð	N
ð
   $™;ðN
ð # T™kðN
ð ˜D‘[ðN
ð 
�Ñ	 ðN
ð N
ð N
ñ „^ðN
ð N
ð N
ð N
ð N
r#   rM  z=
    The standalone decoder part of the ProphetNetModel.
    c                   ó  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 	 ddej	        dz  dej	        dz  dej	        dz  d	ej	        dz  d
e
dz  dej	        dz  dedz  dedz  dedz  dedz  deez  fd„¦   «         Zd„ Zd„ Zd„ Zˆ xZS )ÚProphetNetDecoderr�   c                 ób  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        ‰j        | _        ‰j        | _        ‰j        | _        t          j	        ‰j
        ‰j        ‰j        ¬¦  «        | _        t          ‰¦  «        | _        t          j	        | j        ‰j        d ¦  «        | _        t          j        ˆfd„t%          ‰j        ¦  «        D ¦   «         ¦  «        | _        t+          ‰j        ¦  «        | _        d| _        |                      ¦   «          d S )NrO  c                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS )r@  )r>  )rÿ   Úir�   s     €r"   r  z.ProphetNetDecoder.__init__.<locals>.<listcomp>/  s'   ø€ ÐcÐcÐc¸QÕ# F°aÐ8Ñ8Ô8ÐcÐcÐcr#   F)rš   r›   r3   rH   rð   r¶   r˜   Úmax_target_positionsr   rR  rS  rœ   r†   rT  r”   rU  Úngram_embeddingsrW  rX  Únum_decoder_layersrË   r   rV  rZ  r[  r�   s    `€r"   r›   zProphetNetDecoder.__init__!  s  øø€ Ý‰Œ×Ò˜Ñ Ô Ð à”\ˆŒ
Ø!Ô-ˆÔØ%+Ô%AˆÔ"Ø”~ˆŒØ$*Ô$BˆÔ!å!œ|¨FÔ,=¸vÔ?QÐ_eÔ_rÐsÑsÔsˆÔÝ#AÀ&Ñ#IÔ#IˆÔ å "¤¨T¬Z¸Ô9KÈTÑ RÔ RˆÔÝ”mØcÐcÐcÐcÅ%ÈÔHaÑBbÔBbÐcÑcÔcñ
ô 
ˆŒõ &/¨vÔ/AÑ%BÔ%BˆÔ"à&+ˆÔ#à�ŠÑÔÐÐÐr#   c                 ó   — | j         S r—   r]  rú   s    r"   r^  z&ProphetNetDecoder.get_input_embeddings7  r_  r#   c                 ó   — || _         d S r—   r]  ra  s     r"   rc  z&ProphetNetDecoder.set_input_embeddings:  rd  r#   NrŽ   r©   rj   Úencoder_attention_maskrc   re  rE  r¾   rf  rg  r•   c                 ó"	  ‡!‡"‡#— |�|n| j         j        }|�|n| j         j        }|	�|	n| j         j        }	|
�|
n| j         j        }
|€|€t          d¦  «        ‚|�|�t          d¦  «        ‚|�|€|                      |¦  «        }|j        dd…         \  Š!}| j        r%| j	        r|rt                               d¦  «         d}|r[|€Y|€| j         j        r6t          t          | j         ¬¦  «        t          | j         ¬¦  «        ¦  «        nt          | j         ¬¦  «        }|�|                     ¦   «         nd}|                      ‰!|f|j        |¬	¦  «        \  }}|dk    rd
\  }}n|                      |¦  «        \  }}| j                             |dz   ¦  «        Š#||z   }| j        j        Š"|dk    rJ|                     d¦  «        dk    s
J d¦   «         ‚ˆ!ˆ"ˆ#fd„t/          | j        ¦  «        D ¦   «         }d}d}nMˆ"ˆ#fd„t/          | j        ¦  «        D ¦   «         }|                      ||¦  «        }|                      ||¦  «        }|�md|dd…dddd…f                              d| j         j        dd¦  «        z
  t;          j        | j        ¦  «        j         z  }| !                    |j        ¦  «        }nd}t;          j"        |g|z   d¦  «        }| j#        r|  #                    |¦  «        }tH          j%         &                    || j&        | j	        ¬¦  «        }|	rdnd}|	r| j         j        dk    rdnd}|rdnd}|rdnd}|r| j         j'        rdnd}tQ          | j)        ¦  «        D ]‹\  }}|	r4||dd…d|…f         fz  }| j         j        dk    r||dd…|d…f         fz  } ||||||||||||¬¦  «        }|d         }|r0||d         fz  }||d         fz  }| j         j'        r||d         fz  }ŒŒ|	r4||dd…d|…f         fz  }| j         j        dk    r||dd…|d…f         fz  }|dd…d|…f         }| j         j        dk    r|dd…|d…f         nd} |
s!tU          d„ || ||||||fD ¦   «         ¦  «        S tW          || ||||||¬¦  «        S )a  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, ProphetNetDecoder
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased")
        >>> model = ProphetNetDecoder.from_pretrained("microsoft/prophetnet-large-uncased", add_cross_attention=False)
        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
        >>> outputs = model(**inputs)

        >>> last_hidden_states = outputs.last_hidden_state
        ```NzGEither `decoder_input_ids` or `decoder_inputs_embeds` has to be passed.zFMake sure to only pass `decoder_input_ids` or `decoder_inputs_embeds`.r(   zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...F)r�   r   )r%   rc   )NNr   zOAt the moment `use_cache` is only supported for `decoder_input_ids` of length 1c                 óV   •— g | ]%}‰|d z
           ‰z                         ‰d d ¦  «        ‘Œ&S ©r   )rU   )rÿ   r3   rÔ   rz  Úpredicting_stream_pos_embeds     €€€r"   r  z-ProphetNetDecoder.forward.<locals>.<listcomp>”  sM   ø€ ð #ð #ð #àð " %¨!¡)Ô,Ð/JÑJ×RÒRÐS]Ð_`ÐbcÑdÔdð#ð #ð #r#   c                 ó,   •— g | ]}‰|d z
           ‰z   ‘ŒS r�  r'   )rÿ   r3   rz  r‚  s     €€r"   r  z-ProphetNetDecoder.forward.<locals>.<listcomp>›  s6   ø€ ð #ð #ð #ØPUÐ! %¨!¡)Ô,Ð/JÑJð#ð #ð #r#   ri  rÁ   r'   )rG  r  rZ   r[   rW   rc   rE  r¾   r   c              3   ó   K  — | ]}|®|V — Œ	d S r—   r'   rk  s     r"   rm  z,ProphetNetDecoder.forward.<locals>.<genexpr>à  s4   è è € ð ð àð �=ð ð !�=�=�=ðð r#   )ru   rv   rc   ry   rz   r{   r|   rh   ),r�   rE  r¾   rf  rg  rÑ   rT  rˆ   rZ  rÃ   ÚloggerÚwarning_onceÚis_encoder_decoderr   r
   r¡   rU  r%   Ú!compute_buffered_relative_bucketsr­   rz  ÚweightrV   rX  r3   Úprepare_attention_maskÚprepare_predict_attention_maskrU   rñ   r   r)   r   r*   r  r1   rV  r   r   r¶   rA  rn  rË   rr   rx   )$r�   rŽ   r©   rj   r~  rc   re  rE  r¾   rf  rg  rÓ   r2   Úpast_key_values_lengthÚmain_stream_pos_embedrW   rZ   r[   ry   Úngram_hidden_statesro  r  Úextended_encoder_attention_maskÚall_main_stream_hidden_statesÚall_ngram_stream_hidden_statesÚall_main_stream_attnsÚall_ngram_stream_attnsÚall_cross_attnsrq  Údecoder_layerrs  ru   rv   rÔ   rz  r‚  s$                                    @@@r"   r§   zProphetNetDecoder.forward=  s•  øøø€ ð: "+Ð!6�I�I¸D¼KÔ<Qˆ	Ø1BÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàÐ Ð!6ÝÐfÑgÔgÐgØÐ" }Ð'@ÝÐeÑfÔfÐfØÐ" }Ð'<Ø ×0Ò0°Ñ;Ô;ˆMà&3Ô&9¸"¸1¸"Ô&=Ñ#ˆ
�OàÔ&ð 	"¨4¬=ð 	"Øð "Ý×#Ò#Øpñô ð ð "�	àð 	˜Ð0ð )Ð4¸¼Ô8VÐ4õ $¥L¸¼Ð$DÑ$DÔ$DÅlÐZ^ÔZeÐFfÑFfÔFfÑgÔgÐgå!¨¬Ð5Ñ5Ô5ð ð FUÐE` ×!?Ò!?Ñ!AÔ!AÐ!AÐfgÐà.2×.FÒ.FØ˜Ð)Ø Ô'Ø+ð /Gñ /
ô /
Ñ+Ð˜|ð " QÒ&Ð&ØPZÑMÐ*Ð,MÐ,Mð
 ×6Ò6°|ÑDÔDñØ.Ø1à&*Ô&>×&GÒ&GÈÐWXÑHXÑ&YÔ&YÐ#ð &Ð(=Ñ=ˆàÔ0Ô7Ðð " QÒ&Ð&Ø ×%Ò% aÑ(Ô(¨AÒ-Ð-Ð-Øañ .Ô-Ð-ð#ð #ð #ð #ð #ð #å" 4¤:Ñ.Ô.ð#ñ #ô #Ðð '+Ð#Ø.2Ð+Ð+ð#ð #ð #ð #ð #ÝY^Ð_cÔ_iÑYjÔYjð#ñ #ô #Ðð '+×&AÒ&AÀ-ÐQ_Ñ&`Ô&`Ð#Ø.2×.QÒ.QÐR_ÐaoÑ.pÔ.pÐ+ð "Ð-àÐ,¨Q¨Q¨Q°°d¸A¸A¸AÐ-=Ô>×EÒEÀaÈÌÔIpÐrsÐuvÑwÔwÑwÝ”˜DœJÑ'Ô'Ô+ñ/,Ð+ð /N×.PÒ.PÐQ^ÔQdÑ.eÔ.eÐ+Ð+à.2Ð+åœ	 = /Ð4GÑ"GÈÑKÔKˆàÔ%ð 	FØ ×6Ò6°}ÑEÔEˆMåœ×-Ò-¨m¸t¼|ÐVZÔVcÐ-ÑdÔdˆð /CÐ(L¨¨ÈÐ%Ø/CÐ)gÈÌÔHYÐ\]ÒH]ÐH]¨¨ÐcgÐ&à&7Ð A  ¸TÐØ'8Ð!B  ¸dÐØ 1Ð_°d´kÔ6UÐ_˜"˜"Ð[_ˆå"+¨D¬KÑ"8Ô"8ð 	;ð 	;ÑˆC�Ø#ð \à-°-ÀÀÀÐCSÀOÐCSÐ@SÔ2TÐ1VÑVÐ-Ø”;Ô$ qÒ(Ð(Ø2°}ÀQÀQÀQÈÐHXÐHXÐEXÔ7YÐ6[Ñ[Ð2à)˜MØØ'Ø%Ø"AØ0OØ/MØ2SØ)Ø /Ø#Ø"3ðñ ô ˆMð *¨!Ô,ˆMØ ð ;Ø%¨-¸Ô*:Ð)<Ñ<Ð%Ø&¨=¸Ô+;Ð*=Ñ=Ð&à”;Ô2ð ;Ø#¨°aÔ(8Ð':Ñ:�Oøàð 	XØ)¨m¸A¸A¸AÐ?OÀÐ?OÐ<OÔ.PÐ-RÑRÐ)ØŒ{Ô  1Ò$Ð$Ø.°=ÀÀÀÀOÐDTÐDTÐATÔ3UÐ2WÑWÐ.ð *¨!¨!¨!Ð-=¨oÐ-=Ð*=Ô>ÐØHLÌÔHYÐ\]ÒH]ÐH] -°°°°?Ð3CÐ3CÐ0CÔ"DÐ"DÐcgÐàð 	Ýð ð ð &Ø+Ø#Ø1Ø2Ø)Ø*Ø#ð	ðñ ô ñ ô ð õ ,Ø/Ø$;Ø+Ø7Ø >Ø,Ø3Ø,ð	
ñ 	
ô 	
ð 		
r#   c           	      óà  — |j         \  }}t          j        d| j        ¦  «                             |j        ¦  «                             dd¦  «        }t          | j        | j	        |¦  «        \  }}|d d …d |…d |…f                              |dd¦  «        }t          j
        |d d …d |…d |…f         |d d …d |…| j        | j        |z   …f         gd¦  «                             |dd¦  «        }||fS rõ   )rˆ   r   r+   ry  r  r%   rU   r\   rH   rð   r1   )r�   rW   rÔ   r2   Úmain_relative_bucketsÚpredict_relative_bucketss         r"   rˆ  z3ProphetNetDecoder.compute_buffered_relative_bucketsù  s-  € Ø&2Ô&8Ñ#ˆ
�Oå”| A tÔ'@ÑAÔA×DÒDÀ\ÔEXÑYÔY×`Ò`ÐabÐdeÑfÔfˆÝ:]ØÔ˜dÔ8¸,ñ;
ô ;
Ñ7ÐÐ7ð
 !6°a°a°aÐ9I¸/Ð9IÐK[ÈOÐK[Ð6[Ô \× cÒ cÐdnÐpqÐstÑ uÔ uÐÝ#(¤9à(¨¨¨Ð,<¨_Ð,<Ð>N¸Ð>NÐ)NÔOØ(Ø�A�AÐ'˜Ð'¨Ô)BÀTÔE^ÐapÑEpÐ)pÐpôðð ñ$
ô $
÷ Š&�˜Q Ñ
"Ô
"ð 	!ð %Ð&>Ð>Ð>r#   c                 ó  — |j         d d…         \  }}t          j        ||ft          j        |j        ¦  «        j        |j        |j        ¬¦  «        }t          j        |d¦  «        }|d |…d |…f         d d d d …d d …f                              || j	        j
        f|j         z   ¦  «        }|�8d|d d …d d d d …f         z
  t          j        | j        ¦  «        j        z  }||z   }n|}|                     |j        ¦  «        S )Nr(   r&   r   ri  )rˆ   r   Úfullr)   r   r*   r%   Útriur-   r�   rñ   r  )r�   ry   r©   rÔ   Ú
seq_lengthÚcausal_maskÚextended_causal_maskro  s           r"   rŠ  z(ProphetNetDecoder.prepare_attention_mask  s0  € Ø!.Ô!4°R°a°RÔ!8Ñˆ
�Jõ ”jØ˜Ð$ÝŒK˜Ô+Ñ,Ô,Ô0ØÔ%Ø Ô'ð	
ñ 
ô 
ˆõ ”j ¨aÑ0Ô0ˆà*¨;¨J¨;¸¸¸Ð+CÔDÀTÈ4ÐQRÐQRÐQRÐTUÐTUÐTUÐEUÔV×]Ò]Ø˜œÔ@ÐAÀKÔDUÑUñ 
ô  
Ðð
 Ð%Ø'*¨^¸A¸A¸A¸tÀTÈ1È1È1Ð<LÔ-MÑ'MÕQVÔQ\Ð]aÔ]gÑQhÔQhÔQlÑ&lÐ#Ø&:Ð=TÑ&TÐ#Ð#à&:Ð#Ø&×)Ò)¨-Ô*=Ñ>Ô>Ð>r#   c           	      óÀ  — |j         d d…         \  }}t          | j        | j        |j        |j        ¦  «        }t          j        |d d …d |…d |…f         |d d …d |…| j        | j        |z   …f         gd¬¦  «        }|d d d d …d d …d d …f                              || j	        j
        f|j         z   ¦  «        }|�Œd|d d …d d d d d …f         z
  t          j        | j        ¦  «        j        z  }|                     || j	        j
        | j        ||f¦  «        }t          j        |t          j        |¦  «        gd¬¦  «        }||z   }n|}|                     |j        ¦  «        S )Nr(   rS   r   ri  )rˆ   r>   ry  r3   r%   r   r   r1   r-   r�   rñ   r)   r*   rA   r  )	r�   ry   r©   rÔ   rœ  Úpredict_causal_maskÚextended_predict_causal_maskro  r  s	            r"   r‹  z0ProphetNetDecoder.prepare_predict_attention_mask'  sÃ  € Ø!.Ô!4°R°a°RÔ!8Ñˆ
�Jõ 3ØÔ% t¤z°=Ô3GÈÔI\ñ
ô 
Ðõ $œià# A A A {¨
 {°K°Z°KÐ$?Ô@Ø#Ø�A�A�{˜
�{ DÔ$=ÀÔ@YÐ\fÑ@fÐ$fÐfôðð ð
ñ 
ô 
Ðð (;¸4ÀÀqÀqÀqÈ!È!È!ÈQÈQÈQÐ;NÔ'O×'VÒ'VØ˜œÔ@ÐAÐDWÔD]Ñ]ñ(
ô (
Ð$ð
 Ð%Ø'*¨^¸A¸A¸A¸tÀTÈ4ÐQRÐQRÐQRÐ<RÔ-SÑ'SÕW\ÔWbÐcgÔcmÑWnÔWnÔWrÑ&rÐ#Ø&=×&DÒ&DØ˜Tœ[ÔDÀdÄjÐR\Ð^hÐiñ'ô 'Ð#õ ',¤iØ(­%Ô*:Ð;RÑ*SÔ*SÐTÐZ\ð'ñ 'ô 'Ð#ð /KÐMdÑ.dÐ+Ð+à.JÐ+Ø.×1Ò1°-Ô2EÑFÔFÐFr#   )
NNNNNNNNNN)rl   rm   rn   r   r›   r^  rc  r   r   r   r	   râ   rr   rx   r§   rˆ  rŠ  r‹  r®   r¯   s   @r"   ru  ru    s–  ø€ € € € € ðÐ/ð ð ð ð ð ð ð,$ð $ð $ð%ð %ð %ð ð *.Ø.2Ø59Ø6:Ø(,Ø-1Ø!%Ø)-Ø,0Ø#'ðy
ð y
à”< $Ñ&ðy
ð œ tÑ+ðy
ð  %œ|¨dÑ2ð	y
ð
 !&¤¨tÑ 3ðy
ð  ™ðy
ð ”| dÑ*ðy
ð ˜$‘;ðy
ð   $™;ðy
ð # T™kðy
ð ˜D‘[ðy
ð 
Ð-Ñ	-ðy
ð y
ð y
ñ „^ðy
ðv?ð ?ð ?ð,?ð ?ð ?ð0!Gð !Gð !Gð !Gð !Gð !Gð !Gr#   ru  c                   ó*  ‡ — e Zd ZdddœZdefˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 	 	 	 dde	j
        dz  d	e	j
        dz  d
e	j
        dz  de	j        dz  dedz  dedz  de	j
        dz  de	j
        dz  dedz  dedz  dedz  dedz  deez  fd„¦   «         Zˆ xZS )ÚProphetNetModelúword_embeddings.weight)zencoder.word_embeddings.weightúdecoder.word_embeddings.weightr�   c                 ó„  •— t          ¦   «                              |¦  «         t          j        |j        |j        |j        ¬¦  «        | _        t          j	        |¦  «        }d|_
        t          |¦  «        | _        t          j	        |¦  «        }d|_        t          |¦  «        | _        |                      ¦   «          d S )NrO  FT)rš   r›   r   rR  rS  rœ   r†   rT  ÚcopyÚdeepcopyrE  rM  ÚencoderÚ
is_decoderru  Údecoderr[  )r�   r�   Úencoder_configÚdecoder_configrž   s       €r"   r›   zProphetNetModel.__init__R  s¢   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý!œ|¨FÔ,=¸vÔ?QÐ_eÔ_rÐsÑsÔsˆÔåœ vÑ.Ô.ˆØ#(ˆÔ Ý(¨Ñ8Ô8ˆŒåœ vÑ.Ô.ˆØ$(ˆÔ!Ý(¨Ñ8Ô8ˆŒð 	�ŠÑÔÐÐÐr#   c                 ó   — | j         S r—   r]  rú   s    r"   r^  z$ProphetNetModel.get_input_embeddingsa  r_  r#   c                 óX   — || _         | j         | j        _         | j         | j        _         d S r—   )rT  r©  r«  ra  s     r"   rc  z$ProphetNetModel.set_input_embeddingsd  s*   € Ø$ˆÔØ'+Ô';ˆŒÔ$Ø'+Ô';ˆŒÔ$Ð$Ð$r#   NrŽ   r©   Údecoder_input_idsÚdecoder_attention_maskÚencoder_outputsrc   re  Údecoder_inputs_embedsrE  r¾   rf  rg  r•   c                 ó¶  — |	�|	n| j         j        }	|
�|
n| j         j        }
|�|n| j         j        }|�|n| j         j        }|€|                      ||||
||¬¦  «        }|                      |||d         ||||
||	|¬¦
  «
        }|s||z   S t          |j        |j	        |j
        |j        |j        |j        |j        |j        |j        |j        |j        ¬¦  «        S )añ  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            ProphetNet uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If
            `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
            `past_key_values`).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, ProphetNetModel

        >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased")
        >>> model = ProphetNetModel.from_pretrained("microsoft/prophetnet-large-uncased")

        >>> input_ids = tokenizer(
        ...     "Studies have been shown that owning a dog is good for you", return_tensors="pt"
        ... ).input_ids  # Batch size 1
        >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids  # Batch size 1
        >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)

        >>> last_hidden_states = outputs.last_hidden_state  # main stream hidden states
        >>> last_hidden_states_ngram = outputs.last_hidden_state_ngram  # predict hidden states
        ```N)rŽ   r©   re  r¾   rf  rg  r   )
rŽ   r©   rj   r~  rc   re  r¾   rf  rE  rg  )ru   rv   rc   rd   re   rf   rg   rh   ri   rj   rk   )r�   rE  r¾   rf  rg  r©  r«  rt   ru   rv   rc   ry   rz   r{   r|   rh   )r�   rŽ   r©   r°  r±  r²  rc   re  r³  rE  r¾   rf  rg  rÓ   Údecoder_outputss                  r"   r§   zProphetNetModel.forwardi  s8  € ðd "+Ð!6�I�I¸D¼KÔ<Qˆ	Ø1BÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàÐ"Ø"ŸlšlØ#Ø-Ø+Ø"3Ø%9Ø'ð +ñ ô ˆOð Ÿ,š,Ø'Ø1Ø"1°!Ô"4Ø#1Ø+Ø/Ø/Ø!5ØØ#ð 'ñ 
ô 
ˆð ð 	5Ø" _Ñ4Ð4Ý+Ø-Ô?Ø$3Ô$KØ+Ô;Ø"1Ô"?Ø(7Ô(KØ.Ô9Ø%4Ô%EØ,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð
ñ 
ô 
ð 	
r#   )NNNNNNNNNNNN)rl   rm   rn   Ú_tied_weights_keysr   r›   r^  rc  r   r   r   Ú
BoolTensorrr   r	   râ   rt   r§   r®   r¯   s   @r"   r£  r£  K  s£  ø€ € € € € ð +CØ*Bðð Ðð
Ð/ð ð ð ð ð ð ð$ð $ð $ð<ð <ð <ð
 ð *.Ø.2Ø15Ø:>Ø(,Ø(,Ø-1Ø59Ø!%Ø)-Ø,0Ø#'ð^
ð ^
à”< $Ñ&ð^
ð œ tÑ+ð^
ð !œ<¨$Ñ.ð	^
ð
 !&Ô 0°4Ñ 7ð^
ð  ™ð^
ð  ™ð^
ð ”| dÑ*ð^
ð  %œ|¨dÑ2ð^
ð ˜$‘;ð^
ð   $™;ð^
ð # T™kð^
ð ˜D‘[ð^
ð 
Ð-Ñ	-ð^
ð ^
ð ^
ñ „^ð^
ð ^
ð ^
ð ^
ð ^
r#   r£  zh
    The ProphetNet Model with a language modeling head. Can be used for sequence generation tasks.
    c                   ól  ‡ — e Zd ZddiZdefˆ fd„Zd„ Ze	 	 	 	 	 	 	 	 	 	 	 	 	 ddej	        dz  dej	        dz  d	ej	        dz  d
ej
        dz  dej	        dz  dedz  dej	        dz  dej	        dz  dej	        dz  dedz  dedz  dedz  dedz  deez  fd„¦   «         Zdd„Zdej	        fd„Zdˆ fd„	Zˆ xZS )Ú"ProphetNetForConditionalGenerationúlm_head.weightú!prophetnet.word_embeddings.weightr�   c                 ó  •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        |j        | _        t          j        |j	        |j
        d¬¦  «        | _        |                      ¦   «          d S )NF©Úbias)rš   r›   r£  r‚   r†   r    Údisable_ngram_lossr   r¸   rœ   rS  Úlm_headr[  r�   s     €r"   r›   z+ProphetNetForConditionalGeneration.__init__Õ  sv   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý)¨&Ñ1Ô1ˆŒØ!Ô.ˆÔØ"(Ô";ˆÔå”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒð 	�ŠÑÔÐÐÐr#   c                 ó   — | j         j        S r—   )r‚   rT  rú   s    r"   r^  z7ProphetNetForConditionalGeneration.get_input_embeddingsà  s   € ØŒÔ.Ð.r#   NrŽ   r©   r°  r±  r²  rc   re  r³  ÚlabelsrE  r¾   rf  rg  r•   c                 ó0  — |�|n| j         j        }|	�|€|€|                      |	¦  «        }|                      |||||||||
|||¬¦  «        }|�|j        n|j        dd…         \  }}|d                              || j         j        |d¦  «        }|                      |¦  «        }|dd…df         }| j         j        dk    r|dd…dd…f         nd}|                     ¦   «         s| 	                    ¦   «         }d}|	�|  
                    ||	¦  «        }|s;t          d„ ||fD ¦   «         ¦  «        }|�|f|z   |dd…         z   n||dd…         z   S t          ||||j        |j        |j        |j        |j        |j        |j        |j        |j        ¬¦  «        S )	a…  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            ProphetNet uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If
            `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
            `past_key_values`).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
            config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
            labels in `[0, ..., config.vocab_size]`

        Example:

        ```python
        >>> from transformers import AutoTokenizer, ProphetNetForConditionalGeneration

        >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased")
        >>> model = ProphetNetForConditionalGeneration.from_pretrained("microsoft/prophetnet-large-uncased")

        >>> input_ids = tokenizer(
        ...     "Studies have been shown that owning a dog is good for you", return_tensors="pt"
        ... ).input_ids  # Batch size 1
        >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids  # Batch size 1
        >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)

        >>> logits_next_token = outputs.logits  # logits to predict next token as usual
        >>> logits_ngram_next_tokens = outputs.logits_ngram  # logits to predict 2nd, 3rd, ... next tokens
        ```N)rŽ   r©   r°  r±  r²  rc   re  r³  rE  r¾   rf  rg  r(   r   rS   r   c              3   ó   K  — | ]}|®|V — Œ	d S r—   r'   rk  s     r"   rm  z=ProphetNetForConditionalGeneration.forward.<locals>.<genexpr>A  ó"   è è € ÐRÐR QÀAÀM˜qÀMÀMÀMÀMÐRÐRr#   )r`   ra   rb   rc   rd   re   rf   rg   rh   ri   rj   rk   )r�   rg  r�   r‚   rˆ   r,   r3   rÀ  Úis_contiguousrö   Ú_compute_lossrr   r_   rc   rd   re   rf   rg   rh   ri   rj   rk   )r�   rŽ   r©   r°  r±  r²  rc   re  r³  rÂ  rE  r¾   rf  rg  rÓ   r:  rÔ   r2   Úpredicting_streamsÚpredict_logitsra   rb   r`   Ú
all_logitss                           r"   r§   z*ProphetNetForConditionalGeneration.forwardã  s  € ðn &1Ð%<�k�kÀ$Ä+ÔBYˆàÐÐ"3Ð";Ð@UÐ@]à $× 1Ò 1°&Ñ 9Ô 9Ðà—/’/ØØ)Ø/Ø#9Ø+Ø+Ø'Ø"7ØØ/Ø!5Ø#ð "ñ 
ô 
ˆð (9Ð'DÐÔ#Ð#ÐJ_ÔJeÐfhÐghÐfhÔJiñ 	$ˆ
�Oð % QœZŸ_š_¨Z¸¼Ô9JÈOÐ]_Ñ`Ô`ÐØŸšÐ&8Ñ9Ô9ˆà    1 Ô%ˆØ04´Ô0AÀAÒ0EÐ0E�~ a a a¨¨¨ eÔ,Ð,È4ˆð ×#Ò#Ñ%Ô%ð 	)Ø×&Ò&Ñ(Ô(ˆFàˆØÐØ×%Ò% n°fÑ=Ô=ˆDàð 	ÝÐRÐR¨6°<Ð*@ÐRÑRÔRÑRÔRˆJØ9=Ð9I�D�7˜ZÑ'¨'°!°"°"¬+Ñ5Ð5ÈzÐ\cÐdeÐdfÐdfÔ\gÑOgÐgå,ØØØ)Ø 'Ô 7Ø&-Ô&CØ,3Ô,OØ#*Ô#=Ø)0Ô)IØ!(Ô!9Ø*1Ô*KØ&-Ô&CØ#*Ô#=ðñ ô ð r#   r„   c                 óÔ  — |                      | j        j        |                     d¦  «        |                     d¦  «        ¦  «                             |¦  «        }t          | j        j        ¦  «        D ]}|dk    r	| j        r n|||d d …d d …f<   Œ|                     dd¦  «                             ¦   «         }t          j
                             |                     d|                     d¦  «        ¦  «        dt          j        ¬¦  «        }t          j
                             ||                     d¦  «        d¬¦  «        }| j        j        dk    r–|                     dd¬	¦  «         }|                     |¦  «                             d¦  «        }	||	         }|                     ¦   «         }| j        j        |                     d¦  «        z  }
d
| j        j        z
  |z  |
|z  z   }|S ©Nr   r   rS   r   Úmean)Ú	reductiong        T)r   Úkeepdimri  ©r‡   r�   r3   rV   Úfill_rX  r¿  rÎ   rö   r   r   Úlog_softmaxr,   r   r   Únll_lossÚepsÚsumÚnerÍ  ©r�   ra   rÂ  Úignore_indexÚexpend_targetsrx  Úlprobsr`   Úsmooth_lossÚnon_masked_tokensÚeps_is              r"   rÇ  z0ProphetNetForConditionalGeneration._compute_lossS  óº  € Ø×)Ò)¨$¬+Ô*;¸V¿[º[È¹^¼^ÈVÏ[Ê[ÐYZÉ^Ì^Ñ\Ô\×bÒbÐcoÑpÔpˆå�t”{Ô(Ñ)Ô)ð 	-ð 	-ˆAØ�1Šuˆu˜Ô0ˆuØ�Ø&,ˆN˜1˜a˜a˜a   ˜7Ñ#Ð#à×!Ò! ! QÑ'Ô'×2Ò2Ñ4Ô4ˆÝ”×*Ò*Ø�KŠK˜˜FŸKšK¨™OœOÑ,Ô,ØÝ”-ð +ñ 
ô 
ˆõ Œ}×%Ò% f¨n×.AÒ.AÀ"Ñ.EÔ.EÐQWÐ%ÑXÔXˆàŒ;Œ?˜SÒ Ð Ø!Ÿ:š:¨"°d˜:Ñ;Ô;Ð;ˆKØ .× 1Ò 1°,Ñ ?Ô ?× DÒ DÀRÑ HÔ HÐØ%Ð&7Ô8ˆKØ%×*Ò*Ñ,Ô,ˆKà”K”O f§k¢k°"¡o¤oÑ5ˆEØ˜$œ+œ/Ñ)¨TÑ1°E¸KÑ4GÑGˆDàˆr#   c                 ó,   — |                       |¦  «        S r—   )r�   )r�   rÂ  s     r"   Ú%prepare_decoder_input_ids_from_labelszHProphetNetForConditionalGeneration.prepare_decoder_input_ids_from_labelso  s   € Ø× Ò  Ñ(Ô(Ð(r#   c                 ód   •— |€| j         j        S t          ¦   «                              |¬¦  «        S )N)Úmodality)r‚   r©  rš   Úget_encoder)r�   râ  rž   s     €r"   rã  z.ProphetNetForConditionalGeneration.get_encoderr  s/   ø€ ØÐØ”?Ô*Ð*å‘7”7×&Ò&°Ð&Ñ9Ô9Ð9r#   )NNNNNNNNNNNNN©r„   r—   )rl   rm   rn   r¶  r   r›   r^  r   r   r   r·  r	   râ   rr   r_   r§   rÇ  rà  rã  r®   r¯   s   @r"   r¹  r¹  Ë  sò  ø€ € € € € ð 	Ð=ðÐð	Ð/ð 	ð 	ð 	ð 	ð 	ð 	ð/ð /ð /ð ð *.Ø.2Ø15Ø:>Ø/3Ø(,Ø-1Ø59Ø&*Ø!%Ø)-Ø,0Ø#'ðmð mà”< $Ñ&ðmð œ tÑ+ðmð !œ<¨$Ñ.ð	mð
 !&Ô 0°4Ñ 7ðmð œ¨Ñ,ðmð  ™ðmð ”| dÑ*ðmð  %œ|¨dÑ2ðmð ”˜tÑ#ðmð ˜$‘;ðmð   $™;ðmð # T™kðmð ˜D‘[ðmð  
Ð*Ñ	*ð!mð mð mñ „^ðmð^ð ð ð ð8)¸E¼Lð )ð )ð )ð )ð:ð :ð :ð :ð :ð :ð :ð :ð :ð :r#   r¹  zt
    The standalone decoder part of the ProphetNetModel with a lm head on top. The model can be used for causal
    c                   ó&  ‡ — e Zd ZdddœZdefˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 	 	 dde	j
        dz  d	e	j
        dz  d
e	j
        dz  de	j
        dz  dedz  de	j
        dz  de	j
        dz  dedz  dedz  dedz  dedz  deez  fd„¦   «         Zdd„Zˆ xZS )ÚProphetNetForCausalLMr»  )rº  z)prophetnet.decoder.word_embeddings.weightr�   c                 óZ  •— t          j        |¦  «        }d|_        d|_        t	          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _	        |j
        | _
        t          j        |j        |j        d¬¦  «        | _        |                      ¦   «          d S )NTFr½  )r§  r¨  rª  r‡  rš   r›   ÚProphetNetDecoderWrapperr‚   r†   r    r¿  r   r¸   rœ   rS  rÀ  r[  r�   s     €r"   r›   zProphetNetForCausalLM.__init__„  s•   ø€ å”˜vÑ&Ô&ˆØ ˆÔØ$)ˆÔ!Ý‰Œ×Ò˜Ñ Ô Ð Ý2°6Ñ:Ô:ˆŒà!Ô.ˆÔØ"(Ô";ˆÔå”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒð 	�ŠÑÔÐÐÐr#   c                 ó$   — | j         j        j        S r—   ©r‚   r«  rT  rú   s    r"   r^  z*ProphetNetForCausalLM.get_input_embeddings”  s   € ØŒÔ&Ô6Ð6r#   c                 ó(   — || j         j        _        d S r—   rê  ra  s     r"   rc  z*ProphetNetForCausalLM.set_input_embeddings—  s   € Ø27ˆŒÔÔ/Ð/Ð/r#   NrŽ   r©   rj   r~  rc   re  rÂ  rE  r¾   rf  rg  r•   c                 óŒ  — |�|n| j         j        }| j                             ||||||||	|
|¬¦
  «
        }|�|j        n|j        dd…         \  }}|d                              || j         j        |d¦  «        }|                      |¦  «        }|dd…df         }| j         j        dk    r|dd…dd…f         nd}d}|�|                      ||¦  «        }|s;t          d„ ||fD ¦   «         ¦  «        }|�|f|z   |dd…         z   n||dd…         z   S t          ||||j        |j        |j        |j        |j        |j        ¬¦	  «	        S )	aª  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`

        Example:

        ```python
        >>> from transformers import AutoTokenizer, ProphetNetForCausalLM
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased")
        >>> model = ProphetNetForCausalLM.from_pretrained("microsoft/prophetnet-large-uncased")
        >>> assert model.config.is_decoder, f"{model.__class__} has to be configured as a decoder."
        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
        >>> outputs = model(**inputs)

        >>> logits = outputs.logits

        >>> # Model can also be used with EncoderDecoder framework
        >>> from transformers import BertTokenizer, EncoderDecoderModel, AutoTokenizer
        >>> import torch

        >>> tokenizer_enc = BertTokenizer.from_pretrained("google-bert/bert-large-uncased")
        >>> tokenizer_dec = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased")
        >>> model = EncoderDecoderModel.from_encoder_decoder_pretrained(
        ...     "google-bert/bert-large-uncased", "microsoft/prophetnet-large-uncased"
        ... )

        >>> ARTICLE = (
        ...     "the us state department said wednesday it had received no "
        ...     "formal word from bolivia that it was expelling the us ambassador there "
        ...     "but said the charges made against him are `` baseless ."
        ... )
        >>> input_ids = tokenizer_enc(ARTICLE, return_tensors="pt").input_ids
        >>> labels = tokenizer_dec(
        ...     "us rejects charges against its ambassador in bolivia", return_tensors="pt"
        ... ).input_ids
        >>> outputs = model(input_ids=input_ids, decoder_input_ids=labels[:, :-1], labels=labels[:, 1:])

        >>> loss = outputs.loss
        ```N)
rŽ   r©   rj   r~  rc   re  rE  r¾   rf  rg  r(   r   rS   r   c              3   ó   K  — | ]}|®|V — Œ	d S r—   r'   rk  s     r"   rm  z0ProphetNetForCausalLM.forward.<locals>.<genexpr>ò  rÅ  r#   )	r`   ra   rb   rc   ry   rz   r{   r|   rh   )r�   rg  r‚   r«  rˆ   r,   r3   rÀ  rÇ  rr   r~   rc   ry   rz   r{   r|   rh   )r�   rŽ   r©   rj   r~  rc   re  rÂ  rE  r¾   rf  rg  rÓ   r:  rÔ   r2   rÈ  rÉ  ra   rb   r`   rÊ  s                         r"   r§   zProphetNetForCausalLM.forwardš  s¯  € ðv &1Ð%<�k�kÀ$Ä+ÔBYˆð ”/×)Ò)ØØ)Ø"7Ø#9Ø+Ø'ØØ/Ø!5Ø#ð *ñ 
ô 
ˆð :CÐ9N i¤o oÐTaÔTgÐhjÐijÐhjÔTkÑ#ˆ
�Oà$ QœZŸ_š_¨Z¸¼Ô9JÈOÐ]_Ñ`Ô`ÐØŸšÐ&8Ñ9Ô9ˆà    1 Ô%ˆØ04´Ô0AÀAÒ0EÐ0E�~ a a a¨¨¨ eÔ,Ð,È4ˆàˆØÐØ×%Ò% n°fÑ=Ô=ˆDàð 	ÝÐRÐR¨6°<Ð*@ÐRÑRÔRÑRÔRˆJØ9=Ð9I�D�7˜ZÑ'¨'°!°"°"¬+Ñ5Ð5ÈzÐ\cÐdeÐdfÐdfÔ\gÑOgÐgå,ØØØ)Ø 'Ô 7Ø%Ô3Ø$+Ô$?Ø"Ô-Ø!(Ô!9Ø!(Ô!9ð
ñ 
ô 
ð 
r#   r„   c                 óÔ  — |                      | j        j        |                     d¦  «        |                     d¦  «        ¦  «                             |¦  «        }t          | j        j        ¦  «        D ]}|dk    r	| j        r n|||d d …d d …f<   Œ|                     dd¦  «                             ¦   «         }t          j
                             |                     d|                     d¦  «        ¦  «        dt          j        ¬¦  «        }t          j
                             ||                     d¦  «        d¬¦  «        }| j        j        dk    r–|                     dd¬	¦  «         }|                     |¦  «                             d¦  «        }	||	         }|                     ¦   «         }| j        j        |                     d¦  «        z  }
d
| j        j        z
  |z  |
|z  z   }|S rÌ  rÐ  r×  s              r"   rÇ  z#ProphetNetForCausalLM._compute_loss  rÞ  r#   )NNNNNNNNNNNrä  )rl   rm   rn   r¶  r   r›   r^  rc  r   r   r   r	   râ   rr   r~   r§   rÇ  r®   r¯   s   @r"   ræ  ræ  y  s   ø€ € € € € ð >Ø5Xðð Ðð
Ð/ð ð ð ð ð ð ð 7ð 7ð 7ð8ð 8ð 8ð ð *.Ø.2Ø59Ø6:Ø(,Ø-1Ø&*Ø!%Ø)-Ø,0Ø#'ðdð dà”< $Ñ&ðdð œ tÑ+ðdð  %œ|¨dÑ2ð	dð
 !&¤¨tÑ 3ðdð  ™ðdð ”| dÑ*ðdð ”˜tÑ#ðdð ˜$‘;ðdð   $™;ðdð # T™kðdð ˜D‘[ðdð 
Ð*Ñ	*ðdð dð dñ „^ðdðLð ð ð ð ð ð ð r#   ræ  c                   ó6   ‡ — e Zd ZdZddiZdefˆ fd„Zd„ Zˆ xZS )rè  z„
    This is a wrapper class, so that [`ProphetNetForCausalLM`] can correctly be loaded from pretrained prophetnet
    classes.
    r¥  r¤  r�   c                 óð   •— t          ¦   «                              |¦  «         t          j        |j        |j        |j        ¬¦  «        | _        t          |¦  «        | _	        |  
                    ¦   «          d S )NrO  )rš   r›   r   rR  rS  rœ   r†   rT  ru  r«  r[  r�   s     €r"   r›   z!ProphetNetDecoderWrapper.__init__(  se   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å!œ|¨FÔ,=¸vÔ?QÐ_eÔ_rÐsÑsÔsˆÔÝ(¨Ñ0Ô0ˆŒð 	�ŠÑÔÐÐÐr#   c                 ó   —  | j         |i |¤ŽS r—   )r«  )r�   ÚargsrÓ   s      r"   r§   z ProphetNetDecoderWrapper.forward1  s   € ØˆtŒ|˜TÐ, VÐ,Ð,Ð,r#   )	rl   rm   rn   ro   r¶  r   r›   r§   r®   r¯   s   @r"   rè  rè    sn   ø€ € € € € ðð ð 	)Ð*BðÐðÐ/ð ð ð ð ð ð ð-ð -ð -ð -ð -ð -ð -r#   rè  )ru  rM  ræ  r¹  r£  r€   r;  ):ro   r§  rF   Údataclassesr   r   r   r   Útorch.nnr   Úactivationsr   Úcache_utilsr	   r
   r   Ú
generationr   Úmodeling_layersr   Úmodeling_outputsr   Úmodeling_utilsr   Úutilsr   r   r   r   Úconfiguration_prophetnetr   Ú
get_loggerrl   r…  r   r>   rQ   r\   r_   rt   rx   r~   r€   rR  r”   ÚModuler±   rä   rî   r/  r>  rM  ru  r£  r¹  ræ  rè  Ú__all__r'   r#   r"   ú<module>r      s-  ðð YÐ Xà €€€Ø €€€Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à !Ð !Ð !Ð !Ð !Ð !Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ )Ð )Ð )Ð )Ð )Ð )Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø /Ð /Ð /Ð /Ð /Ð /Ø -Ð -Ð -Ð -Ð -Ð -Ø QÐ QÐ QÐ QÐ QÐ QÐ QÐ QÐ QÐ QÐ QÐ QØ 6Ð 6Ð 6Ð 6Ð 6Ð 6ð 
ˆÔ	˜HÑ	%Ô	%€ðQð Qð Qð Qð7ð 7ð 7ð, ð  ð  ð  ð6Mð Mð Mð. €ððñ ô ð
 ð*?ð *?ð *?ð *?ð *? ñ *?ô *?ñ „ñô ð*?ðZ €ððñ ô ð ð(?ð (?ð (?ð (?ð (? ;ñ (?ô (?ñ „ñô ð(?ðV €ððñ ô ð
 ð#=ð #=ð #=ð #=ð #= ;ñ #=ô #=ñ „ñô ð#=ðL €ððñ ô ð
 ð+=ð +=ð +=ð +=ð += ñ +=ô +=ñ „ñô ð+=ð\ ð!ð !ð !ð !ð ! ñ !ô !ñ „ð!ð8(-ð (-ð (-ð (-ð (- R¤\ñ (-ô (-ð (-ðVs2ð s2ð s2ð s2ð s2˜"œ)ñ s2ô s2ð s2ðlð ð ð ð ˜BœIñ ô ð ð.o/ð o/ð o/ð o/ð o/ 2¤9ñ o/ô o/ð o/ðd	&ð &ð &ð &ð &Ð7ñ &ô &ð &ðRDð Dð Dð Dð DÐ7ñ Dô Dð DðN €ððñ ô ð
c
ð c
ð c
ð c
ð c
Ð1ñ c
ô c
ñô ð
c
ðL €ððñ ô ð
hGð hGð hGð hGð hGÐ1ñ hGô hGñô ð
hGðV	 ð|
ð |
ð |
ð |
ð |
Ð/ñ |
ô |
ñ „ð|
ð~ €ððñ ô ð
f:ð f:ð f:ð f:ð f:Ð)BÀOñ f:ô f:ñô ð
f:ðR €ððñ ô ð
]ð ]ð ]ð ]ð ]Ð5°ñ ]ô ]ñô ð
]ð@-ð -ð -ð -ð -Ð8ñ -ô -ð -ð.ð ð €€€r#   