§
    ‚ŠtjéG ã                   óV  — d Z ddlZddlZddlmZ ddlZddlmZ ddlmZ ddl	m
Z ddlmZ dd	lmZmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZmZmZ ddlmZ ddl m!Z!m"Z"m#Z#m$Z$m%Z% ddl&m'Z'  e%j(        e)¦  «        Z*dNdej+        de,de,de,dej+        f
d„Z-dej+        de,de,dej+        fd„Z.dNdej+        de,de,de,dej+        f
d„Z/de,dej+        fd„Z0dej+        de,dej+        fd„Z1dej+        de,d ej2        dej+        fd!„Z3dej+        d"e,de4ej+        ej+        f         fd#„Z5dej+        d"e,dej+        fd$„Z6d%ej+        d&ej+        d'e,dej+        fd(„Z7 G d)„ d*ej8        ¦  «        Z9 G d+„ d,ej8        ¦  «        Z: G d-„ d.ej8        ¦  «        Z; G d/„ d0ej8        ¦  «        Z< G d1„ d2ej8        ¦  «        Z= G d3„ d4ej8        ¦  «        Z> G d5„ d6ej8        ¦  «        Z? G d7„ d8ej8        ¦  «        Z@ G d9„ d:ej8        ¦  «        ZA G d;„ d<ej8        ¦  «        ZB G d=„ d>ej8        ¦  «        ZC G d?„ d@e¦  «        ZDe# G dA„ dBe¦  «        ¦   «         ZE G dC„ dDeE¦  «        ZFe# G dE„ dFeE¦  «        ¦   «         ZG e#dG¬H¦  «         G dI„ dJeEe¦  «        ¦   «         ZHe# G dK„ dLeE¦  «        ¦   «         ZIg dM¢ZJdS )OzPyTorch LongT5 model.é    N)ÚAny)Únn)ÚCrossEntropyLossé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚPreTrainedModel)ÚDUMMY_INPUTSÚ
DUMMY_MASKÚauto_docstringÚis_torchdynamo_compilingÚloggingé   )ÚLongT5ConfigÚxÚ	block_lenÚdimÚ	pad_valueÚreturnc                 ój  — | j         |          |z  }t          | j         ¦  «        s?t          | j         ¦  «        }||xx         |z  cc<   t          j        || j        ¬¦  «        S dg| j        z  }d|f||<   t          |ddd…         d¦  «        }t          j	         
                    | |d|¬¦  «        } | S )	zHPad a tensor so that a sequence length will be a multiple of `block_len`©Údtype©r   r   r   Néÿÿÿÿ© Úconstant©ÚpadÚmodeÚvalue)ÚshapeÚallÚlistÚtorchÚzerosr#   ÚndimÚsumr   Ú
functionalr)   )r   r   r   r   Úpad_lenÚ	new_shaper)   s          úh/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/longt5/modeling_longt5.pyÚ_pad_to_multipler7   5   sµ   € àŒw�sŒ|ˆm˜iÑ'€GåˆqŒw‰<Œ<ð 5Ý˜œ‘M”Mˆ	Ø�#ˆˆŒ˜'Ñ!ˆˆ‰ÝŒ{˜9¨A¬GÐ4Ñ4Ô4Ð4àˆ(�Q”VÑ
€CØ�7ˆ|€Cˆ�HÝ
ˆc�$�$�B�$Œi˜Ñ
Ô
€CÝ
Œ×Ò˜! ¨:¸YÐÑGÔG€AØ€Hó    c                 ó2  — | j         |         |z  dk    rt          | ||d¬¦  «        } | j         |         |z  }| j         d|…         ||fz   | j         |dz   d…         z   }d|v r!t          j        || j        | j        ¬¦  «        S |                      |¦  «        S )zÅSplit an input tensor into blocks of a given `block_len` along the given `dim`. If the dimension length
    is not a multiple of `block_len`, it will be padded first with selected `pad_value`.
    r   )r   Nr   ©r#   Údevice)r,   r7   r/   Úemptyr#   r;   Úreshape)r   r   r   Ú
num_blocksÚoutput_shapes        r6   Ú_split_into_blocksr@   E   s¥   € ð
 	„wˆs„|�iÑ 1Ò$Ð$Ý˜Q 	¨3¸!Ð<Ñ<Ô<ˆØ”˜” Ñ*€JØ”7˜4˜C˜4”= J°	Ð#:Ñ:¸Q¼WÀcÈAÁgÀ[À[Ô=QÑQ€LàˆLÐÐÝŒ{˜<¨q¬w¸q¼xÐHÑHÔHÐHØ�9Š9�\Ñ"Ô"Ð"r8   Ú	block_dimÚsequence_dimc                 óº  — | j         |         }dg| j        z  }d||<   t          |ddd…         d¦  «        }t          j                             | |d|¬¦  «        } g }t          d¦  «        D ][}t          d	d¦  «        g| j        z  }t          |||z   ¦  «        ||<   t          |¦  «        }| 	                    | |         ¦  «         Œ\t          j        ||¬
¦  «        S )zžConcatenate three consecutive blocks for each input block for local attentiont.

    For more information, see: https://huggingface.co/papers/2112.07916.
    r$   )r   r   Nr%   r&   r'   r(   r   r   ©r   )r,   r1   r2   r   r3   r)   ÚrangeÚsliceÚtupleÚappendr/   Úcat)	r   rA   rB   r   r>   r)   Úblocks_listÚiÚindicess	            r6   Ú_concatenate_3_blocksrM   T   sä   € ð
 ”˜Ô#€Jàˆ(�Q”VÑ
€CØ€Cˆ	�NÝ
ˆc�$�$�B�$Œi˜Ñ
Ô
€Cå
Œ×Ò˜! ¨:¸YÐÑGÔG€Aà&(€KÝ�1‰XŒXð 'ð 'ˆõ ˜˜D‘>”>Ð" Q¤VÑ+ˆÝ" 1 a¨*¡nÑ5Ô5ˆ�	ÑÝ˜‘.”.ˆØ×Ò˜1˜Wœ:Ñ&Ô&Ð&Ð&åŒ9�[ lÐ3Ñ3Ô3Ð3r8   c                 ó¸   — t          j        d| z  t           j        ¬¦  «        }|| |  …         }|                     d¦  «        |                     d¦  «        z
  }|S )z:Makes 3-blocked relative position ids for local attention.r   r"   r   r   )r/   ÚarangeÚint32Ú	unsqueeze)r   Úposition_idsÚcenter_position_idsÚrelative_position_idss       r6   Ú"_make_3block_relative_position_idsrU   m   s]   € å”<  I¡µU´[ÐAÑAÔA€LØ& y°)°Ð';Ô<Ðà(×2Ò2°1Ñ5Ô5Ð8K×8UÒ8UÐVWÑ8XÔ8XÑXÐØ Ð r8   Úlocal_attention_maskc                 óÎ   — t          |¦  «        }t          j        |¦  «        |k     }|dddd…dd…f         }|                     | j        ¦  «        }t          j        | |¦  «        S )znMask local attention mask to enforce that tokens are not allowed to attend tokens farther than ``local_radius.N)rU   r/   ÚabsÚtor;   Úlogical_and)rV   r   rT   Úlocality_masks       r6   Ú_mask_local_attention_maskr\   v   sk   € å>¸yÑIÔIÐÝ”IÐ3Ñ4Ô4°yÒ@€MØ! $¨¨a¨a¨a°°°Ð"2Ô3€MØ!×$Ò$Ð%9Ô%@ÑAÔA€MÝÔÐ1°=ÑAÔAÐAr8   Úattention_maskr;   c                 ó8  — t          | |d¬¦  «        }t          |dd¬¦  «        }|                     d¦  «        }|                     d¦  «        }t          j        ||¦  «        }t          ||¦  «        }|                     d¦  «                             |¦  «        S )z;Prepare attention mask to be applied for a local attention.r   rD   é   ©rA   rB   r%   éþÿÿÿ)r@   rM   rQ   r/   rZ   r\   rY   )r]   r   r;   Ú_blocked_attention_maskÚ_3blocked_attention_maskrV   s         r6   Ú_get_local_attention_maskrd      s¢   € õ 1°ÀÐPQÐRÑRÔRÐå4Ð5LÐXYÐhiÐjÑjÔjÐà5×?Ò?ÀÑCÔCÐØ7×AÒAÀ"ÑEÔEÐå Ô,Ð-DÐF^Ñ_Ô_ÐÝ5Ð6JÈIÑVÔVÐà×)Ò)¨!Ñ,Ô,×/Ò/°Ñ7Ô7Ð7r8   Úglobal_block_sizec                 ó^  ‡‡— | j         dd…         \  }Šdt          j        dt          j        fˆˆfd„}t          j        | | j        ¬¦  «        ‰z  }t          j        |d¬¦  «        |z
  }t          j        | d	k    d
d¦  «                             | j        ¦  «        }t          j	        ||z   d
z
  ¦  «                             | j        ¦  «        }t          j
        d|j        |j        ¬¦  «        }t          j        ||k    ||¦  «        }|| z  | dz
  z   } ||¦  «        }‰‰z  }|dk    rDt          j        |d¬¦  «        j                             |d¦  «                             dd¦  «        }	n"t          j        |d|j        |j        ¬¦  «        }	t          j        t          j        ||¦  «        d¬¦  «        dz
  }
|
                     | j        ¦  «        }
t          j        |
|	k    dd¦  «        }
|                     t          j        ¦  «        |
                     t          j        ¦  «        fS )a  Obtain the "fixed block" global id corresponding to each input token.

    This implementation is a simplified version of the original Flaxformr implementation adopted from:
    https://github.com/google/flaxformer/blob/main/flaxformer/architectures/longt5/long_attention.py.

    In our scenario, as we use this strategy only for a decoder, orphan tokens, i.e. those tokens which do not make for
    the whole fixed block, are assigned to the preceding block.

    Padding tokens from the original sequence are represented by -1.
    Nr_   Ú	block_idsr    c                 ód  •— t          j        ‰¦  «        ‰z  ‰dz
  k    }|                     | j        ¦  «        }t          j        || dk    ¦  «        }|                     d¦  «                             d¦  «                             | j        ¦  «        dz
  }t          j	        | |k     | |¦  «        } | S )Nr   r   r%   )
r/   rO   rY   r;   rZ   r2   rQ   Útyper#   Úwhere)rg   Ú
block_endsÚtrue_block_endsÚfull_blocksre   Úseq_lens       €€r6   Úhandle_orphan_tokensz:_make_global_fixed_block_ids.<locals>.handle_orphan_tokensž   s£   ø€ Ý”l 7Ñ+Ô+Ð.?Ñ?ÐDUÐXYÑDYÒYˆ
Ø—]’] 9Ô#3Ñ4Ô4ˆ
ÝÔ+¨J¸	ÀQºÑGÔGˆØ%×)Ò)¨"Ñ-Ô-×7Ò7¸Ñ;Ô;×@Ò@ÀÄÑQÔQÐTUÑUˆÝ”K 	¨KÒ 7¸ÀKÑPÔPˆ	ØÐr8   ©r;   r   )Úaxisç        ç      ð?g     @�Àr%   r:   r   rD   )r,   r/   ÚTensorÚ	ones_liker;   Úcumsumrj   ri   r#   ÚfloorÚtensorÚmaxÚvaluesÚrepeatÚ	transposer0   ÚonesrY   Úint)r]   re   Ú
batch_sizero   Úfixed_block_maskÚmaskÚglobal_block_idsÚ_global_block_ids_lower_boundÚnum_globalsÚ_sequence_block_ids_maxÚglobal_segment_idsrn   s    `         @r6   Ú_make_global_fixed_block_idsr‡   �   sP  øø€ ð )Ô.¨r°¨rÔ2Ñ€J�ð­¬ð ½¼ð ð ð ð ð ð ð õ ” ~¸nÔ>SÐTÑTÔTÐWhÑhÐÝ”|Ð$4¸1Ð=Ñ=Ô=Ð@PÑPÐÝŒ;�~¨Ò,¨c°7Ñ;Ô;×@Ò@ÀÔAUÑVÔV€DÝ”{ 4Ð*:Ñ#:¸SÑ#@ÑAÔA×FÒFÀ~ÔG[Ñ\Ô\ÐÝ$)¤L°Ð;KÔ;QÐZjÔZqÐ$rÑ$rÔ$rÐ!Ý”{ØÐ8Ò8Ð:JÐLiñô Ðð )¨>Ñ9¸nÈqÑ>PÑQÐà+Ð+Ð,<Ñ=Ô=ÐØÐ.Ñ.€Kà�Q‚€Ý"'¤)Ð,<À"Ð"EÑ"EÔ"EÔ"L×"SÒ"SÐT_ÐabÑ"cÔ"c×"mÒ"mÐnoÐqrÑ"sÔ"sÐÐå"'¤+Ø˜Ð!1Ô!7Ð@PÔ@Wð#
ñ #
ô #
Ðõ œ¥e¤j°¸[Ñ&IÔ&IÈrÐRÑRÔRÐUVÑVÐØ+×.Ò.¨~Ô/DÑEÔEÐÝœÐ%7Ð;RÒ%RÐTUÐWXÑYÔYÐØ× Ò ¥¤Ñ+Ô+Ð-?×-DÒ-DÅUÄYÑ-OÔ-OÐOÐOr8   c                 óÌ   — t          | |¦  «        \  }}|j        d         }t          j        ||j        ¬¦  «        }||d         z
  }|                     t          j        ¦  «        S )zBCreate the relative position tensor for local -> global attention.r%   rp   ©.N)r‡   r,   r/   rO   r;   ri   Úint64)r]   re   rg   r†   Úglobal_seq_lenÚglobal_positionsÚside_relative_positions          r6   Ú _make_side_relative_position_idsrŽ   À   sc   € å$@ÀÐQbÑ$cÔ$cÑ!€IÐ!Ø'Ô-¨bÔ1€NÝ”| N¸9Ô;KÐLÑLÔLÐØ-°	¸)Ô0DÑDÐØ!×&Ò&¥u¤{Ñ3Ô3Ð3r8   Úhidden_statesrg   r‹   c                 ón  — |                      |dk    t          j        ||j        |j        ¬¦  «        ¦  «        }t
          j                             |                     t          j	        ¦  «        |dz   ¦  «        dd…dd…dd…f         }t          j
        d| |                     | j        ¦  «        ¦  «        S )zFCompute individual block aggregates by summing over individual blocks.r   r:   r   Nr%   z...nd,...ng->...gd)rj   r/   rx   r#   r;   r   r3   Úone_hotri   rŠ   Úeinsum)r�   rg   r‹   Úone_hot_block_idss       r6   Ú_create_global_aggregatesr”   É   s®   € ð
 —’Ø�QŠ�œ ^¸9¼?ÐS\ÔScÐdÑdÔdñô €Iõ œ×-Ò-¨i¯nªn½U¼[Ñ.IÔ.IÈ>Ð\]ÑK]Ñ^Ô^Ð_`Ð_`Ð_`ÐbcÐbcÐbcÐehÐfhÐehÐ_hÔiÐÝŒ<Ð,¨mÐ=N×=SÒ=SÐTaÔTgÑ=hÔ=hÑiÔiÐir8   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚLongT5LayerNormç�íµ ÷Æ°>c                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )zg
        Construct a layernorm module in the LongT5 style. No bias and no subtraction of mean.
        N)ÚsuperÚ__init__r   Ú	Parameterr/   r}   ÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚ	__class__s      €r6   rš   zLongT5LayerNorm.__init__×   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr8   c                 óh  — |                      t          j        ¦  «                             d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        j        t          j	        t          j
        fv r|                      | j        j        ¦  «        }| j        |z  S )Nr_   r%   T)Úkeepdim)rY   r/   Úfloat32ÚpowÚmeanÚrsqrtr�   rœ   r#   Úfloat16Úbfloat16)rž   r�   Úvariances      r6   ÚforwardzLongT5LayerNorm.forwardß   s–   € ð !×#Ò#¥E¤MÑ2Ô2×6Ò6°qÑ9Ô9×>Ò>¸rÈ4Ð>ÑPÔPˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆð Œ;Ô¥¤µ´Ð ?Ð?Ð?Ø)×,Ò,¨T¬[Ô->Ñ?Ô?ˆMàŒ{˜]Ñ*Ð*r8   )r—   )Ú__name__Ú
__module__Ú__qualname__rš   r«   Ú__classcell__©r¡   s   @r6   r–   r–   Ö   sL   ø€ € € € € ð$ð $ð $ð $ð $ð $ð+ð +ð +ð +ð +ð +ð +r8   r–   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚLongT5DenseActDenseÚconfigc                 óJ  •— t          ¦   «                              ¦   «          t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j        |j	        ¦  «        | _
        t          |j                 | _        d S ©NF©Úbias)r™   rš   r   ÚLinearÚd_modelÚd_ffÚwiÚwoÚDropoutÚdropout_rateÚdropoutr   Údense_act_fnÚact©rž   r³   r¡   s     €r6   rš   zLongT5DenseActDense.__init__ñ   sx   ø€ Ý‰Œ×ÒÑÔÐÝ”)˜FœN¨F¬K¸eÐDÑDÔDˆŒÝ”)˜FœK¨¬¸eÐDÑDÔDˆŒÝ”z &Ô"5Ñ6Ô6ˆŒÝ˜&Ô-Ô.ˆŒˆˆr8   c                 ó°  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }t          | j        j        t          j        ¦  «        r]|j        | j        j        j        k    rC| j        j        j        t          j	        k    r$| 
                    | j        j        j        ¦  «        }|                      |¦  «        }|S ©N)r»   rÁ   r¿   Ú
isinstancer¼   rœ   r/   rt   r#   Úint8rY   )rž   r�   s     r6   r«   zLongT5DenseActDense.forwardø   s¨   € ØŸš Ñ.Ô.ˆØŸš Ñ/Ô/ˆØŸš ]Ñ3Ô3ˆå�t”w”~¥u¤|Ñ4Ô4ð	CàÔ# t¤w¤~Ô';Ò;Ð;Ø””Ô$­¬
Ò2Ð2à)×,Ò,¨T¬W¬^Ô-AÑBÔBˆMØŸš Ñ.Ô.ˆØÐr8   ©r¬   r­   r®   r   rš   r«   r¯   r°   s   @r6   r²   r²   ð   sS   ø€ € € € € ð/˜|ð /ð /ð /ð /ð /ð /ðð ð ð ð ð ð r8   r²   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚLongT5DenseGatedActDenser³   c                 ó–  •— t          ¦   «                              ¦   «          t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j	        |j
        ¦  «        | _        t          |j                 | _        d S rµ   )r™   rš   r   r¸   r¹   rº   Úwi_0Úwi_1r¼   r½   r¾   r¿   r   rÀ   rÁ   rÂ   s     €r6   rš   z!LongT5DenseGatedActDense.__init__  s”   ø€ Ý‰Œ×ÒÑÔÐÝ”I˜fœn¨f¬kÀÐFÑFÔFˆŒ	Ý”I˜fœn¨f¬kÀÐFÑFÔFˆŒ	Ý”)˜FœK¨¬¸eÐDÑDÔDˆŒÝ”z &Ô"5Ñ6Ô6ˆŒÝ˜&Ô-Ô.ˆŒˆˆr8   c                 óÞ   — |                       |                      |¦  «        ¦  «        }|                      |¦  «        }||z  }|                      |¦  «        }|                      |¦  «        }|S rÄ   )rÁ   rË   rÌ   r¿   r¼   )rž   r�   Úhidden_geluÚhidden_linears       r6   r«   z LongT5DenseGatedActDense.forward  sb   € Ø—h’h˜tŸyšy¨Ñ7Ô7Ñ8Ô8ˆØŸ	š	 -Ñ0Ô0ˆØ# mÑ3ˆØŸš ]Ñ3Ô3ˆØŸš Ñ.Ô.ˆØÐr8   rÇ   r°   s   @r6   rÉ   rÉ     sS   ø€ € € € € ð/˜|ð /ð /ð /ð /ð /ð /ðð ð ð ð ð ð r8   rÉ   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚLongT5LayerFFr³   c                 ó$  •— t          ¦   «                              ¦   «          |j        rt          |¦  «        | _        nt          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          j        |j        ¦  «        | _        d S )N©r    )r™   rš   Úis_gated_actrÉ   ÚDenseReluDenser²   r–   r¹   Úlayer_norm_epsilonÚ
layer_normr   r½   r¾   r¿   rÂ   s     €r6   rš   zLongT5LayerFF.__init__  sx   ø€ Ý‰Œ×ÒÑÔÐØÔð 	>Ý":¸6Ñ"BÔ"BˆDÔÐå"5°fÑ"=Ô"=ˆDÔå)¨&¬.¸fÔ>WÐXÑXÔXˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr8   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }||                      |¦  «        z   }|S rÄ   )r×   rÕ   r¿   )rž   r�   Úforwarded_statess      r6   r«   zLongT5LayerFF.forward$  sF   € ØŸ?š?¨=Ñ9Ô9ÐØ×.Ò.Ð/?Ñ@Ô@ÐØ%¨¯ªÐ5EÑ(FÔ(FÑFˆØÐr8   rÇ   r°   s   @r6   rÑ   rÑ     sS   ø€ € € € € ð7˜|ð 7ð 7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð r8   rÑ   c                   óf   ‡ — e Zd Z	 	 ddededz  fˆ fd„Zedd	„¦   «         Zdd„Z	 	 	 	 	 dd„Z	ˆ xZ
S )ÚLongT5AttentionFNr³   Ú	layer_idxc                 ó*  •— t          ¦   «                              ¦   «          |j        | _        || _        |j        | _        |j        | _        |j        | _        |j        | _        |j	        | _
        |j        | _        | j
        | j        z  | _        || _        |€/| j        r(t                               d| j        j        › d�¦  «         t'          j        | j        | j        d¬¦  «        | _        t'          j        | j        | j        d¬¦  «        | _        t'          j        | j        | j        d¬¦  «        | _        t'          j        | j        | j        d¬¦  «        | _        | j        r$t'          j        | j        | j
        ¦  «        | _        d| _        d S )NzInstantiating a decoder z³ without passing `layer_idx` is not recommended and will to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.Fr¶   )r™   rš   Ú
is_decoderÚhas_relative_attention_biasÚrelative_attention_num_bucketsÚrelative_attention_max_distancer¹   Úd_kvÚkey_value_proj_dimÚ	num_headsÚn_headsr¾   r¿   Ú	inner_dimrÜ   ÚloggerÚwarning_oncer¡   r¬   r   r¸   ÚqÚkÚvÚoÚ	EmbeddingÚrelative_attention_biasÚgradient_checkpointing©rž   r³   rß   rÜ   r¡   s       €r6   rš   zLongT5Attention.__init__-  si  ø€ õ 	‰Œ×ÒÑÔÐØ Ô+ˆŒØ+FˆÔ(Ø.4Ô.SˆÔ+Ø/5Ô/UˆÔ,Ø”~ˆŒØ"(¤+ˆÔØÔ'ˆŒØÔ*ˆŒØœ¨Ô(?Ñ?ˆŒØ"ˆŒØÐ ¤ÐÝ×Òð,¨4¬>Ô+Bð ,ð ,ð ,ñô ð õ ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ>¨4¬<¸eÐDÑDÔDˆŒàÔ+ð 	kÝ+-¬<¸Ô8[Ð]aÔ]iÑ+jÔ+jˆDÔ(à&+ˆÔ#Ð#Ð#r8   Té    é€   c                 óP  — d}|rC|dz  }|| dk                          t          j        ¦  «        |z  z  }t          j        | ¦  «        } n(t          j        | t          j        | ¦  «        ¦  «         } |dz  }| |k     }|t          j        |                      ¦   «         |z  ¦  «        t          j        ||z  ¦  «        z  ||z
  z                        t          j        ¦  «        z   }t          j        |t          j	        ||dz
  ¦  «        ¦  «        }|t          j
        || |¦  «        z  }|S ©aÒ  
        Adapted from Mesh Tensorflow:
        https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593

        Translate relative position to a bucket number for relative attention. The relative position is defined as
        memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to
        position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for
        small absolute relative_position and larger buckets for larger absolute relative_positions. All relative
        positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket.
        This should allow for more graceful generalization to longer sequences than the model has been trained on

        Args:
            relative_position: an int32 Tensor
            bidirectional: a boolean - whether the attention is bidirectional
            num_buckets: an integer
            max_distance: an integer

        Returns:
            a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets)
        r   r_   r   ©rY   r/   ÚlongrX   ÚminÚ
zeros_likeÚlogÚfloatÚmathÚ	full_likerj   ©Úrelative_positionÚbidirectionalÚnum_bucketsÚmax_distanceÚrelative_bucketsÚ	max_exactÚis_smallÚrelative_position_if_larges           r6   Ú_relative_position_bucketz)LongT5Attention._relative_position_bucketO  s>  € ð, ÐØð 	cØ˜AÑˆKØÐ!2°QÒ!6× :Ò :½5¼:Ñ FÔ FÈÑ TÑTÐÝ %¤	Ð*;Ñ <Ô <ÐÐå!&¤Ð+<½eÔ>NÐO`Ñ>aÔ>aÑ!bÔ!bÐ bÐð   1Ñ$ˆ	Ø$ yÒ0ˆð &/ÝŒIÐ'×-Ò-Ñ/Ô/°)Ñ;Ñ<Ô<ÝŒh�| iÑ/Ñ0Ô0ñ1à˜YÑ&ñ(÷ Š"�UŒZ‰.Œ.ñ	&Ð"õ
 &+¤YØ&­¬Ð8RÐT_ÐbcÑTcÑ(dÔ(dñ&
ô &
Ð"ð 	�EœK¨Ð2CÐE_Ñ`Ô`Ñ`ÐØÐr8   r   c                 ó¸  — |€| j         j        j        }t          j        |t          j        |¬¦  «        dd…df         |z   }t          j        |t          j        |¬¦  «        ddd…f         }||z
  }|                      || j         | j        | j	        ¬¦  «        }|                       |¦  «        }	|	 
                    g d¢¦  «                             d¦  «        }	|	S )ú%Compute binned relative position biasNr:   ©rÿ   r   r  ©r_   r   r   r   )rî   rœ   r;   r/   rO   rö   r  rÞ   rà   rá   ÚpermuterQ   )
rž   Úquery_lengthÚ
key_lengthr;   Úpast_seen_tokensÚcontext_positionÚmemory_positionrþ   Úrelative_position_bucketrz   s
             r6   Úcompute_biaszLongT5Attention.compute_bias  sí   € àˆ>ØÔ1Ô8Ô?ˆFÝ œ<¨½E¼JÈvÐVÑVÔVÐWXÐWXÐWXÐZ^ÐW^Ô_ÐbrÑrÐÝœ, z½¼ÈFÐSÑSÔSÐTXÐZ[ÐZ[ÐZ[ÐT[Ô\ˆØ+Ð.>Ñ>ÐØ#'×#AÒ#AØØ#œÐ.ØÔ;ØÔ=ð	 $Bñ $
ô $
Ð ð ×-Ò-Ð.FÑGÔGˆØ—’ 	 	 	Ñ*Ô*×4Ò4°QÑ7Ô7ˆØˆr8   c                 ó�  — |j         dd…         }g |¢d‘| j        ‘R }	|�|                     | j        ¦  «        nd}
t	          |
t
          j        ¦  «        r|
                     ¦   «         n|
}
|du}|                      |¦  «         	                    |	¦  «         
                    dd¦  «        }d}t	          |t          ¦  «        r1|j                             | j        ¦  «        }|r|j        }n
|j        }n|}|r|n|}|r3|�1|r/|j        | j                 j        }|j        | j                 j        }nÚg |j         dd…         ¢d‘| j        ‘R }|                      |¦  «         	                    |¦  «         
                    dd¦  «        }|                      |¦  «         	                    |¦  «         
                    dd¦  «        }|�E|                     ||| j        ¦  «        \  }}|r$t	          |t          ¦  «        rd|j        | j        <   t          j        || 
                    dd¦  «        ¦  «        }|€ª|j         d	         }| j        sLt          j        d|j         d         |d         |f|j        |j        ¬
¦  «        }| j        r| j        rd|_        n$|                      |d         ||j        |
¬¦  «        }|�$|dd…dd…dd…d|j         d	         …f         }||z   }|}||z  }t>          j          !                    | "                    ¦   «         d¬¦  «         #                    |¦  «        }t>          j          $                    || j$        | j        ¬¦  «        }t          j        ||¦  «        }| 
                    dd¦  «         %                    ¦   «         } |j&        g |¢d‘R Ž }|  '                    |¦  «        }||f}|r||fz   }|S )z€
        Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states).
        Nr%   r   r   r_   FTr   ra   ©r;   r#   )r;   r  rD   ©ÚpÚtraining)(r,   rã   Úget_seq_lengthrÜ   rÅ   r/   rt   Úcloneré   Úviewr|   r   Ú
is_updatedÚgetÚcross_attention_cacheÚself_attention_cacheÚlayersÚkeysrz   rê   rë   ÚupdateÚmatmulrß   r0   r;   r#   rï   r  Úrequires_gradr  r   r3   Úsoftmaxrú   Útype_asr¿   Ú
contiguousr=   rì   )rž   r�   r�   Úkey_value_statesÚposition_biasÚpast_key_valuesÚoutput_attentionsÚkwargsÚinput_shapeÚhidden_shaper  Úis_cross_attentionÚquery_statesr  Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚkv_shapeÚscoresr  Úcausal_maskÚposition_bias_maskedÚattn_weightsÚattn_outputÚoutputss                             r6   r«   zLongT5Attention.forward�  s  € ð $Ô)¨#¨2¨#Ô.ˆØB˜ÐB bÐB¨$Ô*AÐBÐBˆØM\ÐMh˜?×9Ò9¸$¼.ÑIÔIÐIÐnoÐå7AÐBRÕTYÔT`Ñ7aÔ7aÐwÐ+×1Ò1Ñ3Ô3Ð3ÐgwÐð .°TÐ9Ðà—v’v˜mÑ,Ô,×1Ò1°,Ñ?Ô?×IÒIÈ!ÈQÑOÔOˆð ˆ
Ý�oÕ':Ñ;Ô;ð 	3Ø(Ô3×7Ò7¸¼ÑGÔGˆJØ!ð Là'6Ô'LÐ$Ð$à'6Ô'KÐ$Ð$à#2Ð à-?ÐRÐ)Ð)À]ˆØð 	F /Ð"=À*Ð"=à-Ô4°T´^ÔDÔIˆJØ/Ô6°t´~ÔFÔMˆLˆLàP˜Ô-¨c¨r¨cÔ2ÐP°BÐP¸Ô8OÐPÐPˆHØŸš Ñ/Ô/×4Ò4°XÑ>Ô>×HÒHÈÈAÑNÔNˆJØŸ6š6 .Ñ1Ô1×6Ò6°xÑ@Ô@×JÒJÈ1ÈaÑPÔPˆLàÐ*Ø+?×+FÒ+FÀzÐS_ÐaeÔaoÑ+pÔ+pÑ(�
˜Là%ð F­*°_ÕFYÑ*ZÔ*Zð FØAE�OÔ.¨t¬~Ñ>õ ”˜l¨J×,@Ò,@ÀÀAÑ,FÔ,FÑGÔGˆàÐ Ø#Ô)¨"Ô-ˆJØÔ3ð 	Ý %¤Ø˜Ô*¨1Ô-¨{¸1¬~¸zÐJÐSYÔS`ÐhnÔhtð!ñ !ô !�ð Ô.ð 7°4´=ð 7Ø26�MÔ/øà $× 1Ò 1Ø ”N J°v´}ÐWgð !2ñ !ô !�ð ÐØ" 1 1 1 a a a¨¨¨Ð,B¨jÔ.>¸rÔ.BÐ,BÐ#BÔC�Ø -°Ñ ;�à,ÐØÐ&Ñ&ˆõ ”}×,Ò,¨V¯\ª\©^¬^ÀÐ,ÑDÔD×LÒLÈVÑTÔTˆÝ”}×,Ò,¨\¸T¼\ÐTXÔTaÐ,ÑbÔbˆå”l <°Ñ>Ô>ˆà!×+Ò+¨A¨qÑ1Ô1×<Ò<Ñ>Ô>ˆØ)�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;ˆØ—f’f˜[Ñ)Ô)ˆà Ð.ˆàð 	0Ø  Ñ/ˆGØˆr8   ©FN©Trñ   rò   )Nr   )NNNNF)r¬   r­   r®   r   r~   rš   Ústaticmethodr  r  r«   r¯   r°   s   @r6   rÛ   rÛ   ,  sÃ   ø€ € € € € ð %*Ø $ð	 ,ð  ,àð ,ð ˜‘:ð	 ,ð  ,ð  ,ð  ,ð  ,ð  ,ðD ð- ð - ð - ñ „\ð- ð^ð ð ð ð( ØØØØð[ð [ð [ð [ð [ð [ð [ð [r8   rÛ   c                   ó`   ‡ — e Zd Zddededdfˆ fd„Zedd
„¦   «         Zdefd„Z		 	 	 dd„Z
ˆ xZS )ÚLongT5LocalAttentionFr³   rß   r    Nc                 óð  •— t          ¦   «                              ¦   «          |j        | _        || _        |j        | _        |j        | _        |j        | _        |j        | _        |j	        | _
        |j        | _        | j        dz   | _        |j        | _        | j
        | j        z  | _        t!          j        | j        | j        d¬¦  «        | _        t!          j        | j        | j        d¬¦  «        | _        t!          j        | j        | j        d¬¦  «        | _        t!          j        | j        | j        d¬¦  «        | _        | j        r$t!          j        | j        | j
        ¦  «        | _        d| _        d S )Nr   Fr¶   )r™   rš   rÞ   rß   rà   rá   r¹   râ   rã   rä   rå   Úlocal_radiusr   r¾   r¿   ræ   r   r¸   ré   rê   rë   rì   rí   rî   rï   ©rž   r³   rß   r¡   s      €r6   rš   zLongT5LocalAttention.__init__ï  s8  ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒØ+FˆÔ(Ø.4Ô.SˆÔ+Ø/5Ô/UˆÔ,Ø”~ˆŒØ"(¤+ˆÔØÔ'ˆŒØ"Ô/ˆÔØÔ*¨QÑ.ˆŒØÔ*ˆŒØœ¨Ô(?Ñ?ˆŒå”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ>¨4¬<¸eÐDÑDÔDˆŒàÔ+ð 	kÝ+-¬<¸Ô8[Ð]aÔ]iÑ+jÔ+jˆDÔ(à&+ˆÔ#Ð#Ð#r8   Trñ   rò   c                 óP  — d}|rC|dz  }|| dk                          t          j        ¦  «        |z  z  }t          j        | ¦  «        } n(t          j        | t          j        | ¦  «        ¦  «         } |dz  }| |k     }|t          j        |                      ¦   «         |z  ¦  «        t          j        ||z  ¦  «        z  ||z
  z                        t          j        ¦  «        z   }t          j        |t          j	        ||dz
  ¦  «        ¦  «        }|t          j
        || |¦  «        z  }|S rô   rõ   rý   s           r6   r  z.LongT5LocalAttention._relative_position_bucket  ó>  € ð. ÐØð 	cØ˜AÑˆKØÐ!2°QÒ!6× :Ò :½5¼:Ñ FÔ FÈÑ TÑTÐÝ %¤	Ð*;Ñ <Ô <ÐÐå!&¤Ð+<½eÔ>NÐO`Ñ>aÔ>aÑ!bÔ!bÐ bÐð   1Ñ$ˆ	Ø$ yÒ0ˆð &/ÝŒIÐ'×-Ò-Ñ/Ô/°)Ñ;Ñ<Ô<ÝŒh�| iÑ/Ñ0Ô0ñ1à˜YÑ&ñ(÷ Š"�UŒZ‰.Œ.ñ	&Ð"õ
 &+¤YØ&­¬Ð8RÐT_ÐbcÑTcÑ(dÔ(dñ&
ô &
Ð"ð 	�EœK¨Ð2CÐE_Ñ`Ô`Ñ`ÐØÐr8   Úblock_lengthc                 óæ  — | j         j        j        j        dk    r| j         j        j        nd}t	          j        d|z  t          j        |¬¦  «        }||| …         }|ddd…f         |dd…df         z
  }|                      || j         | j	        | j
        ¬¦  «        }|                       |¦  «        }|                     g d¢¦  «                             d¦  «                             d¦  «        }|S ©r  ÚmetaNr   r:   r	  r
  r   ©rî   rœ   r;   ri   r/   rO   rö   r  rÞ   rà   rá   r  rQ   ©rž   rE  Útarget_devicer  r  rþ   r  rz   s           r6   r  z!LongT5LocalAttention.compute_bias8  ó  € ð Ô+Ô2Ô9Ô>À&ÒHÐHð Ô(Ô/Ô6Ð6àð 	õ
  œ, q¨<Ñ'7½u¼zÐR_Ð`Ñ`Ô`ˆØ*¨<¸¸Ð+EÔFÐð ,¨D°!°!°!¨GÔ4Ð7GÈÈÈÈ4ÈÔ7PÑPÐØ#'×#AÒ#AØØ#œÐ.ØÔ;ØÔ=ð	 $Bñ $
ô $
Ð ð ×-Ò-Ð.FÑGÔGˆà—’ 	 	 	Ñ*Ô*×4Ò4°QÑ7Ô7×AÒAÀ!ÑDÔDˆØˆr8   c                 ó  ‡ ‡— |j         d d…         \  Š}ˆˆ fd„}ˆˆ fd„} |‰                      |¦  «        ¦  «        } |‰                      |¦  «        ¦  «        }	 |‰                      |¦  «        ¦  «        }
t	          |‰ j        d¬¦  «        }t	          |	‰ j        d¬¦  «        }	t	          |
‰ j        d¬¦  «        }
t          |	dd¬¦  «        }	t          |
dd¬¦  «        }
t          j        d||	¦  «        }|€¤‰ j	        sNt          j
        dd‰ j        ‰ j        d‰ j        z  f|j        |j        ¬	¦  «        }‰ j        r‰ j        rd
|_        n‰                      ‰ j        ¦  «        }|�3t          j        |dk    dd¦  «        }||                     dd¦  «        z   }||z  }t(          j                             |                     ¦   «         d¬¦  «                             |¦  «        }t(          j                             |‰ j        ‰ j        ¬¦  «        }|                     |
j        ¦  «        } |t          j        d||
¦  «        ¦  «        }|d d …d |…d d …f         }‰                      |¦  «        }||f}|r||fz   }|S )Nr_   c                 óH   •— |                       ‰d‰j        ‰j        ¦  «        S ©Ú
projectionr%   ©r  rå   rã   ©Ústatesr   rž   s    €€r6   r,   z+LongT5LocalAttention.forward.<locals>.shapeY  ó    ø€ à—;’;˜z¨2¨t¬|¸TÔ=TÑUÔUÐUr8   c                 ó`   •— |                       ¦   «                              ‰d‰j        ¦  «        S ©r=   r%   ©r&  r  ræ   rR  s    €€r6   Úunshapez-LongT5LocalAttention.forward.<locals>.unshape]  ó)   ø€ à×$Ò$Ñ&Ô&×+Ò+¨J¸¸D¼NÑKÔKÐKr8   r   rD   r`   ú...qhd,...khd->...hqkr   r  Tr   rr   ç    _ Âr%   r  ú...hqk,...khd->...qhd)r,   ré   rê   rë   r@   r   rM   r/   r’   rß   r0   rå   r;   r#   rï   r  r#  r  rj   r|   r   r3   r$  rú   r%  r¿   ri   rì   )rž   r�   r�   r(  r*  Ú
seq_lengthr,   rX  r/  r2  r3  r5  r8  r9  r:  r   s   `              @r6   r«   zLongT5LocalAttention.forwardP  sâ  øø€ ð "/Ô!4°R°a°RÔ!8Ñˆ
�Jð	Vð 	Vð 	Vð 	Vð 	Vð 	Vð	Lð 	Lð 	Lð 	Lð 	Lð 	Lð
 �u˜TŸVšV MÑ2Ô2Ñ3Ô3ˆØ�U˜4Ÿ6š6 -Ñ0Ô0Ñ1Ô1ˆ
Ø�u˜TŸVšV MÑ2Ô2Ñ3Ô3ˆõ *¨,¸¼ÈAÐNÑNÔNˆÝ'¨
°D´NÈÐJÑJÔJˆ
Ý)¨,¸¼ÈAÐNÑNÔNˆõ +¨:ÀÐQRÐSÑSÔSˆ
Ý,¨\ÀQÐUVÐWÑWÔWˆõ ”Ø# \°:ñ
ô 
ˆð Ð àÔ3ð BÝ %¤Ø˜˜4œ<¨¬¸¸T¼^Ñ9KÐLÐU[ÔUbÐjpÔjvð!ñ !ô !�ð Ô.ð 7°4´=ð 7Ø26�MÔ/øà $× 1Ò 1°$´.Ñ AÔ A�àÐå”{ 4¨!¢8¨S°%Ñ8Ô8�à -°·²¸qÀ!Ñ0DÔ0DÑ D�à�-Ñˆå”}×,Ò,¨V¯\ª\©^¬^ÀÐ,ÑDÔD×LÒLÈVÑTÔTˆå”}×,Ò,¨\¸T¼\ÐTXÔTaÐ,ÑbÔbˆà#×(Ò(¨Ô);Ñ<Ô<ˆØ�g�eœlÐ+BÀLÐR^Ñ_Ô_Ñ`Ô`ˆØ! ! ! ! [ j [°!°!°!Ð"3Ô4ˆØ—f’f˜[Ñ)Ô)ˆð Øð
ˆð
 ð 	0Ø  Ñ/ˆGØˆr8   ©Fr<  ©NNF)r¬   r­   r®   r   Úboolrš   r=  r  r~   r  r«   r¯   r°   s   @r6   r?  r?  î  s»   ø€ € € € € ð,ð ,˜|ð ,È$ð ,Ð[_ð ,ð ,ð ,ð ,ð ,ð ,ð0 ð- ð - ð - ñ „\ð- ð^¨ð ð ð ð ð6 ØØðGð Gð Gð Gð Gð Gð Gð Gr8   r?  c                   ó’   ‡ — e Zd Zddededdfˆ fd„Zedd
„¦   «         Zdefd„Z	de
j        de
j        de
j        fd„Z	 	 	 dd„Zˆ xZS )ÚLongT5TransientGlobalAttentionFr³   rß   r    Nc                 ó�  •— t          ¦   «                              ¦   «          |j        | _        || _        |j        | _        |j        | _        |j        | _        |j        | _        |j	        | _
        |j        | _        | j        dz   | _        |j        | _        |j        | _        | j
        | j        z  | _        t#          j        | j        | j        d¬¦  «        | _        t#          j        | j        | j        d¬¦  «        | _        t#          j        | j        | j        d¬¦  «        | _        t#          j        | j        | j        d¬¦  «        | _        | j        r$t#          j        | j        | j
        ¦  «        | _        | j        r$t#          j        | j        | j
        ¦  «        | _        t5          |j        |j        ¬¦  «        | _        d S )Nr   Fr¶   rÓ   )r™   rš   rÞ   rß   rà   rá   r¹   râ   rã   rä   rå   rA  r   re   r¾   r¿   ræ   r   r¸   ré   rê   rë   rì   rí   rî   Úglobal_relative_attention_biasr–   rÖ   Úglobal_input_layer_normrB  s      €r6   rš   z'LongT5TransientGlobalAttention.__init__›  s~  ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒØ+FˆÔ(Ø.4Ô.SˆÔ+Ø/5Ô/UˆÔ,Ø”~ˆŒØ"(¤+ˆÔØÔ'ˆŒØ"Ô/ˆÔØÔ*¨QÑ.ˆŒØ!'Ô!9ˆÔØÔ*ˆŒØœ¨Ô(?Ñ?ˆŒå”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ<¨¬¸eÐDÑDÔDˆŒÝ”˜4œ>¨4¬<¸eÐDÑDÔDˆŒàÔ+ð 	kÝ+-¬<¸Ô8[Ð]aÔ]iÑ+jÔ+jˆDÔ(ð Ô+ð 	rÝ24´,¸tÔ?bÐdhÔdpÑ2qÔ2qˆDÔ/Ý'6°v´~È6ÔKdÐ'eÑ'eÔ'eˆÔ$Ð$Ð$r8   Trñ   rò   c                 óP  — d}|rC|dz  }|| dk                          t          j        ¦  «        |z  z  }t          j        | ¦  «        } n(t          j        | t          j        | ¦  «        ¦  «         } |dz  }| |k     }|t          j        |                      ¦   «         |z  ¦  «        t          j        ||z  ¦  «        z  ||z
  z                        t          j        ¦  «        z   }t          j        |t          j	        ||dz
  ¦  «        ¦  «        }|t          j
        || |¦  «        z  }|S rô   rõ   rý   s           r6   r  z8LongT5TransientGlobalAttention._relative_position_bucket·  rD  r8   rE  c                 óæ  — | j         j        j        j        dk    r| j         j        j        nd}t	          j        d|z  t          j        |¬¦  «        }||| …         }|ddd…f         |dd…df         z
  }|                      || j         | j	        | j
        ¬¦  «        }|                       |¦  «        }|                     g d¢¦  «                             d¦  «                             d¦  «        }|S rG  rI  rJ  s           r6   r  z+LongT5TransientGlobalAttention.compute_biasè  rL  r8   r�   r†   c                 ó~  — t          j        |d         |d d …d d d …f         ¦  «        d d …d df         }t          j        |dk    dd¦  «        }t          || j        ¦  «        }|                      || j         | j        | j        ¬¦  «        }|  	                    |¦  «        }| 
                    g d¢¦  «        }||z   }|S )Nr‰   .r   rr   r[  r	  )r   r   r   r_   )r/   Úeqrj   rŽ   re   r  rÞ   rà   rá   rd  r  )rž   r�   r†   Úside_attention_maskÚattention_side_biasr�   Úside_relative_position_bucketÚ	side_biass           r6   Úcompute_side_biasz0LongT5TransientGlobalAttention.compute_side_bias   sä   € å#œh t¨I¤Ð8JÈ1È1È1ÈdÐTUÐTUÐTUÈ:Ô8VÑWÔWÐXYÐXYÐXYÐ[_ÐadÐXdÔeÐÝ#œkÐ*=ÀÒ*AÀ3ÈÑNÔNÐå!AÀ$ÈÔH^Ñ!_Ô!_ÐØ(,×(FÒ(FØ"Ø#œÐ.ØÔ;ØÔ=ð	 )Gñ )
ô )
Ð%ð ×7Ò7Ð8UÑVÔVˆ	ð ×%Ò% l l lÑ3Ô3ˆ	à1°IÑ=ÐØ"Ð"r8   c                 óR	  ‡ ‡— |j         d d…         \  Š}ˆˆ fd„}ˆˆ fd„}t          |�|n t          j        |j         d d…         ¦  «        ‰ j        ¦  «        \  }}	|	j         d         }
t          |||
¦  «        }‰                      |¦  «        } |‰                      |¦  «        ¦  «        } |‰                      |¦  «        ¦  «        } |‰  	                    |¦  «        ¦  «        } |‰                      |¦  «        ¦  «        } |‰  	                    |¦  «        ¦  «        }t          |‰ j        d¬¦  «        }t          |‰ j        d¬¦  «        }t          |‰ j        d¬¦  «        }t          |dd¬¦  «        }t          |dd¬¦  «        }dg|j        dz   z  }|j         d         |d<   |                     d¦  «                             |¦  «        }|                     d¦  «                             |¦  «        }t          j        ||gd¬¦  «        }t          j        ||gd¬¦  «        }t          j        d||¦  «        }|�6t%          |‰ j        |j        ¦  «        }t          j        |d	k    d
d¦  «        }nd }|�€F‰ j        sNt          j        dd‰ j        ‰ j        d‰ j        z  f|j        |j        ¬¦  «        }‰ j        r‰ j        rd|_        n‰                      ‰ j        ¦  «        }|�||                     dd¦  «        z   }|                     |j        ¦  «        }|€t          j        ‰|¦  «        }‰                      ||	¦  «        }t          |‰ j        d¬¦  «                             dd¦  «        }|                     |j        ¦  «                              |j        ¦  «        }t          j        ||gd¬¦  «        }||z  }tB          j"         #                    | $                    ¦   «         d¬¦  «         %                    |¦  «        }tB          j"         &                    |‰ j&        ‰ j        ¬¦  «        }|                     |j        ¦  «        } |t          j        d||¦  «        ¦  «        }|d d …d |…d d …f         }‰  '                    |¦  «        }||f}|r||fz   }|S )Nr_   c                 óH   •— |                       ‰d‰j        ‰j        ¦  «        S rO  rQ  rR  s    €€r6   r,   z5LongT5TransientGlobalAttention.forward.<locals>.shape  rT  r8   c                 ó`   •— |                       ¦   «                              ‰d‰j        ¦  «        S rV  rW  rR  s    €€r6   rX  z7LongT5TransientGlobalAttention.forward.<locals>.unshape"  rY  r8   r%   r   rD   r`   rZ  r   rr   r[  r   r  Tra   r  r\  )(r,   r‡   r/   r}   re   r”   re  ré   rê   rë   r@   r   rM   r1   rQ   r{   rI   r’   rd   r;   rj   rß   r0   rå   r#   rï   r  r#  r  r|   ri   rn  rY   r   r3   r$  rú   r%  r¿   rì   )rž   r�   r�   r(  r*  r]  r,   rX  rg   r†   Ú_global_seq_lenÚglobal_inputsr/  r2  r3  Úside_key_statesÚside_value_statesÚrepsr5  rV   Úside_position_biasr8  r9  r:  r   s   `                       @r6   r«   z&LongT5TransientGlobalAttention.forward  sù  øø€ ð "/Ô!4°R°a°RÔ!8Ñˆ
�Jð	Vð 	Vð 	Vð 	Vð 	Vð 	Vð	Lð 	Lð 	Lð 	Lð 	Lð 	Lõ )EØÐ$ˆDˆD­%¬*°]Ô5HÈÈ"ÈÔ5MÑ*NÔ*NØÔ"ñ)
ô )
Ñ%ˆ	Ð%ð
 -Ô2°2Ô6ˆÝ1°-ÀÈOÑ\Ô\ˆØ×4Ò4°]ÑCÔCˆð �u˜TŸVšV MÑ2Ô2Ñ3Ô3ˆØ�U˜4Ÿ6š6 -Ñ0Ô0Ñ1Ô1ˆ
Ø�u˜TŸVšV MÑ2Ô2Ñ3Ô3ˆà˜% §¢ }Ñ 5Ô 5Ñ6Ô6ˆØ!˜E $§&¢&¨Ñ"7Ô"7Ñ8Ô8Ðõ *¨,¸¼ÈAÐNÑNÔNˆÝ'¨
°D´NÈÐJÑJÔJˆ
Ý)¨,¸¼ÈAÐNÑNÔNˆõ +¨:ÀÐQRÐSÑSÔSˆ
Ý,¨\ÀQÐUVÐWÑWÔWˆð ˆs�oÔ*¨QÑ.Ñ/ˆØÔ" 1Ô%ˆˆQ‰Ø)×3Ò3°AÑ6Ô6×=Ò=¸dÑCÔCˆØ-×7Ò7¸Ñ:Ô:×AÒAÀ$ÑGÔGÐõ ”Y 
¨OÐ<À!ÐDÑDÔDˆ
Ý”y ,Ð0AÐ!BÈÐJÑJÔJˆõ ”Ð5°|ÀZÑPÔPˆàÐå#<¸TÀ4Ä>ÐS`ÔSgÑ#hÔ#hÐ å#(¤;Ð/CÀaÒ/GÈÈeÑ#TÔ#TÐ Ð à#'Ð àÑ àÔ3ð 	BÝ %¤Ø˜˜4œ<¨¬¸¸T¼^Ñ9KÐLØ!œ=Ø œ,ð!ñ !ô !�ð
 Ô.ð 7°4´=ð 7Ø26�MÔ/øà $× 1Ò 1°$´.Ñ AÔ A�à#Ð/à -Ð0D×0NÒ0NÈqÐRSÑ0TÔ0TÑ T�Ø)×.Ò.¨v¬|Ñ<Ô<ˆMð ˆ|Ý”z *¨jÑ9Ô9�à!%×!7Ò!7¸Ð>PÑ!QÔ!QÐå!3Ð4FÈÌÐ\^Ð!_Ñ!_Ô!_×!iÒ!iÐjkÐmnÑ!oÔ!oÐØ!3×!8Ò!8¸¼Ñ!FÔ!F×!IÒ!IÈ&Ì-Ñ!XÔ!XÐå!œI }Ð6HÐ&IÈrÐRÑRÔRˆMà�-Ñˆå”}×,Ò,¨V¯\ª\©^¬^ÀÐ,ÑDÔD×LÒLÈVÑTÔTˆÝ”}×,Ò,¨\¸T¼\ÐTXÔTaÐ,ÑbÔbˆà#×(Ò(¨Ô);Ñ<Ô<ˆØ�g�eœlÐ+BÀLÐR^Ñ_Ô_Ñ`Ô`ˆØ! ! ! ! [ j [°!°!°!Ð"3Ô4ˆØ—f’f˜[Ñ)Ô)ˆà Ð.ˆàð 	0Ø  Ñ/ˆGØˆr8   r^  r<  r_  )r¬   r­   r®   r   r`  rš   r=  r  r~   r  r/   rt   rn  r«   r¯   r°   s   @r6   rb  rb  š  sö   ø€ € € € € ðfð f˜|ð fÈ$ð fÐ[_ð fð fð fð fð fð fð8 ð- ð - ð - ñ „\ð- ð^¨ð ð ð ð ð0# e¤lð #ÈÌð #ÐY^ÔYeð #ð #ð #ð #ð0 ØØðqð qð qð qð qð qð qð qr8   rb  c                   ó>   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 	 	 dd„Zˆ xZS )ÚLongT5LayerSelfAttentionFNrÜ   c                 óò   •— t          ¦   «                              ¦   «          t          |||¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S )N©rß   rÜ   rÓ   )r™   rš   rÛ   ÚSelfAttentionr–   r¹   rÖ   r×   r   r½   r¾   r¿   rð   s       €r6   rš   z!LongT5LayerSelfAttention.__init__‹  sl   ø€ Ý‰Œ×ÒÑÔÐÝ,ØÐ0KÐW`ð
ñ 
ô 
ˆÔõ *¨&¬.¸fÔ>WÐXÑXÔXˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr8   c                 ó¾   — |                       |¦  «        }|                      ||||||¬¦  «        }	||                      |	d         ¦  «        z   }|f|	dd …         z   }
|
S )N)r�   r(  r)  Ú	use_cacher*  r   r   )r×   r|  r¿   )rž   r�   r]   r(  r)  r~  r*  r+  Únormed_hidden_statesÚattention_outputr:  s              r6   r«   z LongT5LayerSelfAttention.forward“  s}   € ð  $Ÿš¨}Ñ=Ô=ÐØ×-Ò-Ø ØØ'Ø+ØØ/ð .ñ 
ô 
Ðð &¨¯ªÐ5EÀaÔ5HÑ(IÔ(IÑIˆØ Ð"Ð%5°a°b°bÔ%9Ñ9ˆØˆr8   r;  )NNNFF©r¬   r­   r®   r~   rš   r«   r¯   r°   s   @r6   ry  ry  Š  ss   ø€ € € € € ð7ð 7ÈSÐSWÉZð 7ð 7ð 7ð 7ð 7ð 7ð ØØØØðð ð ð ð ð ð ð r8   ry  c                   óD   ‡ — e Zd ZdZddedz  fˆ fd„Z	 	 	 d	defd„Zˆ xZS )
ÚLongT5LayerLocalSelfAttentionz$Local self attention used in encoderFNrÜ   c                 óð   •— t          ¦   «                              ¦   «          t          ||¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S ©N)rß   rÓ   )r™   rš   r?  ÚLocalSelfAttentionr–   r¹   rÖ   r×   r   r½   r¾   r¿   rð   s       €r6   rš   z&LongT5LayerLocalSelfAttention.__init__®  s`   ø€ Ý‰Œ×ÒÑÔÐÝ"6°vÐ[vÐ"wÑ"wÔ"wˆÔÝ)¨&¬.¸fÔ>WÐXÑXÔXˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr8   r+  c                 óº   — |                       |¦  «        }|                      ||||¬¦  «        }||                      |d         ¦  «        z   }|f|dd …         z   }|S ©N)r�   r(  r*  r   r   )r×   r†  r¿   ©	rž   r�   r]   r(  r*  r+  r  r€  r:  s	            r6   r«   z%LongT5LayerLocalSelfAttention.forward´  sw   € ð  $Ÿš¨}Ñ=Ô=ÐØ×2Ò2Ø ØØ'Ø/ð	 3ñ 
ô 
Ðð &¨¯ªÐ5EÀaÔ5HÑ(IÔ(IÑIˆØ Ð"Ð%5°a°b°bÔ%9Ñ9ˆØˆr8   r;  r_  ©	r¬   r­   r®   Ú__doc__r~   rš   r   r«   r¯   r°   s   @r6   rƒ  rƒ  «  s‚   ø€ € € € € Ø.Ð.ð7ð 7ÈSÐSWÉZð 7ð 7ð 7ð 7ð 7ð 7ð ØØðð ð ðð ð ð ð ð ð ð r8   rƒ  c                   óD   ‡ — e Zd ZdZddedz  fˆ fd„Z	 	 	 d	defd„Zˆ xZS )
Ú'LongT5LayerTransientGlobalSelfAttentionz/Transient-Global self attention used in encoderFNrÜ   c                 óð   •— t          ¦   «                              ¦   «          t          ||¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S r…  )r™   rš   rb  ÚTransientGlobalSelfAttentionr–   r¹   rÖ   r×   r   r½   r¾   r¿   rð   s       €r6   rš   z0LongT5LayerTransientGlobalSelfAttention.__init__Ë  si   ø€ Ý‰Œ×ÒÑÔÐÝ,JØÐ0Kð-
ñ -
ô -
ˆÔ)õ *¨&¬.¸fÔ>WÐXÑXÔXˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr8   r+  c                 óº   — |                       |¦  «        }|                      ||||¬¦  «        }||                      |d         ¦  «        z   }|f|dd …         z   }|S rˆ  )r×   r�  r¿   r‰  s	            r6   r«   z/LongT5LayerTransientGlobalSelfAttention.forwardÓ  sw   € ð  $Ÿš¨}Ñ=Ô=ÐØ×<Ò<Ø ØØ'Ø/ð	 =ñ 
ô 
Ðð &¨¯ªÐ5EÀaÔ5HÑ(IÔ(IÑIˆØ Ð"Ð%5°a°b°bÔ%9Ñ9ˆØˆr8   r;  r_  rŠ  r°   s   @r6   r�  r�  È  s‚   ø€ € € € € Ø9Ð9ð7ð 7ÈSÐSWÉZð 7ð 7ð 7ð 7ð 7ð 7ð ØØðð ð ðð ð ð ð ð ð ð r8   r�  c                   ó<   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 	 dd„Zˆ xZS )ÚLongT5LayerCrossAttentionNrÜ   c                 óò   •— t          ¦   «                              ¦   «          t          |d|¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S )NFr{  rÓ   )r™   rš   rÛ   ÚEncDecAttentionr–   r¹   rÖ   r×   r   r½   r¾   r¿   )rž   r³   rÜ   r¡   s      €r6   rš   z"LongT5LayerCrossAttention.__init__é  sc   ø€ Ý‰Œ×ÒÑÔÐÝ.¨vÐSXÐdmÐnÑnÔnˆÔÝ)¨&¬.¸fÔ>WÐXÑXÔXˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr8   Fc                 ó¾   — |                       |¦  «        }|                      ||||||¬¦  «        }	||                      |	d         ¦  «        z   }
|
f|	dd …         z   }|S )N)r�   r'  r(  r)  r*  r   r   )r×   r”  r¿   )rž   r�   r'  r]   r(  r)  r*  r+  r  r€  Úlayer_outputr:  s               r6   r«   z!LongT5LayerCrossAttention.forwardï  s|   € ð  $Ÿš¨}Ñ=Ô=ÐØ×/Ò/Ø ØØ-Ø'Ø+Ø/ð 0ñ 
ô 
Ðð % t§|¢|Ð4DÀQÔ4GÑ'HÔ'HÑHˆØ�/Ð$4°Q°R°RÔ$8Ñ8ˆØˆr8   rÄ   )NNNFr�  r°   s   @r6   r’  r’  è  so   ø€ € € € € ð7ð 7¨#°©*ð 7ð 7ð 7ð 7ð 7ð 7ð ØØØðð ð ð ð ð ð ð r8   r’  c                   óF   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 	 	 	 	 	 	 dd„Zˆ xZS )	ÚLongT5BlockFNrÜ   c                 ó$  •— t          ¦   «                              ¦   «          |j        | _        |j        rt          }n>|j        dk    rt
          }n+|j        dk    rt          }nt          d|j        › d�¦  «        ‚t          j	        ¦   «         | _
        | j
                              ||||¬¦  «        ¦  «         | j        r)| j
                             t          ||¬¦  «        ¦  «         | j
                             t          |¦  «        ¦  «         d S )NÚlocalztransient-globalzjFor encoder attention mechanism, either `local` or `transient-global` attention type is expected, but got ú.r{  )rÜ   )r™   rš   rÞ   ry  Úencoder_attention_typerƒ  r�  Ú
ValueErrorr   Ú
ModuleListÚlayerrH   r’  rÑ   )rž   r³   rß   rÜ   Úattention_layerr¡   s        €r6   rš   zLongT5Block.__init__  s   ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒØÔð 
	Ý6ˆOˆOØÔ*¨gÒ5Ð5Ý;ˆOˆOØÔ*Ð.@Ò@Ð@ÝEˆOˆOåð<Ø!Ô8ð<ð <ð <ñô ð õ ”]‘_”_ˆŒ
ØŒ
×ÒØˆO˜FÐ@[ÐgpÐqÑqÔqñ	
ô 	
ð 	
ð Œ?ð 	VØŒJ×ÒÕ7¸È)ÐTÑTÔTÑUÔUÐUàŒ
×Ò�-¨Ñ/Ô/Ñ0Ô0Ð0Ð0Ð0r8   Tc                 óÊ  —  | j         d         ||||||	¬¦  «        }|d         }|dd …         }|j        t          j        k    r_t          j        |¦  «                             ¦   «         r9t          j        |j        ¦  «        j        dz
  }t          j        || |¬¦  «        }| j	        o|d u}|r¥ | j         d         ||||||	¬¦  «        }|d         }|j        t          j        k    r_t          j        |¦  «                             ¦   «         r9t          j        |j        ¦  «        j        dz
  }t          j        || |¬¦  «        }||dd …         z   } | j         d         |¦  «        }|j        t          j        k    r_t          j        |¦  «                             ¦   «         r9t          j        |j        ¦  «        j        dz
  }t          j        || |¬¦  «        }|f|z   S )Nr   )r]   r(  r)  r~  r*  r   iè  )r÷   ry   )r'  r]   r(  r)  r*  r%   )
rŸ  r#   r/   r¨   ÚisinfÚanyÚfinfory   ÚclamprÞ   )rž   r�   r]   r(  Úencoder_hidden_statesÚencoder_attention_maskÚencoder_decoder_position_biasr)  r~  r*  Úreturn_dictr+  Úself_attention_outputsÚattention_outputsÚclamp_valueÚdo_cross_attentionÚcross_attention_outputss                    r6   r«   zLongT5Block.forward  s  € ð "/ ¤¨A¤ØØ)Ø'Ø+ØØ/ð"
ñ "
ô "
Ðð /¨qÔ1ˆØ2°1°2°2Ô6Ðð Ô¥%¤-Ò/Ð/µE´KÀÑ4NÔ4N×4RÒ4RÑ4TÔ4TÐ/Ýœ+ mÔ&9Ñ:Ô:Ô>ÀÑEˆKÝ!œK¨¸K¸<È[ÐYÑYÔYˆMà!œ_ÐRÐ1FÈdÐ1RÐØð 	PØ&3 d¤j°¤mØØ!6Ø5Ø;Ø /Ø"3ð'ñ 'ô 'Ð#ð 4°AÔ6ˆMð Ô"¥e¤mÒ3Ð3½¼ÀMÑ8RÔ8R×8VÒ8VÑ8XÔ8XÐ3Ý#œk¨-Ô*=Ñ>Ô>ÔBÀTÑI�Ý %¤¨MÀ¸|ÐQ\Ð ]Ñ ]Ô ]�ð !2Ð4KÈAÈBÈBÔ4OÑ OÐð '˜œ
 2œ }Ñ5Ô5ˆð Ô¥%¤-Ò/Ð/µE´KÀÑ4NÔ4N×4RÒ4RÑ4TÔ4TÐ/Ýœ+ mÔ&9Ñ:Ô:Ô>ÀÑEˆKÝ!œK¨¸K¸<È[ÐYÑYÔYˆMð ÐÐ0Ñ0ð	
r8   r;  )	NNNNNNFFTr�  r°   s   @r6   r˜  r˜    s   ø€ € € € € ð1ð 1ÈSÐSWÉZð 1ð 1ð 1ð 1ð 1ð 1ð4 ØØ"Ø#Ø&*ØØØØð<
ð <
ð <
ð <
ð <
ð <
ð <
ð <
r8   r˜  c                   ó‚   ‡ — e Zd ZU eed<   dZdZdgZdZe	d„ ¦   «         Z
 ej        ¦   «         ˆ fd„¦   «         Zd„ Zˆ xZS )	ÚLongT5PreTrainedModelr³   ÚtransformerTr˜  Fc                 óv   — t          j        t          ¦  «        }t          j        t          ¦  «        }|||dœ}|S )N)Údecoder_input_idsÚ	input_idsÚdecoder_attention_mask)r/   rx   r   r   )rž   r´  Ú
input_maskÚdummy_inputss       r6   r·  z"LongT5PreTrainedModel.dummy_inputsg  s?   € õ ”L¥Ñ.Ô.ˆ	Ý”\¥*Ñ-Ô-ˆ
à!*Ø"Ø&0ð
ð 
ˆð
 Ðr8   c                 ób	  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          ¦  «        rt          j        |j        |dz  ¦  «         dS t	          |t          t          t          f¦  «        rjt          j        |j        j        d|dz  ¬¦  «         t          |d¦  «        r2| j        j        s(t          j        |j        j        d|dz  ¬¦  «         dS dS dS t	          |t"          ¦  «        ræt          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         dS dS dS t	          |t0          ¦  «        �rVt          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r*|j        j        �t          j        |j        j        ¦  «         t          j        |j        j        d|| j        j        dz  z  ¬¦  «         t          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         dS dS dS t	          |t6          t8          t:          f¦  «        �r2| j        j        }| j        j        }| j        j        }t          j        |j         j        d|||z  dz  z  ¬¦  «         t          j        |j!        j        d||dz  z  ¬¦  «         t          j        |j"        j        d||dz  z  ¬¦  «         t          j        |j#        j        d|||z  dz  z  ¬¦  «         |j$        rgt          j        |j%        j        d||dz  z  ¬¦  «         t	          |t:          ¦  «        r-t          j        |j&        j        d||dz  z  ¬¦  «         dS dS dS dS )zInitialize the weightsrs   rr   )r¦   ÚstdÚlm_headç      à¿r·   N)'r™   Ú_init_weightsr³   Úinitializer_factorrÅ   r–   ÚinitÚ	constant_rœ   ÚLongT5ModelÚLongT5ForConditionalGenerationÚLongT5EncoderModelÚnormal_ÚsharedÚhasattrÚtie_word_embeddingsrº  r²   r»   r¹   r·   Úzeros_r¼   rº   rÉ   rË   rÌ   rÛ   r?  rb  râ   rä   ré   rê   rë   rì   rß   rî   rd  )rž   ÚmoduleÚfactorr¹   rã   rå   r¡   s         €r6   r¼  z#LongT5PreTrainedModel._init_weightss  s”  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ø”Ô/ˆÝ�f�oÑ.Ô.ð $	ÝŒN˜6œ=¨&°3©,Ñ7Ô7Ð7Ð7Ð7Ý˜¥Õ.LÕN`Ð aÑbÔbð "	ÝŒL˜œÔ-°C¸VÀc¹\ÐJÑJÔJÐJÝ�v˜yÑ)Ô)ð P°$´+Ô2Qð PÝ”˜Vœ^Ô2¸À&È3Á,ÐOÑOÔOÐOÐOÐOðPð Pð Pð På˜Õ 3Ñ4Ô4ð 	ÝŒL˜œÔ)°¸ÀDÄKÔDWÐ\`ÑC`Ñ9aÐbÑbÔbÐbÝ�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+ÝŒL˜œÔ)°¸ÀDÄKÔDTÐY]ÑC]Ñ9^Ð_Ñ_Ô_Ð_Ý�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+Ð+Ð+ð,ð ,Ð.HÐ.Hå˜Õ 8Ñ9Ô9ñ 	ÝŒL˜œÔ+°#¸6ÀdÄkÔFYÐ^bÑEbÑ;cÐdÑdÔdÐdÝ�v”{ FÑ+Ô+ð .°´Ô0@Ð0LÝ”˜FœKÔ,Ñ-Ô-Ð-ÝŒL˜œÔ+°#¸6ÀdÄkÔFYÐ^bÑEbÑ;cÐdÑdÔdÐdÝ�v”{ FÑ+Ô+ð .°´Ô0@Ð0LÝ”˜FœKÔ,Ñ-Ô-Ð-ÝŒL˜œÔ)°¸ÀDÄKÔDTÐY]ÑC]Ñ9^Ð_Ñ_Ô_Ð_Ý�v”y &Ñ)Ô)ð ,¨f¬i¬nÐ.HÝ”˜FœIœNÑ+Ô+Ð+Ð+Ð+ð,ð ,Ð.HÐ.Hå˜¥Õ2FÕHfÐ gÑhÔhñ 	Ø”kÔ)ˆGØ!%¤Ô!1ÐØ”kÔ+ˆGÝŒL˜œœ¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÐjÑjÔjÐjÝŒL˜œœ¨s¸À'È4Á-Ñ8PÐQÑQÔQÐQÝŒL˜œœ¨s¸À'È4Á-Ñ8PÐQÑQÔQÐQÝŒL˜œœ¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÐjÑjÔjÐjØÔ1ð Ý”˜VÔ;ÔBÈÐRXÐ]dÐimÑ\mÑRnÐoÑoÔoÐoÝ˜fÕ&DÑEÔEð Ý”LØÔ=ÔDÈ3ÐTZÐ_fÐkoÑ^oÑTpðñ ô ð ð ð ð	ð 	ðð ðð r8   c                 ó6  — | j         j        }| j         j        }|€t          d¦  «        ‚|                     |j        ¦  «        }|dd d…f                              ¦   «         |ddd …f<   ||d<   |€t          d¦  «        ‚|                     |dk    |¦  «         |S )Nz’self.model.config.decoder_start_token_id has to be defined. In LongT5 it is usually set to the pad_token_id. See LongT5 docs for more information..r%   r   ).r   z1self.model.config.pad_token_id has to be defined.éœÿÿÿ)r³   Údecoder_start_token_idÚpad_token_idr�  Ú	new_zerosr,   r  Úmasked_fill_)rž   r´  rÌ  rÍ  Úshifted_input_idss        r6   Ú_shift_rightz"LongT5PreTrainedModel._shift_rightŸ  sº   € Ø!%¤Ô!CÐØ”{Ô/ˆà!Ð)Ýð8ñô ð ð
 &×/Ò/°	´Ñ@Ô@ÐØ%.¨s°C°R°C¨xÔ%8×%>Ò%>Ñ%@Ô%@Ð˜#˜q˜r˜r˜'Ñ"Ø$:Ð˜&Ñ!àÐÝÐPÑQÔQÐQà×&Ò&Ð'8¸DÒ'@À,ÑOÔOÐOà Ð r8   )r¬   r­   r®   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_can_compile_fullgraphÚpropertyr·  r/   Úno_gradr¼  rÑ  r¯   r°   s   @r6   r°  r°  ^  s�   ø€ € € € € € àÐÐÑØ%ÐØ&*Ð#Ø&˜Ðà"Ðàðð ñ „Xðð €U„]�_„_ð(ð (ð (ð (ñ „_ð(ðV!ð !ð !ð !ð !ð !ð !r8   r°  c                   ó@   ‡ — e Zd Zˆ fd„Zd„ Z	 	 	 	 	 	 	 	 	 	 dd„Zˆ xZS )ÚLongT5Stackc                 ó  •‡— t          ¦   «                              ‰¦  «         t          j        ‰j        ‰j        ¦  «        | _        ‰j        | _        ‰j        | _        | j        dz   | _	        t          j
        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t          j        ‰j        ¦  «        | _        d| _        |                      ¦   «          d S )Nr   c           	      óV   •— g | ]%}t          ‰t          |d k    ¦  «        |¬¦  «        ‘Œ&S )r   r{  )r˜  r`  )Ú.0rK   r³   s     €r6   ú
<listcomp>z(LongT5Stack.__init__.<locals>.<listcomp>À  sC   ø€ ð ð ð àõ ˜FÅÀQÈ!ÂVÁÄÐXYÐZÑZÔZðð ð r8   rÓ   F)r™   rš   r   rí   Ú
vocab_sizer¹   Úembed_tokensrÞ   rA  r   rž  rE   Ú
num_layersÚblockr–   rÖ   Úfinal_layer_normr½   r¾   r¿   rï   Ú	post_initrÂ   s    `€r6   rš   zLongT5Stack.__init__¶  sí   øø€ Ý‰Œ×Ò˜Ñ Ô Ð åœL¨Ô):¸F¼NÑKÔKˆÔØ Ô+ˆŒà"Ô/ˆÔØÔ*¨QÑ.ˆŒå”]ðð ð ð å˜vÔ0Ñ1Ô1ðñ ô ñ
ô 
ˆŒ
õ !0°´ÀFÔD]Ð ^Ñ ^Ô ^ˆÔÝ”z &Ô"5Ñ6Ô6ˆŒà&+ˆÔ#ð 	�ŠÑÔÐÐÐr8   c                 ó   — || _         d S rÄ   )rà  ©rž   Únew_embeddingss     r6   Úset_input_embeddingsz LongT5Stack.set_input_embeddingsÎ  s   € Ø*ˆÔÐÐr8   Nc                 óÊ  — |�|n| j         j        }|�|n| j         j        }|	�|	n| j         j        }	|
�|
n| j         j        }
|�#|�!| j        rdnd}t          d|› d|› d�¦  «        ‚|�1|                     ¦   «         }|                     d|d         ¦  «        }n@|�|                     ¦   «         d d…         }n!| j        rdnd}t          d|› d|› d	�¦  «        ‚| j	        r%| j
        r|rt                               d
¦  «         d}|€&| j        €
J d¦   «         ‚|                      |¦  «        }|\  }}| j        r]|rZ|€X| j         j        r7t          t!          | j         ¬¦  «        t!          | j         ¬¦  «        ¦  «        }nt!          | j         ¬¦  «        }n	| j        sd }|�|                     ¦   «         nd}|€/t%          ¦   «         s!||z   }t'          j        |||j        ¬¦  «        }| j        rt-          | j         |||¬¦  «        }n.| j         j        dk    rt1          || j        |j        ¦  «        }n|}|�t5          | j         |||¬¦  «        }|	rdnd }|rdnd }|r	| j        rdnd }d }d }|                      |¦  «        }t9          | j        ¦  «        D ]h\  }}|	r||fz   } |||||||||||
¬¦
  «
        }|d         }|d         }| j        r|�||rdnd         }|r||d         fz   }| j        r||d         fz   }Œi|                      |¦  «        }|                      |¦  «        }|	r||fz   }|
st?          d„ |||||fD ¦   «         ¦  «        S tA          |||||¬¦  «        S )NÚdecoder_Ú zYou cannot specify both zinput_ids and zinputs_embeds at the same timer%   zYou have to specify either zinput_ids or Úinputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fz<You have to initialize the model with valid token embeddings)r³   r   rp   )r³   rì  r]   r)  rš  )r³   rì  r]   r¦  r&   )r)  r~  r*  r©  r   r   r_   é   c              3   ó   K  — | ]}|®|V — Œ	d S rÄ   r&   )rÝ  rë   s     r6   ú	<genexpr>z&LongT5Stack.forward.<locals>.<genexpr>[  s4   è è € ð 
ð 
àð �=ð ð !�=�=�=ð
ð 
r8   )Úlast_hidden_stater)  r�   Ú
attentionsÚcross_attentions)!r³   r~  r*  Úoutput_hidden_statesr©  rÞ   r�  Úsizer  rï   r  rç   rè   rà  Úis_encoder_decoderr   r
   r  r   r/   r}   r;   r   rœ  rd   r   r   r¿   Ú	enumeraterâ  rã  rG   r   )rž   r´  r]   r¦  r§  rì  r)  r~  r*  ró  r©  r+  Úerr_msg_prefixr,  r   r]  Úpast_key_values_lengthÚmask_seq_lengthr6  Úall_hidden_statesÚall_attentionsÚall_cross_attentionsr(  r¨  r�   rK   Úlayer_moduleÚlayer_outputss                               r6   r«   zLongT5Stack.forwardÑ  sÖ  € ð "+Ð!6�I�I¸D¼KÔ<Qˆ	Ø1BÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàÐ  ]Ð%>Ø+/¬?ÐB˜Z˜ZÀˆNÝØw¨>ÐwÐwÈÐwÐwÐwñô ð ð Ð"Ø#Ÿ.š.Ñ*Ô*ˆKØ!Ÿš r¨;°r¬?Ñ;Ô;ˆIˆIØÐ&Ø'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKˆKà+/¬?ÐB˜Z˜ZÀˆNÝÐu¸>ÐuÐuÐXfÐuÐuÐuÑvÔvÐvàÔ&ð 	"¨4¬=ð 	"Øð "Ý×#Ò#Øpñô ð ð "�	àÐ ØÔ$Ð0Ð0Ð2pÑ0Ô0Ð0Ø ×-Ò-¨iÑ8Ô8ˆMà!,Ñˆ
�JàŒ?ð 	#Øð G˜_Ð4Ø”;Ô1ð GÝ&9Ý$¨D¬KÐ8Ñ8Ô8½,ÈdÌkÐ:ZÑ:ZÔ:Zñ'ô '�O�Oõ '3¸$¼+Ð&FÑ&FÔ&F�OøØ”ð 	#ð #ˆOàETÐE` ×!?Ò!?Ñ!AÔ!AÐ!AÐfgÐØÐ!Õ*BÑ*DÔ*DÐ!à4°zÑAˆOÝ"œZ¨
°OÈMÔL`ÐaÑaÔaˆNàŒ?ð 	)Ý,Ø”{Ø+Ø-Ø /ð	ñ ô ˆKˆKð Œ[Ô/°7Ò:Ð:Ý3°NÀDÄNÐTaÔThÑiÔiˆKˆKà(ˆKà!Ð-Ý%>Ø”{Ø+Ø5Ø&;ð	&ñ &ô &Ð"ð #7Ð@˜B˜B¸DÐØ0Ð:˜˜°dˆØ&7ÐV¸D¼OÐV˜r˜rÐRVÐØˆØ(,Ð%àŸš ]Ñ3Ô3ˆå(¨¬Ñ4Ô4ð  	Vð  	V‰OˆAˆ|Ø#ð IØ$5¸Ð8HÑ$HÐ!à(˜LØØØØ%Ø&Ø-Ø /Ø#Ø"3Ø'ðñ ô ˆMð  *¨!Ô,ˆMð
 *¨!Ô,ˆMØŒð ]Ð#8Ð#DØ0=ÐCTÐ>[¸a¸aÐZ[Ô0\Ð-à ð VØ!/°=ÀÔ3CÐ2EÑ!E�Ø”?ð VØ+?À=ÐQRÔCSÐBUÑ+UÐ(øà×-Ò-¨mÑ<Ô<ˆØŸš ]Ñ3Ô3ˆð  ð 	EØ 1°]Ð4DÑ DÐàð 	Ýð 
ð 
ð "Ø#Ø%Ø"Ø(ðð
ñ 
ô 
ñ 
ô 
ð 
õ 9Ø+Ø+Ø+Ø%Ø1ð
ñ 
ô 
ð 	
r8   )
NNNNNNNNNN)r¬   r­   r®   rš   rè  r«   r¯   r°   s   @r6   rÚ  rÚ  µ  sƒ   ø€ € € € € ðð ð ð ð ð0+ð +ð +ð
 ØØ"Ø#ØØØØØ!Øð[
ð [
ð [
ð [
ð [
ð [
ð [
ð [
r8   rÚ  c                   óh  ‡ — e Zd ZdgZdddœZdefˆ fd„Zd„ Zd„ Ze		 	 	 	 	 	 	 	 	 	 	 	 dd	e
j        dz  d
e
j        dz  de
j        dz  de
j        dz  deee
j                          dz  dedz  de
j        dz  de
j        dz  dedz  dedz  dedz  dedz  dee
j                 ez  fd„¦   «         Zˆ xZS )rÀ  úFdecoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weightúshared.weight)úencoder.embed_tokens.weightúdecoder.embed_tokens.weightr³   c                 óœ  •— t          ¦   «                              |¦  «         t          j        |j        |j        ¦  «        | _        t          j        |¦  «        }d|_	        d|_
        t          |¦  «        | _        t          j        |¦  «        }d|_	        |j        |_        t          |¦  «        | _        |                      ¦   «          d S )NFT)r™   rš   r   rí   rß  r¹   rÄ  ÚcopyÚdeepcopyrÞ   r~  rÚ  ÚencoderÚnum_decoder_layersrá  Údecoderrä  ©rž   r³   Úencoder_configÚdecoder_configr¡   s       €r6   rš   zLongT5Model.__init__y  sª   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”l 6Ô#4°f´nÑEÔEˆŒåœ vÑ.Ô.ˆØ$)ˆÔ!Ø#(ˆÔ Ý" >Ñ2Ô2ˆŒåœ vÑ.Ô.ˆØ$(ˆÔ!Ø$*Ô$=ˆÔ!Ý" >Ñ2Ô2ˆŒð 	�ŠÑÔÐÐÐr8   c                 ó   — | j         S rÄ   ©rÄ  ©rž   s    r6   Úget_input_embeddingsz LongT5Model.get_input_embeddingsŠ  ó
   € ØŒ{Ðr8   c                 ó|   — || _         | j                             |¦  «         | j                             |¦  «         d S rÄ   ©rÄ  r  rè  r	  ræ  s     r6   rè  z LongT5Model.set_input_embeddings�  ó;   € Ø$ˆŒØŒ×)Ò)¨.Ñ9Ô9Ð9ØŒ×)Ò)¨.Ñ9Ô9Ð9Ð9Ð9r8   Nr´  r]   r³  rµ  Úencoder_outputsr)  rì  Údecoder_inputs_embedsr~  r*  ró  r©  r    c                 ó"  — |	�|	n| j         j        }	|�|n| j         j        }|€|                      ||||
||¬¦  «        }ne|rct	          |t
          ¦  «        sNt          |d         t          |¦  «        dk    r|d         ndt          |¦  «        dk    r|d         nd¬¦  «        }|d         }|                      |||||||	|
||¬¦
  «
        }|s||z   S t          |j	        |j
        |j        |j        |j        |j	        |j        |j        ¬¦  «        S )	aµ	  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5
            Training](./longt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            LONGT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If
            `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
            `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [LONGT5
            Training](./longt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, LongT5Model

        >>> tokenizer = AutoTokenizer.from_pretrained("google/long-t5-local-base")
        >>> model = LongT5Model.from_pretrained("google/long-t5-local-base")

        >>> # Let's try a very long encoder input.
        >>> input_ids = tokenizer(
        ...     100 * "Studies have been shown that owning a dog is good for you", return_tensors="pt"
        ... ).input_ids  # Batch size 1

        >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids  # Batch size 1

        >>> # forward pass
        >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
        >>> last_hidden_states = outputs.last_hidden_state
        ```N©r´  r]   rì  r*  ró  r©  r   r   r_   ©rð  r�   rñ  ©
r´  r]   rì  r)  r¦  r§  r~  r*  ró  r©  )rð  r)  Údecoder_hidden_statesÚdecoder_attentionsrò  Úencoder_last_hidden_stater¦  Úencoder_attentions)r³   r~  r©  r  rÅ   r   Úlenr	  r   rð  r)  r�   rñ  rò  )rž   r´  r]   r³  rµ  r  r)  rì  r  r~  r*  ró  r©  r+  r�   Údecoder_outputss                   r6   r«   zLongT5Model.forward’  su  € ðD "+Ð!6�I�I¸D¼KÔ<Qˆ	Ø%0Ð%<�k�kÀ$Ä+ÔBYˆð Ð"Ø"ŸlšlØ#Ø-Ø+Ø"3Ø%9Ø'ð +ñ ô ˆOˆOð ð 	¥¨O½_Ñ!MÔ!Mð 	Ý-Ø"1°!Ô"4Ý47¸Ñ4HÔ4HÈ1Ò4LÐ4L˜o¨aÔ0Ð0ÐRVÝ14°_Ñ1EÔ1EÈÒ1IÐ1I˜?¨1Ô-Ð-Ètðñ ô ˆOð (¨Ô*ˆð Ÿ,š,Ø'Ø1Ø/Ø+Ø"/Ø#1ØØ/Ø!5Ø#ð 'ñ 
ô 
ˆð ð 	5Ø" _Ñ4Ð4å!Ø-Ô?Ø+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð	
ñ 	
ô 	
ð 		
r8   )NNNNNNNNNNNN)r¬   r­   r®   Ú"_keys_to_ignore_on_load_unexpectedÚ_tied_weights_keysr   rš   r  rè  r   r/   Ú
LongTensorÚFloatTensorÚ
BoolTensorrG   r	   rt   r`  r   r«   r¯   r°   s   @r6   rÀ  rÀ  o  sÈ  ø€ € € € € ð 	Rð*Ð&ð (7Ø'6ðð Ðð
˜|ð ð ð ð ð ð ð"ð ð ð:ð :ð :ð
 ð .2Ø37Ø59Ø:>ØBFØ(,Ø-1Ø59Ø!%Ø)-Ø,0Ø#'ðq
ð q
àÔ# dÑ*ðq
ð Ô)¨DÑ0ðq
ð !Ô+¨dÑ2ð	q
ð
 !&Ô 0°4Ñ 7ðq
ð ˜u UÔ%6Ô7Ô8¸4Ñ?ðq
ð  ™ðq
ð ”| dÑ*ðq
ð  %œ|¨dÑ2ðq
ð ˜$‘;ðq
ð   $™;ðq
ð # T™kðq
ð ˜D‘[ðq
ð 
ˆuÔ Ô	!Ð$6Ñ	6ðq
ð q
ð q
ñ „^ðq
ð q
ð q
ð q
ð q
r8   rÀ  z>
    LONGT5 Model with a `language modeling` head on top.
    )Úcustom_introc                   ó–  ‡ — e Zd ZdgZddddœZdefˆ fd„Zd„ Zd„ Ze		 	 	 	 	 	 	 	 	 	 	 	 	 dd	e
j        dz  d
e
j        dz  de
j        dz  de
j        dz  deee
j                          dz  dedz  de
j        dz  de
j        dz  de
j        dz  dedz  dedz  dedz  dedz  dee
j                 ez  fd„¦   «         Zde
j        fd„Zˆ xZS )rÁ  r   r  )r  r  zlm_head.weightr³   c                 ó   •— t          ¦   «                              |¦  «         |j        | _        t	          j        |j        |j        ¦  «        | _        t          j	        |¦  «        }d|_
        d|_        t          |¦  «        | _        t          j	        |¦  «        }d|_
        |j        |_        t          |¦  «        | _        t	          j        |j        |j        d¬¦  «        | _        |                      ¦   «          d S )NFTr¶   )r™   rš   r¹   Ú	model_dimr   rí   rß  rÄ  r  r  rÞ   r~  rÚ  r  r  rá  r	  r¸   rº  rä  r
  s       €r6   rš   z'LongT5ForConditionalGeneration.__init__  sÐ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØœˆŒå”l 6Ô#4°f´nÑEÔEˆŒåœ vÑ.Ô.ˆØ$)ˆÔ!Ø#(ˆÔ Ý" >Ñ2Ô2ˆŒåœ vÑ.Ô.ˆØ$(ˆÔ!Ø$*Ô$=ˆÔ!Ý" >Ñ2Ô2ˆŒå”y ¤°Ô1BÈÐOÑOÔOˆŒð 	�ŠÑÔÐÐÐr8   c                 ó   — | j         S rÄ   r  r  s    r6   r  z3LongT5ForConditionalGeneration.get_input_embeddings+  r  r8   c                 ó|   — || _         | j                             |¦  «         | j                             |¦  «         d S rÄ   r  ræ  s     r6   rè  z3LongT5ForConditionalGeneration.set_input_embeddings.  r  r8   Nr´  r]   r³  rµ  r  r)  rì  r  Úlabelsr~  r*  ró  r©  r    c                 óÐ  — |
�|
n| j         j        }
|�|n| j         j        }|€|                      ||||||¬¦  «        }ne|rct	          |t
          ¦  «        sNt          |d         t          |¦  «        dk    r|d         ndt          |¦  «        dk    r|d         nd¬¦  «        }|d         }|	�|€|€|                      |	¦  «        }|                      |||||||
|||¬¦
  «
        }|d         }| j         j	        r|| j
        dz  z  }|                      |¦  «        }d}|	�pt          d	¬
¦  «        }|	                     |j        ¦  «        }	 ||                     d|                     d¦  «        ¦  «        |	                     d¦  «        ¦  «        }|s|f|dd…         z   |z   }|�|f|z   n|S t#          |||j        |j        |j        |j        |j        |j        |j        ¬¦	  «	        S )a7  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5
            Training](./longt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            LONGT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If
            `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
            `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [LONGT5
            Training](./longt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
            config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
            labels in `[0, ..., config.vocab_size]`

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, LongT5ForConditionalGeneration

        >>> tokenizer = AutoTokenizer.from_pretrained("Stancld/longt5-tglobal-large-16384-pubmed-3k_steps")
        >>> model = LongT5ForConditionalGeneration.from_pretrained(
        ...     "Stancld/longt5-tglobal-large-16384-pubmed-3k_steps"
        ... )

        >>> # Let's try a very long input.
        >>> inputs = tokenizer(100 * "studies have shown that owning a dog is good for you ", return_tensors="pt")
        >>> input_ids = inputs.input_ids

        >>> outputs = model.generate(input_ids)
        >>> print(tokenizer.decode(outputs[0], skip_special_tokens=True))
        abstractthe aim of this article is to provide an overview of the literature on the role of dog
        ```Nr  r   r   r_   r  r  r»  rË  )Úignore_indexr%   )	ÚlossÚlogitsr)  r  r  rò  r  r¦  r  )r³   r~  r©  r  rÅ   r   r  rÑ  r	  rÆ  r)  rº  r   rY   r;   r  rô  r   r)  r�   rñ  rò  rð  )rž   r´  r]   r³  rµ  r  r)  rì  r  r,  r~  r*  ró  r©  r+  r�   r   Úsequence_outputÚ	lm_logitsr/  Úloss_fctÚoutputs                         r6   r«   z&LongT5ForConditionalGeneration.forward3  sa  € ðL "+Ð!6�I�I¸D¼KÔ<Qˆ	Ø%0Ð%<�k�kÀ$Ä+ÔBYˆð Ð"à"ŸlšlØ#Ø-Ø+Ø"3Ø%9Ø'ð +ñ ô ˆOˆOð ð 	¥¨O½_Ñ!MÔ!Mð 	Ý-Ø"1°!Ô"4Ý47¸Ñ4HÔ4HÈ1Ò4LÐ4L˜o¨aÔ0Ð0ÐRVÝ14°_Ñ1EÔ1EÈÒ1IÐ1I˜?¨1Ô-Ð-Ètðñ ô ˆOð (¨Ô*ˆàÐÐ"3Ð";Ð@UÐ@]à $× 1Ò 1°&Ñ 9Ô 9Ðð Ÿ,š,Ø'Ø1Ø/Ø+Ø"/Ø#1ØØ/Ø!5Ø#ð 'ñ 
ô 
ˆð *¨!Ô,ˆàŒ;Ô*ð 	GØ-°´ÀÑ1EÑFˆOà—L’L Ñ1Ô1ˆ	àˆØÐÝ'°TÐ:Ñ:Ô:ˆHà—Y’Y˜yÔ/Ñ0Ô0ˆFØ�8˜IŸNšN¨2¨y¯~ª~¸bÑ/AÔ/AÑBÔBÀFÇKÂKÐPRÁOÄOÑTÔTˆDàð 	FØ�\ O°A°B°BÔ$7Ñ7¸/ÑIˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEåØØØ+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð

ñ 

ô 

ð 
	
r8   c                 ó,   — |                       |¦  «        S rÄ   )rÑ  )rž   r,  s     r6   Ú%prepare_decoder_input_ids_from_labelszDLongT5ForConditionalGeneration.prepare_decoder_input_ids_from_labelsÀ  s   € Ø× Ò  Ñ(Ô(Ð(r8   )NNNNNNNNNNNNN)r¬   r­   r®   r!  r"  r   rš   r  rè  r   r/   r#  r$  r%  rG   rt   r	   r`  r   r«   r6  r¯   r°   s   @r6   rÁ  rÁ    sû  ø€ € € € € ð 	Rð*Ð&ð (7Ø'6Ø)ðð Ðð˜|ð ð ð ð ð ð ð*ð ð ð:ð :ð :ð
 ð .2Ø37Ø59Ø:>Ø=AØ(,Ø26Ø:>Ø*.Ø!%Ø)-Ø,0Ø#'ðJ
ð J
àÔ# dÑ*ðJ
ð Ô)¨DÑ0ðJ
ð !Ô+¨dÑ2ð	J
ð
 !&Ô 0°4Ñ 7ðJ
ð ˜u U¤\Ô2Ô3°dÑ:ðJ
ð  ™ðJ
ð Ô(¨4Ñ/ðJ
ð  %Ô0°4Ñ7ðJ
ð Ô  4Ñ'ðJ
ð ˜$‘;ðJ
ð   $™;ðJ
ð # T™kðJ
ð ˜D‘[ðJ
ð  
ˆuÔ Ô	! OÑ	3ð!J
ð J
ð J
ñ „^ðJ
ðX)¸E¼Lð )ð )ð )ð )ð )ð )ð )ð )r8   rÁ  c                   óÞ   ‡ — e Zd ZddiZdgZdefˆ fd„Zd„ Zd„ Ze		 	 	 	 	 	 dd	e
j        dz  d
e
j        dz  de
j        dz  dedz  dedz  dedz  dee
j                 ez  fd„¦   «         Zˆ xZS )rÂ  r  r  r	  r³   c                 ó  •— t          ¦   «                              |¦  «         t          j        |j        |j        ¦  «        | _        t          j        |¦  «        }d|_	        t          |¦  «        | _        |                      ¦   «          d S )NF)r™   rš   r   rí   rß  r¹   rÄ  r  r  r~  rÚ  r  rä  )rž   r³   r  r¡   s      €r6   rš   zLongT5EncoderModel.__init__Ë  sq   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”l 6Ô#4°f´nÑEÔEˆŒåœ vÑ.Ô.ˆØ#(ˆÔ Ý" >Ñ2Ô2ˆŒð 	�ŠÑÔÐÐÐr8   c                 ó   — | j         S rÄ   r  r  s    r6   r  z'LongT5EncoderModel.get_input_embeddingsÖ  r  r8   c                 óH   — || _         | j                             |¦  «         d S rÄ   )rÄ  r  rè  ræ  s     r6   rè  z'LongT5EncoderModel.set_input_embeddingsÙ  s%   € Ø$ˆŒØŒ×)Ò)¨.Ñ9Ô9Ð9Ð9Ð9r8   Nr´  r]   rì  r*  ró  r©  r    c                 ó\   — |�|n| j         j        }|                      ||||||¬¦  «        }|S )aŠ  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5
            Training](./longt5#training).

        Example:

        ```python
        >>> from transformers import AutoTokenizer, LongT5ForConditionalGeneration

        >>> tokenizer = AutoTokenizer.from_pretrained("google/long-t5-local-base")
        >>> model = LongT5EncoderModel.from_pretrained("google/long-t5-local-base")
        >>> input_ids = tokenizer(
        ...     100 * "Studies have been shown that owning a dog is good for you ", return_tensors="pt"
        ... ).input_ids  # Batch size 1
        >>> outputs = model(input_ids=input_ids)
        >>> last_hidden_states = outputs.last_hidden_state
        ```Nr  )r³   r©  r  )	rž   r´  r]   rì  r*  ró  r©  r+  r  s	            r6   r«   zLongT5EncoderModel.forwardÝ  sJ   € ðF &1Ð%<�k�kÀ$Ä+ÔBYˆàŸ,š,ØØ)Ø'Ø/Ø!5Ø#ð 'ñ 
ô 
ˆð Ðr8   )NNNNNN)r¬   r­   r®   r"  r!  r   rš   r  rè  r   r/   r#  r$  r`  rG   r   r«   r¯   r°   s   @r6   rÂ  rÂ  Ä  s)  ø€ € € € € ð 	& ðÐð +5¨Ð&ð	˜|ð 	ð 	ð 	ð 	ð 	ð 	ðð ð ð:ð :ð :ð ð .2Ø37Ø26Ø)-Ø,0Ø#'ð-ð -àÔ# dÑ*ð-ð Ô)¨DÑ0ð-ð Ô(¨4Ñ/ð	-ð
   $™;ð-ð # T™kð-ð ˜D‘[ð-ð 
ˆuÔ Ô	! OÑ	3ð-ð -ð -ñ „^ð-ð -ð -ð -ð -r8   rÂ  )rÂ  rÁ  rÀ  r°  )r   )Kr‹  r  rû   Útypingr   r/   r   Útorch.nnr   rë  r   r¾  Úactivationsr   Úcache_utilsr	   r
   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   Úutilsr   r   r   r   r   Úconfiguration_longt5r   Ú
get_loggerr¬   rç   rt   r~   r7   r@   rM   rU   r\   r;   rd   rG   r‡   rŽ   r”   ÚModuler–   r²   rÉ   rÑ   rÛ   r?  rb  ry  rƒ  r�  r’  r˜  r°  rÚ  rÀ  rÁ  rÂ  Ú__all__r&   r8   r6   ú<module>rJ     s2  ðð Ð à €€€Ø €€€Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ø %Ð %Ð %Ð %Ð %Ð %à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ )Ð )Ð )Ð )Ð )Ð )Ø JÐ JÐ JÐ JÐ JÐ JÐ JÐ JØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð .Ð -Ð -Ð -Ð -Ð -ðð ð ð ð ð ð ð ð ð ð ð ð ð ð /Ð .Ð .Ð .Ð .Ð .ð 
ˆÔ	˜HÑ	%Ô	%€ðð ˜œð °ð ¸3ð È3ð ÐW\ÔWcð ð ð ð ð #˜%œ,ð #°3ð #¸Sð #ÀUÄ\ð #ð #ð #ð #ð4ð 4˜Uœ\ð 4°cð 4Èð 4ÐY\ð 4ÐejÔeqð 4ð 4ð 4ð 4ð2!°#ð !¸%¼,ð !ð !ð !ð !ðB°U´\ð BÈcð BÐV[ÔVbð Bð Bð Bð Bð8¨e¬lð 8Àsð 8ÐTYÔT`ð 8ÐejÔeqð 8ð 8ð 8ð 8ð .PØ”Lð.PØ58ð.Pà
ˆ5Œ<˜œÐ%Ô&ð.Pð .Pð .Pð .Pðb4°U´\ð 4ÐVYð 4Ð^cÔ^jð 4ð 4ð 4ð 4ð	jØ”<ð	jØ,1¬Lð	jØJMð	jà
„\ð	jð 	jð 	jð 	jð+ð +ð +ð +ð +�b”iñ +ô +ð +ð4ð ð ð ð ˜"œ)ñ ô ð ð,ð ð ð ð ˜rœyñ ô ð ð&ð ð ð ð �B”Iñ ô ð ð&ð ð ð ð �b”iñ ô ð ðDið ið ið ið i˜2œ9ñ iô ið iðXlð lð lð lð l R¤Yñ lô lð lð`ð ð ð ð ˜rœyñ ô ð ðBð ð ð ð  B¤Iñ ô ð ð:ð ð ð ð ¨b¬iñ ô ð ð@ð ð ð ð  ¤	ñ ô ð ð>T
ð T
ð T
ð T
ð T
Ð,ñ T
ô T
ð T
ðn ðS!ð S!ð S!ð S!ð S!˜Oñ S!ô S!ñ „ðS!ðlw
ð w
ð w
ð w
ð w
Ð'ñ w
ô w
ð w
ðt ðT
ð T
ð T
ð T
ð T
Ð'ñ T
ô T
ñ „ðT
ðn €ððñ ô ð
u)ð u)ð u)ð u)ð u)Ð%:¸Oñ u)ô u)ñô ð
u)ðp ðFð Fð Fð Fð FÐ.ñ Fô Fñ „ðFðR kÐ
jÐ
j€€€r8   