§
    ‚Štj8} ã                   óª  — d Z ddlmZ ddlmZ ddlZddlmZ ddlmZm	Z	m
Z
 ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZ ddlmZ  ej        e¦  «        Z G d„ dej         ¦  «        Z! G d„ dej         ¦  «        Z" G d„ dej         ¦  «        Z# G d„ dej         ¦  «        Z$ G d„ dej         ¦  «        Z% G d„ dej         ¦  «        Z& G d„ dej         ¦  «        Z'e G d„ de¦  «        ¦   «         Z( ed ¬!¦  «        e G d"„ d#e¦  «        ¦   «         ¦   «         Z) ed$¬!¦  «        e G d%„ d&e¦  «        ¦   «         ¦   «         Z* ed'¬!¦  «        e G d(„ d)e¦  «        ¦   «         ¦   «         Z+ ed*¬!¦  «        e G d+„ d,e¦  «        ¦   «         ¦   «         Z, ed-¬!¦  «        e G d.„ d/e¦  «        ¦   «         ¦   «         Z- ed0¬!¦  «        e G d1„ d2e¦  «        ¦   «         ¦   «         Z. ed3¬!¦  «        e G d4„ d5e¦  «        ¦   «         ¦   «         Z/e G d6„ d7e(¦  «        ¦   «         Z0 ed8¬!¦  «         G d9„ d:e(e¦  «        ¦   «         Z1 ed;¬!¦  «         G d<„ d=e(¦  «        ¦   «         Z2e G d>„ d?e(¦  «        ¦   «         Z3e G d@„ dAe(¦  «        ¦   «         Z4 edB¬!¦  «         G dC„ dDe(¦  «        ¦   «         Z5e G dE„ dFe(¦  «        ¦   «         Z6g dG¢Z7dS )Hz
PyTorch XLNet model.
é    )ÚCallable)Ú	dataclassN)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FNÚget_activation)ÚGenerationMixin)ÚPreTrainedModel)Úapply_chunking_to_forward)ÚModelOutputÚauto_docstringÚloggingé   )ÚXLNetConfigc                   ór   ‡ — e Zd Zˆ fd„Zedd„¦   «         Zedd„¦   «         Z	 	 	 dd„Zdd	„Z	 	 	 dd
„Z	ˆ xZ
S )ÚXLNetRelativeAttentionc                 ó¨  •— t          ¦   «                              ¦   «          |j        |j        z  dk    rt	          d|j        › d|j        › �¦  «        ‚|j        | _        |j        | _        |j        | _        d|j        dz  z  | _        t          j        t          j
        |j        | j        | j        ¦  «        ¦  «        | _        t          j        t          j
        |j        | j        | j        ¦  «        ¦  «        | _        t          j        t          j
        |j        | j        | j        ¦  «        ¦  «        | _        t          j        t          j
        |j        | j        | j        ¦  «        ¦  «        | _        t          j        t          j
        |j        | j        | j        ¦  «        ¦  «        | _        t          j        t          j
        | j        | j        ¦  «        ¦  «        | _        t          j        t          j
        | j        | j        ¦  «        ¦  «        | _        t          j        t          j
        | j        | j        ¦  «        ¦  «        | _        t          j        t          j
        d| j        | j        ¦  «        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j        ¦  «        | _        d S )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads (r   ç      à?é   ©Úeps)ÚsuperÚ__init__Úd_modelÚn_headÚ
ValueErrorÚd_headÚscaler   Ú	ParameterÚtorchÚFloatTensorÚqÚkÚvÚoÚrÚr_r_biasÚr_s_biasÚr_w_biasÚ	seg_embedÚ	LayerNormÚlayer_norm_epsÚ
layer_normÚDropoutÚdropout©ÚselfÚconfigÚ	__class__s     €úf/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/xlnet/modeling_xlnet.pyr   zXLNetRelativeAttention.__init__'   s  ø€ Ý‰Œ×ÒÑÔÐàŒ>˜FœMÑ)¨QÒ.Ð.Ýð* F¤Nð *ð *Ø œ-ð*ð *ñô ð ð
 ”mˆŒØ”mˆŒØ”~ˆŒØ˜&œ-¨Ñ,Ñ-ˆŒ
å”�eÔ/°´ÀÄÈTÌ[ÑYÔYÑZÔZˆŒÝ”�eÔ/°´ÀÄÈTÌ[ÑYÔYÑZÔZˆŒÝ”�eÔ/°´ÀÄÈTÌ[ÑYÔYÑZÔZˆŒÝ”�eÔ/°´ÀÄÈTÌ[ÑYÔYÑZÔZˆŒÝ”�eÔ/°´ÀÄÈTÌ[ÑYÔYÑZÔZˆŒåœ¥UÔ%6°t´{ÀDÄKÑ%PÔ%PÑQÔQˆŒÝœ¥UÔ%6°t´{ÀDÄKÑ%PÔ%PÑQÔQˆŒÝœ¥UÔ%6°t´{ÀDÄKÑ%PÔ%PÑQÔQˆŒÝœ¥eÔ&7¸¸4¼;ÈÌÑ&TÔ&TÑUÔUˆŒåœ, v¤~¸6Ô;PÐQÑQÔQˆŒÝ”z &¤.Ñ1Ô1ˆŒˆˆó    éÿÿÿÿc           	      óf  — | j         }|                      |d         |d         |d         |d         ¦  «        } | dd…df         } |                      |d         |d         dz
  |d         |d         ¦  «        } t          j        | dt          j        || j        t          j        ¬¦  «        ¦  «        } | S )z<perform relative shift to form the relative attention score.r   r   r   r	   N.©ÚdeviceÚdtype©ÚshapeÚreshaper$   Úindex_selectÚaranger=   Úlong©ÚxÚklenÚx_sizes      r8   Ú	rel_shiftz XLNetRelativeAttention.rel_shiftC   sœ   € ð ”ˆà�IŠI�f˜Q”i ¨¤¨F°1¬I°v¸a´yÑAÔAˆØˆaˆbˆb�#ˆgŒJˆØ�IŠI�f˜Q”i ¨¤¨Q¡°°q´	¸6À!¼9ÑEÔEˆåÔ˜q !¥U¤\°$¸q¼xÍuÌzÐ%ZÑ%ZÔ%ZÑ[Ô[ˆàˆr9   c           	      óv  — | j         }|                      |d         |d         |d         |d         ¦  «        } | d d …d d …dd …d d …f         } |                      |d         |d         |d         |d         dz
  ¦  «        } t          j        | dt          j        || j        t          j        ¬¦  «        ¦  «        } | S )Nr   r   r	   r   r<   r?   rE   s      r8   Úrel_shift_bnijz%XLNetRelativeAttention.rel_shift_bnijP   s®   € à”ˆà�IŠI�f˜Q”i ¨¤¨F°1¬I°v¸a´yÑAÔAˆØˆaˆaˆa����A�B�B˜˜˜ˆkŒNˆØ�IŠI�f˜Q”i ¨¤¨F°1¬I°v¸a´yÀ1±}ÑEÔEˆõ Ô˜q !¥U¤\°$¸q¼xÍuÌzÐ%ZÑ%ZÔ%ZÑ[Ô[ˆð ˆr9   NFc                 óÄ  — t          j        d|| j        z   |¦  «        }t          j        d|| j        z   |¦  «        }	|                      |	|j        d         ¬¦  «        }	|€d}
n9t          j        d|| j        z   | j        ¦  «        }
t          j        d||
¦  «        }
||	z   |
z   | j        z  }|�L|j	        t           j
        k    r|dt          j        d	|¦  «        z  z
  }n|d
t          j        d	|¦  «        z  z
  }t          j                             |d¬¦  «        }|                      |¦  «        }t          j        d||¦  «        }|r|t          j        d|¦  «        fS |S )z.Core relative positional attention operations.zibnd,jbnd->bnijr	   )rG   Nr   zibnd,snd->ibnszijbs,ibns->bnijéÜÿ  z
ijbn->bnijçêŒ 9Y>)F©Údimzbnij,jbnd->ibndz
bnij->ijbn)r$   Úeinsumr-   r+   rK   r@   r,   r.   r"   r>   Úfloat16r   Ú
functionalÚsoftmaxr3   )r5   Úq_headÚk_head_hÚv_head_hÚk_head_rÚseg_matÚ	attn_maskÚoutput_attentionsÚacÚbdÚefÚ
attn_scoreÚ	attn_probÚattn_vecs                 r8   Úrel_attn_corez$XLNetRelativeAttention.rel_attn_core_   sd  € õ Œ\Ð+¨V°d´mÑ-CÀXÑNÔNˆõ Œ\Ð+¨V°d´mÑ-CÀXÑNÔNˆØ× Ò  ¨"¬(°1¬+Ð Ñ6Ô6ˆð ˆ?ØˆBˆBå”Ð.°¸¼Ñ0FÈÌÑWÔWˆBÝ”Ð/°¸"Ñ=Ô=ˆBð ˜2‘g ‘l d¤jÑ0ˆ
ØÐ àŒ¥%¤-Ò/Ð/Ø'¨%µ%´,¸|ÈYÑ2WÔ2WÑ*WÑW�
�
à'¨$µ´¸lÈIÑ1VÔ1VÑ*VÑV�
õ ”M×)Ò)¨*¸!Ð)Ñ<Ô<ˆ	Ø—L’L Ñ+Ô+ˆ	õ ”<Ð 1°9¸hÑGÔGˆàð 	CØ�Uœ\¨,¸	ÑBÔBÐBÐBàˆr9   Tc                 óž   — t          j        d|| j        ¦  «        }|                      |¦  «        }|r||z   }|                      |¦  «        }|S )zPost-attention processing.zibnd,hnd->ibh)r$   rQ   r)   r3   r1   )r5   Úhra   ÚresidualÚattn_outÚoutputs         r8   Úpost_attentionz%XLNetRelativeAttention.post_attentionŽ   sQ   € õ ”< °¸4¼6ÑBÔBˆà—<’< Ñ)Ô)ˆØð 	$Ø !‘|ˆHØ—’ Ñ*Ô*ˆàˆr9   c
           	      ó$  — |��‡|�1|                      ¦   «         dk    rt          j        ||gd¬¦  «        }
n|}
t          j        d|
| j        ¦  «        }t          j        d|
| j        ¦  «        }t          j        d|| j        ¦  «        }t          j        d|| j        ¦  «        }|                      |||||||	¬¦  «        }|	r|\  }}|  	                    ||¦  «        }t          j        d|| j        ¦  «        }|�Pt          j        d||¦  «        }|                      |||||||	¬¦  «        }|	r|\  }}t          j        d||¦  «        }n#|                      |||||||	¬¦  «        }|	r|\  }}|  	                    ||¦  «        }|	r||f}nù|�1|                      ¦   «         dk    rt          j        ||gd¬¦  «        }
n|}
t          j        d|| j        ¦  «        }t          j        d|
| j        ¦  «        }t          j        d|
| j        ¦  «        }t          j        d| 
                    | j        j        ¦  «        | j        ¦  «        }|                      |||||||	¬¦  «        }|	r|\  }}|  	                    ||¦  «        }d }||f}|	r||fz   }|S )Nr   r   rO   zibh,hnd->ibnd)rY   rZ   r[   zmbnd,mlb->lbndzlbnd,mlb->mbnd)rP   r$   ÚcatrQ   r'   r(   r*   r&   rb   rh   Útyper>   )r5   rd   ÚgÚattn_mask_hÚattn_mask_gr*   rY   ÚmemsÚtarget_mappingr[   rj   rV   rW   rX   Úq_head_hÚ
attn_vec_hÚattn_prob_hÚoutput_hÚq_head_gÚ
attn_vec_gÚattn_prob_gÚoutput_gr`   ra   Úoutputss                            r8   ÚforwardzXLNetRelativeAttention.forwardš   s  € ð ‰=ð Ð D§H¢H¡J¤J°¢N NÝ”i  q 	¨qÐ1Ñ1Ô1��à�õ ”| O°S¸$¼&ÑAÔAˆHõ ”| O°S¸$¼&ÑAÔAˆHõ ”| O°Q¸¼Ñ?Ô?ˆHõ ”| O°Q¸¼Ñ?Ô?ˆHð ×+Ò+ØØØØØØ%Ø"3ð ,ñ ô ˆJð !ð 5Ø*4Ñ'�
˜Kð ×*Ò*¨1¨jÑ9Ô9ˆHõ ”| O°Q¸¼Ñ?Ô?ˆHð Ð)Ý œ<Ð(8¸(ÀNÑSÔS�Ø!×/Ò/ØØØØØ#Ø)Ø&7ð 0ñ ô �
ð %ð 9Ø.8Ñ+�J å"œ\Ð*:¸JÈÑWÔW�
�
à!×/Ò/ØØØØØ#Ø)Ø&7ð 0ñ ô �
ð %ð 9Ø.8Ñ+�J ð ×*Ò*¨1¨jÑ9Ô9ˆHà ð 5Ø'¨Ð4�	øð Ð D§H¢H¡J¤J°¢N NÝ”i  q 	¨qÐ1Ñ1Ô1��à�õ ”| O°Q¸¼Ñ?Ô?ˆHÝ”| O°S¸$¼&ÑAÔAˆHÝ”| O°S¸$¼&ÑAÔAˆHõ ”| O°Q·V²V¸D¼F¼LÑ5IÔ5IÈ4Ì6ÑRÔRˆHð ×)Ò)ØØØØØØ%Ø"3ð *ñ ô ˆHð !ð /Ø&.Ñ#�˜)ð ×*Ò*¨1¨hÑ7Ô7ˆHØˆHà˜XÐ&ˆØð 	-Ø  Ñ,ˆGØˆr9   ©r:   ©NNF)T)Ú__name__Ú
__module__Ú__qualname__r   ÚstaticmethodrI   rK   rb   rh   rz   Ú__classcell__©r7   s   @r8   r   r   &   sÓ   ø€ € € € € ð2ð 2ð 2ð 2ð 2ð8 ð
ð 
ð 
ñ „\ð
ð ðð ð ñ „\ðð( ØØð-ð -ð -ð -ð^
ð 
ð 
ð 
ð( ØØð@ð @ð @ð @ð @ð @ð @ð @r9   r   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚXLNetFeedForwardc                 óÜ  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¬¦  «        | _        t          j        |j        |j        ¦  «        | _	        t          j        |j        |j        ¦  «        | _
        t          j        |j        ¦  «        | _        t          |j        t          ¦  «        rt           |j                 | _        d S |j        | _        d S )Nr   )r   r   r   r/   r   r0   r1   ÚLinearÚd_innerÚlayer_1Úlayer_2r2   r3   Ú
isinstanceÚff_activationÚstrr   Úactivation_functionr4   s     €r8   r   zXLNetFeedForward.__init__  s±   ø€ Ý‰Œ×ÒÑÔÐÝœ, v¤~¸6Ô;PÐQÑQÔQˆŒÝ”y ¤°´Ñ@Ô@ˆŒÝ”y ¤°´Ñ@Ô@ˆŒÝ”z &¤.Ñ1Ô1ˆŒÝ�fÔ*­CÑ0Ô0ð 	<Ý'-¨fÔ.BÔ'CˆDÔ$Ð$Ð$à'-Ô';ˆDÔ$Ð$Ð$r9   c                 ó  — |}|                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|S ©N)rˆ   r�   r3   r‰   r1   )r5   Úinprg   s      r8   rz   zXLNetFeedForward.forward)  sx   € ØˆØ—’˜fÑ%Ô%ˆØ×)Ò)¨&Ñ1Ô1ˆØ—’˜fÑ%Ô%ˆØ—’˜fÑ%Ô%ˆØ—’˜fÑ%Ô%ˆØ—’ ¨#¡Ñ.Ô.ˆØˆr9   )r}   r~   r   r   rz   r�   r‚   s   @r8   r„   r„     sG   ø€ € € € € ð	<ð 	<ð 	<ð 	<ð 	<ðð ð ð ð ð ð r9   r„   c                   ó2   ‡ — e Zd Zˆ fd„Z	 	 	 dd„Zd„ Zˆ xZS )Ú
XLNetLayerc                 óú   •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          |¦  «        | _        t          j        |j        ¦  «        | _        |j	        | _	        d| _
        d S ©Nr   )r   r   r   Úrel_attnr„   Úffr   r2   r3   Úchunk_size_feed_forwardÚseq_len_dimr4   s     €r8   r   zXLNetLayer.__init__5  sc   ø€ Ý‰Œ×ÒÑÔÐÝ.¨vÑ6Ô6ˆŒÝ" 6Ñ*Ô*ˆŒÝ”z &¤.Ñ1Ô1ˆŒØ'-Ô'EˆÔ$ØˆÔÐÐr9   NFc
                 ó  — |                       |||||||||	¬¦	  «	        }
|
d d…         \  }}|�!t          | j        | j        | j        |¦  «        }t          | j        | j        | j        |¦  «        }||f|
dd …         z   }
|
S )N)ro   rp   r[   r   )r•   r   Úff_chunkr—   r˜   )r5   rt   rx   rm   rn   r*   rY   ro   rp   r[   ry   s              r8   rz   zXLNetLayer.forward=  s¬   € ð —-’-ØØØØØØØØ)Ø/ð  ñ 

ô 

ˆð % R a Rœ[Ñˆ�(àÐÝ0Ø”˜tÔ;¸TÔ=MÈxñô ˆHõ -¨T¬]¸DÔ<XÐZ^ÔZjÐltÑuÔuˆà˜XÐ&¨°°°¬Ñ4ˆØˆr9   c                 ó0   — |                       |¦  «        }|S r�   )r–   )r5   Úoutput_xs     r8   rš   zXLNetLayer.ff_chunk_  s   € Ø—7’7˜8Ñ$Ô$ˆØˆr9   r|   )r}   r~   r   r   rz   rš   r�   r‚   s   @r8   r’   r’   4  sg   ø€ € € € € ðð ð ð ð ð  ØØð ð  ð  ð  ðDð ð ð ð ð ð r9   r’   c                   ób   ‡ — e Zd ZdZdefˆ fd„Zd	dej        dej        dz  dej        fd„Zˆ xZ	S )
ÚXLNetPoolerStartLogitszÍ
    Compute SQuAD start logits from sequence hidden states.

    Args:
        config ([`XLNetConfig`]):
            The config used by the model, will be used to grab the `hidden_size` of the model.
    r6   c                 ó†   •— t          ¦   «                              ¦   «          t          j        |j        d¦  «        | _        d S r”   )r   r   r   r†   Úhidden_sizeÚdenser4   s     €r8   r   zXLNetPoolerStartLogits.__init__n  s3   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°1Ñ5Ô5ˆŒ
ˆ
ˆ
r9   NÚhidden_statesÚp_maskÚreturnc                 ó¾   — |                       |¦  «                             d¦  «        }|�2|j        t          j        k    r|d|z
  z  d|z  z
  }n|d|z
  z  d|z  z
  }|S )aì  
        Args:
            hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
                The final hidden states of the model.
            p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*):
                Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token
                should be masked.

        Returns:
            `torch.FloatTensor`: The start logits for SQuAD.
        r:   Nr   rM   rN   )r¡   Úsqueezer>   r$   rR   )r5   r¢   r£   rF   s       r8   rz   zXLNetPoolerStartLogits.forwardr  sm   € ð �JŠJ�}Ñ%Ô%×-Ò-¨bÑ1Ô1ˆàÐØŒ|�uœ}Ò,Ð,Ø˜˜V™Ñ$ u¨v¡~Ñ5��à˜˜V™Ñ$ t¨f¡}Ñ4�àˆr9   r�   )
r}   r~   r   Ú__doc__r   r   r$   r%   rz   r�   r‚   s   @r8   rž   rž   e  sŒ   ø€ € € € € ðð ð6˜{ð 6ð 6ð 6ð 6ð 6ð 6ðð  UÔ%6ð ÀÔ@QÐTXÑ@Xð ÐdiÔduð ð ð ð ð ð ð ð r9   rž   c                   ó�   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 ddej        dej        dz  dej        dz  dej        dz  d	ej        f
d
„Z	ˆ xZ
S )ÚXLNetPoolerEndLogitsz÷
    Compute SQuAD end logits from sequence hidden states.

    Args:
        config ([`XLNetConfig`]):
            The config used by the model, will be used to grab the `hidden_size` of the model and the `layer_norm_eps`
            to use.
    r6   c                 óN  •— t          ¦   «                              ¦   «          t          j        |j        dz  |j        ¦  «        | _        t          j        ¦   «         | _        t          j        |j        |j	        ¬¦  «        | _        t          j        |j        d¦  «        | _
        d S )Nr   r   r   )r   r   r   r†   r    Údense_0ÚTanhÚ
activationr/   r0   Údense_1r4   s     €r8   r   zXLNetPoolerEndLogits.__init__”  sz   ø€ Ý‰Œ×ÒÑÔÐÝ”y Ô!3°aÑ!7¸Ô9KÑLÔLˆŒÝœ'™)œ)ˆŒÝœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”y Ô!3°QÑ7Ô7ˆŒˆˆr9   Nr¢   Ústart_statesÚstart_positionsr£   r¤   c                 óJ  — |€|€
J d¦   «         ‚|�a|j         dd…         \  }}|dd…ddf                              dd|¦  «        }|                     d|¦  «        }|                     d|d¦  «        }|                      t	          j        ||gd¬¦  «        ¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «         	                    d¦  «        }|�2|j
        t          j        k    r|d|z
  z  d|z  z
  }n|d|z
  z  d|z  z
  }|S )	aë  
        Args:
            hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
                The final hidden states of the model.
            start_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`, *optional*):
                The hidden states of the first tokens for the labeled span.
            start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
                The position of the first token for the labeled span.
            p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*):
                Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token
                should be masked.

        <Tip>

        One of `start_states` or `start_positions` should be not `None`. If both are set, `start_positions` overrides
        `start_states`.

        </Tip>

        Returns:
            `torch.FloatTensor`: The end logits for SQuAD.
        Nú7One of start_states, start_positions should be not Noneéþÿÿÿr:   rO   r   rM   rN   )r@   ÚexpandÚgatherr«   r$   rj   r­   r/   r®   r¦   r>   rR   )r5   r¢   r¯   r°   r£   ÚslenÚhszrF   s           r8   rz   zXLNetPoolerEndLogits.forward›  s>  € ð: Ð'¨?Ð+FÐ+FØEñ ,GÔ+FÐFð Ð&Ø%Ô+¨B¨C¨CÔ0‰IˆD�#Ø-¨a¨a¨a°°t¨mÔ<×CÒCÀBÈÈCÑPÔPˆOØ(×/Ò/°°OÑDÔDˆLØ'×.Ò.¨r°4¸Ñ<Ô<ˆLà�LŠL�œ M°<Ð#@ÀbÐIÑIÔIÑJÔJˆØ�OŠO˜AÑÔˆØ�NŠN˜1ÑÔˆØ�LŠL˜‰OŒO×#Ò# BÑ'Ô'ˆàÐØŒ|�uœ}Ò,Ð,Ø˜˜V™Ñ$ u¨v¡~Ñ5��à˜˜V™Ñ$ t¨f¡}Ñ4�àˆr9   ©NNN©r}   r~   r   r§   r   r   r$   r%   Ú
LongTensorrz   r�   r‚   s   @r8   r©   r©   Š  sÀ   ø€ € € € € ðð ð8˜{ð 8ð 8ð 8ð 8ð 8ð 8ð 26Ø37Ø+/ð1ð 1àÔ(ð1ð Ô'¨$Ñ.ð1ð Ô)¨DÑ0ð	1ð
 Ô! DÑ(ð1ð 
Ô	ð1ð 1ð 1ð 1ð 1ð 1ð 1ð 1r9   r©   c                   ó�   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 ddej        dej        dz  dej        dz  dej        dz  d	ej        f
d
„Z	ˆ xZ
S )ÚXLNetPoolerAnswerClasszè
    Compute SQuAD 2.0 answer class from classification and start tokens hidden states.

    Args:
        config ([`XLNetConfig`]):
            The config used by the model, will be used to grab the `hidden_size` of the model.
    r6   c                 ó  •— t          ¦   «                              ¦   «          t          j        |j        dz  |j        ¦  «        | _        t          j        ¦   «         | _        t          j        |j        dd¬¦  «        | _        d S )Nr   r   F©Úbias)	r   r   r   r†   r    r«   r¬   r­   r®   r4   s     €r8   r   zXLNetPoolerAnswerClass.__init__Ù  sc   ø€ Ý‰Œ×ÒÑÔÐÝ”y Ô!3°aÑ!7¸Ô9KÑLÔLˆŒÝœ'™)œ)ˆŒÝ”y Ô!3°Q¸UÐCÑCÔCˆŒˆˆr9   Nr¢   r¯   r°   Ú	cls_indexr¤   c                 ó`  — |j         d         }|€|€
J d¦   «         ‚|�K|dd…ddf                              dd|¦  «        }|                     d|¦  «                             d¦  «        }|�L|dd…ddf                              dd|¦  «        }|                     d|¦  «                             d¦  «        }n|dd…ddd…f         }|                      t          j        ||gd¬¦  «        ¦  «        }|                      |¦  «        }|                      |¦  «                             d¦  «        }|S )a¸  
        Args:
            hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
                The final hidden states of the model.
            start_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`, *optional*):
                The hidden states of the first tokens for the labeled span.
            start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
                The position of the first token for the labeled span.
            cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
                Position of the CLS token for each sentence in the batch. If `None`, takes the last token.

        <Tip>

        One of `start_states` or `start_positions` should be not `None`. If both are set, `start_positions` overrides
        `start_states`.

        </Tip>

        Returns:
            `torch.FloatTensor`: The SQuAD 2.0 answer class.
        r:   Nr²   r³   rO   )	r@   r´   rµ   r¦   r«   r$   rj   r­   r®   )r5   r¢   r¯   r°   rÀ   r·   Úcls_token_staterF   s           r8   rz   zXLNetPoolerAnswerClass.forwardß  s@  € ð: Ô! "Ô%ˆØÐ'¨?Ð+FÐ+FØEñ ,GÔ+FÐFð Ð&Ø-¨a¨a¨a°°t¨mÔ<×CÒCÀBÈÈCÑPÔPˆOØ(×/Ò/°°OÑDÔD×LÒLÈRÑPÔPˆLàÐ Ø! ! ! ! T¨4 -Ô0×7Ò7¸¸BÀÑDÔDˆIØ+×2Ò2°2°yÑAÔA×IÒIÈ"ÑMÔMˆOˆOà+¨A¨A¨A¨r°1°1°1¨HÔ5ˆOà�LŠL�œ L°/Ð#BÈÐKÑKÔKÑLÔLˆØ�OŠO˜AÑÔˆØ�LŠL˜‰OŒO×#Ò# BÑ'Ô'ˆàˆr9   r¸   r¹   r‚   s   @r8   r¼   r¼   Ð  sÇ   ø€ € € € € ðð ðD˜{ð Dð Dð Dð Dð Dð Dð 26Ø37Ø-1ð/ð /àÔ(ð/ð Ô'¨$Ñ.ð/ð Ô)¨DÑ0ð	/ð
 Ô# dÑ*ð/ð 
Ô	ð/ð /ð /ð /ð /ð /ð /ð /r9   r¼   c                   ód   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej        dej        dz  dej        fd„Z	ˆ xZ
S )
ÚXLNetSequenceSummaryaÍ  
    Compute a single vector summary of a sequence hidden states.

    Args:
        config ([`XLNetConfig`]):
            The config used by the model. Relevant arguments in the config class of the model are (refer to the actual
            config class of your model for the default values it uses):

            - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are:

                - `"last"` -- Take the last token hidden state (like XLNet)
                - `"first"` -- Take the first token hidden state (like Bert)
                - `"mean"` -- Take the mean of all tokens hidden states
                - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2)
                - `"attn"` -- Not implemented now, use multi-head attention

            - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction.
            - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes
              (otherwise to `config.hidden_size`).
            - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output,
              another string or `None` will add no activation.
            - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation.
            - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation.
    r6   c                 óV  •— t          ¦   «                              ¦   «          t          |dd¦  «        | _        | j        dk    rt          ‚t          j        ¦   «         | _        t          |d¦  «        rW|j	        rPt          |d¦  «        r|j
        r|j        dk    r|j        }n|j        }t          j        |j        |¦  «        | _        t          |dd ¦  «        }|rt          |¦  «        nt          j        ¦   «         | _        t          j        ¦   «         | _        t          |d¦  «        r)|j        dk    rt          j        |j        ¦  «        | _        t          j        ¦   «         | _        t          |d	¦  «        r+|j        dk    r"t          j        |j        ¦  «        | _        d S d S d S )
NÚsummary_typeÚlastÚattnÚsummary_use_projÚsummary_proj_to_labelsr   Úsummary_activationÚsummary_first_dropoutÚsummary_last_dropout)r   r   ÚgetattrrÆ   ÚNotImplementedErrorr   ÚIdentityÚsummaryÚhasattrrÉ   rÊ   Ú
num_labelsr    r†   r   r­   Úfirst_dropoutrÌ   r2   Úlast_dropoutrÍ   )r5   r6   Únum_classesÚactivation_stringr7   s       €r8   r   zXLNetSequenceSummary.__init__,  sœ  ø€ Ý‰Œ×ÒÑÔÐå# F¨N¸FÑCÔCˆÔØÔ Ò&Ð&õ &Ð%å”{‘}”}ˆŒÝ�6Ð-Ñ.Ô.ð 	F°6Ô3Jð 	FÝ�vÐ7Ñ8Ô8ð 1¸VÔ=Zð 1Ð_eÔ_pÐstÒ_tÐ_tØ$Ô/��à$Ô0�Ýœ9 VÔ%7¸ÑEÔEˆDŒLå# FÐ,@À$ÑGÔGÐØIZÐ$m¥NÐ3DÑ$EÔ$EÐ$EÕ`bÔ`kÑ`mÔ`mˆŒåœ[™]œ]ˆÔÝ�6Ð2Ñ3Ô3ð 	J¸Ô8TÐWXÒ8XÐ8XÝ!#¤¨FÔ,HÑ!IÔ!IˆDÔåœK™MœMˆÔÝ�6Ð1Ñ2Ô2ð 	H°vÔ7RÐUVÒ7VÐ7VÝ "¤
¨6Ô+FÑ GÔ GˆDÔÐÐð	Hð 	HÐ7VÐ7Vr9   Nr¢   rÀ   r¤   c                 ó:  — | j         dk    r|dd…df         }�n-| j         dk    r|dd…df         }�n| j         dk    r|                     d¬¦  «        }nò| j         d	k    rÕ|€=t          j        |d
dd…dd…f         |j        d         dz
  t          j        ¬¦  «        }nl|                     d¦  «                             d¦  «        }|                     d|                     ¦   «         dz
  z  | 	                    d¦  «        fz   ¦  «        }| 
                    d|¦  «                             d¦  «        }n| j         dk    rt          ‚|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S )ak  
        Compute a single vector summary of a sequence hidden states.

        Args:
            hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`):
                The hidden states of the last layer.
            cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*):
                Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token.

        Returns:
            `torch.FloatTensor`: The summary of the sequence hidden states.
        rÇ   Nr:   Úfirstr   Úmeanr   rO   rÀ   .r³   )r>   r{   rÈ   )rÆ   rÚ   r$   Ú	full_liker@   rD   Ú	unsqueezer´   rP   Úsizerµ   r¦   rÏ   rÔ   rÑ   r­   rÕ   )r5   r¢   rÀ   rg   s       r8   rz   zXLNetSequenceSummary.forwardI  sª  € ð Ô Ò&Ð&Ø" 1 1 1 b 5Ô)ˆF‰FØÔ 'Ò)Ð)Ø" 1 1 1 a 4Ô(ˆF‰FØÔ &Ò(Ð(Ø"×'Ò'¨AÐ'Ñ.Ô.ˆFˆFØÔ +Ò-Ð-ØÐ Ý!œOØ! # r¨ r¨1¨1¨1 *Ô-Ø!Ô'¨Ô+¨aÑ/Ýœ*ðñ ô �	�	ð &×/Ò/°Ñ3Ô3×=Ò=¸bÑAÔA�	Ø%×,Ò,¨U°i·m²m±o´oÈÑ6IÑ-JÈm×N`ÒN`ÐacÑNdÔNdÐMfÑ-fÑgÔg�	à"×)Ò)¨"¨iÑ8Ô8×@Ò@ÀÑDÔDˆFˆFØÔ &Ò(Ð(Ý%Ð%à×#Ò# FÑ+Ô+ˆØ—’˜fÑ%Ô%ˆØ—’ Ñ(Ô(ˆØ×"Ò" 6Ñ*Ô*ˆàˆr9   r�   r¹   r‚   s   @r8   rÄ   rÄ     s›   ø€ € € € € ðð ð2H˜{ð Hð Hð Hð Hð Hð Hð< VZð)ð )Ø"Ô.ð)Ø;@Ô;KÈdÑ;Rð)à	Ô	ð)ð )ð )ð )ð )ð )ð )ð )r9   rÄ   c                   óX   ‡ — e Zd ZU eed<   dZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZ	S )ÚXLNetPreTrainedModelr6   Útransformerc           	      óª  •— t          ¦   «                              |¦  «         t          |t          ¦  «        r^|j        |j        |j        |j        |j        |j	        |j
        |j        |j        f	D ]#}t          j        |d| j        j        ¬¦  «         Œ$dS t          |t"          ¦  «        r(t          j        |j        d| j        j        ¬¦  «         dS dS )zInitialize the weights.g        )rÚ   ÚstdN)r   Ú_init_weightsrŠ   r   r&   r'   r(   r)   r*   r+   r,   r-   r.   ÚinitÚnormal_r6   Úinitializer_rangeÚ
XLNetModelÚmask_emb)r5   ÚmoduleÚparamr7   s      €r8   rã   z"XLNetPreTrainedModel._init_weightsz  sä   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ4Ñ5Ô5ð 	Wà”Ø”Ø”Ø”Ø”Ø”Ø”Ø”ØÔ ð
ð Qð Q�õ ”˜U¨°$´+Ô2OÐPÑPÔPÐPÐPðQð Qõ ˜¥
Ñ+Ô+ð 	WÝŒL˜œ¨s¸¼Ô8UÐVÑVÔVÐVÐVÐVð	Wð 	Wr9   )
r}   r~   r   r   Ú__annotations__Úbase_model_prefixr$   Úno_gradrã   r�   r‚   s   @r8   rß   rß   u  sg   ø€ € € € € € àÐÐÑØ%Ðà€U„]�_„_ðWð Wð Wð Wñ „_ðWð Wð Wð Wð Wr9   rß   z(
    Output type of [`XLNetModel`].
    )Úcustom_introc                   ó®   — e Zd ZU dZej        ed<   dZeej                 dz  ed<   dZ	e
ej        df         dz  ed<   dZe
ej        df         dz  ed<   dS )ÚXLNetModelOutputa‚  
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_predict, hidden_size)`):
        Sequence of hidden-states at the last layer of the model.

        `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then `num_predict`
        corresponds to `sequence_length`.
    mems (`list[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    Úlast_hidden_stateNro   .r¢   Ú
attentions)r}   r~   r   r§   r$   r%   rë   ro   Úlistr¢   Útuplerò   © r9   r8   rð   rð   �  sŽ   € € € € € € ð
ð 
ð Ô(Ð(Ð(Ñ(Ø+/€Dˆ$ˆuÔ Ô
! DÑ
(Ð/Ð/Ñ/Ø:>€M�5˜Ô*¨CÐ/Ô0°4Ñ7Ð>Ð>Ñ>Ø7;€J��eÔ'¨Ð,Ô-°Ñ4Ð;Ð;Ñ;Ð;Ð;r9   rð   z.
    Output type of [`XLNetLMHeadModel`].
    c                   óÖ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej                 dz  ed<   dZeej        df         dz  ed<   dZeej        df         dz  ed<   dS )	ÚXLNetLMHeadModelOutputaB  
    loss (`torch.FloatTensor` of shape *(1,)*, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, num_predict, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).

        `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then `num_predict`
        corresponds to `sequence_length`.
    mems (`list[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    NÚlossÚlogitsro   .r¢   rò   ©r}   r~   r   r§   rø   r$   r%   rë   rù   ro   ró   r¢   rô   rò   rõ   r9   r8   r÷   r÷   ¨  s¯   € € € € € € ðð ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø+/€Dˆ$ˆuÔ Ô
! DÑ
(Ð/Ð/Ñ/Ø:>€M�5˜Ô*¨CÐ/Ô0°4Ñ7Ð>Ð>Ñ>Ø7;€J��eÔ'¨Ð,Ô-°Ñ4Ð;Ð;Ñ;Ð;Ð;r9   r÷   z<
    Output type of [`XLNetForSequenceClassification`].
    c                   óÖ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej                 dz  ed<   dZeej        df         dz  ed<   dZeej        df         dz  ed<   dS )	Ú$XLNetForSequenceClassificationOutputa�  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `label` is provided):
        Classification (or regression if config.num_labels==1) loss.
    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Classification (or regression if config.num_labels==1) scores (before SoftMax).
    mems (`list[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    Nrø   rù   ro   .r¢   rò   rú   rõ   r9   r8   rü   rü   Ä  ó¯   € € € € € € ð	ð 	ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø+/€Dˆ$ˆuÔ Ô
! DÑ
(Ð/Ð/Ñ/Ø:>€M�5˜Ô*¨CÐ/Ô0°4Ñ7Ð>Ð>Ñ>Ø7;€J��eÔ'¨Ð,Ô-°Ñ4Ð;Ð;Ñ;Ð;Ð;r9   rü   z?
    Output type of [`XLNetForTokenClassificationOutput`].
    c                   óÖ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej                 dz  ed<   dZeej        df         dz  ed<   dZeej        df         dz  ed<   dS )	Ú!XLNetForTokenClassificationOutputaO  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Classification loss.
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_labels)`):
        Classification scores (before SoftMax).
    mems (`list[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    Nrø   rù   ro   .r¢   rò   rú   rõ   r9   r8   rÿ   rÿ   Ý  rý   r9   rÿ   z4
    Output type of [`XLNetForMultipleChoice`].
    c                   óÖ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej                 dz  ed<   dZeej        df         dz  ed<   dZeej        df         dz  ed<   dS )	ÚXLNetForMultipleChoiceOutputa–  
    loss (`torch.FloatTensor` of shape *(1,)*, *optional*, returned when `labels` is provided):
        Classification loss.
    logits (`torch.FloatTensor` of shape `(batch_size, num_choices)`):
        *num_choices* is the second dimension of the input tensors. (see *input_ids* above).

        Classification scores (before SoftMax).
    mems (`list[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    Nrø   rù   ro   .r¢   rò   rú   rõ   r9   r8   r  r  ö  s¯   € € € € € € ðð ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø+/€Dˆ$ˆuÔ Ô
! DÑ
(Ð/Ð/Ñ/Ø:>€M�5˜Ô*¨CÐ/Ô0°4Ñ7Ð>Ð>Ñ>Ø7;€J��eÔ'¨Ð,Ô-°Ñ4Ð;Ð;Ñ;Ð;Ð;r9   r  z=
    Output type of [`XLNetForQuestionAnsweringSimple`].
    c                   óô   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dZ
eej                 dz  ed<   dZeej        df         dz  ed<   dZeej        df         dz  ed	<   dS )
Ú%XLNetForQuestionAnsweringSimpleOutputaþ  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Total span extraction loss is the sum of a Cross-Entropy for the start and end positions.
    start_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length,)`):
        Span-start scores (before SoftMax).
    end_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length,)`):
        Span-end scores (before SoftMax).
    mems (`list[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    Nrø   Ústart_logitsÚ
end_logitsro   .r¢   rò   )r}   r~   r   r§   rø   r$   r%   rë   r  r  ro   ró   r¢   rô   rò   rõ   r9   r8   r  r    sÇ   € € € € € € ðð ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø-1€L�%Ô# dÑ*Ð1Ð1Ñ1Ø+/€J�Ô! DÑ(Ð/Ð/Ñ/Ø+/€Dˆ$ˆuÔ Ô
! DÑ
(Ð/Ð/Ñ/Ø:>€M�5˜Ô*¨CÐ/Ô0°4Ñ7Ð>Ð>Ñ>Ø7;€J��eÔ'¨Ð,Ô-°Ñ4Ð;Ð;Ñ;Ð;Ð;r9   r  z7
    Output type of [`XLNetForQuestionAnswering`].
    c                   óN  — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej
        dz  ed<   dZej        dz  ed<   dZej
        dz  ed<   dZej        dz  ed<   dZeej                 dz  ed	<   dZeej        d
f         dz  ed<   dZeej        d
f         dz  ed<   dS )ÚXLNetForQuestionAnsweringOutputax  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned if both `start_positions` and `end_positions` are provided):
        Classification loss as the sum of start token, end token (and is_impossible if provided) classification
        losses.
    start_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
        Log probabilities for the top config.start_n_top start token possibilities (beam-search).
    start_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
        Indices for the top config.start_n_top start token possibilities (beam-search).
    end_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
        Log probabilities for the top `config.start_n_top * config.end_n_top` end token possibilities
        (beam-search).
    end_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
        Indices for the top `config.start_n_top * config.end_n_top` end token possibilities (beam-search).
    cls_logits (`torch.FloatTensor` of shape `(batch_size,)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
        Log probabilities for the `is_impossible` label of the answers.
    mems (`list[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    Nrø   Ústart_top_log_probsÚstart_top_indexÚend_top_log_probsÚend_top_indexÚ
cls_logitsro   .r¢   rò   )r}   r~   r   r§   rø   r$   r%   rë   r  r	  rº   r
  r  r  ro   ró   r¢   rô   rò   rõ   r9   r8   r  r  -  s  € € € € € € ðð ð* &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø48Ð˜Ô*¨TÑ1Ð8Ð8Ñ8Ø/3€O�UÔ%¨Ñ,Ð3Ð3Ñ3Ø26Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø-1€M�5Ô# dÑ*Ð1Ð1Ñ1Ø+/€J�Ô! DÑ(Ð/Ð/Ñ/Ø+/€Dˆ$ˆuÔ Ô
! DÑ
(Ð/Ð/Ñ/Ø:>€M�5˜Ô*¨CÐ/Ô0°4Ñ7Ð>Ð>Ñ>Ø7;€J��eÔ'¨Ð,Ô-°Ñ4Ð;Ð;Ñ;Ð;Ð;r9   r  c                   óZ  ‡ — e Zd Zˆ fd„Zd„ Zd„ Zd„ Zd„ Zedd„¦   «         Z	dd„Z
e	 	 	 	 	 	 	 	 	 	 	 	 dd	ej        dz  d
ej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dedz  dedz  dedz  deez  fd„¦   «         Zˆ xZS )rç   c                 óŠ  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        ‰j        | _        ‰j        | _        ‰j        | _        ‰j        | _        ‰j        | _        ‰j	        | _	        t          j        ‰j        ‰j        ¦  «        | _        t          j        t          j        dd‰j        ¦  «        ¦  «        | _        t          j        ˆfd„t'          ‰j	        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j        ¦  «        | _        |                      ¦   «          d S )Nr   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS rõ   )r’   )Ú.0Ú_r6   s     €r8   ú
<listcomp>z'XLNetModel.__init__.<locals>.<listcomp>d  s!   ø€ Ð#VÐ#VÐ#V¸1¥J¨vÑ$6Ô$6Ð#VÐ#VÐ#Vr9   )r   r   Úmem_lenÚ	reuse_lenr   Úsame_lengthÚ	attn_typeÚbi_dataÚ	clamp_lenÚn_layerr   Ú	EmbeddingÚ
vocab_sizeÚword_embeddingr#   r$   r%   rè   Ú
ModuleListÚrangeÚlayerr2   r3   Ú	post_initr4   s    `€r8   r   zXLNetModel.__init__V  sÿ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð à”~ˆŒØÔ)ˆŒØ”~ˆŒØ!Ô-ˆÔØÔ)ˆŒØ”~ˆŒØÔ)ˆŒØ”~ˆŒå œl¨6Ô+<¸f¼nÑMÔMˆÔÝœ¥UÔ%6°q¸!¸V¼^Ñ%LÔ%LÑMÔMˆŒÝ”]Ð#VÐ#VÐ#VÐ#VÅÀfÄnÑ@UÔ@UÐ#VÑ#VÔ#VÑWÔWˆŒ
Ý”z &¤.Ñ1Ô1ˆŒð 	�ŠÑÔÐÐÐr9   c                 ó   — | j         S r�   ©r  ©r5   s    r8   Úget_input_embeddingszXLNetModel.get_input_embeddingsj  s   € ØÔ"Ð"r9   c                 ó   — || _         d S r�   r"  ©r5   Únew_embeddingss     r8   Úset_input_embeddingszXLNetModel.set_input_embeddingsm  s   € Ø,ˆÔÐÐr9   c                 ó$  — t          j        |||z   f| j        ¬¦  «        }| j        rP|dd…d|…f                              d¦  «        }|                     |dz   ¦  «         |dd…d|…fxx         |z  cc<   n|                     |dz   ¦  «         |S )aD  
        Creates causal attention mask. Float mask where 1.0 indicates masked, 0.0 indicates not-masked.

        Args:
            qlen: Sequence length
            mlen: Mask length

        ::

                  same_length=False: same_length=True: <mlen > < qlen > <mlen > < qlen >
               ^ [0 0 0 0 0 1 1 1 1] [0 0 0 0 0 1 1 1 1]
                 [0 0 0 0 0 0 1 1 1] [1 0 0 0 0 0 1 1 1]
            qlen [0 0 0 0 0 0 0 1 1] [1 1 0 0 0 0 0 1 1]
                 [0 0 0 0 0 0 0 0 1] [1 1 1 0 0 0 0 0 1]
               v [0 0 0 0 0 0 0 0 0] [1 1 1 1 0 0 0 0 0]

        )r=   Nr:   r   )r$   Úonesr=   r  ÚtrilÚtriu_)r5   ÚqlenÚmlenÚmaskÚmask_los        r8   Úcreate_maskzXLNetModel.create_maskp  s¦   € õ$ Œz˜4 ¨¡Ð-°d´kÐBÑBÔBˆØÔð 	!Ø˜1˜1˜1˜e˜t˜e˜8”n×)Ò)¨"Ñ-Ô-ˆGØ�JŠJ�t˜a‘xÑ Ô Ð Ø����E�T�E�ˆNˆNŒN˜gÑ%ˆNˆN‰NˆNà�JŠJ�t˜a‘xÑ Ô Ð àˆr9   c                 ó   — | j         �| j         dk    r|d | j         …         }| j        �| j        dk    rd}n| j         }|€||d …         }n t          j        ||gd¬¦  «        |d …         }|                     ¦   «         S )Nr   rO   )r  r  r$   rj   Údetach)r5   Úcurr_outÚprev_memÚcutoffÚnew_mems        r8   Ú	cache_memzXLNetModel.cache_memŒ  s˜   € àŒ>Ð%¨$¬.¸1Ò*<Ð*<ØÐ 0 $¤.Ð 0Ô1ˆHàŒ<Ð 4¤<°1Ò#4Ð#4ð ˆFˆFð ”l�]ˆFØÐà˜v˜w˜wÔ'ˆGˆGå”i ¨8Ð 4¸!Ð<Ñ<Ô<¸V¸W¸WÔEˆGà�~Š~ÑÔÐr9   Nc                 óú   — t          j        d| |¦  «        }t          j        t          j        |¦  «        t          j        |¦  «        gd¬¦  «        }|d d …d d d …f         }|�|                     d|d¦  «        }|S )Nzi,d->idr:   rO   )r$   rQ   rj   ÚsinÚcosr´   )Úpos_seqÚinv_freqÚbszÚsinusoid_inpÚpos_embs        r8   Úpositional_embeddingzXLNetModel.positional_embedding¡  sz   € å”| I¨w¸ÑAÔAˆÝ”)�UœY |Ñ4Ô4µe´iÀÑ6MÔ6MÐNÐTVÐWÑWÔWˆØ˜!˜!˜!˜T 1 1 1˜*Ô%ˆàˆ?Ø—n’n R¨¨bÑ1Ô1ˆGàˆr9   c                 ó–  — t          j        d| j        dt           j        |¬¦  «                             ¦   «         }dt          j        d|| j        z  ¦  «        z  }| j        dk    r|| }}n(| j        dk    r|d}}nt          d	| j        › d
�¦  «        ‚| j        �r5t          j        ||dt           j        |¬¦  «                             ¦   «         }	t          j        | | dt           j        |¬¦  «                             ¦   «         }
| j	        dk    rB|	 
                    | j	         | j	        ¦  «        }	|
 
                    | j	         | j	        ¦  «        }
|�5|                      |	||dz  ¦  «        }|                      |
||dz  ¦  «        }n,|                      |	|¦  «        }|                      |
|¦  «        }t          j        ||gd¬¦  «        }nxt          j        ||dt           j        |¬¦  «                             ¦   «         }	| j	        dk    r!|	 
                    | j	         | j	        ¦  «        }	|                      |	||¦  «        }|S )Nr   g       @©r>   r=   r   i'  ÚbiÚunir:   zUnknown `attn_type` ú.g      ð¿ç      ð?r   rO   )r$   rC   r   Úint64ÚfloatÚpowr  r    r  r  ÚclamprA  rj   )r5   r-  rG   r>  r=   Úfreq_seqr=  ÚbegÚendÚfwd_pos_seqÚbwd_pos_seqÚfwd_pos_embÚbwd_pos_embr@  s                 r8   Úrelative_positional_encodingz'XLNetModel.relative_positional_encoding¬  s-  € å”<  4¤<°½E¼KÐPVÐWÑWÔW×]Ò]Ñ_Ô_ˆØ•u”y ¨°D´LÑ)@ÑBÔBÑBˆàŒ>˜TÒ!Ð!à˜d˜U�ˆCˆCØŒ^˜uÒ$Ð$à˜R�ˆCˆCåÐE°D´NÐEÐEÐEÑFÔFÐFàŒ<ñ 	LÝœ, s¨C°½U¼[ÐQWÐXÑXÔX×^Ò^Ñ`Ô`ˆKÝœ,¨ t¨c¨T°3½e¼kÐRXÐYÑYÔY×_Ò_ÑaÔaˆKàŒ~ Ò!Ð!Ø)×/Ò/°´°ÀÄÑPÔP�Ø)×/Ò/°´°ÀÄÑPÔP�àˆØ"×7Ò7¸ÀXÈsÐVWÉxÑXÔX�Ø"×7Ò7¸ÀXÈsÐVWÉxÑXÔX��à"×7Ò7¸ÀXÑNÔN�Ø"×7Ò7¸ÀXÑNÔN�å”i ¨kÐ :ÀÐBÑBÔBˆGˆGåœ, s¨C°½U¼[ÐQWÐXÑXÔX×^Ò^Ñ`Ô`ˆKØŒ~ Ò!Ð!Ø)×/Ò/°´°ÀÄÑPÔP�Ø×/Ò/°¸XÀsÑKÔKˆGàˆr9   Ú	input_idsÚattention_maskro   Ú	perm_maskrp   Útoken_type_idsÚ
input_maskÚinputs_embedsÚuse_memsr[   Úoutput_hidden_statesÚreturn_dictr¤   c                 óè  — |
�|
n| j         j        }
|�|n| j         j        }|�|n| j         j        }| j        r|	�|	n| j         j        }	n|	�|	n| j         j        }	|�|�t          d¦  «        ‚|�C|                     dd¦  «         	                    ¦   «         }|j
        d         |j
        d         }}nT|�C|                     dd¦  «         	                    ¦   «         }|j
        d         |j
        d         }}nt          d¦  «        ‚|�(|                     dd¦  «         	                    ¦   «         nd}|�(|                     dd¦  «         	                    ¦   «         nd}|�(|                     dd¦  «         	                    ¦   «         nd}|�)|                     ddd¦  «         	                    ¦   «         nd}|�)|                     ddd¦  «         	                    ¦   «         nd}|�|d         �|d         j
        d         nd}||z   }| j        }| j        }| j        dk    r'|                      ||¦  «        }|dd…dd…ddf         }n%| j        dk    rd}nt          d	| j        › �¦  «        ‚|�|�
J d
¦   «         ‚	 |€|�d|z
  }|�|�|d         |z   }n|�|€	|d         }n	|€|�|}nd}|�}|dk    rMt!          j        |j
        d         ||g¦  «                             |¦  «        }t!          j        ||gd¬¦  «        }|€|dd…dd…dd…df         }n||dd…dd…dd…df         z  }|�|dk                         |¦  «        }|�˜t!          j        |¦  «                             |¦  «         }|dk    r?t!          j        t!          j        ||g¦  «                             |¦  «        |gd¬¦  «        }||dd…dd…ddf         z   dk                         |¦  «        }nd}|�|}n|                      |¦  «        }|                      |¦  «        }|�=| j                             |j
        d         |d¦  «        }|                      |¦  «        }nd}|�¥|dk    r<t!          j        ||gt           j        |¬¦  «        }t!          j        ||gd¬¦  «        }n|}|dd…df         |ddd…f         k                         ¦   «         }t4          j                             |d¬¦  «                             |¦  «        }nd}|                      ||||j        ¬¦  «        }|                      |¦  «        }d} |€dgt=          | j        ¦  «        z  }|
rg nd}!|rg nd}"tA          | j        ¦  «        D ]ˆ\  }#}$|	r | |  !                    |||#         ¦  «        fz   } |r|" "                    |�||fn|¦  «          |$||||||||#         ||
¬¦	  «	        }%|%dd…         \  }}|
r|! "                    |%d         ¦  «         Œ‰|r|" "                    |�||fn|¦  «         |                      |�|n|¦  «        }&|&                     ddd¦  «         	                    ¦   «         }&|	sd} |r5|�tG          d„ |"D ¦   «         ¦  «        }"ntG          d„ |"D ¦   «         ¦  «        }"|
r5|�tG          d„ |!D ¦   «         ¦  «        }!ntG          d„ |!D ¦   «         ¦  «        }!|stG          d„ |&| |"|!fD ¦   «         ¦  «        S tI          |&| |"|!¬¦  «        S )áª  
        mems (`list[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
            decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
            they have already been computed.

            `use_mems` has to be set to `True` to make use of `mems`.
        perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
            Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`:

            - if `perm_mask[k, i, j] = 0`, i attend to j in batch k;
            - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k.

            If not set, each token attends to all the others (full bidirectional attention). Only used during
            pretraining (to define factorization order) or for sequential decoding (generation).
        target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
            Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
            on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
            (generation).
        input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*):
            Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
            real tokens and 1 for padding which is kept for compatibility with the original code base.

            Mask values selected in `[0, 1]`:

            - 1 for tokens that are **masked**,
            - 0 for tokens that are **not masked**.

            You can only uses one of `input_mask` and `attention_mask`.
        use_mems (`bool`, *optional*):
            Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden
            states from previous forward passes to compute attention, which can significantly improve performance for
            sequential decoding tasks.
        NzDYou cannot specify both input_ids and inputs_embeds at the same timer   r   z5You have to specify either input_ids or inputs_embedsr   rE  rD  zUnsupported attention type: z8You can only use one of input_mask (uses 1 for padding) rG  rO   r:   rC  )rÖ   )r>  r=   rõ   )rm   rn   r*   rY   ro   rp   r[   c              3   ór   K  — | ]2}|D ]-}|                      d dd¦  «                             ¦   «         V — Œ.Œ3dS ©r   r   r   N©ÚpermuteÚ
contiguous)r  Úhsrd   s      r8   ú	<genexpr>z%XLNetModel.forward.<locals>.<genexpr>£  sM   è è € Ð%jÐ%jÈ"ÐgiÐ%jÐ%jÐbc a§i¢i°°1°aÑ&8Ô&8×&CÒ&CÑ&EÔ&EÐ%jÐ%jÐ%jÐ%jÐ%jÐ%jÐ%jr9   c              3   óh   K  — | ]-}|                      d dd¦  «                             ¦   «         V — Œ.dS r`  ra  )r  rd  s     r8   re  z%XLNetModel.forward.<locals>.<genexpr>¥  s>   è è € Ð%_Ð%_È2 b§j¢j°°A°qÑ&9Ô&9×&DÒ&DÑ&FÔ&FÐ%_Ð%_Ð%_Ð%_Ð%_Ð%_r9   c              3   óH   K  — | ]}t          d „ |D ¦   «         ¦  «        V — ŒdS )c              3   ój   K  — | ].}|                      d ddd¦  «                             ¦   «         V — Œ/dS ©r   r	   r   r   Nra  )r  Ú
att_streams     r8   re  z/XLNetModel.forward.<locals>.<genexpr>.<genexpr>«  sB   è è € ÐZÐZÈ*˜*×,Ò,¨Q°°1°aÑ8Ô8×CÒCÑEÔEÐZÐZÐZÐZÐZÐZr9   N)rô   ©r  Úts     r8   re  z%XLNetModel.forward.<locals>.<genexpr>ª  sK   è è € ð #ð #Ø_`•EÐZÐZÐXYÐZÑZÔZÑZÔZð#ð #ð #ð #ð #ð #r9   c              3   ój   K  — | ].}|                      d ddd¦  «                             ¦   «         V — Œ/dS ri  ra  rk  s     r8   re  z%XLNetModel.forward.<locals>.<genexpr>®  s@   è è € Ð"ZÐ"ZÈ! 1§9¢9¨Q°°1°aÑ#8Ô#8×#CÒ#CÑ#EÔ#EÐ"ZÐ"ZÐ"ZÐ"ZÐ"ZÐ"Zr9   c              3   ó   K  — | ]}|®|V — Œ	d S r�   rõ   )r  r(   s     r8   re  z%XLNetModel.forward.<locals>.<genexpr>±  s(   è è € ÐcÐc˜qÐUVÐUb˜ÐUbÐUbÐUbÐUbÐcÐcr9   )rñ   ro   r¢   rò   )%r6   r[   r[  r\  ÚtrainingÚuse_mems_trainÚuse_mems_evalr    Ú	transposerc  r@   rb  r>   r=   r  r1  r$   ÚzerosÚtorj   Úeyer  r3   rè   r´   rD   r   rS   Úone_hotrS  Úlenr  Ú	enumerater8  Úappendrô   rð   )'r5   rT  rU  ro   rV  rp   rW  rX  rY  rZ  r[   r[  r\  Úkwargsr-  r>  r.  rG   Údtype_floatr=   rZ   Ú	data_maskÚ	mems_maskÚnon_tgt_maskÚ
word_emb_krt   Ú
word_emb_qrx   Úmem_padÚcat_idsrY   r@  Únew_memsrò   r¢   ÚiÚlayer_modulery   rg   s'                                          r8   rz   zXLNetModel.forwardÒ  sc  € ðf 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàŒ=ð 	WØ#+Ð#7�x�x¸T¼[Ô=WˆHˆHà#+Ð#7�x�x¸T¼[Ô=VˆHð
 Ð  ]Ð%>ÝÐcÑdÔdÐdØÐ"Ø!×+Ò+¨A¨qÑ1Ô1×<Ò<Ñ>Ô>ˆIØ!œ¨Ô*¨I¬O¸AÔ,>�#ˆDˆDØÐ&Ø)×3Ò3°A°qÑ9Ô9×DÒDÑFÔFˆMØ%Ô+¨AÔ.°Ô0CÀAÔ0F�#ˆDˆDåÐTÑUÔUÐUàHVÐHb˜×1Ò1°!°QÑ7Ô7×BÒBÑDÔDÐDÐhlˆØ@JÐ@V�Z×)Ò)¨!¨QÑ/Ô/×:Ò:Ñ<Ô<Ð<Ð\`ˆ
ØHVÐHb˜×1Ò1°!°QÑ7Ô7×BÒBÑDÔDÐDÐhlˆØ?HÐ?T�I×%Ò% a¨¨AÑ.Ô.×9Ò9Ñ;Ô;Ð;ÐZ^ˆ	ØIWÐIc˜×/Ò/°°1°aÑ8Ô8×CÒCÑEÔEÐEÐimˆà#'Ð#3¸¸Q¼Ð8Kˆt�AŒwŒ}˜QÔÐÐQRˆØ�d‰{ˆà”jˆØ”ˆð Œ>˜UÒ"Ð"Ø×(Ò(¨¨tÑ4Ô4ˆIØ! ! ! ! Q Q Q¨¨dÐ"2Ô3ˆIˆIØŒ^˜tÒ#Ð#ØˆIˆIåÐL¸D¼NÐLÐLÑMÔMÐMð Ð! ^Ð%;Ð%;Ð=wÑ%;Ô%;Ð;ØgØÐ .Ð"<Ø˜~Ñ-ˆJØÐ! iÐ&;Ø" 4Ô(¨9Ñ4ˆIˆIØÐ#¨	Ð(9Ø" 4Ô(ˆIˆIØÐ IÐ$9Ø!ˆIˆIàˆIàÐ à�aŠxˆxÝ!œK¨¬¸Ô);¸TÀ3Ð(GÑHÔH×KÒKÈIÑVÔV�	Ý!œI y°)Ð&<À!ÐDÑDÔD�	ØÐ Ø% a a a¨¨¨¨A¨A¨A¨t mÔ4�	�	à˜Y q q q¨!¨!¨!¨Q¨Q¨Q° }Ô5Ñ5�	àÐ Ø" Qš×*Ò*¨;Ñ7Ô7ˆIàÐ Ý!œI d™OœO×.Ò.¨yÑ9Ô9Ð9ˆLØ�aŠxˆxÝ$œy­%¬+°t¸T°lÑ*CÔ*C×*FÒ*FÀyÑ*QÔ*QÐS_Ð)`ÐfhÐiÑiÔi�Ø&¨°a°a°a¸¸¸¸DÀ$Ð6FÔ)GÑGÈ1ÒL×PÒPÐQZÑ[Ô[ˆLˆLàˆLð Ð$Ø&ˆJˆJà×,Ò,¨YÑ7Ô7ˆJØ—<’< 
Ñ+Ô+ˆØÐ%Øœ×-Ò-¨nÔ.BÀ1Ô.EÀsÈBÑOÔOˆJð —|’| JÑ/Ô/ˆHˆHàˆHð Ð%à�aŠxˆxÝœ+ t¨S k½¼ÈFÐSÑSÔS�Ýœ) W¨nÐ$=À1ÐEÑEÔE��à(�ð & a a a¨ gÔ.°'¸$ÀÀÀ¸'Ô2BÒB×HÒHÑJÔJˆGÝ”m×+Ò+¨GÀÐ+ÑCÔC×FÒFÀ{ÑSÔSˆGˆGàˆGð ×3Ò3°D¸$ÀCÐPXÔP_Ð3Ñ`Ô`ˆØ—,’,˜wÑ'Ô'ˆàˆØˆ<Ø�6�C ¤
™OœOÑ+ˆDà,Ð6�R�R°$ˆ
Ø2Ð<˜˜¸ˆÝ(¨¬Ñ4Ô4ð 	.ð 	.‰OˆAˆ|Øð Kà# t§~¢~°hÀÀQÄÑ'HÔ'HÐ&JÑJ�Ø#ð aØ×$Ò$¸XÐ=Q h°Ð%9Ð%9ÐW_Ñ`Ô`Ð`à"�lØØØ(Ø%ØØØ˜!”WØ-Ø"3ð
ñ 
ô 
ˆGð ")¨¨!¨¤ÑˆH�hØ ð .Ø×!Ò! '¨!¤*Ñ-Ô-Ð-øð  ð 	]Ø× Ò ¸Ð9M (¨HÐ!5Ð!5ÐS[Ñ\Ô\Ð\à—’¨(Ð*>˜h˜hÀHÑMÔMˆð —’  1 aÑ(Ô(×3Ò3Ñ5Ô5ˆàð 	ØˆHàð 	`ØÐ#Ý %Ð%jÐ%jÐP]Ð%jÑ%jÔ%jÑ jÔ j��å %Ð%_Ð%_ÐQ^Ð%_Ñ%_Ô%_Ñ _Ô _�àð 	[ØÐ)å"ð #ð #Ødnð#ñ #ô #ñ ô �
�
õ #Ð"ZÐ"ZÈzÐ"ZÑ"ZÔ"ZÑZÔZ�
àð 	dÝÐcÐc V¨X°}ÀjÐ$QÐcÑcÔcÑcÔcÐcåØ$¨8À=Ð]gð
ñ 
ô 
ð 	
r9   r�   )NN)NNNNNNNNNNNN)r}   r~   r   r   r$  r(  r1  r8  r€   rA  rS  r   r$   ÚTensorÚboolrô   rð   rz   r�   r‚   s   @r8   rç   rç   T  sØ  ø€ € € € € ðð ð ð ð ð(#ð #ð #ð-ð -ð -ðð ð ð8 ð  ð  ð* ðð ð ñ „\ðð$ð $ð $ð $ðL ð *.Ø.2Ø$(Ø)-Ø.2Ø.2Ø*.Ø-1Ø $Ø)-Ø,0Ø#'ðb
ð b
à”< $Ñ&ðb
ð œ tÑ+ðb
ð Œl˜TÑ!ð	b
ð
 ”< $Ñ&ðb
ð œ tÑ+ðb
ð œ tÑ+ðb
ð ”L 4Ñ'ðb
ð ”| dÑ*ðb
ð ˜‘+ðb
ð   $™;ðb
ð # T™kðb
ð ˜D‘[ðb
ð 
Ð!Ñ	!ðb
ð b
ð b
ñ „^ðb
ð b
ð b
ð b
ð b
r9   rç   zt
    XLNet Model with a language modeling head on top (linear layer with weights tied to the input embeddings).
    c            !       óÆ  ‡ — e Zd ZddiZˆ fd„Zd„ Zd„ Z	 dd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd
e	j
        dz  de	j
        dz  de	j
        dz  de	j
        dz  de	j
        dz  de	j
        dz  de	j
        dz  de	j
        dz  de	j
        dz  dedz  dedz  dedz  dedz  dee	j
        z  deez  fd„¦   «         Zedee	j
                 de	j
        dee	j
                 fd„¦   «         Zˆ xZS )ÚXLNetLMHeadModelzlm_loss.weightz!transformer.word_embedding.weightc                 ó  •— t          ¦   «                              |¦  «         |j        | _        |j        | _        t	          |¦  «        | _        t          j        |j        |j	        d¬¦  «        | _
        |                      ¦   «          d S )NTr¾   )r   r   r  r  rç   rà   r   r†   r   r  Úlm_lossr   r4   s     €r8   r   zXLNetLMHeadModel.__init__À  su   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØÔ)ˆŒØ!Ô-ˆÔå% fÑ-Ô-ˆÔÝ”y ¤°Ô1BÈÐNÑNÔNˆŒð 	�ŠÑÔÐÐÐr9   c                 ó   — | j         S r�   ©r‹  r#  s    r8   Úget_output_embeddingsz&XLNetLMHeadModel.get_output_embeddingsË  s
   € ØŒ|Ðr9   c                 ó   — || _         d S r�   r�  r&  s     r8   Úset_output_embeddingsz&XLNetLMHeadModel.set_output_embeddingsÎ  s   € Ø%ˆŒˆˆr9   NFc                 óÌ  ‡— |j         d         }t          j        |dft          j        |j        ¬¦  «        }dŠ|r&t          j        |d d …‰ d …f         |gd¬¦  «        }nt          j        ||gd¬¦  «        }|j         d         }t          j        |||ft          j        |j        ¬¦  «        }	d|	d d …d d …df<   t          j        |d|ft          j        |j        ¬¦  «        }
d|
d d …ddf<   ||	|
|dœ}|rt          ˆfd	„|D ¦   «         ¦  «        |d
<   |                     dd ¦  «         |                     dd ¦  «         | 	                    ¦   «         D ]\  }}||vr|||<   Œ|S )Nr   r   rC  r   rO   rG  r:   )rT  rV  rp   rZ  c              3   ó<   •K  — | ]}|d ‰ …d d …d d …f         V — Œd S r�   rõ   )r  Ú
layer_pastÚoffsets     €r8   re  zAXLNetLMHeadModel.prepare_inputs_for_generation.<locals>.<genexpr>û  s<   øè è € Ð(fÐ(fÈ
¨°H°f°W°H¸a¸a¸aÀÀÀ°NÔ)CÐ(fÐ(fÐ(fÐ(fÐ(fÐ(fr9   ro   rU  Ú	use_cache)
r@   r$   rs  rD   r=   rj   rI  rô   ÚpopÚitems)r5   rT  Úpast_key_valuesrZ  Úis_first_iterationrz  Úeffective_batch_sizeÚdummy_tokenÚsequence_lengthrV  rp   Úmodel_inputsÚkeyÚvaluer”  s                 @r8   Úprepare_inputs_for_generationz.XLNetLMHeadModel.prepare_inputs_for_generationÑ  sÎ  ø€ ð  )œ¨qÔ1ÐÝ”kÐ#7¸Ð";Å5Ä:ÐV_ÔVfÐgÑgÔgˆð
 ˆàð 	CÝœ	 9¨Q¨Q¨Q°°°°¨[Ô#9¸;Ð"GÈQÐOÑOÔOˆIˆIåœ	 9¨kÐ":ÀÐBÑBÔBˆIð $œ/¨!Ô,ˆÝ”KØ! ?°OÐDÍEÌKÐ`iÔ`pð
ñ 
ô 
ˆ	ð "ˆ	�!�!�!�Q�Q�Q˜�(Ñõ œØ! 1 oÐ6½e¼kÐR[ÔRbð
ñ 
ô 
ˆð $'ˆ�q�q�q˜!˜R�xÑ ð #Ø"Ø,Ø ð	
ð 
ˆð ð 	gÝ#(Ð(fÐ(fÐ(fÐ(fÐVeÐ(fÑ(fÔ(fÑ#fÔ#fˆL˜Ñ ð 	�
Š
Ð# TÑ*Ô*Ð*à�
Š
�; Ñ%Ô%Ð%à Ÿ,š,™.œ.ð 	*ð 	*‰JˆC�Ø˜,Ð&Ð&Ø$)�˜SÑ!øàÐr9   r   rT  rU  ro   rV  rp   rW  rX  rY  ÚlabelsrZ  r[   r[  r\  Úlogits_to_keepr¤   c                 ó$  — |�|n| j         j        } | j        |f||||||||
|||dœ|¤Ž}|d         }t          |t          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|	�Tt          ¦   «         } ||                     d| 	                    d¦  «        ¦  «        |	                     d¦  «        ¦  «        }|s|f|dd…         z   }|�|f|z   n|S t          |||j        |j        |j        ¬¦  «        S )aÙ  
        mems (`list[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
            decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
            they have already been computed.

            `use_mems` has to be set to `True` to make use of `mems`.
        perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
            Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`:

            - if `perm_mask[k, i, j] = 0`, i attend to j in batch k;
            - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k.

            If not set, each token attends to all the others (full bidirectional attention). Only used during
            pretraining (to define factorization order) or for sequential decoding (generation).
        target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
            Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
            on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
            (generation).
        input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*):
            Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
            real tokens and 1 for padding which is kept for compatibility with the original code base.

            Mask values selected in `[0, 1]`:

            - 1 for tokens that are **masked**,
            - 0 for tokens that are **not masked**.

            You can only uses one of `input_mask` and `attention_mask`.
        labels (`torch.LongTensor` of shape `(batch_size, num_predict)`, *optional*):
            Labels for masked language modeling. `num_predict` corresponds to `target_mapping.shape[1]`. If
            `target_mapping` is `None`, then `num_predict` corresponds to `sequence_length`.

            The labels should correspond to the masked input words that should be predicted and depends on
            `target_mapping`. Note in order to perform standard auto-regressive language modeling a *<mask>* token has
            to be added to the `input_ids` (see the `prepare_inputs_for_generation` function and examples below)

            Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored, the loss
            is only computed for labels in `[0, ..., config.vocab_size]`
        use_mems (`bool`, *optional*):
            Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden
            states from previous forward passes to compute attention, which can significantly improve performance for
            sequential decoding tasks.

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, XLNetLMHeadModel
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("xlnet/xlnet-large-cased")
        >>> model = XLNetLMHeadModel.from_pretrained("xlnet/xlnet-large-cased")

        >>> # We show how to setup inputs to predict a next token using a bi-directional context.
        >>> input_ids = torch.tensor(
        ...     tokenizer.encode("Hello, my dog is very <mask>", add_special_tokens=False)
        ... ).unsqueeze(
        ...     0
        ... )  # We will predict the masked token
        >>> perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float)
        >>> perm_mask[:, :, -1] = 1.0  # Previous tokens don't see last token
        >>> target_mapping = torch.zeros(
        ...     (1, 1, input_ids.shape[1]), dtype=torch.float
        ... )  # Shape [1, 1, seq_length] => let's predict one token
        >>> target_mapping[
        ...     0, 0, -1
        ... ] = 1.0  # Our first (and only) prediction will be the last token of the sequence (the masked token)

        >>> outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping)
        >>> next_token_logits = outputs[
        ...     0
        ... ]  # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size]

        >>> # The same way can the XLNetLMHeadModel be used to be trained by standard auto-regressive language modeling.
        >>> input_ids = torch.tensor(
        ...     tokenizer.encode("Hello, my dog is very <mask>", add_special_tokens=False)
        ... ).unsqueeze(
        ...     0
        ... )  # We will predict the masked token
        >>> labels = torch.tensor(tokenizer.encode("cute", add_special_tokens=False)).unsqueeze(0)
        >>> assert labels.shape[0] == 1, "only one word will be predicted"
        >>> perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float)
        >>> perm_mask[
        ...     :, :, -1
        ... ] = 1.0  # Previous tokens don't see last token as is done in standard auto-regressive lm training
        >>> target_mapping = torch.zeros(
        ...     (1, 1, input_ids.shape[1]), dtype=torch.float
        ... )  # Shape [1, 1, seq_length] => let's predict one token
        >>> target_mapping[
        ...     0, 0, -1
        ... ] = 1.0  # Our first (and only) prediction will be the last token of the sequence (the masked token)

        >>> outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping, labels=labels)
        >>> loss = outputs.loss
        >>> next_token_logits = (
        ...     outputs.logits
        ... )  # Logits have shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size]
        ```N©rU  ro   rV  rp   rW  rX  rY  rZ  r[   r[  r\  r   r:   r   ©rø   rù   ro   r¢   rò   )r6   r\  rà   rŠ   ÚintÚslicer‹  r   ÚviewrÝ   r÷   ro   r¢   rò   )r5   rT  rU  ro   rV  rp   rW  rX  rY  r¡  rZ  r[   r[  r\  r¢  rz  Útransformer_outputsr¢   Úslice_indicesrù   rø   Úloss_fctrg   s                          r8   rz   zXLNetLMHeadModel.forward  sk  € ðj &1Ð%<�k�kÀ$Ä+ÔBYˆà.˜dÔ.Øð
à)ØØØ)Ø)Ø!Ø'ØØ/Ø!5Ø#ð
ð 
ð ð
ð 
Ðð  ,¨AÔ.ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐå'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨F¯KªK¸©O¬OÑ<Ô<¸f¿kºkÈ"¹o¼oÑNÔNˆDàð 	FØ�YÐ!4°Q°R°RÔ!8Ñ8ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå%ØØØ$Ô)Ø-Ô;Ø*Ô5ð
ñ 
ô 
ð 	
r9   Úbeam_idxc                 ó    ‡— ˆfd„| D ¦   «         S )zü
        This function is used to re-order the `mems` cache if [`~PreTrainedModel.beam_search`] or
        [`~PreTrainedModel.beam_sample`] is called. This is required to match `mems` with the correct beam_idx at every
        generation step.
        c                 ól   •— g | ]0}|                      d ‰                     |j        ¦  «        ¦  «        ‘Œ1S )r   )rB   rt  r=   )r  r“  r¬  s     €r8   r  z3XLNetLMHeadModel._reorder_cache.<locals>.<listcomp>­  s8   ø€ ÐbÐbÐbÈz�
×'Ò'¨¨8¯;ª;°zÔ7HÑ+IÔ+IÑJÔJÐbÐbÐbr9   rõ   )ro   r¬  s    `r8   Ú_reorder_cachezXLNetLMHeadModel._reorder_cache¦  s!   ø€ ð cÐbÐbÐbÐ]aÐbÑbÔbÐbr9   r|   )NNNNNNNNNNNNNr   )r}   r~   r   Ú_tied_weights_keysr   rŽ  r�  r   r   r$   r†  r‡  r¦  rô   r÷   rz   r€   ró   r¯  r�   r‚   s   @r8   r‰  r‰  ¸  s(  ø€ € € € € ð +Ð,OÐPÐð	ð 	ð 	ð 	ð 	ðð ð ð&ð &ð &ð RWð5ð 5ð 5ð 5ðn ð *.Ø.2Ø$(Ø)-Ø.2Ø.2Ø*.Ø-1Ø&*Ø $Ø)-Ø,0Ø#'Ø-.ð[
ð [
à”< $Ñ&ð[
ð œ tÑ+ð[
ð Œl˜TÑ!ð	[
ð
 ”< $Ñ&ð[
ð œ tÑ+ð[
ð œ tÑ+ð[
ð ”L 4Ñ'ð[
ð ”| dÑ*ð[
ð ”˜tÑ#ð[
ð ˜‘+ð[
ð   $™;ð[
ð # T™kð[
ð ˜D‘[ð[
ð ˜eœlÑ*ð[
ð" 
Ð'Ñ	'ð#[
ð [
ð [
ñ „^ð[
ðz ðc˜T %¤,Ô/ð c¸5¼<ð cÈDÐQVÔQ]ÔL^ð cð cð cñ „\ðcð cð cð cð cr9   r‰  z‘
    XLNet Model with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g.
    for GLUE tasks.
    c                   ó8  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
ej        dz  dej        dz  dedz  dedz  dedz  dedz  dee	z  fd„¦   «         Z
ˆ xZS )ÚXLNetForSequenceClassificationc                 ó0  •— t          ¦   «                              |¦  «         |j        | _        || _        t	          |¦  «        | _        t          |¦  «        | _        t          j	        |j
        |j        ¦  «        | _        |                      ¦   «          d S r�   )r   r   rÓ   r6   rç   rà   rÄ   Úsequence_summaryr   r†   r   Úlogits_projr   r4   s     €r8   r   z'XLNetForSequenceClassification.__init__·  s}   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒØˆŒå% fÑ-Ô-ˆÔÝ 4°VÑ <Ô <ˆÔÝœ9 V¤^°VÔ5FÑGÔGˆÔð 	�ŠÑÔÐÐÐr9   NrT  rU  ro   rV  rp   rW  rX  rY  r¡  rZ  r[   r[  r\  r¤   c                 óò  — |�|n| j         j        } | j        |f||||||||
|||dœ|¤Ž}|d         }|                      |¦  «        }|                      |¦  «        }d}|	��Z| j         j        €f| j        dk    rd| j         _        nN| j        dk    r7|	j        t          j	        k    s|	j        t          j
        k    rd| j         _        nd| j         _        | j         j        dk    rWt          ¦   «         }| j        dk    r1 ||                     ¦   «         |	                     ¦   «         ¦  «        }nŽ |||	¦  «        }n�| j         j        dk    rGt          ¦   «         } ||                     d| j        ¦  «        |	                     d¦  «        ¦  «        }n*| j         j        dk    rt          ¦   «         } |||	¦  «        }|s|f|dd…         z   }|�|f|z   n|S t!          |||j        |j        |j        ¬	¦  «        S )
a.
  
        mems (`list[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
            decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
            they have already been computed.

            `use_mems` has to be set to `True` to make use of `mems`.
        perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
            Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`:

            - if `perm_mask[k, i, j] = 0`, i attend to j in batch k;
            - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k.

            If not set, each token attends to all the others (full bidirectional attention). Only used during
            pretraining (to define factorization order) or for sequential decoding (generation).
        target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
            Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
            on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
            (generation).
        input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*):
            Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
            real tokens and 1 for padding which is kept for compatibility with the original code base.

            Mask values selected in `[0, 1]`:

            - 1 for tokens that are **masked**,
            - 0 for tokens that are **not masked**.

            You can only uses one of `input_mask` and `attention_mask`.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        use_mems (`bool`, *optional*):
            Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden
            states from previous forward passes to compute attention, which can significantly improve performance for
            sequential decoding tasks.
        Nr¤  r   r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr:   r¥  )r6   r\  rà   r´  rµ  Úproblem_typerÓ   r>   r$   rD   r¦  r   r¦   r   r¨  r   rü   ro   r¢   rò   )r5   rT  rU  ro   rV  rp   rW  rX  rY  r¡  rZ  r[   r[  r\  rz  r©  rg   rù   rø   r«  s                       r8   rz   z&XLNetForSequenceClassification.forwardÃ  sD  € ðp &1Ð%<�k�kÀ$Ä+ÔBYˆà.˜dÔ.Øð
à)ØØØ)Ø)Ø!Ø'ØØ/Ø!5Ø#ð
ð 
ð ð
ð 
Ðð % QÔ'ˆà×&Ò& vÑ.Ô.ˆØ×!Ò! &Ñ)Ô)ˆàˆØÑØŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ñ @Ô @À&Ç+Â+ÈbÁ/Ä/ÑRÔR��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�àð 	FØ�YÐ!4°Q°R°RÔ!8Ñ8ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå3ØØØ$Ô)Ø-Ô;Ø*Ô5ð
ñ 
ô 
ð 	
r9   ©NNNNNNNNNNNNN)r}   r~   r   r   r   r$   r†  r‡  rô   rü   rz   r�   r‚   s   @r8   r²  r²  °  s|  ø€ € € € € ð
ð 
ð 
ð 
ð 
ð ð *.Ø.2Ø$(Ø)-Ø.2Ø.2Ø*.Ø-1Ø&*Ø $Ø)-Ø,0Ø#'ðn
ð n
à”< $Ñ&ðn
ð œ tÑ+ðn
ð Œl˜TÑ!ð	n
ð
 ”< $Ñ&ðn
ð œ tÑ+ðn
ð œ tÑ+ðn
ð ”L 4Ñ'ðn
ð ”| dÑ*ðn
ð ”˜tÑ#ðn
ð ˜‘+ðn
ð   $™;ðn
ð # T™kðn
ð ˜D‘[ðn
ð  
Ð5Ñ	5ð!n
ð n
ð n
ñ „^ðn
ð n
ð n
ð n
ð n
r9   r²  c                   ó8  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
ej        dz  dej        dz  dedz  dedz  dedz  dedz  dee	z  fd„¦   «         Z
ˆ xZS )ÚXLNetForTokenClassificationc                 óú   •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        |j        ¦  «        | _        |  	                    ¦   «          d S r�   )
r   r   rÓ   rç   rà   r   r†   r    Ú
classifierr   r4   s     €r8   r   z$XLNetForTokenClassification.__init__7  óf   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå% fÑ-Ô-ˆÔÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐr9   NrT  rU  ro   rV  rp   rW  rX  rY  r¡  rZ  r[   r[  r\  r¤   c                 ó¬  — |�|n| j         j        }|                      |||||||||
|||¬¦  «        }|d         }|                      |¦  «        }d}|	�Ft	          ¦   «         } ||                     d| j        ¦  «        |	                     d¦  «        ¦  «        }|s|f|dd…         z   }|�|f|z   n|S t          |||j        |j	        |j
        ¬¦  «        S )a×
  
        mems (`list[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
            decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
            they have already been computed.

            `use_mems` has to be set to `True` to make use of `mems`.
        perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
            Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`:

            - if `perm_mask[k, i, j] = 0`, i attend to j in batch k;
            - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k.

            If not set, each token attends to all the others (full bidirectional attention). Only used during
            pretraining (to define factorization order) or for sequential decoding (generation).
        target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
            Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
            on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
            (generation).
        input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*):
            Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
            real tokens and 1 for padding which is kept for compatibility with the original code base.

            Mask values selected in `[0, 1]`:

            - 1 for tokens that are **masked**,
            - 0 for tokens that are **not masked**.

            You can only uses one of `input_mask` and `attention_mask`.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
            where *num_choices* is the size of the second dimension of the input tensors. (see *input_ids* above)
        use_mems (`bool`, *optional*):
            Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden
            states from previous forward passes to compute attention, which can significantly improve performance for
            sequential decoding tasks.emory states to speed up sequential decoding. If set to `True`, the model will use the hidden
            states from previous forward passes to compute attention, which can significantly improve performance for
            sequential decoding tasks.
        Nr¤  r   r:   r   r¥  )r6   r\  rà   r¿  r   r¨  rÓ   rÿ   ro   r¢   rò   )r5   rT  rU  ro   rV  rp   rW  rX  rY  r¡  rZ  r[   r[  r\  rz  ry   Úsequence_outputrù   rø   r«  rg   s                        r8   rz   z#XLNetForTokenClassification.forwardA  s  € ðr &1Ð%<�k�kÀ$Ä+ÔBYˆà×"Ò"ØØ)ØØØ)Ø)Ø!Ø'ØØ/Ø!5Ø#ð #ñ 
ô 
ˆð " !œ*ˆà—’ Ñ1Ô1ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬OÑ<Ô<¸f¿kºkÈ"¹o¼oÑNÔNˆDàð 	FØ�Y ¨¨¨¤Ñ,ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå0ØØØ”Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r9   r»  )r}   r~   r   r   r   r$   r†  r‡  rô   rÿ   rz   r�   r‚   s   @r8   r½  r½  5  s|  ø€ € € € € ðð ð ð ð ð ð *.Ø.2Ø$(Ø)-Ø.2Ø.2Ø*.Ø-1Ø&*Ø $Ø)-Ø,0Ø#'ð\
ð \
à”< $Ñ&ð\
ð œ tÑ+ð\
ð Œl˜TÑ!ð	\
ð
 ”< $Ñ&ð\
ð œ tÑ+ð\
ð œ tÑ+ð\
ð ”L 4Ñ'ð\
ð ”| dÑ*ð\
ð ”˜tÑ#ð\
ð ˜‘+ð\
ð   $™;ð\
ð # T™kð\
ð ˜D‘[ð\
ð  
Ð2Ñ	2ð!\
ð \
ð \
ñ „^ð\
ð \
ð \
ð \
ð \
r9   r½  c                   ó8  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
ej        dz  dej        dz  dedz  dedz  dedz  dedz  dee	z  fd„¦   «         Z
ˆ xZS )ÚXLNetForMultipleChoicec                 ó   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¦  «        | _        t          j        |j        d¦  «        | _	        |  
                    ¦   «          d S r”   )r   r   rç   rà   rÄ   r´  r   r†   r   rµ  r   r4   s     €r8   r   zXLNetForMultipleChoice.__init__£  si   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å% fÑ-Ô-ˆÔÝ 4°VÑ <Ô <ˆÔÝœ9 V¤^°QÑ7Ô7ˆÔð 	�ŠÑÔÐÐÐr9   NrT  rW  rX  rU  ro   rV  rp   rY  r¡  rZ  r[   r[  r\  r¤   c                 óä  — |�|n| j         j        }|�|j        d         n|j        d         }|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�=|                     d|                     d¦  «        |                     d¦  «        ¦  «        nd} | j        |f||||||||
|||dœ|¤Ž}|d         }|                      |¦  «        }|                      |¦  «        }|                     d|¦  «        }d}|	�-t          ¦   «         } |||	                     d¦  «        ¦  «        }|s|f|dd…         z   }|�|f|z   n|S t          |||j
        |j        |j        ¬¦  «        S )a&  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        input_mask (`torch.FloatTensor` of shape `batch_size, num_choices, sequence_length`, *optional*):
            Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
            real tokens and 1 for padding which is kept for compatibility with the original code base.

            Mask values selected in `[0, 1]`:

            - 1 for tokens that are **masked**,
            - 0 for tokens that are **not masked**.

            You can only uses one of `input_mask` and `attention_mask`.
        mems (`list[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
            decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
            they have already been computed.

            `use_mems` has to be set to `True` to make use of `mems`.
        perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
            Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`:

            - if `perm_mask[k, i, j] = 0`, i attend to j in batch k;
            - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k.

            If not set, each token attends to all the others (full bidirectional attention). Only used during
            pretraining (to define factorization order) or for sequential decoding (generation).
        target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
            Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
            on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
            (generation).
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
        use_mems (`bool`, *optional*):
            Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden
            states from previous forward passes to compute attention, which can significantly improve performance for
            sequential decoding tasks.
        Nr   r:   r³   )rW  rX  rU  ro   rV  rp   rY  rZ  r[   r[  r\  r   r¥  )r6   r\  r@   r¨  rÝ   rà   r´  rµ  r   r  ro   r¢   rò   )r5   rT  rW  rX  rU  ro   rV  rp   rY  r¡  rZ  r[   r[  r\  rz  Únum_choicesÚflat_input_idsÚflat_token_type_idsÚflat_attention_maskÚflat_input_maskÚflat_inputs_embedsr©  rg   rù   Úreshaped_logitsrø   r«  s                              r8   rz   zXLNetForMultipleChoice.forward­  sZ  € ðR &1Ð%<�k�kÀ$Ä+ÔBYˆà,5Ð,A�i”o aÔ(Ð(À}ÔGZÐ[\ÔG]ˆàCLÐCX˜Ÿš¨¨I¯NªN¸2Ñ,>Ô,>Ñ?Ô?Ð?Ð^bˆØR`ÐRl˜n×1Ò1°"°n×6IÒ6IÈ"Ñ6MÔ6MÑNÔNÐNÐrvÐØR`ÐRl˜n×1Ò1°"°n×6IÒ6IÈ"Ñ6MÔ6MÑNÔNÐNÐrvÐØFPÐF\˜*Ÿ/š/¨"¨j¯oªo¸bÑ.AÔ.AÑBÔBÐBÐbfˆð Ð(ð ×Ò˜r =×#5Ò#5°bÑ#9Ô#9¸=×;MÒ;MÈbÑ;QÔ;QÑRÔRÐRàð 	ð /˜dÔ.Øð
à.Ø&Ø.ØØØ)Ø,ØØ/Ø!5Ø#ð
ð 
ð ð
ð 
Ðð  % QÔ'ˆà×&Ò& vÑ.Ô.ˆØ×!Ò! &Ñ)Ô)ˆØ Ÿ+š+ b¨+Ñ6Ô6ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜O¨V¯[ª[¸©_¬_Ñ=Ô=ˆDàð 	FØ%Ð'Ð*=¸a¸b¸bÔ*AÑAˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå+ØØ"Ø$Ô)Ø-Ô;Ø*Ô5ð
ñ 
ô 
ð 	
r9   r»  )r}   r~   r   r   r   r$   r†  r‡  rô   r  rz   r�   r‚   s   @r8   rÄ  rÄ  ¡  s|  ø€ € € € € ðð ð ð ð ð ð *.Ø.2Ø*.Ø.2Ø$(Ø)-Ø.2Ø-1Ø&*Ø $Ø)-Ø,0Ø#'ð{
ð {
à”< $Ñ&ð{
ð œ tÑ+ð{
ð ”L 4Ñ'ð	{
ð
 œ tÑ+ð{
ð Œl˜TÑ!ð{
ð ”< $Ñ&ð{
ð œ tÑ+ð{
ð ”| dÑ*ð{
ð ”˜tÑ#ð{
ð ˜‘+ð{
ð   $™;ð{
ð # T™kð{
ð ˜D‘[ð{
ð  
Ð-Ñ	-ð!{
ð {
ð {
ñ „^ð{
ð {
ð {
ð {
ð {
r9   rÄ  zÞ
    XLNet Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear
    layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c            !       óN  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
ej        dz  dej        dz  dej        dz  dedz  dedz  dedz  dedz  dee	z  fd„¦   «         Z
ˆ xZS )ÚXLNetForQuestionAnsweringSimplec                 óú   •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        |j        ¦  «        | _        |  	                    ¦   «          d S r�   )
r   r   rÓ   rç   rà   r   r†   r    Ú
qa_outputsr   r4   s     €r8   r   z(XLNetForQuestionAnsweringSimple.__init__3  rÀ  r9   NrT  rU  ro   rV  rp   rW  rX  rY  r°   Úend_positionsrZ  r[   r[  r\  r¤   c                 ó´  — |�|n| j         j        } | j        |f|||||||||||dœ|¤Ž}|d         }|                      |¦  «        }|                     dd¬¦  «        \  }}|                     d¦  «                             ¦   «         }|                     d¦  «                             ¦   «         }d}|	�ç|
�åt          |	                     ¦   «         ¦  «        dk    r|	                     d¦  «        }	t          |
                     ¦   «         ¦  «        dk    r|
                     d¦  «        }
|                     d¦  «        }|	 	                    d|¦  «        }	|
 	                    d|¦  «        }
t          |¬¦  «        } |||	¦  «        } |||
¦  «        }||z   dz  }|s||f|dd…         z   }|�|f|z   n|S t          ||||j        |j        |j        ¬	¦  «        S )
r^  Nr¤  r   r   r:   rO   )Úignore_indexr   )rø   r  r  ro   r¢   rò   )r6   r\  rà   rÑ  Úsplitr¦   rc  rw  rÝ   rK  r   r  ro   r¢   rò   )r5   rT  rU  ro   rV  rp   rW  rX  rY  r°   rÒ  rZ  r[   r[  r\  rz  ry   rÂ  rù   r  r  Ú
total_lossÚignored_indexr«  Ú
start_lossÚend_lossrg   s                              r8   rz   z'XLNetForQuestionAnsweringSimple.forward=  s2  € ðj &1Ð%<�k�kÀ$Ä+ÔBYˆà"�$Ô"Øð
à)ØØØ)Ø)Ø!Ø'ØØ/Ø!5Ø#ð
ð 
ð ð
ð 
ˆð  " !œ*ˆà—’ Ñ1Ô1ˆØ#)§<¢<°°r <Ñ#:Ô#:Ñ ˆ�jØ#×+Ò+¨BÑ/Ô/×:Ò:Ñ<Ô<ˆØ×'Ò'¨Ñ+Ô+×6Ò6Ñ8Ô8ˆ
àˆ
ØÐ&¨=Ð+Då�?×'Ò'Ñ)Ô)Ñ*Ô*¨QÒ.Ð.Ø"1×"9Ò"9¸"Ñ"=Ô"=�Ý�=×%Ò%Ñ'Ô'Ñ(Ô(¨1Ò,Ð,Ø -× 5Ò 5°bÑ 9Ô 9�à(×-Ò-¨aÑ0Ô0ˆMØ-×3Ò3°A°}ÑEÔEˆOØ)×/Ò/°°=ÑAÔAˆMå'°]ÐCÑCÔCˆHØ!˜ ,°Ñ@Ô@ˆJØ�x 
¨MÑ:Ô:ˆHØ$ xÑ/°1Ñ4ˆJàð 	RØ" JÐ/°'¸!¸"¸"´+Ñ=ˆFØ/9Ð/E�Z�M FÑ*Ð*È6ÐQå4ØØ%Ø!Ø”Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r9   )NNNNNNNNNNNNNN)r}   r~   r   r   r   r$   r†  r‡  rô   r  rz   r�   r‚   s   @r8   rÏ  rÏ  ,  s‘  ø€ € € € € ðð ð ð ð ð ð *.Ø.2Ø$(Ø)-Ø.2Ø.2Ø*.Ø-1Ø/3Ø-1Ø $Ø)-Ø,0Ø#'ði
ð i
à”< $Ñ&ði
ð œ tÑ+ði
ð Œl˜TÑ!ð	i
ð
 ”< $Ñ&ði
ð œ tÑ+ði
ð œ tÑ+ði
ð ”L 4Ñ'ði
ð ”| dÑ*ði
ð œ¨Ñ,ði
ð ”| dÑ*ði
ð ˜‘+ði
ð   $™;ði
ð # T™kði
ð ˜D‘[ði
ð" 
Ð6Ñ	6ð#i
ð i
ð i
ñ „^ði
ð i
ð i
ð i
ð i
r9   rÏ  c            '       ó�  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
ej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dedz  dedz  dedz  dee	z  f$d„¦   «         Z
ˆ xZS )ÚXLNetForQuestionAnsweringc                 óB  •— t          ¦   «                              |¦  «         |j        | _        |j        | _        t	          |¦  «        | _        t          |¦  «        | _        t          |¦  «        | _	        t          |¦  «        | _        |                      ¦   «          d S r�   )r   r   Ústart_n_topÚ	end_n_toprç   rà   rž   r  r©   r  r¼   Úanswer_classr   r4   s     €r8   r   z"XLNetForQuestionAnswering.__init__¬  s‡   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô-ˆÔØÔ)ˆŒå% fÑ-Ô-ˆÔÝ2°6Ñ:Ô:ˆÔÝ.¨vÑ6Ô6ˆŒÝ2°6Ñ:Ô:ˆÔð 	�ŠÑÔÐÐÐr9   NrT  rU  ro   rV  rp   rW  rX  rY  r°   rÒ  Úis_impossiblerÀ   r£   rZ  r[   r[  r\  r¤   c                 ó®  — |�|n| j         j        } | j        |f|||||||||||dœ|¤Ž}|d         }|                      ||¬¦  «        }|dd…         }|	�õ|
�ó|	|
||fD ]1}|�-|                     ¦   «         dk    r|                     d¦  «         Œ2|                      ||	|¬¦  «        }t          ¦   «         } |||	¦  «        } |||
¦  «        }||z   dz  }|�A|�?|                      ||	|¬	¦  «        }t          j
        ¦   «         } |||¦  «        }||d
z  z  }|s|f|dd…         z   S t          ||j        |j        |j        ¬¦  «        S |                     ¦   «         \  } }!}"t          j                             |d¬¦  «        }#t%          j        |#| j        d¬¦  «        \  }$}%|%                     d¦  «                             dd|"¦  «        }&t%          j        |d|&¦  «        }'|'                     d¦  «                             d|!dd¦  «        }'|                     d¦  «                             |'¦  «        }(|�|                     d¦  «        nd}|                      |(|'|¬¦  «        }t          j                             |d¬¦  «        })t%          j        |)| j        d¬¦  «        \  }*}+|*                     d| j        | j        z  ¦  «        }*|+                     d| j        | j        z  ¦  «        }+t%          j        d||#¦  «        }'|                      ||'|¬¦  «        }|s|$|%|*|+|f}||dd…         z   S t          |$|%|*|+||j        |j        |j        ¬¦  «        S )aß  
        mems (`list[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
            decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
            they have already been computed.

            `use_mems` has to be set to `True` to make use of `mems`.
        perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
            Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`:

            - if `perm_mask[k, i, j] = 0`, i attend to j in batch k;
            - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k.

            If not set, each token attends to all the others (full bidirectional attention). Only used during
            pretraining (to define factorization order) or for sequential decoding (generation).
        target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
            Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
            on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
            (generation).
        input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*):
            Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
            real tokens and 1 for padding which is kept for compatibility with the original code base.

            Mask values selected in `[0, 1]`:

            - 1 for tokens that are **masked**,
            - 0 for tokens that are **not masked**.

            You can only uses one of `input_mask` and `attention_mask`.
        is_impossible (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels whether a question has an answer or no answer (SQuAD 2.0)
        cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the classification token to use as input for computing plausibility of the
            answer.
        p_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Optional mask of tokens which can't be in answers (e.g. [CLS], [PAD], ...). 1.0 means token should be
            masked. 0.0 mean token is not masked.
        use_mems (`bool`, *optional*):
            Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden
            states from previous forward passes to compute attention, which can significantly improve performance for
            sequential decoding tasks.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, XLNetForQuestionAnswering
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("xlnet/xlnet-base-cased")
        >>> model = XLNetForQuestionAnswering.from_pretrained("xlnet/xlnet-base-cased")

        >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(
        ...     0
        ... )  # Batch size 1
        >>> start_positions = torch.tensor([1])
        >>> end_positions = torch.tensor([3])
        >>> outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions)

        >>> loss = outputs.loss
        ```Nr¤  r   )r£   r   r:   )r°   r£   r   )r°   rÀ   r   )rø   ro   r¢   rò   rO   r³   )r¯   r£   z
blh,bl->bh)r¯   rÀ   )r  r	  r
  r  r  ro   r¢   rò   )r6   r\  rà   r  rP   Úsqueeze_r  r   rß  r   r   r  ro   r¢   rò   rÝ   rS   rT   r$   ÚtopkrÝ  rÜ   r´   rµ   Ú	expand_asrÞ  r¨  rQ   ),r5   rT  rU  ro   rV  rp   rW  rX  rY  r°   rÒ  rà  rÀ   r£   rZ  r[   r[  r\  rz  r©  r¢   r  ry   rF   r  r«  rØ  rÙ  rÖ  r  Úloss_fct_clsÚcls_lossr>  r¶   r·   Ústart_log_probsr  r	  Ústart_top_index_expr¯   Úhidden_states_expandedÚend_log_probsr
  r  s,                                               r8   rz   z!XLNetForQuestionAnswering.forward¹  s  € ðd &1Ð%<�k�kÀ$Ä+ÔBYˆà.˜dÔ.Øð
à)ØØØ)Ø)Ø!Ø'ØØ/Ø!5Ø#ð
ð 
ð ð
ð 
Ðð ,¨AÔ.ˆØ×(Ò(¨¸vÐ(ÑFÔFˆà% a b bÔ)ˆàÐ&¨=Ð+Dà% }°iÀÐOð #ð #�Ø�= Q§U¢U¡W¤W¨q¢[ [Ø—J’J˜r‘N”N�Nøð Ÿš¨ÈÐ`f˜ÑgÔgˆJå'Ñ)Ô)ˆHØ!˜ ,°Ñ@Ô@ˆJØ�x 
¨MÑ:Ô:ˆHØ$ xÑ/°1Ñ4ˆJàÐ$¨Ð)Bà!×.Ò.¨}ÈoÐirÐ.ÑsÔs�
Ý!Ô3Ñ5Ô5�Ø'˜<¨
°MÑBÔB�ð ˜h¨™nÑ,�
àð Ø"�}Ð':¸1¸2¸2Ô'>Ñ>Ð>å6Ø#Ø,Ô1Ø"5Ô"CØ2Ô=ð	ñ ô ð ð +×/Ò/Ñ1Ô1‰NˆC��sÝ œm×3Ò3°LÀbÐ3ÑIÔIˆOå38´:Ø Ô!1°rð4ñ 4ô 4Ñ0Ð ð #2×";Ò";¸BÑ"?Ô"?×"FÒ"FÀrÈ2ÈsÑ"SÔ"SÐÝ œ<¨°rÐ;NÑOÔOˆLØ'×1Ò1°!Ñ4Ô4×;Ò;¸BÀÀbÈ"ÑMÔMˆLà%2×%<Ò%<¸QÑ%?Ô%?×%IÒ%IØñ&ô &Ð"ð .4Ð-?�V×%Ò% bÑ)Ô)Ð)ÀTˆFØŸšÐ)?ÈlÐci˜ÑjÔjˆJÝœM×1Ò1°*À!Ð1ÑDÔDˆMå/4¬zØ˜tœ~°1ð0ñ 0ô 0Ñ,Ð˜}ð !2× 6Ò 6°r¸4Ô;KÈdÌnÑ;\Ñ ]Ô ]ÐØ)×.Ò.¨r°4Ô3CÀdÄnÑ3TÑUÔUˆMå œ<Ø˜m¨_ñô ˆLð ×*Ò*Ø¨LÀIð +ñ ô ˆJð ð Ø.°ÐARÐTaÐcmÐn�ØÐ!4°Q°R°RÔ!8Ñ8Ð8å6Ø(;Ø$3Ø&7Ø"/Ø)Ø,Ô1Ø"5Ô"CØ2Ô=ð	ñ 	ô 	ð 	r9   )NNNNNNNNNNNNNNNNN)r}   r~   r   r   r   r$   r†  r‡  rô   r  rz   r�   r‚   s   @r8   rÛ  rÛ  ª  sÐ  ø€ € € € € ðð ð ð ð ð ð *.Ø.2Ø$(Ø)-Ø.2Ø.2Ø*.Ø-1Ø/3Ø-1Ø-1Ø)-Ø&*Ø $Ø)-Ø,0Ø#'ð%uð uà”< $Ñ&ðuð œ tÑ+ðuð Œl˜TÑ!ð	uð
 ”< $Ñ&ðuð œ tÑ+ðuð œ tÑ+ðuð ”L 4Ñ'ðuð ”| dÑ*ðuð œ¨Ñ,ðuð ”| dÑ*ðuð ”| dÑ*ðuð ”< $Ñ&ðuð ”˜tÑ#ðuð ˜‘+ðuð    $™;ð!uð" # T™kð#uð$ ˜D‘[ð%uð( 
Ð0Ñ	0ð)uð uð uñ „^ðuð uð uð uð ur9   rÛ  )rÄ  rÛ  rÏ  r²  r½  r‰  rç   rß   )8r§   Úcollections.abcr   Údataclassesr   r$   r   Útorch.nnr   r   r   Ú r
   rä   Úactivationsr   r   Ú
generationr   Úmodeling_utilsr   Úpytorch_utilsr   Úutilsr   r   r   Úconfiguration_xlnetr   Ú
get_loggerr}   ÚloggerÚModuler   r„   r’   rž   r©   r¼   rÄ   rß   rð   r÷   rü   rÿ   r  r  r  rç   r‰  r²  r½  rÄ  rÏ  rÛ  Ú__all__rõ   r9   r8   ú<module>rù     sJ  ððð ð %Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø )Ð )Ð )Ð )Ð )Ð )Ø -Ð -Ð -Ð -Ð -Ð -Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,ð 
ˆÔ	˜HÑ	%Ô	%€ðtð tð tð tð t˜RœYñ tô tð tðnð ð ð ð �r”yñ ô ð ð.-ð -ð -ð -ð -�”ñ -ô -ð -ðb!ð !ð !ð !ð !˜RœYñ !ô !ð !ðJBð Bð Bð Bð B˜2œ9ñ Bô Bð BðL>ð >ð >ð >ð >˜RœYñ >ô >ð >ðD`ð `ð `ð `ð `˜2œ9ñ `ô `ð `ðF ðWð Wð Wð Wð W˜?ñ Wô Wñ „ðWð2 €ððñ ô ð
 ð<ð <ð <ð <ð <�{ñ <ô <ñ „ñô ð<ð& €ððñ ô ð
 ð<ð <ð <ð <ð <˜[ñ <ô <ñ „ñô ð<ð, €ððñ ô ð
 ð<ð <ð <ð <ð <¨;ñ <ô <ñ „ñô ð<ð& €ððñ ô ð
 ð<ð <ð <ð <ð <¨ñ <ô <ñ „ñô ð<ð& €ððñ ô ð
 ð<ð <ð <ð <ð < ;ñ <ô <ñ „ñô ð<ð* €ððñ ô ð
 ð<ð <ð <ð <ð <¨Kñ <ô <ñ „ñô ð<ð, €ððñ ô ð
 ð<ð <ð <ð <ð < kñ <ô <ñ „ñô ð<ðB ð`
ð `
ð `
ð `
ð `
Ð%ñ `
ô `
ñ „ð`
ðF €ððñ ô ð
pcð pcð pcð pcð pcÐ+¨_ñ pcô pcñô ð
pcðf €ððñ ô ð|
ð |
ð |
ð |
ð |
Ð%9ñ |
ô |
ñô ð|
ð~ ðh
ð h
ð h
ð h
ð h
Ð"6ñ h
ô h
ñ „ðh
ðV ðG
ð G
ð G
ð G
ð G
Ð1ñ G
ô G
ñ „ðG
ðT €ððñ ô ðu
ð u
ð u
ð u
ð u
Ð&:ñ u
ô u
ñô ðu
ðp ðDð Dð Dð Dð DÐ 4ñ Dô Dñ „ðDðN	ð 	ð 	€€€r9   