§
    ‚ŠtjÎ†  ã                   ó�  — d Z ddlZddlZddlmZ ddlmZmZmZ ddlm	Z
 ddlmZmZ ddlmZ dd	lmZmZmZmZmZmZmZ dd
lmZ ddlmZmZ ddlmZ  ej        e ¦  «        Z!e G d„ de¦  «        ¦   «         Z" G d„ dej#        ¦  «        Z$ G d„ dej#        ¦  «        Z% G d„ dej#        ¦  «        Z& G d„ dej#        ¦  «        Z' G d„ dej#        ¦  «        Z( G d„ dej#        ¦  «        Z) G d„ dej#        ¦  «        Z* G d„ dej#        ¦  «        Z+e G d „ d!e"¦  «        ¦   «         Z, G d"„ d#e"¦  «        Z- G d$„ d%ej#        ¦  «        Z. ed&¬'¦  «         G d(„ d)e"¦  «        ¦   «         Z/e G d*„ d+e"¦  «        ¦   «         Z0e G d,„ d-e"¦  «        ¦   «         Z1 G d.„ d/ej#        ¦  «        Z2e G d0„ d1e"¦  «        ¦   «         Z3d2„ Z4g d3¢Z5dS )4zPyTorch MPNet model.é    N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FNÚgelu)Úcreate_bidirectional_mask)ÚBaseModelOutputÚBaseModelOutputWithPoolingÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)Úauto_docstringÚloggingé   )ÚMPNetConfigc                   óX   ‡ — e Zd ZU eed<   dZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZ	S )ÚMPNetPreTrainedModelÚconfigÚmpnetc                 óv  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rt	          j        |j        ¦  «         dS t          |t          ¦  «        rQt	          j        |j	        t          j        |j	        j        d         ¦  «                             d¦  «        ¦  «         dS dS )zInitialize the weightséÿÿÿÿ©r   r   N)ÚsuperÚ_init_weightsÚ
isinstanceÚMPNetLMHeadÚinitÚzeros_ÚbiasÚMPNetEmbeddingsÚcopy_Úposition_idsÚtorchÚarangeÚshapeÚexpand)ÚselfÚmoduleÚ	__class__s     €úf/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/mpnet/modeling_mpnet.pyr    z"MPNetPreTrainedModel._init_weights0   s§   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�kÑ*Ô*ð 	iÝŒK˜œÑ$Ô$Ð$Ð$Ð$Ý˜¥Ñ0Ô0ð 	iÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÑhÔhÐhÐhÐhð	ið 	ió    )
Ú__name__Ú
__module__Ú__qualname__r   Ú__annotations__Úbase_model_prefixr)   Úno_gradr    Ú__classcell__©r/   s   @r0   r   r   +   sg   ø€ € € € € € àÐÐÑØÐà€U„]�_„_ðið ið ið iñ „_ðið ið ið ið ir1   r   c                   ó,   ‡ — e Zd Zˆ fd„Zdd„Zd„ Zˆ xZS )r&   c                 ó  •— t          ¦   «                              ¦   «          d| _        t          j        |j        |j        | j        ¬¦  «        | _        t          j        |j        |j        | j        ¬¦  «        | _	        t          j
        |j        |j        ¬¦  «        | _
        t          j        |j        ¦  «        | _        |                      dt!          j        |j        ¦  «                             d¦  «        d¬¦  «         d S )Nr   )Úpadding_idx©Úepsr(   r   F)Ú
persistent)r   Ú__init__r<   r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferr)   r*   r,   ©r-   r   r/   s     €r0   r@   zMPNetEmbeddings.__init__;   sê   ø€ Ý‰Œ×ÒÑÔÐØˆÔÝ!œ|¨FÔ,=¸vÔ?QÐ_cÔ_oÐpÑpÔpˆÔÝ#%¤<ØÔ*¨FÔ,>ÈDÔL\ð$
ñ $
ô $
ˆÔ õ œ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒØ×ÒØ�EœL¨Ô)GÑHÔH×OÒOÐPWÑXÔXÐejð 	ñ 	
ô 	
ð 	
ð 	
ð 	
r1   Nc                 óº  — |€-|�t          || j        ¦  «        }n|                      |¦  «        }|�|                     ¦   «         }n|                     ¦   «         d d…         }|d         }|€| j        d d …d |…f         }|€|                      |¦  «        }|                      |¦  «        }||z   }|                      |¦  «        }|                      |¦  «        }|S )Nr   r   )	Ú"create_position_ids_from_input_idsr<   Ú&create_position_ids_from_inputs_embedsÚsizer(   rD   rF   rG   rK   )	r-   Ú	input_idsr(   Úinputs_embedsÚkwargsÚinput_shapeÚ
seq_lengthrF   Ú
embeddingss	            r0   ÚforwardzMPNetEmbeddings.forwardI   sî   € ØÐØÐ$ÝAÀ)ÈTÔM]Ñ^Ô^��à#×JÒJÈ=ÑYÔY�àÐ Ø#Ÿ.š.Ñ*Ô*ˆKˆKà'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKà  ”^ˆ
àÐØÔ,¨Q¨Q¨Q°°°¨^Ô<ˆLàÐ Ø ×0Ò0°Ñ;Ô;ˆMØ"×6Ò6°|ÑDÔDÐà"Ð%8Ñ8ˆ
Ø—^’^ JÑ/Ô/ˆ
Ø—\’\ *Ñ-Ô-ˆ
ØÐr1   c                 ó  — |                      ¦   «         dd…         }|d         }t          j        | j        dz   || j        z   dz   t          j        |j        ¬¦  «        }|                     d¦  «                             |¦  «        S )z×
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nr   r   )ÚdtypeÚdevicer   )rQ   r)   r*   r<   Úlongr[   Ú	unsqueezer,   )r-   rS   rU   Úsequence_lengthr(   s        r0   rP   z6MPNetEmbeddings.create_position_ids_from_inputs_embedsc   s‡   € ð $×(Ò(Ñ*Ô*¨3¨B¨3Ô/ˆØ% aœ.ˆå”|ØÔ˜qÑ  /°DÔ4DÑ"DÀqÑ"HÕPUÔPZÐcpÔcwð
ñ 
ô 
ˆð ×%Ò% aÑ(Ô(×/Ò/°Ñ<Ô<Ð<r1   )NNN)r2   r3   r4   r@   rX   rP   r8   r9   s   @r0   r&   r&   :   s[   ø€ € € € € ð
ð 
ð 
ð 
ð 
ðð ð ð ð4=ð =ð =ð =ð =ð =ð =r1   r&   c                   ó,   ‡ — e Zd Zˆ fd„Z	 	 	 dd„Zˆ xZS )ÚMPNetSelfAttentionc                 ó¬  •— t          ¦   «                              ¦   «          |j        |j        z  dk    r0t	          |d¦  «        s t          d|j        › d|j        › d�¦  «        ‚|j        | _        t          |j        |j        z  ¦  «        | _        | j        | j        z  | _        t          j
        |j        | j        ¦  «        | _        t          j
        |j        | j        ¦  «        | _        t          j
        |j        | j        ¦  «        | _        t          j
        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        d S )Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú))r   r@   rC   Únum_attention_headsÚhasattrÚ
ValueErrorÚintÚattention_head_sizeÚall_head_sizer   ÚLinearÚqÚkÚvÚorI   Úattention_probs_dropout_probrK   rM   s     €r0   r@   zMPNetSelfAttention.__init__v   s2  ø€ Ý‰Œ×ÒÑÔÐØÔ Ô :Ñ:¸aÒ?Ð?ÍÐPVÐXhÑHiÔHiÐ?Ýð8 FÔ$6ð 8ð 8Ø Ô4ð8ð 8ð 8ñô ð ð
 $*Ô#=ˆÔ Ý#& vÔ'9¸FÔ<VÑ'VÑ#WÔ#WˆÔ Ø!Ô5¸Ô8PÑPˆÔå”˜6Ô-¨tÔ/AÑBÔBˆŒÝ”˜6Ô-¨tÔ/AÑBÔBˆŒÝ”˜6Ô-¨tÔ/AÑBÔBˆŒÝ”˜6Ô-¨vÔ/AÑBÔBˆŒå”z &Ô"EÑFÔFˆŒˆˆr1   NFc                 óÒ  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
t          j        ||	                     dd¦  «        ¦  «        }|t          j
        | j        ¦  «        z  }|�||z  }|�||z   }t          j                             |d¬¦  «        }|                      |¦  «        }t          j        ||
¦  «        }|                     dddd¦  «                             ¦   «         }|                     ¦   «         d d…         | j        fz   } |j        |Ž }|                      |¦  «        }|r||fn|f}|S )Nr   r   é   éþÿÿÿ©Údimr   r   )r+   rh   rk   ÚviewÚ	transposerl   rm   r)   ÚmatmulÚmathÚsqrtr   Ú
functionalÚsoftmaxrK   ÚpermuteÚ
contiguousrQ   ri   rn   )r-   Úhidden_statesÚattention_maskÚposition_biasÚoutput_attentionsrT   rU   Úhidden_shaperk   rl   rm   Úattention_scoresÚattention_probsÚcÚnew_c_shapern   Úoutputss                    r0   rX   zMPNetSelfAttention.forward‰   sÔ  € ð $Ô)¨#¨2¨#Ô.ˆØC˜ÐC bÐC¨$Ô*BÐCÐCˆØ�FŠF�=Ñ!Ô!×&Ò& |Ñ4Ô4×>Ò>¸qÀ!ÑDÔDˆØ�FŠF�=Ñ!Ô!×&Ò& |Ñ4Ô4×>Ò>¸qÀ!ÑDÔDˆØ�FŠF�=Ñ!Ô!×&Ò& |Ñ4Ô4×>Ò>¸qÀ!ÑDÔDˆõ !œ<¨¨1¯;ª;°r¸2Ñ+>Ô+>Ñ?Ô?ÐØ+­d¬i¸Ô8PÑ.QÔ.QÑQÐð Ð$Ø Ñ-ÐàÐ%Ø/°.Ñ@Ðõ œ-×/Ò/Ð0@ÀbÐ/ÑIÔIˆàŸ,š, Ñ7Ô7ˆåŒL˜¨!Ñ,Ô,ˆà�IŠI�a˜˜A˜qÑ!Ô!×,Ò,Ñ.Ô.ˆØ—f’f‘h”h˜s ˜s”m tÔ'9Ð&;Ñ;ˆØˆAŒF�KÐ ˆà�FŠF�1‰IŒIˆà*;ÐE�1�oÐ&Ð&À!ÀˆØˆr1   ©NNF©r2   r3   r4   r@   rX   r8   r9   s   @r0   r`   r`   u   s\   ø€ € € € € ðGð Gð Gð Gð Gð, ØØð'ð 'ð 'ð 'ð 'ð 'ð 'ð 'r1   r`   c                   ó,   ‡ — e Zd Zˆ fd„Z	 	 	 dd„Zˆ xZS )ÚMPNetAttentionc                 óö   •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          j        |j        |j        ¬¦  «        | _        t	          j        |j	        ¦  «        | _
        d S ©Nr=   )r   r@   r`   Úattnr   rG   rC   rH   rI   rJ   rK   rM   s     €r0   r@   zMPNetAttention.__init__´   s\   ø€ Ý‰Œ×ÒÑÔÐÝ& vÑ.Ô.ˆŒ	Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr1   NFc                 ó¶   — |                       ||||¬¦  «        }|                      |                      |d         ¦  «        |z   ¦  «        }|f|dd …         z   }|S )N)r�   r   r   )rŽ   rG   rK   )	r-   r~   r   r€   r�   rT   Úself_outputsÚattention_outputr‡   s	            r0   rX   zMPNetAttention.forwardº   sl   € ð —y’yØØØØ/ð	 !ñ 
ô 
ˆð  Ÿ>š>¨$¯,ª,°|ÀA´Ñ*GÔ*GÈ-Ñ*WÑXÔXÐØ#Ð%¨°Q°R°RÔ(8Ñ8ˆØˆr1   rˆ   r‰   r9   s   @r0   r‹   r‹   ³   sW   ø€ € € € € ð>ð >ð >ð >ð >ð ØØðð ð ð ð ð ð ð r1   r‹   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚMPNetIntermediatec                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j        t          ¦  «        rt          |j                 | _        d S |j        | _        d S ©N)r   r@   r   rj   rC   Úintermediate_sizeÚdenser!   Ú
hidden_actÚstrr	   Úintermediate_act_fnrM   s     €r0   r@   zMPNetIntermediate.__init__Ï   sn   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3KÑLÔLˆŒ
Ý�fÔ'­Ñ-Ô-ð 	9Ý'-¨fÔ.?Ô'@ˆDÔ$Ð$Ð$à'-Ô'8ˆDÔ$Ð$Ð$r1   r~   Úreturnc                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r•   )r—   rš   )r-   r~   s     r0   rX   zMPNetIntermediate.forward×   s,   € ØŸ
š
 =Ñ1Ô1ˆØ×0Ò0°Ñ?Ô?ˆØÐr1   ©r2   r3   r4   r@   r)   ÚTensorrX   r8   r9   s   @r0   r“   r“   Î   s^   ø€ € € € € ð9ð 9ð 9ð 9ð 9ð U¤\ð °e´lð ð ð ð ð ð ð ð r1   r“   c                   óP   ‡ — e Zd Zˆ fd„Zdej        dej        dej        fd„Zˆ xZS )ÚMPNetOutputc                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S r�   )r   r@   r   rj   r–   rC   r—   rG   rH   rI   rJ   rK   rM   s     €r0   r@   zMPNetOutput.__init__ß   sf   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ7¸Ô9KÑLÔLˆŒ
Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr1   r~   Úinput_tensorr›   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|S r•   )r—   rK   rG   )r-   r~   r¢   s      r0   rX   zMPNetOutput.forwardå   s@   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØŸš }°|Ñ'CÑDÔDˆØÐr1   r�   r9   s   @r0   r    r    Þ   si   ø€ € € € € ð>ð >ð >ð >ð >ð U¤\ð ÀÄð ÐRWÔR^ð ð ð ð ð ð ð ð r1   r    c                   ó,   ‡ — e Zd Zˆ fd„Z	 	 	 dd„Zˆ xZS )Ú
MPNetLayerc                 óÀ   •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          |¦  «        | _        t          |¦  «        | _        d S r•   )r   r@   r‹   Ú	attentionr“   Úintermediater    ÚoutputrM   s     €r0   r@   zMPNetLayer.__init__í   sK   ø€ Ý‰Œ×ÒÑÔÐÝ'¨Ñ/Ô/ˆŒÝ-¨fÑ5Ô5ˆÔÝ! &Ñ)Ô)ˆŒˆˆr1   NFc                 ó¾   — |                       ||||¬¦  «        }|d         }|dd …         }|                      |¦  «        }	|                      |	|¦  «        }
|
f|z   }|S )N)r€   r�   r   r   )r§   r¨   r©   )r-   r~   r   r€   r�   rT   Úself_attention_outputsr‘   r‡   Úintermediate_outputÚlayer_outputs              r0   rX   zMPNetLayer.forwardó   s   € ð "&§¢ØØØ'Ø/ð	 "0ñ "
ô "
Ðð 2°!Ô4ÐØ(¨¨¨Ô,ˆà"×/Ò/Ð0@ÑAÔAÐØ—{’{Ð#6Ð8HÑIÔIˆØ�/ GÑ+ˆØˆr1   rˆ   r‰   r9   s   @r0   r¥   r¥   ì   sW   ø€ € € € € ð*ð *ð *ð *ð *ð ØØðð ð ð ð ð ð ð r1   r¥   c                   ó~   ‡ — e Zd Zˆ fd„Z	 	 	 	 ddej        dej        dz  dededef
d	„Zdd„Ze	dd„¦   «         Z
ˆ xZS )ÚMPNetEncoderc                 ó&  •‡— t          ¦   «                              ¦   «          ‰| _        ‰j        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _	        t          j
        ‰j        | j        ¦  «        | _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS © )r¥   )Ú.0Ú_r   s     €r0   ú
<listcomp>z)MPNetEncoder.__init__.<locals>.<listcomp>  s!   ø€ Ð#`Ð#`Ð#`¸1¥J¨vÑ$6Ô$6Ð#`Ð#`Ð#`r1   )r   r@   r   rd   Ún_headsr   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayerrA   Úrelative_attention_num_bucketsÚrelative_attention_biasrM   s    `€r0   r@   zMPNetEncoder.__init__  s}   øø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ1ˆŒÝ”]Ð#`Ð#`Ð#`Ð#`ÅÀfÔF^Ñ@_Ô@_Ð#`Ñ#`Ô#`ÑaÔaˆŒ
Ý')¤|°FÔ4YÐ[_Ô[gÑ'hÔ'hˆÔ$Ð$Ð$r1   NFr~   r   r�   Úoutput_hidden_statesÚreturn_dictc                 ó>  — |                       |¦  «        }|rdnd }|rdnd }	t          | j        ¦  «        D ]0\  }
}|r||fz   } ||||fd|i|¤Ž}|d         }|r|	|d         fz   }	Œ1|r||fz   }|st          d„ |||	fD ¦   «         ¦  «        S t	          |||	¬¦  «        S )Nr²   r�   r   r   c              3   ó   K  — | ]}|®|V — Œ	d S r•   r²   )r³   rm   s     r0   ú	<genexpr>z'MPNetEncoder.forward.<locals>.<genexpr>3  s(   è è € ÐhÐh˜qÐZ[ÐZg˜ÐZgÐZgÐZgÐZgÐhÐhr1   )Úlast_hidden_stater~   Ú
attentions)Úcompute_position_biasÚ	enumeraterº   Útupler   )r-   r~   r   r�   r½   r¾   rT   r€   Úall_hidden_statesÚall_attentionsÚiÚlayer_moduleÚlayer_outputss                r0   rX   zMPNetEncoder.forward  s,  € ð ×2Ò2°=ÑAÔAˆØ"6Ð@˜B˜B¸DÐØ0Ð:˜˜°dˆÝ(¨¬Ñ4Ô4ð 	Fð 	F‰OˆAˆ|Ø#ð IØ$5¸Ð8HÑ$HÐ!à(˜LØØØðð ð #4ð	ð
 ðð ˆMð *¨!Ô,ˆMà ð FØ!/°=ÀÔ3CÐ2EÑ!E�øð  ð 	EØ 1°]Ð4DÑ DÐàð 	iÝÐhÐh ]Ð4EÀ~Ð$VÐhÑhÔhÑhÔhÐhÝØ+Ø+Ø%ð
ñ 
ô 
ð 	
r1   é    c                 ó®  — |                      d¦  «        |                      d¦  «        |                      d¦  «        }}}|�|d d …d d …d f         }|d d …d d d …f         }nTt          j        |t          j        ¬¦  «        d d …d f         }t          j        |t          j        ¬¦  «        d d d …f         }||z
  }	|                      |	|¬¦  «        }
|
                     |j        ¦  «        }
|                      |
¦  «        }|                     g d¢¦  «         	                    d¦  «        }| 
                    |d||f¦  «                             ¦   «         }|S )Nr   r   )rZ   )Únum_buckets)rq   r   r   r   )rQ   r)   r*   r\   Úrelative_position_bucketÚtor[   r¼   r|   r]   r,   r}   )r-   Úxr(   rÎ   ÚbszÚqlenÚklenÚcontext_positionÚmemory_positionÚrelative_positionÚ	rp_bucketÚvaluess               r0   rÄ   z"MPNetEncoder.compute_position_bias:  sG  € ØŸ&š& ™)œ) Q§V¢V¨A¡Y¤Y°·²°q±	´	�4ˆTˆØÐ#Ø+¨A¨A¨A¨q¨q¨q°$¨JÔ7ÐØ*¨1¨1¨1¨d°A°A°A¨:Ô6ˆOˆOå$œ|¨D½¼
ÐCÑCÔCÀAÀAÀAÀtÀGÔLÐÝ#œl¨4µu´zÐBÑBÔBÀ4ÈÈÈÀ7ÔKˆOà+Ð.>Ñ>Ðà×1Ò1Ð2CÐQ\Ð1Ñ]Ô]ˆ	Ø—L’L ¤Ñ*Ô*ˆ	Ø×-Ò-¨iÑ8Ô8ˆØ—’ 	 	 	Ñ*Ô*×4Ò4°QÑ7Ô7ˆØ—’  R¨¨tÐ4Ñ5Ô5×@Ò@ÑBÔBˆØˆr1   é€   c                 ó   — d}|  }|dz  }||dk                           t          j        ¦  «        |z  z  }t          j        |¦  «        }|dz  }||k     }|t          j        |                     ¦   «         |z  ¦  «        t          j        ||z  ¦  «        z  ||z
  z                        t          j        ¦  «        z   }t          j        |t          j        ||dz
  ¦  «        ¦  «        }|t          j	        |||¦  «        z  }|S )Nr   rq   r   )
rÐ   r)   r\   ÚabsÚlogÚfloatrx   ÚminÚ	full_likeÚwhere)r×   rÎ   Úmax_distanceÚretÚnÚ	max_exactÚis_smallÚval_if_larges           r0   rÏ   z%MPNetEncoder.relative_position_bucketL  sì   € àˆØÐˆà˜ÑˆØ��A’�zŠz�%œ*Ñ%Ô%¨Ñ3Ñ3ˆÝŒI�a‰LŒLˆà 1Ñ$ˆ	Ø�y’=ˆà ÝŒI�a—g’g‘i”i )Ñ+Ñ,Ô,­t¬x¸ÀyÑ8PÑ/QÔ/QÑQÐU`ÐclÑUlÑmß
Š"�UŒZ‰.Œ.ñˆõ ”y ­u¬¸|È[Ð[\É_Ñ/]Ô/]Ñ^Ô^ˆØ�uŒ{˜8 Q¨Ñ5Ô5Ñ5ˆØˆ
r1   )NFFF)NrÌ   )rÌ   rÚ   )r2   r3   r4   r@   r)   rž   ÚboolrX   rÄ   ÚstaticmethodrÏ   r8   r9   s   @r0   r¯   r¯   
  sÒ   ø€ € € € € ðið ið ið ið ið /3Ø"'Ø%*Ø!ð&
ð &
à”|ð&
ð œ tÑ+ð&
ð  ð	&
ð
 #ð&
ð ð&
ð &
ð &
ð &
ðPð ð ð ð$ ðð ð ñ „\ðð ð ð ð r1   r¯   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚMPNetPoolerc                 óÀ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        ¦   «         | _        d S r•   )r   r@   r   rj   rC   r—   ÚTanhÚ
activationrM   s     €r0   r@   zMPNetPooler.__init__c  sC   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ'™)œ)ˆŒˆˆr1   r~   r›   c                 ór   — |d d …df         }|                       |¦  «        }|                      |¦  «        }|S ©Nr   )r—   rî   )r-   r~   Úfirst_token_tensorÚpooled_outputs       r0   rX   zMPNetPooler.forwardh  s@   € ð +¨1¨1¨1¨a¨4Ô0ÐØŸ
š
Ð#5Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐr1   r�   r9   s   @r0   rë   rë   b  s^   ø€ € € € € ð$ð $ð $ð $ð $ð
 U¤\ð °e´lð ð ð ð ð ð ð ð r1   rë   c                   óâ   ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Ze	 	 	 	 	 	 	 ddej        dz  dej	        dz  dej        dz  d	ej	        dz  d
e
dz  de
dz  de
dz  deej                 ez  fd„¦   «         Zˆ xZS )Ú
MPNetModelTc                 ó   •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        |rt          |¦  «        nd| _        |  	                    ¦   «          dS )zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        N)
r   r@   r   r&   rW   r¯   Úencoderrë   ÚpoolerÚ	post_init)r-   r   Úadd_pooling_layerr/   s      €r0   r@   zMPNetModel.__init__s  ss   ø€ õ
 	‰Œ×Ò˜Ñ Ô Ð ØˆŒå)¨&Ñ1Ô1ˆŒÝ# FÑ+Ô+ˆŒØ->ÐH•k &Ñ)Ô)Ð)ÀDˆŒð 	�ŠÑÔÐÐÐr1   c                 ó   — | j         j        S r•   ©rW   rD   ©r-   s    r0   Úget_input_embeddingszMPNetModel.get_input_embeddings‚  s   € ØŒÔ.Ð.r1   c                 ó   — || j         _        d S r•   rû   )r-   Úvalues     r0   Úset_input_embeddingszMPNetModel.set_input_embeddings…  s   € Ø*/ˆŒÔ'Ð'Ð'r1   NrR   r   r(   rS   r�   r½   r¾   r›   c                 óè  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|�|�t	          d¦  «        ‚|€|€t	          d¦  «        ‚|                      |||¬¦  «        }	t          | j         |	|¬¦  «        }|                      |	||||¬¦  «        }
|
d         }| j        �|                      |¦  «        nd }|s||f|
dd …         z   S t          |||
j
        |
j        ¬¦  «        S )	NzDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embeds)rR   r(   rS   )r   rS   r   )r   r�   r½   r¾   r   r   )rÂ   Úpooler_outputr~   rÃ   )r   r�   r½   r¾   rf   rW   r   rö   r÷   r   r~   rÃ   )r-   rR   r   r(   rS   r�   r½   r¾   rT   Úembedding_outputÚencoder_outputsÚsequence_outputrò   s                r0   rX   zMPNetModel.forwardˆ  sT  € ð 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàÐ  ]Ð%>ÝÐcÑdÔdÐdØÐ =Ð#8ÝÐTÑUÔUÐUàŸ?š?°YÈ\Ðiv˜?ÑwÔwÐå2Ø”;Ø*Ø)ð
ñ 
ô 
ˆð Ÿ,š,ØØ)Ø/Ø!5Ø#ð 'ñ 
ô 
ˆð *¨!Ô,ˆØ8<¼Ð8O˜Ÿš OÑ4Ô4Ð4ÐUYˆàð 	JØ# ]Ð3°oÀaÀbÀbÔ6IÑIÐIå)Ø-Ø'Ø)Ô7Ø&Ô1ð	
ñ 
ô 
ð 	
r1   )T)NNNNNNN)r2   r3   r4   r@   rý   r   r   r)   Ú
LongTensorÚFloatTensorrè   rÆ   rž   r   rX   r8   r9   s   @r0   rô   rô   q  s  ø€ € € € € ðð ð ð ð ð ð/ð /ð /ð0ð 0ð 0ð ð .2Ø37Ø04Ø26Ø)-Ø,0Ø#'ð0
ð 0
àÔ# dÑ*ð0
ð Ô)¨DÑ0ð0
ð Ô&¨Ñ-ð	0
ð
 Ô(¨4Ñ/ð0
ð   $™;ð0
ð # T™kð0
ð ˜D‘[ð0
ð 
ˆuŒ|Ô	Ð9Ñ	9ð0
ð 0
ð 0
ñ „^ð0
ð 0
ð 0
ð 0
ð 0
r1   rô   c                   ó   ‡ — e Zd ZdddœZˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 ddej	        dz  d	ej
        dz  d
ej	        dz  dej
        dz  dej	        dz  dedz  dedz  dedz  deej                 ez  fd„¦   «         Zˆ xZS )ÚMPNetForMaskedLMz'mpnet.embeddings.word_embeddings.weightzlm_head.bias)zlm_head.decoder.weightzlm_head.decoder.biasc                 óÆ   •— t          ¦   «                              |¦  «         t          |d¬¦  «        | _        t	          |¦  «        | _        |                      ¦   «          d S ©NF)rù   )r   r@   rô   r   r"   Úlm_headrø   rM   s     €r0   r@   zMPNetForMaskedLM.__init__Â  sV   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å ¸%Ð@Ñ@Ô@ˆŒ
Ý" 6Ñ*Ô*ˆŒð 	�ŠÑÔÐÐÐr1   c                 ó   — | j         j        S r•   )r  Údecoderrü   s    r0   Úget_output_embeddingsz&MPNetForMaskedLM.get_output_embeddingsË  s   € ØŒ|Ô#Ð#r1   c                 ó@   — || j         _        |j        | j         _        d S r•   )r  r  r%   )r-   Únew_embeddingss     r0   Úset_output_embeddingsz&MPNetForMaskedLM.set_output_embeddingsÎ  s   € Ø-ˆŒÔØ*Ô/ˆŒÔÐÐr1   NrR   r   r(   rS   Úlabelsr�   r½   r¾   r›   c	           	      ó   — |�|n| j         j        }|                      |||||||¬¦  «        }
|
d         }|                      |¦  «        }d}|�Kt	          ¦   «         } ||                     d| j         j        ¦  «        |                     d¦  «        ¦  «        }|s|f|
dd…         z   }|�|f|z   n|S t          |||
j        |
j	        ¬¦  «        S )a¢  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        N©r   r(   rS   r�   r½   r¾   r   r   rq   ©ÚlossÚlogitsr~   rÃ   )
r   r¾   r   r  r   ru   rB   r   r~   rÃ   )r-   rR   r   r(   rS   r  r�   r½   r¾   rT   r‡   r  Úprediction_scoresÚmasked_lm_lossÚloss_fctr©   s                   r0   rX   zMPNetForMaskedLM.forwardÒ  s  € ð& &1Ð%<�k�kÀ$Ä+ÔBYˆà—*’*ØØ)Ø%Ø'Ø/Ø!5Ø#ð ñ 
ô 
ˆð " !œ*ˆØ ŸLšL¨Ñ9Ô9ÐàˆØÐÝ'Ñ)Ô)ˆHØ%˜XÐ&7×&<Ò&<¸RÀÄÔAWÑ&XÔ&XÐZ`×ZeÒZeÐfhÑZiÔZiÑjÔjˆNàð 	ZØ'Ð)¨G°A°B°B¬KÑ7ˆFØ3AÐ3M�^Ð%¨Ñ.Ð.ÐSYÐYåØØ$Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r1   ©NNNNNNNN)r2   r3   r4   Ú_tied_weights_keysr@   r  r  r   r)   r  r  rè   rÆ   rž   r   rX   r8   r9   s   @r0   r	  r	  ¼  s@  ø€ € € € € à"KØ .ðð Ðð
ð ð ð ð ð$ð $ð $ð0ð 0ð 0ð ð .2Ø37Ø04Ø26Ø*.Ø)-Ø,0Ø#'ð/
ð /
àÔ# dÑ*ð/
ð Ô)¨DÑ0ð/
ð Ô&¨Ñ-ð	/
ð
 Ô(¨4Ñ/ð/
ð Ô  4Ñ'ð/
ð   $™;ð/
ð # T™kð/
ð ˜D‘[ð/
ð 
ˆuŒ|Ô	˜~Ñ	-ð/
ð /
ð /
ñ „^ð/
ð /
ð /
ð /
ð /
r1   r	  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )r"   z5MPNet Head for masked and permuted language modeling.c                 ó†  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j        |j	        d¬¦  «        | _
        t          j        t          j        |j	        ¦  «        ¦  «        | _        d S )Nr=   T)r%   )r   r@   r   rj   rC   r—   rG   rH   Ú
layer_normrB   r  Ú	Parameterr)   Úzerosr%   rM   s     €r0   r@   zMPNetLMHead.__init__  sŽ   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒå”y Ô!3°VÔ5FÈTÐRÑRÔRˆŒÝ”L¥¤¨VÔ->Ñ!?Ô!?Ñ@Ô@ˆŒ	ˆ	ˆ	r1   c                 ó¢   — |                       |¦  «        }t          |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r•   )r—   r
   r   r  ©r-   ÚfeaturesrT   rÑ   s       r0   rX   zMPNetLMHead.forward  sE   € Ø�JŠJ�xÑ Ô ˆÝ�‰GŒGˆØ�OŠO˜AÑÔˆð �LŠL˜‰OŒOˆàˆr1   ©r2   r3   r4   Ú__doc__r@   rX   r8   r9   s   @r0   r"   r"     sR   ø€ € € € € Ø?Ð?ðAð Að Að Að Aðð ð ð ð ð ð r1   r"   z�
    MPNet Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    )Úcustom_introc                   óê   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  d	edz  d
edz  de	ej
                 ez  fd„¦   «         Zˆ xZS )ÚMPNetForSequenceClassificationc                 óÞ   •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S r  )r   r@   Ú
num_labelsrô   r   ÚMPNetClassificationHeadÚ
classifierrø   rM   s     €r0   r@   z'MPNetForSequenceClassification.__init__"  s`   ø€ Ý‰Œ×Ò˜Ñ Ô Ð à Ô+ˆŒÝ ¸%Ð@Ñ@Ô@ˆŒ
Ý1°&Ñ9Ô9ˆŒð 	�ŠÑÔÐÐÐr1   NrR   r   r(   rS   r  r�   r½   r¾   r›   c	           	      óÀ  — |�|n| j         j        }|                      |||||||¬¦  «        }
|
d         }|                      |¦  «        }d}|��Z| j         j        €f| j        dk    rd| j         _        nN| j        dk    r7|j        t          j        k    s|j        t          j	        k    rd| j         _        nd| j         _        | j         j        dk    rWt          ¦   «         }| j        dk    r1 ||                     ¦   «         |                     ¦   «         ¦  «        }nŽ |||¦  «        }n�| j         j        dk    rGt          ¦   «         } ||                     d| j        ¦  «        |                     d¦  «        ¦  «        }n*| j         j        dk    rt          ¦   «         } |||¦  «        }|s|f|
d	d…         z   }|�|f|z   n|S t          |||
j        |
j        ¬
¦  «        S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr  r   r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr   rq   r  )r   r¾   r   r.  Úproblem_typer,  rZ   r)   r\   rg   r   Úsqueezer   ru   r   r   r~   rÃ   ©r-   rR   r   r(   rS   r  r�   r½   r¾   rT   r‡   r  r  r  r  r©   s                   r0   rX   z&MPNetForSequenceClassification.forward,  s  € ð( &1Ð%<�k�kÀ$Ä+ÔBYˆà—*’*ØØ)Ø%Ø'Ø/Ø!5Ø#ð ñ 
ô 
ˆð " !œ*ˆØ—’ Ñ1Ô1ˆàˆØÑØŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ñ @Ô @À&Ç+Â+ÈbÁ/Ä/ÑRÔR��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�Øð 	FØ�Y ¨¨¨¤Ñ,ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r1   r  )r2   r3   r4   r@   r   r)   r  r  rè   rÆ   rž   r   rX   r8   r9   s   @r0   r*  r*    s"  ø€ € € € € ðð ð ð ð ð ð .2Ø37Ø04Ø26Ø*.Ø)-Ø,0Ø#'ð@
ð @
àÔ# dÑ*ð@
ð Ô)¨DÑ0ð@
ð Ô&¨Ñ-ð	@
ð
 Ô(¨4Ñ/ð@
ð Ô  4Ñ'ð@
ð   $™;ð@
ð # T™kð@
ð ˜D‘[ð@
ð 
ˆuŒ|Ô	Ð7Ñ	7ð@
ð @
ð @
ñ „^ð@
ð @
ð @
ð @
ð @
r1   r*  c                   óê   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  d	edz  d
edz  de	ej
                 ez  fd„¦   «         Zˆ xZS )ÚMPNetForMultipleChoicec                 ó  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        ¦  «        | _        t	          j        |j	        d¦  «        | _
        |                      ¦   «          d S )Nr   )r   r@   rô   r   r   rI   rJ   rK   rj   rC   r.  rø   rM   s     €r0   r@   zMPNetForMultipleChoice.__init__r  sl   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å Ñ'Ô'ˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒÝœ) FÔ$6¸Ñ:Ô:ˆŒð 	�ŠÑÔÐÐÐr1   NrR   r   r(   rS   r  r�   r½   r¾   r›   c	           	      ó\  — |�|n| j         j        }|�|j        d         n|j        d         }
|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�=|                     d|                     d¦  «        |                     d¦  «        ¦  «        nd}|                      |||||||¬¦  «        }|d         }|                      |¦  «        }|                      |¦  «        }|                     d|
¦  «        }d}|�t          ¦   «         } |||¦  «        }|s|f|dd…         z   }|�|f|z   n|S t          |||j
        |j        ¬¦  «        S )a¸  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   r   rr   )r(   r   rS   r�   r½   r¾   rq   r  )r   r¾   r+   ru   rQ   r   rK   r.  r   r   r~   rÃ   )r-   rR   r   r(   rS   r  r�   r½   r¾   rT   Únum_choicesÚflat_input_idsÚflat_position_idsÚflat_attention_maskÚflat_inputs_embedsr‡   rò   r  Úreshaped_logitsr  r  r©   s                         r0   rX   zMPNetForMultipleChoice.forward|  sù  € ðH &1Ð%<�k�kÀ$Ä+ÔBYˆØ,5Ð,A�i”o aÔ(Ð(À}ÔGZÐ[\ÔG]ˆàCLÐCX˜Ÿš¨¨I¯NªN¸2Ñ,>Ô,>Ñ?Ô?Ð?Ð^bˆØLXÐLd˜L×-Ò-¨b°,×2CÒ2CÀBÑ2GÔ2GÑHÔHÐHÐjnÐØR`ÐRl˜n×1Ò1°"°n×6IÒ6IÈ"Ñ6MÔ6MÑNÔNÐNÐrvÐð Ð(ð ×Ò˜r =×#5Ò#5°bÑ#9Ô#9¸=×;MÒ;MÈbÑ;QÔ;QÑRÔRÐRàð 	ð —*’*ØØ*Ø.Ø,Ø/Ø!5Ø#ð ñ 
ô 
ˆð   œ
ˆàŸš ]Ñ3Ô3ˆØ—’ Ñ/Ô/ˆØ Ÿ+š+ b¨+Ñ6Ô6ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜O¨VÑ4Ô4ˆDàð 	FØ%Ð'¨'°!°"°"¬+Ñ5ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå(ØØ"Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r1   r  )r2   r3   r4   r@   r   r)   r  r  rè   rÆ   rž   r   rX   r8   r9   s   @r0   r7  r7  p  s"  ø€ € € € € ðð ð ð ð ð ð .2Ø37Ø04Ø26Ø*.Ø)-Ø,0Ø#'ðL
ð L
àÔ# dÑ*ðL
ð Ô)¨DÑ0ðL
ð Ô&¨Ñ-ð	L
ð
 Ô(¨4Ñ/ðL
ð Ô  4Ñ'ðL
ð   $™;ðL
ð # T™kðL
ð ˜D‘[ðL
ð 
ˆuŒ|Ô	Ð8Ñ	8ðL
ð L
ð L
ñ „^ðL
ð L
ð L
ð L
ð L
r1   r7  c                   óê   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  d	edz  d
edz  de	ej
                 ez  fd„¦   «         Zˆ xZS )ÚMPNetForTokenClassificationc                 ó:  •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        t          j        |j        ¦  «        | _        t          j	        |j
        |j        ¦  «        | _        |                      ¦   «          d S r  )r   r@   r,  rô   r   r   rI   rJ   rK   rj   rC   r.  rø   rM   s     €r0   r@   z$MPNetForTokenClassification.__init__Î  s~   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå ¸%Ð@Ñ@Ô@ˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐr1   NrR   r   r(   rS   r  r�   r½   r¾   r›   c	           	      óÀ  — |�|n| j         j        }|                      |||||||¬¦  «        }
|
d         }|                      |¦  «        }|                      |¦  «        }d}|�Ft          ¦   «         } ||                     d| j        ¦  «        |                     d¦  «        ¦  «        }|s|f|
dd…         z   }|�|f|z   n|S t          |||
j	        |
j
        ¬¦  «        S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Nr  r   r   rq   r  )r   r¾   r   rK   r.  r   ru   r,  r   r~   rÃ   r5  s                   r0   rX   z#MPNetForTokenClassification.forwardÙ  s  € ð$ &1Ð%<�k�kÀ$Ä+ÔBYˆà—*’*ØØ)Ø%Ø'Ø/Ø!5Ø#ð ñ 
ô 
ˆð " !œ*ˆàŸ,š, Ñ7Ô7ˆØ—’ Ñ1Ô1ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬OÑ<Ô<¸f¿kºkÈ"¹o¼oÑNÔNˆDàð 	FØ�Y ¨¨¨¤Ñ,ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r1   r  )r2   r3   r4   r@   r   r)   r  r  rè   rÆ   rž   r   rX   r8   r9   s   @r0   rA  rA  Ì  s  ø€ € € € € ð	ð 	ð 	ð 	ð 	ð ð .2Ø37Ø04Ø26Ø*.Ø)-Ø,0Ø#'ð0
ð 0
àÔ# dÑ*ð0
ð Ô)¨DÑ0ð0
ð Ô&¨Ñ-ð	0
ð
 Ô(¨4Ñ/ð0
ð Ô  4Ñ'ð0
ð   $™;ð0
ð # T™kð0
ð ˜D‘[ð0
ð 
ˆuŒ|Ô	Ð4Ñ	4ð0
ð 0
ð 0
ñ „^ð0
ð 0
ð 0
ð 0
ð 0
r1   rA  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )r-  z-Head for sentence-level classification tasks.c                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        t          j        |j        |j	        ¦  «        | _
        d S r•   )r   r@   r   rj   rC   r—   rI   rJ   rK   r,  Úout_projrM   s     €r0   r@   z MPNetClassificationHead.__init__  sc   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒÝœ	 &Ô"4°fÔ6GÑHÔHˆŒˆˆr1   c                 óô   — |d d …dd d …f         }|                       |¦  «        }|                      |¦  «        }t          j        |¦  «        }|                       |¦  «        }|                      |¦  «        }|S rð   )rK   r—   r)   ÚtanhrF  r$  s       r0   rX   zMPNetClassificationHead.forward  sj   € Ø�Q�Q�Q˜˜1˜1˜1�WÔˆØ�LŠL˜‰OŒOˆØ�JŠJ�q‰MŒMˆÝŒJ�q‰MŒMˆØ�LŠL˜‰OŒOˆØ�MŠM˜!ÑÔˆØˆr1   r&  r9   s   @r0   r-  r-    sR   ø€ € € € € Ø7Ð7ðIð Ið Ið Ið Iðð ð ð ð ð ð r1   r-  c                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	edz  d
edz  dedz  de	ej
                 ez  fd„¦   «         Zˆ xZS )ÚMPNetForQuestionAnsweringc                 óþ   •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        t          j        |j        |j        ¦  «        | _        |  	                    ¦   «          d S r  )
r   r@   r,  rô   r   r   rj   rC   Ú
qa_outputsrø   rM   s     €r0   r@   z"MPNetForQuestionAnswering.__init__"  sj   ø€ Ý‰Œ×Ò˜Ñ Ô Ð à Ô+ˆŒÝ ¸%Ð@Ñ@Ô@ˆŒ
Ýœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐr1   NrR   r   r(   rS   Ústart_positionsÚend_positionsr�   r½   r¾   r›   c
           	      ó¬  — |	�|	n| j         j        }	|                      |||||||	¬¦  «        }|d         }|                      |¦  «        }|                     dd¬¦  «        \  }}|                     d¦  «                             ¦   «         }|                     d¦  «                             ¦   «         }d }|�ç|�åt          |                     ¦   «         ¦  «        dk    r|                     d¦  «        }t          |                     ¦   «         ¦  «        dk    r|                     d¦  «        }|                     d¦  «        }| 	                    d|¦  «        }| 	                    d|¦  «        }t          |¬¦  «        } |||¦  «        } |||¦  «        }||z   dz  }|	s||f|dd …         z   }|�|f|z   n|S t          ||||j        |j        ¬¦  «        S )	Nr  r   r   r   rs   )Úignore_indexrq   )r  Ústart_logitsÚ
end_logitsr~   rÃ   )r   r¾   r   rL  Úsplitr4  r}   ÚlenrQ   Úclampr   r   r~   rÃ   )r-   rR   r   r(   rS   rM  rN  r�   r½   r¾   rT   r‡   r  r  rQ  rR  Ú
total_lossÚignored_indexr  Ú
start_lossÚend_lossr©   s                         r0   rX   z!MPNetForQuestionAnswering.forward,  s  € ð &1Ð%<�k�kÀ$Ä+ÔBYˆà—*’*ØØ)Ø%Ø'Ø/Ø!5Ø#ð ñ 
ô 
ˆð " !œ*ˆà—’ Ñ1Ô1ˆØ#)§<¢<°°r <Ñ#:Ô#:Ñ ˆ�jØ#×+Ò+¨BÑ/Ô/×:Ò:Ñ<Ô<ˆØ×'Ò'¨Ñ+Ô+×6Ò6Ñ8Ô8ˆ
àˆ
ØÐ&¨=Ð+Då�?×'Ò'Ñ)Ô)Ñ*Ô*¨QÒ.Ð.Ø"1×"9Ò"9¸"Ñ"=Ô"=�Ý�=×%Ò%Ñ'Ô'Ñ(Ô(¨1Ò,Ð,Ø -× 5Ò 5°bÑ 9Ô 9�à(×-Ò-¨aÑ0Ô0ˆMØ-×3Ò3°A°}ÑEÔEˆOØ)×/Ò/°°=ÑAÔAˆMå'°]ÐCÑCÔCˆHØ!˜ ,°Ñ@Ô@ˆJØ�x 
¨MÑ:Ô:ˆHØ$ xÑ/°1Ñ4ˆJàð 	RØ" JÐ/°'¸!¸"¸"´+Ñ=ˆFØ/9Ð/E�Z�M FÑ*Ð*È6ÐQå+ØØ%Ø!Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r1   )	NNNNNNNNN)r2   r3   r4   r@   r   r)   r  r  rè   rÆ   rž   r   rX   r8   r9   s   @r0   rJ  rJ     s%  ø€ € € € € ðð ð ð ð ð ð .2Ø37Ø04Ø26Ø37Ø15Ø)-Ø,0Ø#'ð;
ð ;
àÔ# dÑ*ð;
ð Ô)¨DÑ0ð;
ð Ô&¨Ñ-ð	;
ð
 Ô(¨4Ñ/ð;
ð Ô)¨DÑ0ð;
ð Ô'¨$Ñ.ð;
ð   $™;ð;
ð # T™kð;
ð ˜D‘[ð;
ð 
ˆuŒ|Ô	Ð;Ñ	;ð;
ð ;
ð ;
ñ „^ð;
ð ;
ð ;
ð ;
ð ;
r1   rJ  c                 óÖ   — |                       |¦  «                             ¦   «         }t          j        |d¬¦  «                             |¦  «        |z  }|                     ¦   «         |z   S )zñ
    Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
    are ignored. This is modified from fairseq's `utils.make_positions`. :param torch.Tensor x: :return torch.Tensor:
    r   rs   )Únerg   r)   ÚcumsumÚtype_asr\   )rR   r<   ÚmaskÚincremental_indicess       r0   rO   rO   k  s`   € ð �<Š<˜Ñ$Ô$×(Ò(Ñ*Ô*€DÝœ, t°Ð3Ñ3Ô3×;Ò;¸DÑAÔAÀDÑHÐØ×#Ò#Ñ%Ô%¨Ñ3Ð3r1   )r	  r7  rJ  r*  rA  r¥   rô   r   )6r'  rx   r)   r   Útorch.nnr   r   r   Ú r   r#   Úactivationsr	   r
   Úmasking_utilsr   Úmodeling_outputsr   r   r   r   r   r   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_mpnetr   Ú
get_loggerr2   Úloggerr   ÚModuler&   r`   r‹   r“   r    r¥   r¯   rë   rô   r	  r"   r*  r7  rA  r-  rJ  rO   Ú__all__r²   r1   r0   ú<module>rl     sá  ðð Ð à €€€à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø 'Ð 'Ð 'Ð 'Ð 'Ð 'Ð 'Ð 'Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð .Ð -Ð -Ð -Ð -Ð -Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,ð 
ˆÔ	˜HÑ	%Ô	%€ð ðið ið ið ið i˜?ñ iô iñ „ðið8=ð 8=ð 8=ð 8=ð 8=�b”iñ 8=ô 8=ð 8=ðv;ð ;ð ;ð ;ð ;˜œñ ;ô ;ð ;ð|ð ð ð ð �R”Yñ ô ð ð6ð ð ð ð ˜œ	ñ ô ð ð ð ð ð ð �"”)ñ ô ð ðð ð ð ð �”ñ ô ð ð<Tð Tð Tð Tð T�2”9ñ Tô Tð Tðpð ð ð ð �"”)ñ ô ð ð ðG
ð G
ð G
ð G
ð G
Ð%ñ G
ô G
ñ „ðG
ðTF
ð F
ð F
ð F
ð F
Ð+ñ F
ô F
ð F
ðRð ð ð ð �"”)ñ ô ð ð, €ððñ ô ðL
ð L
ð L
ð L
ð L
Ð%9ñ L
ô L
ñô ðL
ð^ ðX
ð X
ð X
ð X
ð X
Ð1ñ X
ô X
ñ „ðX
ðv ð=
ð =
ð =
ð =
ð =
Ð"6ñ =
ô =
ñ „ð=
ð@ð ð ð ð ˜bœiñ ô ð ð& ðG
ð G
ð G
ð G
ð G
Ð 4ñ G
ô G
ñ „ðG
ðT4ð 4ð 4ð	ð 	ð 	€€€r1   