§
    ‚Štjæ  ã                   óD  — d Z ddlmZ ddlZddlmZ ddlmZmZmZ ddl	m
Z ddlmZmZ dd	lmZmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZmZmZmZm Z m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+m,Z,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2 ddl3m4Z4  e-j5        e6¦  «        Z7 G d„ dej8        ¦  «        Z9	 	 dOdej8        dej:        dej:        dej:        dej:        dz  de;dz  de;d e'e+         fd!„Z< G d"„ d#ej8        ¦  «        Z= G d$„ d%ej8        ¦  «        Z> G d&„ d'ej8        ¦  «        Z? G d(„ d)ej8        ¦  «        Z@ G d*„ d+ej8        ¦  «        ZA G d,„ d-ej8        ¦  «        ZB G d.„ d/ej8        ¦  «        ZC G d0„ d1e¦  «        ZD G d2„ d3ej8        ¦  «        ZE G d4„ d5ej8        ¦  «        ZFe, G d6„ d7e%¦  «        ¦   «         ZG e,d8¬9¦  «         G d:„ d;eG¦  «        ¦   «         ZH e,d<¬9¦  «         G d=„ d>eGe¦  «        ¦   «         ZIe, G d?„ d@eG¦  «        ¦   «         ZJ G dA„ dBej8        ¦  «        ZK e,dC¬9¦  «         G dD„ dEeG¦  «        ¦   «         ZLe, G dF„ dGeG¦  «        ¦   «         ZMe, G dH„ dIeG¦  «        ¦   «         ZN G dJ„ dKej8        ¦  «        ZOe, G dL„ dMeG¦  «        ¦   «         ZPg dN¢ZQdS )PzPyTorch X-MOD model.é    )ÚCallableN)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FNÚgelu)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úapply_chunking_to_forward)ÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputsé   )Ú
XmodConfigc                   óÆ   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  d	ed
ej	        fd„Z
ed„ ¦   «         Zedd„¦   «         Zˆ xZS )ÚXmodEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 óø  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        ¬¦  «        | _        t          j        |j        |j        ¦  «        | _	        t          j
        |j        |j        ¬¦  «        | _
        t          j        |j        ¦  «        | _        |                      dt!          j        |j        ¦  «                             d¦  «        d¬¦  «         |                      dt!          j        | j                             ¦   «         t           j        ¬¦  «        d¬¦  «         |j        | _        t          j        |j        |j        | j        ¬¦  «        | _        d S )	N)Úpadding_idx©ÚepsÚposition_ids©r%   éÿÿÿÿF)Ú
persistentÚtoken_type_ids)Údtype)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferÚtorchÚarangeÚmax_position_embeddingsÚexpandÚzerosr-   ÚsizeÚlongr*   Úposition_embeddings©ÚselfÚconfigÚ	__class__s     €úd/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/xmod/modeling_xmod.pyr4   zXmodEmbeddings.__init__6   sJ  ø€ Ý‰Œ×ÒÑÔÐÝ!œ|¨FÔ,=¸vÔ?QÐ_eÔ_rÐsÑsÔsˆÔÝ%'¤\°&Ô2HÈ&ÔJ\Ñ%]Ô%]ˆÔ"åœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒà×ÒØ�EœL¨Ô)GÑHÔH×OÒOÐPWÑXÔXÐejð 	ñ 	
ô 	
ð 	
ð 	×ÒØ�eœk¨$Ô*;×*@Ò*@Ñ*BÔ*BÍ%Ì*ÐUÑUÔUÐbgð 	ñ 	
ô 	
ð 	
ð "Ô.ˆÔÝ#%¤<ØÔ*¨FÔ,>ÈDÔL\ð$
ñ $
ô $
ˆÔ Ð Ð ó    Nr   Ú	input_idsr1   r-   Úinputs_embedsÚpast_key_values_lengthÚreturnc                 ó*  — |€:|�|                       || j        |¦  «        }n|                      || j        ¦  «        }|�|                     ¦   «         }n|                     ¦   «         d d…         }|\  }}|€§t	          | d¦  «        rl| j                             |j        ¦  «                             |j	        d         d¦  «        }	t          j        |	d|¬¦  «        }	|	                     ||¦  «        }n+t          j        |t          j        | j        j        ¬¦  «        }|€|                      |¦  «        }|                      |¦  «        }
||
z   }|                      |¦  «        }||z   }|                      |¦  «        }|                      |¦  «        }|S )Nr/   r1   r   r%   )ÚdimÚindex©r2   Údevice)Ú"create_position_ids_from_input_idsr*   Ú&create_position_ids_from_inputs_embedsrG   Úhasattrr1   ÚtorX   rE   ÚshaperB   ÚgatherrF   rH   r-   r9   r;   rI   r<   r@   )rK   rP   r1   r-   rQ   rR   Úinput_shapeÚ
batch_sizeÚ
seq_lengthÚbuffered_token_type_idsr;   Ú
embeddingsrI   s                rN   ÚforwardzXmodEmbeddings.forwardJ   s¨  € ð ÐØÐ$à#×FÒFØ˜tÔ/Ð1Gñ ô  ��ð  $×JÒJÈ=ÐZ^ÔZjÑkÔk�àÐ Ø#Ÿ.š.Ñ*Ô*ˆKˆKà'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKà!,Ñˆ
�Jð
 Ð!Ý�tÐ-Ñ.Ô.ð mà*.Ô*=×*@Ò*@ÀÔATÑ*UÔ*U×*\Ò*\Ð]iÔ]oÐpqÔ]rÐtvÑ*wÔ*wÐ'Ý*/¬,Ð7NÐTUÐ]iÐ*jÑ*jÔ*jÐ'Ø!8×!?Ò!?À
ÈJÑ!WÔ!W��å!&¤¨[ÅÄ
ÐSWÔSdÔSkÐ!lÑ!lÔ!l�àÐ Ø ×0Ò0°Ñ;Ô;ˆMØ $× :Ò :¸>Ñ JÔ JÐØ"Ð%:Ñ:ˆ
à"×6Ò6°|ÑDÔDÐØÐ"5Ñ5ˆ
à—^’^ JÑ/Ô/ˆ
Ø—\’\ *Ñ-Ô-ˆ
ØÐrO   c                 óú   — |                       ¦   «         dd…         }|d         }t          j        |dz   ||z   dz   t          j        | j        ¬¦  «        }|                     d¦  «                             |¦  «        S )z×
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nr/   r%   rW   r   )rG   rB   rC   rH   rX   Ú	unsqueezerE   )rQ   r*   r_   Úsequence_lengthr-   s        rN   rZ   z5XmodEmbeddings.create_position_ids_from_inputs_embedsz   s~   € ð $×(Ò(Ñ*Ô*¨3¨B¨3Ô/ˆØ% aœ.ˆå”|Ø˜!‰O˜_¨{Ñ:¸QÑ>ÅeÄjÐYfÔYmð
ñ 
ô 
ˆð ×%Ò% aÑ(Ô(×/Ò/°Ñ<Ô<Ð<rO   c                 óÜ   — |                       |¦  «                             ¦   «         }t          j        |d¬¦  «                             |¦  «        |z   |z  }|                     ¦   «         |z   S )a  
        Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
        are ignored. This is modified from fairseq's `utils.make_positions`.

        Args:
            x: torch.Tensor x:

        Returns: torch.Tensor
        r%   ©rU   )ÚneÚintrB   ÚcumsumÚtype_asrH   )rP   r*   rR   ÚmaskÚincremental_indicess        rN   rY   z1XmodEmbeddings.create_position_ids_from_input_idsŒ   sg   € ð �|Š|˜KÑ(Ô(×,Ò,Ñ.Ô.ˆÝ$œ|¨D°aÐ8Ñ8Ô8×@Ò@ÀÑFÔFÐI_Ñ_ÐcgÑgÐØ"×'Ò'Ñ)Ô)¨KÑ7Ð7rO   )NNNNr   )r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r4   rB   Ú
LongTensorÚFloatTensorrk   ÚTensorrd   ÚstaticmethodrZ   rY   Ú__classcell__©rM   s   @rN   r(   r(   3   s   ø€ € € € € ØQÐQð
ð 
ð 
ð 
ð 
ð, .2Ø26Ø04Ø26Ø&'ð.ð .àÔ# dÑ*ð.ð Ô(¨4Ñ/ð.ð Ô&¨Ñ-ð	.ð
 Ô(¨4Ñ/ð.ð !$ð.ð 
Œð.ð .ð .ð .ð` ð=ð =ñ „\ð=ð" ð8ð 8ð 8ñ „\ð8ð 8ð 8ð 8ð 8rO   r(   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingr@   Úkwargsc                 ó®  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |d¬¦  «        }t          j                             ||| j        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «         	                    ¦   «         }	|	|fS )Nr/   ç      à¿é   r   ri   )ÚpÚtrainingr%   )
rG   rB   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr@   r†   Ú
contiguous)
r{   r|   r}   r~   r   r€   r@   r�   Úattn_weightsÚattn_outputs
             rN   Úeager_attention_forwardrŽ   ž   sÈ   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$rO   c                   ó„   ‡ — e Zd Zd
ˆ fd„	Z	 	 ddej        dej        dz  dedz  dee	         de
ej                 f
d	„Zˆ xZS )ÚXmodSelfAttentionFNc                 óÄ  •— t          ¦   «                              ¦   «          |j        |j        z  dk    r0t	          |d¦  «        s t          d|j        › d|j        › d�¦  «        ‚|| _        |j        | _        t          |j        |j        z  ¦  «        | _        | j        | j        z  | _	        | j        dz  | _
        t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        t          j        |j        ¦  «        | _        |j        | _        || _        || _        d S ©Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)rƒ   )r3   r4   r7   Únum_attention_headsr[   Ú
ValueErrorrL   rk   Úattention_head_sizeÚall_head_sizer€   r   ÚLinearr|   r}   r~   r>   Úattention_probs_dropout_probr@   Ú
is_decoderÚ	is_causalÚ	layer_idx©rK   rL   rœ   r�   rM   s       €rN   r4   zXmodSelfAttention.__init__¼   sG  ø€ Ý‰Œ×ÒÑÔÐØÔ Ô :Ñ:¸aÒ?Ð?ÍÐPVÐXhÑHiÔHiÐ?Ýð8 FÔ$6ð 8ð 8Ø Ô4ð8ð 8ð 8ñô ð ð ˆŒà#)Ô#=ˆÔ Ý#& vÔ'9¸FÔ<VÑ'VÑ#WÔ#WˆÔ Ø!Ô5¸Ô8PÑPˆÔØÔ/°Ñ5ˆŒå”Y˜vÔ1°4Ô3EÑFÔFˆŒ
Ý”9˜VÔ/°Ô1CÑDÔDˆŒÝ”Y˜vÔ1°4Ô3EÑFÔFˆŒ
å”z &Ô"EÑFÔFˆŒà Ô+ˆŒØ"ˆŒØ"ˆŒˆˆrO   Úhidden_statesr   Úpast_key_valuesr�   rS   c                 óÈ  — |j         d d…         }g |¢d‘| j        ‘R } |                      |¦  «        j        |Ž                      dd¦  «        } |                      |¦  «        j        |Ž                      dd¦  «        } |                      |¦  «        j        |Ž                      dd¦  «        }	|�=|}
t          |t          ¦  «        r|j	        }
|
 
                    ||	| j        ¦  «        \  }}	t          j        | j        j        t           ¦  «        } || |||	|f| j        sdn| j        j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }||fS )Nr/   r%   r„   rz   ©r@   r€   )r]   r—   r|   Úviewrˆ   r}   r~   Ú
isinstancer   Úself_attention_cacheÚupdater�   r   Úget_interfacerL   Ú_attn_implementationrŽ   r†   r@   r…   r€   Úreshaper‹   )rK   rŸ   r   r    r�   r_   Úhidden_shapeÚquery_layerÚ	key_layerÚvalue_layerÚcurrent_past_key_valuesÚattention_interfacer�   rŒ   s                 rN   rd   zXmodSelfAttention.forwardÔ   s¨  € ð $Ô)¨#¨2¨#Ô.ˆØC˜ÐC bÐC¨$Ô*BÐCÐCˆð 5�d—j’j Ñ/Ô/Ô4°lÐC×MÒMÈaÐQRÑSÔSˆØ0�D—H’H˜]Ñ+Ô+Ô0°,Ð?×IÒIÈ!ÈQÑOÔOˆ	Ø4�d—j’j Ñ/Ô/Ô4°lÐC×MÒMÈaÐQRÑSÔSˆàÐ&à&5Ð#Ý˜/Õ+>Ñ?Ô?ð OØ*9Ô*NÐ'ð &=×%CÒ%CÀIÈ{Ð\`Ô\jÑ%kÔ%kÑ"ˆI�{å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}Ð@�C�C°$´,´.Ø”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ˜LÐ(Ð(rO   ©FN)NN)rp   rq   rr   r4   rB   rv   ru   r   r   r   Útuplerd   rx   ry   s   @rN   r�   r�   »   s©   ø€ € € € € ð#ð #ð #ð #ð #ð #ð6 48Ø(,ð	')ð ')à”|ð')ð Ô)¨DÑ0ð')ð  ™ð	')ð
 Ð+Ô,ð')ð 
ˆuŒ|Ô	ð')ð ')ð ')ð ')ð ')ð ')ð ')ð ')rO   r�   c                   óš   ‡ — e Zd Zdˆ fd„	Z	 	 	 ddej        dej        dz  dej        dz  dedz  dee	         d	e
ej                 fd
„Zˆ xZS )ÚXmodCrossAttentionFNc                 ó¬  •— t          ¦   «                              ¦   «          |j        |j        z  dk    r0t	          |d¦  «        s t          d|j        › d|j        › d�¦  «        ‚|| _        |j        | _        t          |j        |j        z  ¦  «        | _        | j        | j        z  | _	        | j        dz  | _
        t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        t          j        |j        ¦  «        | _        || _        || _        d S r’   )r3   r4   r7   r•   r[   r–   rL   rk   r—   r˜   r€   r   r™   r|   r}   r~   r>   rš   r@   rœ   r�   rž   s       €rN   r4   zXmodCrossAttention.__init__   s=  ø€ Ý‰Œ×ÒÑÔÐØÔ Ô :Ñ:¸aÒ?Ð?ÍÐPVÐXhÑHiÔHiÐ?Ýð8 FÔ$6ð 8ð 8Ø Ô4ð8ð 8ð 8ñô ð ð ˆŒà#)Ô#=ˆÔ Ý#& vÔ'9¸FÔ<VÑ'VÑ#WÔ#WˆÔ Ø!Ô5¸Ô8PÑPˆÔØÔ/°Ñ5ˆŒå”Y˜vÔ1°4Ô3EÑFÔFˆŒ
Ý”9˜VÔ/°Ô1CÑDÔDˆŒÝ”Y˜vÔ1°4Ô3EÑFÔFˆŒ
å”z &Ô"EÑFÔFˆŒà"ˆŒØ"ˆŒˆˆrO   rŸ   Úencoder_hidden_statesr   r    r�   rS   c                 óì  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|�|j                             | j        ¦  «        nd}	|�;|	r9|j        j	        | j                 j
        }
|j        j	        | j                 j        }nÈg |j         d d…         ¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }
|                      |¦  «                             |¦  «                             dd¦  «        }|�3|j                             |
|| j        ¦  «        \  }
}d|j        | j        <   t          j        | j        j        t&          ¦  «        } || ||
||f| j        sdn| j        j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }||fS )Nr/   r%   r„   FTrz   r¢   )r]   r—   r|   r£   rˆ   Ú
is_updatedÚgetr�   Úcross_attention_cacheÚlayersÚkeysÚvaluesr}   r~   r¦   r   r§   rL   r¨   rŽ   r†   r@   r…   r€   r©   r‹   )rK   rŸ   rµ   r   r    r�   r_   rª   r«   r·   r¬   r­   Úkv_shaper¯   r�   rŒ   s                   rN   rd   zXmodCrossAttention.forward  s-  € ð $Ô)¨#¨2¨#Ô.ˆàC˜ÐC bÐC¨$Ô*BÐCÐCˆð —j’j Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆàGVÐGb�_Ô/×3Ò3°D´NÑCÔCÐCÐhmˆ
ØÐ&¨:Ð&à'Ô=ÔDÀTÄ^ÔTÔYˆIØ)Ô?ÔFÀtÄ~ÔVÔ]ˆKˆKàXÐ.Ô4°S°b°SÔ9ÐX¸2ÐX¸tÔ?WÐXÐXˆHØŸšÐ!6Ñ7Ô7×<Ò<¸XÑFÔF×PÒPÐQRÐTUÑVÔVˆIØŸ*š*Ð%:Ñ;Ô;×@Ò@ÀÑJÔJ×TÒTÐUVÐXYÑZÔZˆKàÐ*à)8Ô)N×)UÒ)UØ˜{¨D¬Nñ*ô *Ñ&�	˜;ð >B�Ô*¨4¬>Ñ:å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}Ð@�C�C°$´,´.Ø”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ˜LÐ(Ð(rO   r°   )NNN)rp   rq   rr   r4   rB   rv   ru   r   r   r   r±   rd   rx   ry   s   @rN   r³   r³   ÿ   s¿   ø€ € € € € ð#ð #ð #ð #ð #ð #ð4 ;?Ø37Ø6:ð1)ð 1)à”|ð1)ð  %Ô0°4Ñ7ð1)ð Ô)¨DÑ0ð	1)ð
 -¨tÑ3ð1)ð Ð+Ô,ð1)ð 
ˆuŒ|Ô	ð1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)rO   r³   c                   óP   ‡ — e Zd Zˆ fd„Zdej        dej        dej        fd„Zˆ xZS )ÚXmodSelfOutputc                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j	        ¦  «        | _
        d S ©Nr+   )r3   r4   r   r™   r7   Údenser<   r=   r>   r?   r@   rJ   s     €rN   r4   zXmodSelfOutput.__init__M  sf   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆrO   rŸ   Úinput_tensorrS   c                 ód   — |                       |¦  «        }|                      |¦  «        }||z   }|S ©N)rÂ   r@   )rK   rŸ   rÃ   s      rN   rd   zXmodSelfOutput.forwardS  s4   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØ%¨Ñ4ˆØÐrO   ©rp   rq   rr   r4   rB   rv   rd   rx   ry   s   @rN   r¿   r¿   K  si   ø€ € € € € ð>ð >ð >ð >ð >ð U¤\ð ÀÄð ÐRWÔR^ð ð ð ð ð ð ð ð rO   r¿   c                   óÒ   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej        dej        dz  dej        dz  dej        dz  deeej                          dz  d	ee	         d
eej                 fd„Z
ˆ xZS )ÚXmodAttentionFNc                 óÜ   •— t          ¦   «                              ¦   «          || _        |rt          nt          } ||||¬¦  «        | _        t          |¦  «        | _        |j        | _        d S )N©rœ   r�   )	r3   r4   Úis_cross_attentionr³   r�   rK   r¿   ÚoutputÚpre_norm)rK   rL   rœ   r�   rË   Úattention_classrM   s         €rN   r4   zXmodAttention.__init__[  sf   ø€ Ý‰Œ×ÒÑÔÐØ"4ˆÔØ0BÐYÕ,Ð,ÕHYˆØ#�O F°iÈ9ÐUÑUÔUˆŒ	Ý$ VÑ,Ô,ˆŒàœˆŒˆˆrO   rŸ   r   rµ   Úencoder_attention_maskr    r�   rS   c                 óþ   — |}| j         r| j                             |¦  «        }| j        s|n|} | j        |f|||dœ|¤Ž\  }}	|                      ||¦  «        }| j         s| j                             |¦  «        }||	fS )N)rµ   r   r    )rÍ   rÌ   r<   rË   rK   )
rK   rŸ   r   rµ   rÏ   r    r�   ÚresidualÚattention_outputrŒ   s
             rN   rd   zXmodAttention.forwardd  s¸   € ð !ˆØŒ=ð 	AØ œK×1Ò1°-Ñ@Ô@ˆMà/3Ô/FÐb˜˜ÐLbˆØ)2¨¬Øð*
à"7Ø)Ø+ð	*
ð *
ð
 ð*
ð *
Ñ&Ð˜,ð  Ÿ;š;Ð'7¸ÑBÔBÐàŒ}ð 	GØ#œ{×4Ò4Ð5EÑFÔFÐà Ð-Ð-rO   )FNF©NNNN)rp   rq   rr   r4   rB   rv   ru   r±   r   r   rd   rx   ry   s   @rN   rÈ   rÈ   Z  sá   ø€ € € € € ð(ð (ð (ð (ð (ð (ð 48Ø:>Ø;?ØBFð.ð .à”|ð.ð Ô)¨DÑ0ð.ð  %Ô0°4Ñ7ð	.ð
 !&Ô 1°DÑ 8ð.ð ˜u UÔ%6Ô7Ô8¸4Ñ?ð.ð Ð+Ô,ð.ð 
ˆuŒ|Ô	ð.ð .ð .ð .ð .ð .ð .ð .rO   rÈ   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚXmodIntermediatec                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j        t          ¦  «        rt          |j                 | _        d S |j        | _        d S rÅ   )r3   r4   r   r™   r7   Úintermediate_sizerÂ   r¤   Ú
hidden_actÚstrr
   Úintermediate_act_fnrJ   s     €rN   r4   zXmodIntermediate.__init__ƒ  sn   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3KÑLÔLˆŒ
Ý�fÔ'­Ñ-Ô-ð 	9Ý'-¨fÔ.?Ô'@ˆDÔ$Ð$Ð$à'-Ô'8ˆDÔ$Ð$Ð$rO   rŸ   rS   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rÅ   )rÂ   rÚ   ©rK   rŸ   s     rN   rd   zXmodIntermediate.forward‹  s,   € ØŸ
š
 =Ñ1Ô1ˆØ×0Ò0°Ñ?Ô?ˆØÐrO   rÆ   ry   s   @rN   rÕ   rÕ   ‚  s^   ø€ € € € € ð9ð 9ð 9ð 9ð 9ð U¤\ð °e´lð ð ð ð ð ð ð ð rO   rÕ   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚXmodAdapterc                 ó~  •— t          ¦   «                              ¦   «          |j        |j        z  | _        t          j        |j        | j        ¦  «        | _        t          j        | j        |j        ¦  «        | _        t          |j
        t          ¦  «        rt          |j
                 | _        d S |j
        | _        d S rÅ   )r3   r4   r7   Úadapter_reduction_factorÚbottleneck_sizer   r™   Údense1Údense2r¤   rØ   rÙ   r
   Úadapter_act_fnrJ   s     €rN   r4   zXmodAdapter.__init__’  sš   ø€ Ý‰Œ×ÒÑÔÐØ%Ô1°VÔ5TÑTˆÔÝ”i Ô 2°DÔ4HÑIÔIˆŒÝ”i Ô 4°fÔ6HÑIÔIˆŒÝ�fÔ'­Ñ-Ô-ð 	4Ý"(¨Ô):Ô";ˆDÔÐÐà"(Ô"3ˆDÔÐÐrO   rŸ   rS   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rÅ   )râ   rä   rã   rÜ   s     rN   rd   zXmodAdapter.forwardœ  s=   € ØŸš MÑ2Ô2ˆØ×+Ò+¨MÑ:Ô:ˆØŸš MÑ2Ô2ˆØÐrO   rÆ   ry   s   @rN   rÞ   rÞ   ‘  s^   ø€ € € € € ð4ð 4ð 4ð 4ð 4ð U¤\ð °e´lð ð ð ð ð ð ð ð rO   rÞ   c                   ó‚   ‡ — e Zd Zˆ fd„Zdej        dej        dej        dej        fd„Zdej        dej        fd„Zˆ xZS )Ú
XmodOutputc                 ó<  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        |j	        | _	        t          j
        |j        ¦  «        | _        |j        r&t          j        |j        |j        ¬¦  «        | _        nd | _        |j        | _        t          j        i ¦  «        | _        |j        D ]&}t%          |¦  «        | j        t'          |¦  «        <   Œ'd S rÁ   )r3   r4   r   r™   r×   r7   rÂ   r<   r=   Úln_before_adapterr>   r?   r@   Úadapter_layer_normÚadapter_reuse_layer_normÚ
ModuleDictÚadapter_modulesÚ	languagesrÞ   rÙ   )rK   rL   ÚlanguagerM   s      €rN   r4   zXmodOutput.__init__¤  sû   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ7¸Ô9KÑLÔLˆŒ
Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒØ!'Ô!9ˆÔÝ”z &Ô"<Ñ=Ô=ˆŒØÔ$ð 	+Ý&(¤l°6Ô3EÈ6ÔK`Ð&aÑ&aÔ&aˆDÔ#Ð#à&*ˆDÔ#Ø(.Ô(GˆÔ%Ý!œ}¨RÑ0Ô0ˆÔØÔ(ð 	Fð 	FˆHÝ2=¸fÑ2EÔ2EˆDÔ ¥ X¡¤Ñ/Ð/ð	Fð 	FrO   rŸ   rÃ   Úlang_idsrS   c                 ó�   — |                       |¦  «        }|                      |¦  «        }||z   }|                      ||¦  «        }|S rÅ   )rÂ   r@   Úlang_adapter)rK   rŸ   rÃ   rð   s       rN   rd   zXmodOutput.forward³  sI   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØ%¨Ñ4ˆØ×)Ò)¨(°MÑBÔBˆØÐrO   c                 ó¢  — | j         s|}| j        �|                      |¦  «        }n| j        r|                      |¦  «        }| j         r|}t	          j        |¦  «        }t          | j                             ¦   «         ¦  «        D ].\  }}||k    }||         } | j        |         |¦  «        }	|	||<   Œ/|  	                    |¦  «        }||z  }|S rÅ   )
ré   rê   rë   r<   rB   Ú
zeros_likeÚ	enumeraterí   r»   r@   )
rK   rð   rŸ   rÑ   Únew_hidden_statesÚadapter_idxÚlang_keyÚ	lang_maskÚlang_hidden_statesÚadapted_lang_hidden_statess
             rN   rò   zXmodOutput.lang_adapterº  sø   € ØÔ%ð 	%Ø$ˆHàÔ"Ð.Ø ×3Ò3°MÑBÔBˆMˆMØÔ*ð 	:Ø ŸNšN¨=Ñ9Ô9ˆMàÔ!ð 	%Ø$ˆHå!Ô,¨]Ñ;Ô;ÐÝ%.¨tÔ/C×/HÒ/HÑ/JÔ/JÑ%KÔ%Kð 	Fð 	FÑ!ˆK˜Ø  KÒ/ˆIØ!.¨yÔ!9ÐØ)G¨Ô)=¸hÔ)GÐHZÑ)[Ô)[Ð&Ø+EÐ˜iÑ(Ð(àŸšÐ%6Ñ7Ô7ˆØ˜Ñ!ˆØÐrO   )	rp   rq   rr   r4   rB   rv   rd   rò   rx   ry   s   @rN   rç   rç   £  sŸ   ø€ € € € € ðFð Fð Fð Fð Fð U¤\ð ÀÄð ÐY^ÔYeð ÐjoÔjvð ð ð ð ð U¤\ð À%Ä,ð ð ð ð ð ð ð ð rO   rç   c                   óÚ   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej        dej        dej        dz  dej        dz  dej        dz  deeej                          dz  d	ee	         d
ej        fd„Z
d„ Zˆ xZS )Ú	XmodLayerNc                 ó®  •— t          ¦   «                              ¦   «          |j        | _        d| _        t	          ||j        |¬¦  «        | _        |j        | _        |j        | _        | j        r1| j        st          | › d�¦  «        ‚t	          |d|d¬¦  «        | _	        t          |¦  «        | _        t          |¦  «        | _        |j        | _        d S )Nr%   rÊ   z> should be used as a decoder model if cross attention is addedFT)rœ   r�   rË   )r3   r4   Úchunk_size_feed_forwardÚseq_len_dimrÈ   r›   Ú	attentionÚadd_cross_attentionr–   ÚcrossattentionrÕ   Úintermediaterç   rÌ   rÍ   )rK   rL   r�   rM   s      €rN   r4   zXmodLayer.__init__Ó  sÙ   ø€ Ý‰Œ×ÒÑÔÐØ'-Ô'EˆÔ$ØˆÔÝ& v¸Ô9JÐV_Ð`Ñ`Ô`ˆŒØ Ô+ˆŒØ#)Ô#=ˆÔ ØÔ#ð 	Ø”?ð jÝ  DÐ!hÐ!hÐ!hÑiÔiÐiÝ"/ØØØ#Ø#'ð	#ñ #ô #ˆDÔõ -¨VÑ4Ô4ˆÔÝ  Ñ(Ô(ˆŒØœˆŒˆˆrO   rŸ   rð   r   rµ   rÏ   r    r�   rS   c                 ó²  —  | j         ||fd|i|¤Ž\  }}	|}
| j        r=|�;t          | d¦  «        st          d| › d�¦  «        ‚ | j        |
d ||fd|i|¤Ž\  }}	|}
|
}| j        r| j                             |
¦  «        }
t          | j	        | j
        | j        |
¦  «        }|                      |||¦  «        }| j        s| j                             |¦  «        }|S )Nr    r  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`)r  r›   r[   r–   r  rÍ   rÌ   r<   r   Úfeed_forward_chunkrÿ   r   )rK   rŸ   rð   r   rµ   rÏ   r    r�   Úself_attention_outputÚ_rÒ   Úcross_attention_outputrÑ   Úintermediate_outputÚlayer_outputs                  rN   rd   zXmodLayer.forwardç  se  € ð $2 4¤>ØØð$
ð $
ð ,ð$
ð ð	$
ð $
Ñ Ð˜qð 1ÐàŒ?ð 	6Ð4Ð@Ý˜4Ð!1Ñ2Ô2ð Ý ðD¸dð Dð Dð Dñô ð ð
 )<¨Ô(;Ø ØØ%Ø&ð	)ð )ð
 !0ð)ð ð)ð )Ñ%Ð" Að  6Ðà#ˆØŒ=ð 	GØ#œ{×4Ò4Ð5EÑFÔFÐÝ7ØÔ#ØÔ(ØÔØñ	
ô 
Ðð —{’{Ð#6¸À(ÑKÔKˆØŒ}ð 	?Øœ;×0Ò0°Ñ>Ô>ˆLàÐrO   c                 ó,   — |                       |¦  «        S rÅ   )r  )rK   rÒ   s     rN   r  zXmodLayer.feed_forward_chunk  s   € Ø× Ò Ð!1Ñ2Ô2Ð2rO   rÅ   rÓ   )rp   rq   rr   r4   rB   rv   ru   r±   r   r   rd   r  rx   ry   s   @rN   rý   rý   Ò  sø   ø€ € € € € ð(ð (ð (ð (ð (ð (ð0 48Ø:>Ø;?ØBFð0ð 0à”|ð0ð ”,ð0ð Ô)¨DÑ0ð	0ð
  %Ô0°4Ñ7ð0ð !&Ô 1°DÑ 8ð0ð ˜u UÔ%6Ô7Ô8¸4Ñ?ð0ð Ð+Ô,ð0ð 
Œð0ð 0ð 0ð 0ðd3ð 3ð 3ð 3ð 3ð 3ð 3rO   rý   c                   óð   ‡ — e Zd Zˆ fd„Z	 	 	 	 	 ddej        dej        dej        dz  dej        dz  dej        dz  deeej                          dz  d	edz  d
e	e
         deej                 ez  fd„Zˆ xZS )ÚXmodEncoderc                 ó:  •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        ‰j        | _	        | j	        r't          j
        ‰j        ‰j        ¬¦  «        | _
        d S d S )Nc                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS ))r�   )rý   )Ú.0ÚirL   s     €rN   ú
<listcomp>z(XmodEncoder.__init__.<locals>.<listcomp>!  s&   ø€ Ð#lÐ#lÐ#lÀq¥I¨fÀÐ$BÑ$BÔ$BÐ#lÐ#lÐ#lrO   r+   )r3   r4   rL   r   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayerrÍ   Úis_pre_normr<   r7   r=   rJ   s    `€rN   r4   zXmodEncoder.__init__  s•   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”]Ð#lÐ#lÐ#lÐ#lÍEÐRXÔRjÑLkÔLkÐ#lÑ#lÔ#lÑmÔmˆŒ
Ø!œ?ˆÔØÔð 	YÝœ\¨&Ô*<À&ÔBWÐXÑXÔXˆDŒNˆNˆNð	Yð 	YrO   NrŸ   rð   r   rµ   rÏ   r    Ú	use_cacher�   rS   c           	      ó´   — t          | j        ¦  «        D ]\  }	}
 |
||||||fi |¤Ž}Œ| j        r|                      |¦  «        }t	          ||r|nd ¬¦  «        S )N)Úlast_hidden_stater    )rõ   r  r  r<   r   )rK   rŸ   rð   r   rµ   rÏ   r    r  r�   r  Úlayer_modules              rN   rd   zXmodEncoder.forward&  s™   € õ  )¨¬Ñ4Ô4ð 		ð 		‰OˆAˆ|Ø(˜LØØØØ%Ø&Øðð ð ðð ˆMˆMð Ôð 	:Ø ŸNšN¨=Ñ9Ô9ˆMå8Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
rO   )NNNNN)rp   rq   rr   r4   rB   rv   ru   r±   Úboolr   r   r   rd   rx   ry   s   @rN   r  r    s  ø€ € € € € ðYð Yð Yð Yð Yð 48Ø:>Ø;?ØBFØ!%ð
ð 
à”|ð
ð ”,ð
ð Ô)¨DÑ0ð	
ð
  %Ô0°4Ñ7ð
ð !&Ô 1°DÑ 8ð
ð ˜u UÔ%6Ô7Ô8¸4Ñ?ð
ð ˜$‘;ð
ð Ð+Ô,ð
ð 
ˆuŒ|Ô	ÐHÑ	Hð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
rO   r  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )Ú
XmodPoolerc                 óÀ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        ¦   «         | _        d S rÅ   )r3   r4   r   r™   r7   rÂ   ÚTanhÚ
activationrJ   s     €rN   r4   zXmodPooler.__init__G  sC   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ'™)œ)ˆŒˆˆrO   rŸ   rS   c                 ór   — |d d …df         }|                       |¦  «        }|                      |¦  «        }|S ©Nr   )rÂ   r"  )rK   rŸ   Úfirst_token_tensorÚpooled_outputs       rN   rd   zXmodPooler.forwardL  s@   € ð +¨1¨1¨1¨a¨4Ô0ÐØŸ
š
Ð#5Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐrO   rÆ   ry   s   @rN   r  r  F  s^   ø€ € € € € ð$ð $ð $ð $ð $ð
 U¤\ð °e´lð ð ð ð ð ð ð ð rO   r  c                   óŠ   ‡ — e Zd ZeZdZdZg d¢ZdZdZ	dZ
dZeeedœZ ej        ¦   «         ˆ fd„¦   «         Zdefd„Zd„ Zˆ xZS )	ÚXmodPreTrainedModelÚrobertaT)r(   r�   r³   )rŸ   Ú
attentionsÚcross_attentionsc                 ó¨  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rt	          j        |j        ¦  «         dS t          |t          ¦  «        rjt	          j        |j	        t          j        |j	        j        d         ¦  «                             d¦  «        ¦  «         t	          j        |j        ¦  «         dS dS )zInitialize the weightsr/   r.   N)r3   Ú_init_weightsr¤   Ú
XmodLMHeadÚinitÚzeros_Úbiasr(   Úcopy_r-   rB   rC   r]   rE   r1   )rK   r{   rM   s     €rN   r-  z!XmodPreTrainedModel._init_weightse  s¶   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�jÑ)Ô)ð 	/ÝŒK˜œÑ$Ô$Ð$Ð$Ð$Ý˜¥Ñ/Ô/ð 	/ÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÑhÔhÐhÝŒK˜Ô-Ñ.Ô.Ð.Ð.Ð.ð	/ð 	/rO   rï   c           	      ó–   — || j         j        vr.t          | › d|› dt          | j         j        ¦  «        › �¦  «        ‚|| j         _        dS )zÜ
        Set the default language code for the model. This is used when the language is not specified in the input.

        Args:
            language (`str`): The language code, such as `"en_XX"` or `"de_DE"`.
        z does not have an adapter for z. Supported languages: N)rL   rî   r–   ÚlistÚdefault_language)rK   rï   s     rN   Úset_default_languagez(XmodPreTrainedModel.set_default_languageo  sa   € ð ˜4œ;Ô0Ð0Ð0ÝØÐuÐu°xÐuÐuÕX\Ð]aÔ]hÔ]rÑXsÔXsÐuÐuñô ð ð (0ˆŒÔ$Ð$Ð$rO   c                 óž  — t                                d¦  «         | j        j                             ¦   «         D ]	}d|_        Œ
t                                d¦  «         | j        j        j        D ]^}|j        j	        �(|j        j	                             ¦   «         D ]	}d|_        Œ
|j        j
                             ¦   «         D ]	}d|_        Œ
Œ_dS )z¡
        Freeze the embeddings and language adapters of the model. Usually, this is applied before the model is
        fine-tuned on a downstream task.
        zFreezing embeddingsFzFreezing adaptersN)ÚloggerÚinfor)  rc   Ú
parametersÚrequires_gradÚencoderr  rÌ   rê   rí   )rK   Ú	parameterr  s      rN   Ú'freeze_embeddings_and_language_adaptersz;XmodPreTrainedModel.freeze_embeddings_and_language_adapters|  sÜ   € õ
 	�ŠÐ)Ñ*Ô*Ð*ØœÔ0×;Ò;Ñ=Ô=ð 	,ð 	,ˆIØ&+ˆIÔ#Ð#Ý�ŠÐ'Ñ(Ô(Ð(Ø”\Ô)Ô/ð 	0ð 	0ˆEØŒ|Ô.Ð:Ø!&¤Ô!@×!KÒ!KÑ!MÔ!Mð 4ð 4�IØ.3�IÔ+Ð+Ø"œ\Ô9×DÒDÑFÔFð 0ð 0�	Ø*/�	Ô'Ð'ð0ð		0ð 	0rO   )rp   rq   rr   r&   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚno_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendrý   r�   r³   Ú_can_record_outputsrB   Úno_gradr-  rÙ   r6  r>  rx   ry   s   @rN   r(  r(  U  sÀ   ø€ € € € € à€LØ!ÐØ&*Ð#ØTÐTÐTÐØÐØ€NØÐØ"&Ðà"Ø'Ø.ðð Ðð €U„]�_„_ð/ð /ð /ð /ñ „_ð/ð0¨Sð 0ð 0ð 0ð 0ð0ð 0ð 0ð 0ð 0ð 0ð 0rO   r(  a0  
    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in *Attention is
    all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz
    Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.

    .. _*Attention is all you need*: https://huggingface.co/papers/1706.03762
    )Úcustom_introc                   óz  ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Zeee	 	 	 	 	 	 	 	 	 	 dde	j
        dz  de	j        dz  de	j
        dz  d	e	j
        dz  d
e	j
        dz  de	j
        dz  de	j
        dz  de	j
        dz  dee	j                 dz  dedz  dee         dee	j
                 ez  fd„¦   «         ¦   «         ¦   «         Zd„ Zˆ xZS )Ú	XmodModelTc                 ó  •— t          ¦   «                              |¦  «         || _        d| _        t	          |¦  «        | _        t          |¦  «        | _        |rt          |¦  «        nd| _	        |  
                    ¦   «          dS )zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        FN)r3   r4   rL   Úgradient_checkpointingr(   rc   r  r<  r  ÚpoolerÚ	post_init)rK   rL   Úadd_pooling_layerrM   s      €rN   r4   zXmodModel.__init__œ  s{   ø€ õ
 	‰Œ×Ò˜Ñ Ô Ð ØˆŒØ&+ˆÔ#å(¨Ñ0Ô0ˆŒÝ" 6Ñ*Ô*ˆŒà,=ÐG•j Ñ(Ô(Ð(À4ˆŒð 	�ŠÑÔÐÐÐrO   c                 ó   — | j         j        S rÅ   ©rc   r9   ©rK   s    rN   Úget_input_embeddingszXmodModel.get_input_embeddings®  s   € ØŒÔ.Ð.rO   c                 ó   — || j         _        d S rÅ   rR  )rK   r~   s     rN   Úset_input_embeddingszXmodModel.set_input_embeddings²  s   € Ø*/ˆŒÔ'Ð'Ð'rO   NrP   rð   r   r1   r-   rQ   rµ   rÏ   r    r  r�   rS   c                 óæ  — |du |duz  rt          d¦  «        ‚| j        j        r|
�|
n| j        j        }
nd}
|
r[|	€Y|€| j        j        r6t          t          | j        ¬¦  «        t          | j        ¬¦  «        ¦  «        nt          | j        ¬¦  «        }	|�|j        d         n|j        d         }|�|j        n|j        }|	�|	 	                    ¦   «         nd}|€Ž| j        j
        €t          d¦  «        ‚t          | j        j        d         j        j                             ¦   «         ¦  «        }|                     | j        j
        ¦  «        }|t%          j        ||¬¦  «        z  }|                      |||||¬¦  «        }|                      |||||	¬	¦  «        \  }} | j        |f|||||	|
|d
œ|¤Ž}|d         }| j        �|                      |¦  «        nd}t/          |||j        ¬¦  «        S )á  
        lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of the language adapters that should be activated for each sample, respectively. Default: the index
            that corresponds to `self.config.default_language`.
        Nz:You must specify exactly one of input_ids or inputs_embedsF)rL   r   zPInput language unknown. Please call `XmodPreTrainedModel.set_default_language()`)rX   )rP   r-   r1   rQ   rR   )r   rÏ   Úembedding_outputrµ   r    )rð   r   rµ   rÏ   r    r  r-   )r  Úpooler_outputr    )r–   rL   r›   r  Úis_encoder_decoderr   r   r]   rX   Úget_seq_lengthr5  r4  r<  r  rÌ   rí   r»   rV   rB   Úonesrc   Ú_create_attention_masksrN  r   r    )rK   rP   rð   r   r1   r-   rQ   rµ   rÏ   r    r  r�   r`   rX   rR   Úadapter_languagesÚdefault_lang_idrY  Úencoder_outputsÚsequence_outputr&  s                        rN   rd   zXmodModel.forwardµ  sj  € ð, ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàŒ;Ô!ð 	Ø%.Ð%:˜	˜	ÀÄÔ@UˆIˆIàˆIàð 	˜Ð0ð )Ð4¸¼Ô8VÐ4õ $¥L¸¼Ð$DÑ$DÔ$DÅlÐZ^ÔZeÐFfÑFfÔFfÑgÔgÐgå!¨¬Ð5Ñ5Ô5ð ð ,5Ð+@�Y”_ QÔ'Ð'ÀmÔFYÐZ[ÔF\ˆ
Ø%.Ð%:�Ô!Ð!ÀÔ@TˆØETÐE` ×!?Ò!?Ñ!AÔ!AÐ!AÐfgÐàÐØŒ{Ô+Ð3Ý Ð!sÑtÔtÐtÝ $ T¤\Ô%7¸Ô%:Ô%AÔ%Q×%VÒ%VÑ%XÔ%XÑ YÔ YÐØ/×5Ò5°d´kÔ6RÑSÔSˆOØ&­¬°JÀvÐ)NÑ)NÔ)NÑNˆHàŸ?š?ØØ%Ø)Ø'Ø#9ð +ñ 
ô 
Ðð 26×1MÒ1MØ)Ø#9Ø-Ø"7Ø+ð 2Nñ 2
ô 2
Ñ.ˆÐ.ð '˜$œ,Øð

àØ)Ø"7Ø#9Ø+ØØ%ð

ð 

ð ð

ð 

ˆð *¨!Ô,ˆØ8<¼Ð8O˜Ÿš OÑ4Ô4Ð4ÐUYˆå;Ø-Ø'Ø+Ô;ð
ñ 
ô 
ð 	
rO   c                 ó¶   — | j         j        rt          | j         |||¬¦  «        }nt          | j         ||¬¦  «        }|�t          | j         |||¬¦  «        }||fS )N)rL   rQ   r   r    )rL   rQ   r   )rL   rQ   r   rµ   )rL   r›   r   r   )rK   r   rÏ   rY  rµ   r    s         rN   r^  z!XmodModel._create_attention_masks
  s�   € ð Œ;Ô!ð 	Ý/Ø”{Ø.Ø-Ø /ð	ñ ô ˆNˆNõ 7Ø”{Ø.Ø-ðñ ô ˆNð "Ð-Ý%>Ø”{Ø.Ø5Ø&;ð	&ñ &ô &Ð"ð Ð5Ð5Ð5rO   )T)
NNNNNNNNNN)rp   rq   rr   r4   rT  rV  r#   r$   r    rB   rv   rt   r4  ru   r  r   r   r±   r   rd   r^  rx   ry   s   @rN   rK  rK  �  s©  ø€ € € € € ðð ð ð ð ð ð$/ð /ð /ð0ð 0ð 0ð  ØØð *.Ø,0Ø.2Ø.2Ø,0Ø-1Ø59Ø6:Ø:>Ø!%ðO
ð O
à”< $Ñ&ðO
ð Ô" TÑ)ðO
ð œ tÑ+ð	O
ð
 œ tÑ+ðO
ð ”l TÑ)ðO
ð ”| dÑ*ðO
ð  %œ|¨dÑ2ðO
ð !&¤¨tÑ 3ðO
ð ˜eÔ/Ô0°4Ñ7ðO
ð ˜$‘;ðO
ð Ð+Ô,ðO
ð 
ˆuŒ|Ô	ÐKÑ	KðO
ð O
ð O
ñ „^ñ „_ñ  ÔðO
ðd6ð 6ð 6ð 6ð 6ð 6ð 6rO   rK  zQ
    X-MOD Model with a `language modeling` head on top for CLM fine-tuning.
    c                    ó¤  ‡ — e Zd ZdddœZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 	 	 	 dd	e	j
        dz  d
e	j
        dz  de	j        dz  de	j
        dz  de	j
        dz  de	j        dz  de	j        dz  de	j        dz  de	j
        dz  deee	j                          dz  dedz  dee	j        z  dee         dee	j                 ez  fd„¦   «         ¦   «         Zˆ xZS )ÚXmodForCausalLMú)roberta.embeddings.word_embeddings.weightúlm_head.bias©zlm_head.decoder.weightzlm_head.decoder.biasc                 ó  •— t          ¦   «                              |¦  «         |j        st                               d¦  «         t          |d¬¦  «        | _        t          |¦  «        | _        |  	                    ¦   «          d S )NzLIf you want to use `XmodLMHeadModel` as a standalone, add `is_decoder=True.`F©rP  ©
r3   r4   r›   r8  ÚwarningrK  r)  r.  Úlm_headrO  rJ   s     €rN   r4   zXmodForCausalLM.__init__7  su   ø€ Ý‰Œ×Ò˜Ñ Ô Ð àÔ ð 	kÝ�NŠNÐiÑjÔjÐjå  ¸5ÐAÑAÔAˆŒÝ! &Ñ)Ô)ˆŒð 	�ŠÑÔÐÐÐrO   c                 ó   — | j         j        S rÅ   ©rm  ÚdecoderrS  s    rN   Úget_output_embeddingsz%XmodForCausalLM.get_output_embeddingsD  ó   € ØŒ|Ô#Ð#rO   c                 ó   — || j         _        d S rÅ   ro  ©rK   Únew_embeddingss     rN   Úset_output_embeddingsz%XmodForCausalLM.set_output_embeddingsH  ó   € Ø-ˆŒÔÐÐrO   Nr   rP   rð   r   r1   r-   rQ   rµ   rÏ   Úlabelsr    r  Úlogits_to_keepr�   rS   c                 ón  — |	�d} | j         |f||||||||
|ddœ
|¤Ž}|j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|	� | j        d||	| j        j        dœ|¤Ž}t          |||j
        |j        |j        |j        ¬¦  «        S )aS  
        lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of the language adapters that should be activated for each sample, respectively. Default: the index
            that corresponds to `self.config.default_language`.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`

        Example:

        ```python
        >>> from transformers import AutoTokenizer, XmodForCausalLM, AutoConfig
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("FacebookAI/xlm-roberta-base")
        >>> config = AutoConfig.from_pretrained("facebook/xmod-base")
        >>> config.is_decoder = True
        >>> model = XmodForCausalLM.from_pretrained("facebook/xmod-base", config=config)
        >>> model.set_default_language("en_XX")

        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
        >>> outputs = model(**inputs)

        >>> prediction_logits = outputs.logits
        ```NFT)
rð   r   r1   r-   rQ   rµ   rÏ   r    r  Úreturn_dict)Úlogitsrx  r6   )Úlossr|  r    rŸ   r*  r+  © )r)  r  r¤   rk   Úslicerm  Úloss_functionrL   r6   r   r    rŸ   r*  r+  )rK   rP   rð   r   r1   r-   rQ   rµ   rÏ   rx  r    r  ry  r�   ÚoutputsrŸ   Úslice_indicesr|  r}  s                      rN   rd   zXmodForCausalLM.forwardK  s  € ðX ÐØˆIà@LÀÄØðA
àØ)Ø)Ø%Ø'Ø"7Ø#9Ø+ØØðA
ð A
ð ðA
ð A
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDå0ØØØ#Ô3Ø!Ô/ØÔ)Ø$Ô5ð
ñ 
ô 
ð 	
rO   )NNNNNNNNNNNr   )rp   rq   rr   Ú_tied_weights_keysr4   rq  rv  r"   r    rB   rt   ru   r±   r  rk   rv   r   r   r   rd   rx   ry   s   @rN   re  re  +  sÛ  ø€ € € € € ð #NØ .ðð Ðð
ð 
ð 
ð 
ð 
ð$ð $ð $ð.ð .ð .ð Øð .2Ø,0Ø37Ø26Ø04Ø26Ø:>Ø;?Ø*.ØBFØ!%Ø-.ðL
ð L
àÔ# dÑ*ðL
ð Ô" TÑ)ðL
ð Ô)¨DÑ0ð	L
ð
 Ô(¨4Ñ/ðL
ð Ô&¨Ñ-ðL
ð Ô(¨4Ñ/ðL
ð  %Ô0°4Ñ7ðL
ð !&Ô 1°DÑ 8ðL
ð Ô  4Ñ'ðL
ð ˜u UÔ%6Ô7Ô8¸4Ñ?ðL
ð ˜$‘;ðL
ð ˜eœlÑ*ðL
ð Ð+Ô,ðL
ð 
ˆuŒ|Ô	Ð@Ñ	@ðL
ð L
ð L
ñ „^ñ ÔðL
ð L
ð L
ð L
ð L
rO   re  c                   óT  ‡ — e Zd ZdddœZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 dde	j
        dz  d	e	j
        dz  d
e	j        dz  de	j
        dz  de	j
        dz  de	j        dz  de	j        dz  de	j        dz  de	j
        dz  dee         dee	j                 ez  fd„¦   «         ¦   «         Zˆ xZS )ÚXmodForMaskedLMrf  rg  rh  c                 ó  •— t          ¦   «                              |¦  «         |j        rt                               d¦  «         t          |d¬¦  «        | _        t          |¦  «        | _        |  	                    ¦   «          d S )NzkIf you want to use `XmodForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.Frj  rk  rJ   s     €rN   r4   zXmodForMaskedLM.__init__¤  s~   ø€ Ý‰Œ×Ò˜Ñ Ô Ð àÔð 	Ý�NŠNð1ñô ð õ
 ! ¸5ÐAÑAÔAˆŒÝ! &Ñ)Ô)ˆŒð 	�ŠÑÔÐÐÐrO   c                 ó   — | j         j        S rÅ   ro  rS  s    rN   rq  z%XmodForMaskedLM.get_output_embeddings´  rr  rO   c                 ó   — || j         _        d S rÅ   ro  rt  s     rN   rv  z%XmodForMaskedLM.set_output_embeddings¸  rw  rO   NrP   rð   r   r1   r-   rQ   rµ   rÏ   rx  r�   rS   c
                 óB  —  | j         |f|||||||ddœ|
¤Ž}|d         }|                      |¦  «        }d}|	�Kt          ¦   «         } ||                     d| j        j        ¦  «        |	                     d¦  «        ¦  «        }t          |||j        |j        ¬¦  «        S )a¶  
        lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of the language adapters that should be activated for each sample, respectively. Default: the index
            that corresponds to `self.config.default_language`.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        T)rð   r   r1   r-   rQ   rµ   rÏ   r{  r   Nr/   ©r}  r|  rŸ   r*  )	r)  rm  r   r£   rL   r6   r   rŸ   r*  )rK   rP   rð   r   r1   r-   rQ   rµ   rÏ   rx  r�   r�  rb  Úprediction_scoresÚmasked_lm_lossÚloss_fcts                   rN   rd   zXmodForMaskedLM.forward»  sÙ   € ð0 �$”,Øð
àØ)Ø)Ø%Ø'Ø"7Ø#9Øð
ð 
ð ð
ð 
ˆð " !œ*ˆØ ŸLšL¨Ñ9Ô9ÐàˆØÐÝ'Ñ)Ô)ˆHØ%˜XÐ&7×&<Ò&<¸RÀÄÔAWÑ&XÔ&XÐZ`×ZeÒZeÐfhÑZiÔZiÑjÔjˆNåØØ$Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rO   )	NNNNNNNNN)rp   rq   rr   rƒ  r4   rq  rv  r"   r    rB   rt   ru   r   r   r±   rv   r   rd   rx   ry   s   @rN   r…  r…  œ  s  ø€ € € € € ð #NØ .ðð Ððð ð ð ð ð $ð $ð $ð.ð .ð .ð Øð .2Ø,0Ø37Ø26Ø04Ø26Ø:>Ø;?Ø*.ð/
ð /
àÔ# dÑ*ð/
ð Ô" TÑ)ð/
ð Ô)¨DÑ0ð	/
ð
 Ô(¨4Ñ/ð/
ð Ô&¨Ñ-ð/
ð Ô(¨4Ñ/ð/
ð  %Ô0°4Ñ7ð/
ð !&Ô 1°DÑ 8ð/
ð Ô  4Ñ'ð/
ð Ð+Ô,ð/
ð 
ˆuŒ|Ô	˜~Ñ	-ð/
ð /
ð /
ñ „^ñ Ôð/
ð /
ð /
ð /
ð /
rO   r…  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )r.  z*Roberta Head for masked language modeling.c                 ó‚  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j        |j	        ¦  «        | _
        t          j        t          j        |j	        ¦  «        ¦  «        | _        d S rÁ   )r3   r4   r   r™   r7   rÂ   r<   r=   Ú
layer_normr6   rp  Ú	ParameterrB   rF   r1  rJ   s     €rN   r4   zXmodLMHead.__init__ó  s‰   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒå”y Ô!3°VÔ5FÑGÔGˆŒÝ”L¥¤¨VÔ->Ñ!?Ô!?Ñ@Ô@ˆŒ	ˆ	ˆ	rO   c                 ó¢   — |                       |¦  «        }t          |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rÅ   )rÂ   r   r�  rp  ©rK   Úfeaturesr�   Úxs       rN   rd   zXmodLMHead.forwardû  sE   € Ø�JŠJ�xÑ Ô ˆÝ�‰GŒGˆØ�OŠO˜AÑÔˆð �LŠL˜‰OŒOˆàˆrO   ©rp   rq   rr   rs   r4   rd   rx   ry   s   @rN   r.  r.  ð  sR   ø€ € € € € Ø4Ð4ðAð Að Að Að Aðð ð ð ð ð ð rO   r.  z�
    X-MOD Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    c                   ó  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
e	e
         deej                 ez  fd„¦   «         ¦   «         Zˆ xZS )ÚXmodForSequenceClassificationc                 óì   •— t          ¦   «                              |¦  «         |j        | _        || _        t	          |d¬¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S ©NFrj  )	r3   r4   Ú
num_labelsrL   rK  r)  ÚXmodClassificationHeadÚ
classifierrO  rJ   s     €rN   r4   z&XmodForSequenceClassification.__init__  sg   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒØˆŒå  ¸5ÐAÑAÔAˆŒÝ0°Ñ8Ô8ˆŒð 	�ŠÑÔÐÐÐrO   NrP   rð   r   r1   r-   rQ   rx  r�   rS   c           
      ó^  —  | j         |f|||||ddœ|¤Ž}	|	d         }
|                      |
¦  «        }d}|��Z| j        j        €f| j        dk    rd| j        _        nN| j        dk    r7|j        t          j        k    s|j        t          j        k    rd| j        _        nd| j        _        | j        j        dk    rWt          ¦   «         }| j        dk    r1 || 
                    ¦   «         | 
                    ¦   «         ¦  «        }nŽ |||¦  «        }n�| j        j        dk    rGt          ¦   «         } ||                     d	| j        ¦  «        |                     d	¦  «        ¦  «        }n*| j        j        dk    rt          ¦   «         } |||¦  «        }t          |||	j        |	j        ¬
¦  «        S )a¡  
        lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of the language adapters that should be activated for each sample, respectively. Default: the index
            that corresponds to `self.config.default_language`.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        T©rð   r   r1   r-   rQ   r{  r   Nr%   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr/   rŠ  )r)  r�  rL   Úproblem_typer›  r2   rB   rH   rk   r   Úsqueezer   r£   r   r   rŸ   r*  ©rK   rP   rð   r   r1   r-   rQ   rx  r�   r�  rb  r|  r}  r�  s                 rN   rd   z%XmodForSequenceClassification.forward  sÍ  € ð, �$”,Øð	
àØ)Ø)Ø%Ø'Øð	
ð 	
ð ð	
ð 	
ˆð " !œ*ˆØ—’ Ñ1Ô1ˆàˆØÑØŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ñ @Ô @À&Ç+Â+ÈbÁ/Ä/ÑRÔR��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�å'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rO   ©NNNNNNN)rp   rq   rr   r4   r"   r    rB   rt   ru   r   r   r±   rv   r   rd   rx   ry   s   @rN   r˜  r˜    s"  ø€ € € € € ð	ð 	ð 	ð 	ð 	ð Øð .2Ø,0Ø37Ø26Ø04Ø26Ø*.ð=
ð =
àÔ# dÑ*ð=
ð Ô" TÑ)ð=
ð Ô)¨DÑ0ð	=
ð
 Ô(¨4Ñ/ð=
ð Ô&¨Ñ-ð=
ð Ô(¨4Ñ/ð=
ð Ô  4Ñ'ð=
ð Ð+Ô,ð=
ð 
ˆuŒ|Ô	Ð7Ñ	7ð=
ð =
ð =
ñ „^ñ Ôð=
ð =
ð =
ð =
ð =
rO   r˜  c                   ó  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
e	e
         deej                 ez  fd„¦   «         ¦   «         Zˆ xZS )ÚXmodForMultipleChoicec                 ó  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        ¦  «        | _        t	          j        |j	        d¦  «        | _
        |                      ¦   «          d S )Nr%   )r3   r4   rK  r)  r   r>   r?   r@   r™   r7   r�  rO  rJ   s     €rN   r4   zXmodForMultipleChoice.__init__^  sl   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å  Ñ(Ô(ˆŒÝ”z &Ô"<Ñ=Ô=ˆŒÝœ) FÔ$6¸Ñ:Ô:ˆŒð 	�ŠÑÔÐÐÐrO   NrP   rð   r1   r   rx  r-   rQ   r�   rS   c           
      óØ  — |�|j         d         n|j         d         }	|�)|                     d|                     d¦  «        ¦  «        nd}
|�>|                     |                     d¦  «        |                     d¦  «        z  ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�=|                     d|                     d¦  «        |                     d¦  «        ¦  «        nd} | j        |
f|||||ddœ|¤Ž}|d         }|                      |¦  «        }|                      |¦  «        }|                     d|	¦  «        }d}|�t          ¦   «         } |||¦  «        }t          |||j	        |j
        ¬¦  «        S )	a|  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        lang_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of the language adapters that should be activated for each sample, respectively. Default: the index
            that corresponds to `self.config.default_language`.
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        Nr%   r/   r   éþÿÿÿT)rð   r-   r1   r   rQ   r{  rŠ  )r]   r£   rG   Úrepeatr)  r@   r�  r   r   rŸ   r*  )rK   rP   rð   r1   r   rx  r-   rQ   r�   Únum_choicesÚflat_input_idsÚflat_lang_idsÚflat_position_idsÚflat_token_type_idsÚflat_attention_maskÚflat_inputs_embedsr�  r&  r|  Úreshaped_logitsr}  r�  s                         rN   rd   zXmodForMultipleChoice.forwardh  s(  € ð\ -6Ð,A�i”o aÔ(Ð(À}ÔGZÐ[\ÔG]ˆàCLÐCX˜Ÿš¨¨I¯NªN¸2Ñ,>Ô,>Ñ?Ô?Ð?Ð^bˆØRZÐRf˜Ÿš¨	¯ª°qÑ(9Ô(9¸I¿NºNÈ1Ñ<MÔ<MÑ(MÑNÔNÐNÐlpˆØLXÐLd˜L×-Ò-¨b°,×2CÒ2CÀBÑ2GÔ2GÑHÔHÐHÐjnÐØR`ÐRl˜n×1Ò1°"°n×6IÒ6IÈ"Ñ6MÔ6MÑNÔNÐNÐrvÐØR`ÐRl˜n×1Ò1°"°n×6IÒ6IÈ"Ñ6MÔ6MÑNÔNÐNÐrvÐð Ð(ð ×Ò˜r =×#5Ò#5°bÑ#9Ô#9¸=×;MÒ;MÈbÑ;QÔ;QÑRÔRÐRàð 	ð �$”,Øð	
à"Ø*Ø.Ø.Ø,Øð	
ð 	
ð ð	
ð 	
ˆð   œ
ˆàŸš ]Ñ3Ô3ˆØ—’ Ñ/Ô/ˆØ Ÿ+š+ b¨+Ñ6Ô6ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜O¨VÑ4Ô4ˆDå(ØØ"Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rO   r¦  )rp   rq   rr   r4   r"   r    rB   rt   ru   r   r   r±   rv   r   rd   rx   ry   s   @rN   r¨  r¨  [  s4  ø€ € € € € ðð ð ð ð ð Øð .2Ø,0Ø26Ø37Ø*.Ø04Ø26ðS
ð S
àÔ# dÑ*ðS
ð Ô" TÑ)ðS
ð Ô(¨4Ñ/ð	S
ð
 Ô)¨DÑ0ðS
ð Ô  4Ñ'ðS
ð Ô&¨Ñ-ðS
ð Ô(¨4Ñ/ðS
ð Ð+Ô,ðS
ð 
ˆuŒ|Ô	Ð8Ñ	8ðS
ð S
ð S
ñ „^ñ ÔðS
ð S
ð S
ð S
ð S
rO   r¨  c                   ó  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
e	e
         deej                 ez  fd„¦   «         ¦   «         Zˆ xZS )ÚXmodForTokenClassificationc                 óZ  •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        |j        �|j        n|j        }t          j        |¦  «        | _	        t          j
        |j        |j        ¦  «        | _        |                      ¦   «          d S rš  )r3   r4   r›  rK  r)  Úclassifier_dropoutr?   r   r>   r@   r™   r7   r�  rO  ©rK   rL   r¸  rM   s      €rN   r4   z#XmodForTokenClassification.__init__Ã  sš   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå  ¸5ÐAÑAÔAˆŒà)/Ô)BÐ)NˆFÔ%Ð%ÐTZÔTnð 	õ ”zÐ"4Ñ5Ô5ˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐrO   NrP   rð   r   r1   r-   rQ   rx  r�   rS   c           
      ó^  —  | j         |f|||||ddœ|¤Ž}	|	d         }
|                      |
¦  «        }
|                      |
¦  «        }d}|�Ft          ¦   «         } ||                     d| j        ¦  «        |                     d¦  «        ¦  «        }t          |||	j        |	j        ¬¦  «        S )aï  
        lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of the language adapters that should be activated for each sample, respectively. Default: the index
            that corresponds to `self.config.default_language`.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        TrŸ  r   Nr/   rŠ  )	r)  r@   r�  r   r£   r›  r   rŸ   r*  r¥  s                 rN   rd   z"XmodForTokenClassification.forwardÑ  s×   € ð( �$”,Øð	
àØ)Ø)Ø%Ø'Øð	
ð 	
ð ð	
ð 	
ˆð " !œ*ˆàŸ,š, Ñ7Ô7ˆØ—’ Ñ1Ô1ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬OÑ<Ô<¸f¿kºkÈ"¹o¼oÑNÔNˆDå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rO   r¦  )rp   rq   rr   r4   r"   r    rB   rt   ru   r   r   r±   rv   r   rd   rx   ry   s   @rN   r¶  r¶  À  s"  ø€ € € € € ðð ð ð ð ð Øð .2Ø,0Ø37Ø26Ø04Ø26Ø*.ð,
ð ,
àÔ# dÑ*ð,
ð Ô" TÑ)ð,
ð Ô)¨DÑ0ð	,
ð
 Ô(¨4Ñ/ð,
ð Ô&¨Ñ-ð,
ð Ô(¨4Ñ/ð,
ð Ô  4Ñ'ð,
ð Ð+Ô,ð,
ð 
ˆuŒ|Ô	Ð4Ñ	4ð,
ð ,
ð ,
ñ „^ñ Ôð,
ð ,
ð ,
ð ,
ð ,
rO   r¶  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )rœ  z-Head for sentence-level classification tasks.c                 ó4  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        |j        �|j        n|j        }t          j        |¦  «        | _	        t          j        |j        |j
        ¦  «        | _        d S rÅ   )r3   r4   r   r™   r7   rÂ   r¸  r?   r>   r@   r›  Úout_projr¹  s      €rN   r4   zXmodClassificationHead.__init__  s   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
à)/Ô)BÐ)NˆFÔ%Ð%ÐTZÔTnð 	õ ”zÐ"4Ñ5Ô5ˆŒÝœ	 &Ô"4°fÔ6GÑHÔHˆŒˆˆrO   c                 óô   — |d d …dd d …f         }|                       |¦  «        }|                      |¦  «        }t          j        |¦  «        }|                       |¦  «        }|                      |¦  «        }|S r$  )r@   rÂ   rB   Útanhr½  r“  s       rN   rd   zXmodClassificationHead.forward  sj   € Ø�Q�Q�Q˜˜1˜1˜1�WÔˆØ�LŠL˜‰OŒOˆØ�JŠJ�q‰MŒMˆÝŒJ�q‰MŒMˆØ�LŠL˜‰OŒOˆØ�MŠM˜!ÑÔˆØˆrO   r–  ry   s   @rN   rœ  rœ    sR   ø€ € € € € Ø7Ð7ðIð Ið Ið Ið Iðð ð ð ð ð ð rO   rœ  c                   ó(  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
ej        dz  de	e
         deej                 ez  fd„¦   «         ¦   «         Zˆ xZS )ÚXmodForQuestionAnsweringc                 óþ   •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        t          j        |j        |j        ¦  «        | _        |  	                    ¦   «          d S rš  )
r3   r4   r›  rK  r)  r   r™   r7   Ú
qa_outputsrO  rJ   s     €rN   r4   z!XmodForQuestionAnswering.__init__  sj   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå  ¸5ÐAÑAÔAˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐrO   NrP   rð   r   r1   r-   rQ   Ústart_positionsÚend_positionsr�   rS   c	           
      óH  —  | j         |f|||||ddœ|	¤Ž}
|
d         }|                      |¦  «        }|                     dd¬¦  «        \  }}|                     d¦  «                             ¦   «         }|                     d¦  «                             ¦   «         }d}|�ç|�åt          |                     ¦   «         ¦  «        dk    r|                     d¦  «        }t          |                     ¦   «         ¦  «        dk    r|                     d¦  «        }|                     d¦  «        }|                     d|¦  «        }|                     d|¦  «        }t          |¬¦  «        } |||¦  «        } |||¦  «        }||z   d	z  }t          ||||
j
        |
j        ¬
¦  «        S )rX  TrŸ  r   r%   r/   ri   N)Úignore_indexr„   )r}  Ústart_logitsÚ
end_logitsrŸ   r*  )r)  rÃ  Úsplitr¤  r‹   ÚlenrG   Úclampr   r   rŸ   r*  )rK   rP   rð   r   r1   r-   rQ   rÄ  rÅ  r�   r�  rb  r|  rÈ  rÉ  Ú
total_lossÚignored_indexr�  Ú
start_lossÚend_losss                       rN   rd   z XmodForQuestionAnswering.forward&  sÒ  € ð& �$”,Øð	
àØ)Ø)Ø%Ø'Øð	
ð 	
ð ð	
ð 	
ˆð " !œ*ˆà—’ Ñ1Ô1ˆØ#)§<¢<°°r <Ñ#:Ô#:Ñ ˆ�jØ#×+Ò+¨BÑ/Ô/×:Ò:Ñ<Ô<ˆØ×'Ò'¨Ñ+Ô+×6Ò6Ñ8Ô8ˆ
àˆ
ØÐ&¨=Ð+Då�?×'Ò'Ñ)Ô)Ñ*Ô*¨QÒ.Ð.Ø"1×"9Ò"9¸"Ñ"=Ô"=�Ý�=×%Ò%Ñ'Ô'Ñ(Ô(¨1Ò,Ð,Ø -× 5Ò 5°bÑ 9Ô 9�à(×-Ò-¨aÑ0Ô0ˆMØ-×3Ò3°A°}ÑEÔEˆOØ)×/Ò/°°=ÑAÔAˆMå'°]ÐCÑCÔCˆHØ!˜ ,°Ñ@Ô@ˆJØ�x 
¨MÑ:Ô:ˆHØ$ xÑ/°1Ñ4ˆJå+ØØ%Ø!Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
rO   )NNNNNNNN)rp   rq   rr   r4   r"   r    rB   rt   ru   r   r   r±   rv   r   rd   rx   ry   s   @rN   rÁ  rÁ    s7  ø€ € € € € ðð ð ð ð ð Øð .2Ø,0Ø37Ø26Ø04Ø26Ø37Ø15ð:
ð :
àÔ# dÑ*ð:
ð Ô" TÑ)ð:
ð Ô)¨DÑ0ð	:
ð
 Ô(¨4Ñ/ð:
ð Ô&¨Ñ-ð:
ð Ô(¨4Ñ/ð:
ð Ô)¨DÑ0ð:
ð Ô'¨$Ñ.ð:
ð Ð+Ô,ð:
ð 
ˆuŒ|Ô	Ð;Ñ	;ð:
ð :
ð :
ñ „^ñ Ôð:
ð :
ð :
ð :
ð :
rO   rÁ  )re  r…  r¨  rÁ  r˜  r¶  rK  r(  )Nrz   )Rrs   Úcollections.abcr   rB   r   Útorch.nnr   r   r   Ú r	   r/  Úactivationsr
   r   Úcache_utilsr   r   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r    r!   Úutils.genericr"   r#   Úutils.output_capturingr$   Úconfiguration_xmodr&   Ú
get_loggerrp   r8  ÚModuler(   rv   ÚfloatrŽ   r�   r³   r¿   rÈ   rÕ   rÞ   rç   rý   r  r  r(  rK  re  r…  r.  r˜  r¨  r¶  rœ  rÁ  Ú__all__r~  rO   rN   ú<module>rå     s&  ðð Ð à $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø 'Ð 'Ð 'Ð 'Ð 'Ð 'Ð 'Ð 'Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ )Ð )Ð )Ð )Ð )Ð )Ø JÐ JÐ JÐ JÐ JÐ JÐ JÐ JØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ð	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð GÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø *Ð *Ð *Ð *Ð *Ð *ð 
ˆÔ	˜HÑ	%Ô	%€ðg8ð g8ð g8ð g8ð g8�R”Yñ g8ô g8ð g8ðb !Øð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð:@)ð @)ð @)ð @)ð @)˜œ	ñ @)ô @)ð @)ðHI)ð I)ð I)ð I)ð I)˜œñ I)ô I)ð I)ðXð ð ð ð �R”Yñ ô ð ð$.ð $.ð $.ð $.ð $.�B”Iñ $.ô $.ð $.ðPð ð ð ð �r”yñ ô ð ðð ð ð ð �"”)ñ ô ð ð$,ð ,ð ,ð ,ð ,�”ñ ,ô ,ð ,ð^H3ð H3ð H3ð H3ð H3Ð*ñ H3ô H3ð H3ðV%
ð %
ð %
ð %
ð %
�"”)ñ %
ô %
ð %
ðRð ð ð ð �”ñ ô ð ð ð40ð 40ð 40ð 40ð 40˜/ñ 40ô 40ñ „ð40ðn €ððñ ô ðM6ð M6ð M6ð M6ð M6Ð#ñ M6ô M6ñô ðM6ð` €ððñ ô ð
i
ð i
ð i
ð i
ð i
Ð)¨?ñ i
ô i
ñô ð
i
ðX ðO
ð O
ð O
ð O
ð O
Ð)ñ O
ô O
ñ „ðO
ðfð ð ð ð �”ñ ô ð ð, €ððñ ô ðL
ð L
ð L
ð L
ð L
Ð$7ñ L
ô L
ñô ðL
ð^ ða
ð a
ð a
ð a
ð a
Ð/ñ a
ô a
ñ „ða
ðH ð>
ð >
ð >
ð >
ð >
Ð!4ñ >
ô >
ñ „ð>
ðDð ð ð ð ˜RœYñ ô ð ð, ðH
ð H
ð H
ð H
ð H
Ð2ñ H
ô H
ñ „ðH
ðV	ð 	ð 	€€€rO   