§
    ‚Štj÷Ñ  ã                   óÀ  — d Z ddlZddlZddlmZ ddlmZmZmZ ddlm	Z
 ddlmZ ddlmZ dd	lmZ dd
lmZmZmZmZmZmZ ddlmZ ddlmZ ddlmZmZmZm Z m!Z!m"Z" ddl#m$Z$  e"j%        e&¦  «        Z'da(d„ Z)d„ Z*dId„Z+dId„Z,dId„Z-d„ Z. G d„ dej/        j0        ¦  «        Z1 G d„ dej/        j0        ¦  «        Z2 G d„ d¦  «        Z3dJd„Z4d„ Z5	 	 	 dKd„Z6 G d „ d!ej7        ¦  «        Z8 G d"„ d#ej7        ¦  «        Z9 G d$„ d%ej7        ¦  «        Z: G d&„ d'ej7        ¦  «        Z; G d(„ d)ej7        ¦  «        Z< G d*„ d+ej7        ¦  «        Z= G d,„ d-e¦  «        Z> G d.„ d/ej7        ¦  «        Z? G d0„ d1ej7        ¦  «        Z@ G d2„ d3ej7        ¦  «        ZA G d4„ d5ej7        ¦  «        ZBe G d6„ d7e¦  «        ¦   «         ZCe G d8„ d9eC¦  «        ¦   «         ZDe G d:„ d;eC¦  «        ¦   «         ZE G d<„ d=ej7        ¦  «        ZF ed>¬?¦  «         G d@„ dAeC¦  «        ¦   «         ZGe G dB„ dCeC¦  «        ¦   «         ZHe G dD„ dEeC¦  «        ¦   «         ZIe G dF„ dGeC¦  «        ¦   «         ZJg dH¢ZKdS )LzPyTorch MRA model.é    N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)Ú"BaseModelOutputWithCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)Úapply_chunking_to_forward)Úauto_docstringÚis_cuda_platformÚis_kernels_availableÚis_ninja_availableÚis_torch_cuda_availableÚloggingé   )Ú	MraConfigc                  óf   — t          ¦   «         st          d¦  «        ‚ddlm}   | dd¬¦  «        ad S )NzFkernels is not installed, please install it with `pip install kernels`r   ©Ú
get_kernelzkernels-community/mrar   )Úversion)r   ÚImportErrorÚintegrations.hub_kernelsr   Úmra_cuda_kernelr   s    úb/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/mra/modeling_mra.pyÚload_cuda_kernelsr$   4   sK   € åÑ!Ô!ð dÝÐbÑcÔcÐcØ6Ð6Ð6Ð6Ð6Ð6à �jÐ!8À!ÐDÑDÔD€O€O€Oó    c                 óÜ  — t          |                      ¦   «         ¦  «        dk    rt          d¦  «        ‚t          |                     ¦   «         ¦  «        dk    rt          d¦  «        ‚|                      d¦  «        dk    rt          d¦  «        ‚|                      d¦  «        dk    rt          d¦  «        ‚|                      d	¬
¦  «        j                             dd	¦  «        }|                     ¦   «         }|                     ¦   «         }|                     ¦   «         }t           	                    ||||¦  «        \  }}|                     dd	¦  «        dd…dd…ddd…f         }||fS )z8
    Computes maximum values for softmax stability.
    é   z.sparse_qk_prod must be a 4-dimensional tensor.é   ú'indices must be a 2-dimensional tensor.é    z>The size of the second dimension of sparse_qk_prod must be 32.r   z=The size of the third dimension of sparse_qk_prod must be 32.éþÿÿÿ©ÚdiméÿÿÿÿN)
ÚlenÚsizeÚ
ValueErrorÚmaxÚvaluesÚ	transposeÚ
contiguousÚintr"   Ú	index_max)Úsparse_qk_prodÚindicesÚquery_num_blockÚkey_num_blockÚ
index_valsÚmax_valsÚmax_vals_scatters          r#   Ú
sparse_maxr?   =   s`  € õ ˆ>×ÒÑ Ô Ñ!Ô! QÒ&Ð&ÝÐIÑJÔJÐJå
ˆ7�<Š<‰>Œ>ÑÔ˜aÒÐÝÐBÑCÔCÐCà×Ò˜1ÑÔ Ò#Ð#ÝÐYÑZÔZÐZà×Ò˜1ÑÔ Ò#Ð#ÝÐXÑYÔYÐYà×#Ò#¨Ð#Ñ+Ô+Ô2×<Ò<¸RÀÑDÔD€JØ×&Ò&Ñ(Ô(€Jà�kŠk‰mŒm€GØ× Ò Ñ"Ô"€Gå!0×!:Ò!:¸:ÀwÐP_ÐanÑ!oÔ!oÑ€HÐØ'×1Ò1°"°bÑ9Ô9¸!¸!¸!¸Q¸Q¸QÀÀaÀaÀa¸-ÔHÐàÐ%Ð%Ð%r%   r*   c                 óB  — t          |                      ¦   «         ¦  «        dk    rt          d¦  «        ‚t          |                     ¦   «         ¦  «        dk    rt          d¦  «        ‚| j        d         |j        d         k    rt          d¦  «        ‚| j        \  }}||z  }t	          j        |                     d¦  «        t          j        |j        ¬¦  «        }|                      |||¦  «        } | |dd…df         ||z                       ¦   «         dd…f         } | S )zN
    Converts attention mask to a sparse mask for high resolution logits.
    r(   z$mask must be a 2-dimensional tensor.r)   r   zBmask and indices must have the same size in the zero-th dimension.©ÚdtypeÚdeviceN)	r/   r0   r1   ÚshapeÚtorchÚarangeÚlongrC   Úreshape)Úmaskr9   Ú
block_sizeÚ
batch_sizeÚseq_lenÚ	num_blockÚ	batch_idxs          r#   Úsparse_maskrO   Y   s  € õ ˆ4�9Š9‰;Œ;ÑÔ˜1ÒÐÝÐ?Ñ@Ô@Ð@å
ˆ7�<Š<‰>Œ>ÑÔ˜aÒÐÝÐBÑCÔCÐCà„z�!„}˜œ aÔ(Ò(Ð(ÝÐ]Ñ^Ô^Ð^àœ*Ñ€J�Ø˜:Ñ%€Iå”˜WŸ\š\¨!™_œ_µE´JÀwÄ~ÐVÑVÔV€IØ�<Š<˜
 I¨zÑ:Ô:€DØ�	˜!˜!˜!˜T˜'Ô" W¨yÑ%8×$>Ò$>Ñ$@Ô$@À!À!À!ÐCÔD€Dà€Kr%   c                 óR  — |                       ¦   «         \  }}}|                      ¦   «         \  }}}||z  dk    rt          d¦  «        ‚||z  dk    rt          d¦  «        ‚|                      |||z  ||¦  «                             dd¦  «        } |                     |||z  ||¦  «                             dd¦  «        }t	          |                       ¦   «         ¦  «        dk    rt          d¦  «        ‚t	          |                      ¦   «         ¦  «        dk    rt          d¦  «        ‚t	          |                      ¦   «         ¦  «        d	k    rt          d
¦  «        ‚|                       d¦  «        dk    rt          d¦  «        ‚|                      d¦  «        dk    rt          d¦  «        ‚|                      ¦   «         } |                     ¦   «         }|                     ¦   «         }|                     ¦   «         }t                               | ||                     ¦   «         ¦  «        S )z7
    Performs Sampled Dense Matrix Multiplication.
    r   zTquery_size (size of first dimension of dense_query) must be divisible by block_size.úPkey_size (size of first dimension of dense_key) must be divisible by block_size.r.   r+   r'   z+dense_query must be a 4-dimensional tensor.ú)dense_key must be a 4-dimensional tensor.r(   r)   r   r*   z.The third dimension of dense_query must be 32.z,The third dimension of dense_key must be 32.)	r0   r1   rH   r4   r/   r5   r6   r"   Úmm_to_sparse)	Údense_queryÚ	dense_keyr9   rJ   rK   Ú
query_sizer-   Ú_Úkey_sizes	            r#   rS   rS   p   s  € ð #.×"2Ò"2Ñ"4Ô"4Ñ€J�
˜CØ —~’~Ñ'Ô'Ñ€A€x�à�JÑ !Ò#Ð#ÝÐoÑpÔpÐpà�*Ñ Ò!Ð!ÝÐkÑlÔlÐlà×%Ò% j°*À
Ñ2JÈJÐX[Ñ\Ô\×fÒfÐgiÐkmÑnÔn€KØ×!Ò! *¨h¸*Ñ.DÀjÐRUÑVÔV×`Ò`ÐacÐegÑhÔh€Iå
ˆ;×ÒÑÔÑÔ !Ò#Ð#ÝÐFÑGÔGÐGå
ˆ9�>Š>ÑÔÑÔ Ò!Ð!ÝÐDÑEÔEÐEå
ˆ7�<Š<‰>Œ>ÑÔ˜aÒÐÝÐBÑCÔCÐCà×Ò˜ÑÔ˜bÒ Ð ÝÐIÑJÔJÐJà‡~‚~�aÑÔ˜BÒÐÝÐGÑHÔHÐHà×(Ò(Ñ*Ô*€KØ×$Ò$Ñ&Ô&€Ià�kŠk‰mŒm€GØ× Ò Ñ"Ô"€Gå×'Ò'¨°YÀÇÂÁÄÑNÔNÐNr%   c                 ó"  — |                      ¦   «         \  }}}||z  dk    rt          d¦  «        ‚|                       d¦  «        |k    rt          d¦  «        ‚|                       d¦  «        |k    rt          d¦  «        ‚|                     |||z  ||¦  «                             dd¦  «        }t	          |                       ¦   «         ¦  «        d	k    rt          d
¦  «        ‚t	          |                      ¦   «         ¦  «        d	k    rt          d¦  «        ‚t	          |                      ¦   «         ¦  «        dk    rt          d¦  «        ‚|                      d¦  «        dk    rt          d¦  «        ‚|                      ¦   «         } |                     ¦   «         }|                     ¦   «         }|                     ¦   «         }t                               | |||¦  «        }|                     dd¦  «                             |||z  |¦  «        }|S )zP
    Performs matrix multiplication of a sparse matrix with a dense matrix.
    r   rQ   r(   zQThe size of the second dimension of sparse_query must be equal to the block_size.r   zPThe size of the third dimension of sparse_query must be equal to the block_size.r.   r+   r'   ú,sparse_query must be a 4-dimensional tensor.rR   r)   r*   z8The size of the third dimension of dense_key must be 32.)	r0   r1   rH   r4   r/   r5   r6   r"   Úsparse_dense_mm)	Úsparse_queryr9   rU   r:   rJ   rK   rX   r-   Údense_qk_prods	            r#   r[   r[   ˜   sò  € ð !*§¢Ñ 0Ô 0Ñ€J�˜#à�*Ñ Ò!Ð!ÝÐkÑlÔlÐlà×Ò˜ÑÔ˜zÒ)Ð)ÝÐlÑmÔmÐmà×Ò˜ÑÔ˜zÒ)Ð)ÝÐkÑlÔlÐlà×!Ò! *¨h¸*Ñ.DÀjÐRUÑVÔV×`Ò`ÐacÐegÑhÔh€Iå
ˆ<×ÒÑÔÑÔ 1Ò$Ð$ÝÐGÑHÔHÐHå
ˆ9�>Š>ÑÔÑÔ Ò!Ð!ÝÐDÑEÔEÐEå
ˆ7�<Š<‰>Œ>ÑÔ˜aÒÐÝÐBÑCÔCÐCà‡~‚~�aÑÔ˜BÒÐÝÐSÑTÔTÐTà×*Ò*Ñ,Ô,€Là�kŠk‰mŒm€GØ× Ò Ñ"Ô"€GØ×$Ò$Ñ&Ô&€Iå#×3Ò3°LÀ'È9ÐVeÑfÔf€MØ!×+Ò+¨B°Ñ3Ô3×;Ò;¸JÈÐZdÑHdÐfiÑjÔj€MØÐr%   c                 óf   — | |z  |z  t          j        | |d¬¦  «        z                        ¦   «         S )NÚfloor©Úrounding_mode)rE   ÚdivrG   )r9   Údim_1_blockÚdim_2_blocks      r#   Útranspose_indicesre   À   s5   € Ø�{Ñ" kÑ1µE´I¸gÀ{ÐbiÐ4jÑ4jÔ4jÑj×pÒpÑrÔrÐrr%   c                   óR   — e Zd Zed„ ¦   «         Zed„ ¦   «         Zedd„¦   «         ZdS )ÚMraSampledDenseMatMulc                 óf   — t          ||||¦  «        }|                      |||¦  «         || _        |S ©N)rS   Úsave_for_backwardrJ   )ÚctxrT   rU   r9   rJ   r8   s         r#   ÚforwardzMraSampledDenseMatMul.forwardÅ   s:   € å% k°9¸gÀzÑRÔRˆØ×Ò˜k¨9°gÑ>Ô>Ð>Ø#ˆŒØÐr%   c                 ó$  — | j         \  }}}| j        }|                     d¦  «        |z  }|                     d¦  «        |z  }t          |||¦  «        }t	          |                     dd¦  «        |||¦  «        }	t	          ||||¦  «        }
|
|	d d fS ©Nr   r.   r+   )Úsaved_tensorsrJ   r0   re   r[   r4   )rk   ÚgradrT   rU   r9   rJ   r:   r;   Ú	indices_TÚgrad_keyÚ
grad_querys              r#   ÚbackwardzMraSampledDenseMatMul.backwardÌ   s›   € à*-Ô*;Ñ'ˆ�Y Ø”^ˆ
Ø%×*Ò*¨1Ñ-Ô-°Ñ;ˆØ!Ÿš qÑ)Ô)¨ZÑ7ˆÝ% g¨ÀÑNÔNˆ	Ý" 4§>¢>°"°bÑ#9Ô#9¸9ÀkÐS`ÑaÔaˆÝ$ T¨7°I¸ÑOÔOˆ
Ø˜8 T¨4Ð/Ð/r%   r*   c                 ó<   — t                                | |||¦  «        S ri   )rg   Úapply)rT   rU   r9   rJ   s       r#   Úoperator_callz#MraSampledDenseMatMul.operator_call×   s   € å$×*Ò*¨;¸	À7ÈJÑWÔWÐWr%   N©r*   ©Ú__name__Ú
__module__Ú__qualname__Ústaticmethodrl   rt   rw   © r%   r#   rg   rg   Ä   sn   € € € € € Øðð ñ „\ðð ð0ð 0ñ „\ð0ð ðXð Xð Xñ „\ðXð Xð Xr%   rg   c                   óP   — e Zd Zed„ ¦   «         Zed„ ¦   «         Zed„ ¦   «         ZdS )ÚMraSparseDenseMatMulc                 óf   — t          ||||¦  «        }|                      |||¦  «         || _        |S ri   )r[   rj   r:   )rk   r\   r9   rU   r:   r8   s         r#   rl   zMraSparseDenseMatMul.forwardÝ   s;   € å(¨°wÀ	È?Ñ[Ô[ˆØ×Ò˜l¨G°YÑ?Ô?Ð?Ø-ˆÔØÐr%   c                 ó  — | j         \  }}}| j        }|                     d¦  «        |                     d¦  «        z  }t          |||¦  «        }t	          |                     dd¦  «        |||¦  «        }t          |||¦  «        }	|	d |d fS rn   )ro   r:   r0   re   r[   r4   rS   )
rk   rp   r\   r9   rU   r:   r;   rq   rr   rs   s
             r#   rt   zMraSparseDenseMatMul.backwardä   s’   € à+.Ô+<Ñ(ˆ�g˜yØÔ-ˆØ!Ÿš qÑ)Ô)¨\×->Ò->¸rÑ-BÔ-BÑBˆÝ% g¨ÀÑNÔNˆ	Ý" <×#9Ò#9¸"¸bÑ#AÔ#AÀ9ÈdÐTaÑbÔbˆÝ! $¨	°7Ñ;Ô;ˆ
Ø˜4 ¨4Ð/Ð/r%   c                 ó<   — t                                | |||¦  «        S ri   )r€   rv   )r\   r9   rU   r:   s       r#   rw   z"MraSparseDenseMatMul.operator_callî   s   € å#×)Ò)¨,¸ÀÈOÑ\Ô\Ð\r%   Nry   r~   r%   r#   r€   r€   Ü   sh   € € € € € Øðð ñ „\ðð ð0ð 0ñ „\ð0ð ð]ð ]ñ „\ð]ð ]ð ]r%   r€   c                   ó$   — e Zd Zed„ ¦   «         ZdS )ÚMraReduceSumc                 ó¨  — |                       ¦   «         \  }}}}t          |                       ¦   «         ¦  «        dk    rt          d¦  «        ‚t          |                      ¦   «         ¦  «        dk    rt          d¦  «        ‚|                       ¦   «         \  }}}}|                      ¦   «         \  }}|                      d¬¦  «                             ||z  |¦  «        } t          j        |                      d¦  «        t
          j        |j        ¬¦  «        }t          j	        ||d¬	¦  «                             ¦   «         |d d …d f         |z  z                        ||z  ¦  «        }	t          j
        ||z  |f| j        | j        ¬¦  «        }
|
                     d|	| ¦  «                             |||¦  «        }|                     |||z  ¦  «        }|S )
Nr'   rZ   r(   r)   r,   r   rA   r_   r`   )r0   r/   r1   ÚsumrH   rE   rF   rG   rC   rb   ÚzerosrB   Ú	index_add)r\   r9   r:   r;   rK   rM   rJ   rW   rN   Úglobal_idxesÚtempÚoutputs               r#   rw   zMraReduceSum.operator_callô   s»  € à/;×/@Ò/@Ñ/BÔ/BÑ,ˆ
�I˜z¨1åˆ|× Ò Ñ"Ô"Ñ#Ô# qÒ(Ð(ÝÐKÑLÔLÐLåˆw�|Š|‰~Œ~ÑÔ !Ò#Ð#ÝÐFÑGÔGÐGà*×/Ò/Ñ1Ô1Ñˆˆ1ˆj˜!Ø '§¢¡¤Ñˆ
�Ià#×'Ò'¨AÐ'Ñ.Ô.×6Ò6°zÀIÑ7MÈzÑZÔZˆå”L §¢¨a¡¤½¼
È7Ì>ÐZÑZÔZˆ	åŒI�g˜}¸GÐDÑDÔD×IÒIÑKÔKÈiÐXYÐXYÐXYÐ[_ÐX_ÔN`ÐcrÑNrÑrß
Š'�*˜yÑ(Ñ
)Ô
)ð 	õ Œ{Ø˜/Ñ)¨:Ð6¸lÔ>PÐYeÔYlð
ñ 
ô 
ˆð —’  <°Ñ>Ô>×FÒFÀzÐSbÐdnÑoÔoˆà—’ 
¨O¸jÑ,HÑIÔIˆØˆr%   N)rz   r{   r|   r}   rw   r~   r%   r#   r…   r…   ó   s-   € € € € € Øðð ñ „\ðð ð r%   r…   c                 ó´  — |                       ¦   «         \  }}}||z  }d}	|�ë|                     |||¦  «                             d¬¦  «        }
|                      ||||¦  «                             d¬¦  «        |
dd…dd…df         dz   z  }|                     ||||¦  «                             d¬¦  «        |
dd…dd…df         dz   z  }|�?|                     ||||¦  «                             d¬¦  «        |
dd…dd…df         dz   z  }	n°|t          j        ||t          j        | j        ¬¦  «        z  }
|                      ||||¦  «                             d¬¦  «        }|                     ||||¦  «                             d¬¦  «        }|�,|                     ||||¦  «                             d¬¦  «        }	t          j        || 	                    dd¦  «        ¦  «        t          j        |¦  «        z  }|                     dd¬¦  «        j        }|�;|d	|
dd…ddd…f         |
dd…dd…df         z  d
k                          ¦   «         z  z
  }||
||	fS )z/
    Compute low resolution approximation.
    Nr.   r,   r+   ç�íµ ÷Æ°>rA   T)r-   Úkeepdimsç     ˆÃ@g      à?)r0   rH   r‡   rE   ÚonesÚfloatrC   ÚmeanÚmatmulr4   ÚmathÚsqrtr2   r3   )ÚqueryÚkeyrJ   rI   ÚvaluerK   rL   Úhead_dimÚnum_block_per_rowÚ	value_hatÚtoken_countÚ	query_hatÚkey_hatÚlow_resolution_logitÚlow_resolution_logit_row_maxs                  r#   Úget_low_resolution_logitr¢     s¬  € ð %*§J¢J¡L¤LÑ!€J�˜à :Ñ-Ðà€IØÐØ—l’l :Ð/@À*ÑMÔM×QÒQÐVXÐQÑYÔYˆØ—M’M *Ð.?ÀÈXÑVÔV×ZÒZÐ_aÐZÑbÔbØ˜˜˜˜1˜1˜1˜d˜
Ô# dÑ*ñ
ˆ	ð —+’+˜jÐ*;¸ZÈÑRÔR×VÒVÐ[]ÐVÑ^Ô^Ø˜˜˜˜1˜1˜1˜d˜
Ô# dÑ*ñ
ˆð ÐØŸš jÐ2CÀZÐQYÑZÔZ×^Ò^ÐceÐ^ÑfÔfØ˜A˜A˜A˜q˜q˜q $˜JÔ'¨$Ñ.ñˆIøð !¥5¤:¨jÐ:KÕSXÔS^ÐglÔgsÐ#tÑ#tÔ#tÑtˆØ—M’M *Ð.?ÀÈXÑVÔV×[Ò[Ð`bÐ[ÑcÔcˆ	Ø—+’+˜jÐ*;¸ZÈÑRÔR×WÒWÐ\^ÐWÑ_Ô_ˆØÐØŸš jÐ2CÀZÐQYÑZÔZ×_Ò_ÐdfÐ_ÑgÔgˆIå œ<¨	°7×3DÒ3DÀRÈÑ3LÔ3LÑMÔMÕPTÔPYÐZbÑPcÔPcÑcÐà#7×#;Ò#;ÀÈTÐ#;Ñ#RÔ#RÔ#YÐ àÐà  3¨;°q°q°q¸$ÀÀÀ°zÔ+BÀ[ÐQRÐQRÐQRÐTUÐTUÐTUÐW[ÐQ[ÔE\Ñ+\Ð`cÒ*c×)jÒ)jÑ)lÔ)lÑ#lÑlð 	ð   Ð.JÈIÐUÐUr%   c                 ó¬  — | j         \  }}}|dk    ra|dz  }t          j        ||| j        ¬¦  «        }	t          j        t          j        |	| ¬¦  «        |¬¦  «        }
| |
ddd…dd…f         dz  z   } |dk    r@| dd…d|…dd…f         dz   | dd…d|…dd…f<   | dd…dd…d|…f         dz   | dd…dd…d|…f<   t          j        |                      |d¦  «        |ddd	¬
¦  «        }|j        }|dk    rD|j	         
                    d¬¦  «        j	        }| |dd…ddf         k                         ¦   «         }n|dk    rd}nt          |› d�¦  «        ‚||fS )zZ
    Compute the indices of the subset of components to be used in the approximation.
    r   r(   ©rC   )ÚdiagonalNg     ˆ³@r.   TF)r-   ÚlargestÚsortedÚfullr,   Úsparsez# is not a valid approx_model value.)rD   rE   r‘   rC   ÚtrilÚtriuÚtopkrH   r9   r3   Úminr’   r1   )r    Ú
num_blocksÚapprox_modeÚinitial_prior_first_n_blocksÚinitial_prior_diagonal_n_blocksrK   Útotal_blocks_per_rowrW   ÚoffsetÚ	temp_maskÚdiagonal_maskÚ
top_k_valsr9   Ú	thresholdÚhigh_resolution_masks                  r#   Úget_block_idxesr¹   8  sè  € ð +?Ô*DÑ'€JÐ$ aà&¨Ò*Ð*Ø0°AÑ5ˆÝ”JÐ3Ð5IÐRfÔRmÐnÑnÔnˆ	Ýœ
¥5¤:¨iÀ6À'Ð#JÑ#JÔ#JÐU[Ð\Ñ\Ô\ˆØ3°mÀDÈ!È!È!ÈQÈQÈQÀJÔ6OÐRUÑ6UÑUÐà# aÒ'Ð'à    Ð$AÐ%AÐ$AÀ1À1À1Ð!DÔEÈÑKð 	˜Q˜Q˜QÐ =Ð!=Ð =¸q¸q¸qÐ@ÑAð !    A A AÐ'DÐ(DÐ'DÐ!DÔEÈÑKð 	˜Q˜Q˜Q   Ð#@Ð$@Ð#@Ð@ÑAõ ”Ø×$Ò$ Z°Ñ4Ô4°jÀbÐRVÐ_dðñ ô €Jð Ô €Gà�fÒÐØÔ%×)Ò)¨bÐ)Ñ1Ô1Ô8ˆ	Ø 4¸	À!À!À!ÀTÈ4À-Ô8PÒ P×WÒWÑYÔYÐÐØ	˜Ò	 Ð	 Ø#ÐÐå˜KÐLÐLÐLÑMÔMÐMàÐ(Ð(Ð(r%   c	                 óò  — t           €&t          j        | ¦  «                             ¦   «         S |                      ¦   «         \  }	}
}}|	|
z  }||z  dk    rt          d¦  «        ‚||z  }|                      |||¦  «        } |                     |||¦  «        }|                     |||¦  «        }|�6| |dd…dd…df         z  } ||dd…dd…df         z  }||dd…dd…df         z  }|dk    rt          | ||||¦  «        \  }}}}nX|dk    rCt          j        ¦   «         5  t          | |||¦  «        \  }}}}ddd¦  «         n# 1 swxY w Y   nt          d¦  «        ‚t          j        ¦   «         5  ||z
  }t          |||||¦  «        \  }}ddd¦  «         n# 1 swxY w Y   t                               | |||¬¦  «        t          j        |¦  «        z  }t          ||||¦  «        \  }}||z
  }|�)|dd	t!          ||¦  «        dd…dd…dd…df         z
  z  z
  }t          j        |¦  «        }t$                               ||||¦  «        }t&                               ||||¦  «        }|dk    �r°t          j        ||z
  d|z  z
  ¦  «        |dd…ddd…f         z  }t          j        ||¦  «        dd…dd…ddd…f                              d	d	|d	¦  «                             |||¦  «        }|                     d
¬¦  «        dd…dd…df                              d	d	|¦  «                             ||¦  «        }|                     d	d	|¦  «                             ||¦  «        |z
  } |�| |z  } t          j        | | dk                         ¦   «         z  ¦  «        }!||!dd…dd…df         z  }||!z  }t          j        |  | dk                         ¦   «         z  ¦  «        }"||"dd…dd…df         z  }||"z  }||z   |dd…dd…df         |dd…dd…df         z   dz   z  }#n+|dk    r||dd…dd…df         dz   z  }#nt          d¦  «        ‚|�|#|dd…dd…df         z  }#|#                     |	|
||¦  «        }#|#S )z0
    Use Mra to approximate self-attention.
    Nr   z4sequence length must be divisible by the block_size.r¨   r©   z&approx_mode must be "full" or "sparse")rJ   r�   r   r.   r,   rŽ   z-config.approx_mode must be "full" or "sparse")r"   rE   Ú
zeros_likeÚrequires_grad_r0   r1   rH   r¢   Úno_gradÚ	Exceptionr¹   rg   rw   r•   r–   r?   rO   Úexpr€   r…   r”   Úrepeatr‡   r’   )$r—   r˜   r™   rI   r®   r¯   rJ   r°   r±   rK   Únum_headrL   rš   Ú
meta_batchr›   r    r�   r¡   rœ   rW   Úlow_resolution_logit_normalizedr9   r¸   Úhigh_resolution_logitr=   r>   Úhigh_resolution_attnÚhigh_resolution_attn_outÚhigh_resolution_normalizerÚlow_resolution_attnÚlow_resolution_attn_outÚlow_resolution_normalizerÚlog_correctionÚlow_resolution_corrÚhigh_resolution_corrÚcontext_layers$                                       r#   Úmra2_attentionrÏ   ^  s2  € õ ÐÝÔ Ñ&Ô&×5Ò5Ñ7Ô7Ð7à.3¯jªj©l¬lÑ+€J�˜' 8Ø˜hÑ&€Jà�Ñ˜qÒ Ð ÝÐOÑPÔPÐPà :Ñ-Ðà�MŠM˜* g¨xÑ8Ô8€EØ
�+Š+�j '¨8Ñ
4Ô
4€CØ�MŠM˜* g¨xÑ8Ô8€EàÐØ˜˜Q˜Q˜Q    4˜ZÔ(Ñ(ˆØ�D˜˜˜˜A˜A˜A˜t˜Ô$Ñ$ˆØ˜˜Q˜Q˜Q    4˜ZÔ(Ñ(ˆà�fÒÐÝUmØ�3˜
 D¨%ñV
ô V
ÑRÐ˜kÐ+GÈÈð 
˜Ò	 Ð	 ÝŒ]‰_Œ_ð 	ð 	ÝQiØ�s˜J¨ñRô RÑNÐ  +Ð/KÈQð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	øõ
 Ð@ÑAÔAÐAå	Œ‰Œð 
ð 
Ø*>ÐA]Ñ*]Ð'Ý(7Ø+ØØØ(Ø+ñ)
ô )
Ñ%ˆÐ%ð
ð 
ð 
ñ 
ô 
ð 
ð 
ð 
ð 
ð 
ð 
øøøð 
ð 
ð 
ð 
õ 2×?Ò?Øˆs�G¨
ð @ñ ô åŒ	�(ÑÔñÐõ ",Ð,AÀ7ÐL]Ð_pÑ!qÔ!qÑ€HÐØ1Ð4DÑDÐØÐØ 5¸¸qÅ;ÈtÐU\ÑC]ÔC]Ð^_Ð^_Ð^_ÐabÐabÐabÐdeÐdeÐdeÐgkÐ^kÔClÑ?lÑ8mÑ mÐÝ œ9Ð%:Ñ;Ô;ÐÝ3×AÒAØ˜g uÐ.?ñ ô  Ðõ ".×!;Ò!;Ø˜gÐ'8Ð:Kñ"ô "Ðð �fÒÑåŒIÐ*Ð-IÑIÈCÐRfÑLfÑfÑgÔgØ˜!˜!˜!˜T 1 1 1˜*Ô%ñ&ð 	õ ŒLÐ,¨iÑ8Ô8¸¸¸¸A¸A¸A¸tÀQÀQÀQ¸ÔGßŠV�A�q˜* aÑ(Ô(ßŠW�Z ¨(Ñ3Ô3ð 	 ð  ×#Ò#¨Ð#Ñ+Ô+¨A¨A¨A¨q¨q¨q°$¨JÔ7×>Ò>¸qÀ!ÀZÑPÔP×XÒXÐYcÐelÑmÔmð 	"ð 6×<Ò<¸QÀÀ:ÑNÔN×VÒVÐWaÐcjÑkÔkÐnvÑvˆØÐØ+¨dÑ2ˆNå#œi¨¸.ÈAÒ:M×9TÒ9TÑ9VÔ9VÑ(VÑWÔWÐØ"9Ð<OÐPQÐPQÐPQÐSTÐSTÐSTÐVZÐPZÔ<[Ñ"[ÐØ$=Ð@SÑ$SÐ!å$œy¨.¨¸NÈQÒ<N×;UÒ;UÑ;WÔ;WÑ)WÑXÔXÐØ#;Ð>RÐSTÐSTÐSTÐVWÐVWÐVWÐY]ÐS]Ô>^Ñ#^Ð Ø%?ÐBVÑ%VÐ"à1Ð4KÑKØ& q q q¨!¨!¨!¨T zÔ2Ð5NÈqÈqÈqÐRSÐRSÐRSÐUYÈzÔ5ZÑZÐ]aÑañ
ˆˆð 
˜Ò	 Ð	 Ø0Ð4NÈqÈqÈqÐRSÐRSÐRSÐUYÈzÔ4ZÐ]aÑ4aÑbˆˆåÐGÑHÔHÐHàÐØ%¨¨Q¨Q¨Q°°°°4¨ZÔ(8Ñ8ˆà!×)Ò)¨*°hÀÈÑRÔR€MàÐs$   ÄEÅEÅ	EÅ0FÆFÆFc                   ó*   ‡ — e Zd ZdZˆ fd„Zdd„Zˆ xZS )ÚMraEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 óð  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        ¬¦  «        | _        t          j        |j        dz   |j        ¦  «        | _	        t          j        |j
        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j        ¦  «        | _        |                      dt%          j        |j        ¦  «                             d¦  «        dz   ¦  «         |                      dt%          j        | j                             ¦   «         t$          j        | j        j        ¬¦  «        d¬	¦  «         d S )
N)Úpadding_idxr(   ©ÚepsÚposition_ids©r   r.   Útoken_type_idsrA   F)Ú
persistent)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferrE   rF   Úexpandrˆ   rÖ   r0   rG   rC   ©ÚselfÚconfigÚ	__class__s     €r#   rÛ   zMraEmbeddings.__init__Ô  s9  ø€ Ý‰Œ×ÒÑÔÐÝ!œ|¨FÔ,=¸vÔ?QÐ_eÔ_rÐsÑsÔsˆÔÝ#%¤<°Ô0NÐQRÑ0RÐTZÔTfÑ#gÔ#gˆÔ Ý%'¤\°&Ô2HÈ&ÔJ\Ñ%]Ô%]ˆÔ"åœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒð 	×Ò˜^­U¬\¸&Ô:XÑ-YÔ-Y×-`Ò-`ÐahÑ-iÔ-iÐlmÑ-mÑnÔnÐnØ×ÒØÝŒK˜Ô)×.Ò.Ñ0Ô0½¼
È4ÔK\ÔKcÐdÑdÔdØð 	ñ 	
ô 	
ð 	
ð 	
ð 	
r%   Nc                 ón  — |�|                      ¦   «         }n|                      ¦   «         d d…         }|d         }|€| j        d d …d |…f         }|€mt          | d¦  «        r2| j        d d …d |…f         }|                     |d         |¦  «        }|}n+t          j        |t
          j        | j        j        ¬¦  «        }|€|  	                    |¦  «        }|  
                    |¦  «        }	||	z   }
|                      |¦  «        }|
|z  }
|                      |
¦  «        }
|                      |
¦  «        }
|
S )Nr.   r   rØ   r   rA   )r0   rÖ   ÚhasattrrØ   rë   rE   rˆ   rG   rC   rà   rä   râ   rå   ré   )rí   Ú	input_idsrØ   rÖ   Úinputs_embedsÚinput_shapeÚ
seq_lengthÚbuffered_token_type_idsÚ buffered_token_type_ids_expandedrä   Ú
embeddingsrâ   s               r#   rl   zMraEmbeddings.forwardå  sT  € ØÐ Ø#Ÿ.š.Ñ*Ô*ˆKˆKà'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKà  ”^ˆ
àÐØÔ,¨Q¨Q¨Q°°°¨^Ô<ˆLð
 Ð!Ý�tÐ-Ñ.Ô.ð mØ*.Ô*=¸a¸a¸aÀÀ*À¸nÔ*MÐ'Ø3J×3QÒ3QÐR]Ð^_ÔR`ÐblÑ3mÔ3mÐ0Ø!A��å!&¤¨[ÅÄ
ÐSWÔSdÔSkÐ!lÑ!lÔ!l�àÐ Ø ×0Ò0°Ñ;Ô;ˆMØ $× :Ò :¸>Ñ JÔ JÐØ"Ð%:Ñ:ˆ
à"×6Ò6°|ÑDÔDÐØÐ)Ñ)ˆ
à—^’^ JÑ/Ô/ˆ
Ø—\’\ *Ñ-Ô-ˆ
ØÐr%   )NNNN©rz   r{   r|   Ú__doc__rÛ   rl   Ú__classcell__©rï   s   @r#   rÑ   rÑ   Ñ  sR   ø€ € € € € ØQÐQð
ð 
ð 
ð 
ð 
ð" ð  ð  ð  ð  ð  ð  ð  r%   rÑ   c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )ÚMraSelfAttentionc                 ó0  •— t          ¦   «                              ¦   «          |j        |j        z  dk    r0t	          |d¦  «        s t          d|j        › d|j        › d�¦  «        ‚t          d u}t          ¦   «         rbt          ¦   «         rTt          ¦   «         rF|sD	 t          ¦   «          n4# t          $ r'}t                               d|› �¦  «         Y d }~nd }~ww xY w|j        | _        t          |j        |j        z  ¦  «        | _        | j        | j        z  | _        t#          j        |j        | j        ¦  «        | _        t#          j        |j        | j        ¦  «        | _        t#          j        |j        | j        ¦  «        | _        t#          j        |j        ¦  «        | _        |j        dz  |j        z  | _        t9          | j        t          |j        dz  dz  ¦  «        ¦  «        | _        |j        | _        |j        | _        |j        | _        d S )	Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)zGCould not load the custom kernel for multi-scale deformable attention: r*   r(   ) rÚ   rÛ   rÞ   Únum_attention_headsrñ   r1   r"   r   r   r   r$   r¾   ÚloggerÚwarningr6   Úattention_head_sizeÚall_head_sizer   ÚLinearr—   r˜   r™   rç   Úattention_probs_dropout_probré   rá   Úblock_per_rowrM   r­   r¯   r°   r±   )rí   rî   Úkernel_loadedÚerï   s       €r#   rÛ   zMraSelfAttention.__init__	  s"  ø€ Ý‰Œ×ÒÑÔÐØÔ Ô :Ñ:¸aÒ?Ð?ÍÐPVÐXhÑHiÔHiÐ?Ýð8 FÔ$6ð 8ð 8Ø Ô4ð8ð 8ð 8ñô ð õ
 (¨tÐ3ˆÝ"Ñ$Ô$ð 	nÕ)9Ñ);Ô);ð 	nÕ@RÑ@TÔ@Tð 	nÐ]jð 	nðnÝ!Ñ#Ô#Ð#Ð#øÝð nð nð nÝ—’ÐlÐijÐlÐlÑmÔmÐmÐmÐmÐmÐmÐmøøøøðnøøøð $*Ô#=ˆÔ Ý#& vÔ'9¸FÔ<VÑ'VÑ#WÔ#WˆÔ Ø!Ô5¸Ô8PÑPˆÔå”Y˜vÔ1°4Ô3EÑFÔFˆŒ
Ý”9˜VÔ/°Ô1CÑDÔDˆŒÝ”Y˜vÔ1°4Ô3EÑFÔFˆŒ
å”z &Ô"EÑFÔFˆŒà Ô8¸BÑ>À&ÔBVÑVˆŒÝ˜Tœ^­S°&Ô2PÐTVÑ2VÐ[\Ñ1\Ñ-]Ô-]Ñ^Ô^ˆŒà!Ô-ˆÔØ,2Ô,OˆÔ)Ø/5Ô/UˆÔ,Ð,Ð,s   ÂB* Â*
CÂ4CÃCNc           
      ó*  — |j         \  }}}|                      |¦  «                             |d| j        | j        ¦  «                             dd¦  «        }|                      |¦  «                             |d| j        | j        ¦  «                             dd¦  «        }|                      |¦  «                             |d| j        | j        ¦  «                             dd¦  «        }d|dz  z   }|                     ¦   «          	                    d| j        d¦  «         
                    || j        z  |¦  «                             ¦   «         }d}	| j        |	k     r¦|| j        ||	| j        z
  f}
t          j        |t          j        |
|j        ¬¦  «        gd¬¦  «        }t          j        |t          j        |
|j        ¬¦  «        gd¬¦  «        }t          j        |t          j        |
|j        ¬¦  «        gd¬¦  «        }t!          |                     ¦   «         |                     ¦   «         |                     ¦   «         |                     ¦   «         | j        | j        | j        | j        ¬	¦  «        }| j        |	k     r|d d …d d …d d …d | j        …f         }| 
                    || j        || j        ¦  «        }|                     d
ddd¦  «                             ¦   «         }|                     ¦   «         d d…         | j        fz   } |j        |Ž }|f}|S )Nr.   r   r(   g      ð?r�   r*   r¤   r,   )r¯   r°   r±   r   r   r+   )rD   r—   Úviewr  r  r4   r˜   r™   ÚsqueezerÀ   rH   r6   rE   Úcatrˆ   rC   rÏ   r’   rM   r¯   r°   r±   Úpermuter5   r0   r  )rí   Úhidden_statesÚattention_maskrK   rL   rW   Úquery_layerÚ	key_layerÚvalue_layerÚgpu_warp_sizeÚpad_sizerÎ   Únew_context_layer_shapeÚoutputss                 r#   rl   zMraSelfAttention.forward)  sõ  € Ø!.Ô!4Ñˆ
�G˜Qà�JŠJ�}Ñ%Ô%ßŠT�*˜b $Ô":¸DÔ<TÑUÔUßŠY�q˜!‰_Œ_ð 	ð �HŠH�]Ñ#Ô#ßŠT�*˜b $Ô":¸DÔ<TÑUÔUßŠY�q˜!‰_Œ_ð 	ð �JŠJ�}Ñ%Ô%ßŠT�*˜b $Ô":¸DÔ<TÑUÔUßŠY�q˜!‰_Œ_ð 	ð ˜~°Ñ7Ñ7ˆà×"Ò"Ñ$Ô$ßŠV�A�tÔ/°Ñ3Ô3ßŠW�Z $Ô":Ñ:¸GÑDÔDßŠS‰UŒUð	 	ð ˆàÔ# mÒ3Ð3Ø! 4Ô#;¸WÀmÐVZÔVnÑFnÐnˆHåœ) [µ%´+¸hÈ{ÔOaÐ2bÑ2bÔ2bÐ$cÐikÐlÑlÔlˆKÝœ	 9­e¬k¸(È9ÔK[Ð.\Ñ.\Ô.\Ð"]ÐceÐfÑfÔfˆIÝœ) [µ%´+¸hÈ{ÔOaÐ2bÑ2bÔ2bÐ$cÐikÐlÑlÔlˆKå&Ø×ÒÑÔØ�OŠOÑÔØ×ÒÑÔØ× Ò Ñ"Ô"ØŒNØÔ(Ø)-Ô)JØ,0Ô,Pð	
ñ 	
ô 	
ˆð Ô# mÒ3Ð3Ø)¨!¨!¨!¨Q¨Q¨Q°°°Ð3M°TÔ5MÐ3MÐ*MÔNˆMà%×-Ò-¨j¸$Ô:RÐT[Ð]aÔ]uÑvÔvˆà%×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔFˆØ"/×"4Ò"4Ñ"6Ô"6°s¸°sÔ";¸tÔ?QÐ>SÑ"SÐØ*˜Ô*Ð,CÐDˆà Ð"ˆàˆr%   ri   ©rz   r{   r|   rÛ   rl   rû   rü   s   @r#   rþ   rþ     sR   ø€ € € € € ðVð Vð Vð Vð Vð@<ð <ð <ð <ð <ð <ð <ð <r%   rþ   c                   óP   ‡ — e Zd Zˆ fd„Zdej        dej        dej        fd„Zˆ xZS )ÚMraSelfOutputc                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j	        ¦  «        | _
        d S ©NrÔ   )rÚ   rÛ   r   r  rÞ   Údenserå   ræ   rç   rè   ré   rì   s     €r#   rÛ   zMraSelfOutput.__init__j  sf   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr%   r  Úinput_tensorÚreturnc                 óŠ   — |                       |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|S ri   ©r  ré   rå   ©rí   r  r   s      r#   rl   zMraSelfOutput.forwardp  ó@   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØŸš }°|Ñ'CÑDÔDˆØÐr%   ©rz   r{   r|   rÛ   rE   ÚTensorrl   rû   rü   s   @r#   r  r  i  ói   ø€ € € € € ð>ð >ð >ð >ð >ð U¤\ð ÀÄð ÐRWÔR^ð ð ð ð ð ð ð ð r%   r  c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )ÚMraAttentionc                 ó˜   •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          |¦  «        | _        d S ri   )rÚ   rÛ   rþ   rí   r  rŒ   rì   s     €r#   rÛ   zMraAttention.__init__x  s;   ø€ Ý‰Œ×ÒÑÔÐÝ$ VÑ,Ô,ˆŒ	Ý# FÑ+Ô+ˆŒˆˆr%   Nc                 ó†   — |                       ||¦  «        }|                      |d         |¦  «        }|f|dd …         z   }|S ©Nr   r   )rí   rŒ   )rí   r  r  Úself_outputsÚattention_outputr  s         r#   rl   zMraAttention.forward}  sH   € Ø—y’y °Ñ?Ô?ˆØŸ;š; |°A¤¸ÑFÔFÐØ#Ð%¨°Q°R°RÔ(8Ñ8ˆØˆr%   ri   r  rü   s   @r#   r*  r*  w  sL   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð
ð ð ð ð ð ð ð r%   r*  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚMraIntermediatec                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j        t          ¦  «        rt          |j                 | _        d S |j        | _        d S ri   )rÚ   rÛ   r   r  rÞ   Úintermediate_sizer  Ú
isinstanceÚ
hidden_actÚstrr	   Úintermediate_act_fnrì   s     €r#   rÛ   zMraIntermediate.__init__†  sn   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3KÑLÔLˆŒ
Ý�fÔ'­Ñ-Ô-ð 	9Ý'-¨fÔ.?Ô'@ˆDÔ$Ð$Ð$à'-Ô'8ˆDÔ$Ð$Ð$r%   r  r!  c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S ri   )r  r7  ©rí   r  s     r#   rl   zMraIntermediate.forwardŽ  s,   € ØŸ
š
 =Ñ1Ô1ˆØ×0Ò0°Ñ?Ô?ˆØÐr%   r&  rü   s   @r#   r1  r1  …  s^   ø€ € € € € ð9ð 9ð 9ð 9ð 9ð U¤\ð °e´lð ð ð ð ð ð ð ð r%   r1  c                   óP   ‡ — e Zd Zˆ fd„Zdej        dej        dej        fd„Zˆ xZS )Ú	MraOutputc                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S r  )rÚ   rÛ   r   r  r3  rÞ   r  rå   ræ   rç   rè   ré   rì   s     €r#   rÛ   zMraOutput.__init__–  sf   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ7¸Ô9KÑLÔLˆŒ
Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr%   r  r   r!  c                 óŠ   — |                       |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|S ri   r#  r$  s      r#   rl   zMraOutput.forwardœ  r%  r%   r&  rü   s   @r#   r;  r;  •  r(  r%   r;  c                   ó,   ‡ — e Zd Zˆ fd„Zdd„Zd„ Zˆ xZS )ÚMraLayerc                 óþ   •— t          ¦   «                              ¦   «          |j        | _        d| _        t	          |¦  «        | _        |j        | _        t          |¦  «        | _        t          |¦  «        | _
        d S ©Nr   )rÚ   rÛ   Úchunk_size_feed_forwardÚseq_len_dimr*  Ú	attentionÚadd_cross_attentionr1  Úintermediater;  rŒ   rì   s     €r#   rÛ   zMraLayer.__init__¤  si   ø€ Ý‰Œ×ÒÑÔÐØ'-Ô'EˆÔ$ØˆÔÝ% fÑ-Ô-ˆŒØ#)Ô#=ˆÔ Ý+¨FÑ3Ô3ˆÔÝ Ñ'Ô'ˆŒˆˆr%   Nc                 ó¤   — |                       ||¦  «        }|d         }|dd …         }t          | j        | j        | j        |¦  «        }|f|z   }|S r-  )rD  r   Úfeed_forward_chunkrB  rC  )rí   r  r  Úself_attention_outputsr/  r  Úlayer_outputs          r#   rl   zMraLayer.forward­  sd   € Ø!%§¢°¸~Ñ!NÔ!NÐØ1°!Ô4Ðà(¨¨¨Ô,ˆå0ØÔ# TÔ%AÀ4ÔCSÐUeñ
ô 
ˆð  �/ GÑ+ˆàˆr%   c                 ó\   — |                       |¦  «        }|                      ||¦  «        }|S ri   )rF  rŒ   )rí   r/  Úintermediate_outputrJ  s       r#   rH  zMraLayer.feed_forward_chunkº  s2   € Ø"×/Ò/Ð0@ÑAÔAÐØ—{’{Ð#6Ð8HÑIÔIˆØÐr%   ri   )rz   r{   r|   rÛ   rl   rH  rû   rü   s   @r#   r?  r?  £  s[   ø€ € € € € ð(ð (ð (ð (ð (ðð ð ð ðð ð ð ð ð ð r%   r?  c                   ó,   ‡ — e Zd Zˆ fd„Z	 	 	 dd„Zˆ xZS )Ú
MraEncoderc                 óÔ   •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r~   )r?  )Ú.0rW   rî   s     €r#   ú
<listcomp>z'MraEncoder.__init__.<locals>.<listcomp>Ä  s!   ø€ Ð#^Ð#^Ð#^¸¥H¨VÑ$4Ô$4Ð#^Ð#^Ð#^r%   F)	rÚ   rÛ   rî   r   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayerÚgradient_checkpointingrì   s    `€r#   rÛ   zMraEncoder.__init__Á  s`   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”]Ð#^Ð#^Ð#^Ð#^½eÀFÔD\Ñ>]Ô>]Ð#^Ñ#^Ô#^Ñ_Ô_ˆŒ
Ø&+ˆÔ#Ð#Ð#r%   NFTc                 óæ   — |rdnd }t          | j        ¦  «        D ]!\  }}|r||fz   } |||¦  «        }|d         }Œ"|r||fz   }|st          d„ ||fD ¦   «         ¦  «        S t          ||¬¦  «        S )Nr~   r   c              3   ó   K  — | ]}|®|V — Œ	d S ri   r~   )rQ  Úvs     r#   ú	<genexpr>z%MraEncoder.forward.<locals>.<genexpr>Ü  s"   è è € ÐXÐX˜qÈ!È-˜È-È-È-È-ÐXÐXr%   )Úlast_hidden_stater  )Ú	enumeraterV  Útupler   )	rí   r  r  Úoutput_hidden_statesÚreturn_dictÚall_hidden_statesÚiÚlayer_moduleÚlayer_outputss	            r#   rl   zMraEncoder.forwardÇ  sÄ   € ð #7Ð@˜B˜B¸DÐå(¨¬Ñ4Ô4ð 	-ð 	-‰OˆAˆ|Ø#ð IØ$5¸Ð8HÑ$HÐ!à(˜L¨¸ÑGÔGˆMà)¨!Ô,ˆMˆMàð 	EØ 1°]Ð4DÑ DÐàð 	YÝÐXÐX ]Ð4EÐ$FÐXÑXÔXÑXÔXÐXÝ1Ø+Ø+ð
ñ 
ô 
ð 	
r%   )NFTr  rü   s   @r#   rN  rN  À  sW   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð Ø"Øð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r%   rN  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚMraPredictionHeadTransformc                 óV  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j        t          ¦  «        rt          |j                 | _
        n|j        | _
        t          j        |j        |j        ¬¦  «        | _        d S r  )rÚ   rÛ   r   r  rÞ   r  r4  r5  r6  r	   Útransform_act_fnrå   ræ   rì   s     €r#   rÛ   z#MraPredictionHeadTransform.__init__å  s…   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ý�fÔ'­Ñ-Ô-ð 	6Ý$*¨6Ô+<Ô$=ˆDÔ!Ð!à$*Ô$5ˆDÔ!Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒˆˆr%   r  r!  c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S ri   )r  rh  rå   r9  s     r#   rl   z"MraPredictionHeadTransform.forwardî  s=   € ØŸ
š
 =Ñ1Ô1ˆØ×-Ò-¨mÑ<Ô<ˆØŸš }Ñ5Ô5ˆØÐr%   r&  rü   s   @r#   rf  rf  ä  sc   ø€ € € € € ðUð Uð Uð Uð Uð U¤\ð °e´lð ð ð ð ð ð ð ð r%   rf  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚMraLMPredictionHeadc                 ó  •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          j        |j        |j        d¬¦  «        | _        t	          j	        t          j        |j        ¦  «        ¦  «        | _        d S )NT)Úbias)rÚ   rÛ   rf  Ú	transformr   r  rÞ   rÝ   ÚdecoderÚ	ParameterrE   rˆ   rm  rì   s     €r#   rÛ   zMraLMPredictionHead.__init__÷  sj   ø€ Ý‰Œ×ÒÑÔÐÝ3°FÑ;Ô;ˆŒõ ”y Ô!3°VÔ5FÈTÐRÑRÔRˆŒÝ”L¥¤¨VÔ->Ñ!?Ô!?Ñ@Ô@ˆŒ	ˆ	ˆ	r%   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S ri   )rn  ro  r9  s     r#   rl   zMraLMPredictionHead.forward   s*   € ØŸš }Ñ5Ô5ˆØŸš ]Ñ3Ô3ˆØÐr%   r  rü   s   @r#   rk  rk  ö  sL   ø€ € € € € ðAð Að Að Að Aðð ð ð ð ð ð r%   rk  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚMraOnlyMLMHeadc                 óp   •— t          ¦   «                              ¦   «          t          |¦  «        | _        d S ri   )rÚ   rÛ   rk  Úpredictionsrì   s     €r#   rÛ   zMraOnlyMLMHead.__init__  s/   ø€ Ý‰Œ×ÒÑÔÐÝ.¨vÑ6Ô6ˆÔÐÐr%   Úsequence_outputr!  c                 ó0   — |                       |¦  «        }|S ri   )ru  )rí   rv  Úprediction_scoress      r#   rl   zMraOnlyMLMHead.forward  s   € Ø ×,Ò,¨_Ñ=Ô=ÐØ Ð r%   r&  rü   s   @r#   rs  rs    s^   ø€ € € € € ð7ð 7ð 7ð 7ð 7ð! u¤|ð !¸¼ð !ð !ð !ð !ð !ð !ð !ð !r%   rs  c                   ól   ‡ — e Zd ZU eed<   dZdZ ej        ¦   «         de	j
        fˆ fd„¦   «         Zˆ xZS )ÚMraPreTrainedModelrî   ÚmraTÚmodulec                 ó®  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rt	          j        |j        ¦  «         dS t          |t          ¦  «        rmt	          j        |j	        t          j        |j	        j        d         ¦  «                             d¦  «        dz   ¦  «         t	          j        |j        ¦  «         dS dS )zInitialize the weightsr.   r×   r(   N)rÚ   Ú_init_weightsr4  rk  ÚinitÚzeros_rm  rÑ   Úcopy_rÖ   rE   rF   rD   rë   rØ   )rí   r|  rï   s     €r#   r~  z MraPreTrainedModel._init_weights  s½   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ1Ñ2Ô2ð 	/ÝŒK˜œÑ$Ô$Ð$Ð$Ð$Ý˜¥Ñ.Ô.ð 	/ÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÐjkÑ,kÑlÔlÐlÝŒK˜Ô-Ñ.Ô.Ð.Ð.Ð.ð	/ð 	/r%   )rz   r{   r|   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingrE   r½   r   ÚModuler~  rû   rü   s   @r#   rz  rz    st   ø€ € € € € € ð ÐÐÑØÐØ&*Ð#à€U„]�_„_ð/ B¤Ið /ð /ð /ð /ð /ñ „_ð/ð /ð /ð /ð /r%   rz  c                   óÔ   ‡ — e Zd Zˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
e	dz  de	dz  de
ez  fd„¦   «         Zˆ xZS )ÚMraModelc                 óÐ   •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S ri   )rÚ   rÛ   rî   rÑ   rø   rN  ÚencoderÚ	post_initrì   s     €r#   rÛ   zMraModel.__init__%  sX   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒå'¨Ñ/Ô/ˆŒÝ! &Ñ)Ô)ˆŒð 	�ŠÑÔÐÐÐr%   c                 ó   — | j         j        S ri   ©rø   rà   ©rí   s    r#   Úget_input_embeddingszMraModel.get_input_embeddings/  s   € ØŒÔ.Ð.r%   c                 ó   — || j         _        d S ri   rŒ  )rí   r™   s     r#   Úset_input_embeddingszMraModel.set_input_embeddings2  s   € Ø*/ˆŒÔ'Ð'Ð'r%   Nrò   r  rØ   rÖ   ró   r_  r`  r!  c                 óv  — |�|n| j         j        }|�|n| j         j        }|�|�t          d¦  «        ‚|�+|                      ||¦  «         |                     ¦   «         }	n.|�|                     ¦   «         d d…         }	nt          d¦  «        ‚|	\  }
}|�|j        n|j        }|€t          j        |
|f|¬¦  «        }|€gt          | j
        d¦  «        r1| j
        j        d d …d |…f         }|                     |
|¦  «        }|}n!t          j        |	t          j        |¬¦  «        }|  
                    ||||¬¦  «        }t          | j         |d d …dd	…d d …f         |d
„ ¬¦  «        }|                      ||||¬¦  «        }|d         }|s|f|d	d …         z   S t#          ||j        |j        |j        ¬¦  «        S )NzDYou cannot specify both input_ids and inputs_embeds at the same timer.   z5You have to specify either input_ids or inputs_embedsr¤   rØ   rA   )rò   rÖ   rØ   ró   r   r   c                  óB   — t          j        dt           j        ¬¦  «        S )NT)rB   )rE   ÚtensorÚbool)Úargss    r#   ú<lambda>z"MraModel.forward.<locals>.<lambda>j  s   € ­E¬L¸ÅUÄZÐ,PÑ,PÔ,P€ r%   )rî   ró   r  Úand_mask_function)r  r_  r`  )r\  r  Ú
attentionsÚcross_attentions)rî   r_  r`  r1   Ú%warn_if_padding_and_no_attention_maskr0   rC   rE   r‘   rñ   rø   rØ   rë   rˆ   rG   r
   r‰  r   r  r˜  r™  )rí   rò   r  rØ   rÖ   ró   r_  r`  Úkwargsrô   rK   rõ   rC   rö   r÷   Úembedding_outputÚencoder_outputsrv  s                     r#   rl   zMraModel.forward5  s7  € ð %9Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàÐ  ]Ð%>ÝÐcÑdÔdÐdØÐ"Ø×6Ò6°yÀ.ÑQÔQÐQØ#Ÿ.š.Ñ*Ô*ˆKˆKØÐ&Ø'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKˆKåÐTÑUÔUÐUà!,Ñˆ
�JØ%.Ð%:�Ô!Ð!ÀÔ@TˆàÐ!Ý"œZ¨*°jÐ)AÈ6ÐRÑRÔRˆNàÐ!Ý�t”Ð(8Ñ9Ô9ð [Ø*.¬/Ô*HÈÈÈÈKÈZÈKÈÔ*XÐ'Ø3J×3QÒ3QÐR\Ð^hÑ3iÔ3iÐ0Ø!A��å!&¤¨[ÅÄ
ÐSYÐ!ZÑ!ZÔ!Z�àŸ?š?ØØ%Ø)Ø'ð	 +ñ 
ô 
Ðõ 3Ø”;Ø*¨1¨1¨1¨a°¨c°1°1°1¨9Ô5Ø)àPÐPð
ñ 
ô 
ˆð Ÿ,š,ØØ)Ø!5Ø#ð	 'ñ 
ô 
ˆð *¨!Ô,ˆàð 	<Ø#Ð%¨¸¸¸Ô(;Ñ;Ð;å1Ø-Ø)Ô7Ø&Ô1Ø,Ô=ð	
ñ 
ô 
ð 	
r%   )NNNNNNN)rz   r{   r|   rÛ   rŽ  r�  r   rE   r'  r”  r^  r   rl   rû   rü   s   @r#   r‡  r‡  #  s"  ø€ € € € € ðð ð ð ð ð/ð /ð /ð0ð 0ð 0ð ð *.Ø.2Ø.2Ø,0Ø-1Ø,0Ø#'ðG
ð G
à”< $Ñ&ðG
ð œ tÑ+ðG
ð œ tÑ+ð	G
ð
 ”l TÑ)ðG
ð ”| dÑ*ðG
ð # T™kðG
ð ˜D‘[ðG
ð 
Ð3Ñ	3ðG
ð G
ð G
ñ „^ðG
ð G
ð G
ð G
ð G
r%   r‡  c                   óô   ‡ — e Zd ZdddœZˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 ddej	        dz  d	ej	        dz  d
ej	        dz  dej	        dz  dej	        dz  dej	        dz  de
dz  de
dz  deez  fd„¦   «         Zˆ xZS )ÚMraForMaskedLMzcls.predictions.biasz%mra.embeddings.word_embeddings.weight)zcls.predictions.decoder.biaszcls.predictions.decoder.weightc                 óÂ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¦  «        | _        |                      ¦   «          d S ri   )rÚ   rÛ   r‡  r{  rs  ÚclsrŠ  rì   s     €r#   rÛ   zMraForMaskedLM.__init__‡  sQ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å˜FÑ#Ô#ˆŒÝ! &Ñ)Ô)ˆŒð 	�ŠÑÔÐÐÐr%   c                 ó$   — | j         j        j        S ri   )r¡  ru  ro  r�  s    r#   Úget_output_embeddingsz$MraForMaskedLM.get_output_embeddings�  s   € ØŒxÔ#Ô+Ð+r%   c                 óT   — || j         j        _        |j        | j         j        _        d S ri   )r¡  ru  ro  rm  )rí   Únew_embeddingss     r#   Úset_output_embeddingsz$MraForMaskedLM.set_output_embeddings“  s%   € Ø'5ˆŒÔÔ$Ø$2Ô$7ˆŒÔÔ!Ð!Ð!r%   Nrò   r  rØ   rÖ   ró   Úlabelsr_  r`  r!  c	           	      ó   — |�|n| j         j        }|                      |||||||¬¦  «        }
|
d         }|                      |¦  «        }d}|�Kt	          ¦   «         } ||                     d| j         j        ¦  «        |                     d¦  «        ¦  «        }|s|f|
dd…         z   }|�|f|z   n|S t          |||
j        |
j	        ¬¦  «        S )a£  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        N©r  rØ   rÖ   ró   r_  r`  r   r.   r   ©ÚlossÚlogitsr  r˜  )
rî   r`  r{  r¡  r   r  rÝ   r   r  r˜  )rí   rò   r  rØ   rÖ   ró   r§  r_  r`  r›  r  rv  rx  Úmasked_lm_lossÚloss_fctrŒ   s                   r#   rl   zMraForMaskedLM.forward—  s  € ð& &1Ð%<�k�kÀ$Ä+ÔBYˆà—(’(ØØ)Ø)Ø%Ø'Ø!5Ø#ð ñ 
ô 
ˆð " !œ*ˆØ ŸHšH _Ñ5Ô5ÐàˆØÐÝ'Ñ)Ô)ˆHØ%˜XÐ&7×&<Ò&<¸RÀÄÔAWÑ&XÔ&XÐZ`×ZeÒZeÐfhÑZiÔZiÑjÔjˆNàð 	ZØ'Ð)¨G°A°B°B¬KÑ7ˆFØ3AÐ3M�^Ð%¨Ñ.Ð.ÐSYÐYåØØ$Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r%   ©NNNNNNNN)rz   r{   r|   Ú_tied_weights_keysrÛ   r£  r¦  r   rE   r'  r”  r^  r   rl   rû   rü   s   @r#   rŸ  rŸ  €  s9  ø€ € € € € ð )?Ø*Qðð Ðð
ð ð ð ð ð,ð ,ð ,ð8ð 8ð 8ð ð *.Ø.2Ø.2Ø,0Ø-1Ø&*Ø,0Ø#'ð/
ð /
à”< $Ñ&ð/
ð œ tÑ+ð/
ð œ tÑ+ð	/
ð
 ”l TÑ)ð/
ð ”| dÑ*ð/
ð ”˜tÑ#ð/
ð # T™kð/
ð ˜D‘[ð/
ð 
�Ñ	ð/
ð /
ð /
ñ „^ð/
ð /
ð /
ð /
ð /
r%   rŸ  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚMraClassificationHeadz-Head for sentence-level classification tasks.c                 ó"  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        t          j        |j        |j	        ¦  «        | _
        || _        d S ri   )rÚ   rÛ   r   r  rÞ   r  rç   rè   ré   Ú
num_labelsÚout_projrî   rì   s     €r#   rÛ   zMraClassificationHead.__init__Î  sj   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒÝœ	 &Ô"4°fÔ6GÑHÔHˆŒàˆŒˆˆr%   c                 ó
  — |d d …dd d …f         }|                       |¦  «        }|                      |¦  «        }t          | j        j                 |¦  «        }|                       |¦  «        }|                      |¦  «        }|S )Nr   )ré   r  r	   rî   r5  rµ  )rí   Úfeaturesr›  Úxs       r#   rl   zMraClassificationHead.forwardÖ  st   € Ø�Q�Q�Q˜˜1˜1˜1�WÔˆØ�LŠL˜‰OŒOˆØ�JŠJ�q‰MŒMˆÝ�4”;Ô)Ô*¨1Ñ-Ô-ˆØ�LŠL˜‰OŒOˆØ�MŠM˜!ÑÔˆØˆr%   rù   rü   s   @r#   r²  r²  Ë  sM   ø€ € € € € Ø7Ð7ðð ð ð ð ðð ð ð ð ð ð r%   r²  z›
    MRA Model transformer with a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks.
    )Úcustom_introc                   óÞ   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	edz  d
edz  dee	z  fd„¦   «         Z
ˆ xZS )ÚMraForSequenceClassificationc                 óÚ   •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S ri   )rÚ   rÛ   r´  r‡  r{  r²  Ú
classifierrŠ  rì   s     €r#   rÛ   z%MraForSequenceClassification.__init__ç  s[   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒÝ˜FÑ#Ô#ˆŒÝ/°Ñ7Ô7ˆŒð 	�ŠÑÔÐÐÐr%   Nrò   r  rØ   rÖ   ró   r§  r_  r`  r!  c	           	      óÀ  — |�|n| j         j        }|                      |||||||¬¦  «        }
|
d         }|                      |¦  «        }d}|��Z| j         j        €f| j        dk    rd| j         _        nN| j        dk    r7|j        t          j        k    s|j        t          j	        k    rd| j         _        nd| j         _        | j         j        dk    rWt          ¦   «         }| j        dk    r1 ||                     ¦   «         |                     ¦   «         ¦  «        }nŽ |||¦  «        }n�| j         j        dk    rGt          ¦   «         } ||                     d| j        ¦  «        |                     d¦  «        ¦  «        }n*| j         j        dk    rt          ¦   «         } |||¦  «        }|s|f|
dd…         z   }|�|f|z   n|S t          |||
j        |
j        ¬	¦  «        S )
a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr©  r   r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr.   rª  )rî   r`  r{  r½  Úproblem_typer´  rB   rE   rG   r6   r   r  r   r  r   r   r  r˜  )rí   rò   r  rØ   rÖ   ró   r§  r_  r`  r›  r  rv  r¬  r«  r®  rŒ   s                   r#   rl   z$MraForSequenceClassification.forwardð  s  € ð& &1Ð%<�k�kÀ$Ä+ÔBYˆà—(’(ØØ)Ø)Ø%Ø'Ø!5Ø#ð ñ 
ô 
ˆð " !œ*ˆØ—’ Ñ1Ô1ˆàˆØÑØŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ñ @Ô @À&Ç+Â+ÈbÁ/Ä/ÑRÔR��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�Øð 	FØ�Y ¨¨¨¤Ñ,ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r%   r¯  )rz   r{   r|   rÛ   r   rE   r'  r”  r^  r   rl   rû   rü   s   @r#   r»  r»  à  s  ø€ € € € € ðð ð ð ð ð ð *.Ø.2Ø.2Ø,0Ø-1Ø&*Ø,0Ø#'ð@
ð @
à”< $Ñ&ð@
ð œ tÑ+ð@
ð œ tÑ+ð	@
ð
 ”l TÑ)ð@
ð ”| dÑ*ð@
ð ”˜tÑ#ð@
ð # T™kð@
ð ˜D‘[ð@
ð 
Ð)Ñ	)ð@
ð @
ð @
ñ „^ð@
ð @
ð @
ð @
ð @
r%   r»  c                   óÞ   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	edz  d
edz  dee	z  fd„¦   «         Z
ˆ xZS )ÚMraForMultipleChoicec                 ó   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        |j        ¦  «        | _        t	          j        |j        d¦  «        | _        |  	                    ¦   «          d S rA  )
rÚ   rÛ   r‡  r{  r   r  rÞ   Úpre_classifierr½  rŠ  rì   s     €r#   rÛ   zMraForMultipleChoice.__init__6  sr   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å˜FÑ#Ô#ˆŒÝ œi¨Ô(:¸FÔ<NÑOÔOˆÔÝœ) FÔ$6¸Ñ:Ô:ˆŒð 	�ŠÑÔÐÐÐr%   Nrò   r  rØ   rÖ   ró   r§  r_  r`  r!  c	           	      ó  — |�|n| j         j        }|�|j        d         n|j        d         }
|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�=|                     d|                     d¦  «        |                     d¦  «        ¦  «        nd}|                      |||||||¬¦  «        }|d         }|dd…df         }|                      |¦  «        } t          j        ¦   «         |¦  «        }|  	                    |¦  «        }|                     d|
¦  «        }d}|�t          ¦   «         } |||¦  «        }|s|f|dd…         z   }|�|f|z   n|S t          |||j        |j        ¬¦  «        S )a[  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   r.   r+   r©  r   rª  )rî   r`  rD   r  r0   r{  rÆ  r   ÚReLUr½  r   r   r  r˜  )rí   rò   r  rØ   rÖ   ró   r§  r_  r`  r›  Únum_choicesr  Úhidden_stateÚpooled_outputr¬  Úreshaped_logitsr«  r®  rŒ   s                      r#   rl   zMraForMultipleChoice.forward@  sN  € ðV &1Ð%<�k�kÀ$Ä+ÔBYˆØ,5Ð,A�i”o aÔ(Ð(À}ÔGZÐ[\ÔG]ˆà>GÐ>S�I—N’N 2 y§~¢~°bÑ'9Ô'9Ñ:Ô:Ð:ÐY]ˆ	ØM[ÐMg˜×,Ò,¨R°×1DÒ1DÀRÑ1HÔ1HÑIÔIÐIÐmqˆØM[ÐMg˜×,Ò,¨R°×1DÒ1DÀRÑ1HÔ1HÑIÔIÐIÐmqˆØGSÐG_�|×(Ò(¨¨\×->Ò->¸rÑ-BÔ-BÑCÔCÐCÐeiˆð Ð(ð ×Ò˜r =×#5Ò#5°bÑ#9Ô#9¸=×;MÒ;MÈbÑ;QÔ;QÑRÔRÐRàð 	ð —(’(ØØ)Ø)Ø%Ø'Ø!5Ø#ð ñ 
ô 
ˆð ˜q”zˆØ$ Q Q Q¨ TÔ*ˆØ×+Ò+¨MÑ:Ô:ˆØ!�œ™	œ	 -Ñ0Ô0ˆØ—’ Ñ/Ô/ˆà Ÿ+š+ b¨+Ñ6Ô6ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜O¨VÑ4Ô4ˆDàð 	FØ%Ð'¨'°!°"°"¬+Ñ5ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå(ØØ"Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r%   r¯  )rz   r{   r|   rÛ   r   rE   r'  r”  r^  r   rl   rû   rü   s   @r#   rÄ  rÄ  4  s  ø€ € € € € ðð ð ð ð ð ð *.Ø.2Ø.2Ø,0Ø-1Ø&*Ø,0Ø#'ðW
ð W
à”< $Ñ&ðW
ð œ tÑ+ðW
ð œ tÑ+ð	W
ð
 ”l TÑ)ðW
ð ”| dÑ*ðW
ð ”˜tÑ#ðW
ð # T™kðW
ð ˜D‘[ðW
ð 
Ð*Ñ	*ðW
ð W
ð W
ñ „^ðW
ð W
ð W
ð W
ð W
r%   rÄ  c                   óÞ   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	edz  d
edz  dee	z  fd„¦   «         Z
ˆ xZS )ÚMraForTokenClassificationc                 ó6  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        ¦  «        | _        t          j	        |j
        |j        ¦  «        | _        |                      ¦   «          d S ri   )rÚ   rÛ   r´  r‡  r{  r   rç   rè   ré   r  rÞ   r½  rŠ  rì   s     €r#   rÛ   z"MraForTokenClassification.__init__�  sy   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå˜FÑ#Ô#ˆŒÝ”z &Ô"<Ñ=Ô=ˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐr%   Nrò   r  rØ   rÖ   ró   r§  r_  r`  r!  c	           	      óì  — |�|n| j         j        }|                      |||||||¬¦  «        }
|
d         }|                      |¦  «        }|                      |¦  «        }d}|�Üt          ¦   «         }|�”|                     d¦  «        dk    }|                     d| j        ¦  «        }t          j	        ||                     d¦  «        t          j
        |j        ¦  «                             |¦  «        ¦  «        } |||¦  «        }n8 ||                     d| j        ¦  «        |                     d¦  «        ¦  «        }|s|f|
dd…         z   }|�|f|z   n|S t          |||
j        |
j        ¬¦  «        S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Nr©  r   r.   r   rª  )rî   r`  r{  ré   r½  r   r  r´  rE   Úwherer“  Úignore_indexÚtype_asr   r  r˜  )rí   rò   r  rØ   rÖ   ró   r§  r_  r`  r›  r  rv  r¬  r«  r®  Úactive_lossÚactive_logitsÚactive_labelsrŒ   s                      r#   rl   z!MraForTokenClassification.forward¨  s•  € ð" &1Ð%<�k�kÀ$Ä+ÔBYˆà—(’(ØØ)Ø)Ø%Ø'Ø!5Ø#ð ñ 
ô 
ˆð " !œ*ˆàŸ,š, Ñ7Ô7ˆØ—’ Ñ1Ô1ˆàˆØÐÝ'Ñ)Ô)ˆHàÐ)Ø,×1Ò1°"Ñ5Ô5¸Ò:�Ø &§¢¨B°´Ñ @Ô @�Ý %¤Ø §¢¨R¡¤µ%´,¸xÔ?TÑ2UÔ2U×2]Ò2]Ð^dÑ2eÔ2eñ!ô !�ð  �x ¨}Ñ=Ô=��à�x §¢¨B°´Ñ @Ô @À&Ç+Â+ÈbÁ/Ä/ÑRÔR�àð 	FØ�Y ¨¨¨¤Ñ,ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r%   r¯  )rz   r{   r|   rÛ   r   rE   r'  r”  r^  r   rl   rû   rü   s   @r#   rÎ  rÎ  ›  s  ø€ € € € € ð	ð 	ð 	ð 	ð 	ð ð *.Ø.2Ø.2Ø,0Ø-1Ø&*Ø,0Ø#'ð8
ð 8
à”< $Ñ&ð8
ð œ tÑ+ð8
ð œ tÑ+ð	8
ð
 ”l TÑ)ð8
ð ”| dÑ*ð8
ð ”˜tÑ#ð8
ð # T™kð8
ð ˜D‘[ð8
ð 
Ð&Ñ	&ð8
ð 8
ð 8
ñ „^ð8
ð 8
ð 8
ð 8
ð 8
r%   rÎ  c                   óô   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
edz  dedz  dee	z  fd„¦   «         Z
ˆ xZS )ÚMraForQuestionAnsweringc                 ó  •— t          ¦   «                              |¦  «         d|_        |j        | _        t          |¦  «        | _        t          j        |j        |j        ¦  «        | _        |  	                    ¦   «          d S )Nr(   )
rÚ   rÛ   r´  r‡  r{  r   r  rÞ   Ú
qa_outputsrŠ  rì   s     €r#   rÛ   z MraForQuestionAnswering.__init__æ  sm   ø€ Ý‰Œ×Ò˜Ñ Ô Ð àˆÔØ Ô+ˆŒå˜FÑ#Ô#ˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐr%   Nrò   r  rØ   rÖ   ró   Ústart_positionsÚend_positionsr_  r`  r!  c
           	      ód  — |	�|	n| j         j        }	|                      |||||||	¬¦  «        }|d         }|                      |¦  «        }|                     dd¬¦  «        \  }}|                     d¦  «        }|                     d¦  «        }d }|�ç|�åt          |                     ¦   «         ¦  «        dk    r|                     d¦  «        }t          |                     ¦   «         ¦  «        dk    r|                     d¦  «        }|                     d¦  «        }|                     d|¦  «        }|                     d|¦  «        }t          |¬¦  «        } |||¦  «        } |||¦  «        }||z   dz  }|	s||f|dd …         z   }|�|f|z   n|S t          ||||j        |j        ¬¦  «        S )	Nr©  r   r   r.   r,   )rÒ  r(   )r«  Ústart_logitsÚ
end_logitsr  r˜  )rî   r`  r{  rÚ  Úsplitr  r/   r0   Úclampr   r   r  r˜  )rí   rò   r  rØ   rÖ   ró   rÛ  rÜ  r_  r`  r›  r  rv  r¬  rÞ  rß  Ú
total_lossÚignored_indexr®  Ú
start_lossÚend_lossrŒ   s                         r#   rl   zMraForQuestionAnswering.forwardò  sõ  € ð &1Ð%<�k�kÀ$Ä+ÔBYˆà—(’(ØØ)Ø)Ø%Ø'Ø!5Ø#ð ñ 
ô 
ˆð " !œ*ˆà—’ Ñ1Ô1ˆØ#)§<¢<°°r <Ñ#:Ô#:Ñ ˆ�jØ#×+Ò+¨BÑ/Ô/ˆØ×'Ò'¨Ñ+Ô+ˆ
àˆ
ØÐ&¨=Ð+Då�?×'Ò'Ñ)Ô)Ñ*Ô*¨QÒ.Ð.Ø"1×"9Ò"9¸"Ñ"=Ô"=�Ý�=×%Ò%Ñ'Ô'Ñ(Ô(¨1Ò,Ð,Ø -× 5Ò 5°bÑ 9Ô 9�à(×-Ò-¨aÑ0Ô0ˆMØ-×3Ò3°A°}ÑEÔEˆOØ)×/Ò/°°=ÑAÔAˆMå'°]ÐCÑCÔCˆHØ!˜ ,°Ñ@Ô@ˆJØ�x 
¨MÑ:Ô:ˆHØ$ xÑ/°1Ñ4ˆJàð 	RØ" JÐ/°'¸!¸"¸"´+Ñ=ˆFØ/9Ð/E�Z�M FÑ*Ð*È6ÐQå+ØØ%Ø!Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r%   )	NNNNNNNNN)rz   r{   r|   rÛ   r   rE   r'  r”  r^  r   rl   rû   rü   s   @r#   rØ  rØ  ä  s  ø€ € € € € ð
ð 
ð 
ð 
ð 
ð ð *.Ø.2Ø.2Ø,0Ø-1Ø/3Ø-1Ø,0Ø#'ð;
ð ;
à”< $Ñ&ð;
ð œ tÑ+ð;
ð œ tÑ+ð	;
ð
 ”l TÑ)ð;
ð ”| dÑ*ð;
ð œ¨Ñ,ð;
ð ”| dÑ*ð;
ð # T™kð;
ð ˜D‘[ð;
ð 
Ð-Ñ	-ð;
ð ;
ð ;
ñ „^ð;
ð ;
ð ;
ð ;
ð ;
r%   rØ  )rŸ  rÄ  rØ  r»  rÎ  r?  r‡  rz  rx   )NN)r*   r   r   )Lrú   r•   rE   r   Útorch.nnr   r   r   Ú r   r  Úactivationsr	   Úmasking_utilsr
   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   Úpytorch_utilsr   Úutilsr   r   r   r   r   r   Úconfiguration_mrar   Ú
get_loggerrz   r  r"   r$   r?   rO   rS   r[   re   ÚautogradÚFunctionrg   r€   r…   r¢   r¹   rÏ   r…  rÑ   rþ   r  r*  r1  r;  r?  rN  rf  rk  rs  rz  r‡  rŸ  r²  r»  rÄ  rÎ  rØ  Ú__all__r~   r%   r#   ú<module>rô     sà  ðð Ð à €€€à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð .Ð -Ð -Ð -Ð -Ð -Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð )Ð (Ð (Ð (Ð (Ð (ð 
ˆÔ	˜HÑ	%Ô	%€à€ðEð Eð Eð&ð &ð &ð8ð ð ð ð.%Oð %Oð %Oð %OðP%ð %ð %ð %ðPsð sð sðXð Xð Xð Xð X˜EœNÔ3ñ Xô Xð Xð0]ð ]ð ]ð ]ð ]˜5œ>Ô2ñ ]ô ]ð ]ð.ð ð ð ð ñ ô ð ð:%Vð %Vð %Vð %VðP#)ð #)ð #)ðZ Ø!"Ø$%ðpð pð pð pðf4ð 4ð 4ð 4ð 4�B”Iñ 4ô 4ð 4ðn]ð ]ð ]ð ]ð ]�r”yñ ]ô ]ð ]ðBð ð ð ð �B”Iñ ô ð ð
ð 
ð 
ð 
ð 
�2”9ñ 
ô 
ð 
ðð ð ð ð �b”iñ ô ð ð ð ð ð ð �”	ñ ô ð ðð ð ð ð Ð)ñ ô ð ð: 
ð  
ð  
ð  
ð  
�”ñ  
ô  
ð  
ðHð ð ð ð  ¤ñ ô ð ð$ð ð ð ð ˜"œ)ñ ô ð ð"!ð !ð !ð !ð !�R”Yñ !ô !ð !ð ð/ð /ð /ð /ð /˜ñ /ô /ñ „ð/ð  ðY
ð Y
ð Y
ð Y
ð Y
Ð!ñ Y
ô Y
ñ „ðY
ðx ðF
ð F
ð F
ð F
ð F
Ð'ñ F
ô F
ñ „ðF
ðTð ð ð ð ˜BœIñ ô ð ð* €ððñ ô ðK
ð K
ð K
ð K
ð K
Ð#5ñ K
ô K
ñô ðK
ð\ ðc
ð c
ð c
ð c
ð c
Ð-ñ c
ô c
ñ „ðc
ðL ðE
ð E
ð E
ð E
ð E
Ð 2ñ E
ô E
ñ „ðE
ðP ðI
ð I
ð I
ð I
ð I
Ð0ñ I
ô I
ñ „ðI
ðX	ð 	ð 	€€€r%   