§
    ‚Štjòº  ã                   ój  — d Z ddlZddlZddlmZ ddlmZmZmZ ddlm	Z
 ddlmZ ddlmZmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZmZmZmZmZmZ ddl m!Z! ddl"m#Z# ddl$m%Z%m&Z& ddl'm(Z(  e&j)        e*¦  «        Z+ G d„ dej,        ¦  «        Z- G d„ dej,        ¦  «        Z. G d„ dej,        ¦  «        Z/ G d„ dej,        ¦  «        Z0 G d„ dej,        ¦  «        Z1 G d„ dej,        ¦  «        Z2 G d„ dej,        ¦  «        Z3 G d „ d!e¦  «        Z4 G d"„ d#ej,        ¦  «        Z5 G d$„ d%ej,        ¦  «        Z6 G d&„ d'ej,        ¦  «        Z7 G d(„ d)ej,        ¦  «        Z8e% G d*„ d+e!¦  «        ¦   «         Z9 e%d,¬-¦  «         G d.„ d/e9¦  «        ¦   «         Z:e% G d0„ d1e9¦  «        ¦   «         Z; e%d2¬-¦  «         G d3„ d4e9e¦  «        ¦   «         Z< e%d5¬-¦  «         G d6„ d7e9¦  «        ¦   «         Z=e% G d8„ d9e9¦  «        ¦   «         Z>e% G d:„ d;e9¦  «        ¦   «         Z?e% G d<„ d=e9¦  «        ¦   «         Z@g d>¢ZAdS )?zPyTorch RemBERT model.é    N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)Úapply_chunking_to_forward)Úauto_docstringÚloggingé   )ÚRemBertConfigc                   ó˜   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  d	ed
ej	        fd„Z
ˆ xZS )ÚRemBertEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 ó8  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        ¬¦  «        | _        t          j        |j        |j        ¦  «        | _	        t          j        |j
        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j        ¦  «        | _        |                      dt%          j        |j        ¦  «                             d¦  «        d¬¦  «         d S )N)Úpadding_idx©ÚepsÚposition_ids©r   éÿÿÿÿF)Ú
persistent)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚinput_embedding_sizeÚpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferÚtorchÚarangeÚexpand©ÚselfÚconfigÚ	__class__s     €új/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/rembert/modeling_rembert.pyr)   zRemBertEmbeddings.__init__2   sö   ø€ Ý‰Œ×ÒÑÔÐÝ!œ|ØÔ˜vÔ:ÈÔH[ð 
ñ  
ô  
ˆÔõ $&¤<°Ô0NÐPVÔPkÑ#lÔ#lˆÔ Ý%'¤\°&Ô2HÈ&ÔJeÑ%fÔ%fˆÔ"åœ fÔ&AÀvÔG\Ð]Ñ]Ô]ˆŒÝ”z &Ô"<Ñ=Ô=ˆŒð 	×ÒØ�EœL¨Ô)GÑHÔH×OÒOÐPWÑXÔXÐejð 	ñ 	
ô 	
ð 	
ð 	
ð 	
ó    Nr   Ú	input_idsÚtoken_type_idsr$   Úinputs_embedsÚpast_key_values_lengthÚreturnc                 óð  — |�|                      ¦   «         }n|                      ¦   «         d d…         }|d         }|€| j        d d …|||z   …f         }|€+t          j        |t          j        | j        j        ¬¦  «        }|€|                      |¦  «        }|                      |¦  «        }||z   }	|                      |¦  «        }
|	|
z  }	|  	                    |	¦  «        }	|  
                    |	¦  «        }	|	S )Nr&   r   ©ÚdtypeÚdevice)Úsizer$   r9   ÚzerosÚlongrJ   r.   r2   r0   r3   r7   )r=   rB   rC   r$   rD   rE   Úinput_shapeÚ
seq_lengthr2   Ú
embeddingsr0   s              r@   ÚforwardzRemBertEmbeddings.forwardB   s	  € ð Ð Ø#Ÿ.š.Ñ*Ô*ˆKˆKà'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKà  ”^ˆ
àÐØÔ,¨Q¨Q¨QÐ0FÈÐVlÑIlÐ0lÐ-lÔmˆLàÐ!Ý"œ[¨½E¼JÈtÔO`ÔOgÐhÑhÔhˆNàÐ Ø ×0Ò0°Ñ;Ô;ˆMØ $× :Ò :¸>Ñ JÔ JÐà"Ð%:Ñ:ˆ
Ø"×6Ò6°|ÑDÔDÐØÐ)Ñ)ˆ
Ø—^’^ JÑ/Ô/ˆ
Ø—\’\ *Ñ-Ô-ˆ
ØÐrA   )NNNNr   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r)   r9   Ú
LongTensorÚFloatTensorÚintÚTensorrQ   Ú__classcell__©r?   s   @r@   r   r   /   sÄ   ø€ € € € € ØQÐQð
ð 
ð 
ð 
ð 
ð$ .2Ø26Ø04Ø26Ø&'ðð àÔ# dÑ*ðð Ô(¨4Ñ/ðð Ô&¨Ñ-ð	ð
 Ô(¨4Ñ/ðð !$ðð 
Œðð ð ð ð ð ð ð rA   r   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚRemBertPoolerc                 óÀ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        ¦   «         | _        d S ©N)r(   r)   r   ÚLinearÚhidden_sizeÚdenseÚTanhÚ
activationr<   s     €r@   r)   zRemBertPooler.__init__e   sC   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ'™)œ)ˆŒˆˆrA   Úhidden_statesrF   c                 ór   — |d d …df         }|                       |¦  «        }|                      |¦  «        }|S )Nr   )rb   rd   )r=   re   Úfirst_token_tensorÚpooled_outputs       r@   rQ   zRemBertPooler.forwardj   s@   € ð +¨1¨1¨1¨a¨4Ô0ÐØŸ
š
Ð#5Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐrA   ©rR   rS   rT   r)   r9   rY   rQ   rZ   r[   s   @r@   r]   r]   d   s^   ø€ € € € € ð$ð $ð $ð $ð $ð
 U¤\ð °e´lð ð ð ð ð ð ð ð rA   r]   c                   óz   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej        dej        dz  dej        dz  dedz  ded	e	fd
„Z
ˆ xZS )ÚRemBertSelfAttentionNc                 óŠ  •— t          ¦   «                              ¦   «          |j        |j        z  dk    r0t	          |d¦  «        s t          d|j        › d|j        › d�¦  «        ‚|j        | _        t          |j        |j        z  ¦  «        | _        | j        | j        z  | _        t          j
        |j        | j        ¦  «        | _        t          j
        |j        | j        ¦  «        | _        t          j
        |j        | j        ¦  «        | _        t          j        |j        ¦  «        | _        |j        | _        || _        d S )Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú))r(   r)   ra   Únum_attention_headsÚhasattrÚ
ValueErrorrX   Úattention_head_sizeÚall_head_sizer   r`   ÚqueryÚkeyÚvaluer5   Úattention_probs_dropout_probr7   Ú
is_decoderÚ	layer_idx©r=   r>   ry   r?   s      €r@   r)   zRemBertSelfAttention.__init__t   s*  ø€ Ý‰Œ×ÒÑÔÐØÔ Ô :Ñ:¸aÒ?Ð?ÍÐPVÐXhÑHiÔHiÐ?Ýð8 FÔ$6ð 8ð 8Ø Ô4ð8ð 8ð 8ñô ð ð
 $*Ô#=ˆÔ Ý#& vÔ'9¸FÔ<VÑ'VÑ#WÔ#WˆÔ Ø!Ô5¸Ô8PÑPˆÔå”Y˜vÔ1°4Ô3EÑFÔFˆŒ
Ý”9˜VÔ/°Ô1CÑDÔDˆŒÝ”Y˜vÔ1°4Ô3EÑFÔFˆŒ
å”z &Ô"EÑFÔFˆŒà Ô+ˆŒØ"ˆŒˆˆrA   Fre   Úattention_maskÚencoder_hidden_statesÚpast_key_valuesÚoutput_attentionsrF   c                 óf  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }	d}
|d u}|�Ht          |t          ¦  «        r1|j                             | j	        ¦  «        }
|r|j
        }n
|j        }n|}|r|n|}|r3|�1|
r/|j        | j	                 j        }|j        | j	                 j        }nÚg |j         d d…         ¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|�E|                     ||| j	        ¦  «        \  }}|r$t          |t          ¦  «        rd|j        | j	        <   t%          j        |	|                     dd¦  «        ¦  «        }|t)          j        | j        ¦  «        z  }|�||z   }t,          j                             |d¬¦  «        }|                      |¦  «        }t%          j        ||¦  «        }|                     dddd	¦  «                             ¦   «         }|                     ¦   «         d d…         | j        fz   } |j        |Ž }||fS )
Nr&   r   é   FTéþÿÿÿ©Údimr   r   )Úshaperr   rt   ÚviewÚ	transposeÚ
isinstancer   Ú
is_updatedÚgetry   Úcross_attention_cacheÚself_attention_cacheÚlayersÚkeysÚvaluesru   rv   Úupdater9   ÚmatmulÚmathÚsqrtr   Ú
functionalÚsoftmaxr7   ÚpermuteÚ
contiguousrK   rs   )r=   re   r{   r|   r}   r~   ÚkwargsrN   Úhidden_shapeÚquery_layerrˆ   Úis_cross_attentionÚcurr_past_key_valuesÚcurrent_statesÚ	key_layerÚvalue_layerÚkv_shapeÚattention_scoresÚattention_probsÚcontext_layerÚnew_context_layer_shapes                        r@   rQ   zRemBertSelfAttention.forward‰   sÛ  € ð $Ô)¨#¨2¨#Ô.ˆØC˜ÐC bÐC¨$Ô*BÐCÐCˆØ—j’j Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆàˆ
Ø2¸$Ð>ÐØÐ&Ý˜/Õ+>Ñ?Ô?ð 7Ø,Ô7×;Ò;¸D¼NÑKÔK�
Ø%ð Pà+:Ô+PÐ(Ð(à+:Ô+OÐ(Ð(à'6Ð$à2DÐWÐ.Ð.È-ˆØð 	F /Ð"=À*Ð"=à,Ô3°D´NÔCÔHˆIØ.Ô5°d´nÔEÔLˆKˆKàQ˜Ô-¨c¨r¨cÔ2ÐQ°BÐQ¸Ô8PÐQÐQˆHØŸš Ñ0Ô0×5Ò5°hÑ?Ô?×IÒIÈ!ÈQÑOÔOˆIØŸ*š* ^Ñ4Ô4×9Ò9¸(ÑCÔC×MÒMÈaÐQRÑSÔSˆKàÐ*à)=×)DÒ)DÀYÐP[Ð]aÔ]kÑ)lÔ)lÑ&�	˜;à%ð F­*°_ÕFYÑ*ZÔ*Zð FØAE�OÔ.¨t¬~Ñ>õ !œ<¨°Y×5HÒ5HÈÈRÑ5PÔ5PÑQÔQÐà+­d¬i¸Ô8PÑ.QÔ.QÑQÐØÐ%à/°.Ñ@Ðõ œ-×/Ò/Ð0@ÀbÐ/ÑIÔIˆð Ÿ,š, Ñ7Ô7ˆåœ _°kÑBÔBˆà%×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔFˆØ"/×"4Ò"4Ñ"6Ô"6°s¸°sÔ";¸tÔ?QÐ>SÑ"SÐØ*˜Ô*Ð,CÐDˆà˜oÐ-Ð-rA   r_   ©NNNF©rR   rS   rT   r)   r9   rY   rW   r
   ÚboolÚtuplerQ   rZ   r[   s   @r@   rk   rk   s   sÃ   ø€ € € € € ð#ð #ð #ð #ð #ð #ð0 48Ø:>Ø(,Ø"'ð@.ð @.à”|ð@.ð Ô)¨DÑ0ð@.ð  %Ô0°4Ñ7ð	@.ð
  ™ð@.ð  ð@.ð 
ð@.ð @.ð @.ð @.ð @.ð @.ð @.ð @.rA   rk   c                   óP   ‡ — e Zd Zˆ fd„Zdej        dej        dej        fd„Zˆ xZS )ÚRemBertSelfOutputc                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j	        ¦  «        | _
        d S ©Nr"   )r(   r)   r   r`   ra   rb   r3   r4   r5   r6   r7   r<   s     €r@   r)   zRemBertSelfOutput.__init__Î   sf   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆrA   re   Úinput_tensorrF   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|S r_   ©rb   r7   r3   ©r=   re   r¬   s      r@   rQ   zRemBertSelfOutput.forwardÔ   ó@   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØŸš }°|Ñ'CÑDÔDˆØÐrA   ri   r[   s   @r@   r©   r©   Í   ói   ø€ € € € € ð>ð >ð >ð >ð >ð U¤\ð ÀÄð ÐRWÔR^ð ð ð ð ð ð ð ð rA   r©   c                   ó–   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej        dej        dz  dej        dz  dedz  dedz  d	e	ej                 fd
„Z
ˆ xZS )ÚRemBertAttentionNc                 óœ   •— t          ¦   «                              ¦   «          t          ||¬¦  «        | _        t	          |¦  «        | _        d S )N©ry   )r(   r)   rk   r=   r©   Úoutputrz   s      €r@   r)   zRemBertAttention.__init__Ü   s@   ø€ Ý‰Œ×ÒÑÔÐÝ(¨¸9ÐEÑEÔEˆŒ	Ý'¨Ñ/Ô/ˆŒˆˆrA   Fre   r{   r|   r}   r~   rF   c                 óŽ   — |                       |||||¬¦  «        }|                      |d         |¦  «        }|f|dd …         z   }	|	S )N©r{   r|   r}   r~   r   r   )r=   r¶   )
r=   re   r{   r|   r}   r~   r—   Úself_outputsÚattention_outputÚoutputss
             r@   rQ   zRemBertAttention.forwardá   s`   € ð —y’yØØ)Ø"7Ø+Ø/ð !ñ 
ô 
ˆð  Ÿ;š; |°A¤¸ÑFÔFÐØ#Ð%¨°Q°R°RÔ(8Ñ8ˆØˆrA   r_   r¤   r¥   r[   s   @r@   r³   r³   Û   s¿   ø€ € € € € ð0ð 0ð 0ð 0ð 0ð 0ð 48Ø:>Ø(,Ø).ðð à”|ðð Ô)¨DÑ0ðð  %Ô0°4Ñ7ð	ð
  ™ðð   $™;ðð 
ˆuŒ|Ô	ðð ð ð ð ð ð ð rA   r³   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚRemBertIntermediatec                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j        t          ¦  «        rt          |j                 | _        d S |j        | _        d S r_   )r(   r)   r   r`   ra   Úintermediate_sizerb   r‡   Ú
hidden_actÚstrr	   Úintermediate_act_fnr<   s     €r@   r)   zRemBertIntermediate.__init__ø   sn   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3KÑLÔLˆŒ
Ý�fÔ'­Ñ-Ô-ð 	9Ý'-¨fÔ.?Ô'@ˆDÔ$Ð$Ð$à'-Ô'8ˆDÔ$Ð$Ð$rA   re   rF   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r_   )rb   rÂ   ©r=   re   s     r@   rQ   zRemBertIntermediate.forward   s,   € ØŸ
š
 =Ñ1Ô1ˆØ×0Ò0°Ñ?Ô?ˆØÐrA   ri   r[   s   @r@   r½   r½   ÷   s^   ø€ € € € € ð9ð 9ð 9ð 9ð 9ð U¤\ð °e´lð ð ð ð ð ð ð ð rA   r½   c                   óP   ‡ — e Zd Zˆ fd„Zdej        dej        dej        fd„Zˆ xZS )ÚRemBertOutputc                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S r«   )r(   r)   r   r`   r¿   ra   rb   r3   r4   r5   r6   r7   r<   s     €r@   r)   zRemBertOutput.__init__  sf   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ7¸Ô9KÑLÔLˆŒ
Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆrA   re   r¬   rF   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|S r_   r®   r¯   s      r@   rQ   zRemBertOutput.forward  r°   rA   ri   r[   s   @r@   rÆ   rÆ     r±   rA   rÆ   c                   ó²   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 	 ddej        dej        dz  dej        dz  dej        dz  dedz  d	edz  d
e	ej                 fd„Z
d„ Zˆ xZS )ÚRemBertLayerNc                 ó„  •— t          ¦   «                              ¦   «          |j        | _        d| _        t	          ||¦  «        | _        |j        | _        |j        | _        | j        r/| j        st          | › d�¦  «        ‚t	          ||¬¦  «        | _	        t          |¦  «        | _        t          |¦  «        | _        d S )Nr   z> should be used as a decoder model if cross attention is addedrµ   )r(   r)   Úchunk_size_feed_forwardÚseq_len_dimr³   Ú	attentionrx   Úadd_cross_attentionrq   Úcrossattentionr½   ÚintermediaterÆ   r¶   rz   s      €r@   r)   zRemBertLayer.__init__  s¸   ø€ Ý‰Œ×ÒÑÔÐØ'-Ô'EˆÔ$ØˆÔÝ)¨&°)Ñ<Ô<ˆŒØ Ô+ˆŒØ#)Ô#=ˆÔ ØÔ#ð 	PØ”?ð jÝ  DÐ!hÐ!hÐ!hÑiÔiÐiÝ"2°6ÀYÐ"OÑ"OÔ"OˆDÔÝ/°Ñ7Ô7ˆÔÝ# FÑ+Ô+ˆŒˆˆrA   Fre   r{   r|   Úencoder_attention_maskr}   r~   rF   c                 ó`  — |                       ||||¬¦  «        }|d         }	|dd …         }
| j        rT|�Rt          | d¦  «        st          d| › d�¦  «        ‚|                      |	||||¬¦  «        }|d         }	|
|dd …         z   }
t          | j        | j        | j        |	¦  «        }|f|
z   }
|
S )N)r{   r~   r}   r   r   rÐ   z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r¸   )	rÎ   rx   rp   rq   rÐ   r   Úfeed_forward_chunkrÌ   rÍ   )r=   re   r{   r|   rÒ   r}   r~   r—   Úself_attention_outputsrº   r»   Úcross_attention_outputsÚlayer_outputs                r@   rQ   zRemBertLayer.forward%  s  € ð "&§¢ØØ)Ø/Ø+ð	 "0ñ "
ô "
Ðð 2°!Ô4ÐØ(¨¨¨Ô,ˆàŒ?ð 	<Ð4Ð@Ý˜4Ð!1Ñ2Ô2ð Ý ðD¸dð Dð Dð Dñô ð ð
 '+×&9Ò&9Ø Ø5Ø&;Ø /Ø"3ð ':ñ 'ô 'Ð#ð  7°qÔ9ÐØÐ 7¸¸¸Ô ;Ñ;ˆGå0ØÔ# TÔ%AÀ4ÔCSÐUeñ
ô 
ˆð  �/ GÑ+ˆàˆrA   c                 ó\   — |                       |¦  «        }|                      ||¦  «        }|S r_   )rÑ   r¶   )r=   rº   Úintermediate_outputr×   s       r@   rÔ   zRemBertLayer.feed_forward_chunkQ  s2   € Ø"×/Ò/Ð0@ÑAÔAÐØ—{’{Ð#6Ð8HÑIÔIˆØÐrA   r_   )NNNNF)rR   rS   rT   r)   r9   rY   rW   r
   r¦   r§   rQ   rÔ   rZ   r[   s   @r@   rÊ   rÊ     sä   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð ,ð$ 48Ø:>Ø;?Ø(,Ø).ð)ð )à”|ð)ð Ô)¨DÑ0ð)ð  %Ô0°4Ñ7ð	)ð
 !&Ô 1°DÑ 8ð)ð  ™ð)ð   $™;ð)ð 
ˆuŒ|Ô	ð)ð )ð )ð )ðXð ð ð ð ð ð rA   rÊ   c                   ó¬   ‡ — e Zd Zˆ fd„Z	 	 	 	 	 	 	 	 ddej        dej        dz  dej        dz  dej        dz  d	edz  d
edz  dededede	e
z  fd„Zˆ xZS )ÚRemBertEncoderc                 ó  •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ‰j        ‰j        ¦  «        | _        t          j        ˆfd„t          ‰j
        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nc                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS )rµ   )rÊ   )Ú.0Úir>   s     €r@   ú
<listcomp>z+RemBertEncoder.__init__.<locals>.<listcomp>]  s&   ø€ Ð#oÐ#oÐ#oÈ!¥L°À1Ð$EÑ$EÔ$EÐ#oÐ#oÐ#orA   F)r(   r)   r>   r   r`   r,   ra   Úembedding_hidden_mapping_inÚ
ModuleListÚrangeÚnum_hidden_layersÚlayerÚgradient_checkpointingr<   s    `€r@   r)   zRemBertEncoder.__init__X  s|   øø€ Ý‰Œ×ÒÑÔÐØˆŒå+-¬9°VÔ5PÐRXÔRdÑ+eÔ+eˆÔ(Ý”]Ð#oÐ#oÐ#oÐ#oÍuÐU[ÔUmÑOnÔOnÐ#oÑ#oÔ#oÑpÔpˆŒ
Ø&+ˆÔ#Ð#Ð#rA   NFTre   r{   r|   rÒ   r}   Ú	use_cacher~   Úoutput_hidden_statesÚreturn_dictrF   c
           	      ól  — | j         r%| j        r|rt                               d¦  «         d}|r8|€6t	          t          | j        ¬¦  «        t          | j        ¬¦  «        ¦  «        }|                      |¦  «        }|rdnd }|rdnd }|r| j        j        rdnd }t          | j
        ¦  «        D ]K\  }}|r||fz   } |||||||¦  «        }|d         }|r$||d         fz   }| j        j        r||d         fz   }ŒL|r||fz   }|	st          d„ |||||fD ¦   «         ¦  «        S t          |||||¬	¦  «        S )
NzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...F)r>   © r   r   r€   c              3   ó   K  — | ]}|®|V — Œ	d S r_   rë   )rÞ   Úvs     r@   ú	<genexpr>z)RemBertEncoder.forward.<locals>.<genexpr>“  s4   è è € ð 
ð 
àð �=ð ð !�=�=�=ð
ð 
rA   )Úlast_hidden_stater}   re   Ú
attentionsÚcross_attentions)ræ   ÚtrainingÚloggerÚwarning_oncer   r   r>   rá   rÏ   Ú	enumeraterå   r§   r   )r=   re   r{   r|   rÒ   r}   rç   r~   rè   ré   r—   Úall_hidden_statesÚall_self_attentionsÚall_cross_attentionsrß   Úlayer_moduleÚlayer_outputss                    r@   rQ   zRemBertEncoder.forward`  só  € ð Ô&ð 	"¨4¬=ð 	"Øð "Ý×#Ò#Øpñô ð ð "�	àð 	v˜Ð0Ý1µ,ÀdÄkÐ2RÑ2RÔ2RÕT`ÐhlÔhsÐTtÑTtÔTtÑuÔuˆOà×8Ò8¸ÑGÔGˆØ"6Ð@˜B˜B¸DÐØ$5Ð?˜b˜b¸4ÐØ%6Ðd¸4¼;Ô;ZÐd˜r˜rÐ`dÐå(¨¬Ñ4Ô4ð 	Vð 	V‰OˆAˆ|Ø#ð IØ$5¸Ð8HÑ$HÐ!à(˜LØØØ%Ø&ØØ!ñô ˆMð *¨!Ô,ˆMØ ð VØ&9¸]È1Ô=MÐ<OÑ&OÐ#Ø”;Ô2ð VØ+?À=ÐQRÔCSÐBUÑ+UÐ(øàð 	EØ 1°]Ð4DÑ DÐàð 	Ýð 
ð 
ð "Ø#Ø%Ø'Ø(ðð
ñ 
ô 
ñ 
ô 
ð 
õ 9Ø+Ø+Ø+Ø*Ø1ð
ñ 
ô 
ð 	
rA   )NNNNNFFT)rR   rS   rT   r)   r9   rY   rW   r
   r¦   r§   r   rQ   rZ   r[   s   @r@   rÛ   rÛ   W  s  ø€ € € € € ð,ð ,ð ,ð ,ð ,ð 48Ø:>Ø;?Ø(,Ø!%Ø"'Ø%*Ø ðD
ð D
à”|ðD
ð Ô)¨DÑ0ðD
ð  %Ô0°4Ñ7ð	D
ð
 !&Ô 1°DÑ 8ðD
ð  ™ðD
ð ˜$‘;ðD
ð  ðD
ð #ðD
ð ðD
ð 
Ð:Ñ	:ðD
ð D
ð D
ð D
ð D
ð D
ð D
ð D
rA   rÛ   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚRemBertPredictionHeadTransformc                 óV  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j        t          ¦  «        rt          |j                 | _
        n|j        | _
        t          j        |j        |j        ¬¦  «        | _        d S r«   )r(   r)   r   r`   ra   rb   r‡   rÀ   rÁ   r	   Útransform_act_fnr3   r4   r<   s     €r@   r)   z'RemBertPredictionHeadTransform.__init__©  s…   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ý�fÔ'­Ñ-Ô-ð 	6Ý$*¨6Ô+<Ô$=ˆDÔ!Ð!à$*Ô$5ˆDÔ!Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒˆˆrA   re   rF   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r_   )rb   rþ   r3   rÄ   s     r@   rQ   z&RemBertPredictionHeadTransform.forward²  s=   € ØŸ
š
 =Ñ1Ô1ˆØ×-Ò-¨mÑ<Ô<ˆØŸš }Ñ5Ô5ˆØÐrA   ri   r[   s   @r@   rü   rü   ¨  sc   ø€ € € € € ðUð Uð Uð Uð Uð U¤\ð °e´lð ð ð ð ð ð ð ð rA   rü   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚRemBertLMPredictionHeadc                 óP  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¦  «        | _        t          |j
                 | _        t          j        |j        |j        ¬¦  «        | _        d S r«   )r(   r)   r   r`   ra   Úoutput_embedding_sizerb   r+   Údecoderr	   rÀ   rd   r3   r4   r<   s     €r@   r)   z RemBertLMPredictionHead.__init__º  sz   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3OÑPÔPˆŒ
Ý”y Ô!=¸vÔ?PÑQÔQˆŒÝ  Ô!2Ô3ˆŒÝœ fÔ&BÈÔH]Ð^Ñ^Ô^ˆŒˆˆrA   re   rF   c                 ó®   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r_   )rb   rd   r3   r  rÄ   s     r@   rQ   zRemBertLMPredictionHead.forwardÁ  sL   € ØŸ
š
 =Ñ1Ô1ˆØŸš¨Ñ6Ô6ˆØŸš }Ñ5Ô5ˆØŸš ]Ñ3Ô3ˆØÐrA   ri   r[   s   @r@   r  r  ¹  sc   ø€ € € € € ð_ð _ð _ð _ð _ð U¤\ð °e´lð ð ð ð ð ð ð ð rA   r  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚRemBertOnlyMLMHeadc                 óp   •— t          ¦   «                              ¦   «          t          |¦  «        | _        d S r_   )r(   r)   r  Úpredictionsr<   s     €r@   r)   zRemBertOnlyMLMHead.__init__Ë  s/   ø€ Ý‰Œ×ÒÑÔÐÝ2°6Ñ:Ô:ˆÔÐÐrA   Úsequence_outputrF   c                 ó0   — |                       |¦  «        }|S r_   )r	  )r=   r
  Úprediction_scoress      r@   rQ   zRemBertOnlyMLMHead.forwardÏ  s   € Ø ×,Ò,¨_Ñ=Ô=ÐØ Ð rA   ri   r[   s   @r@   r  r  Ê  s^   ø€ € € € € ð;ð ;ð ;ð ;ð ;ð! u¤|ð !¸¼ð !ð !ð !ð !ð !ð !ð !ð !rA   r  c                   ó2   ‡ — e Zd ZU eed<   dZdZˆ fd„Zˆ xZS )ÚRemBertPreTrainedModelr>   ÚrembertTc                 ó  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rQt	          j        |j        t          j        |j        j	        d         ¦  «         
                    d¦  «        ¦  «         d S d S )Nr&   r%   )r(   Ú_init_weightsr‡   r   ÚinitÚcopy_r$   r9   r:   r„   r;   )r=   Úmoduler?   s     €r@   r  z$RemBertPreTrainedModel._init_weightsÚ  s{   ø€ Ý‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ/Ñ0Ô0ð 	iÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÑhÔhÐhÐhÐhð	ið 	irA   )	rR   rS   rT   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingr  rZ   r[   s   @r@   r  r  Ô  s[   ø€ € € € € € àÐÐÑØ!ÐØ&*Ð#ðið ið ið ið ið ið ið ið irA   r  a
  
    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in [Attention is
    all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
    )Úcustom_introc                   ó&  ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  d	ej        dz  d
ej	        dz  dej	        dz  dej	        dz  de
dz  dedz  dedz  dedz  dedz  deez  fd„¦   «         Zˆ xZS )ÚRemBertModelTc                 ó   •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        |rt          |¦  «        nd| _        |  	                    ¦   «          dS )zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        N)
r(   r)   r>   r   rP   rÛ   Úencoderr]   ÚpoolerÚ	post_init)r=   r>   Úadd_pooling_layerr?   s      €r@   r)   zRemBertModel.__init__í  ss   ø€ õ
 	‰Œ×Ò˜Ñ Ô Ð ØˆŒå+¨FÑ3Ô3ˆŒÝ% fÑ-Ô-ˆŒà/@ÐJ•m FÑ+Ô+Ð+ÀdˆŒð 	�ŠÑÔÐÐÐrA   c                 ó   — | j         j        S r_   ©rP   r.   ©r=   s    r@   Úget_input_embeddingsz!RemBertModel.get_input_embeddingsý  s   € ØŒÔ.Ð.rA   c                 ó   — || j         _        d S r_   r!  )r=   rv   s     r@   Úset_input_embeddingsz!RemBertModel.set_input_embeddings   s   € Ø*/ˆŒÔ'Ð'Ð'rA   NrB   r{   rC   r$   rD   r|   rÒ   r}   rç   r~   rè   ré   rF   c                 óè  — |
�|
n| j         j        }
|�|n| j         j        }|�|n| j         j        }| j         j        r|	�|	n| j         j        }	nd}	|�|�t          d¦  «        ‚|�+|                      ||¦  «         |                     ¦   «         }n.|�|                     ¦   «         d d…         }nt          d¦  «        ‚|\  }}|�|j	        n|j	        }|€dn| 
                    ¦   «         }|€t          j        |||z   f|¬¦  «        }|€!t          j        |t          j        |¬¦  «        }|                      |||||¬¦  «        }t!          | j         ||¬	¦  «        }|�t!          | j         |||¬
¦  «        }|                      ||||||	|
||¬¦	  «	        }|d         }| j        �|                      |¦  «        nd }|s||f|dd …         z   S t'          |||j        |j        |j        |j        ¬¦  «        S )NFzDYou cannot specify both input_ids and inputs_embeds at the same timer&   z5You have to specify either input_ids or inputs_embedsr   )rJ   rH   )rB   r$   rC   rD   rE   )r>   rD   r{   )r>   rD   r{   r|   )r{   r|   rÒ   r}   rç   r~   rè   ré   r   )rï   Úpooler_outputr}   re   rð   rñ   )r>   r~   rè   ré   rx   rç   rq   Ú%warn_if_padding_and_no_attention_maskrK   rJ   Úget_seq_lengthr9   ÚonesrL   rM   rP   r   r  r  r   r}   re   rð   rñ   )r=   rB   r{   rC   r$   rD   r|   rÒ   r}   rç   r~   rè   ré   r—   rN   Ú
batch_sizerO   rJ   rE   Úembedding_outputÚencoder_outputsr
  rh   s                          r@   rQ   zRemBertModel.forward  s‘  € ð" 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàŒ;Ô!ð 	Ø%.Ð%:˜	˜	ÀÄÔ@UˆIˆIàˆIàÐ  ]Ð%>ÝÐcÑdÔdÐdØÐ"Ø×6Ò6°yÀ.ÑQÔQÐQØ#Ÿ.š.Ñ*Ô*ˆKˆKØÐ&Ø'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKˆKåÐTÑUÔUÐUà!,Ñˆ
�JØ%.Ð%:�Ô!Ð!ÀÔ@Tˆà&5Ð&=  À?×CaÒCaÑCcÔCcÐàÐ!Ý"œZ¨*°jÐCYÑ6YÐ)ZÐdjÐkÑkÔkˆNØÐ!Ý"œ[¨½E¼JÈvÐVÑVÔVˆNàŸ?š?ØØ%Ø)Ø'Ø#9ð +ñ 
ô 
Ðõ 3Ø”;Ø*Ø)ð
ñ 
ô 
ˆð "Ð-Ý%>Ø”{Ø.Ø5Ø&;ð	&ñ &ô &Ð"ð Ÿ,š,ØØ)Ø"7Ø#9Ø+ØØ/Ø!5Ø#ð 'ñ 

ô 

ˆð *¨!Ô,ˆØ8<¼Ð8O˜Ÿš OÑ4Ô4Ð4ÐUYˆàð 	JØ# ]Ð3°oÀaÀbÀbÔ6IÑIÐIå;Ø-Ø'Ø+Ô;Ø)Ô7Ø&Ô1Ø,Ô=ð
ñ 
ô 
ð 	
rA   )T)NNNNNNNNNNNN)rR   rS   rT   r)   r#  r%  r   r9   rV   rW   r
   r¦   r§   r   rQ   rZ   r[   s   @r@   r  r  à  sŽ  ø€ € € € € ðð ð ð ð ð ð /ð /ð /ð0ð 0ð 0ð ð .2Ø26Ø26Ø04Ø26Ø:>Ø;?Ø(,Ø!%Ø)-Ø,0Ø#'ð]
ð ]
àÔ# dÑ*ð]
ð Ô(¨4Ñ/ð]
ð Ô(¨4Ñ/ð	]
ð
 Ô&¨Ñ-ð]
ð Ô(¨4Ñ/ð]
ð  %Ô0°4Ñ7ð]
ð !&Ô 1°DÑ 8ð]
ð  ™ð]
ð ˜$‘;ð]
ð   $™;ð]
ð # T™kð]
ð ˜D‘[ð]
ð 
Ð=Ñ	=ð]
ð ]
ð ]
ñ „^ð]
ð ]
ð ]
ð ]
ð ]
rA   r  c                   ó"  ‡ — e Zd Zˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  d	ej	        dz  d
ej	        dz  dej	        dz  dej        dz  de
dz  de
dz  de
dz  deez  fd„¦   «         Zˆ xZS )ÚRemBertForMaskedLMc                 ó  •— t          ¦   «                              |¦  «         |j        rt                               d¦  «         t          |d¬¦  «        | _        t          |¦  «        | _        |  	                    ¦   «          d S )NznIf you want to use `RemBertForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.F©r  ©
r(   r)   rx   ró   Úwarningr  r  r  Úclsr  r<   s     €r@   r)   zRemBertForMaskedLM.__init__f  s~   ø€ Ý‰Œ×Ò˜Ñ Ô Ð àÔð 	Ý�NŠNð1ñô ð õ
 $ F¸eÐDÑDÔDˆŒÝ% fÑ-Ô-ˆŒð 	�ŠÑÔÐÐÐrA   c                 ó$   — | j         j        j        S r_   ©r4  r	  r  r"  s    r@   Úget_output_embeddingsz(RemBertForMaskedLM.get_output_embeddingsu  ó   € ØŒxÔ#Ô+Ð+rA   c                 ó(   — || j         j        _        d S r_   r6  ©r=   Únew_embeddingss     r@   Úset_output_embeddingsz(RemBertForMaskedLM.set_output_embeddingsx  ó   € Ø'5ˆŒÔÔ$Ð$Ð$rA   NrB   r{   rC   r$   rD   r|   rÒ   Úlabelsr~   rè   ré   rF   c                 ó¦  — |�|n| j         j        }|                      ||||||||	|
|¬¦
  «
        }|d         }|                      |¦  «        }d}|�Kt	          ¦   «         } ||                     d| j         j        ¦  «        |                     d¦  «        ¦  «        }|s|f|dd…         z   }|�|f|z   n|S t          |||j        |j	        ¬¦  «        S )a£  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        N)	r{   rC   r$   rD   r|   rÒ   r~   rè   ré   r   r&   r€   ©ÚlossÚlogitsre   rð   )
r>   ré   r  r4  r   r…   r+   r   re   rð   )r=   rB   r{   rC   r$   rD   r|   rÒ   r>  r~   rè   ré   r—   r»   r
  r  Úmasked_lm_lossÚloss_fctr¶   s                      r@   rQ   zRemBertForMaskedLM.forward{  s  € ð, &1Ð%<�k�kÀ$Ä+ÔBYˆà—,’,ØØ)Ø)Ø%Ø'Ø"7Ø#9Ø/Ø!5Ø#ð ñ 
ô 
ˆð " !œ*ˆØ ŸHšH _Ñ5Ô5ÐàˆØÐÝ'Ñ)Ô)ˆHØ%˜XÐ&7×&<Ò&<¸RÀÄÔAWÑ&XÔ&XÐZ`×ZeÒZeÐfhÑZiÔZiÑjÔjˆNàð 	ZØ'Ð)¨G°A°B°B¬KÑ7ˆFØ3AÐ3M�^Ð%¨Ñ.Ð.ÐSYÐYåØØ$Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rA   )NNNNNNNNNNN)rR   rS   rT   r)   r7  r<  r   r9   rV   rW   r¦   r§   r   rQ   rZ   r[   s   @r@   r/  r/  d  se  ø€ € € € € ðð ð ð ð ð,ð ,ð ,ð6ð 6ð 6ð ð .2Ø26Ø26Ø04Ø26Ø:>Ø;?Ø*.Ø)-Ø,0Ø#'ð5
ð 5
àÔ# dÑ*ð5
ð Ô(¨4Ñ/ð5
ð Ô(¨4Ñ/ð	5
ð
 Ô&¨Ñ-ð5
ð Ô(¨4Ñ/ð5
ð  %Ô0°4Ñ7ð5
ð !&Ô 1°DÑ 8ð5
ð Ô  4Ñ'ð5
ð   $™;ð5
ð # T™kð5
ð ˜D‘[ð5
ð 
�Ñ	ð5
ð 5
ð 5
ñ „^ð5
ð 5
ð 5
ð 5
ð 5
rA   r/  zS
    RemBERT Model with a `language modeling` head on top for CLM fine-tuning.
    c            !       óP  ‡ — e Zd Zˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  d	ej        dz  d
ej	        dz  dej	        dz  dej	        dz  de
dz  dej        dz  dedz  dedz  dedz  dedz  deej        z  deez  fd„¦   «         Zˆ xZS )ÚRemBertForCausalLMc                 ó  •— t          ¦   «                              |¦  «         |j        st                               d¦  «         t          |d¬¦  «        | _        t          |¦  «        | _        |  	                    ¦   «          d S )NzOIf you want to use `RemBertForCausalLM` as a standalone, add `is_decoder=True.`Fr1  r2  r<   s     €r@   r)   zRemBertForCausalLM.__init__º  su   ø€ Ý‰Œ×Ò˜Ñ Ô Ð àÔ ð 	nÝ�NŠNÐlÑmÔmÐmå# F¸eÐDÑDÔDˆŒÝ% fÑ-Ô-ˆŒð 	�ŠÑÔÐÐÐrA   c                 ó$   — | j         j        j        S r_   r6  r"  s    r@   r7  z(RemBertForCausalLM.get_output_embeddingsÆ  r8  rA   c                 ó(   — || j         j        _        d S r_   r6  r:  s     r@   r<  z(RemBertForCausalLM.set_output_embeddingsÉ  r=  rA   Nr   rB   r{   rC   r$   rD   r|   rÒ   r}   r>  rç   r~   rè   ré   Úlogits_to_keeprF   c                 óÌ  — |�|n| j         j        }|                      |||||||||
|||¬¦  «        }|d         }t          |t          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|	� | j        d||	| j         j        dœ|¤Ž}|s|f|dd…         z   }|�|f|z   n|S t          |||j
        |j        |j        |j        ¬¦  «        S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, RemBertForCausalLM, RemBertConfig
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("google/rembert")
        >>> config = RemBertConfig.from_pretrained("google/rembert")
        >>> config.is_decoder = True
        >>> model = RemBertForCausalLM.from_pretrained("google/rembert", config=config)

        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
        >>> outputs = model(**inputs)

        >>> prediction_logits = outputs.logits
        ```N)r{   rC   r$   rD   r|   rÒ   r}   rç   r~   rè   ré   r   )rB  r>  r+   r€   )rA  rB  r}   re   rð   rñ   rë   )r>   ré   r  r‡   rX   Úslicer4  Úloss_functionr+   r   r}   re   rð   rñ   )r=   rB   r{   rC   r$   rD   r|   rÒ   r}   r>  rç   r~   rè   ré   rJ  r—   r»   re   Úslice_indicesrB  rA  r¶   s                         r@   rQ   zRemBertForCausalLM.forwardÌ  sF  € ðR &1Ð%<�k�kÀ$Ä+ÔBYˆà—,’,ØØ)Ø)Ø%Ø'Ø"7Ø#9Ø+ØØ/Ø!5Ø#ð ñ 
ô 
ˆð   œ
ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜-¨¨¨¨=¸!¸!¸!Ð(;Ô<Ñ=Ô=ˆàˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDàð 	FØ�Y ¨¨¨¤Ñ,ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå0ØØØ#Ô3Ø!Ô/ØÔ)Ø$Ô5ð
ñ 
ô 
ð 	
rA   )NNNNNNNNNNNNNr   )rR   rS   rT   r)   r7  r<  r   r9   rV   rW   r
   r¦   rX   rY   r§   r   rQ   rZ   r[   s   @r@   rF  rF  ´  s´  ø€ € € € € ð
ð 
ð 
ð 
ð 
ð,ð ,ð ,ð6ð 6ð 6ð ð .2Ø26Ø26Ø04Ø26Ø:>Ø;?Ø(,Ø*.Ø!%Ø)-Ø,0Ø#'Ø-.ðM
ð M
àÔ# dÑ*ðM
ð Ô(¨4Ñ/ðM
ð Ô(¨4Ñ/ð	M
ð
 Ô&¨Ñ-ðM
ð Ô(¨4Ñ/ðM
ð  %Ô0°4Ñ7ðM
ð !&Ô 1°DÑ 8ðM
ð  ™ðM
ð Ô  4Ñ'ðM
ð ˜$‘;ðM
ð   $™;ðM
ð # T™kðM
ð ˜D‘[ðM
ð ˜eœlÑ*ðM
ð" 
Ð2Ñ	2ð#M
ð M
ð M
ñ „^ðM
ð M
ð M
ð M
ð M
rA   rF  zŸ
    RemBERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the
    pooled output) e.g. for GLUE tasks.
    c                   óê   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	edz  d
edz  dedz  de	e
z  fd„¦   «         Zˆ xZS )Ú RemBertForSequenceClassificationc                 ó6  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        ¦  «        | _        t          j	        |j
        |j        ¦  «        | _        |                      ¦   «          d S r_   ©r(   r)   Ú
num_labelsr  r  r   r5   Úclassifier_dropout_probr7   r`   ra   Ú
classifierr  r<   s     €r@   r)   z)RemBertForSequenceClassification.__init__$  sy   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒÝ# FÑ+Ô+ˆŒÝ”z &Ô"@ÑAÔAˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐrA   NrB   r{   rC   r$   rD   r>  r~   rè   ré   rF   c
           
      óì  — |	�|	n| j         j        }	|                      ||||||||	¬¦  «        }|d         }|                      |¦  «        }|                      |¦  «        }d}|��Z| j         j        €f| j        dk    rd| j         _        nN| j        dk    r7|j        t          j	        k    s|j        t          j
        k    rd| j         _        nd| j         _        | j         j        dk    rWt          ¦   «         }| j        dk    r1 ||                     ¦   «         |                     ¦   «         ¦  «        }nŽ |||¦  «        }n�| j         j        dk    rGt          ¦   «         } ||                     d| j        ¦  «        |                     d¦  «        ¦  «        }n*| j         j        dk    rt          ¦   «         } |||¦  «        }|	s|f|dd…         z   }|�|f|z   n|S t!          |||j        |j        ¬	¦  «        S )
a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N©r{   rC   r$   rD   r~   rè   ré   r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr&   r€   r@  )r>   ré   r  r7   rU  Úproblem_typerS  rI   r9   rM   rX   r   Úsqueezer   r…   r   r   re   rð   )r=   rB   r{   rC   r$   rD   r>  r~   rè   ré   r—   r»   rh   rB  rA  rD  r¶   s                    r@   rQ   z(RemBertForSequenceClassification.forward.  s  € ð( &1Ð%<�k�kÀ$Ä+ÔBYˆà—,’,ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ñ 	
ô 	
ˆð   œ
ˆàŸš ]Ñ3Ô3ˆØ—’ Ñ/Ô/ˆàˆØÑØŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ñ @Ô @À&Ç+Â+ÈbÁ/Ä/ÑRÔR��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�Øð 	FØ�Y ¨¨¨¤Ñ,ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rA   ©	NNNNNNNNN)rR   rS   rT   r)   r   r9   rW   rV   r¦   r§   r   rQ   rZ   r[   s   @r@   rP  rP    s1  ø€ € € € € ðð ð ð ð ð ð /3Ø37Ø26Ø15Ø26Ø*.Ø)-Ø,0Ø#'ðD
ð D
àÔ$ tÑ+ðD
ð Ô)¨DÑ0ðD
ð Ô(¨4Ñ/ð	D
ð
 Ô'¨$Ñ.ðD
ð Ô(¨4Ñ/ðD
ð Ô  4Ñ'ðD
ð   $™;ðD
ð # T™kðD
ð ˜D‘[ðD
ð 
Ð)Ñ	)ðD
ð D
ð D
ñ „^ðD
ð D
ð D
ð D
ð D
rA   rP  c                   óê   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	edz  d
edz  dedz  de	e
z  fd„¦   «         Zˆ xZS )ÚRemBertForMultipleChoicec                 ó  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        ¦  «        | _        t	          j        |j	        d¦  «        | _
        |                      ¦   «          d S )Nr   )r(   r)   r  r  r   r5   rT  r7   r`   ra   rU  r  r<   s     €r@   r)   z!RemBertForMultipleChoice.__init__x  sl   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å# FÑ+Ô+ˆŒÝ”z &Ô"@ÑAÔAˆŒÝœ) FÔ$6¸Ñ:Ô:ˆŒð 	�ŠÑÔÐÐÐrA   NrB   r{   rC   r$   rD   r>  r~   rè   ré   rF   c
           
      ó¸  — |	�|	n| j         j        }	|�|j        d         n|j        d         }|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�=|                     d|                     d¦  «        |                     d¦  «        ¦  «        nd}|                      ||||||||	¬¦  «        }|d         }|                      |¦  «        }|                      |¦  «        }|                     d|¦  «        }d}|�t          ¦   «         } |||¦  «        }|	s|f|dd…         z   }|�|f|z   n|S t          |||j
        |j        ¬¦  «        S )a[  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   r&   r�   rW  r€   r@  )r>   ré   r„   r…   rK   r  r7   rU  r   r   re   rð   )r=   rB   r{   rC   r$   rD   r>  r~   rè   ré   r—   Únum_choicesr»   rh   rB  Úreshaped_logitsrA  rD  r¶   s                      r@   rQ   z RemBertForMultipleChoice.forward‚  s(  € ðX &1Ð%<�k�kÀ$Ä+ÔBYˆØ,5Ð,A�i”o aÔ(Ð(À}ÔGZÐ[\ÔG]ˆà>GÐ>S�I—N’N 2 y§~¢~°bÑ'9Ô'9Ñ:Ô:Ð:ÐY]ˆ	ØM[ÐMg˜×,Ò,¨R°×1DÒ1DÀRÑ1HÔ1HÑIÔIÐIÐmqˆØM[ÐMg˜×,Ò,¨R°×1DÒ1DÀRÑ1HÔ1HÑIÔIÐIÐmqˆØGSÐG_�|×(Ò(¨¨\×->Ò->¸rÑ-BÔ-BÑCÔCÐCÐeiˆð Ð(ð ×Ò˜r =×#5Ò#5°bÑ#9Ô#9¸=×;MÒ;MÈbÑ;QÔ;QÑRÔRÐRàð 	ð —,’,ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ñ 	
ô 	
ˆð   œ
ˆàŸš ]Ñ3Ô3ˆØ—’ Ñ/Ô/ˆØ Ÿ+š+ b¨+Ñ6Ô6ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜O¨VÑ4Ô4ˆDàð 	FØ%Ð'¨'°!°"°"¬+Ñ5ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå(ØØ"Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rA   r]  )rR   rS   rT   r)   r   r9   rW   rV   r¦   r§   r   rQ   rZ   r[   s   @r@   r_  r_  v  s1  ø€ € € € € ðð ð ð ð ð ð /3Ø37Ø26Ø15Ø26Ø*.Ø)-Ø,0Ø#'ðW
ð W
àÔ$ tÑ+ðW
ð Ô)¨DÑ0ðW
ð Ô(¨4Ñ/ð	W
ð
 Ô'¨$Ñ.ðW
ð Ô(¨4Ñ/ðW
ð Ô  4Ñ'ðW
ð   $™;ðW
ð # T™kðW
ð ˜D‘[ðW
ð 
Ð*Ñ	*ðW
ð W
ð W
ñ „^ðW
ð W
ð W
ð W
ð W
rA   r_  c                   óê   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	edz  d
edz  dedz  de	e
z  fd„¦   «         Zˆ xZS )ÚRemBertForTokenClassificationc                 ó:  •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        t          j        |j        ¦  «        | _        t          j	        |j
        |j        ¦  «        | _        |                      ¦   «          d S ©NFr1  rR  r<   s     €r@   r)   z&RemBertForTokenClassification.__init__ß  s~   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå# F¸eÐDÑDÔDˆŒÝ”z &Ô"@ÑAÔAˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐrA   NrB   r{   rC   r$   rD   r>  r~   rè   ré   rF   c
           
      óÂ  — |	�|	n| j         j        }	|                      ||||||||	¬¦  «        }|d         }|                      |¦  «        }|                      |¦  «        }d}|�Ft          ¦   «         } ||                     d| j        ¦  «        |                     d¦  «        ¦  «        }|	s|f|dd…         z   }|�|f|z   n|S t          |||j	        |j
        ¬¦  «        S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        NrW  r   r&   r€   r@  )r>   ré   r  r7   rU  r   r…   rS  r   re   rð   )r=   rB   r{   rC   r$   rD   r>  r~   rè   ré   r—   r»   r
  rB  rA  rD  r¶   s                    r@   rQ   z%RemBertForTokenClassification.forwardê  s  € ð$ &1Ð%<�k�kÀ$Ä+ÔBYˆà—,’,ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ñ 	
ô 	
ˆð " !œ*ˆàŸ,š, Ñ7Ô7ˆØ—’ Ñ1Ô1ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬OÑ<Ô<¸f¿kºkÈ"¹o¼oÑNÔNˆDàð 	FØ�Y ¨¨¨¤Ñ,ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rA   r]  )rR   rS   rT   r)   r   r9   rW   rV   r¦   r§   r   rQ   rZ   r[   s   @r@   re  re  Ý  s  ø€ € € € € ð	ð 	ð 	ð 	ð 	ð ð /3Ø37Ø26Ø15Ø26Ø*.Ø)-Ø,0Ø#'ð1
ð 1
àÔ$ tÑ+ð1
ð Ô)¨DÑ0ð1
ð Ô(¨4Ñ/ð	1
ð
 Ô'¨$Ñ.ð1
ð Ô(¨4Ñ/ð1
ð Ô  4Ñ'ð1
ð   $™;ð1
ð # T™kð1
ð ˜D‘[ð1
ð 
Ð&Ñ	&ð1
ð 1
ð 1
ñ „^ð1
ð 1
ð 1
ð 1
ð 1
rA   re  c                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
edz  dedz  dedz  de	e
z  fd„¦   «         Zˆ xZS )ÚRemBertForQuestionAnsweringc                 óþ   •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        t          j        |j        |j        ¦  «        | _        |  	                    ¦   «          d S rg  )
r(   r)   rS  r  r  r   r`   ra   Ú
qa_outputsr  r<   s     €r@   r)   z$RemBertForQuestionAnswering.__init__!  sj   ø€ Ý‰Œ×Ò˜Ñ Ô Ð à Ô+ˆŒå# F¸eÐDÑDÔDˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐrA   NrB   r{   rC   r$   rD   Ústart_positionsÚend_positionsr~   rè   ré   rF   c           
      óf  — |
�|
n| j         j        }
|                      |||||||	|
¬¦  «        }|d         }|                      |¦  «        }|                     dd¬¦  «        \  }}|                     d¦  «        }|                     d¦  «        }d }|�ç|�åt          |                     ¦   «         ¦  «        dk    r|                     d¦  «        }t          |                     ¦   «         ¦  «        dk    r|                     d¦  «        }|                     d¦  «        }|                     d|¦  «         |                     d|¦  «         t          |¬¦  «        } |||¦  «        } |||¦  «        }||z   dz  }|
s||f|dd …         z   }|�|f|z   n|S t          ||||j        |j        ¬¦  «        S )	NrW  r   r   r&   r‚   )Úignore_indexr€   )rA  Ústart_logitsÚ
end_logitsre   rð   )r>   ré   r  rl  Úsplitr\  ÚlenrK   Úclamp_r   r   re   rð   )r=   rB   r{   rC   r$   rD   rm  rn  r~   rè   ré   r—   r»   r
  rB  rq  rr  Ú
total_lossÚignored_indexrD  Ú
start_lossÚend_lossr¶   s                          r@   rQ   z#RemBertForQuestionAnswering.forward,  sú  € ð &1Ð%<�k�kÀ$Ä+ÔBYˆà—,’,ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ñ 	
ô 	
ˆð " !œ*ˆà—’ Ñ1Ô1ˆØ#)§<¢<°°r <Ñ#:Ô#:Ñ ˆ�jØ#×+Ò+¨BÑ/Ô/ˆØ×'Ò'¨Ñ+Ô+ˆ
àˆ
ØÐ&¨=Ð+Då�?×'Ò'Ñ)Ô)Ñ*Ô*¨QÒ.Ð.Ø"1×"9Ò"9¸"Ñ"=Ô"=�Ý�=×%Ò%Ñ'Ô'Ñ(Ô(¨1Ò,Ð,Ø -× 5Ò 5°bÑ 9Ô 9�à(×-Ò-¨aÑ0Ô0ˆMØ×"Ò" 1 mÑ4Ô4Ð4Ø× Ò   MÑ2Ô2Ð2å'°]ÐCÑCÔCˆHØ!˜ ,°Ñ@Ô@ˆJØ�x 
¨MÑ:Ô:ˆHØ$ xÑ/°1Ñ4ˆJàð 	RØ" JÐ/°'¸!¸"¸"´+Ñ=ˆFØ/9Ð/E�Z�M FÑ*Ð*È6ÐQå+ØØ%Ø!Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
rA   )
NNNNNNNNNN)rR   rS   rT   r)   r   r9   rW   rV   r¦   r§   r   rQ   rZ   r[   s   @r@   rj  rj    s3  ø€ € € € € ð	ð 	ð 	ð 	ð 	ð ð /3Ø37Ø26Ø15Ø26Ø37Ø15Ø)-Ø,0Ø#'ð=
ð =
àÔ$ tÑ+ð=
ð Ô)¨DÑ0ð=
ð Ô(¨4Ñ/ð	=
ð
 Ô'¨$Ñ.ð=
ð Ô(¨4Ñ/ð=
ð Ô)¨DÑ0ð=
ð Ô'¨$Ñ.ð=
ð   $™;ð=
ð # T™kð=
ð ˜D‘[ð=
ð 
Ð-Ñ	-ð=
ð =
ð =
ñ „^ð=
ð =
ð =
ð =
ð =
rA   rj  )	rF  r/  r_  rj  rP  re  rÊ   r  r  )BrU   r‘   r9   r   Útorch.nnr   r   r   Ú r   r  Úactivationsr	   Úcache_utilsr
   r   r   Ú
generationr   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   r   r   Úmodeling_utilsr   Úpytorch_utilsr   Úutilsr   r   Úconfiguration_rembertr   Ú
get_loggerrR   ró   ÚModuler   r]   rk   r©   r³   r½   rÆ   rÊ   rÛ   rü   r  r  r  r  r/  rF  rP  r_  re  rj  Ú__all__rë   rA   r@   ú<module>r‰     s
  ðð Ð à €€€à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ )Ð )Ð )Ð )Ð )Ð )Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9ð	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð .Ð -Ð -Ð -Ð -Ð -Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0ð 
ˆÔ	˜HÑ	%Ô	%€ð1ð 1ð 1ð 1ð 1˜œ	ñ 1ô 1ð 1ðjð ð ð ð �B”Iñ ô ð ðV.ð V.ð V.ð V.ð V.˜2œ9ñ V.ô V.ð V.ðtð ð ð ð ˜œ	ñ ô ð ðð ð ð ð �r”yñ ô ð ð8ð ð ð ð ˜"œ)ñ ô ð ð ð ð ð ð �B”Iñ ô ð ð?ð ?ð ?ð ?ð ?Ð-ñ ?ô ?ð ?ðDM
ð M
ð M
ð M
ð M
�R”Yñ M
ô M
ð M
ðbð ð ð ð  R¤Yñ ô ð ð"ð ð ð ð ˜bœiñ ô ð ð"!ð !ð !ð !ð !˜œñ !ô !ð !ð ðið ið ið ið i˜_ñ iô iñ „ðið €ð	ðñ ô ðu
ð u
ð u
ð u
ð u
Ð)ñ u
ô u
ñô ðu
ðp ðL
ð L
ð L
ð L
ð L
Ð/ñ L
ô L
ñ „ðL
ð^ €ððñ ô ð
a
ð a
ð a
ð a
ð a
Ð/°ñ a
ô a
ñô ð
a
ðH €ððñ ô ðP
ð P
ð P
ð P
ð P
Ð'=ñ P
ô P
ñô ðP
ðf ðc
ð c
ð c
ð c
ð c
Ð5ñ c
ô c
ñ „ðc
ðL ð>
ð >
ð >
ð >
ð >
Ð$:ñ >
ô >
ñ „ð>
ðB ðJ
ð J
ð J
ð J
ð J
Ð"8ñ J
ô J
ñ „ðJ
ðZ
ð 
ð 
€€€rA   