§
    ‚Štjo¯  ã                   ót  — d Z ddlZddlmZ ddlZddlmZ ddlmZmZm	Z	 ddl
mZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZmZ ddlmZ ddlmZ ddlm Z  ddl!m"Z"m#Z#m$Z$m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+  e%j,        e-¦  «        Z. G d„ dej/        ¦  «        Z0 G d„ dej/        ¦  «        Z1 G d„ dej/        ¦  «        Z2 G d„ dej/        ¦  «        Z3 G d„ dej/        ¦  «        Z4 G d„ dej/        ¦  «        Z5 G d „ d!ej/        ¦  «        Z6 G d"„ d#ej/        ¦  «        Z7 G d$„ d%e¦  «        Z8e# G d&„ d'e¦  «        ¦   «         Z9 G d(„ d)ej/        ¦  «        Z: G d*„ d+ej/        ¦  «        Z; G d,„ d-ej/        ¦  «        Z<e# G d.„ d/e9¦  «        ¦   «         Z= G d0„ d1ej/        ¦  «        Z>e# G d2„ d3e9¦  «        ¦   «         Z? G d4„ d5ej/        ¦  «        Z@ e#d6¬7¦  «         G d8„ d9e9¦  «        ¦   «         ZAe# G d:„ d;e9¦  «        ¦   «         ZBe# G d<„ d=e9¦  «        ¦   «         ZCe# G d>„ d?e9¦  «        ¦   «         ZDg d@¢ZEdS )AzPyTorch ConvBERT model.é    N)ÚCallable)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FNÚget_activation)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)Ú"BaseModelOutputWithCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)ÚUnpack)Úapply_chunking_to_forward)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚConvBertConfigc                   ó’   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 d
dej        dz  dej        dz  dej        dz  dej        dz  dej        f
d	„Zˆ xZ	S )ÚConvBertEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 óÒ  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        ¬¦  «        | _        t          j        |j        |j        ¦  «        | _	        t          j        |j
        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j        ¦  «        | _        |                      dt%          j        |j        ¦  «                             d¦  «        d¬¦  «         |                      dt%          j        | j                             ¦   «         t$          j        ¬¦  «        d¬¦  «         d S )	N)Úpadding_idx©ÚepsÚposition_ids©r   éÿÿÿÿF)Ú
persistentÚtoken_type_ids©Údtype)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚembedding_sizeÚpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferÚtorchÚarangeÚexpandÚzerosr%   ÚsizeÚlong©ÚselfÚconfigÚ	__class__s     €úl/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/convbert/modeling_convbert.pyr-   zConvBertEmbeddings.__init__7   s3  ø€ Ý‰Œ×ÒÑÔÐÝ!œ|¨FÔ,=¸vÔ?TÐbhÔbuÐvÑvÔvˆÔÝ#%¤<°Ô0NÐPVÔPeÑ#fÔ#fˆÔ Ý%'¤\°&Ô2HÈ&ÔJ_Ñ%`Ô%`ˆÔ"åœ fÔ&;ÀÔAVÐWÑWÔWˆŒÝ”z &Ô"<Ñ=Ô=ˆŒà×ÒØ�EœL¨Ô)GÑHÔH×OÒOÐPWÑXÔXÐejð 	ñ 	
ô 	
ð 	
ð 	×ÒØ�eœk¨$Ô*;×*@Ò*@Ñ*BÔ*BÍ%Ì*ÐUÑUÔUÐbgð 	ñ 	
ô 	
ð 	
ð 	
ð 	
ó    NÚ	input_idsr)   r%   Úinputs_embedsÚreturnc                 ój  — |�|                      ¦   «         }n|                      ¦   «         d d…         }|d         }|€| j        d d …d |…f         }|€mt          | d¦  «        r2| j        d d …d |…f         }|                     |d         |¦  «        }|}n+t          j        |t
          j        | j        j        ¬¦  «        }|€|  	                    |¦  «        }|  
                    |¦  «        }	|                      |¦  «        }
||	z   |
z   }|                      |¦  «        }|                      |¦  «        }|S )Nr'   r   r)   r   ©r+   Údevice)rA   r%   Úhasattrr)   r?   r=   r@   rB   rN   r2   r4   r6   r7   r;   )rD   rI   r)   r%   rJ   Úinput_shapeÚ
seq_lengthÚbuffered_token_type_idsÚ buffered_token_type_ids_expandedr4   r6   Ú
embeddingss               rG   ÚforwardzConvBertEmbeddings.forwardG   sQ  € ð Ð Ø#Ÿ.š.Ñ*Ô*ˆKˆKà'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKà  ”^ˆ
àÐØÔ,¨Q¨Q¨Q°°°¨^Ô<ˆLð
 Ð!Ý�tÐ-Ñ.Ô.ð mØ*.Ô*=¸a¸a¸aÀÀ*À¸nÔ*MÐ'Ø3J×3QÒ3QÐR]Ð^_ÔR`ÐblÑ3mÔ3mÐ0Ø!A��å!&¤¨[ÅÄ
ÐSWÔSdÔSkÐ!lÑ!lÔ!l�àÐ Ø ×0Ò0°Ñ;Ô;ˆMØ"×6Ò6°|ÑDÔDÐØ $× :Ò :¸>Ñ JÔ JÐà"Ð%8Ñ8Ð;PÑPˆ
Ø—^’^ JÑ/Ô/ˆ
Ø—\’\ *Ñ-Ô-ˆ
ØÐrH   )NNNN)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r-   r=   Ú
LongTensorÚFloatTensorrU   Ú__classcell__©rF   s   @rG   r    r    4   s¸   ø€ € € € € ØQÐQð
ð 
ð 
ð 
ð 
ð$ .2Ø26Ø04Ø26ð$ð $àÔ# dÑ*ð$ð Ô(¨4Ñ/ð$ð Ô&¨Ñ-ð	$ð
 Ô(¨4Ñ/ð$ð 
Ô	ð$ð $ð $ð $ð $ð $ð $ð $rH   r    c                   óF   ‡ — e Zd ZdZˆ fd„Zdej        dej        fd„Zˆ xZS )ÚSeparableConv1DzSThis class implements separable convolution, i.e. a depthwise and a pointwise layerc                 óÊ  •— t          ¦   «                              ¦   «          t          j        |||||dz  d¬¦  «        | _        t          j        ||dd¬¦  «        | _        t          j        t          j        |d¦  «        ¦  «        | _	        | j        j
        j                             d|j        ¬¦  «         | j        j
        j                             d|j        ¬¦  «         d S )Né   F)Úkernel_sizeÚgroupsÚpaddingÚbiasr   )rb   re   ç        ©ÚmeanÚstd)r,   r-   r   ÚConv1dÚ	depthwiseÚ	pointwiseÚ	Parameterr=   r@   re   ÚweightÚdataÚnormal_Úinitializer_range)rD   rE   Úinput_filtersÚoutput_filtersrb   ÚkwargsrF   s         €rG   r-   zSeparableConv1D.__init__q   sÐ   ø€ Ý‰Œ×ÒÑÔÐÝœØØØ#Ø Ø 1Ñ$Øð
ñ 
ô 
ˆŒõ œ =°.ÈaÐV[Ð\Ñ\Ô\ˆŒÝ”L¥¤¨^¸QÑ!?Ô!?Ñ@Ô@ˆŒ	àŒÔÔ"×*Ò*°¸Ô9QÐ*ÑRÔRÐRØŒÔÔ"×*Ò*°¸Ô9QÐ*ÑRÔRÐRÐRÐRrH   Úhidden_statesrK   c                 ón   — |                       |¦  «        }|                      |¦  «        }|| j        z  }|S ©N)rk   rl   re   )rD   ru   Úxs      rG   rU   zSeparableConv1D.forward�   s4   € Ø�NŠN˜=Ñ)Ô)ˆØ�NŠN˜1ÑÔˆØ	ˆTŒY‰ˆØˆrH   ©	rV   rW   rX   rY   r-   r=   ÚTensorrU   r\   r]   s   @rG   r_   r_   n   si   ø€ € € € € Ø]Ð]ðSð Sð Sð Sð Sð  U¤\ð °e´lð ð ð ð ð ð ð ð rH   r_   c                   óš   ‡ — e Zd Zˆ fd„Z	 	 d	dej        dej        dz  dej        dz  dee         de	ej        ej        f         f
d„Z
ˆ xZS )
ÚConvBertSelfAttentionc                 óp  •— t          ¦   «                              ¦   «          |j        |j        z  dk    r0t	          |d¦  «        s t          d|j        › d|j        › d�¦  «        ‚|j        |j        z  }|dk     r|j        | _        d| _        n|| _        |j        | _        |j        | _        |j        | j        z  dk    rt          d¦  «        ‚|j        | j        z  dz  | _        | j        | j        z  | _	        t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        t          ||j        | j	        | j        ¦  «        | _        t          j        | j	        | j        | j        z  ¦  «        | _        t          j        |j        | j	        ¦  «        | _        t          j        | j        dgt)          | j        dz
  dz  ¦  «        dg¬	¦  «        | _        t          j        |j        ¦  «        | _        d S )
Nr   r0   zThe hidden size (z6) is not a multiple of the number of attention heads (ú)r   z6hidden_size should be divisible by num_attention_headsra   )rb   rd   )r,   r-   Úhidden_sizeÚnum_attention_headsrO   Ú
ValueErrorÚ
head_ratioÚconv_kernel_sizeÚattention_head_sizeÚall_head_sizer   ÚLinearÚqueryÚkeyÚvaluer_   Úkey_conv_attn_layerÚconv_kernel_layerÚconv_out_layerÚUnfoldÚintÚunfoldr9   Úattention_probs_dropout_probr;   )rD   rE   Únew_num_attention_headsrF   s      €rG   r-   zConvBertSelfAttention.__init__‰   s&  ø€ Ý‰Œ×ÒÑÔÐØÔ Ô :Ñ:¸aÒ?Ð?ÍÐPVÐXhÑHiÔHiÐ?Ýð8 FÔ$6ð 8ð 8Ø Ô4ð8ð 8ð 8ñô ð ð
 #)Ô"<ÀÔ@QÑ"QÐØ" QÒ&Ð&Ø$Ô8ˆDŒOØ'(ˆDÔ$Ð$à'>ˆDÔ$Ø$Ô/ˆDŒOà &Ô 7ˆÔØÔ Ô 8Ñ8¸AÒ=Ð=ÝÐUÑVÔVÐVà$*Ô$6¸$Ô:RÑ$RÐWXÑ#XˆÔ Ø!Ô5¸Ô8PÑPˆÔå”Y˜vÔ1°4Ô3EÑFÔFˆŒ
Ý”9˜VÔ/°Ô1CÑDÔDˆŒÝ”Y˜vÔ1°4Ô3EÑFÔFˆŒ
å#2Ø�FÔ&¨Ô(:¸DÔ<Qñ$
ô $
ˆÔ õ "$¤¨4Ô+=¸tÔ?WÐZ^ÔZoÑ?oÑ!pÔ!pˆÔÝ œi¨Ô(:¸DÔ<NÑOÔOˆÔå”iØÔ.°Ð2½SÀ$ÔBWÐZ[ÑB[Ð_`ÑA`Ñ=aÔ=aÐcdÐ<eð
ñ 
ô 
ˆŒõ ”z &Ô"EÑFÔFˆŒˆˆrH   Nru   Úattention_maskÚencoder_hidden_statesrt   rK   c                 ó4  — |j         d d…         }g |¢d‘| j        ‘R }|�+|                      |¦  «        }|                      |¦  «        }n*|                      |¦  «        }|                      |¦  «        }|                      |                     dd¦  «        ¦  «        }	|	                     dd¦  «        }	|                      |¦  «        }
|
                     |¦  «                             dd¦  «        }|                     |¦  «                             dd¦  «        }|                     |¦  «                             dd¦  «        }t          j	        |	|
¦  «        }|  
                    |¦  «        }t          j        |d| j        dg¦  «        }t          j        |d¬¦  «        }|                      |¦  «        }t          j        ||d         d| j        g¦  «        }|                     dd¦  «                             ¦   «                              d¦  «        }t$          j                             || j        dgd| j        dz
  dz  dgd¬¦  «        }|                     dd¦  «                             |d         d| j        | j        ¦  «        }t          j        |d| j        | j        g¦  «        }t          j        ||¦  «        }t          j        |d| j        g¦  «        }t          j        ||                     dd¦  «        ¦  «        }|t-          j        | j        ¦  «        z  }|�||z   }t$          j                             |d¬¦  «        }|                      |¦  «        }t          j        ||¦  «        }|                     dddd¦  «                             ¦   «         }t          j        ||d         d| j        | j        g¦  «        }t          j        ||gd¦  «        }|                     ¦   «         d d…         | j        | j        z  dz  fz   } |j        |Ž }||fS )	Nr'   r   ra   ©Údimr   )rb   Údilationrd   Ústrideéþÿÿÿr   )Úshaper„   rˆ   r‰   rŠ   Ú	transposer‡   Úviewr=   Úmultiplyr‹   Úreshaperƒ   ÚsoftmaxrŒ   r…   Ú
contiguousÚ	unsqueezer   Ú
functionalr�   ÚmatmulÚmathÚsqrtr;   Úpermuter€   ÚcatrA   )rD   ru   r’   r“   rt   rP   Úhidden_shapeÚmixed_key_layerÚmixed_value_layerÚmixed_key_conv_attn_layerÚmixed_query_layerÚquery_layerÚ	key_layerÚvalue_layerÚconv_attn_layerr‹   rŒ   Úattention_scoresÚattention_probsÚcontext_layerÚconv_outÚnew_context_layer_shapes                         rG   rU   zConvBertSelfAttention.forward°   s  € ð $Ô)¨#¨2¨#Ô.ˆØC˜ÐC bÐC¨$Ô*BÐCÐCˆð !Ð,Ø"ŸhšhÐ'<Ñ=Ô=ˆOØ $§
¢
Ð+@Ñ AÔ AÐÐà"Ÿhšh }Ñ5Ô5ˆOØ $§
¢
¨=Ñ 9Ô 9Ðà$(×$<Ò$<¸]×=TÒ=TÐUVÐXYÑ=ZÔ=ZÑ$[Ô$[Ð!Ø$=×$GÒ$GÈÈ1Ñ$MÔ$MÐ!à ŸJšJ }Ñ5Ô5ÐØ'×,Ò,¨\Ñ:Ô:×DÒDÀQÈÑJÔJˆà#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆ	Ø'×,Ò,¨\Ñ:Ô:×DÒDÀQÈÑJÔJˆåœ.Ð)BÐDUÑVÔVˆà ×2Ò2°?ÑCÔCÐÝ!œMÐ*;¸bÀ$ÔBWÐYZÐ=[Ñ\Ô\ÐÝ!œMÐ*;ÀÐCÑCÔCÐà×,Ò,¨]Ñ;Ô;ˆÝœ ~¸ÀA¼ÈÈDÔL^Ð7_Ñ`Ô`ˆØ'×1Ò1°!°QÑ7Ô7×BÒBÑDÔD×NÒNÈrÑRÔRˆÝœ×-Ò-ØØÔ.°Ð2ØØÔ+¨aÑ/°AÑ5°qÐ9Øð .ñ 
ô 
ˆð (×1Ò1°!°QÑ7Ô7×?Ò?Ø˜ŒN˜B Ô 2°DÔ4Iñ
ô 
ˆõ œ ~¸¸DÔ<TÐVZÔVkÐ7lÑmÔmˆÝœ nÐ6GÑHÔHˆÝœ ~¸¸DÔ<NÐ7OÑPÔPˆõ !œ<¨°Y×5HÒ5HÈÈRÑ5PÔ5PÑQÔQÐØ+­d¬i¸Ô8PÑ.QÔ.QÑQÐØÐ%à/°.Ñ@Ðõ œ-×/Ò/Ð0@ÀbÐ/ÑIÔIˆð Ÿ,š, Ñ7Ô7ˆåœ _°kÑBÔBˆØ%×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔFˆå”=Ø˜[¨œ^¨R°Ô1IÈ4ÔKcÐdñ
ô 
ˆõ œ	 =°(Ð";¸QÑ?Ô?ˆð #0×"4Ò"4Ñ"6Ô"6°s¸°sÔ";ØÔ$ tÔ'?Ñ?À!ÑCð?
ñ #
Ðð +˜Ô*Ð,CÐDˆà˜oÐ-Ð-rH   ©NN)rV   rW   rX   r-   r=   rz   r[   r   r   ÚtuplerU   r\   r]   s   @rG   r|   r|   ˆ   sÂ   ø€ € € € € ð%Gð %Gð %Gð %Gð %GðT 48Ø59ð	O.ð O.à”|ðO.ð Ô)¨DÑ0ðO.ð  %œ|¨dÑ2ð	O.ð
 Ð+Ô,ðO.ð 
ˆuŒ|˜Uœ\Ð)Ô	*ðO.ð O.ð O.ð O.ð O.ð O.ð O.ð O.rH   r|   c                   óP   ‡ — e Zd Zˆ fd„Zdej        dej        dej        fd„Zˆ xZS )ÚConvBertSelfOutputc                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j	        ¦  «        | _
        d S ©Nr#   )r,   r-   r   r†   r   Údenser7   r8   r9   r:   r;   rC   s     €rG   r-   zConvBertSelfOutput.__init__  sf   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆrH   ru   Úinput_tensorrK   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|S rw   ©r¼   r;   r7   ©rD   ru   r½   s      rG   rU   zConvBertSelfOutput.forward	  ó@   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØŸš }°|Ñ'CÑDÔDˆØÐrH   ©rV   rW   rX   r-   r=   rz   rU   r\   r]   s   @rG   r¹   r¹     si   ø€ € € € € ð>ð >ð >ð >ð >ð U¤\ð ÀÄð ÐRWÔR^ð ð ð ð ð ð ð ð rH   r¹   c                   ó€   ‡ — e Zd Zˆ fd„Z	 	 d	dej        dej        dz  dej        dz  dee         dej        f
d„Z	ˆ xZ
S )
ÚConvBertAttentionc                 ó˜   •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          |¦  «        | _        d S rw   )r,   r-   r|   rD   r¹   ÚoutputrC   s     €rG   r-   zConvBertAttention.__init__  s;   ø€ Ý‰Œ×ÒÑÔÐÝ)¨&Ñ1Ô1ˆŒ	Ý(¨Ñ0Ô0ˆŒˆˆrH   Nru   r’   r“   rt   rK   c                 óZ   —  | j         ||fd|i|¤Ž\  }}|                      ||¦  «        }|S )Nr“   )rD   rÆ   )rD   ru   r’   r“   rt   r³   Ú_Úattention_outputs           rG   rU   zConvBertAttention.forward  sW   € ð %˜4œ9ØØð
ð 
ð #8ð
ð ð	
ð 
Ñˆ�qð  Ÿ;š; }°mÑDÔDÐØÐrH   r¶   )rV   rW   rX   r-   r=   rz   r[   r   r   rU   r\   r]   s   @rG   rÄ   rÄ     s¢   ø€ € € € € ð1ð 1ð 1ð 1ð 1ð 48Ø59ð	 ð  à”|ð ð Ô)¨DÑ0ð ð  %œ|¨dÑ2ð	 ð
 Ð+Ô,ð ð 
Œð ð  ð  ð  ð  ð  ð  ð  rH   rÄ   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚGroupedLinearLayerc                 ó�  •— t          ¦   «                              ¦   «          || _        || _        || _        | j        | j        z  | _        | j        | j        z  | _        t          j        t          j
        | j        | j        | j        ¦  «        ¦  «        | _        t          j        t          j
        |¦  «        ¦  «        | _        d S rw   )r,   r-   Ú
input_sizeÚoutput_sizeÚ
num_groupsÚgroup_in_dimÚgroup_out_dimr   rm   r=   Úemptyrn   re   )rD   rÍ   rÎ   rÏ   rF   s       €rG   r-   zGroupedLinearLayer.__init__(  s™   ø€ Ý‰Œ×ÒÑÔÐØ$ˆŒØ&ˆÔØ$ˆŒØ œO¨t¬Ñ>ˆÔØ!Ô-°´Ñ@ˆÔÝ”l¥5¤;¨t¬ÀÔ@QÐSWÔSeÑ#fÔ#fÑgÔgˆŒÝ”L¥¤¨[Ñ!9Ô!9Ñ:Ô:ˆŒ	ˆ	ˆ	rH   ru   rK   c                 óv  — t          |                     ¦   «         ¦  «        d         }t          j        |d| j        | j        g¦  «        }|                     ddd¦  «        }t          j        || j        ¦  «        }|                     ddd¦  «        }t          j        ||d| j	        g¦  «        }|| j
        z   }|S )Nr   r'   r   ra   )ÚlistrA   r=   rž   rÏ   rÐ   r¦   r£   rn   rÎ   re   )rD   ru   Ú
batch_sizerx   s       rG   rU   zGroupedLinearLayer.forward2  s¥   € Ý˜-×,Ò,Ñ.Ô.Ñ/Ô/°Ô2ˆ
ÝŒM˜-¨"¨d¬o¸tÔ?PÐ)QÑRÔRˆØ�IŠI�a˜˜AÑÔˆÝŒL˜˜DœKÑ(Ô(ˆØ�IŠI�a˜˜AÑÔˆÝŒM˜!˜j¨"¨dÔ.>Ð?Ñ@Ô@ˆØ�”	‰MˆØˆrH   rÂ   r]   s   @rG   rË   rË   '  s^   ø€ € € € € ð;ð ;ð ;ð ;ð ;ð U¤\ð °e´lð ð ð ð ð ð ð ð rH   rË   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚConvBertIntermediatec                 ór  •— t          ¦   «                              ¦   «          |j        dk    r%t          j        |j        |j        ¦  «        | _        n&t          |j        |j        |j        ¬¦  «        | _        t          |j
        t          ¦  «        rt          |j
                 | _        d S |j
        | _        d S )Nr   ©rÍ   rÎ   rÏ   )r,   r-   rÏ   r   r†   r   Úintermediate_sizer¼   rË   Ú
isinstanceÚ
hidden_actÚstrr
   Úintermediate_act_fnrC   s     €rG   r-   zConvBertIntermediate.__init__>  s§   ø€ Ý‰Œ×ÒÑÔÐØÔ Ò!Ð!Ýœ 6Ô#5°vÔ7OÑPÔPˆDŒJˆJå+Ø!Ô-¸6Ô;SÐ`fÔ`qðñ ô ˆDŒJõ �fÔ'­Ñ-Ô-ð 	9Ý'-¨fÔ.?Ô'@ˆDÔ$Ð$Ð$à'-Ô'8ˆDÔ$Ð$Ð$rH   ru   rK   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rw   )r¼   rÞ   ©rD   ru   s     rG   rU   zConvBertIntermediate.forwardK  s,   € ØŸ
š
 =Ñ1Ô1ˆØ×0Ò0°Ñ?Ô?ˆØÐrH   rÂ   r]   s   @rG   r×   r×   =  s^   ø€ € € € € ð9ð 9ð 9ð 9ð 9ð U¤\ð °e´lð ð ð ð ð ð ð ð rH   r×   c                   óP   ‡ — e Zd Zˆ fd„Zdej        dej        dej        fd„Zˆ xZS )ÚConvBertOutputc                 óz  •— t          ¦   «                              ¦   «          |j        dk    r%t          j        |j        |j        ¦  «        | _        n&t          |j        |j        |j        ¬¦  «        | _        t          j	        |j        |j
        ¬¦  «        | _	        t          j        |j        ¦  «        | _        d S )Nr   rÙ   r#   )r,   r-   rÏ   r   r†   rÚ   r   r¼   rË   r7   r8   r9   r:   r;   rC   s     €rG   r-   zConvBertOutput.__init__R  sŸ   ø€ Ý‰Œ×ÒÑÔÐØÔ Ò!Ð!Ýœ 6Ô#;¸VÔ=OÑPÔPˆDŒJˆJå+Ø!Ô3ÀÔASÐ`fÔ`qðñ ô ˆDŒJõ œ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆrH   ru   r½   rK   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|S rw   r¿   rÀ   s      rG   rU   zConvBertOutput.forward]  rÁ   rH   rÂ   r]   s   @rG   râ   râ   Q  si   ø€ € € € € ð	>ð 	>ð 	>ð 	>ð 	>ð U¤\ð ÀÄð ÐRWÔR^ð ð ð ð ð ð ð ð rH   râ   c                   óœ   ‡ — e Zd Zˆ fd„Z	 	 	 ddej        dej        dz  dej        dz  dej        dz  dee         dej        fd	„Z	d
„ Z
ˆ xZS )ÚConvBertLayerc                 ó~  •— t          ¦   «                              ¦   «          |j        | _        d| _        t	          |¦  «        | _        |j        | _        |j        | _        | j        r-| j        st          | › d�¦  «        ‚t	          |¦  «        | _	        t          |¦  «        | _        t          |¦  «        | _        d S )Nr   z> should be used as a decoder model if cross attention is added)r,   r-   Úchunk_size_feed_forwardÚseq_len_dimrÄ   Ú	attentionÚ
is_decoderÚadd_cross_attentionÚ	TypeErrorÚcrossattentionr×   Úintermediaterâ   rÆ   rC   s     €rG   r-   zConvBertLayer.__init__e  s°   ø€ Ý‰Œ×ÒÑÔÐØ'-Ô'EˆÔ$ØˆÔÝ*¨6Ñ2Ô2ˆŒØ Ô+ˆŒØ#)Ô#=ˆÔ ØÔ#ð 	<Ø”?ð iÝ 4Ð gÐ gÐ gÑhÔhÐhÝ"3°FÑ";Ô";ˆDÔÝ0°Ñ8Ô8ˆÔÝ$ VÑ,Ô,ˆŒˆˆrH   Nru   r’   r“   Úencoder_attention_maskrt   rK   c                 óà   —  | j         ||fi |¤Ž}| j        r6|�4t          | d¦  «        st          d| › d�¦  «        ‚ | j        ||fd|i|¤Ž}t          | j        | j        | j        |¦  «        }|S )Nrî   z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r“   )	rê   rë   rO   ÚAttributeErrorrî   r   Úfeed_forward_chunkrè   ré   )rD   ru   r’   r“   rð   rt   rÉ   Úlayer_outputs           rG   rU   zConvBertLayer.forwards  sà   € ð *˜4œ>ØØð
ð 
ð ð
ð 
Ðð Œ?ð 	Ð4Ð@Ý˜4Ð!1Ñ2Ô2ð Ý$ðD¸dð Dð Dð Dñô ð ð  3˜tÔ2Ø Ø&ð ð  ð '<ð ð ð	 ð  Ðõ 1ØÔ# TÔ%AÀ4ÔCSÐUeñ
ô 
ˆð ÐrH   c                 ó\   — |                       |¦  «        }|                      ||¦  «        }|S rw   )rï   rÆ   )rD   rÉ   Úintermediate_outputrô   s       rG   ró   z ConvBertLayer.feed_forward_chunk“  s2   € Ø"×/Ò/Ð0@ÑAÔAÐØ—{’{Ð#6Ð8HÑIÔIˆØÐrH   ©NNN)rV   rW   rX   r-   r=   rz   r[   r   r   rU   ró   r\   r]   s   @rG   ræ   ræ   d  sÆ   ø€ € € € € ð-ð -ð -ð -ð -ð" 48Ø59Ø6:ðð à”|ðð Ô)¨DÑ0ðð  %œ|¨dÑ2ð	ð
 !&¤¨tÑ 3ðð Ð+Ô,ðð 
Œðð ð ð ð@ð ð ð ð ð ð rH   ræ   c                   óf   ‡ — e Zd ZU eed<   dZdZeedœZ	 e
j        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚConvBertPreTrainedModelrE   ÚconvbertT)ru   Ú
attentionsc                 óT  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rt	          j        |j        ¦  «         dS t          |t          ¦  «        rAt	          j        |j	        d| j
        j        ¬¦  «         t	          j        |j        ¦  «         dS t          |t          ¦  «        rjt	          j        |j        t          j        |j        j        d         ¦  «                             d¦  «        ¦  «         t	          j        |j        ¦  «         dS dS )zInitialize the weightsrf   rg   r'   r&   N)r,   Ú_init_weightsrÛ   r_   ÚinitÚzeros_re   rË   rp   rn   rE   rq   r    Úcopy_r%   r=   r>   rš   r?   r)   )rD   ÚmodulerF   s     €rG   rý   z%ConvBertPreTrainedModel._init_weights£  s   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�oÑ.Ô.ð 	/ÝŒK˜œÑ$Ô$Ð$Ð$Ð$Ý˜Õ 2Ñ3Ô3ð 	/ÝŒL˜œ¨S°d´kÔ6SÐTÑTÔTÐTÝŒK˜œÑ$Ô$Ð$Ð$Ð$Ý˜Õ 2Ñ3Ô3ð 	/ÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÑhÔhÐhÝŒK˜Ô-Ñ.Ô.Ð.Ð.Ð.ð	/ð 	/rH   )rV   rW   rX   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingræ   r|   Ú_can_record_outputsr=   Úno_gradrý   r\   r]   s   @rG   rù   rù   ™  sy   ø€ € € € € € àÐÐÑØ"ÐØ&*Ð#à&Ø+ðð Ðð
 €U„]�_„_ð
/ð 
/ð 
/ð 
/ñ „_ð
/ð 
/ð 
/ð 
/ð 
/rH   rù   c                   ó|   ‡ — e Zd Zˆ fd„Z	 	 	 d	dej        dej        dz  dej        dz  dej        dz  def
d„Zˆ xZ	S )
ÚConvBertEncoderc                 óÔ   •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS © )ræ   )Ú.0rÈ   rE   s     €rG   ú
<listcomp>z,ConvBertEncoder.__init__.<locals>.<listcomp>µ  s!   ø€ Ð#cÐ#cÐ#c¸a¥M°&Ñ$9Ô$9Ð#cÐ#cÐ#crH   F)	r,   r-   rE   r   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayerÚgradient_checkpointingrC   s    `€rG   r-   zConvBertEncoder.__init__²  s`   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”]Ð#cÐ#cÐ#cÐ#cÅ5ÈÔIaÑCbÔCbÐ#cÑ#cÔ#cÑdÔdˆŒ
Ø&+ˆÔ#Ð#Ð#rH   Nru   r’   r“   rð   rK   c                 óP   — | j         D ]} |||f||dœ|¤Ž}Œt          |¬¦  «        S )N)r“   rð   )Úlast_hidden_state)r  r   )rD   ru   r’   r“   rð   rt   Úlayer_modules          rG   rU   zConvBertEncoder.forward¸  sf   € ð !œJð 	ð 	ˆLØ(˜LØØðð '<Ø'=ð	ð ð
 ðð ˆMˆMõ 2Ø+ð
ñ 
ô 
ð 	
rH   r÷   )
rV   rW   rX   r-   r=   rz   r[   r   rU   r\   r]   s   @rG   r  r  ±  s¤   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð 48Ø59Ø6:ð
ð 
à”|ð
ð Ô)¨DÑ0ð
ð  %œ|¨dÑ2ð	
ð
 !&¤¨tÑ 3ð
ð 
,ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
rH   r  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚConvBertPredictionHeadTransformc                 óV  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j        t          ¦  «        rt          |j                 | _
        n|j        | _
        t          j        |j        |j        ¬¦  «        | _        d S r»   )r,   r-   r   r†   r   r¼   rÛ   rÜ   rÝ   r
   Útransform_act_fnr7   r8   rC   s     €rG   r-   z(ConvBertPredictionHeadTransform.__init__Ï  s…   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ý�fÔ'­Ñ-Ô-ð 	6Ý$*¨6Ô+<Ô$=ˆDÔ!Ð!à$*Ô$5ˆDÔ!Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒˆˆrH   ru   rK   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rw   )r¼   r  r7   rà   s     rG   rU   z'ConvBertPredictionHeadTransform.forwardØ  s=   € ØŸ
š
 =Ñ1Ô1ˆØ×-Ò-¨mÑ<Ô<ˆØŸš }Ñ5Ô5ˆØÐrH   rÂ   r]   s   @rG   r  r  Î  sc   ø€ € € € € ðUð Uð Uð Uð Uð U¤\ð °e´lð ð ð ð ð ð ð ð rH   r  c                   ód   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej        dej        dz  dej        fd„Z	ˆ xZ
S )
ÚConvBertSequenceSummaryaÐ  
    Compute a single vector summary of a sequence hidden states.

    Args:
        config ([`ConvBertConfig`]):
            The config used by the model. Relevant arguments in the config class of the model are (refer to the actual
            config class of your model for the default values it uses):

            - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are:

                - `"last"` -- Take the last token hidden state (like XLNet)
                - `"first"` -- Take the first token hidden state (like Bert)
                - `"mean"` -- Take the mean of all tokens hidden states
                - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2)
                - `"attn"` -- Not implemented now, use multi-head attention

            - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction.
            - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes
              (otherwise to `config.hidden_size`).
            - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output,
              another string or `None` will add no activation.
            - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation.
            - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation.
    rE   c                 óV  •— t          ¦   «                              ¦   «          t          |dd¦  «        | _        | j        dk    rt          ‚t          j        ¦   «         | _        t          |d¦  «        rW|j	        rPt          |d¦  «        r|j
        r|j        dk    r|j        }n|j        }t          j        |j        |¦  «        | _        t          |dd ¦  «        }|rt          |¦  «        nt          j        ¦   «         | _        t          j        ¦   «         | _        t          |d¦  «        r)|j        dk    rt          j        |j        ¦  «        | _        t          j        ¦   «         | _        t          |d	¦  «        r+|j        dk    r"t          j        |j        ¦  «        | _        d S d S d S )
NÚsummary_typeÚlastÚattnÚsummary_use_projÚsummary_proj_to_labelsr   Úsummary_activationÚsummary_first_dropoutÚsummary_last_dropout)r,   r-   Úgetattrr  ÚNotImplementedErrorr   ÚIdentityÚsummaryrO   r!  r"  Ú
num_labelsr   r†   r   Ú
activationÚfirst_dropoutr$  r9   Úlast_dropoutr%  )rD   rE   Únum_classesÚactivation_stringrF   s       €rG   r-   z ConvBertSequenceSummary.__init__ú  sœ  ø€ Ý‰Œ×ÒÑÔÐå# F¨N¸FÑCÔCˆÔØÔ Ò&Ð&õ &Ð%å”{‘}”}ˆŒÝ�6Ð-Ñ.Ô.ð 	F°6Ô3Jð 	FÝ�vÐ7Ñ8Ô8ð 1¸VÔ=Zð 1Ð_eÔ_pÐstÒ_tÐ_tØ$Ô/��à$Ô0�Ýœ9 VÔ%7¸ÑEÔEˆDŒLå# FÐ,@À$ÑGÔGÐØIZÐ$m¥NÐ3DÑ$EÔ$EÐ$EÕ`bÔ`kÑ`mÔ`mˆŒåœ[™]œ]ˆÔÝ�6Ð2Ñ3Ô3ð 	J¸Ô8TÐWXÒ8XÐ8XÝ!#¤¨FÔ,HÑ!IÔ!IˆDÔåœK™MœMˆÔÝ�6Ð1Ñ2Ô2ð 	H°vÔ7RÐUVÒ7VÐ7VÝ "¤
¨6Ô+FÑ GÔ GˆDÔÐÐð	Hð 	HÐ7VÐ7VrH   Nru   Ú	cls_indexrK   c                 ó:  — | j         dk    r|dd…df         }�n-| j         dk    r|dd…df         }�n| j         dk    r|                     d¬¦  «        }nò| j         d	k    rÕ|€=t          j        |d
dd…dd…f         |j        d         dz
  t          j        ¬¦  «        }nl|                     d¦  «                             d¦  «        }|                     d|                     ¦   «         dz
  z  | 	                    d¦  «        fz   ¦  «        }| 
                    d|¦  «                             d¦  «        }n| j         dk    rt          ‚|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S )ak  
        Compute a single vector summary of a sequence hidden states.

        Args:
            hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`):
                The hidden states of the last layer.
            cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*):
                Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token.

        Returns:
            `torch.FloatTensor`: The summary of the sequence hidden states.
        r  Nr'   Úfirstr   rh   r   r•   r0  .r™   r*   )r'   r   )r  rh   r=   Ú	full_likerš   rB   r¡   r?   r–   rA   ÚgatherÚsqueezer'  r,  r)  r+  r-  )rD   ru   r0  rÆ   s       rG   rU   zConvBertSequenceSummary.forward  sª  € ð Ô Ò&Ð&Ø" 1 1 1 b 5Ô)ˆF‰FØÔ 'Ò)Ð)Ø" 1 1 1 a 4Ô(ˆF‰FØÔ &Ò(Ð(Ø"×'Ò'¨AÐ'Ñ.Ô.ˆFˆFØÔ +Ò-Ð-ØÐ Ý!œOØ! # r¨ r¨1¨1¨1 *Ô-Ø!Ô'¨Ô+¨aÑ/Ýœ*ðñ ô �	�	ð &×/Ò/°Ñ3Ô3×=Ò=¸bÑAÔA�	Ø%×,Ò,¨U°i·m²m±o´oÈÑ6IÑ-JÈm×N`ÒN`ÐacÑNdÔNdÐMfÑ-fÑgÔg�	à"×)Ò)¨"¨iÑ8Ô8×@Ò@ÀÑDÔDˆFˆFØÔ &Ò(Ð(Ý%Ð%à×#Ò# FÑ+Ô+ˆØ—’˜fÑ%Ô%ˆØ—’ Ñ(Ô(ˆØ×"Ò" 6Ñ*Ô*ˆàˆrH   rw   )rV   rW   rX   rY   r   r-   r=   r[   rZ   rU   r\   r]   s   @rG   r  r  à  s›   ø€ € € € € ðð ð2H˜~ð Hð Hð Hð Hð Hð Hð< VZð)ð )Ø"Ô.ð)Ø;@Ô;KÈdÑ;Rð)à	Ô	ð)ð )ð )ð )ð )ð )ð )ð )rH   r  c                   óæ   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zeee	 	 	 	 	 dde	j
        dz  de	j        dz  de	j
        dz  de	j
        dz  d	e	j        dz  d
ee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚConvBertModelc                 ó8  •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        |j        k    r$t          j        |j        |j        ¦  «        | _        t          |¦  «        | _
        || _        |                      ¦   «          d S rw   )r,   r-   r    rT   r0   r   r   r†   Úembeddings_projectr  ÚencoderrE   Ú	post_initrC   s     €rG   r-   zConvBertModel.__init__E  s�   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý,¨VÑ4Ô4ˆŒàÔ  FÔ$6Ò6Ð6Ý&(¤i°Ô0EÀvÔGYÑ&ZÔ&ZˆDÔ#å& vÑ.Ô.ˆŒØˆŒà�ŠÑÔÐÐÐrH   c                 ó   — | j         j        S rw   ©rT   r2   ©rD   s    rG   Úget_input_embeddingsz"ConvBertModel.get_input_embeddingsQ  s   € ØŒÔ.Ð.rH   c                 ó   — || j         _        d S rw   r=  )rD   r‰   s     rG   Úset_input_embeddingsz"ConvBertModel.set_input_embeddingsT  s   € Ø*/ˆŒÔ'Ð'Ð'rH   NrI   r’   r)   r%   rJ   rt   rK   c                 óØ  — |�|�t          d¦  «        ‚|�+|                      ||¦  «         |                     ¦   «         }n.|�|                     ¦   «         d d…         }nt          d¦  «        ‚|\  }}	|�|j        n|j        }
|€t	          j        ||
¬¦  «        }|€gt          | j        d¦  «        r1| j        j        d d …d |	…f         }| 	                    ||	¦  «        }|}n!t	          j
        |t          j        |
¬¦  «        }|                      ||||¬¦  «        }t          | d¦  «        r|                      |¦  «        }t          | j        ||¬	¦  «        } | j        |fd
|i|¤Ž}|S )NzDYou cannot specify both input_ids and inputs_embeds at the same timer'   z5You have to specify either input_ids or inputs_embeds)rN   r)   rM   )rI   r%   r)   rJ   r9  )rE   rJ   r’   r’   )r�   Ú%warn_if_padding_and_no_attention_maskrA   rN   r=   ÚonesrO   rT   r)   r?   r@   rB   r9  r   rE   r:  )rD   rI   r’   r)   r%   rJ   rt   rP   rÕ   rQ   rN   rR   rS   ru   Úencoder_outputss                  rG   rU   zConvBertModel.forwardW  sÇ  € ð Ð  ]Ð%>ÝÐcÑdÔdÐdØÐ"Ø×6Ò6°yÀ.ÑQÔQÐQØ#Ÿ.š.Ñ*Ô*ˆKˆKØÐ&Ø'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKˆKåÐTÑUÔUÐUà!,Ñˆ
�JØ%.Ð%:�Ô!Ð!ÀÔ@TˆàÐ!Ý"œZ¨¸FÐCÑCÔCˆNØÐ!Ý�t”Ð(8Ñ9Ô9ð [Ø*.¬/Ô*HÈÈÈÈKÈZÈKÈÔ*XÐ'Ø3J×3QÒ3QÐR\Ð^hÑ3iÔ3iÐ0Ø!A��å!&¤¨[ÅÄ
ÐSYÐ!ZÑ!ZÔ!Z�àŸšØ¨lÈ>Ðivð (ñ 
ô 
ˆõ �4Ð-Ñ.Ô.ð 	CØ ×3Ò3°MÑBÔBˆMå2Ø”;Ø'Ø)ð
ñ 
ô 
ˆð ?K¸d¼lØð?
ð ?
à)ð?
ð ð?
ð ?
ˆð ÐrH   )NNNNN)rV   rW   rX   r-   r?  rA  r   r   r   r=   rZ   r[   r   r   r   rU   r\   r]   s   @rG   r7  r7  C  s  ø€ € € € € ð
ð 
ð 
ð 
ð 
ð/ð /ð /ð0ð 0ð 0ð  ØØð .2Ø37Ø26Ø04Ø26ð3ð 3àÔ# dÑ*ð3ð Ô)¨DÑ0ð3ð Ô(¨4Ñ/ð	3ð
 Ô&¨Ñ-ð3ð Ô(¨4Ñ/ð3ð Ð+Ô,ð3ð 
,ð3ð 3ð 3ñ „^ñ „_ñ  Ôð3ð 3ð 3ð 3ð 3rH   r7  c                   óF   ‡ — e Zd ZdZˆ fd„Zdej        dej        fd„Zˆ xZS )ÚConvBertGeneratorPredictionszAPrediction module for the generator, made up of two dense layers.c                 ó  •— t          ¦   «                              ¦   «          t          d¦  «        | _        t	          j        |j        |j        ¬¦  «        | _        t	          j        |j	        |j        ¦  «        | _
        d S )NÚgelur#   )r,   r-   r   r+  r   r7   r0   r8   r†   r   r¼   rC   s     €rG   r-   z%ConvBertGeneratorPredictions.__init__“  sa   ø€ Ý‰Œ×ÒÑÔÐå(¨Ñ0Ô0ˆŒÝœ fÔ&;ÀÔAVÐWÑWÔWˆŒÝ”Y˜vÔ1°6Ô3HÑIÔIˆŒ
ˆ
ˆ
rH   Úgenerator_hidden_statesrK   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rw   )r¼   r+  r7   )rD   rJ  ru   s      rG   rU   z$ConvBertGeneratorPredictions.forwardš  s<   € ØŸ
š
Ð#:Ñ;Ô;ˆØŸš¨Ñ6Ô6ˆØŸš }Ñ5Ô5ˆàÐrH   )	rV   rW   rX   rY   r-   r=   r[   rU   r\   r]   s   @rG   rG  rG  �  sk   ø€ € € € € ØKÐKðJð Jð Jð Jð Jð¨uÔ/@ð ÀUÔEVð ð ð ð ð ð ð ð rH   rG  c                   óú   ‡ — e Zd ZddiZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 dde	j
        dz  de	j        dz  d	e	j
        dz  d
e	j
        dz  de	j        dz  de	j
        dz  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚConvBertForMaskedLMzgenerator_lm_head.weightz*convbert.embeddings.word_embeddings.weightc                 ó
  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¦  «        | _        t          j        |j        |j	        ¦  «        | _
        |                      ¦   «          d S rw   )r,   r-   r7  rú   rG  Úgenerator_predictionsr   r†   r0   r/   Úgenerator_lm_headr;  rC   s     €rG   r-   zConvBertForMaskedLM.__init__¦  sj   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å% fÑ-Ô-ˆŒÝ%AÀ&Ñ%IÔ%IˆÔ"å!#¤¨6Ô+@À&ÔBSÑ!TÔ!TˆÔà�ŠÑÔÐÐÐrH   c                 ó   — | j         S rw   ©rP  r>  s    rG   Úget_output_embeddingsz)ConvBertForMaskedLM.get_output_embeddings°  s   € ØÔ%Ð%rH   c                 ó   — || _         d S rw   rR  )rD   r2   s     rG   Úset_output_embeddingsz)ConvBertForMaskedLM.set_output_embeddings³  s   € Ø!0ˆÔÐÐrH   NrI   r’   r)   r%   rJ   Úlabelsrt   rK   c                 ón  —  | j         |f||||dœ|¤Ž}|d         }	|                      |	¦  «        }
|                      |
¦  «        }
d}|�Pt          j        ¦   «         } ||
                     d| j        j        ¦  «        |                     d¦  «        ¦  «        }t          ||
|j	        |j
        ¬¦  «        S )a¢  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        ©r’   r)   r%   rJ   r   Nr'   ©ÚlossÚlogitsru   rû   )rú   rO  rP  r   r   rœ   rE   r/   r   ru   rû   )rD   rI   r’   r)   r%   rJ   rV  rt   rJ  Úgenerator_sequence_outputÚprediction_scoresrZ  Úloss_fcts                rG   rU   zConvBertForMaskedLM.forward¶  sò   € ð$ GTÀdÄmØðG
à)Ø)Ø%Ø'ðG
ð G
ð ðG
ð G
Ðð %<¸AÔ$>Ð!à ×6Ò6Ð7PÑQÔQÐØ ×2Ò2Ð3DÑEÔEÐàˆàÐÝÔ*Ñ,Ô,ˆHØ�8Ð-×2Ò2°2°t´{Ô7MÑNÔNÐPV×P[ÒP[Ð\^ÑP_ÔP_Ñ`Ô`ˆDåØØ$Ø1Ô?Ø.Ô9ð	
ñ 
ô 
ð 	
rH   ©NNNNNN)rV   rW   rX   Ú_tied_weights_keysr-   rS  rU  r   r   r=   rZ   r[   r   r   r·   r   rU   r\   r]   s   @rG   rM  rM  ¢  s/  ø€ € € € € à4Ð6bÐcÐðð ð ð ð ð&ð &ð &ð1ð 1ð 1ð Øð .2Ø37Ø26Ø04Ø26Ø*.ð(
ð (
àÔ# dÑ*ð(
ð Ô)¨DÑ0ð(
ð Ô(¨4Ñ/ð	(
ð
 Ô&¨Ñ-ð(
ð Ô(¨4Ñ/ð(
ð Ô  4Ñ'ð(
ð Ð+Ô,ð(
ð 
�Ñ	ð(
ð (
ð (
ñ „^ñ Ôð(
ð (
ð (
ð (
ð (
rH   rM  c                   óF   ‡ — e Zd ZdZˆ fd„Zdej        dej        fd„Zˆ xZS )ÚConvBertClassificationHeadz-Head for sentence-level classification tasks.c                 óB  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        |j        �|j        n|j        }t          j        |¦  «        | _	        t          j        |j        |j
        ¦  «        | _        || _        d S rw   )r,   r-   r   r†   r   r¼   Úclassifier_dropoutr:   r9   r;   r*  Úout_projrE   ©rD   rE   rd  rF   s      €rG   r-   z#ConvBertClassificationHead.__init__æ  s†   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
à)/Ô)BÐ)NˆFÔ%Ð%ÐTZÔTnð 	õ ”zÐ"4Ñ5Ô5ˆŒÝœ	 &Ô"4°fÔ6GÑHÔHˆŒàˆŒˆˆrH   ru   rK   c                 ó
  — |d d …dd d …f         }|                       |¦  «        }|                      |¦  «        }t          | j        j                 |¦  «        }|                       |¦  «        }|                      |¦  «        }|S )Nr   )r;   r¼   r
   rE   rÜ   re  )rD   ru   rt   rx   s       rG   rU   z"ConvBertClassificationHead.forwardñ  st   € Ø˜!˜!˜!˜Q   ˜'Ô"ˆØ�LŠL˜‰OŒOˆØ�JŠJ�q‰MŒMˆÝ�4”;Ô)Ô*¨1Ñ-Ô-ˆØ�LŠL˜‰OŒOˆØ�MŠM˜!ÑÔˆØˆrH   ry   r]   s   @rG   rb  rb  ã  sd   ø€ € € € € Ø7Ð7ð	ð 	ð 	ð 	ð 	ð U¤\ð ÀÄð ð ð ð ð ð ð ð rH   rb  z 
    ConvBERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the
    pooled output) e.g. for GLUE tasks.
    )Úcustom_introc                   óæ   ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	e	e
         d
eez  fd„¦   «         ¦   «         Zˆ xZS )Ú!ConvBertForSequenceClassificationc                 óè   •— t          ¦   «                              |¦  «         |j        | _        || _        t	          |¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S rw   )	r,   r-   r*  rE   r7  rú   rb  Ú
classifierr;  rC   s     €rG   r-   z*ConvBertForSequenceClassification.__init__  sb   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒØˆŒÝ% fÑ-Ô-ˆŒÝ4°VÑ<Ô<ˆŒð 	�ŠÑÔÐÐÐrH   NrI   r’   r)   r%   rJ   rV  rt   rK   c                 óZ  —  | j         |f||||dœ|¤Ž}|d         }	|                      |	¦  «        }
d}|��Z| j        j        €f| j        dk    rd| j        _        nN| j        dk    r7|j        t          j        k    s|j        t          j        k    rd| j        _        nd| j        _        | j        j        dk    rWt          ¦   «         }| j        dk    r1 ||
 
                    ¦   «         | 
                    ¦   «         ¦  «        }nŽ ||
|¦  «        }n�| j        j        dk    rGt          ¦   «         } ||
                     d| j        ¦  «        |                     d¦  «        ¦  «        }n*| j        j        dk    rt          ¦   «         } ||
|¦  «        }t          ||
|j        |j        ¬	¦  «        S )
a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        rX  r   Nr   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr'   rY  )rú   rl  rE   Úproblem_typer*  r+   r=   rB   rŽ   r   r5  r   rœ   r   r   ru   rû   ©rD   rI   r’   r)   r%   rJ   rV  rt   ÚoutputsÚsequence_outputr[  rZ  r^  s                rG   rU   z)ConvBertForSequenceClassification.forward  sÈ  € ð$ 7D°d´mØð7
à)Ø)Ø%Ø'ð7
ð 7
ð ð7
ð 7
ˆð " !œ*ˆØ—’ Ñ1Ô1ˆàˆØÑØŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ñ @Ô @À&Ç+Â+ÈbÁ/Ä/ÑRÔR��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�å'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rH   r_  )rV   rW   rX   r-   r   r   r=   rZ   r[   r   r   r·   r   rU   r\   r]   s   @rG   rj  rj  û  s  ø€ € € € € ðð ð ð ð ð Øð .2Ø37Ø26Ø04Ø26Ø*.ð8
ð 8
àÔ# dÑ*ð8
ð Ô)¨DÑ0ð8
ð Ô(¨4Ñ/ð	8
ð
 Ô&¨Ñ-ð8
ð Ô(¨4Ñ/ð8
ð Ô  4Ñ'ð8
ð Ð+Ô,ð8
ð 
Ð)Ñ	)ð8
ð 8
ð 8
ñ „^ñ Ôð8
ð 8
ð 8
ð 8
ð 8
rH   rj  c                   óæ   ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	e	e
         d
eez  fd„¦   «         ¦   «         Zˆ xZS )ÚConvBertForMultipleChoicec                 ó   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¦  «        | _        t          j        |j        d¦  «        | _	        |  
                    ¦   «          d S )Nr   )r,   r-   r7  rú   r  Úsequence_summaryr   r†   r   rl  r;  rC   s     €rG   r-   z"ConvBertForMultipleChoice.__init__K  sh   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å% fÑ-Ô-ˆŒÝ 7¸Ñ ?Ô ?ˆÔÝœ) FÔ$6¸Ñ:Ô:ˆŒð 	�ŠÑÔÐÐÐrH   NrI   r’   r)   r%   rJ   rV  rt   rK   c                 óP  — |�|j         d         n|j         d         }|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�=|                     d|                     d¦  «        |                     d¦  «        ¦  «        nd} | j        |f||||dœ|¤Ž}	|	d         }
|                      |
¦  «        }|                      |¦  «        }|                     d|¦  «        }d}|�t          ¦   «         } |||¦  «        }t          |||	j        |	j	        ¬¦  «        S )a\  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:


            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   r'   r™   rX  r   rY  )
rš   rœ   rA   rú   rx  rl  r   r   ru   rû   )rD   rI   r’   r)   r%   rJ   rV  rt   Únum_choicesrs  rt  Úpooled_outputr[  Úreshaped_logitsrZ  r^  s                   rG   rU   z!ConvBertForMultipleChoice.forwardU  sé  € ðV -6Ð,A�i”o aÔ(Ð(À}ÔGZÐ[\ÔG]ˆà>GÐ>S�I—N’N 2 y§~¢~°bÑ'9Ô'9Ñ:Ô:Ð:ÐY]ˆ	ØM[ÐMg˜×,Ò,¨R°×1DÒ1DÀRÑ1HÔ1HÑIÔIÐIÐmqˆØM[ÐMg˜×,Ò,¨R°×1DÒ1DÀRÑ1HÔ1HÑIÔIÐIÐmqˆØGSÐG_�|×(Ò(¨¨\×->Ò->¸rÑ-BÔ-BÑCÔCÐCÐeiˆð Ð(ð ×Ò˜r =×#5Ò#5°bÑ#9Ô#9¸=×;MÒ;MÈbÑ;QÔ;QÑRÔRÐRàð 	ð 7D°d´mØð7
à)Ø)Ø%Ø'ð7
ð 7
ð ð7
ð 7
ˆð " !œ*ˆà×-Ò-¨oÑ>Ô>ˆØ—’ Ñ/Ô/ˆØ Ÿ+š+ b¨+Ñ6Ô6ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜O¨VÑ4Ô4ˆDå(ØØ"Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rH   r_  )rV   rW   rX   r-   r   r   r=   rZ   r[   r   r   r·   r   rU   r\   r]   s   @rG   rv  rv  I  s  ø€ € € € € ðð ð ð ð ð Øð .2Ø37Ø26Ø04Ø26Ø*.ðN
ð N
àÔ# dÑ*ðN
ð Ô)¨DÑ0ðN
ð Ô(¨4Ñ/ð	N
ð
 Ô&¨Ñ-ðN
ð Ô(¨4Ñ/ðN
ð Ô  4Ñ'ðN
ð Ð+Ô,ðN
ð 
Ð*Ñ	*ðN
ð N
ð N
ñ „^ñ ÔðN
ð N
ð N
ð N
ð N
rH   rv  c                   óæ   ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	e	e
         d
eez  fd„¦   «         ¦   «         Zˆ xZS )ÚConvBertForTokenClassificationc                 óV  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        |j        �|j        n|j        }t          j        |¦  «        | _	        t          j
        |j        |j        ¦  «        | _        |                      ¦   «          d S rw   )r,   r-   r*  r7  rú   rd  r:   r   r9   r;   r†   r   rl  r;  rf  s      €rG   r-   z'ConvBertForTokenClassification.__init__ª  s•   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå% fÑ-Ô-ˆŒà)/Ô)BÐ)NˆFÔ%Ð%ÐTZÔTnð 	õ ”zÐ"4Ñ5Ô5ˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐrH   NrI   r’   r)   r%   rJ   rV  rt   rK   c                 óZ  —  | j         |f||||dœ|¤Ž}|d         }	|                      |	¦  «        }	|                      |	¦  «        }
d}|�Ft          ¦   «         } ||
                     d| j        ¦  «        |                     d¦  «        ¦  «        }t          ||
|j        |j        ¬¦  «        S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        rX  r   Nr'   rY  )	rú   r;   rl  r   rœ   r*  r   ru   rû   rr  s                rG   rU   z&ConvBertForTokenClassification.forward¸  sÒ   € ð  7D°d´mØð7
à)Ø)Ø%Ø'ð7
ð 7
ð ð7
ð 7
ˆð " !œ*ˆàŸ,š, Ñ7Ô7ˆØ—’ Ñ1Ô1ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬OÑ<Ô<¸f¿kºkÈ"¹o¼oÑNÔNˆDå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rH   r_  )rV   rW   rX   r-   r   r   r=   rZ   r[   r   r   r·   r   rU   r\   r]   s   @rG   r~  r~  ¨  s  ø€ € € € € ðð ð ð ð ð Øð .2Ø37Ø26Ø04Ø26Ø*.ð&
ð &
àÔ# dÑ*ð&
ð Ô)¨DÑ0ð&
ð Ô(¨4Ñ/ð	&
ð
 Ô&¨Ñ-ð&
ð Ô(¨4Ñ/ð&
ð Ô  4Ñ'ð&
ð Ð+Ô,ð&
ð 
Ð&Ñ	&ð&
ð &
ð &
ñ „^ñ Ôð&
ð &
ð &
ð &
ð &
rH   r~  c                   óö   ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
e	e
         defd„¦   «         ¦   «         Zˆ xZS )ÚConvBertForQuestionAnsweringc                 óú   •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        |j        ¦  «        | _        |  	                    ¦   «          d S rw   )
r,   r-   r*  r7  rú   r   r†   r   Ú
qa_outputsr;  rC   s     €rG   r-   z%ConvBertForQuestionAnswering.__init__å  se   ø€ Ý‰Œ×Ò˜Ñ Ô Ð à Ô+ˆŒÝ% fÑ-Ô-ˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐrH   NrI   r’   r)   r%   rJ   Ústart_positionsÚend_positionsrt   rK   c                 óD  —  | j         |f||||dœ|¤Ž}	|	d         }
|                      |
¦  «        }|                     dd¬¦  «        \  }}|                     d¦  «                             ¦   «         }|                     d¦  «                             ¦   «         }d }|�ç|�åt          |                     ¦   «         ¦  «        dk    r|                     d¦  «        }t          |                     ¦   «         ¦  «        dk    r|                     d¦  «        }|                     d¦  «        }|                     d|¦  «        }|                     d|¦  «        }t          |¬¦  «        } |||¦  «        } |||¦  «        }||z   dz  }t          ||||	j
        |	j        ¬¦  «        S )	NrX  r   r   r'   r•   )Úignore_indexra   )rZ  Ústart_logitsÚ
end_logitsru   rû   )rú   r„  Úsplitr5  r    ÚlenrA   Úclampr   r   ru   rû   )rD   rI   r’   r)   r%   rJ   r…  r†  rt   rs  rt  r[  r‰  rŠ  Ú
total_lossÚignored_indexr^  Ú
start_lossÚend_losss                      rG   rU   z$ConvBertForQuestionAnswering.forwardï  sÍ  € ð 7D°d´mØð7
à)Ø)Ø%Ø'ð7
ð 7
ð ð7
ð 7
ˆð " !œ*ˆà—’ Ñ1Ô1ˆØ#)§<¢<°°r <Ñ#:Ô#:Ñ ˆ�jØ#×+Ò+¨BÑ/Ô/×:Ò:Ñ<Ô<ˆØ×'Ò'¨Ñ+Ô+×6Ò6Ñ8Ô8ˆ
àˆ
ØÐ&¨=Ð+Då�?×'Ò'Ñ)Ô)Ñ*Ô*¨QÒ.Ð.Ø"1×"9Ò"9¸"Ñ"=Ô"=�Ý�=×%Ò%Ñ'Ô'Ñ(Ô(¨1Ò,Ð,Ø -× 5Ò 5°bÑ 9Ô 9�à(×-Ò-¨aÑ0Ô0ˆMØ-×3Ò3°A°}ÑEÔEˆOØ)×/Ò/°°=ÑAÔAˆMå'°]ÐCÑCÔCˆHØ!˜ ,°Ñ@Ô@ˆJØ�x 
¨MÑ:Ô:ˆHØ$ xÑ/°1Ñ4ˆJå+ØØ%Ø!Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
rH   )NNNNNNN)rV   rW   rX   r-   r   r   r=   rZ   r[   r   r   r   rU   r\   r]   s   @rG   r‚  r‚  ã  s  ø€ € € € € ðð ð ð ð ð Øð .2Ø37Ø26Ø04Ø26Ø37Ø15ð2
ð 2
àÔ# dÑ*ð2
ð Ô)¨DÑ0ð2
ð Ô(¨4Ñ/ð	2
ð
 Ô&¨Ñ-ð2
ð Ô(¨4Ñ/ð2
ð Ô)¨DÑ0ð2
ð Ô'¨$Ñ.ð2
ð Ð+Ô,ð2
ð 
&ð2
ð 2
ð 2
ñ „^ñ Ôð2
ð 2
ð 2
ð 2
ð 2
rH   r‚  )rM  rv  r‚  rj  r~  ræ   r7  rù   )FrY   r¤   Úcollections.abcr   r=   r   Útorch.nnr   r   r   Ú r	   rþ   Úactivationsr
   r   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úconfiguration_convbertr   Ú
get_loggerrV   ÚloggerÚModuler    r_   r|   r¹   rÄ   rË   r×   râ   ræ   rù   r  r  r  r7  rG  rM  rb  rj  rv  r~  r‚  Ú__all__r  rH   rG   ú<module>r¤     s  ðð Ð à €€€Ø $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð .Ð -Ð -Ð -Ð -Ð -Ø &Ð &Ð &Ð &Ð &Ð &Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6ðð ð ð ð ð ð ð ð ð ð ð ð 8Ð 7Ð 7Ð 7Ð 7Ð 7Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø 2Ð 2Ð 2Ð 2Ð 2Ð 2ð 
ˆÔ	˜HÑ	%Ô	%€ð7ð 7ð 7ð 7ð 7˜œñ 7ô 7ð 7ðtð ð ð ð �b”iñ ô ð ð4w.ð w.ð w.ð w.ð w.˜BœIñ w.ô w.ð w.ðtð ð ð ð ˜œñ ô ð ð ð  ð  ð  ð  ˜œ	ñ  ô  ð  ð.ð ð ð ð ˜œñ ô ð ð,ð ð ð ð ˜2œ9ñ ô ð ð(ð ð ð ð �R”Yñ ô ð ð&2ð 2ð 2ð 2ð 2Ð.ñ 2ô 2ð 2ðj ð/ð /ð /ð /ð /˜oñ /ô /ñ „ð/ð.
ð 
ð 
ð 
ð 
�b”iñ 
ô 
ð 
ð:ð ð ð ð  b¤iñ ô ð ð$`ð `ð `ð `ð `˜bœiñ `ô `ð `ðF ðIð Ið Ið Ið IÐ+ñ Iô Iñ „ðIðXð ð ð ð  2¤9ñ ô ð ð$ ð=
ð =
ð =
ð =
ð =
Ð1ñ =
ô =
ñ „ð=
ð@ð ð ð ð  ¤ñ ô ð ð0 €ððñ ô ðE
ð E
ð E
ð E
ð E
Ð(?ñ E
ô E
ñô ðE
ðP ð[
ð [
ð [
ð [
ð [
Ð 7ñ [
ô [
ñ „ð[
ð| ð7
ð 7
ð 7
ð 7
ð 7
Ð%<ñ 7
ô 7
ñ „ð7
ðt ð?
ð ?
ð ?
ð ?
ð ?
Ð#:ñ ?
ô ?
ñ „ð?
ðD	ð 	ð 	€€€rH   