§
    ‚Štj]½  ã                   óä  — d Z ddlZddlZddlmZ ddlmZmZmZ ddlm	Z
 ddlmZ ddlmZ dd	lmZmZmZmZmZmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZm Z m!Z!m"Z"m#Z#m$Z$  ej%        e&¦  «        Z' G d„ dej(        ¦  «        Z) G d„ dej(        ¦  «        Z* G d„ dej(        ¦  «        Z+ G d„ dej(        ¦  «        Z, G d„ dej(        ¦  «        Z- G d„ dej(        ¦  «        Z. G d„ dej(        ¦  «        Z/ G d„ dej(        ¦  «        Z0 G d„ d ej(        ¦  «        Z1e G d!„ d"e¦  «        ¦   «         Z2e G d#„ d$e2¦  «        ¦   «         Z3e G d%„ d&e2¦  «        ¦   «         Z4 G d'„ d(ej(        ¦  «        Z5 ed)¬*¦  «         G d+„ d,e2¦  «        ¦   «         Z6e G d-„ d.e2¦  «        ¦   «         Z7e G d/„ d0e2¦  «        ¦   «         Z8 G d1„ d2ej(        ¦  «        Z9e G d3„ d4e2¦  «        ¦   «         Z:d7d5„Z;g d6¢Z<dS )8zPyTorch I-BERT model.é    N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)Úgelu)Úcreate_bidirectional_mask)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)Úauto_docstringÚloggingé   )ÚIBertConfig)ÚIntGELUÚIntLayerNormÚ
IntSoftmaxÚQuantActÚQuantEmbeddingÚQuantLinearc                   ó2   ‡ — e Zd ZdZˆ fd„Z	 dd„Zd„ Zˆ xZS )ÚIBertEmbeddingszV
    Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
    c                 óÂ  •— t          ¦   «                              ¦   «          |j        | _        d| _        d| _        d| _        d| _        d| _        t          |j	        |j
        |j        | j        | j        ¬¦  «        | _        t          |j        |j
        | j        | j        ¬¦  «        | _        |                      dt!          j        |j        ¦  «                             d¦  «        d	¬
¦  «         |j        | _        t          |j        |j
        | j        | j        | j        ¬¦  «        | _        t-          | j        | j        ¬¦  «        | _        t-          | j        | j        ¬¦  «        | _        t3          |j
        |j        | j        | j        |j        ¬¦  «        | _        t-          | j        | j        ¬¦  «        | _        t=          j        |j         ¦  «        | _!        d S )Né   é   é   é    )Úpadding_idxÚ
weight_bitÚ
quant_mode)r%   r&   Úposition_ids©r   éÿÿÿÿF)Ú
persistent©r&   ©ÚepsÚ
output_bitr&   Úforce_dequant)"ÚsuperÚ__init__r&   Úembedding_bitÚembedding_act_bitÚact_bitÚln_input_bitÚln_output_bitr   Ú
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚregister_bufferÚtorchÚarangeÚmax_position_embeddingsÚexpandr$   Úposition_embeddingsr   Úembeddings_act1Úembeddings_act2r   Úlayer_norm_epsr/   Ú	LayerNormÚoutput_activationr   ÚDropoutÚhidden_dropout_probÚdropout©ÚselfÚconfigÚ	__class__s     €úf/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/ibert/modeling_ibert.pyr1   zIBertEmbeddings.__init__3   sÍ  ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒØˆÔØ!#ˆÔØˆŒØˆÔØˆÔå-ØÔØÔØÔ+ØÔ)Ø”ð 
ñ  
ô  
ˆÔõ &4ØÔ" FÔ$6À4ÔCUÐbfÔbqð&
ñ &
ô &
ˆÔ"ð
 	×ÒØ�EœL¨Ô)GÑHÔH×OÒOÐPWÑXÔXÐejð 	ñ 	
ô 	
ð 	
ð
 "Ô.ˆÔÝ#1ØÔ*ØÔØÔ(ØÔ)Ø”ð$
ñ $
ô $
ˆÔ õ  (¨Ô(>È4Ì?Ð[Ñ[Ô[ˆÔÝ'¨Ô(>È4Ì?Ð[Ñ[Ô[ˆÔå%ØÔØÔ%ØÔ)Ø”Ø Ô.ð
ñ 
ô 
ˆŒõ "*¨$¬,À4Ä?Ð!SÑ!SÔ!SˆÔÝ”z &Ô"<Ñ=Ô=ˆŒˆˆó    Nr   c                 óò  — |€F|�/t          || j        |¦  «                             |j        ¦  «        }n|                      |¦  «        }|�|                     ¦   «         }n|                     ¦   «         d d…         }|€+t          j        |t          j        | j	        j        ¬¦  «        }|€|  
                    |¦  «        \  }}nd }|                      |¦  «        \  }}	|                      ||||	¬¦  «        \  }
}|                      |¦  «        \  }}|                      |
|||¬¦  «        \  }
}|                      |
|¦  «        \  }
}|                      |
¦  «        }
|                      |
|¦  «        \  }
}|
|fS )Nr)   ©ÚdtypeÚdevice©ÚidentityÚidentity_scaling_factor)Ú"create_position_ids_from_input_idsr$   ÚtorT   Ú&create_position_ids_from_inputs_embedsÚsizer>   ÚzerosÚlongr'   r:   r<   rC   rB   rF   rJ   rG   )rL   Ú	input_idsÚtoken_type_idsr'   Úinputs_embedsÚpast_key_values_lengthÚinput_shapeÚinputs_embeds_scaling_factorr<   Ú$token_type_embeddings_scaling_factorÚ
embeddingsÚembeddings_scaling_factorrB   Ú"position_embeddings_scaling_factors                 rO   ÚforwardzIBertEmbeddings.forwardd   sº  € ð ÐØÐ$åAØ˜tÔ/Ð1Gñ ô  ç’"�YÔ%Ñ&Ô&ð �ð  $×JÒJÈ=ÑYÔY�àÐ Ø#Ÿ.š.Ñ*Ô*ˆKˆKà'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKàÐ!Ý"œ[¨½E¼JÈtÔO`ÔOgÐhÑhÔhˆNàÐ Ø:>×:NÒ:NÈyÑ:YÔ:YÑ7ˆMÐ7Ð7à+/Ð(ØFJ×F`ÒF`ÐaoÑFpÔFpÑCÐÐCà04×0DÒ0DØØ(Ø*Ø$Hð	 1Eñ 1
ô 1
Ñ-ˆ
Ð-ð CG×BZÒBZÐ[gÑBhÔBhÑ?ÐÐ?Ø04×0DÒ0DØØ%Ø(Ø$Fð	 1Eñ 1
ô 1
Ñ-ˆ
Ð-ð 15·²¸zÐKdÑ0eÔ0eÑ-ˆ
Ð-Ø—\’\ *Ñ-Ô-ˆ
Ø04×0FÒ0FÀzÐSlÑ0mÔ0mÑ-ˆ
Ð-ØÐ4Ð4Ð4rP   c                 ó  — |                      ¦   «         dd…         }|d         }t          j        | j        dz   || j        z   dz   t          j        |j        ¬¦  «        }|                     d¦  «                             |¦  «        S )z×
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nr)   r   rR   r   )r[   r>   r?   r$   r]   rT   Ú	unsqueezerA   )rL   r`   rb   Úsequence_lengthr'   s        rO   rZ   z6IBertEmbeddings.create_position_ids_from_inputs_embeds’   s‡   € ð $×(Ò(Ñ*Ô*¨3¨B¨3Ô/ˆØ% aœ.ˆå”|ØÔ˜qÑ  /°DÔ4DÑ"DÀqÑ"HÕPUÔPZÐcpÔcwð
ñ 
ô 
ˆð ×%Ò% aÑ(Ô(×/Ò/°Ñ<Ô<Ð<rP   )NNNNr   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r1   rh   rZ   Ú__classcell__©rN   s   @rO   r   r   .   sn   ø€ € € € € ðð ð/>ð />ð />ð />ð />ðd rsð,5ð ,5ð ,5ð ,5ð\=ð =ð =ð =ð =ð =ð =rP   r   c                   ó*   ‡ — e Zd Zˆ fd„Z	 	 dd„Zˆ xZS )ÚIBertSelfAttentionc           	      óR  •— t          ¦   «                              ¦   «          |j        |j        z  dk    r0t	          |d¦  «        s t          d|j        › d|j        › d�¦  «        ‚|j        | _        d| _        d| _        d| _	        |j        | _        t          |j        |j        z  ¦  «        | _        | j        | j        z  | _        t          |j        | j        d| j        | j        | j        d¬	¦  «        | _        t          |j        | j        d| j        | j        | j        d¬	¦  «        | _        t          |j        | j        d| j        | j        | j        d¬	¦  «        | _        t#          | j	        | j        ¬
¦  «        | _        t#          | j	        | j        ¬
¦  «        | _        t#          | j	        | j        ¬
¦  «        | _        t#          | j	        | j        ¬
¦  «        | _        t-          j        |j        ¦  «        | _        t5          | j	        | j        |j        ¬¦  «        | _        d S )Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)r    r#   T©Úbiasr%   Úbias_bitr&   Úper_channelr+   ©r&   r/   )r0   r1   r8   Únum_attention_headsÚhasattrÚ
ValueErrorr&   r%   ry   r4   ÚintÚattention_head_sizeÚall_head_sizer   ÚqueryÚkeyÚvaluer   Úquery_activationÚkey_activationÚvalue_activationrG   r   rH   Úattention_probs_dropout_probrJ   r   r/   ÚsoftmaxrK   s     €rO   r1   zIBertSelfAttention.__init__¥   s  ø€ Ý‰Œ×ÒÑÔÐØÔ Ô :Ñ:¸aÒ?Ð?ÍÐPVÐXhÑHiÔHiÐ?Ýð8 FÔ$6ð 8ð 8Ø Ô4ð8ð 8ð 8ñô ð ð !Ô+ˆŒØˆŒØˆŒØˆŒà#)Ô#=ˆÔ Ý#& vÔ'9¸FÔ<VÑ'VÑ#WÔ#WˆÔ Ø!Ô5¸Ô8PÑPˆÔõ !ØÔØÔØØ”Ø”]Ø”Øð
ñ 
ô 
ˆŒ
õ ØÔØÔØØ”Ø”]Ø”Øð
ñ 
ô 
ˆŒõ !ØÔØÔØØ”Ø”]Ø”Øð
ñ 
ô 
ˆŒ
õ !)¨¬À$Ä/Ð RÑ RÔ RˆÔÝ& t¤|ÀÄÐPÑPÔPˆÔÝ (¨¬À$Ä/Ð RÑ RÔ RˆÔÝ!)¨$¬,À4Ä?Ð!SÑ!SÔ!SˆÔå”z &Ô"EÑFÔFˆŒå! $¤,¸4¼?ÐZ`ÔZnÐoÑoÔoˆŒˆˆrP   NFc                 óÈ  — |                       ||¦  «        \  }}|                      ||¦  «        \  }}|                      ||¦  «        \  }	}
|                      ||¦  «        \  }}|                      ||¦  «        \  }}|                      |	|
¦  «        \  }}|j        d d…         }g |¢d‘| j        ‘R }|                     |¦  «         	                    dd¦  «        }|                     |¦  «         	                    dd¦  «        }|                     |¦  «         	                    dd¦  «        }t          j        || 	                    dd¦  «        ¦  «        }t          j        | j        ¦  «        }||z  }| j        r	||z  |z  }nd }|�||z   }|                      ||¦  «        \  }}|                      |¦  «        }t          j        ||¦  «        }|�||z  }nd }|                     dddd¦  «                             ¦   «         }|                     ¦   «         d d…         | j        fz   } |j        |Ž }|                      ||¦  «        \  }}|r||fn|f}|r||fn|f}||fS )Nr)   r   é   éþÿÿÿr   r   )r‚   rƒ   r„   r…   r†   r‡   Úshaper€   ÚviewÚ	transposer>   ÚmatmulÚmathÚsqrtr&   r‰   rJ   ÚpermuteÚ
contiguousr[   r�   rG   )rL   Úhidden_statesÚhidden_states_scaling_factorÚattention_maskÚoutput_attentionsÚmixed_query_layerÚ mixed_query_layer_scaling_factorÚmixed_key_layerÚmixed_key_layer_scaling_factorÚmixed_value_layerÚ mixed_value_layer_scaling_factorÚquery_layerÚquery_layer_scaling_factorÚ	key_layerÚkey_layer_scaling_factorÚvalue_layerÚvalue_layer_scaling_factorrb   Úhidden_shapeÚattention_scoresÚscaleÚattention_scores_scaling_factorÚattention_probsÚattention_probs_scaling_factorÚcontext_layerÚcontext_layer_scaling_factorÚnew_context_layer_shapeÚoutputsÚoutput_scaling_factors                                rO   rh   zIBertSelfAttention.forwardÜ   sé  € ð ?C¿jºjÈÐXtÑ>uÔ>uÑ;ÐÐ;Ø:>¿(º(À=ÐRnÑ:oÔ:oÑ7ˆÐ7Ø>B¿jºjÈÐXtÑ>uÔ>uÑ;ÐÐ;ð 37×2GÒ2GØÐ?ñ3
ô 3
Ñ/ˆÐ/ð /3×.AÒ.AÀ/ÐSqÑ.rÔ.rÑ+ˆ	Ð+Ø26×2GÒ2GØÐ?ñ3
ô 3
Ñ/ˆÐ/ð
 $Ô)¨#¨2¨#Ô.ˆØC˜ÐC bÐC¨$Ô*BÐCÐCˆØ!×&Ò& |Ñ4Ô4×>Ò>¸qÀ!ÑDÔDˆØ—N’N <Ñ0Ô0×:Ò:¸1¸aÑ@Ô@ˆ	Ø!×&Ò& |Ñ4Ô4×>Ò>¸qÀ!ÑDÔDˆõ !œ<¨°Y×5HÒ5HÈÈRÑ5PÔ5PÑQÔQÐÝ”	˜$Ô2Ñ3Ô3ˆØ+¨eÑ3ÐØŒ?ð 	3Ø.HÐKcÑ.cÐfkÑ.kÐ+Ð+à.2Ð+àÐ%à/°.Ñ@Ðð ;?¿,º,ØÐ=ñ;
ô ;
Ñ7ˆÐ7ð Ÿ,š, Ñ7Ô7ˆåœ _°kÑBÔBˆØ)Ð5Ø+IÐLfÑ+fÐ(Ð(à+/Ð(à%×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔFˆØ"/×"4Ò"4Ñ"6Ô"6°s¸°sÔ";¸tÔ?QÐ>SÑ"SÐØ*˜Ô*Ð,CÐDˆð 7;×6LÒ6LØÐ7ñ7
ô 7
Ñ3ˆÐ3ð 7HÐ]�= /Ð2Ð2ÈmÐM]ˆð !ð1Ð)Ð+IÐJÐJà.Ð0ð 	ð Ð-Ð-Ð-rP   ©NF©rl   rm   rn   r1   rh   rp   rq   s   @rO   rs   rs   ¤   sb   ø€ € € € € ð5pð 5pð 5pð 5pð 5pðv ØðH.ð H.ð H.ð H.ð H.ð H.ð H.ð H.rP   rs   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚIBertSelfOutputc           	      ó.  •— t          ¦   «                              ¦   «          |j        | _        d| _        d| _        d| _        d| _        d| _        t          |j	        |j	        d| j        | j        | j        d¬¦  «        | _
        t          | j        | j        ¬¦  «        | _        t          |j	        |j        | j        | j        |j        ¬¦  «        | _        t          | j        | j        ¬¦  «        | _        t%          j        |j        ¦  «        | _        d S ©Nr    r#   r"   Trw   r+   r,   )r0   r1   r&   r4   r%   ry   r5   r6   r   r8   Údenser   Úln_input_actr   rE   r/   rF   rG   r   rH   rI   rJ   rK   s     €rO   r1   zIBertSelfOutput.__init__(  s  ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒØˆŒØˆŒØˆŒØˆÔØˆÔå ØÔØÔØØ”Ø”]Ø”Øð
ñ 
ô 
ˆŒ
õ % TÔ%6À4Ä?ÐSÑSÔSˆÔÝ%ØÔØÔ%ØÔ)Ø”Ø Ô.ð
ñ 
ô 
ˆŒõ "*¨$¬,À4Ä?Ð!SÑ!SÔ!SˆÔÝ”z &Ô"<Ñ=Ô=ˆŒˆˆrP   c                 ó  — |                       ||¦  «        \  }}|                      |¦  «        }|                      ||||¬¦  «        \  }}|                      ||¦  «        \  }}|                      ||¦  «        \  }}||fS ©NrU   ©r¶   rJ   r·   rF   rG   ©rL   r•   r–   Úinput_tensorÚinput_tensor_scaling_factors        rO   rh   zIBertSelfOutput.forwardE  ó¥   € Ø6:·j²jÀÐPlÑ6mÔ6mÑ3ˆÐ3ØŸš ]Ñ3Ô3ˆØ6:×6GÒ6GØØ(Ø!Ø$?ð	 7Hñ 7
ô 7
Ñ3ˆÐ3ð 7;·n²nÀ]ÐTpÑ6qÔ6qÑ3ˆÐ3à6:×6LÒ6LØÐ7ñ7
ô 7
Ñ3ˆÐ3ð Ð:Ð:Ð:rP   r±   rq   s   @rO   r³   r³   '  óG   ø€ € € € € ð>ð >ð >ð >ð >ð:;ð ;ð ;ð ;ð ;ð ;ð ;rP   r³   c                   ó*   ‡ — e Zd Zˆ fd„Z	 	 dd„Zˆ xZS )ÚIBertAttentionc                 ó°   •— t          ¦   «                              ¦   «          |j        | _        t          |¦  «        | _        t          |¦  «        | _        d S ©N)r0   r1   r&   rs   rL   r³   ÚoutputrK   s     €rO   r1   zIBertAttention.__init__W  sE   ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒÝ& vÑ.Ô.ˆŒ	Ý% fÑ-Ô-ˆŒˆˆrP   NFc                 óÆ   — |                       ||||¦  «        \  }}|                      |d         |d         ||¦  «        \  }}|f|dd …         z   }	|f|dd …         z   }
|	|
fS )Nr   r   )rL   rÄ   )rL   r•   r–   r—   r˜   Úself_outputsÚself_outputs_scaling_factorÚattention_outputÚattention_output_scaling_factorr®   Úoutputs_scaling_factors              rO   rh   zIBertAttention.forward]  s™   € ð 59·I²IØØ(ØØñ	5
ô 5
Ñ1ˆÐ1ð =A¿KºKØ˜ŒOÐ8¸Ô;¸]ÐLhñ=
ô =
Ñ9ÐÐ9ð $Ð%¨°Q°R°RÔ(8Ñ8ˆØ"AÐ!CÐFaÐbcÐbdÐbdÔFeÑ!eÐØÐ.Ð.Ð.rP   r°   r±   rq   s   @rO   rÁ   rÁ   V  sT   ø€ € € € € ð.ð .ð .ð .ð .ð Øð/ð /ð /ð /ð /ð /ð /ð /rP   rÁ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚIBertIntermediatec           	      ó¦  •— t          ¦   «                              ¦   «          |j        | _        d| _        d| _        d| _        t          |j        |j        d| j        | j        | j        d¬¦  «        | _	        |j
        dk    rt          d¦  «        ‚t          | j        |j        ¬¦  «        | _        t          | j        | j        ¬¦  «        | _        d S )	Nr    r#   Trw   r	   z3I-BERT only supports 'gelu' for `config.hidden_act`r{   r+   )r0   r1   r&   r4   r%   ry   r   r8   Úintermediate_sizer¶   Ú
hidden_actr~   r   r/   Úintermediate_act_fnr   rG   rK   s     €rO   r1   zIBertIntermediate.__init__s  sÈ   ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒØˆŒØˆŒØˆŒÝ ØÔØÔ$ØØ”Ø”]Ø”Øð
ñ 
ô 
ˆŒ
ð Ô Ò&Ð&ÝÐRÑSÔSÐSÝ#*°d´oÐU[ÔUiÐ#jÑ#jÔ#jˆÔ Ý!)¨$¬,À4Ä?Ð!SÑ!SÔ!SˆÔÐÐrP   c                 ó    — |                       ||¦  «        \  }}|                      ||¦  «        \  }}|                      ||¦  «        \  }}||fS rÃ   )r¶   rÐ   rG   )rL   r•   r–   s      rO   rh   zIBertIntermediate.forward‡  sn   € Ø6:·j²jÀÐPlÑ6mÔ6mÑ3ˆÐ3Ø6:×6NÒ6NØÐ7ñ7
ô 7
Ñ3ˆÐ3ð
 7;×6LÒ6LØÐ7ñ7
ô 7
Ñ3ˆÐ3ð Ð:Ð:Ð:rP   r±   rq   s   @rO   rÌ   rÌ   r  sL   ø€ € € € € ðTð Tð Tð Tð Tð(
;ð 
;ð 
;ð 
;ð 
;ð 
;ð 
;rP   rÌ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚIBertOutputc           	      ó.  •— t          ¦   «                              ¦   «          |j        | _        d| _        d| _        d| _        d| _        d| _        t          |j	        |j
        d| j        | j        | j        d¬¦  «        | _        t          | j        | j        ¬¦  «        | _        t          |j
        |j        | j        | j        |j        ¬¦  «        | _        t          | j        | j        ¬¦  «        | _        t'          j        |j        ¦  «        | _        d S rµ   )r0   r1   r&   r4   r%   ry   r5   r6   r   rÎ   r8   r¶   r   r·   r   rE   r/   rF   rG   r   rH   rI   rJ   rK   s     €rO   r1   zIBertOutput.__init__•  s  ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒØˆŒØˆŒØˆŒØˆÔØˆÔå ØÔ$ØÔØØ”Ø”]Ø”Øð
ñ 
ô 
ˆŒ
õ % TÔ%6À4Ä?ÐSÑSÔSˆÔÝ%ØÔØÔ%ØÔ)Ø”Ø Ô.ð
ñ 
ô 
ˆŒõ "*¨$¬,À4Ä?Ð!SÑ!SÔ!SˆÔÝ”z &Ô"<Ñ=Ô=ˆŒˆˆrP   c                 ó  — |                       ||¦  «        \  }}|                      |¦  «        }|                      ||||¬¦  «        \  }}|                      ||¦  «        \  }}|                      ||¦  «        \  }}||fS r¹   rº   r»   s        rO   rh   zIBertOutput.forward²  r¾   rP   r±   rq   s   @rO   rÓ   rÓ   ”  r¿   rP   rÓ   c                   ó0   ‡ — e Zd Zˆ fd„Z	 	 dd„Zd„ Zˆ xZS )Ú
IBertLayerc                 ót  •— t          ¦   «                              ¦   «          |j        | _        d| _        d| _        t          |¦  «        | _        t          |¦  «        | _        t          |¦  «        | _
        t          | j        | j        ¬¦  «        | _        t          | j        | j        ¬¦  «        | _        d S )Nr    r   r+   )r0   r1   r&   r4   Úseq_len_dimrÁ   Ú	attentionrÌ   ÚintermediaterÓ   rÄ   r   Úpre_intermediate_actÚpre_output_actrK   s     €rO   r1   zIBertLayer.__init__Ä  s˜   ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒØˆŒàˆÔÝ'¨Ñ/Ô/ˆŒÝ-¨fÑ5Ô5ˆÔÝ! &Ñ)Ô)ˆŒå$,¨T¬\ÀdÄoÐ$VÑ$VÔ$VˆÔ!Ý& t¤|ÀÄÐPÑPÔPˆÔÐÐrP   NFc                 ó°   — |                       ||||¬¦  «        \  }}|d         }|d         }|dd …         }	|                      ||¦  «        \  }
}|
f|	z   }	|	S )N)r˜   r   r   )rÚ   Úfeed_forward_chunk)rL   r•   r–   r—   r˜   Úself_attention_outputsÚ%self_attention_outputs_scaling_factorrÈ   rÉ   r®   Úlayer_outputÚlayer_output_scaling_factors               rO   rh   zIBertLayer.forwardÑ  s�   € ð IMÏÊØØ(ØØ/ð	 IWñ I
ô I
ÑEÐÐ Eð 2°!Ô4ÐØ*OÐPQÔ*RÐ'à(¨¨¨Ô,ˆà48×4KÒ4KØÐ=ñ5
ô 5
Ñ1ˆÐ1ð  �/ GÑ+ˆàˆrP   c                 óÖ   — |                       ||¦  «        \  }}|                      ||¦  «        \  }}|                      ||¦  «        \  }}|                      ||||¦  «        \  }}||fS rÃ   )rÜ   rÛ   rÝ   rÄ   )rL   rÈ   rÉ   Úintermediate_outputÚ"intermediate_output_scaling_factorrâ   rã   s          rO   rß   zIBertLayer.feed_forward_chunkê  s©   € Ø<@×<UÒ<UØÐ=ñ=
ô =
Ñ9ÐÐ9ð CG×BSÒBSØÐ=ñC
ô C
Ñ?ÐÐ?ð CG×BUÒBUØÐ!CñC
ô C
Ñ?ÐÐ?ð 59·K²KØÐ!CÐEUÐWvñ5
ô 5
Ñ1ˆÐ1ð Ð8Ð8Ð8rP   r°   )rl   rm   rn   r1   rh   rß   rp   rq   s   @rO   r×   r×   Ã  sh   ø€ € € € € ðQð Qð Qð Qð Qð" Øðð ð ð ð29ð 9ð 9ð 9ð 9ð 9ð 9rP   r×   c                   ó.   ‡ — e Zd Zˆ fd„Z	 	 	 	 dd„Zˆ xZS )ÚIBertEncoderc                 óÞ   •‡— t          ¦   «                              ¦   «          ‰| _        ‰j        | _        t	          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS © )r×   )Ú.0Ú_rM   s     €rO   ú
<listcomp>z)IBertEncoder.__init__.<locals>.<listcomp>   s!   ø€ Ð#`Ð#`Ð#`¸1¥J¨vÑ$6Ô$6Ð#`Ð#`Ð#`rP   )	r0   r1   rM   r&   r   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayerrK   s    `€rO   r1   zIBertEncoder.__init__ü  s`   øø€ Ý‰Œ×ÒÑÔÐØˆŒØ Ô+ˆŒÝ”]Ð#`Ð#`Ð#`Ð#`ÅÀfÔF^Ñ@_Ô@_Ð#`Ñ#`Ô#`ÑaÔaˆŒ
ˆ
ˆ
rP   NFTc                 ó  — |rdnd }|rdnd }d }	t          | j        ¦  «        D ]1\  }
}|r||fz   } |||||¦  «        }|d         }|r||d         fz   }Œ2|r||fz   }|st          d„ ||||	fD ¦   «         ¦  «        S t          ||||	¬¦  «        S )Nrë   r   r   c              3   ó   K  — | ]}|®|V — Œ	d S rÃ   rë   )rì   Úvs     rO   ú	<genexpr>z'IBertEncoder.forward.<locals>.<genexpr>"  s4   è è € ð 	ð 	àð �=ð ð !�=�=�=ð	ð 	rP   )Úlast_hidden_stater•   Ú
attentionsÚcross_attentions)Ú	enumeraterò   Útupler   )rL   r•   r–   r—   r˜   Úoutput_hidden_statesÚreturn_dictÚall_hidden_statesÚall_self_attentionsÚall_cross_attentionsÚiÚlayer_moduleÚlayer_outputss                rO   rh   zIBertEncoder.forward  s&  € ð #7Ð@˜B˜B¸DÐØ$5Ð?˜b˜b¸4ÐØ#Ðå(¨¬Ñ4Ô4ð 	Pð 	P‰OˆAˆ|Ø#ð IØ$5¸Ð8HÑ$HÐ!à(˜LØØ,ØØ!ñ	ô ˆMð *¨!Ô,ˆMØ ð PØ&9¸]È1Ô=MÐ<OÑ&OÐ#øàð 	EØ 1°]Ð4DÑ DÐàð 
	Ýð 	ð 	ð "Ø%Ø'Ø(ð	ð	ñ 	ô 	ñ 	ô 	ð 	õ 9Ø+Ø+Ø*Ø1ð	
ñ 
ô 
ð 	
rP   )NFFTr±   rq   s   @rO   rè   rè   û  s_   ø€ € € € € ðbð bð bð bð bð ØØ"Øð/
ð /
ð /
ð /
ð /
ð /
ð /
ð /
rP   rè   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚIBertPoolerc                 óØ   •— t          ¦   «                              ¦   «          |j        | _        t          j        |j        |j        ¦  «        | _        t          j        ¦   «         | _        d S rÃ   )	r0   r1   r&   r   ÚLinearr8   r¶   ÚTanhÚ
activationrK   s     €rO   r1   zIBertPooler.__init__5  sM   ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ'™)œ)ˆŒˆˆrP   c                 ór   — |d d …df         }|                       |¦  «        }|                      |¦  «        }|S ©Nr   )r¶   r	  )rL   r•   Úfirst_token_tensorÚpooled_outputs       rO   rh   zIBertPooler.forward;  s@   € ð +¨1¨1¨1¨a¨4Ô0ÐØŸ
š
Ð#5Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐrP   r±   rq   s   @rO   r  r  4  sG   ø€ € € € € ð$ð $ð $ð $ð $ðð ð ð ð ð ð rP   r  c                   ó`   ‡ — e Zd ZU eed<   dZ ej        ¦   «         ˆ fd„¦   «         Zdd„Z	ˆ xZ
S )ÚIBertPreTrainedModelrM   Úibertc                 óâ  •— t          ¦   «                              |¦  «         t          |t          ¦  «        r·t	          j        |j        d| j        j        ¬¦  «         |j	        �t	          j
        |j	        ¦  «         t          |dd¦  «        �2t	          j
        |j        ¦  «         t	          j
        |j        ¦  «         t          |dd¦  «        �t	          j
        |j        ¦  «         dS dS t          |t          ¦  «        r®t	          j        |j        d| j        j        ¬¦  «         |j        �:t          |j        dd¦  «        s$t	          j
        |j        |j                 ¦  «         t          |dd¦  «        �4t	          j
        |j        ¦  «         t	          j
        |j        ¦  «         dS dS t          |t$          ¦  «        r.t          |d	d¦  «        �t	          j
        |j        ¦  «         dS dS t          |t(          ¦  «        rt	          j
        |j	        ¦  «         dS t          |t*          ¦  «        rQt	          j        |j        t1          j        |j        j        d
         ¦  «                             d¦  «        ¦  «         dS t          |t8          ¦  «        rOt	          j        |j        d¦  «         t	          j        |j        d¦  «         t	          j
        |j         ¦  «         dS dS )zInitialize the weightsg        )ÚmeanÚstdNÚweight_integerÚbias_integerÚ_is_hf_initializedFÚweight_scaling_factorÚshiftr)   r(   gñhãˆµøä¾gñhãˆµøä>)!r0   Ú_init_weightsÚ
isinstancer   ÚinitÚnormal_ÚweightrM   Úinitializer_rangerx   Úzeros_Úgetattrr  Úfc_scaling_factorr  r   r$   r  r   r  ÚIBertLMHeadr   Úcopy_r'   r>   r?   r�   rA   r   Ú	constant_Úx_minÚx_maxÚact_scaling_factor)rL   ÚmodulerN   s     €rO   r  z"IBertPreTrainedModel._init_weightsI  s£  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�kÑ*Ô*ð 	3ÝŒL˜œ¨S°d´kÔ6SÐTÑTÔTÐTØŒ{Ð&Ý”˜FœKÑ(Ô(Ð(Ý�vÐ/°Ñ6Ô6ÐBÝ”˜FÔ1Ñ2Ô2Ð2Ý”˜FÔ4Ñ5Ô5Ð5Ý�v˜~¨tÑ4Ô4Ð@Ý”˜FÔ/Ñ0Ô0Ð0Ð0Ð0ð AÐ@å˜¥Ñ/Ô/ð 	3ÝŒL˜œ¨S°d´kÔ6SÐTÑTÔTÐTàÔ!Ð-µg¸f¼mÐMaÐchÑ6iÔ6iÐ-Ý”˜FœM¨&Ô*<Ô=Ñ>Ô>Ð>Ý�vÐ6¸Ñ=Ô=ÐIÝ”˜FÔ8Ñ9Ô9Ð9Ý”˜FÔ1Ñ2Ô2Ð2Ð2Ð2ð JÐIõ ˜¥Ñ-Ô-ð 
	3Ý�v˜w¨Ñ-Ô-Ð9Ý”˜FœLÑ)Ô)Ð)Ð)Ð)ð :Ð9å˜¥Ñ,Ô,ð 	3ÝŒK˜œÑ$Ô$Ð$Ð$Ð$Ý˜¥Ñ0Ô0ð 	3ÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÑhÔhÐhÐhÐhÝ˜¥Ñ)Ô)ð 	3ÝŒN˜6œ<¨Ñ/Ô/Ð/ÝŒN˜6œ<¨Ñ.Ô.Ð.ÝŒK˜Ô1Ñ2Ô2Ð2Ð2Ð2ð	3ð 	3rP   Nc                 ó    — t          d¦  «        ‚)Nz6`resize_token_embeddings` is not supported for I-BERT.)ÚNotImplementedError)rL   Únew_num_tokenss     rO   Úresize_token_embeddingsz,IBertPreTrainedModel.resize_token_embeddingsj  s   € Ý!Ð"ZÑ[Ô[Ð[rP   rÃ   )rl   rm   rn   r   Ú__annotations__Úbase_model_prefixr>   Úno_gradr  r,  rp   rq   s   @rO   r  r  D  s{   ø€ € € € € € àÐÐÑØÐà€U„]�_„_ð3ð 3ð 3ð 3ñ „_ð3ð@\ð \ð \ð \ð \ð \ð \ð \rP   r  c                   óü   ‡ — e Zd ZdZdˆ fd„	Zd„ Zd„ Ze	 	 	 	 	 	 	 	 ddej	        dz  dej
        dz  d	ej	        dz  d
ej	        dz  dej
        dz  dedz  dedz  dedz  deeej
                 z  fd„¦   «         Zˆ xZS )Ú
IBertModela®  

    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in [Attention is
    all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

    Tc                 ó  •— t          ¦   «                              |¦  «         || _        |j        | _        t	          |¦  «        | _        t          |¦  «        | _        |rt          |¦  «        nd| _	        |  
                    ¦   «          dS )zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        N)r0   r1   rM   r&   r   re   rè   Úencoderr  ÚpoolerÚ	post_init)rL   rM   Úadd_pooling_layerrN   s      €rO   r1   zIBertModel.__init__y  s}   ø€ õ
 	‰Œ×Ò˜Ñ Ô Ð ØˆŒØ Ô+ˆŒå)¨&Ñ1Ô1ˆŒÝ# FÑ+Ô+ˆŒà->ÐH•k &Ñ)Ô)Ð)ÀDˆŒð 	�ŠÑÔÐÐÐrP   c                 ó   — | j         j        S rÃ   ©re   r:   ©rL   s    rO   Úget_input_embeddingszIBertModel.get_input_embeddingsŠ  s   € ØŒÔ.Ð.rP   c                 ó   — || j         _        d S rÃ   r8  )rL   r„   s     rO   Úset_input_embeddingszIBertModel.set_input_embeddings�  s   € Ø*/ˆŒÔ'Ð'Ð'rP   Nr^   r—   r_   r'   r`   r˜   rü   rý   Úreturnc	                 ó2  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|�|�t	          d¦  «        ‚|�+|                      ||¦  «         |                     ¦   «         }
n.|�|                     ¦   «         d d…         }
nt	          d¦  «        ‚|
\  }}|�|j        n|j        }|€t          j	        ||f|¬¦  «        }|€!t          j
        |
t          j        |¬¦  «        }|                      ||||¬¦  «        \  }}t          | j         ||¬¦  «        }|                      ||||||¬¦  «        }|d	         }| j        �|                      |¦  «        nd }|s||f|d
d …         z   S t!          |||j        |j        |j        ¬¦  «        S )NzDYou cannot specify both input_ids and inputs_embeds at the same timer)   z5You have to specify either input_ids or inputs_embeds)rT   rR   )r^   r'   r_   r`   )rM   r`   r—   )r—   r˜   rü   rý   r   r   )r÷   Úpooler_outputr•   rø   rù   )rM   r˜   rü   rý   r~   Ú%warn_if_padding_and_no_attention_maskr[   rT   r>   Úonesr\   r]   re   r
   r3  r4  r   r•   rø   rù   )rL   r^   r—   r_   r'   r`   r˜   rü   rý   Úkwargsrb   Ú
batch_sizeÚ
seq_lengthrT   Úembedding_outputÚembedding_output_scaling_factorÚencoder_outputsÚsequence_outputr  s                      rO   rh   zIBertModel.forward�  s  € ð 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàÐ  ]Ð%>ÝÐcÑdÔdÐdØÐ"Ø×6Ò6°yÀ.ÑQÔQÐQØ#Ÿ.š.Ñ*Ô*ˆKˆKØÐ&Ø'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKˆKåÐTÑUÔUÐUà!,Ñˆ
�JØ%.Ð%:�Ô!Ð!ÀÔ@TˆàÐ!Ý"œZ¨*°jÐ)AÈ6ÐRÑRÔRˆNØÐ!Ý"œ[¨½E¼JÈvÐVÑVÔVˆNà<@¿OºOØØ%Ø)Ø'ð	 =Lñ =
ô =
Ñ9ÐÐ9õ 3Ø”;Ø*Ø)ð
ñ 
ô 
ˆð Ÿ,š,ØØ+Ø)Ø/Ø!5Ø#ð 'ñ 
ô 
ˆð *¨!Ô,ˆØ8<¼Ð8O˜Ÿš OÑ4Ô4Ð4ÐUYˆàð 	JØ# ]Ð3°oÀaÀbÀbÔ6IÑIÐIå;Ø-Ø'Ø)Ô7Ø&Ô1Ø,Ô=ð
ñ 
ô 
ð 	
rP   )T)NNNNNNNN)rl   rm   rn   ro   r1   r:  r<  r   r>   Ú
LongTensorÚFloatTensorÚboolr   rû   rh   rp   rq   s   @rO   r1  r1  n  sO  ø€ € € € € ðð ðð ð ð ð ð ð"/ð /ð /ð0ð 0ð 0ð ð .2Ø37Ø26Ø04Ø26Ø)-Ø,0Ø#'ðE
ð E
àÔ# dÑ*ðE
ð Ô)¨DÑ0ðE
ð Ô(¨4Ñ/ð	E
ð
 Ô&¨Ñ-ðE
ð Ô(¨4Ñ/ðE
ð   $™;ðE
ð # T™kðE
ð ˜D‘[ðE
ð 
6¸¸eÔ>OÔ8PÑ	PðE
ð E
ð E
ñ „^ðE
ð E
ð E
ð E
ð E
rP   r1  c                   ó  ‡ — e Zd ZdddœZˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 ddej	        dz  d	ej
        dz  d
ej	        dz  dej	        dz  dej
        dz  dej	        dz  dedz  dedz  dedz  deeej
                 z  fd„¦   «         Zˆ xZS )ÚIBertForMaskedLMz(ibert.embeddings.word_embeddings.weight$zlm_head.bias)zlm_head.decoder.weightzlm_head.decoder.biasc                 óÆ   •— t          ¦   «                              |¦  «         t          |d¬¦  «        | _        t	          |¦  «        | _        |                      ¦   «          d S ©NF)r6  )r0   r1   r1  r  r"  Úlm_headr5  rK   s     €rO   r1   zIBertForMaskedLM.__init__à  sV   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å ¸%Ð@Ñ@Ô@ˆŒ
Ý" 6Ñ*Ô*ˆŒð 	�ŠÑÔÐÐÐrP   c                 ó   — | j         j        S rÃ   )rP  Údecoderr9  s    rO   Úget_output_embeddingsz&IBertForMaskedLM.get_output_embeddingsé  s   € ØŒ|Ô#Ð#rP   c                 ó@   — || j         _        |j        | j         _        d S rÃ   )rP  rR  rx   )rL   Únew_embeddingss     rO   Úset_output_embeddingsz&IBertForMaskedLM.set_output_embeddingsì  s   € Ø-ˆŒÔØ*Ô/ˆŒÔÐÐrP   Nr^   r—   r_   r'   r`   Úlabelsr˜   rü   rý   r=  c
           
      ó¢  — |	�|	n| j         j        }	|                      ||||||||	¬¦  «        }|d         }|                      |¦  «        }d}|�Kt	          ¦   «         } ||                     d| j         j        ¦  «        |                     d¦  «        ¦  «        }|	s|f|dd…         z   }|�|f|z   n|S t          |||j        |j	        ¬¦  «        S )a¢  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        N©r—   r_   r'   r`   r˜   rü   rý   r   r)   r‹   ©ÚlossÚlogitsr•   rø   )
rM   rý   r  rP  r   rŽ   r7   r   r•   rø   )rL   r^   r—   r_   r'   r`   rW  r˜   rü   rý   rB  r®   rH  Úprediction_scoresÚmasked_lm_lossÚloss_fctrÄ   s                    rO   rh   zIBertForMaskedLM.forwardð  s  € ð( &1Ð%<�k�kÀ$Ä+ÔBYˆà—*’*ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ñ 	
ô 	
ˆð " !œ*ˆØ ŸLšL¨Ñ9Ô9ÐàˆØÐÝ'Ñ)Ô)ˆHØ%˜XÐ&7×&<Ò&<¸RÀÄÔAWÑ&XÔ&XÐZ`×ZeÒZeÐfhÑZiÔZiÑjÔjˆNàð 	ZØ'Ð)¨G°A°B°B¬KÑ7ˆFØ3AÐ3M�^Ð%¨Ñ.Ð.ÐSYÐYåØØ$Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rP   ©	NNNNNNNNN)rl   rm   rn   Ú_tied_weights_keysr1   rS  rV  r   r>   rI  rJ  rK  r   rû   rh   rp   rq   s   @rO   rM  rM  Ù  sY  ø€ € € € € ð #MØ .ðð Ðð
ð ð ð ð ð$ð $ð $ð0ð 0ð 0ð ð .2Ø37Ø26Ø04Ø26Ø*.Ø)-Ø,0Ø#'ð0
ð 0
àÔ# dÑ*ð0
ð Ô)¨DÑ0ð0
ð Ô(¨4Ñ/ð	0
ð
 Ô&¨Ñ-ð0
ð Ô(¨4Ñ/ð0
ð Ô  4Ñ'ð0
ð   $™;ð0
ð # T™kð0
ð ˜D‘[ð0
ð 
˜% Ô 1Ô2Ñ	2ð0
ð 0
ð 0
ñ „^ð0
ð 0
ð 0
ð 0
ð 0
rP   rM  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )r"  z)I-BERT Head for masked language modeling.c                 ó‚  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j        |j	        ¦  «        | _
        t          j        t          j        |j	        ¦  «        ¦  «        | _        d S )N)r-   )r0   r1   r   r  r8   r¶   rF   rE   Ú
layer_normr7   rR  Ú	Parameterr>   r\   rx   rK   s     €rO   r1   zIBertLMHead.__init__'  s‰   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒå”y Ô!3°VÔ5FÑGÔGˆŒÝ”L¥¤¨VÔ->Ñ!?Ô!?Ñ@Ô@ˆŒ	ˆ	ˆ	rP   c                 ó¢   — |                       |¦  «        }t          |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rÃ   )r¶   r	   rd  rR  )rL   ÚfeaturesrB  Úxs       rO   rh   zIBertLMHead.forward/  sE   € Ø�JŠJ�xÑ Ô ˆÝ�‰GŒGˆØ�OŠO˜AÑÔˆð �LŠL˜‰OŒOˆàˆrP   ©rl   rm   rn   ro   r1   rh   rp   rq   s   @rO   r"  r"  $  sR   ø€ € € € € Ø3Ð3ðAð Að Að Að Aðð ð ð ð ð ð rP   r"  zž
    I-BERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    )Úcustom_introc                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	edz  d
edz  dedz  de	e
ej                 z  fd„¦   «         Zˆ xZS )ÚIBertForSequenceClassificationc                 óÞ   •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S rO  )r0   r1   Ú
num_labelsr1  r  ÚIBertClassificationHeadÚ
classifierr5  rK   s     €rO   r1   z'IBertForSequenceClassification.__init__A  s`   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå ¸%Ð@Ñ@Ô@ˆŒ
Ý1°&Ñ9Ô9ˆŒð 	�ŠÑÔÐÐÐrP   Nr^   r—   r_   r'   r`   rW  r˜   rü   rý   r=  c
           
      óÂ  — |	�|	n| j         j        }	|                      ||||||||	¬¦  «        }|d         }|                      |¦  «        }d}|��Z| j         j        €f| j        dk    rd| j         _        nN| j        dk    r7|j        t          j        k    s|j        t          j	        k    rd| j         _        nd| j         _        | j         j        dk    rWt          ¦   «         }| j        dk    r1 ||                     ¦   «         |                     ¦   «         ¦  «        }nŽ |||¦  «        }n�| j         j        dk    rGt          ¦   «         } ||                     d| j        ¦  «        |                     d¦  «        ¦  «        }n*| j         j        dk    rt          ¦   «         } |||¦  «        }|	s|f|d	d…         z   }|�|f|z   n|S t          |||j        |j        ¬
¦  «        S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        NrY  r   r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr)   r‹   rZ  )rM   rý   r  rp  Úproblem_typern  rS   r>   r]   r   r   Úsqueezer   rŽ   r   r   r•   rø   ©rL   r^   r—   r_   r'   r`   rW  r˜   rü   rý   rB  r®   rH  r\  r[  r_  rÄ   s                    rO   rh   z&IBertForSequenceClassification.forwardK  s  € ð( &1Ð%<�k�kÀ$Ä+ÔBYˆà—*’*ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ñ 	
ô 	
ˆð " !œ*ˆØ—’ Ñ1Ô1ˆàˆØÑØŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ñ @Ô @À&Ç+Â+ÈbÁ/Ä/ÑRÔR��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�Øð 	FØ�Y ¨¨¨¤Ñ,ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rP   r`  )rl   rm   rn   r1   r   r>   rI  rJ  rK  r   rû   rh   rp   rq   s   @rO   rl  rl  :  s8  ø€ € € € € ðð ð ð ð ð ð .2Ø37Ø26Ø04Ø26Ø*.Ø)-Ø,0Ø#'ðA
ð A
àÔ# dÑ*ðA
ð Ô)¨DÑ0ðA
ð Ô(¨4Ñ/ð	A
ð
 Ô&¨Ñ-ðA
ð Ô(¨4Ñ/ðA
ð Ô  4Ñ'ðA
ð   $™;ðA
ð # T™kðA
ð ˜D‘[ðA
ð 
" E¨%Ô*;Ô$<Ñ	<ðA
ð A
ð A
ñ „^ðA
ð A
ð A
ð A
ð A
rP   rl  c                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	edz  d
edz  dedz  de	e
ej                 z  fd„¦   «         Zˆ xZS )ÚIBertForMultipleChoicec                 ó  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        ¦  «        | _        t	          j        |j	        d¦  «        | _
        |                      ¦   «          d S )Nr   )r0   r1   r1  r  r   rH   rI   rJ   r  r8   rp  r5  rK   s     €rO   r1   zIBertForMultipleChoice.__init__’  sl   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å Ñ'Ô'ˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒÝœ) FÔ$6¸Ñ:Ô:ˆŒð 	�ŠÑÔÐÐÐrP   Nr^   r_   r—   rW  r'   r`   r˜   rü   rý   r=  c
           
      ó¸  — |	�|	n| j         j        }	|�|j        d         n|j        d         }|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�=|                     d|                     d¦  «        |                     d¦  «        ¦  «        nd}|                      ||||||||	¬¦  «        }|d         }|                      |¦  «        }|                      |¦  «        }|                     d|¦  «        }d}|�t          ¦   «         } |||¦  «        }|	s|f|dd…         z   }|�|f|z   n|S t          |||j
        |j        ¬¦  «        S )a[  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        Nr   r)   rŒ   )r'   r_   r—   r`   r˜   rü   rý   r‹   rZ  )rM   rý   r�   rŽ   r[   r  rJ   rp  r   r   r•   rø   )rL   r^   r_   r—   rW  r'   r`   r˜   rü   rý   rB  Únum_choicesÚflat_input_idsÚflat_position_idsÚflat_token_type_idsÚflat_attention_maskÚflat_inputs_embedsr®   r  r\  Úreshaped_logitsr[  r_  rÄ   s                           rO   rh   zIBertForMultipleChoice.forwardœ  s+  € ðX &1Ð%<�k�kÀ$Ä+ÔBYˆØ,5Ð,A�i”o aÔ(Ð(À}ÔGZÐ[\ÔG]ˆàCLÐCX˜Ÿš¨¨I¯NªN¸2Ñ,>Ô,>Ñ?Ô?Ð?Ð^bˆØLXÐLd˜L×-Ò-¨b°,×2CÒ2CÀBÑ2GÔ2GÑHÔHÐHÐjnÐØR`ÐRl˜n×1Ò1°"°n×6IÒ6IÈ"Ñ6MÔ6MÑNÔNÐNÐrvÐØR`ÐRl˜n×1Ò1°"°n×6IÒ6IÈ"Ñ6MÔ6MÑNÔNÐNÐrvÐð Ð(ð ×Ò˜r =×#5Ò#5°bÑ#9Ô#9¸=×;MÒ;MÈbÑ;QÔ;QÑRÔRÐRàð 	ð —*’*ØØ*Ø.Ø.Ø,Ø/Ø!5Ø#ð ñ 	
ô 	
ˆð   œ
ˆàŸš ]Ñ3Ô3ˆØ—’ Ñ/Ô/ˆØ Ÿ+š+ b¨+Ñ6Ô6ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜O¨VÑ4Ô4ˆDàð 	FØ%Ð'¨'°!°"°"¬+Ñ5ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå(ØØ"Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rP   r`  )rl   rm   rn   r1   r   r>   rI  rJ  rK  r   rû   rh   rp   rq   s   @rO   ry  ry  �  s8  ø€ € € € € ðð ð ð ð ð ð .2Ø26Ø37Ø*.Ø04Ø26Ø)-Ø,0Ø#'ðV
ð V
àÔ# dÑ*ðV
ð Ô(¨4Ñ/ðV
ð Ô)¨DÑ0ð	V
ð
 Ô  4Ñ'ðV
ð Ô&¨Ñ-ðV
ð Ô(¨4Ñ/ðV
ð   $™;ðV
ð # T™kðV
ð ˜D‘[ðV
ð 
# U¨5Ô+<Ô%=Ñ	=ðV
ð V
ð V
ñ „^ðV
ð V
ð V
ð V
ð V
rP   ry  c                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	edz  d
edz  dedz  de	e
ej                 z  fd„¦   «         Zˆ xZS )ÚIBertForTokenClassificationc                 ó:  •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        t          j        |j        ¦  «        | _        t          j	        |j
        |j        ¦  «        | _        |                      ¦   «          d S rO  )r0   r1   rn  r1  r  r   rH   rI   rJ   r  r8   rp  r5  rK   s     €rO   r1   z$IBertForTokenClassification.__init__ø  s~   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå ¸%Ð@Ñ@Ô@ˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐrP   Nr^   r—   r_   r'   r`   rW  r˜   rü   rý   r=  c
           
      óÂ  — |	�|	n| j         j        }	|                      ||||||||	¬¦  «        }|d         }|                      |¦  «        }|                      |¦  «        }d}|�Ft          ¦   «         } ||                     d| j        ¦  «        |                     d¦  «        ¦  «        }|	s|f|dd…         z   }|�|f|z   n|S t          |||j	        |j
        ¬¦  «        S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        NrY  r   r)   r‹   rZ  )rM   rý   r  rJ   rp  r   rŽ   rn  r   r•   rø   rw  s                    rO   rh   z#IBertForTokenClassification.forward  s  € ð$ &1Ð%<�k�kÀ$Ä+ÔBYˆà—*’*ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ñ 	
ô 	
ˆð " !œ*ˆàŸ,š, Ñ7Ô7ˆØ—’ Ñ1Ô1ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬OÑ<Ô<¸f¿kºkÈ"¹o¼oÑNÔNˆDàð 	FØ�Y ¨¨¨¤Ñ,ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
rP   r`  )rl   rm   rn   r1   r   r>   rI  rJ  rK  r   rû   rh   rp   rq   s   @rO   r„  r„  ö  s%  ø€ € € € € ð	ð 	ð 	ð 	ð 	ð ð .2Ø37Ø26Ø04Ø26Ø*.Ø)-Ø,0Ø#'ð1
ð 1
àÔ# dÑ*ð1
ð Ô)¨DÑ0ð1
ð Ô(¨4Ñ/ð	1
ð
 Ô&¨Ñ-ð1
ð Ô(¨4Ñ/ð1
ð Ô  4Ñ'ð1
ð   $™;ð1
ð # T™kð1
ð ˜D‘[ð1
ð 
  uÔ'8Ô!9Ñ	9ð1
ð 1
ð 1
ñ „^ð1
ð 1
ð 1
ð 1
ð 1
rP   r„  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ro  z-Head for sentence-level classification tasks.c                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        t          j        |j        |j	        ¦  «        | _
        d S rÃ   )r0   r1   r   r  r8   r¶   rH   rI   rJ   rn  Úout_projrK   s     €rO   r1   z IBertClassificationHead.__init__;  sc   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒÝœ	 &Ô"4°fÔ6GÑHÔHˆŒˆˆrP   c                 óô   — |d d …dd d …f         }|                       |¦  «        }|                      |¦  «        }t          j        |¦  «        }|                       |¦  «        }|                      |¦  «        }|S r  )rJ   r¶   r>   Útanhr‰  )rL   rg  rB  r•   s       rO   rh   zIBertClassificationHead.forwardA  ss   € Ø     A q q q Ô)ˆØŸš ]Ñ3Ô3ˆØŸ
š
 =Ñ1Ô1ˆÝœ
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØŸš mÑ4Ô4ˆØÐrP   ri  rq   s   @rO   ro  ro  8  sR   ø€ € € € € Ø7Ð7ðIð Ið Ið Ið Iðð ð ð ð ð ð rP   ro  c                   ó  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
edz  dedz  dedz  de	e
ej                 z  fd„¦   «         Zˆ xZS )ÚIBertForQuestionAnsweringc                 óþ   •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        t          j        |j        |j        ¦  «        | _        |  	                    ¦   «          d S rO  )
r0   r1   rn  r1  r  r   r  r8   Ú
qa_outputsr5  rK   s     €rO   r1   z"IBertForQuestionAnswering.__init__M  sj   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå ¸%Ð@Ñ@Ô@ˆŒ
Ýœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐrP   Nr^   r—   r_   r'   r`   Ústart_positionsÚend_positionsr˜   rü   rý   r=  c           
      ó®  — |
�|
n| j         j        }
|                      |||||||	|
¬¦  «        }|d         }|                      |¦  «        }|                     dd¬¦  «        \  }}|                     d¦  «                             ¦   «         }|                     d¦  «                             ¦   «         }d }|�ç|�åt          |                     ¦   «         ¦  «        dk    r|                     d¦  «        }t          |                     ¦   «         ¦  «        dk    r|                     d¦  «        }|                     d¦  «        }| 	                    d|¦  «        }| 	                    d|¦  «        }t          |¬¦  «        } |||¦  «        } |||¦  «        }||z   dz  }|
s||f|dd …         z   }|�|f|z   n|S t          ||||j        |j        ¬¦  «        S )	NrY  r   r   r)   ©Údim)Úignore_indexr‹   )r[  Ústart_logitsÚ
end_logitsr•   rø   )rM   rý   r  r�  Úsplitrv  r”   Úlenr[   Úclampr   r   r•   rø   )rL   r^   r—   r_   r'   r`   r�  r‘  r˜   rü   rý   rB  r®   rH  r\  r–  r—  Ú
total_lossÚignored_indexr_  Ú
start_lossÚend_lossrÄ   s                          rO   rh   z!IBertForQuestionAnswering.forwardW  s  € ð &1Ð%<�k�kÀ$Ä+ÔBYˆà—*’*ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ñ 	
ô 	
ˆð " !œ*ˆà—’ Ñ1Ô1ˆØ#)§<¢<°°r <Ñ#:Ô#:Ñ ˆ�jØ#×+Ò+¨BÑ/Ô/×:Ò:Ñ<Ô<ˆØ×'Ò'¨Ñ+Ô+×6Ò6Ñ8Ô8ˆ
àˆ
ØÐ&¨=Ð+Då�?×'Ò'Ñ)Ô)Ñ*Ô*¨QÒ.Ð.Ø"1×"9Ò"9¸"Ñ"=Ô"=�Ý�=×%Ò%Ñ'Ô'Ñ(Ô(¨1Ò,Ð,Ø -× 5Ò 5°bÑ 9Ô 9�à(×-Ò-¨aÑ0Ô0ˆMØ-×3Ò3°A°}ÑEÔEˆOØ)×/Ò/°°=ÑAÔAˆMå'°]ÐCÑCÔCˆHØ!˜ ,°Ñ@Ô@ˆJØ�x 
¨MÑ:Ô:ˆHØ$ xÑ/°1Ñ4ˆJàð 	RØ" JÐ/°'¸!¸"¸"´+Ñ=ˆFØ/9Ð/E�Z�M FÑ*Ð*È6ÐQå+ØØ%Ø!Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
rP   )
NNNNNNNNNN)rl   rm   rn   r1   r   r>   rI  rJ  rK  r   rû   rh   rp   rq   s   @rO   r�  r�  K  s:  ø€ € € € € ðð ð ð ð ð ð .2Ø37Ø26Ø04Ø26Ø37Ø15Ø)-Ø,0Ø#'ð=
ð =
àÔ# dÑ*ð=
ð Ô)¨DÑ0ð=
ð Ô(¨4Ñ/ð	=
ð
 Ô&¨Ñ-ð=
ð Ô(¨4Ñ/ð=
ð Ô)¨DÑ0ð=
ð Ô'¨$Ñ.ð=
ð   $™;ð=
ð # T™kð=
ð ˜D‘[ð=
ð 
&¨¨eÔ.?Ô(@Ñ	@ð=
ð =
ð =
ñ „^ð=
ð =
ð =
ð =
ð =
rP   r�  c                 óÜ   — |                       |¦  «                             ¦   «         }t          j        |d¬¦  «                             |¦  «        |z   |z  }|                     ¦   «         |z   S )aM  
    Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
    are ignored. This is modified from fairseq's *utils.make_positions*.

    Args:
    input_ids (`torch.LongTensor`):
           Indices of input sequence tokens in the vocabulary.

    Returns: torch.Tensor
    r   r“  )Úner   r>   ÚcumsumÚtype_asr]   )r^   r$   ra   ÚmaskÚincremental_indicess        rO   rX   rX   ˜  sg   € ð �<Š<˜Ñ$Ô$×(Ò(Ñ*Ô*€DÝ œ<¨°!Ð4Ñ4Ô4×<Ò<¸TÑBÔBÐE[Ñ[Ð_cÑcÐØ×#Ò#Ñ%Ô%¨Ñ3Ð3rP   )rM  ry  r�  rl  r„  r1  r  )r   )=ro   r‘   r>   r   Útorch.nnr   r   r   Ú r   r  Úactivationsr	   Úmasking_utilsr
   Úmodeling_outputsr   r   r   r   r   r   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_ibertr   Úquant_modulesr   r   r   r   r   r   Ú
get_loggerrl   ÚloggerÚModuler   rs   r³   rÁ   rÌ   rÓ   r×   rè   r  r  r1  rM  r"  rl  ry  r„  ro  r�  rX   Ú__all__rë   rP   rO   ú<module>r²     sU  ðð" Ð à €€€à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø Ð Ð Ð Ð Ð Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð .Ð -Ð -Ð -Ð -Ð -Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø cÐ cÐ cÐ cÐ cÐ cÐ cÐ cÐ cÐ cÐ cÐ cÐ cÐ cÐ cÐ cð 
ˆÔ	˜HÑ	%Ô	%€ðs=ð s=ð s=ð s=ð s=�b”iñ s=ô s=ð s=ðl@.ð @.ð @.ð @.ð @.˜œñ @.ô @.ð @.ðF,;ð ,;ð ,;ð ,;ð ,;�b”iñ ,;ô ,;ð ,;ð^/ð /ð /ð /ð /�R”Yñ /ô /ð /ð8;ð ;ð ;ð ;ð ;˜œ	ñ ;ô ;ð ;ðD,;ð ,;ð ,;ð ,;ð ,;�"”)ñ ,;ô ,;ð ,;ð^59ð 59ð 59ð 59ð 59�”ñ 59ô 59ð 59ðp6
ð 6
ð 6
ð 6
ð 6
�2”9ñ 6
ô 6
ð 6
ðrð ð ð ð �"”)ñ ô ð ð  ð&\ð &\ð &\ð &\ð &\˜?ñ &\ô &\ñ „ð&\ðR ðg
ð g
ð g
ð g
ð g
Ð%ñ g
ô g
ñ „ðg
ðT ðG
ð G
ð G
ð G
ð G
Ð+ñ G
ô G
ñ „ðG
ðTð ð ð ð �"”)ñ ô ð ð, €ððñ ô ðM
ð M
ð M
ð M
ð M
Ð%9ñ M
ô M
ñô ðM
ð` ðb
ð b
ð b
ð b
ð b
Ð1ñ b
ô b
ñ „ðb
ðJ ð>
ð >
ð >
ð >
ð >
Ð"6ñ >
ô >
ñ „ð>
ðBð ð ð ð ˜bœiñ ô ð ð& ðI
ð I
ð I
ð I
ð I
Ð 4ñ I
ô I
ñ „ðI
ðX4ð 4ð 4ð 4ð"ð ð €€€rP   