tjpdZddlmZddlmZddlZddlmZddlmZm Z m Z ddl m Z dd lmZdd lmZmZmZdd lmZdd lmZmZdd lmZddlmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ddl%m&Z&m'Z'ddl(m)Z)ddl*m+Z+ddl,m-Z-m.Z.m/Z/m0Z0ddl1m2Z2m3Z3ddl4m5Z5ddl6m7Z7e0j8e9Z:Gddej;Z< d]dej;dej=dej=dej=dej=dzde>dzd e>d!e)e.fd"Z?Gd#d$ej;Z@Gd%d&ej;ZAGd'd(ej;ZBGd)d*ej;ZCGd+d,ej;ZDGd-d.ej;ZEGd/d0eZFGd1d2ej;ZGGd3d4ej;ZHGd5d6ej;ZIGd7d8ej;ZJGd9d:ej;ZKGd;dej;ZMe/Gd?d@e'ZNe/dABeGdCdDe-ZOe/dEBGdFdGeNZPe/dHBGdIdJeNZQe/dKBGdLdMeNeZRe/GdNdOeNZSe/dPBGdQdReNZTe/dSBGdTdUeNZUe/GdVdWeNZVe/GdXdYeNZWe/GdZd[eNZXgd\ZYdS)^zPyTorch BERT model.)Callable) dataclassN)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)initialization)ACT2FN)Cache DynamicCacheEncoderDecoderCache)GenerationMixin)create_bidirectional_maskcreate_causal_mask)GradientCheckpointingLayer) )BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentions!CausalLMOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputNextSentencePredictorOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)apply_chunking_to_forward) ModelOutputTransformersKwargsauto_docstringlogging)can_return_tuplemerge_with_config_defaults)capture_outputs) BertConfigc eZdZdZfdZ d dejdzdejdzdejdzdejdzd ed ej f d Z xZ S) BertEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.cttj|j|j|j|_tj|j|j|_ tj|j |j|_ tj |j|j |_ tj|j|_|dt%j|jdd|dt%j|jt$jddS) N) padding_idxeps position_idsr'F) persistenttoken_type_ids)dtype)super__init__r Embedding vocab_size hidden_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_buffertorcharangeexpandzerosr/sizelongselfconfig __class__s d/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/bert/modeling_bert.pyr6zBertEmbeddings.__init__8s3 !|F,=v?Q_e_rsss#%<0NPVPb#c#c %'\&2H&J\%]%]"f&8f>STTTz&"<==   EL)GHHOOPWXXej      ek$*;*@*@*B*B%*UUUbg      Nr input_idsr3r/ inputs_embedspast_key_values_lengthreturnc||}n|dd}|\}}||jdd|||zf}|t|drT|j|jdd} t j| d|} | ||}n+t j|t j |jj }|| |}| |} || z} | |} | | z} || } || } | S)Nr1r3rr')dimindex)r4device)rJr/hasattrr3rHshaperFgatherrIrKrYr;r?r=r@rD) rMrRr3r/rSrT input_shape batch_size seq_lengthbuffered_token_type_idsr? embeddingsr=s rPforwardzBertEmbeddings.forwardHsu  #..**KK',,..ss3K!, J  ,QQQ0FVlIl0l-lmL  !t-.. m*.*=*D*D\EWXYEZ\^*_*_'*/,7NTU]i*j*j*j'!8!?!? J!W!W!&[ SWSdSk!l!l!l   00;;M $ : :> J J"%:: "66|DD"55 ^^J// \\*-- rQ)NNNNr) __name__ __module__ __qualname____doc__r6rF LongTensor FloatTensorintTensorrb __classcell__rOs@rPr*r*5sQQ     $.2260426&' ((#d*((4/(&- ( (4/ ( !$ ( ((((((((rQr*modulequerykeyvalueattention_maskscalingrDkwargsc ||ddz}tj||dd|z}|||z}tj|d}tj|||j}tj||} | dd } | |fS)Nr1r rW)ptrainingr') rJrFmatmul transposer functionalsoftmaxrDrz contiguous) rnrorprqrrrsrDrt attn_weights attn_outputs rPeager_attention_forwardrss**R..D(<s}}Q':':;;gEL!#n4 =((2(>>L=((6?([[L,|U33K''1--88::K  $$rQc eZdZd fd Z d dejdejdzdedzdee de ejf d Z xZ S) BertSelfAttentionFNct|j|jzdkr0t |ds t d|jd|jd||_|j|_t|j|jz |_|j|jz|_ |jdz|_ tj |j|j |_ tj |j|j |_tj |j|j |_tj|j|_|j|_||_||_dSNrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()rv)r5r6r9num_attention_headsrZ ValueErrorrNriattention_head_size all_head_sizersrLinearrorprqrBattention_probs_dropout_probrD is_decoder is_causal layer_idxrMrNrrrOs rPr6zBertSelfAttention.__init__sG    : :a ? ?PVXhHiHi ?8F$688 4888  #)#= #&v'9F?? O*9*N'&=%C%CI{\`\j%k%k "I{(?(M K ,.E) ) %8$7      % $}@CC$,.L % %  % % ! \*k);;;;;;FFHH L((rQFN)NN rcrdrer6rFrjrhr rr!tuplerbrkrls@rPrrs######648(, ')')|'))D0') ') +, ') u|  ')')')')')')')')rQrceZdZd fd Z d dejdejdzdejdzdedzdee d e ejf d Z xZ S) BertCrossAttentionFNct|j|jzdkr0t |ds t d|jd|jd||_|j|_t|j|jz |_|j|jz|_ |jdz|_ tj |j|j |_ tj |j|j |_tj |j|j |_tj|j|_||_||_dSr)r5r6r9rrZrrNrirrrsrrrorprqrBrrDrrrs rPr6zBertCrossAttention.__init__s=    : :a ? ?PVXhHiHi ?8F$688 4888  #)#= #&v'9FB*4>:(?(M K ,.E) ) %8$7      % $}@CC$,.L % %  % % ! \*k);;;;;;FFHH L((rQr)NNN) rcrdrer6rFrjrhrrr!rrbrkrls@rPrrs######4;?376: 1)1)|1) %0471))D0 1) -t3 1) +, 1) u| 1)1)1)1)1)1)1)1)rQrcPeZdZfdZdejdejdejfdZxZS)BertSelfOutputcttj|j|j|_tj|j|j|_tj|j |_ dSNr-) r5r6rrr9denser@rArBrCrDrLs rPr6zBertSelfOutput.__init__sf Yv163EFF f&8f>STTTz&"<== rQr input_tensorrUc||}||}|||z}|SNrrDr@rMrrs rPrbzBertSelfOutput.forward%@ =11  ]33 }|'CDD rQrcrdrer6rFrjrbrkrls@rPrri>>>>> U\RWR^rQrceZdZd fd Z d dejdejdzdejdzdejdzdedzd ee d e ejfd Z xZ S) BertAttentionFNct||_|rtnt}|||||_t ||_dS)Nrr)r5r6is_cross_attentionrrrMroutput)rMrNrrrattention_classrOs rPr6zBertAttention.__init__-s] "40BY,,HY#OFi9UUU $V,, rQrrrrencoder_attention_maskrrtrUc v|js|n|}|j|f|||d|\}}|||}||fS)N)rrrr)rrMr) rMrrrrrrrtattention_outputrs rPrbzBertAttention.forward4sq04/FbLb)2 * "7)+ * *  * * &, ;;'7GG--rQ)FNFNNNNrrls@rPrr,s------48:>;?(, ..|.)D0. %047 . !& 1D 8 .  .+,. u| ........rQrcBeZdZfdZdejdejfdZxZS)BertIntermediatecttj|j|j|_t|jtrt|j|_ dS|j|_ dSr) r5r6rrr9intermediate_sizerr hidden_actstrr intermediate_act_fnrLs rPr6zBertIntermediate.__init__Jsn Yv163KLL f' - - 9'-f.?'@D $ $ $'-'8D $ $ $rQrrUcZ||}||}|Sr)rrrMrs rPrbzBertIntermediate.forwardRs, =11 00?? rQrrls@rPrrIs^99999U\elrQrcPeZdZfdZdejdejdejfdZxZS) BertOutputcttj|j|j|_tj|j|j|_tj |j |_ dSr) r5r6rrrr9rr@rArBrCrDrLs rPr6zBertOutput.__init__Ysf Yv79KLL f&8f>STTTz&"<== rQrrrUc||}||}|||z}|Srrrs rPrbzBertOutput.forward_rrQrrls@rPrrXrrQrceZdZd fd Z d dejdejdzdejdzdejdzdedzdee d ejfd Z d Z xZ S) BertLayerNct|j|_d|_t ||j||_|j|_|j|_|jr1|jst|dt |d|d|_ t||_ t||_ dS)Nr'rz> should be used as a decoder model if cross attention is addedFT)rrr)r5r6chunk_size_feed_forward seq_len_dimrr attentionadd_cross_attentionrcrossattentionr intermediaterr)rMrNrrOs rPr6zBertLayer.__init__gs '-'E$&v9JV_``` +#)#=  # ? j D!h!h!hiii"/##' ###D  -V44 (( rQrrrrrrrtrUc |j||fd|i|\}}|} |jr=|;t|dstd|d|j|d||fd|i|\} }| } t |j|j|j| } | S)Nrrz'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`) rrrZrrrfeed_forward_chunkrr) rMrrrrrrrtself_attention_output_rcross_attention_output layer_outputs rPrbzBertLayer.forwardzs$24>  $ $ ,$  $ $  q 1 ? 64@4!122  DdDDD )<(;%%& )) !0 )  )) % "A 6 0  #T%A4CSUe  rQc\||}|||}|Sr)rr)rMrintermediate_outputrs rPrzBertLayer.feed_forward_chunks2"//0@AA{{#68HII rQrr) rcrdrer6rFrjrhr rr!rbrrkrls@rPrrfs)))))),48:>;?(, %%|%)D0% %047 % !& 1D 8 %  %+,% %%%%NrQrceZdZfdZ d dejdejdzdejdzdejdzdedzdedzd e e d e eje zfd Z xZS) BertEncoderct|_tjfdt jD|_dS)Nc2g|]}t|S))r)r).0irNs rP z(BertEncoder.__init__..s&#l#l#lqIf$B$B$B#l#l#lrQ)r5r6rNr ModuleListrangenum_hidden_layerslayerrLs `rPr6zBertEncoder.__init__sW  ]#l#l#l#lERXRjLkLk#l#l#lmm rQNrrrrrr use_cachertrUc |t|jD]\}} | |||f||d|}t||r|ndS)N)rr)last_hidden_stater) enumeraterr) rMrrrrrrrrtr layer_modules rPrbzBertEncoder.forwards )44  OA|(L%(> /   MM9+/8BOOd    rQ)NNNNN)rcrdrer6rFrjrhr boolrr!rrrbrkrls@rPrrsnnnnn48:>;?(,!%  | )D0  %047  !& 1D 8    $; +,  u| H H        rQrcBeZdZfdZdejdejfdZxZS) BertPoolercttj|j|j|_tj|_dSr)r5r6rrr9rTanh activationrLs rPr6zBertPooler.__init__sC Yv163EFF '))rQrrUcr|dddf}||}||}|S)Nr)rr)rMrfirst_token_tensor pooled_outputs rPrbzBertPooler.forwards@+111a40 #566  66 rQrrls@rPrrs^$$$$$ U\elrQrcBeZdZfdZdejdejfdZxZS)BertPredictionHeadTransformcVttj|j|j|_t |jtrt|j|_ n |j|_ tj |j|j |_ dSr) r5r6rrr9rrrrr transform_act_fnr@rArLs rPr6z$BertPredictionHeadTransform.__init__s Yv163EFF f' - - 6$*6+<$=D ! !$*$5D !f&8f>STTTrQrrUc||}||}||}|Sr)rrr@rs rPrbz#BertPredictionHeadTransform.forwards= =11 --m<< }55 rQrrls@rPrrscUUUUUU\elrQrc$eZdZfdZdZxZS)BertLMPredictionHeadctt||_t j|j|jd|_t j tj |j|_ dS)NT)bias) r5r6r transformrrr9r8decoder ParameterrFrIr rLs rPr6zBertLMPredictionHead.__init__sj 4V<<y!3V5FTRRR LV->!?!?@@ rQcZ||}||}|Sr)r r rs rPrbzBertLMPredictionHead.forwards*}55  ]33 rQrcrdrer6rbrkrls@rPrrsLAAAAArQrcBeZdZfdZdejdejfdZxZS)BertOnlyMLMHeadcptt||_dSr)r5r6r predictionsrLs rPr6zBertOnlyMLMHead.__init__s/ /77rQsequence_outputrUc0||}|Sr)r)rMrprediction_scoress rPrbzBertOnlyMLMHead.forwards ,,_==  rQrrls@rPrrs^88888!u|! !!!!!!!!rQrc$eZdZfdZdZxZS)BertOnlyNSPHeadcttj|jd|_dSNrw)r5r6rrr9seq_relationshiprLs rPr6zBertOnlyNSPHead.__init__s6  " &*{.5&; &&& "555rQ)T) NNNNNNNNN)rcrdre_no_split_modulesr6rHrJr%r&r"rFrjr rrr!rrrbrPrkrls@rPr>r>Js*;7"///000 *..2.2,0-1596:(,!%? ? <$&?  t+?  t+ ? lT) ? |d* ?  %|d2? !& t 3? ? $;? +,?  u| K K? ? ? ^_ ? B6666666rQr>z Bert Model with two heads on top as done during the pretraining: a `masked language modeling` head and a `next sentence prediction (classification)` head. c(eZdZdddZfdZdZdZee dde j dzd e j dzd e j dzd e j dzd e j dzd e j dzde j dzde e de e j ezfdZxZS)BertForPreTraining&bert.embeddings.word_embeddings.weightcls.predictions.biaszcls.predictions.decoder.weightzcls.predictions.decoder.biasct|t||_t ||_|dSr)r5r6r>r$rclsrCrLs rPr6zBertForPreTraining.__init__sQ    f%% '// rQc$|jjjSrrZrr rGs rPget_output_embeddingsz(BertForPreTraining.get_output_embeddingsx#++rQcT||jj_|j|jj_dSrrZrr r rMnew_embeddingss rPset_output_embeddingsz(BertForPreTraining.set_output_embeddings%'5$$2$7!!!rQNrRrrr3r/rSlabelsnext_sentence_labelrtrUc |j|f||||dd|} | dd\} } || | \} } d}||t}|| d|jj|d}|| dd|d}||z}t || | | j| jS)am labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` next_sentence_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see `input_ids` docstring) Indices should be in `[0, 1]`: - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. Example: ```python >>> from transformers import AutoTokenizer, BertForPreTraining >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased") >>> model = BertForPreTraining.from_pretrained("google-bert/bert-base-uncased") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) >>> prediction_logits = outputs.prediction_logits >>> seq_relationship_logits = outputs.seq_relationship_logits ``` Trrr3r/rS return_dictNrwr1)r8r9r:rr%) r$rZrrrNr8r7rr%)rMrRrrr3r/rSrerfrtoutputsrrrr total_lossloss_fctmasked_lm_lossnext_sentence_losss rPrbzBertForPreTraining.forwards)R$)  ))%'     *1!&48HH_m4\4\11  "5"A'))H%X&7&<& ,,,888*..2.2,0-1&*37A A <$&A  t+A  t+ A lT) A |d* A  t#A #\D0A +,A  u| 7 7A A A ^A A A A A rQrUzP Bert Model with a `language modeling` head on top for CLM fine-tuning. cleZdZdddZfdZdZdZee dd e j dzd e j dzd e j dzd e j dzd e j dzde j dzde j dzde j dzde dzde dzde e j zdeedee j ezfdZxZS)BertLMHeadModelrVrWrXct||jstdt |d|_t||_| dS)NzLIf you want to use `BertLMHeadModel` as a standalone, add `is_decoder=True.`FrD r5r6rloggerwarningr>r$rrZrCrLs rPr6zBertLMHeadModel.__init__Bsu      k NNi j j jf>>> "6** rQc$|jjjSrr\rGs rPr]z%BertLMHeadModel.get_output_embeddingsNr^rQcT||jj_|j|jj_dSrr`ras rPrcz%BertLMHeadModel.set_output_embeddingsQrdrQNrrRrrr3r/rSrrrerrlogits_to_keeprtrUc l|d} |j|f||||||| | dd | } | j}t| trt | dn| }||dd|ddf}d}||jd|||jjd| }t||| j | j | j | j S)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]` NFT) rrr3r/rSrrrrri)logitsrer8)r8r|rrr%r&r<)r$rrrislicerZ loss_functionrNr8rrrr%r&)rMrRrrr3r/rSrrrerrrzrtrjr slice_indicesr|r8s rPrbzBertLMHeadModel.forwardUs.  I@I  A ))%'"7#9+ A A  A A  1 8B>SV8W8Wk~ot444]k -=!!!(;<==  %4%pVFt{OeppioppD0#3!/)$5     rQ) NNNNNNNNNNr)rcrdrerpr6r]rcr$r"rFrjr rrirr!rrrbrkrls@rPrrrr7s+S(>     ,,,888*..2.2,0-1596:&*(,!%-.6 6 <$&6  t+6  t+ 6 lT) 6 |d* 6  %|d26 !& t 36  t#6 6 $;6 el*6 +,6  u| @ @6 6 6 ^6 6 6 6 6 rQrrc>eZdZdddZfdZdZdZee dde j dzd e j dzd e j dzd e j dzd e j dzd e j dzde j dzde j dzde e de e j ezfdZxZS)BertForMaskedLMrVrWrXct||jrtdt |d|_t||_| dS)NzkIf you want to use `BertForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.FrtrurLs rPr6zBertForMaskedLM.__init__s~       NN1    f>>> "6** rQc$|jjjSrr\rGs rPr]z%BertForMaskedLM.get_output_embeddingsr^rQcT||jj_|j|jj_dSrr`ras rPrcz%BertForMaskedLM.set_output_embeddingsrdrQNrRrrr3r/rSrrrertrUc @|j|f||||||dd| } | d} || } d} |Kt}|| d|jj|d} t | | | j| jS)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` T)rrr3r/rSrrrirNr1r8r|rr%) r$rZrrrNr8rrr%)rMrRrrr3r/rSrrrertrjrrrmrls rPrbzBertForMaskedLM.forwards($)   ))%'"7#9      "!* HH_55  '))H%X&7&<&     ,,,888*..2.2,0-1596:&*+ + <$&+  t++  t+ + lT) + |d* +  %|d2+ !& t 3+  t#+ +,+  u| ~ -+ + + ^+ + + + + rQrzT Bert Model with a `next sentence prediction (classification)` head on top. ceZdZfdZee d dejdzdejdzdejdzdejdzdejdzdejdzd ee d e eje zfd Z xZ S) BertForNextSentencePredictionct|t||_t ||_|dSr)r5r6r>r$rrZrCrLs rPr6z&BertForNextSentencePrediction.__init__sQ    f%% "6** rQNrRrrr3r/rSrertrUc (|j|f||||dd|}|d} || } d} |At} | | dd|d} t | | |j|jS)a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see `input_ids` docstring). Indices should be in `[0, 1]`: - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. Example: ```python >>> from transformers import AutoTokenizer, BertForNextSentencePrediction >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased") >>> model = BertForNextSentencePrediction.from_pretrained("google-bert/bert-base-uncased") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light." >>> encoding = tokenizer(prompt, next_sentence, return_tensors="pt") >>> outputs = model(**encoding, labels=torch.LongTensor([1])) >>> logits = outputs.logits >>> assert logits[0, 0] < logits[0, 1] # next sentence was random ``` Trhr'Nr1rwr)r$rZrrrrr%) rMrRrrr3r/rSrertrjrseq_relationship_scoresrnrls rPrbz%BertForNextSentencePrediction.forwardsN$)  ))%'       "&((="9"9!  '))H!)*A*F*Fr1*M*Mv{{[]!_!_ *#*!/)     rQNNNNNN)rcrdrer6r$r"rFrjrr!rrrbrkrls@rPrrs *..2.2,0-1&*= = <$&=  t+=  t+ = lT) = |d* =  t#= +,=  u| : := = = ^= = = = = rQrz Bert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. ceZdZfdZee d dejdzdejdzdejdzdejdzdejdzdejdzd ee d e eje zfd Z xZ S) BertForSequenceClassificationcdt||j|_||_t ||_|j|jn|j}tj ||_ tj |j |j|_ |dSr)r5r6 num_labelsrNr>r$classifier_dropoutrCrrBrDrr9 classifierrCrMrNrrOs rPr6z&BertForSequenceClassification.__init__5s     + f%% )/)B)NF % %TZTn z"455 )F$68IJJ rQNrRrrr3r/rSrertrUc |j|f||||dd|}|d} || } || } d} |Z|jjf|jdkr d|j_nN|jdkr7|jtjks|jtj kr d|j_n d|j_|jjdkrWt} |jdkr1| | | } n| | |} n|jjdkrGt} | | d|j| d} n*|jjdkrt} | | |} t| | |j|j S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Trhr'N regressionsingle_label_classificationmulti_label_classificationr1r)r$rDrrN problem_typerr4rFrKrirsqueezerrrrrr%) rMrRrrr3r/rSrertrjrr|r8rls rPrbz%BertForSequenceClassification.forwardDs$$)  ))%'        ]33 //  {'/?a''/;DK,,_q((flej.H.HFL\a\eLeLe/LDK,,/KDK,{'<77"99?a''#8FNN$4$4fnn6F6FGGDD#8FF33DD)-JJJ+--x B @ @&++b//RR)-III,..x//'!/)     rQr)rcrdrer6r$r"rFrjrr!rrrbrkrls@rPrr.s     *..2.2,0-1&*; ; <$&;  t+;  t+ ; lT) ; |d* ;  t#; +,;  u| 7 7; ; ; ^; ; ; ; ; rQrceZdZfdZee d dejdzdejdzdejdzdejdzdejdzdejdzd ee d e eje zfd Z xZ S) BertForMultipleChoicec4t|t||_|j|jn|j}t j||_t j |j d|_ | dS)Nr') r5r6r>r$rrCrrBrDrr9rrCrs rPr6zBertForMultipleChoice.__init__s    f%% )/)B)NF % %TZTn z"455 )F$6:: rQNrRrrr3r/rSrertrUc R| |jdn |jd}|)|d|dnd}|)|d|dnd}|)|d|dnd}|)|d|dnd}|=|d|d|dnd}|j|f||||dd|} | d} || } || } | d|} d} |t }|| |} t| | | j| j S)a[ input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) Nr'r1Trhr) r[rrJr$rDrrrrr%)rMrRrrr3r/rSrert num_choicesrjrr|reshaped_logitsr8rls rPrbzBertForMultipleChoice.forwardsT-6,Aioa((}GZ[\G] >G>SINN2y~~b'9'9:::Y] M[Mg,,R1D1DR1H1HIIImqM[Mg,,R1D1DR1H1HIIImqGSG_|((\->->r-B-BCCCei (   r=#5#5b#9#9=;M;Mb;Q;Q R R R  $)  ))%'        ]33 // ++b+66  '))H8OV44D("!/)     rQr)rcrdrer6r$r"rFrjrr!rrrbrkrls@rPrrs     *..2.2,0-1&*N N <$&N  t+N  t+ N lT) N |d* N  t#N +,N  u| 8 8N N N ^N N N N N rQrceZdZfdZee d dejdzdejdzdejdzdejdzdejdzdejdzd ee d e eje zfd Z xZ S) BertForTokenClassificationcZt||j|_t|d|_|j|jn|j}tj||_ tj |j |j|_ | dSNFrt)r5r6rr>r$rrCrrBrDrr9rrCrs rPr6z#BertForTokenClassification.__init__s     +f>>> )/)B)NF % %TZTn z"455 )F$68IJJ rQNrRrrr3r/rSrertrUc \|j|f||||dd|}|d} || } || } d} |Ft} | | d|j|d} t | | |j|jS)z labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. TrhrNr1r) r$rDrrrrrrr%) rMrRrrr3r/rSrertrjrr|r8rls rPrbz"BertForTokenClassification.forwards $)  ))%'     "!*,,7711  '))H8FKKDO<r$rrr9 qa_outputsrCrLs rPr6z!BertForQuestionAnswering.__init__$sj     +f>>> )F$68IJJ rQNrRrrr3r/rSstart_positions end_positionsrtrUc F|j|f||||dd|} | d} || } | dd\} } | d} | d} d}||t |dkr|d}t |dkr|d}| d}|d|}|d|}t|}|| |}|| |}||zdz }t|| | | j | j S) NTrhrr'r1rx) ignore_indexrw)r8 start_logits end_logitsrr%) r$rsplitrrlenrJclamprrrr%)rMrRrrr3r/rSrrrtrjrr|rrrk ignored_indexrl start_lossend_losss rPrbz BertForQuestionAnswering.forward.s$)  ))%'     "!*11#)<<r<#:#: j#++B//::<< ''++6688   &=+D?''))**Q.."1"9"9""="==%%''((1,, - 5 5b 9 9 (--a00M-33A}EEO)//=AAM']CCCH!,@@Jx M::H$x/14J+%!!/)     rQro)rcrdrer6r$r"rFrjrr!rrrbrkrls@rPrr"s*..2.2,0-1/3-13 3 <$&3  t+3  t+ 3 lT) 3 |d* 3 ,3 |d*3 +,3  u| ; ;3 3 3 ^3 3 3 3 3 rQr) rrrrUrrrrrrr>r#)Nrm)Zrfcollections.abcr dataclassesrrFrtorch.nnrrrr r) activationsr cache_utilsr r r generationr masking_utilsrrmodeling_layersrmodeling_outputsrrrrrrrrrmodeling_utilsrrprocessing_utilsr pytorch_utilsrutilsr r!r"r# utils.genericr$r%utils.output_capturingr&configuration_bertr( get_loggerrcrvModuler*rjfloatrrrrrrrrrrrrrrrr#r7r>rUrrrrrrrr__all__r<rQrPrs$$$$$$!!!!!! AAAAAAAAAA&&&&&&!!!!!!CCCCCCCCCC))))))JJJJJJJJ999999                      GFFFFFFF&&&&&&666666MMMMMMMMMMMMIIIIIIII555555******  H % %;;;;;RY;;;H!%% I% <% % < % L4' % T\ %%'(%%%%8@)@)@)@)@) @)@)@)FI)I)I)I)I)I)I)I)X     RY   .....BI...:     ry           >>>>>*>>>B     ")   @        ")"     29    !!!!!bi!!!&&&&&bi&&& 9 9 9 9 929 9 9 9/////////2  77777{77   7&    |6|6|6|6|6#|6|6  |6~ Y Y Y Y Y ,Y Y  Y x Q Q Q Q Q )?Q Q  Q hI I I I I )I I I X I I I I I $7I I  I X M M M M M $7M M  M `^ ^ ^ ^ ^ /^ ^ ^ B8 8 8 8 8 !48 8 8 v@ @ @ @ @ 2@ @ @ F   rQ