[TU]5 [R"URUR UR S9Ul[R"URUR 5Ul [R"URUR 5Ul [R"UR URS9Ul [R"UR5UlUR#S[$R&"UR5R)S5SS9 [+USS5Ulg) N) padding_idxeps position_ids)rF) persistentposition_embedding_typeabsolute)super__init__r Embedding vocab_size hidden_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_buffertorcharangeexpandgetattrr#selfconfig __class__s f/var/www/html/shao/venv/lib/python3.13/site-packages/transformers/models/splinter/modeling_splinter.pyr&SplinterEmbeddings.__init__,s !||F,=,=v?Q?Q_e_r_rs#%<<0N0NPVPbPb#c %'\\&2H2H&J\J\%]"f&8&8f>S>STzz&"<"<=   ELL)G)GHOOPWXej  (/v7PR\']$ input_idstoken_type_idsr inputs_embedsreturncUbUR5nOUR5SSnUSnUcURSS2SU24nUc8[R"U[RURR S9nUcUR U5nURU5nXG-nURS:XaURU5n X- nURU5nURU5nU$)Nr!rdtypedevicer$) sizer r6zeroslongrHr+r/r#r-r0r4) r;rArBr rC input_shape seq_lengthr/ embeddingsr-s r>forwardSplinterEmbeddings.forward=s  #..*K',,.s3K ^  ,,Q  ^ J":  ' ': 5"&":":<"H   -J^^J/ \\*- r@)r0r4r#r-r/r+)NNNN)__name__ __module__ __qualname____firstlineno____doc__r&rr6 LongTensor FloatTensortuplerO__static_attributes__ __classcell__r=s@r>rr)sQ^&15593759 E,,-!!1!12u//0    1 12   r@rmodulequerykeyvalueattention_maskscalingr4 head_maskc [R"XRSS55U-n Ub"USS2SS2SS2SURS24n X-n [R R U S[RS9RUR5n [R RXURS9n UbXRSSSS5-n [R"X5n U RSS5R5n X4$)Nr r!)dimrG)ptrainingr)r6matmul transposeshaper functionalsoftmaxfloat32torGr4rhview contiguous) r\r]r^r_r`rar4rbkwargs attn_weights causal_mask attn_outputs r>eager_attention_forwardrv_s<<}}Q':;gEL!$Q1o " o%=> #1 ==((2U]](SVVW\WbWbcL==((6??([L#nnQAq&AA ,,|3K''1-88:K  $$r@c ^\rSrSrU4SjrS S\R S\\RS\\RS\\ S\ \R 4 Sjjr S r U=r $) SplinterSelfAttention{c6>[TU]5 URUR-S:wa7[ US5(d&[ SURSURS35eXlURUl[URUR- 5UlURUR-Ul [R"URUR5Ul [R"URUR5Ul [R"URUR5Ul[R"UR 5UlUR UlURS-Ulg)Nrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()g)r%r&r)num_attention_headshasattr ValueErrorr<intattention_head_size all_head_sizerLinearr]r^r_r2attention_probs_dropout_probr4attention_dropoutrar:s r>r&SplinterSelfAttention.__init__|sD     : : :a ?PVXhHiHi#F$6$6#78 445Q8   #)#=#= #&v'9'9FrOSplinterSelfAttention.forwards[$))#2. CCbC$*B*BC zz-055lCMMaQRS XXm,11,?II!QO zz-055lCMMaQRS (? ;; + +w 6"9$++:Z:Z"[ $7      % $}}C$2H2HLL %  % ! "));;;;FFH 1B;-JUr@) rrrr<r4r^r}r]rar_NNF)rQrRrSrTr&r6TensorrrWboolrXrOrYrZr[s@r>rxrx{st607;15,1 !||!!!2!23!E--. ! $D> ! u|| !!r@rxcz^\rSrSrU4SjrS\R S\R S\R 4SjrSrU=r $)SplinterSelfOutputc(>[TU]5 [R"URUR5Ul[R "URURS9Ul[R"UR5Ul gNr) r%r&rrr)denser0r1r2r3r4r:s r>r&SplinterSelfOutput.__init__s` YYv1163E3EF f&8&8f>S>STzz&"<"<= r@r input_tensorrDcpURU5nURU5nURX-5nU$Nrr4r0r;rrs r>rOSplinterSelfOutput.forward5 =1  ]3 }'CD r@r0rr4 rQrRrSrTr&r6rrOrYrZr[s@r>rr6> U\\RWR^R^r@rc ^\rSrSrU4SjrSrS S\RS\\RS\\RS\\ S\ \R4 S jjr S r U=r$) SplinterAttentionc>[TU]5 [U5Ul[ U5Ul[ 5Ulgr)r%r&rxr;routputset pruned_headsr:s r>r&SplinterAttention.__init__s0 )&1 (0 Er@c6[U5S:Xag[XRRURRUR 5up[ URRU5URl[ URRU5URl[ URRU5URl [ URRUSS9URl URR[U5- URlURRURR-URl UR RU5Ulg)Nrrrf)lenrr;r}rrrr]r^r_rrrunion)r;headsindexs r> prune_headsSplinterAttention.prune_headss  u:? 7 9900$))2O2OQUQbQb  -TYY__eD *499==%@ ,TYY__eD .t{{/@/@%QO )- (E(EE (R %"&))"?"?$))B_B_"_  --33E:r@rr`rbrrDc pUR"U4UUUS.UD6nURUSU5nU4USS-nU$N)r`rbrrr)r;r) r;rr`rbrrr self_outputsattention_outputrs r>rOSplinterAttention.forwards]yy  )/     ;;|A F#% QR(88r@)rrr;r)rQrRrSrTr&rr6rrrWrrXrOrYrZr[s@r>rrsy" ;*7;15,1 ||!!2!23E--.  $D>  u|| r@rcb^\rSrSrU4SjrS\R S\R 4SjrSrU=r $)SplinterIntermediatec>[TU]5 [R"URUR 5Ul[UR[5(a[URUl gURUl gr) r%r&rrr)intermediate_sizer isinstance hidden_actstrr intermediate_act_fnr:s r>r&SplinterIntermediate.__init__s` YYv1163K3KL f'' - -'-f.?.?'@D $'-'8'8D $r@rrDcJURU5nURU5nU$rrr)r;rs r>rOSplinterIntermediate.forwards& =1 00? r@rrr[s@r>rrs(9U\\ellr@rcz^\rSrSrU4SjrS\R S\R S\R 4SjrSrU=r $)SplinterOutputic(>[TU]5 [R"URUR 5Ul[R"UR URS9Ul[R"UR5Ul gr) r%r&rrrr)rr0r1r2r3r4r:s r>r&SplinterOutput.__init__s` YYv779K9KL f&8&8f>S>STzz&"<"<= r@rrrDcpURU5nURU5nURX-5nU$rrrs r>rOSplinterOutput.forward rr@rrr[s@r>rrrr@rc ^\rSrSrU4SjrS S\R S\\RS\\RS\\ S\ \R 4 Sjjr S r S r U=r$) SplinterLayeric>[TU]5 URUlSUl[ U5Ul[ U5Ul[U5Ul g)Nr) r%r&chunk_size_feed_forward seq_len_dimr attentionr intermediaterrr:s r>r&SplinterLayer.__init__sI '-'E'E$*6208$V, r@rr`rbrrDc UR"U4UUUS.UD6nUSnUSSn[URURURU5n U 4U-nU$r)rrfeed_forward_chunkrr) r;rr`rbrrrself_attention_outputsrr layer_outputs r>rOSplinterLayer.forwards"& " )/ "  " 2!4(,0  # #T%A%A4CSCSUe  /G+r@cJURU5nURX!5nU$r)rr)r;rintermediate_outputrs r>r SplinterLayer.feed_forward_chunk4s)"//0@A{{#6I r@)rrrrrr)rQrRrSrTr&r6rrrWrrXrOrrYrZr[s@r>rrsy-7;15,1 ||!!2!23E--.  $D>  u|| 2r@rc^\rSrSrU4Sjr\S S\RS\\RS\\RS\\ S\\ S\\ S \ \ \R\ 44S jj5rS rU=r$) SplinterEncoderi;c>[TU]5 Xl[R"[ UR 5Vs/sHn[U5PM sn5UlSUl gs snf)NF) r%r&r<r ModuleListrangenum_hidden_layersrlayergradient_checkpointing)r;r<ir=s r>r&SplinterEncoder.__init__<sR  ]]5IaIaCb#cCbaM&$9Cb#cd &+#$dsA%rr`rbroutput_hidden_states return_dictrDc U(aSOSnU(aSOSn [UR5H=upU(aX4-nUbX:OSn U "SUUU US.UD6n U SnU(dM5XS4-n M? U(aX4-n[UUU S9$)N)rr`rbrrrlast_hidden_stater attentions) enumeraterr )r;rr`rbrrrrrall_hidden_statesall_self_attentionsr layer_modulelayer_head_mask layer_outputss r>rOSplinterEncoder.forwardBs#7BD$5b4(4OA#$58H$H!.7.CilO(+-)"3   M*!,M  &91=Mrr;s, 7;15,1/4&*& ||& !!2!23& E--. & $D> & 'tn & d^&  uU\\"O3 4& & r@rc.\rSrSr%\\S'SrSrSrSr g)SplinterPreTrainedModelilr<splinterTc[U[R5(akURRR SUR RS9 URb%URRR5 gg[U[R5(axURRR SUR RS9 URb2URRURR5 gg[U[R5(aJURRR5 URRRS5 gg)zInitialize the weightsr)meanstdNg?)rrrweightdatanormal_r<initializer_rangebiaszero_r'rr0fill_)r;r\s r> _init_weights%SplinterPreTrainedModel._init_weightsrs fbii ( ( MM   & &CT[[5R5R & S{{&   &&('  - - MM   & &CT[[5R5R & S!!- ""6#5#56<<>.  - - KK   " " $ MM   $ $S ).r@rN) rQrRrSrTr__annotations__base_model_prefixsupports_gradient_checkpointingrrYrr@r>rrls "&*#*r@rcR^\rSrSrSrU4SjrSrSrSr\ \ SS\ \ RS\ \ RS \ \ RS \ \ RS \ \ RS \ \ RS \ \S\ \S\ \S\\\44Sjj55rSrU=r$) SplinterModelia" The model is an encoder (with only self-attention) following the architecture described in [Attention is all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. c>[TU]U5 Xl[U5Ul[ U5UlUR5 gr)r%r&r<rrNrencoder post_initr:s r>r&SplinterModel.__init__s9   ,V4&v.  r@c.URR$rrNr+)r;s r>get_input_embeddings"SplinterModel.get_input_embeddingss...r@c$XRlgrr)r;r_s r>set_input_embeddings"SplinterModel.set_input_embeddingss */'r@cUR5H7up#URRURR U5 M9 g)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrrrr)r;heads_to_prunerrs r> _prune_headsSplinterModel._prune_headss< +002LE LL  u % / / ; ;E B3r@rAr`rBr rbrCrrrrDc UbUOURRnUbUOURRnU bU OURRn UbUb [ S5eUb"UR X5 UR 5n O"UbUR 5SSn O [ S5eU upUb URO URn Uc[R"X4U S9nUc$[R"U [RU S9nURX*5nURXPRR5nURUUUUS9nUR!UUUUUSS 9nUS n[#UUR$UR&S 9$) a token_type_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) NzDYou cannot specify both input_ids and inputs_embeds at the same timer!z5You have to specify either input_ids or inputs_embeds)rHrF)rAr rBrCT)r`rbrrrrr)r<rruse_return_dictr%warn_if_padding_and_no_attention_maskrIrHr6onesrJrKget_extended_attention_mask get_head_maskrrNrr rr)r;rAr`rBr rbrCrrrrL batch_sizerMrHextended_attention_maskembedding_outputencoder_outputssequence_outputs r>rOSplinterModel.forwards:2C1N-TXT_T_TqTq$8$D $++JjJj &1%cd d  "  6 6y Q#..*K  &',,.s3KTU U!, %.%:!! @T@T  !"ZZ*)A6RN  !"[[EJJvVN150P0PQ_0m&&y++2O2OP ??%)' +  ,, 2/!5 ' *!,-)77&11  r@)r<rNr) NNNNNNNNN)rQrRrSrTrUr&rrrrrrr6rrrrXr rOrYrZr[s@r>r r s /0C-11515/3,004,0/3&*R ELL)R !.R !. R u||, R ELL) R   -R $D>R 'tnR d^R  uo% &R R r@r cf^\rSrSrSU4SjjrS\R S\R 4SjrSrU=r $)SplinterFullyConnectedLayeric>[TU]5 XlX l[R "URUR5Ul[UUl[R"UR5Ul gr) r%r& input_dim output_dimrrrr act_fnr0)r;r,r-rr=s r>r&$SplinterFullyConnectedLayer.__init__sR "$YYt~~t? Z( doo6r@inputsrDclURU5nURU5nURU5nU$r)rr.r0)r;r0rs r>rO#SplinterFullyConnectedLayer.forwards2 6*  M2 }5 r@)r0r.rr,r-)gelurr[s@r>r*r*s(7ellu||r@r*c2^\rSrSrSrU4SjrSrSrU=r$)QuestionAwareSpanSelectionHeadi z^ Implementation of Question-Aware Span Selection (QASS) head, described in Splinter's paper: c>[TU]5 [URUR5Ul[URUR5Ul[URUR5Ul[URUR5Ul[R"URURSS9Ul [R"URURSS9Ul g)NF)r) r%r&r*r)query_start_transformquery_end_transformstart_transform end_transformrrstart_classifierend_classifierr:s r>r&'QuestionAwareSpanSelectionHead.__init__s %@ASASU[UgUg%h"#>v?Q?QSYSeSe#f :6;M;MvOaOab89K9KVM_M_` " &*<*P>PW\ ] ii(:(:FrO&QuestionAwareSpanSelectionHead.forwardsKKM 1##B'..q!S9 V%@ 55mD11-@))&1 %%f---.>? ''1a0 ||M> ++N; ##Aq!,\\-: ''r@)r<r:r8r7r;r9) rQrRrSrTrUr&rOrYrZr[s@r>r5r5 s \((r@r5c^\rSrSrU4Sjr\SS\\RS\\RS\\RS\\RS\\RS\\RS \\RS \\RS \\ S \\ S \\ S\\RS\ \ \ 44Sjj5rSrU=r$)SplinterForQuestionAnsweringi2c>[TU]U5 [U5Ul[ U5UlUR UlUR5 grr%r&r rr5 splinter_qassquestion_token_idrr:s r>r&%SplinterForQuestionAnswering.__init__4C  %f- ;FC!'!9!9 r@rAr`rBr rbrCstart_positions end_positionsrrrquestion_positionsrDc U bU OURRn Sn U cUbB[R"[R"XR 5R 5SS9nOH[R"URS5[RURURS9nURS5n Sn URUUUUUUU U U S9 nUSnURUU 5unnU (a"URS 5URS 5nnUbfUS U- [R "UR"5R$--nUS U- [R "UR"5R$--nSnUbUb['UR55S :aURS5n['UR55S :aURS5nURS 5nUR)SU5 UR)SU5 [+US 9nU"UU5nU"UU5nUU-S - nU (dUU4US S-nUbU4U-$U$[-UUUUR.UR0S 9$) aI token_type_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) question_positions (`torch.LongTensor` of shape `(batch_size, num_questions)`, *optional*): The positions of all question tokens. If given, start_logits and end_logits will be of shape `(batch_size, num_questions, sequence_length)`. If None, the first question token in each sequence in the batch will be the only one for which start_logits and end_logits are calculated and they will be of shape `(batch_size, sequence_length)`. NFr!rr)rGlayoutrHTr`rBr rbrCrrrr ignore_indexrdlossrJrKrr)r<rr6argmaxeqrRrrJrIrKrYrHr?rrQsqueezefinforGminrclamp_rrrr)r;rAr`rBr rbrCrUrVrrrrWquestion_positions_were_none"question_position_for_each_examplerr'rJrK total_loss ignored_indexloss_fct start_lossend_lossrs r>rO$SplinterForQuestionAnswering.forward>sH&1% @R@RST@U*L  %'1~+=\M_M_A`AdAd*ddL#q>'9U[[IYIY=Z=^=^&^^J  &=+D?'')*Q."1"9"9""==%%'(1, - 5 5b 9 (--a0M  " "1m 4  M 2']CH!,@J M:H$x/14J"J/'!"+=F/9/EZMF* Q6 Q+%!!//))   r@rRrrQ NNNNNNNNNNNN)rQrRrSrTr&rrr6rrVrrrXrrOrYrZr[s@r>rNrN2s?-11515/3,0046:48,0/3&*9=c ELL)c !.c !. c u||, c ELL) c   -c "%"2"23c   0 01c $D>c 'tnc d^c %U%5%56c  u22 3c c r@rNzB Class for outputs of Splinter as a span selection model. ) custom_introc\rSrSr%SrSr\\R\ S'Sr \\R\ S'Sr \\R\ S'Sr \\ \R\ S'Sr\\ \R\ S'S rg) SplinterForPreTrainingOutputia loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when start and end positions are provided): Total span extraction loss is the sum of a Cross-Entropy for the start and end positions. start_logits (`torch.FloatTensor` of shape `(batch_size, num_questions, sequence_length)`): Span-start scores (before SoftMax). end_logits (`torch.FloatTensor` of shape `(batch_size, num_questions, sequence_length)`): Span-end scores (before SoftMax). Nr^rJrKrrr)rQrRrSrTrUr^rr6rWrrJrKrrXrrYrr@r>rqrqs|)-D(5$$ %,04L(5,,-4.2J**+28[TU]U5 [U5Ul[ U5UlUR UlUR5 grrPr:s r>r&SplinterForPreTraining.__init__rTr@rAr`rBr rbrCrUrVrrrrWrDc rU bU OURRn U cUbUb [S5eU cUc [S5eU cURU5n UR UUUUUUU U U S9 n U SnUR 5unnnUR X5unnU R S5nUbURS5RUUU5nUSU- [R"UR5R--nUSU- [R"UR5R--nSnUbUbURS[SUS- 55 URS[SUS- 55 [URR S9nU"UR#UU-U5UR#UU-55nU"UR#UU-U5UR#UU-55nUU-S- nU (dUU4U SS-nUbU4U-$U$[%UUUU R&U R(S 9$) a input_ids (`torch.LongTensor` of shape `(batch_size, num_questions, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) token_type_ids (`torch.LongTensor` of shape `batch_size, num_questions, sequence_length`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `batch_size, num_questions, sequence_length`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_questions, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert *input_ids* indices into associated vectors than the model's internal embedding lookup matrix. start_positions (`torch.LongTensor` of shape `(batch_size, num_questions)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. end_positions (`torch.LongTensor` of shape `(batch_size, num_questions)`, *optional*): Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. question_positions (`torch.LongTensor` of shape `(batch_size, num_questions)`, *optional*): The positions of all question tokens. If given, start_logits and end_logits will be of shape `(batch_size, num_questions, sequence_length)`. If None, the first question token in each sequence in the batch will be the only one for which start_logits and end_logits are calculated and they will be of shape `(batch_size, sequence_length)`. NzCquestion_positions must be specified in order to calculate the lossz>question_positions must be specified when input_embeds is usedrZrrr[rdr])r<r TypeError_prepare_question_positionsrrIrQr?r8r6rbrGrcrdmaxrr*rprqrr)r;rAr`rBr rbrCrUrVrrrrWrr'r#sequence_lengthrfrJrK num_questions attention_mask_for_each_questionrgrirjrkrs r>rOSplinterForPreTraining.forwardsn&1%$? @ (T[[5M5MNH!!!*}"H%x/14J"J/'!"+=F/9/EZMF* Q6 Q+%!!//))   r@c[R"XRR:H5up#[R"U5n[R "UR S5UR54URR[RURS9n[R"UVs/sHn[R"U5PM sn5nX5X'4'U$s snf)NrrF) r6wherer<rRbincountfullrIryr*rKrHcatr7)r;rArowsflat_positionsr{rCncolss r>rx2SplinterForPreTraining._prepare_question_positionsKs${{9 8U8U+UVt, JJ ^^A  1 1 3 4 KK $ $**##  yy=A=a%,,q/=AB .$*Bs;C(rmrn)rQrRrSrTr&rrr6rrVrrrXrqrOrxrYrZr[s@r>rsrss`-11515/3,0046:48,0/3&*9=z ELL)z !.z !. z u||, z ELL) z   -z "%"2"23z   0 01z $D>z 'tnz d^z %U%5%56z  u22 3z z x U\\ ell  r@rs)rNrsrr r)rN)8rU dataclassesrtypingrrrr6torch.utils.checkpointrtorch.nnr activationsr modeling_layersr modeling_outputsr r rmodeling_utilsrr pytorch_utilsrrrutilsrrrconfiguration_splinterr get_loggerrQloggerModulerrfloatrvrxrrrrrrrr r*r5rNrqrs__all__rr@r>rs,!,, %!9ZZFll 3   H %22z(,% II% <<% % << % U\\* %  %% %%87BII7v  * *\ 299  RYY %.%R. bii. b*o**,s +s s l"))$#(RYY#(Lo #:o o d  :;:  :"S4SSl r@