tjdZddlZddlZddlZddlmZddlmcmZddlm Z m Z m Z ddl m ZddlmZddlmZddlmZdd lmZmZmZmZdd lmZdd lmZdd lmZdd l m!Z!m"Z"m#Z#m$Z$ddl%m&Z&m'Z'ddl(m)Z)ddl*m+Z+e#j,e-Z.Gddej/Z0Gddej/Z1Gddej/Z2Gddej/Z3Gddej/Z4GddeZ5Gddej/Z6Gd d!ej/Z7Gd"d#ej/Z8e!Gd$d%eZ9e!Gd&d'e9Z:Gd(d)ej/Z;e!d*+Gd,d-e9Zgd3Z?dS)4zPyTorch LayoutLMv3 model.N)BCEWithLogitsLossCrossEntropyLossMSELoss)initialization)ACT2FN)create_bidirectional_mask)GradientCheckpointingLayer)BaseModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)Unpack)apply_chunking_to_forward)auto_docstringcan_return_tuplelogging torch_int)TransformersKwargsmerge_with_config_defaults)capture_outputs)LayoutLMv3Configc*eZdZdZfdZddZxZS)LayoutLMv3PatchEmbeddingszLayoutLMv3 image (patch) embeddings. This class also automatically interpolates the position embeddings for varying image sizes.ctt|jtjjr|jn |j|jf}t|jtjjr|jn |j|jf}|d|dz|d|dzf|_tj |j |j |||_ dS)Nrr) kernel_sizestride)super__init__ isinstance input_size collectionsabcIterable patch_size patch_shapennConv2d num_channels hidden_sizeproj)selfconfig image_sizer' __class__s p/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/layoutlmv3/modeling_layoutlmv3.pyr!z"LayoutLMv3PatchEmbeddings.__init__7s &+[_-EFF 8F  #V%67 &+[_-EFF 8F  #V%67  'qMZ]:JqMZXY]STTTz&"<==   EL)GHHOOPWXXej    ".#%<  *F,>DL\$ $ $  &(\&2SU[Uk%l%l"%'\&2SU[Uk%l%l"%'\&2SU[Uf%g%g"%'\&2SU[Uf%g%g"""r3c  ||dddddf}||dddddf}||dddddf}||dddddf}n"#t$r}td|d}~wwxYw|t j|dddddf|dddddfz dd}|t j|dddddf|dddddfz dd}t j||||||gd} | S) Nrrr6rz;The `bbox` coordinate values should be within 0-1000 range.ir5dim)rlrm IndexErrorrorecliprpcat) r.bboxleft_position_embeddingsupper_position_embeddingsright_position_embeddingslower_position_embeddingserorpspatial_position_embeddingss r2%calculate_spatial_position_embeddingsz>LayoutLMv3TextEmbeddings.calculate_spatial_position_embeddingsrs c'+'A'A$qqq!!!Qw-'P'P $(,(B(B4111a=(Q(Q %(,(B(B4111a=(Q(Q %(,(B(B4111a=(Q(Q % % c c cZ[[ab b c!% : :5:d111aaaQR7mVZ[\[\[\^_^_^_ab[bVcFcefhl;m;m n n $ : :5:d111aaaQR7mVZ[\[\[\^_^_^_ab[bVcFcefhl;m;m n n',i()))%%   ' ' ' #+*sBB B*B%%B*c||}tj|d||z}||zS)z Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. This is modified from fairseq's `utils.make_positions`. rrs)neintrecumsumtype_aslong)r. input_idsrRmaskincremental_indicess r2"create_position_ids_from_input_idsz;LayoutLMv3TextEmbeddings.create_position_ids_from_input_idss` ||K((,,..$|Da888@@FF$N"''))K77r3c|dd}|d}tj|jdz||jzdztj|j}|d|S)z We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids. Nr5rdtypedevicer)r8rerfrRrr unsqueezerh)r. inputs_embeds input_shapesequence_lengthrUs r2&create_position_ids_from_inputs_embedsz?LayoutLMv3TextEmbeddings.create_position_ids_from_inputs_embedss$((**3B3/ %a.|  q /D4D"Dq"HPUPZcpcw   %%a((// << J J"%:: "66|DD)) &*&P&PQU&V&V#"== ^^J// \\*-- r3NNNNN) rIrJrKrLr!rrrrFrMrNs@r2rPrPVshhhhh.+++4888 = = =  ''''''''r3rPcFeZdZfdZddZ ddeefdZxZS) LayoutLMv3SelfAttentionct|j|jzdkr0t |ds t d|jd|jd|j|_t |j|jz |_|j|jz|_tj |j|j|_ tj |j|j|_ tj |j|j|_ tj|j|_|j|_|j|_dS)Nrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ())r r!r,num_attention_headshasattr ValueErrorrattention_head_size all_head_sizer)Linearquerykeyvalueraattention_probs_dropout_probrchas_relative_attention_biashas_spatial_attention_biasrqs r2r!z LayoutLMv3SelfAttention.__init__s1    : :a ? ?PVXhHiHi ?8F$688 4888  $*#= #&v'9FS"S* *,CD o--r3)rNNN) rIrJrKr!rrrrFrMrNs@r2rrsLLLLL( 8 8 8 8 5.5. +, 5.5.5.5.5.5.5.5.r3rcPeZdZfdZdejdejdejfdZxZS)LayoutLMv3SelfOutputcttj|j|j|_tj|j|j|_tj|j |_ dSNrS) r r!r)rr,denser_r`rarbrcrqs r2r!zLayoutLMv3SelfOutput.__init__'sf Yv163EFF f&8f>STTTz&"<== r3r input_tensorreturnc||}||}|||z}|SrGrrcr_r.rrs r2rFzLayoutLMv3SelfOutput.forward-@ =11  ]33 }|'CDD r3rIrJrKr!reTensorrFrMrNs@r2rr&i>>>>> U\RWR^r3rc>eZdZfdZ ddeefdZxZS)LayoutLMv3Attentionctt||_t ||_dSrG)r r!rr.routputrqs r2r!zLayoutLMv3Attention.__init__6s; +F33 *622 r3Nrc `|}|j||f||d|\}}|||}|SNrr)r.r) r.rrrrrresidualattention_output_s r2rFzLayoutLMv3Attention.forward;sa!'di   !      ! ;;'7BBr3r)rIrJrKr!rrrFrMrNs@r2rr5sl33333   +,         r3rcFeZdZfdZ ddeefdZdZxZS)LayoutLMv3Layerct|j|_d|_t ||_t ||_t||_ dSNr) r r!chunk_size_feed_forward seq_len_dimr attentionLayoutLMv3Intermediate intermediateLayoutLMv3Outputrrqs r2r!zLayoutLMv3Layer.__init__Qs^ '-'E$,V4426::&v.. r3NFrc z|||||}t|j|j|j|}|Sr)rrfeed_forward_chunkrr) r.rroutput_attentionsrrrr layer_outputs r2rFzLayoutLMv3Layer.forwardYsU >>  ! *  1  #T%A4CSUe  r3c\||}|||}|SrG)rr)r.rintermediate_outputrs r2rz"LayoutLMv3Layer.feed_forward_chunkos2"//0@AA{{#68HII r3)NFNN) rIrJrKr!rrrFrrMrNs@r2rrPs~///// +,,r3rcVeZdZfdZd dZdZdZ d d eefd Z xZ S) LayoutLMv3Encodercht|_tjfdt jD|_d|_j |_ j |_ |j r>j |_ j |_ tj |j jd|_|j rfj|_j|_tj |jjd|_tj |jjd|_dSdS)Nc.g|]}tS)r).0rr/s r2 z.LayoutLMv3Encoder.__init__..ys!#e#e#eOF$;$;#e#e#er3F)bias)r r!r/r) ModuleListrangenum_hidden_layerslayergradient_checkpointingrr rel_pos_bins max_rel_posrr rel_pos_biasmax_rel_2d_posrel_2d_pos_binsrel_pos_x_biasrel_pos_y_biasrqs `r2r!zLayoutLMv3Encoder.__init__vs&  ]#e#e#e#eU6KcEdEd#e#e#eff &+#+1+M(*0*K'  + e & 3D %1D  " $*;V=W^c d d dD   * j"("7D #)#9D "$)D,@&B\ch"i"i"iD "$)D,@&B\ch"i"i"iD     j jr3Trc:d}|r8|dz}||dk|zz }tj|}n(tj| tj|}|dz}||k}|tj||z tj||z z ||z ztjz} tj | tj | |dz } |tj ||| z }|S)Nrr6r) rreabsmax zeros_likelogfloatrrmin full_likewhere) r.relative_position bidirectional num_buckets max_distanceretn max_exactis_small val_if_larges r2relative_position_bucketz*LayoutLMv3Encoder.relative_position_buckets  S A K %)//11K? ?C +,,AA ,,e.>?P.Q.QRRA 1$ y=! Iaggii)+ , ,tx y8P/Q/Q QU`clUl m "UZ.. yu|[[\_/]/]^^  u{8Q 555 r3c|d|dz }|||j|j}t j5|jj| dddd}dddn #1swxYwY| }|S)Nrr5rrrrrr6) rrrrreno_gradrweighttr;r)r.rU rel_pos_matrs r2_cal_1d_pos_embz!LayoutLMv3Encoder._cal_1d_pos_embs ",,R00<3I3I"3M3MM // ))0  ]__ P P'.00227;CCAq!QOOG P P P P P P P P P P P P P P P$$&&s!;B((B,/B,c|dddddf}|dddddf}|d|dz }|d|dz }|||j|j}|||j|j}t j5|jj| dddd}|j j| dddd}dddn #1swxYwY| }| }||z}|S)Nrrrr5rrr6) rrrrrerrrrr;rr) r.rxposition_coord_xposition_coord_yrel_pos_x_2d_matrel_pos_y_2d_mat rel_pos_x rel_pos_yrs r2_cal_2d_pos_embz!LayoutLMv3Encoder._cal_2d_pos_embs111a=111a=+55b99STTTz&"<== r3rrrc||}||}|||z}|SrGrrs r2rFzLayoutLMv3Output.forwardrr3rrNs@r2rrrr3rcfeZdZUeed<dZdZeedZ e j fdZ xZ S)LayoutLMv3PreTrainedModelr/ layoutlmv3)imagetext)r attentionscXt|t|tr|jjr2t j|jt j|j t|dr)"r r!r/ text_embedrPrCr>r patch_embedrr#r'r8r) Parameterrerr,rArBrapos_dropr_r`rbrcrrrdrDnormrencoder post_initrqs r2r!zLayoutLMv3Model.__init__s       ?6v>>DO   C 9@@D F-0AABBDI\%+aFs*/'''r3r\c  tjtjd||ddzz||dd}tjtjd||ddzz||dd}tj|dd|dd|dd|dddd|dd|dd|dd|ddddgddd}tjdd|dz |dz gg}tj||gdS) zJ Create the bounding boxes for the visual (patch) tokens. rrtrunc) rounding_modeNr5rs) redivrfstackrepeatr@r:tensorrw)r.r0max_len visual_bbox_x visual_bbox_yr: cls_token_boxs r2rDz"LayoutLMv3Model.create_visual_bboxAs LGz!}q'897 C CZPQ]bi    LGz!}q'897 C CZPQ]bi   kcrc"))*Q-;;crc"))*Q-;;EEaKKabb!((A::abb!((A::DDQJJ       $r1++  ueWq['A+&N%OPP y-51====r3c|j|dd}|||}|Sr)r:rdrtype)r.rrrr:s r2calculate_visual_bboxz%LayoutLMv3Model.calculate_visual_bboxXsA&--j!Q?? !nnV,,11%88 r3c>||}|\}}}|j|dd}t j||fd}|j ||jz}||}||}|S)Nr5rrs) rNr8rArhrerwrBrPrQ)r.rArCrseq_lenr cls_tokenss r2 forward_imagezLayoutLMv3Model.forward_image]s%%l33 ",!2!2 GQ^**:r2>> Y J7Q???  > %#dn4J]]:.. YYz** r3NrrxrrrUrrArrc |!|} | \} } |j} nS|)|dd} | \} } |j} n(|t|} |j} ntd|||t j| | f| }|!t j| tj| }|?t jtt| dgztj| }| |||||} dx}}dx}}|t|j d|j jz t|j d |j jz }}||}t j| |j d ftj| }|t j||gd }n|}|j js |j jr|j jr?|| tj| }|t j||gd }n|}t jd |j d tj| | d }||^t jd | d | d }|| }t j||gd }n|}||t j| |gd } n|} || } || } ng|j js |j jrO|j jr|}|j jr5|j jddd| d f}||}|}t9|j | |}|j| f|||||d|}|j}t?|S)a input_ids (`torch.LongTensor` of shape `(batch_size, token_sequence_length)`): Indices of input sequence tokens in the vocabulary. Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] token. See `pixel_values` for `patch_sequence_length`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) bbox (`torch.LongTensor` of shape `(batch_size, token_sequence_length, 4)`, *optional*): Bounding boxes of each input sequence tokens. Selected in the range `[0, config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, y1) represents the position of the lower right corner. Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] token. See `pixel_values` for `patch_sequence_length`. token_type_ids (`torch.LongTensor` of shape `(batch_size, token_sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] token. See `pixel_values` for `patch_sequence_length`. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `(batch_size, token_sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] token. See `pixel_values` for `patch_sequence_length`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, token_sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert *input_ids* indices into associated vectors than the model's internal embedding lookup matrix. Examples: ```python >>> from transformers import AutoProcessor, AutoModel >>> from datasets import load_dataset >>> processor = AutoProcessor.from_pretrained("microsoft/layoutlmv3-base", apply_ocr=False) >>> model = AutoModel.from_pretrained("microsoft/layoutlmv3-base") >>> dataset = load_dataset("nielsr/funsd-layoutlmv3", split="train") >>> example = dataset[0] >>> image = example["image"] >>> words = example["tokens"] >>> boxes = example["bboxes"] >>> encoding = processor(image, words, boxes=boxes, return_tensors="pt") >>> outputs = model(**encoding) >>> last_hidden_states = outputs.last_hidden_state ```Nr5zEYou have to specify either input_ids or inputs_embeds or pixel_values)rrra)rrxrUrrr6rrrs)rrr)r/rr)rxrUrrDrEr&) r8rlenrreonesrrtuplelistrCrr<r/r'rprwrrrlrfrdrrhr_rcrU expand_asr rRr'r )r.rrxrrrUrrArrr seq_lengthrembedding_output final_bboxfinal_position_idsrDrEvisual_embeddingsvisual_attention_maskr:visual_position_idsencoder_outputssequence_outputs r2rFzLayoutLMv3Model.forwardns_Z  #..**K%0 "J %FF  &',,..ss3K%0 "J ")FF  %\**J!(FFdee e  M$=%!&j*-Ev!V!V!V%!&[ SY!Z!Z!Z|{5k):):aS)@#A#A\bccc##)-+ /   +/. '%)) {  #,,Q/$+2HHII,,Q/$+2HHII&L!% 2 2< @ @ $)J.4Q78 SY%%% !)!&N&#Q#Q#Q#[#[\]#^#^L#/#6#6{#C#CL).LBU3V\])^)^)^&&)<&$ (A#(9.>@Q-RXY#Z#Z#Z  #4 #~~.>?? #||,<==   [ 4 2 8^ 2{5 "! {6 2#;AAA?OQ?O///000>>>>. " .2(,3726042615m m #d*m %m )D0 m (4/ m &- m (4/m '$.m +,m   m m m ^_ m m m m m r3r=c*eZdZdZdfd ZdZxZS)LayoutLMv3ClassificationHeadz\ Head for sentence-level classification tasks. Reference: RobertaClassificationHead Fct||_|r(tj|jdz|j|_n$tj|j|j|_|j|jn|j}tj ||_ tj|j|j |_ dS)Nr) r r! pool_featurer)rr,rclassifier_dropoutrbrarc num_labelsout_proj)r.r/rrr1s r2r!z%LayoutLMv3ClassificationHead.__init__&s (  K6#5#96;MNNDJJ6#5v7IJJDJ)/)B)NF % %TZTn z"455  &"4f6GHH r3c||}||}tj|}||}||}|SrG)rcrretanhr)r.xs r2rFz$LayoutLMv3ClassificationHead.forward3sR LLOO JJqMM JqMM LLOO MM!  r3)FrHrNs@r2rr!s\ I I I I I Ir3ra LayoutLMv3 Model with a token classification head on top (a linear layer on top of the final hidden states) e.g. for sequence labeling (information extraction) tasks such as [FUNSD](https://guillaumejaume.github.io/FUNSD/), [SROIE](https://rrc.cvc.uab.es/?ch=13), [CORD](https://github.com/clovaai/cord) and [Kleister-NDA](https://github.com/applicaai/kleister-nda). ) custom_introceZdZfdZdZdZee ddej dzdej dzdej dzdej dzd ej dzd ej dzd ej dzd ej dzd e e de ezfdZxZS) LayoutLMv3ForTokenClassificationczt||j|_t||_t j|j|_|jdkr%t j |j |j|_ nt|d|_ | dS)N Fr)r r!rr=r5r)rarbrcrr, classifierrrSrqs r2r!z)LayoutLMv3ForTokenClassification.__init__Es     +)&11z&"<==  r ! ! i(:F>> from transformers import AutoProcessor, AutoModelForTokenClassification >>> from datasets import load_dataset >>> processor = AutoProcessor.from_pretrained("microsoft/layoutlmv3-base", apply_ocr=False) >>> model = AutoModelForTokenClassification.from_pretrained("microsoft/layoutlmv3-base", num_labels=7) >>> dataset = load_dataset("nielsr/funsd-layoutlmv3", split="train") >>> example = dataset[0] >>> image = example["image"] >>> words = example["tokens"] >>> boxes = example["bboxes"] >>> word_labels = example["ner_tags"] >>> encoding = processor(image, words, boxes=boxes, word_labels=word_labels, return_tensors="pt") >>> outputs = model(**encoding) >>> loss = outputs.loss >>> logits = outputs.logits ```)rxrrrUrrANr5rrlosslogitsrr8) r5r8rcrrr:rrrr8)r.rrxrrrUrrrAroutputsrrwrrrloss_fcts r2rFz(LayoutLMv3ForTokenClassification.forwardXs#Z"$/   ))%'%        #..**KK',,..ss3K ^ !!*QQQ  ^4,,7711  '))H8FKKDO<>> from transformers import AutoProcessor, AutoModelForQuestionAnswering >>> from datasets import load_dataset >>> import torch >>> processor = AutoProcessor.from_pretrained("microsoft/layoutlmv3-base", apply_ocr=False) >>> model = AutoModelForQuestionAnswering.from_pretrained("microsoft/layoutlmv3-base") >>> dataset = load_dataset("nielsr/funsd-layoutlmv3", split="train") >>> example = dataset[0] >>> image = example["image"] >>> question = "what's his name?" >>> words = example["tokens"] >>> boxes = example["bboxes"] >>> encoding = processor(image, question, words, boxes=boxes, return_tensors="pt") >>> start_positions = torch.tensor([1]) >>> end_positions = torch.tensor([3]) >>> outputs = model(**encoding, start_positions=start_positions, end_positions=end_positions) >>> loss = outputs.loss >>> start_scores = outputs.start_logits >>> end_scores = outputs.end_logits ```rrrUrrxrArrr5rsN) ignore_indexr6)r start_logits end_logitsrr8) r5rsplitsqueezerrrr8clamprr rr8)r.rrrrUrrrrxrArrrrrr total_loss ignored_indexr start_lossend_losss r2rFz&LayoutLMv3ForQuestionAnswering.forwards`$34?  $ ))%'% $ $  $ $ "!*11#)<<r<#:#: j#++B//::<< ''++6688   &=+D?''))**Q.."1"9"9""="==%%''((1,, - 5 5b 9 9 (--a00M-33A}EEO)//=AAM']CCCH!,@@Jx M::H$x/14J+%!!/)     r3) NNNNNNNNN)rIrJrKr!rWrZrrrerrrrrtr rFrMrNs@r2rrsw666444.2372604263715(,04W W #d*W )D0W (4/ W &- W (4/ W )D0W '$.W %W &-W +,W  - -W W W ^W W W W W r3ra  LayoutLMv3 Model with a sequence classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for document image classification tasks such as the [RVL-CDIP](https://www.cs.cmu.edu/~aharley/rvl-cdip/) dataset. ceZdZfdZdZdZee ddej dzdej dzdej dzdej dzd ej dzd ej dzd ej dzd ej dzd e e de ezfdZxZS)#LayoutLMv3ForSequenceClassificationct||j|_||_t ||_t |d|_|dSr) r r!rr/r=r5rrrSrqs r2r!z,LayoutLMv3ForSequenceClassification.__init__se     + )&116vERRR r3c4|jSrGrrVs r2rWz8LayoutLMv3ForSequenceClassification.get_input_embeddings%rr3c:|j|dSrGrrYs r2rZz8LayoutLMv3ForSequenceClassification.set_input_embeddings(rr3NrrrrUrrrxrArrc x|j|f||||||d| } | ddddddf} || } d} |Z|jjf|jdkr d|j_nN|jdkr7|jt jks|jt jkr d|j_n d|j_|jjdkrWt}|jdkr1|| | } n|| |} n|jjdkrGt}|| d|j| d} n*|jjdkrt}|| |} t| | | j| j S) a_ bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): Bounding boxes of each input sequence tokens. Selected in the range `[0, config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, y1) represents the position of the lower right corner. Examples: ```python >>> from transformers import AutoProcessor, AutoModelForSequenceClassification >>> from datasets import load_dataset >>> import torch >>> processor = AutoProcessor.from_pretrained("microsoft/layoutlmv3-base", apply_ocr=False) >>> model = AutoModelForSequenceClassification.from_pretrained("microsoft/layoutlmv3-base") >>> dataset = load_dataset("nielsr/funsd-layoutlmv3", split="train") >>> example = dataset[0] >>> image = example["image"] >>> words = example["tokens"] >>> boxes = example["bboxes"] >>> encoding = processor(image, words, boxes=boxes, return_tensors="pt") >>> sequence_label = torch.tensor([1]) >>> outputs = model(**encoding, labels=sequence_label) >>> loss = outputs.loss >>> logits = outputs.logits ```rrNr regressionsingle_label_classificationmulti_label_classificationr5r)r5rr/ problem_typerrrerrrrrr:rr rr8)r.rrrrUrrrxrArrrrrrs r2rFz+LayoutLMv3ForSequenceClassification.forward+sX$34?  $ ))%'% $ $  $ $ "!*QQQ111W-11  {'/?a''/;DK,,_q((flej.H.HFL\a\eLeLe/LDK,,/KDK,{'<77"99?a''#8FNN$4$4fnn6F6FGGDD#8FF33DD)-JJJ+--x B @ @&++b//RR)-III,..x//'!/)     r3r)rIrJrKr!rWrZrrrerrrrrtr rFrMrNs@r2rrsa666444.237260426*.(,04T T #d*T )D0T (4/ T &- T (4/ T  4'T %T &-T +,T  ) )T T T ^T T T T T r3r)rrrr=r4)@rLr$rretorch.nnr)torch.nn.functional functionalr=rrrrr? activationsr masking_utilsr modeling_layersr modeling_outputsr r r rmodeling_utilsrprocessing_utilsr pytorch_utilsrutilsrrrr utils.genericrrutils.output_capturingrconfiguration_layoutlmv3r get_loggerrIloggerModulerrPrrrrrrrr4r=rrrr__all__rr3r2rs"  AAAAAAAAAA&&&&&&!!!!!!666666999999 .-----&&&&&&666666 LKKKKKKK555555666666  H % %        FsssssryssslV.V.V.V.V.biV.V.V.t     29        ")   6"""""0"""Jn@n@n@n@n@ n@n@n@d     RY        ry   iiiiiiii(C C C C C /C C C L296` ` ` ` ` '@` ` ` Fi i i i i %>i i i Xf f f f f *Cf f f R   r3