tjs`$ddlZddlmZddlmZmZmZddlZddl Z ddl m Z ddl m Z mZddlmZmZmZmZddlmZmZmZdd lmZdd lmZerdd lmZGd d edZedeGddeZdgZdS)N) accumulate) TYPE_CHECKINGOptionalUnion) BatchFeature) ImageInputis_valid_image)MultiModalDataProcessingKwargsProcessorMixinUnpack) AddedToken BatchEncoding TextInput)auto_docstring)requires)PreTokenizedInputc*eZdZddiddddddidZd S) ColModernVBertProcessorKwargspaddinglongestTchannels_first)return_row_col_info data_formatdo_convert_rgbreturn_tensorspt) text_kwargs images_kwargs common_kwargsN)__name__ __module__ __qualname__ _defaultsz/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/colmodernvbert/processing_colmodernvbert.pyrr(sE y $(+"  +D1  IIIr'rF)total)torch)backendsc eZdZeZ d#dededzdedzffd Ze d$de e e ze e e zde e d e e e d fdedzd e ed ef d Z d%de dzde e d e e e d fd e efd Z d%de dzde e d e e e d fd e effd Zdeded efdZde de ed e e efdZd&dZ d&de dzd e ed efdZde e e zd e ed efdZ d'de de dfde de dfdededd e d!efd df d"ZxZS)(ColModernVBertProcessorN@ image_seq_lenvisual_prompt_prefix query_prefixc d}tdddj|_tdddj|_tdddj|_d|_||_|j|_|j|_ |j|_ fd td D|_ tjd |_d |j|j|jgi}||j|_t#j|fd |i||p d|jd|_|pd|_|j|_dS)a image_seq_len (`int`, *optional*, defaults to 64): The length of the image sequence i.e. the number of tokens per image in the input. visual_prompt_prefix (`str`, *optional*): A string that gets tokenized and prepended to the image tokens. query_prefix (`str`, *optional*): A prefix to be used for the query. NzFT) normalizedspecialzzz c xg|]6}tdD]$}d|dzd|dzd%7S))rangeconvert_tokens_to_ids).0ij tokenizers r( z4ColModernVBertProcessor.__init__..Vsn   NOejklemem  `aI + +,HAE,H,HA,H,H,H I I    r'r6z*(\n?\n?|\n?)+additional_special_tokens chat_templatez<|begin_of_text|>User:z0Describe the image. Assistant:)rcontentfake_image_token image_tokenend_of_utterance_tokenglobal_image_tagr/r<image_token_idfake_image_token_idglobal_image_token_idr; row_col_idsrecompile%_regex_to_remove_extra_special_tokensadd_special_tokenssuper__init__r0r1query_augmentation_token) selfimage_processorr@rCr/r0r1kwargs tokens_to_add __class__s ` r(rSz ColModernVBertProcessor.__init__;s$ *+FSXbf g g g o%iE4PPPX&01ERWae&f&f&f&n# .*'==d>NOO#,#B#B4CX#Y#Y %.%D%DTEZ%[%["    SXYZS[S[    68Z@m5n5n2 (% +*  $$]333'==d>NOO)[[=[TZ[[[$8% hT%5 h h h !).B(,(C%%%r'imagestextrrWreturnc $|jd||d|\}}|jd||d||jtfd|jji|}||n|j}|ddd}|ddd}|ddd}ix} } |=|j|fi|d \} } | d d| d d|| || \}} |j|fi|d} |r| | d <g} t| D]v\}}g}|D]W}|d\}}| ||}| ||dz }| ||z dzX| |w|r| | d| | d<||| dgn||jdd|i|d} ti| | |S)a  image_seq_len (`int`, *optional*): The length of the image sequence. If not provided, the default value of self.image_seq_len is used. image_seq_len should be equal to int(((image_size // patch_size) ** 2) / (scale_factor**2)) )rZr[tokenizer_init_kwargsNrreturn_text_replacement_offsetsFreturn_mm_token_type_idsrr rowscols)images_replacementstext_replacement_offsetsnew_spanr8 input_idsmm_token_type_idsimage) modalitiesr[)data tensor_typer&)prepare_inputs_layoutvalidate_inputs _merge_kwargsrr@ init_kwargsr/pop_process_imagesget_text_with_replacements enumerate char_to_tokenappendcreate_mm_token_type_ids_check_special_mm_tokensr)rUrZr[r/rW output_kwargsr_r`r image_inputs text_inputsrcrdbatch_image_seq_lengthsbatch_idtext_replacement_offsetimage_seq_lensrjstartend start_id_pos end_id_poss r(__call__z ColModernVBertProcessor.__call__ps2t1UdUUfUU @F@@@@@** )  "&."<   *7)B HZ *7 *F*J*JKlns*t*t'#0#?#C#CD^`e#f#f &}599:JDQQ%'' {  0D0DV0n0n}]lOm0n0n -L-   VT * * *   VT * * *151P1P.A2Q22..-dnTRR]=5QRR 2W>VK :;*,'9BC[9\9\CC5H5%'N 7MM%)*%5 s'2'@'@5'Q'Q %0%>%>xq%Q%Q &--j<.G!.KLLLL+22>BBBB+7;7T7T#K02I88K 34--dKWI-VVV  ($.SSdSmM6RSSK!@K!@.s('Z'Z'Z6 T5E(F(F'Z'Z'Zr'cBg|]}||dzS)r8r&)r=r>cumsum_images_in_textrZs r(rAzAColModernVBertProcessor.prepare_inputs_layout..sF$$$4Q7:OPQTUPU:VVW$$$r') isinstancestrcopyrV fetch_imagesr listtuplerr;len)rUrZr[rWn_images_in_text split_imagesrs`` @r(rlz-ColModernVBertProcessor.prepare_inputs_layoutsa  $$$ v99;;D  )66v>>Ff%% &!(FT5M22 &~fQi7P7P &#'Z'Z'Z'ZUY'Z'Z'Z$-.C$zBR7S7S2T2T,T)$$$$$!&s+;'<'>>!-8Mb8Q8S8S1T0U!U!-$XFt|r'c xtj||fi|||td|fd|D}|:d|D}||kr&tdjd|djd|d dS|9t |r,td t |djd dSdSdS) Nz+You must provide either `text` or `images`.cDg|]}|jSr&rrs r(rAz;ColModernVBertProcessor.validate_inputs..s(RRR6 T-= > >RRRr'c,g|]}t|Sr&)r)r=sublists r(rAz;ColModernVBertProcessor.validate_inputs..s%I%I%Iwc'll%I%I%Ir'zThe total number of zP tokens in the prompts should be the same as the number of images passed. Found  z tokens and z images per sample.zFound z. tokens in the text but no images were passed.)rRrm ValueErrorrGanysum)rUrZr[rWrn_images_in_imagesrYs` r(rmz'ColModernVBertProcessor.validate_inputssG 77777 .%SSSh(SS3cSSSSr'racg|] }|D]}| Sr&r&)r=col_listcols r(rAz?ColModernVBertProcessor.replace_image_token..rr'rbrrDr7r8r9r: )rFrIrGr/r;)rUryr image_rows image_colstext_split_imagesn_hn_ws r(replace_image_tokenz+ColModernVBertProcessor.replace_image_tokens}SS,v*>SSST]^ SS,v*>SSST]^ ??zQ(**,-%'$*<<=*,- !# Z(( * * ,,C%02:#'::a:::;!-/$2DDE%% "T)!! ,T*,,*,-%'$*<<=*,-   % $r'rfr{cg}t|D]\}}tj||}tj|}tj||jkd}d} |D]@} | t |krn*|| } | | z} d|| | <tj|| } A|| |S)Nrr8) rsnparray zeros_likewhererKr searchsortedrutolist) rUrfr{rgr> seq_lengths array_idsmm_token_typesimage_start_positionsr?seq_lenrrs r(rvz0ColModernVBertProcessor.create_mm_token_type_ids s'(?@@ > >NA{1..I]955N$&HY$:R-R$S$STU$V !A& @ @12222E-a0go,-uSy)O$93??  $ $^%:%:%<%< = = = =  r'c i}|tjdi|fd|D}jdz}jdz}t ddd d z }g}g} |D]K\} } } || zd z} | d kr|nd }|||z| | zz| | L||| d td i|S)a Computes the number of placeholder tokens needed for multimodal inputs with the given sizes. Args: image_sizes (`list[list[int]]`, *optional*): The input sizes formatted as (height, width) per each image. Returns: `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided input modalities, along with other useful data. Nr c8g|]}jjg|RSr&)rVget_number_of_image_patches)r= image_sizer rUs r(rAzFColModernVBertProcessor._get_num_multimodal_tokens..0sE"""A$@\*\m\\\"""r'rz F)rQrfr8r)num_image_tokensnum_image_patchesr&) rr%getupdater/rr@rur )rU image_sizesrW vision_datanum_image_row_colsbase_image_length col_lengthextra_split_newlinerr num_patchesnum_rowsnum_cols row_length split_extrar s` @r(_get_num_multimodal_tokensz2ColModernVBertProcessor._get_num_multimodal_tokenssh  "9CGGY[\\M   ( ( (""""""-"""  !% 2Q 6 +a/J #&dnnVPUn&V&VWb&c"d"dgh"h !  " 3E 6 6/ Xx'(2Q6 5=\\11q  ''(9K(G:X`K`(abbb!((5555   4D[lmm n n n,, ,,,r'c |jtfd|jji|}|ddd}|du}t |r|g}nt |trt |drnZt |tr6t |dtrt |ddstdd|D}| |j gt|z||d|d }|r=|d  |d dkd }| d |i|S)a Prepare for the model one or several image(s). Handles input validation, RGB conversion, and prepends the `visual_prompt_prefix` to each image. Optionally computes labels from `token_type_ids` when a `suffix` is provided in `text_kwargs`. Args: images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`, `list[torch.Tensor]`): The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch tensor. In case of a NumPy array/PyTorch tensor, each image should be of shape (C, H, W), where C is a number of channels, H and W are image height and width. return_tensors (`str` or [`~utils.TensorType`], *optional*): If set, will return tensors of a particular framework. Acceptable values are: - `'pt'`: Return PyTorch `torch.Tensor` objects. - `'np'`: Return NumPy `np.ndarray` objects. Returns: [`BatchFeature`]: A [`BatchFeature`] with the following fields: - **input_ids** -- List of token ids to be fed to a model. - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not `None`). - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`. r^rsuffixNrzAimages must be an image, list of images or list of list of imagesc8g|]}|dS)RGB)convert)r=rhs r(rAz:ColModernVBertProcessor.process_images..zs$;;;5%--&&;;;r'r )r[rZr rrftoken_type_idsilabels)rnrr@rorpr rrrrr0r masked_fillr)rUrZrWrxrreturn_token_type_ids batch_docrs r(process_imagesz&ColModernVBertProcessor.process_imagesIs<+* )  "&."<   }-11(DAA &d 2 & ! ! bXFF  % % b.*C*C b VT** bz&)T/J/J b~^def^ghi^jOkOk b`aa a<;F;;;MM+,s6{{:'8%m4 "   ! 1{+77 BR8SWX8XZ^__F   h/ 0 0 0r'c jtfdjji|}|dddt |t r|g}n?t |trt |dt std j dzfd|D} |d |d }|S) ad Prepare for the model one or several text queries. Handles input validation, prepends the `query_prefix`, and appends query augmentation tokens (used to pad query embeddings for better late-interaction retrieval performance). Args: text (`str`, `list[str]`, `list[list[str]]`): The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set `is_split_into_words=True` (to lift the ambiguity with a batch of sequences). return_tensors (`str` or [`~utils.TensorType`], *optional*): If set, will return tensors of a particular framework. Acceptable values are: - `'pt'`: Return PyTorch `torch.Tensor` objects. - `'np'`: Return NumPy `np.ndarray` objects. Returns: [`BatchFeature`]: A [`BatchFeature`] with the following fields: - **input_ids** -- List of token ids to be fed to a model. - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not `None`). r^rrNrz*Text must be a string or a list of strings c*g|]}j|zzSr&)r1)r=queryrUrs r(rAz;ColModernVBertProcessor.process_queries..s&!W!W!W$"3e";f"D!W!W!Wr'F)r[rr) rnrr@rorprrrrrTr)rUr[rWrx texts_query batch_queryrs` @r(process_queriesz'ColModernVBertProcessor.process_queriess :+* )  "&."<   }-11(DAA dC  K6DDT4(( KZQ-E-E KIJJ J >2R7F"X!W!W!W!WRV!W!W!W mm"'%m4$  r'cpuquery_embeddingsz torch.Tensorpassage_embeddings batch_size output_dtypez torch.dtype output_devicez torch.devicec 8t|dkrtdt|dkrtd|dj|djkrtd|dj|djkrtd| |dj}g}t dt||D]:}g}t jjj ||||zdd} t dt||D]} t jjj || | |zdd} | t j d | |  d d d | t j|d || usually obtained by padding the list of tensors. Args: query_embeddings (`Union[torch.Tensor, list[torch.Tensor]`): Query embeddings. passage_embeddings (`Union[torch.Tensor, list[torch.Tensor]`): Passage embeddings. batch_size (`int`, *optional*, defaults to 128): Batch size for computing scores. output_dtype (`torch.dtype`, *optional*, defaults to `torch.float32`): The dtype of the output tensor. If `None`, the dtype of the input embeddings is used. output_device (`torch.device` or `str`, *optional*, defaults to "cpu"): The device of the output tensor. Returns: `torch.Tensor`: A tensor of shape `(n_queries, n_passages)` containing the scores. The score tensor is saved on the "cpu" device. rzNo queries providedzNo passages providedz/Queries and passages must be on the same devicez-Queries and passages must have the same dtypeNT) batch_first padding_valuez bnd,csd->bcnsr)dimrr8)rrdevicedtyper;r*nnutilsrnn pad_sequencerueinsummaxrcatto) rUrrrrrscoresr> batch_scores batch_queriesr?batch_passagess r(score_retrievalz'ColModernVBertProcessor.score_retrievals*@  A % %233 3 ! " "a ' '344 4 A  %);A)>)E E ENOO O A  $(:1(=(C C CLMM M  +A.4L%'q#.//<< ] ]A/1L!HN.;; Q^!34$VW<M1c"455zBB  !&!3!@!@&q1z>'9:\]"A""##L-PPTTYZT[[\]^bbghbii MM%)La888;;LIILL][[ \ \ \ \yQ''''r')NNr.NN)NNN)NN)N)rNr)r"r#r$rvalid_processor_kwargsintrrSrr rrrrrrrlr rmdictrrvrrrrrr __classcell__)rYs@r(r-r-6s; +/#'3D3D  3D "Dj 3DDj3D3D3D3D3D3DjJNbf$( >^>^T*--T*5E0FF>^I2DOTJ]E^^_>^Tz >^ 67 >^  >^>^>^^>^D%)bf  T! I2DOTJ]E^^_ 67    H%)bfT!I2DOTJ]E^^_)* 2%%%%%%%:!$!QUVYQZ!_cdhildm_n!!!!*)-)-)-)-Z%)@@T!@67@  @@@@D7$y/)7677  7777z0449 >(>(^0D DE>(".$~2F"FG>( >( }- >( ^S01 >( >(>(>(>(>(>(>(>(r'r-) rN itertoolsrtypingrrrnumpyrr*feature_extraction_utilsr image_utilsr r processing_utilsr r r rtokenization_utils_baserrrrrutils.import_utilsrrrr-__all__r&r'r(r s* 1111111111 44444455555555XXXXXXXXXXXXKKKKKKKKKK######******=<<<<<<     $4E     :J(J(J(J(J(nJ(J(J(Z % %r'