tjdDddlmZddlmZmZddlZddlmZddlm Z ddl m Z ddl m Z mZdd lmZdd lmZdd lmZmZmZmZdd lmZdd lmZddlmZddlmZddl m!Z!m"Z"ddl#m$Z$ddl%m&Z&m'Z'ej(e)Z*edeGdde$Z+Gdde'dZ,edeGdde&Z-eGd d!e"Z.ed"#eGd$d%eZ/ed&#Gd'd(e!Z0gd)Z1dS)*) dataclass)OptionalUnionN)strict)PreTrainedConfig) BatchFeature) ImageInputis_valid_image)Unpack) TextInput) ModelOutputTransformersKwargsauto_docstringlogging)can_return_tuple)requires)CONFIG_MAPPING) AutoModel)ColPaliForRetrievalColPaliPreTrainedModel)ColQwen2Config)Idefics3ProcessorIdefics3ProcessorKwargsz!ModernVBERT/colmodernvbert-merged) checkpointcfeZdZUdZdZdeiZdZeezdze d<dZ e e d<dZ e e d<fd ZxZS) ColModernVBertConfigz Example: ```python from transformers import ColModernVBertConfig, ColModernVBertForRetrieval config = ColModernVBertConfig() model = ColModernVBertForRetrieval(config) ``` colmodernvbert vlm_configN embedding_dimg{Gz?initializer_rangec |j4td|_tdnAt |jt r't|jddi|j|_t |jds(|jj|j_tj di|dS)N modernvbertzc`vlm_config` is `None`. Initializing `vlm_config` with the `ModernVBertConfig` with default values. model_type vocab_size) r rloggerinfo isinstancedicthasattrget_text_configr'super __post_init__)selfkwargs __class__s w/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/colmodernvbert/modular_colmodernvbert.pyr0z"ColModernVBertConfig.__post_init__<s ? ",];==DO KKu     . . _,T_\-JK^^do^^DOt 55 V)-)H)H)J)J)UDO &''''''')__name__ __module__ __qualname____doc__r&r sub_configsr r,__annotations__r"intr#floatr0 __classcell__r3s@r4rr's  "J!12K15J''$.555M3#u### ( ( ( ( ( ( ( ( (r5rc*eZdZddiddddddidZd S) ColModernVBertProcessorKwargspaddinglongestTchannels_first)return_row_col_info data_formatdo_convert_rgbreturn_tensorspt) text_kwargs images_kwargs common_kwargsN)r6r7r8 _defaultsr(r5r4rArAKsE y $(+"  +D1  IIIr5rAF)total)torch)backendsc eZdZ ddededzdedzffd Z ddedzdeed e fd Z d e e e zdeed e fd Z ddede dfdede dfdededdedefd df dZxZS)ColModernVBertProcessorN@ image_seq_lenvisual_prompt_prefix query_prefixc d}tj||f||d||p d|jd|_|pd|_|j|_dS)a image_seq_len (`int`, *optional*, defaults to 64): The length of the image sequence i.e. the number of tokens per image in the input. visual_prompt_prefix (`str`, *optional*): A string that gets tokenized and prepended to the image tokens. query_prefix (`str`, *optional*): A prefix to be used for the query. N) chat_templaterTz<|begin_of_text|>User:z0Describe the image. Assistant:)r/__init__ image_tokenrUrVend_of_utterance_tokenquery_augmentation_token) r1image_processor tokenizerrXrTrUrVr2r3s r4rZz ColModernVBertProcessor.__init__\s$    ('       %9% hT%5 h h h !).B(,(C%%%r5imagesr2returnc |jtfd|jji|}|ddd}|du}t |r|g}nt |trt |drnZt |tr6t |dtrt |ddstdd|D}| |j gt|z||d|d }|r=|d  |d dkd }| d |i|S)a Prepare for the model one or several image(s). Handles input validation, RGB conversion, and prepends the `visual_prompt_prefix` to each image. Optionally computes labels from `token_type_ids` when a `suffix` is provided in `text_kwargs`. Args: images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`, `list[torch.Tensor]`): The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch tensor. In case of a NumPy array/PyTorch tensor, each image should be of shape (C, H, W), where C is a number of channels, H and W are image height and width. return_tensors (`str` or [`~utils.TensorType`], *optional*): If set, will return tensors of a particular framework. Acceptable values are: - `'pt'`: Return PyTorch `torch.Tensor` objects. - `'np'`: Return NumPy `np.ndarray` objects. Returns: [`BatchFeature`]: A [`BatchFeature`] with the following fields: - **input_ids** -- List of token ids to be fed to a model. - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not `None`). - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`. tokenizer_init_kwargsrJsuffixNrzAimages must be an image, list of images or list of list of imagesc8g|]}|dS)RGB)convert).0images r4 z:ColModernVBertProcessor.process_images..s$;;;5%--&&;;;r5rK)textr`rKrJ input_idstoken_type_idsilabels) _merge_kwargsrAr_ init_kwargspopr r+list ValueError__call__rUlen masked_fillupdate)r1r`r2 output_kwargsrdreturn_token_type_ids batch_docrns r4process_imagesz&ColModernVBertProcessor.process_images~s<+* )  "&."<   }-11(DAA &d 2 & ! ! bXFF  % % b.*C*C b VT** bz&)T/J/J b~^def^ghi^jOkOk b`aa a<;F;;;MM+,s6{{:'8%m4 "   ! 1{+77 BR8SWX8XZ^__F   h/ 0 0 0r5rkc jtfdjji|}|dddt |t r|g}n?t |trt |dt std j dzfd|D} |d |d }|S) ad Prepare for the model one or several text queries. Handles input validation, prepends the `query_prefix`, and appends query augmentation tokens (used to pad query embeddings for better late-interaction retrieval performance). Args: text (`str`, `list[str]`, `list[list[str]]`): The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set `is_split_into_words=True` (to lift the ambiguity with a batch of sequences). return_tensors (`str` or [`~utils.TensorType`], *optional*): If set, will return tensors of a particular framework. Acceptable values are: - `'pt'`: Return PyTorch `torch.Tensor` objects. - `'np'`: Return NumPy `np.ndarray` objects. Returns: [`BatchFeature`]: A [`BatchFeature`] with the following fields: - **input_ids** -- List of token ids to be fed to a model. - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not `None`). rcrJrdNrz*Text must be a string or a list of strings c*g|]}j|zzSr()rV)rhqueryr1rds r4rjz;ColModernVBertProcessor.process_queries..s&!W!W!W$"3e";f"D!W!W!Wr5F)rkryrJ) rorAr_rprqr+strrrrsr]rt)r1rkr2rx texts_query batch_queryrds` @r4process_queriesz'ColModernVBertProcessor.process_queriess :+* )  "&."<   }-11(DAA dC  K6DDT4(( KZQ-E-E KIJJ J >2R7F"X!W!W!W!WRV!W!W!W mm"'%m4$  r5r!cpuquery_embeddingsz torch.Tensorpassage_embeddings batch_size output_dtypez torch.dtype output_devicez torch.devicec 8t|dkrtdt|dkrtd|dj|djkrtd|dj|djkrtd| |dj}g}t dt||D]:}g}t jjj ||||zdd} t dt||D]} t jjj || | |zdd} | t j d | |  d d d | t j|d || usually obtained by padding the list of tensors. Args: query_embeddings (`Union[torch.Tensor, list[torch.Tensor]`): Query embeddings. passage_embeddings (`Union[torch.Tensor, list[torch.Tensor]`): Passage embeddings. batch_size (`int`, *optional*, defaults to 128): Batch size for computing scores. output_dtype (`torch.dtype`, *optional*, defaults to `torch.float32`): The dtype of the output tensor. If `None`, the dtype of the input embeddings is used. output_device (`torch.device` or `str`, *optional*, defaults to "cpu"): The device of the output tensor. Returns: `torch.Tensor`: A tensor of shape `(n_queries, n_passages)` containing the scores. The score tensor is saved on the "cpu" device. rzNo queries providedzNo passages providedz/Queries and passages must be on the same devicez-Queries and passages must have the same dtypeNT) batch_first padding_valuez bnd,csd->bcnsr)dimr)rursdevicedtyperangerOnnutilsrnn pad_sequenceappendeinsummaxsumcatto) r1rrrrrscoresi batch_scores batch_queriesjbatch_passagess r4score_retrievalz'ColModernVBertProcessor.score_retrievals*@  A % %233 3 ! " "a ' '344 4 A  %);A)>)E E ENOO O A  $(:1(=(C C CLMM M  +A.4L%'q#.//<< ] ]A/1L!HN.;; Q^!34$VW<M1c"455zBB  !&!3!@!@&q1z>'9:\]"A""##L-PPTTYZT[[\]^bbghbii MM%)La888;;LIILL][[ \ \ \ \yQ''''r5)NNrSNNN)r!Nr)r6r7r8r<rrZr r rAr r{r rrrrrrr>r?s@r4rRrRYs +/#' D D  D "Dj DDj D D D D D DH%)@@T!@67@  @@@@D7$y/)7677  7777z0449 >(>(^0D DE>(".$~2F"FG>( >( }- >( ^S01 >( >(>(>(>(>(>(>(>(r5rRceZdZUeed<dS)ColModernVBertPreTrainedModelconfigN)r6r7r8rr;r(r5r4rr:s      r5rz: Base class for ColModernVBert embeddings output. ) custom_introceZdZUdZdZejdzed<dZej dzed<dZ e ejdzed<dZ e ejdzed<dZ e ejdzed<dS) ColModernVBertForRetrievalOutputa loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Language modeling loss (for next-token prediction). embeddings (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): The embeddings of the model. image_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True` and `pixel_values` are provided): Tuple of `torch.FloatTensor` (one for the output of the image modality projection + one for the output of each layer) of shape `(batch_size, num_channels, image_size, image_size)`. Hidden-states of the image encoder at the output of each layer plus the initial modality projection outputs. Nloss embeddings hidden_statesimage_hidden_states attentions)r6r7r8r9rrO FloatTensorr;rTensorrtuplerrr(r5r4rr?s  &*D% d ")))&*J t#***59M5*+d2999;?u01D8???26Je'(4/66666r5ru Following the ColPali approach, ColModernVBert leverages VLMs to construct efficient multi-vector embeddings directly from document images (“screenshots”) for document retrieval. The model is trained to maximize the similarity between these document embeddings and the corresponding query embeddings, using the late interaction method introduced in ColBERT. Using ColModernVBert removes the need for potentially complex and brittle layout recognition and OCR pipelines with a single model that can take into account both the textual and visual content (layout, charts, ...) of a document. ColModernVBert is trained on top of ModernVBert, and was introduced in the following paper: [*ModernVBERT: Towards Smaller Visual Document Retrievers*](https://arxiv.org/abs/2510.01149). ColModernVBert is part of the ColVision model family, which was introduced with ColPali in the following paper: [*ColPali: Efficient Document Retrieval with Vision Language Models*](https://huggingface.co/papers/2407.01449). c eZdZdeffd Zee d dejdzdej dzdej dzde e de f d ZxZS) ColModernVBertForRetrievalrct|tj|j|_|dSr)r/rZr from_configr vlm post_init)r1rr3s r4rZz#ColModernVBertForRetrieval.__init__jsF    ():;; r5Nrl pixel_valuesattention_maskr2rac |jd|||d|}|d}|jjj}|||}||ddz }|9||j|j}||dz}t||j |j |j S) N)rlrrrT)rkeepdim)rr)rrrrr() rembedding_proj_layerweightrrnormr unsqueezerrrr) r1rlrrr2 vlm_outputlast_hidden_states proj_dtypers r4forwardz"ColModernVBertForRetrieval.forwardosTX )%     (].5; ../A/D/DZ/P/PQQ  *//b$/"G"GG  %+..Z5EjN_.``N#n&>&>r&B&BBJ/!$2!, * >     r5)NNN)r6r7r8rrZrrrO LongTensorrrr rrrr>r?s@r4rrXs$3 .215.2   #d* '$.  t+  +,  *    ^     r5r)rrrrR)2 dataclassesrtypingrrrOhuggingface_hub.dataclassesrconfiguration_utilsrfeature_extraction_utilsr image_utilsr r processing_utilsr tokenization_utils_baser rrrrr utils.genericrutils.import_utilsrautorauto.modeling_autorcolpali.modeling_colpalirrcolqwen2.configuration_colqwen2ridefics3.processing_idefics3rr get_loggerr6r)rrArRrrr__all__r(r5r4rsA"!!!!!"""""""" ......33333344444455555555&&&&&&000000MMMMMMMMMMMM------******!!!!!!******RRRRRRRR<<<<<<UUUUUUUU  H % %>???(((((>((@?(D     $;5     :\(\(\(\(\(/\(\(\(~!!!!!$:!!!  77777{77   7&"& & & & & !4& & #"& R   r5