tj8^ddlZddlmZddlmZddlmZmZddlmZddl m Z ddl m Z dd l mZdd lmZmZmZmZmZdd lmZd d lmZmZd dlmZd dlmZd dlmZm Z m!Z!m"Z"m#Z#ej$e%Z&edeGddeZ'Gdde"Z(Gdde#Z)edeGddeZ*GddeZ+GddeZ,Gdd eZ-Gd!d"e!Z.Gd#d$e Z/gd%Z0dS)&N)strict)nn)Cache DynamicCache)GenerationConfig)FlashAttentionKwargs)BaseModelOutputWithPooling)Unpack)TransformersKwargsauto_docstringcan_return_tupleloggingtorch_compilable_check)merge_with_config_defaults)Idefics3ConfigIdefics3VisionConfig)Idefics3ImageProcessor)Idefics3ImageProcessorPil)Idefics3BaseModelOutputWithPast Idefics3ForConditionalGeneration Idefics3ModelIdefics3PreTrainedModelIdefics3VisionTransformerz$HuggingFaceTB/SmolVLM2-2.2B-Instruct) checkpointceZdZdZdZdS)SmolVLMVisionConfiga Example: ```python >>> from transformers.models.smolvlm.modeling_smolvlm import SmolVLMVisionTransformer >>> from transformers.models.smolvlm.configuration_smolvlm import SmolVLMVisionConfig >>> # Initializing a SmolVLMVisionConfig with google/siglip-so400m-patch14-384 style configuration >>> configuration = SmolVLMVisionConfig() >>> # Initializing a SmolVLMVisionTransformer (with random weights) from the google/siglip-so400m-patch14-384 style configuration >>> model = SmolVLMVisionTransformer(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```smolvlm_visionN__name__ __module__ __qualname____doc__ model_typei/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/smolvlm/modular_smolvlm.pyrr+s  ""JJJr'rceZdZdS)SmolVLMPreTrainedModelNr!r"r#r&r'r(r*r*BDr'r*ceZdZdS)SmolVLMVisionTransformerNr+r&r'r(r.r.Fr,r'r.ceZdZdZdZdS) SmolVLMConfiga scale_factor (`int`, *optional*, defaults to 2): The scale factor for the image encoder. Example: ```python >>> from transformers import SmolVLMModel, SmolVLMConfig >>> # Initializing configuration >>> configuration = SmolVLMConfig() >>> # Initializing a model from the configuration >>> model = SmolVLMModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```smolvlmNr r&r'r(r0r0Js   JJJr'r0ceZdZdS)SmolVLMImageProcessorNr+r&r'r(r3r3_r,r'r3ceZdZdS)SmolVLMImageProcessorPilNr+r&r'r(r5r5cr,r'r5ceZdZdS)SmolVLMBaseModelOutputWithPastNr+r&r'r(r7r7gr,r'r7ceZdZdZdejdejdejfdZee d dd ej d ejdzd e e d e ezfd Zeee d ddejdzdejdzdejdzdedzdej dzd ej dzd ejdzdej dzdedzd e ed e ezfdZdS) SmolVLMModelz A subclass of Idefics3Model. We do *not* remove or block the call to inputs_merger in forward. Instead, we override inputs_merger here with custom logic. input_ids inputs_embedsimage_hidden_statesc0|j\}}}|X||tj|jjtj|jk}|d}n||jjk}|d}ttj ||zdkd||z}tj j |ddd} | dd } |d } | dz |z} | dz |z} | d| z}tj|}|||| |ddf||<tj|d ||}|S) Ndtypedevice).rdimrzCAt least one sample has tokens not divisible by patch_size.)rAr)value)shapeget_input_embeddingstorchtensorconfigimage_token_idlongr@sumrallr functionalpadcumsum unsqueeze zeros_likewhere)selfr:r;r<_ patch_size image_masknum_image_tokensblocks_per_sampleoffsets block_offsetrow_cum chunk_idx local_idx block_idx image_embeds merged_embedss r( inputs_mergerzSmolVLMModel.inputs_mergerqs/4:q  &*E$*C*C*E*E T[7uzR_Rfggg++J$F+JJ"dk&@@J%>>a>00 I&3q8 9 9 Q   - :(%))*;*B*Bq*B*I*I6YZ)[[ss| ###++q[Z/ q[J.  **1-- 9 ' 66 #6y7LiXbNcefefef7f#g Z  J$8$8$<$z3SmolVLMModel.get_image_features..s!???l(+???r')rrr)sizer?r@) dimensionrpstep)rErjT)repatch_attention_mask return_dictr&)rFtor?viewnumelrMrHany contiguousonesboolr@rJ vision_configrWunfold vision_modellast_hidden_state connector pooler_output)rUrerfrg batch_size num_images num_channelsheightwidthnb_values_per_imagereal_images_indsrWpatches_subgridrs image_outputsr<image_featuress ` r(get_image_featureszSmolVLMModel.get_image_featuress7 ?K>P; J fe#TZ88 (|(j)@Z (batch_size, max_num_images, 3, max_heights, max_widths) where max_num_images is the maximum number of images among the batch_size samples in the batch. Padding images are not needed beyond padding the pixel_values at the entrance of the model. For efficiency, we only pass through the vision_model's forward the real images by discarding the padding images i.e. pixel_values of size (image_batch_size, 3, height, width) where image_batch_size would be 7 when num_images_per_sample=[1, 3, 1, 2] and max_num_images would be 3. attention_mask position_idspast_key_values use_cachec | |j\} } n| |j\} } } ntd| r|t|j}|:|j||j}||td|8|||dj }||j}n#|!||j |j}|| |||}|jd ||||| d| }t|j |j|j|j| S) Nz5You have to specify either input_ids or inputs_embeds)rJzMYou cannot specify both pixel_values and image_hidden_states at the same timeT)rtr>)r:r;r<)r;rrrr)rr hidden_states attentionsr<r&)rF ValueErrorrrJ text_modelrGrur@rrr?rcr7rrrr)rUr:rrrr;rerfr<rrgr seq_lengthrVoutputss r(forwardzSmolVLMModel.forwards4  %._ "J  &(5(; %J AATUU U  ?0*$+>>>O  BDO@@BB9MMPPQZQabbM  #(;(Glmm m  #"&"9"92#:## #6"8"89M"N"N   ,"5"8"8tzR_Rf"8"g"g   * ..#+$7/M "$/ ')%+      .%7#3!/) 3     r')N) NNNNNNNNN)r!r"r#r$rH LongTensorTensorrcrr FloatTensorr r tupler rrr BoolTensorr{r r7rr&r'r(r9r9ks ):?,]b]iB^m 9=22'2$.52+, 2 + + 2222h ^    .2.204(,26158<8>> import httpx >>> from io import BytesIO >>> import torch >>> from PIL import Image >>> from io import BytesIO >>> from transformers import AutoProcessor, AutoModelForImageTextToText >>> from transformers.image_utils import load_image >>> # Note that passing the image urls (instead of the actual pil images) to the processor is also possible >>> image1 = load_image("https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg") >>> image2 = load_image("https://cdn.britannica.com/59/94459-050-DBA42467/Skyline-Chicago.jpg") >>> image3 = load_image("https://cdn.britannica.com/68/170868-050-8DDE8263/Golden-Gate-Bridge-San-Francisco.jpg") >>> processor = AutoProcessor.from_pretrained("HuggingFaceTB/SmolVLM2-2.2B-Instruct") >>> model = AutoModelForImageTextToText.from_pretrained("HuggingFaceTB/SmolVLM2-2.2B-Instruct", dtype=torch.bfloat16, device_map="auto") >>> # Create inputs >>> messages = [ ... { ... "role": "user", ... "content": [ ... {"type": "video", "path": path/to/video}, ... {"type": "text", "text": "What is happening in this video?"}, ... ] ... } ... ] >>> inputs = processor.apply_chat_template([messages], add_generation_prompt=True) >>> # Generate >>> generated_ids = model.generate(**inputs, max_new_tokens=256) >>> generated_texts = processor.batch_decode(generated_ids, skip_special_tokens=True) >>> print(generated_texts) ```Nr&)rr)rU super_kwargsrs r(rz'SmolVLMForConditionalGeneration.forwards(d '','''''r')r!r"r#_tied_weights_keysrr __classcell__)rs@r(rrs]*,RS2(2(2(2(2(2(2(2(2(r'r)rr0r3r5rr*r9r.)1rHhuggingface_hub.dataclassesrr cache_utilsrr generationrmodeling_flash_attention_utilsr modeling_outputsr processing_utilsr utilsr r rrr utils.genericridefics3.configuration_idefics3rr"idefics3.image_processing_idefics3r&idefics3.image_processing_pil_idefics3ridefics3.modeling_idefics3rrrrr get_loggerr!loggerrr*r.r0r3r5r7r9r__all__r&r'r(rsR" ..............******BBBBBB::::::&&&&&&jjjjjjjjjjjjjj777777RRRRRRRRGGGGGGNNNNNN  H % %ABBB""""".""CB"*     4        8   ABBBNCB&     2        8        %D   g g g g g =g g g T<(<(<(<(<(&F<(<(<(~   r'