§
    ‚Štj+�  ã                   óô  — d dl mZ d dlmZ d dlZd dlmZ ddlmZ ddlm	Z	m
Z
 ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZ ddlmZmZ ddlmZ ddlmZmZm Z m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z)m*Z* e G d„ de¦  «        ¦   «         Z+ G d„ dej,        ¦  «        Z-	 d@dej,        dej.        dej.        dej.        dej.        dz  d e/d!e/fd"„Z0 G d#„ d$ej,        ¦  «        Z1 G d%„ d&ej,        ¦  «        Z2 G d'„ d(e¦  «        Z3 G d)„ d*ej,        ¦  «        Z4 ed+¬,¦  «         G d-„ d.e+¦  «        ¦   «         Z5 ed/¬,¦  «        e G d0„ d1e¦  «        ¦   «         ¦   «         Z6 G d2„ d3ej,        ¦  «        Z7 G d4„ d5ej,        ¦  «        Z8 ed6¬,¦  «         G d7„ d8e+¦  «        ¦   «         Z9 ed9¬,¦  «        e G d:„ d;e¦  «        ¦   «         ¦   «         Z: ed<¬,¦  «         G d=„ d>e+e¦  «        ¦   «         Z;g d?¢Z<dS )Aé    )ÚCallable)Ú	dataclassN)Únné   )ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationConfigÚGenerationMixin)Úcreate_bidirectional_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPoolingÚModelOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚtorch_compilable_check)Úmerge_with_config_defaults)Úcapture_outputsé   )Ú	AutoModelé   )ÚSmolVLMConfigÚSmolVLMVisionConfigc                   óD   — e Zd ZU eed<   dZdZdZddgZdgZ	dZ
dZdZdZdS )	ÚSmolVLMPreTrainedModelÚconfigÚmodel)ÚimageÚtextTÚSmolVLMVisionAttentionÚSmolVLMDecoderLayerÚpast_key_valuesN)Ú__name__Ú
__module__Ú__qualname__r   Ú__annotations__Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backend© ó    új/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/smolvlm/modeling_smolvlm.pyr!   r!   -   s\   € € € € € € àÐÐÑØÐØ(ÐØ&*Ð#Ø1Ð3HÐIÐØ#4Ð"5ÐØÐØ€NØÐØ"&ÐÐÐr7   r!   c                   óZ   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        dej	        fd„Z
ˆ xZS )ÚSmolVLMVisionEmbeddingsaX  
    This is a modified version of `siglip.modelign_siglip.SiglipVisionEmbeddings` to enable images of variable
    resolution.

    The modifications are adapted from [Patch n' Pack: NaViT, a Vision Transformer for any Aspect Ratio and Resolution](https://huggingface.co/papers/2307.06304)
    which allows treating images in their native aspect ratio and without the need to resize them to the same
    fixed size. In particular, we start from the original pre-trained SigLIP model
    (which uses images of fixed-size square images) and adapt it by training on images of variable resolutions.
    r"   c                 óš  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          j        |j        | j        | j        | j        d¬¦  «        | _	        | j        | j        z  | _
        | j
        dz  | _        | j        | _        t          j        | j        | j        ¦  «        | _        d S )NÚvalid)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingr   )ÚsuperÚ__init__Úhidden_sizeÚ	embed_dimÚ
image_sizeÚ
patch_sizer   ÚConv2dÚnum_channelsÚpatch_embeddingÚnum_patches_per_sideÚnum_patchesÚnum_positionsÚ	EmbeddingÚposition_embedding©Úselfr"   Ú	__class__s     €r8   rC   z SmolVLMVisionEmbeddings.__init__F   sµ   ø€ Ý‰Œ×ÒÑÔÐØÔ+ˆŒØ Ô+ˆŒØ Ô+ˆŒå!œyØÔ+ØœØœØ”?Øð 
ñ  
ô  
ˆÔð %)¤O°t´Ñ$FˆÔ!ØÔ4°aÑ7ˆÔØ!Ô-ˆÔÝ"$¤,¨tÔ/AÀ4Ä>Ñ"RÔ"RˆÔÐÐr7   Úpixel_valuesÚpatch_attention_maskÚreturnc                 óz  — |j         \  }}}}|                      |¦  «        }|                     d¦  «                             dd¦  «        }|| j        z  || j        z  }
}	t          j        d| j        z  dd| j        z  |j        ¬¦  «        }t          j	        ||	|
z  fd|j        ¬¦  «        }|d d …d d …df          
                    d¬¦  «        }|d d …dd d …f          
                    d¬¦  «        }d|z  }d|z  }|                     d¦  «        }|                     d¦  «        }t          j        ||j        t
          j        ¬¦  «        }t          j        ||j        t
          j        ¬¦  «        }|d d d …f         |d d …d f         z  }|d d d …f         |d d …d f         z  }t          j        |d	¬
¦  «        }t          j        |d	¬
¦  «        }|                     |j        ¦  «        }|                     |j        ¦  «        }t          j        ||d¬¦  «        }t          j        ||d¬¦  «        }|d d …d d …d f         | j        z  |d d …d d d …f         z   }|                     |d¦  «        }||                     |d¦  «                 ||                     |d¦  «        <   ||                      |¦  «        z   }|S )Nr   r   g      ð?)Údevicer   )ÚsizeÚ
fill_valuerW   ©Údim)rW   Údtypegé!çýÿï?)ÚmaxT)Úrightéÿÿÿÿ)ÚshaperJ   ÚflattenÚ	transposerG   ÚtorchÚarangerK   rW   ÚfullÚsumrX   Úfloat32ÚclampÚtor\   Ú	bucketizeÚreshapeÚviewrO   )rQ   rS   rT   Ú
batch_sizeÚ_Úmax_im_hÚmax_im_wÚpatch_embedsÚ
embeddingsÚmax_nb_patches_hÚmax_nb_patches_wÚ
boundariesÚposition_idsÚnb_patches_hÚnb_patches_wÚstep_hÚstep_wÚmax_patches_hÚmax_patches_wÚ	h_indicesÚ	w_indicesÚfractional_coords_hÚfractional_coords_wÚbucket_coords_hÚbucket_coords_wÚpos_idss                             r8   ÚforwardzSmolVLMVisionEmbeddings.forwardY   s  € Ø,8Ô,>Ñ)ˆ
�A�x à×+Ò+¨LÑ9Ô9ˆØ!×)Ò)¨!Ñ,Ô,×6Ò6°q¸!Ñ<Ô<ˆ
à-5¸¼Ñ-HÈ(ÐVZÔVeÑJeÐ*ÐÝ”\Ø�Ô)Ñ)¨3°°DÔ4MÑ0MÐVbÔVið
ñ 
ô 
ˆ
õ ”zØÐ.Ð1AÑAÐBÈqÐYeÔYlð
ñ 
ô 
ˆð ,¨A¨A¨A¨q¨q¨q°!¨GÔ4×8Ò8¸QÐ8Ñ?Ô?ˆØ+¨A¨A¨A¨q°!°!°!¨GÔ4×8Ò8¸QÐ8Ñ?Ô?ˆà�|Ñ#ˆØ�|Ñ#ˆà,×1Ò1°!Ñ4Ô4ˆØ,×1Ò1°!Ñ4Ô4ˆÝ”L °|Ô7JÕRWÔR_Ð`Ñ`Ô`ˆ	Ý”L °|Ô7JÕRWÔR_Ð`Ñ`Ô`ˆ	à'¨¨a¨a¨a¨Ô0°6¸!¸!¸!¸T¸'´?ÑBÐØ'¨¨a¨a¨a¨Ô0°6¸!¸!¸!¸T¸'´?ÑBÐå#œkÐ*=ÀJÐPÑPÔPÐÝ#œkÐ*=ÀJÐPÑPÔPÐà1×4Ò4°\Ô5GÑHÔHÐØ1×4Ò4°\Ô5GÑHÔHÐåœ/Ð*=¸zÐQUÐVÑVÔVˆÝœ/Ð*=¸zÐQUÐVÑVÔVˆà! ! ! ! Q Q Q¨ *Ô-°Ô0IÑIÈOÐ\]Ð\]Ð\]Ð_cÐefÐefÐefÐ\fÔLgÑgˆØ—/’/ *¨bÑ1Ô1ˆàBIÐJ^×JcÒJcÐdnÐprÑJsÔJsÔBtˆÐ)×.Ò.¨z¸2Ñ>Ô>Ñ?à $×"9Ò"9¸,Ñ"GÔ"GÑGˆ
ØÐr7   )r)   r*   r+   Ú__doc__r   rC   rc   ÚFloatTensorÚ
BoolTensorÚTensorr„   Ú__classcell__©rR   s   @r8   r:   r:   ;   s‰   ø€ € € € € ðð ðSÐ2ð Sð Sð Sð Sð Sð Sð&+ EÔ$5ð +ÈUÔM]ð +ÐbgÔbnð +ð +ð +ð +ð +ð +ð +ð +r7   r:   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutc                 óÀ  — t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |dt           j        ¬¦  «                             |j        ¦  «        }t          j         	                    ||| j
        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «                             ¦   «         }	|	|fS )Nr_   éþÿÿÿ)r[   r\   )ÚpÚtrainingr   r   )rc   Úmatmulrb   r   Ú
functionalÚsoftmaxrg   ri   r\   r’   r–   Ú
contiguous)
rŒ   r�   rŽ   r�   r�   r‘   r’   ÚkwargsÚattn_weightsÚattn_outputs
             r8   Úeager_attention_forwardrž   ‡   sÃ   € õ ”<  s§}¢}°R¸Ñ'<Ô'<Ñ=Ô=ÀÑG€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r7   c            
       ó~   ‡ — e Zd ZdZˆ fd„Z	 ddej        dej        dz  deej        ej        dz  f         fd„Zˆ xZ	S )	r&   z=Multi-headed attention from 'Attention Is All You Need' paperc                 ó‚  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        | j        | j        z  | _        | j        | j        z  | j        k    r t          d| j        › d| j        › d�¦  «        ‚| j        dz  | _	        |j
        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        d| _        d S )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g      à¿F)rB   rC   r"   rD   rE   Únum_attention_headsÚ	num_headsÚhead_dimÚ
ValueErrorÚscaleÚattention_dropoutr’   r   ÚLinearÚk_projÚv_projÚq_projÚout_projÚ	is_causalrP   s     €r8   rC   zSmolVLMVisionAttention.__init__¡   s  ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØÔ3ˆŒØœ¨$¬.Ñ8ˆŒØŒ=˜4œ>Ñ)¨T¬^Ò;Ð;Ýð'ÈdÌnð 'ð 'Ø”Nð'ð 'ð 'ñô ð ð ”] DÑ(ˆŒ
ØÔ/ˆŒå”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝœ	 $¤.°$´.ÑAÔAˆŒð ˆŒˆˆr7   NÚhidden_statesr�   rU   c           
      ó¼  — |j         dd…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }t          j        | j	        j
        t          ¦  «        }	 |	| ||||| j        | j        | j        sdn| j        ¬¦  «        \  }
} |
j        g |¢d‘R Ž                      ¦   «         }
|                      |
¦  «        }
|
|fS )z#Input shape: Batch x Time x ChannelNr_   r   r   r‹   )r¬   r‘   r’   )r`   r£   rª   rl   rb   r¨   r©   r   Úget_interfacer"   Ú_attn_implementationrž   r¬   r¥   r–   r’   rk   rš   r«   )rQ   r­   r�   r›   Úinput_shapeÚhidden_shapeÚqueriesÚkeysÚvaluesÚattention_interfacer�   rœ   s               r8   r„   zSmolVLMVisionAttention.forward·   sg  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆØ—+’+˜mÑ,Ô,×1Ò1°,Ñ?Ô?×IÒIÈ!ÈQÑOÔOˆØ�{Š{˜=Ñ)Ô)×.Ò.¨|Ñ<Ô<×FÒFÀqÈ!ÑLÔLˆØ—’˜]Ñ+Ô+×0Ò0°Ñ>Ô>×HÒHÈÈAÑNÔNˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØØ”nØ”JØ#œ}Ð>�C�C°$´,ð	%
ñ 	%
ô 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—m’m KÑ0Ô0ˆà˜LÐ(Ð(r7   ©N)
r)   r*   r+   r…   rC   rc   rˆ   Útupler„   r‰   rŠ   s   @r8   r&   r&   ž   s”   ø€ € € € € ØGÐGðð ð ð ð ð2 /3ð)ð )à”|ð)ð œ tÑ+ð)ð
 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð)ð )ð )ð )ð )ð )ð )ð )r7   r&   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚSmolVLMVisionMLPc                 ó  •— t          ¦   «                              ¦   «          || _        t          |j                 | _        t          j        |j        |j	        ¦  «        | _
        t          j        |j	        |j        ¦  «        | _        d S r·   )rB   rC   r"   r   Ú
hidden_actÚactivation_fnr   r§   rD   Úintermediate_sizeÚfc1Úfc2rP   s     €r8   rC   zSmolVLMVisionMLP.__init__Ú   sf   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ# FÔ$5Ô6ˆÔÝ”9˜VÔ/°Ô1IÑJÔJˆŒÝ”9˜VÔ5°vÔ7IÑJÔJˆŒˆˆr7   r­   rU   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r·   )r¿   r½   rÀ   )rQ   r­   s     r8   r„   zSmolVLMVisionMLP.forwardá   s=   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆØÐr7   )r)   r*   r+   rC   rc   rˆ   r„   r‰   rŠ   s   @r8   rº   rº   Ù   sc   ø€ € € € € ðKð Kð Kð Kð Kð U¤\ð °e´lð ð ð ð ð ð ð ð r7   rº   c            	       óv   ‡ — e Zd Zdefˆ fd„Zedej        dej        dee	         dej
        fd„¦   «         Zˆ xZS )ÚSmolVLMEncoderLayerr"   c                 óD  •— t          ¦   «                              ¦   «          |j        | _        t	          |¦  «        | _        t          j        | j        |j        ¬¦  «        | _	        t          |¦  «        | _        t          j        | j        |j        ¬¦  «        | _        d S ©N)Úeps)rB   rC   rD   rE   r&   Ú	self_attnr   Ú	LayerNormÚlayer_norm_epsÚlayer_norm1rº   ÚmlpÚlayer_norm2rP   s     €r8   rC   zSmolVLMEncoderLayer.__init__é   s   ø€ Ý‰Œ×ÒÑÔÐØÔ+ˆŒÝ/°Ñ7Ô7ˆŒÝœ<¨¬¸FÔ<QÐRÑRÔRˆÔÝ# FÑ+Ô+ˆŒÝœ<¨¬¸FÔ<QÐRÑRÔRˆÔÐÐr7   r­   r�   r›   rU   c                 óÆ   — |}|                       |¦  «        } | j        d||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N)r­   r�   r6   )rÊ   rÇ   rÌ   rË   )rQ   r­   r�   r›   Úresidualrn   s         r8   r„   zSmolVLMEncoderLayer.forwardñ   s“   € ð !ˆà×(Ò(¨Ñ7Ô7ˆØ)˜4œ>ð 
Ø'Ø)ð
ð 
ð ð
ð 
Ñˆ�qð
 ! =Ñ0ˆà ˆØ×(Ò(¨Ñ7Ô7ˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆàÐr7   )r)   r*   r+   r   rC   r   rc   rˆ   r   r   r†   r„   r‰   rŠ   s   @r8   rÃ   rÃ   è   sŸ   ø€ € € € € ðSÐ2ð Sð Sð Sð Sð Sð Sð ðà”|ðð œðð Ð+Ô,ð	ð
 
Ô	ðð ð ñ „^ðð ð ð ð r7   rÃ   c                   ób   ‡ — e Zd ZdZdefˆ fd„Ze	 ddej        dz  de	e
z  fd„¦   «         Zˆ xZS )	ÚSmolVLMEncoderz³
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`SmolVLMEncoderLayer`].

    Args:
        config: SmolVLMConfig
    r"   c                 óÔ   •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r6   )rÃ   )Ú.0rn   r"   s     €r8   ú
<listcomp>z+SmolVLMEncoder.__init__.<locals>.<listcomp>  s"   ø€ Ð$jÐ$jÐ$jÀQÕ%8¸Ñ%@Ô%@Ð$jÐ$jÐ$jr7   F)	rB   rC   r"   r   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayersÚgradient_checkpointingrP   s    `€r8   rC   zSmolVLMEncoder.__init__  sa   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”mÐ$jÐ$jÐ$jÐ$jÍ%ÐPVÔPhÑJiÔJiÐ$jÑ$jÔ$jÑkÔkˆŒØ&+ˆÔ#Ð#Ð#r7   Nr�   rU   c                 óV   — |}| j         D ]} |||¦  «        }|}Œt          |¬¦  «        S )N©Úlast_hidden_state)rØ   r   )rQ   Úinputs_embedsr�   r­   Úencoder_layerÚlayer_outputss         r8   r„   zSmolVLMEncoder.forward  sK   € ð &ˆØ!œ[ð 	*ð 	*ˆMØ)˜MØØñô ˆMð
 *ˆMˆMå°Ð?Ñ?Ô?Ð?r7   r·   )r)   r*   r+   r…   r   rC   r   rc   rˆ   r¸   r   r„   r‰   rŠ   s   @r8   rÐ   rÐ   
  s£   ø€ € € € € ðð ð,˜}ð ,ð ,ð ,ð ,ð ,ð ,ð ð /3ð@ð @ð œ tÑ+ð@ð 
�Ñ	 ð	@ð @ð @ñ „^ð@ð @ð @ð @ð @r7   rÐ   zN
    The SmolVLM Vision Transformer Model outputting raw image embedding.
    ©Úcustom_introc            
       ó¸   ‡ — e Zd ZU eed<   dZeedœZdefˆ fd„Z	d„ Z
d„ Ze ed¬¦  «        	 dd
ej        d	z  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚSmolVLMVisionTransformerr"   )r$   )r­   Ú
attentionsc                 ó(  •— t          ¦   «                              |¦  «         |j        }t          |¦  «        | _        t          |¦  «        | _        |j        | _        t          j	        ||j
        ¬¦  «        | _        |                      ¦   «          d S rÅ   )rB   rC   rD   r:   rr   rÐ   ÚencoderrG   r   rÈ   rÉ   Úpost_layernormÚ	post_init)rQ   r"   rE   rR   s      €r8   rC   z!SmolVLMVisionTransformer.__init__9  s{   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØÔ&ˆ	å1°&Ñ9Ô9ˆŒÝ% fÑ-Ô-ˆŒØ Ô+ˆŒÝ œl¨9¸&Ô:OÐPÑPÔPˆÔà�ŠÑÔÐÐÐr7   c                 ó   — | j         S r·   ©rr   ©rQ   s    r8   Úget_input_embeddingsz-SmolVLMVisionTransformer.get_input_embeddingsD  s
   € ØŒÐr7   c                 ó   — || _         d S r·   rê   ©rQ   r�   s     r8   Úset_input_embeddingsz-SmolVLMVisionTransformer.set_input_embeddingsG  s   € ØˆŒˆˆr7   F)Útie_last_hidden_statesNrT   r›   rU   c                 ó  — |                      d¦  «        }|€p| j        }t          j        ||                      d¦  «        |z  |                      d¦  «        |z  f¦  «        }|                     t          j        |j        ¬¦  «        }|                      ||¬¦  «        }|                     |d¦  «        }t          | j
        ||¬¦  «        }|                      ||¬¦  «        }|j        }|                      |¦  «        }t          |¬	¦  «        S )
Nr   r   r   ©r\   rW   )rS   rT   r_   )r"   rÝ   r�   )rÝ   r�   rÛ   )rX   rG   rc   Úonesri   ÚboolrW   rr   rl   r   r"   ræ   rÜ   rç   r   )	rQ   rS   rT   r›   rm   rG   r­   Úencoder_outputsrÜ   s	            r8   r„   z SmolVLMVisionTransformer.forwardJ  s1  € ð "×&Ò& qÑ)Ô)ˆ
ØÐ'ØœˆJÝ#(¤:àØ ×%Ò% aÑ(Ô(¨JÑ6Ø ×%Ò% aÑ(Ô(¨JÑ6ðñ$ô $Ð ð $8×#:Ò#:ÅÄÐT`ÔTgÐ#:Ñ#hÔ#hÐ àŸš°\ÐXl˜ÑmÔmˆà3×8Ò8¸ÀRÑHÔHÐå8Ø”;Ø'Ø/ð 
ñ  
ô  
Ðð ,0¯<ª<Ø'Ø/ð ,8ñ ,
ô ,
ˆð
 ,Ô=ÐØ ×/Ò/Ð0AÑBÔBÐåØ/ð
ñ 
ô 
ð 	
r7   r·   )r)   r*   r+   r   r,   r.   rÃ   r&   Ú_can_record_outputsrC   rì   rï   r   r   rc   r‡   r   r   r¸   r   r„   r‰   rŠ   s   @r8   rã   rã   ,  s   ø€ € € € € € ð  ÐÐÑØ!Ðà,Ø,ðð Ðð
	Ð2ð 	ð 	ð 	ð 	ð 	ð 	ðð ð ð ð  ð  ð  Ø€_¨EÐ2Ñ2Ô2ð 9=ð&
ð &
ð $Ô.°Ñ5ð&
ð Ð+Ô,ð	&
ð
 
�Ñ	 ð&
ð &
ð &
ñ 3Ô2ñ  Ôð&
ð &
ð &
ð &
ð &
r7   rã   z{
    Base class for SmolVLM model's outputs that may also contain a past key/values (to speed up sequential decoding).
    c                   óÄ   — e Zd ZU dZdZej        dz  ed<   dZe	dz  ed<   dZ
eej                 dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dS )ÚSmolVLMBaseModelOutputWithPastab  
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Sequence of hidden-states at the output of the last layer of the model.
        If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
        hidden_size)` is output.
    image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
        sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder
    NrÜ   r(   r­   rä   Úimage_hidden_states)r)   r*   r+   r…   rÜ   rc   r†   r,   r(   r   r­   r¸   rä   rù   r6   r7   r8   rø   rø   u  s£   € € € € € € ð	ð 	ð 37Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø;?Ð˜˜uÔ0Ô1°DÑ8Ð?Ð?Ñ?Ð?Ð?r7   rø   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚSmolVLMSimpleMLPc                 óÆ   •— t          ¦   «                              ¦   «          |j        j        |j        dz  z  }|j        j        }t          j        ||d¬¦  «        | _        d S )Nr   F©Úbias)	rB   rC   Úvision_configrD   Úscale_factorÚtext_configr   r§   Úproj)rQ   r"   Ú
input_sizeÚoutput_sizerR   s       €r8   rC   zSmolVLMSimpleMLP.__init__�  sX   ø€ Ý‰Œ×ÒÑÔÐØÔ)Ô5¸Ô9LÈaÑ9OÑPˆ
ØÔ(Ô4ˆÝ”I˜j¨+¸EÐBÑBÔBˆŒ	ˆ	ˆ	r7   c                 ó,   — |                       |¦  «        S r·   )r  )rQ   Úxs     r8   r„   zSmolVLMSimpleMLP.forward•  s   € Ø�yŠy˜‰|Œ|Ðr7   )r)   r*   r+   rC   r„   r‰   rŠ   s   @r8   rû   rû   Ž  sL   ø€ € € € € ðCð Cð Cð Cð Cðð ð ð ð ð ð r7   rû   c                   ó,   ‡ — e Zd Zˆ fd„Zdd„Zd„ Zˆ xZS )ÚSmolVLMConnectorc                 óˆ   •— t          ¦   «                              ¦   «          |j        | _        t          |¦  «        | _        d S r·   )rB   rC   r   rû   Úmodality_projectionrP   s     €r8   rC   zSmolVLMConnector.__init__š  s:   ø€ Ý‰Œ×ÒÑÔÐØ"Ô/ˆÔÝ#3°FÑ#;Ô#;ˆÔ Ð Ð r7   r   c                 ó   — |                      ¦   «         \  }}}t          |dz  ¦  «        x}}|                     ||||¦  «        }|                     ||t          ||z  ¦  «        ||z  ¦  «        }|                     dddd¦  «        }|                     |t          ||z  ¦  «        t          ||z  ¦  «        ||dz  z  ¦  «        }|                     dddd¦  «        }|                     |t          ||dz  z  ¦  «        ||dz  z  ¦  «        }|S )Ng      à?r   r   r   r   )rX   Úintrl   Úpermuterk   )rQ   r  r   ÚbszÚseqrE   ÚheightÚwidths           r8   Úpixel_shufflezSmolVLMConnector.pixel_shuffleŸ  s  € ØŸfšf™hœhÑˆˆS�)Ý˜S #™X™œÐ&ˆ�Ø�FŠF�3˜  yÑ1Ô1ˆØ�FŠF�3˜¥ E¨LÑ$8Ñ 9Ô 9¸9À|Ñ;SÑTÔTˆØ�IŠI�a˜˜A˜qÑ!Ô!ˆØ�IŠI�c�3˜u |Ñ3Ñ4Ô4µc¸&À<Ñ:OÑ6PÔ6PÐR[Ð_kÐmnÑ_nÑRoÑpÔpˆØ�IŠI�a˜˜A˜qÑ!Ô!ˆØ�IŠI�c�3˜s l°A¡oÑ6Ñ7Ô7¸ÀlÐTUÁoÑ9VÑWÔWˆØˆr7   c                 óf   — |                       || j        ¦  «        }|                      |¦  «        }|S r·   )r  r   r
  )rQ   rù   s     r8   r„   zSmolVLMConnector.forwardª  s7   € Ø"×0Ò0Ð1DÀdÔFWÑXÔXÐØ"×6Ò6Ð7JÑKÔKÐØ"Ð"r7   )r   )r)   r*   r+   rC   r  r„   r‰   rŠ   s   @r8   r  r  ™  s[   ø€ € € € € ð<ð <ð <ð <ð <ð
	ð 	ð 	ð 	ð#ð #ð #ð #ð #ð #ð #r7   r  zY
    SmolVLM model consisting of a SIGLIP vision encoder and Llama3 language decoder
    c                   óü  ‡ — e Zd ZdZdefˆ fd„Zd„ Zd„ Zdej	        dej
        dej
        fd	„Ze ed
¬¦  «        	 ddej        dej	        dz  dee         deez  fd„¦   «         ¦   «         Zee ed¬¦  «        	 	 	 	 	 	 	 	 	 ddej	        dz  dej
        dz  dej	        dz  dedz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dee         deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚSmolVLMModelz§
    A subclass of Idefics3Model. We do *not* remove or block the call to inputs_merger
    in forward. Instead, we override inputs_merger here with custom logic.
    r"   c                 ó
  •— t          ¦   «                              |¦  «         | j        j        j        | _        | j        j        j        | _        t                               |j	        ¦  «        | _
        t          |¦  «        | _        t          j        |j        ¦  «        | _        t!          |j	        j        |j	        j        z  dz  |j        dz  z  ¦  «        | _        | j        j        | _        |                      ¦   «          d S )Nr   )rB   rC   r"   r  Úpad_token_idÚpadding_idxÚ
vocab_sizerã   Ú_from_configrÿ   Úvision_modelr  Ú	connectorr   Úfrom_configÚ
text_modelr  rF   rG   r   Úimage_seq_lenÚimage_token_idrè   rP   s     €r8   rC   zSmolVLMModel.__init__»  s×   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Øœ;Ô2Ô?ˆÔØœ+Ô1Ô<ˆŒå4×AÒAÀ&ÔBVÑWÔWˆÔÝ)¨&Ñ1Ô1ˆŒÝ#Ô/°Ô0BÑCÔCˆŒå ØÔ"Ô-°Ô1EÔ1PÑPÐUVÑVÐ[aÔ[nÐpqÑ[qÑrñ
ô 
ˆÔð #œkÔ8ˆÔà�ŠÑÔÐÐÐr7   c                 ó4   — | j                              ¦   «         S r·   )r  rì   rë   s    r8   rì   z!SmolVLMModel.get_input_embeddingsË  s   € ØŒ×3Ò3Ñ5Ô5Ð5r7   c                 ó:   — | j                              |¦  «         d S r·   )r  rï   rî   s     r8   rï   z!SmolVLMModel.set_input_embeddingsÎ  s   € ØŒ×,Ò,¨UÑ3Ô3Ð3Ð3Ð3r7   Ú	input_idsrÝ   rù   c                 ó0  — |j         \  }}}|€X| |                      ¦   «         t          j        | j        j        t          j        |j        ¬¦  «        ¦  «        k    }|d         }n|| j        j        k    }|                     d¬¦  «        }t          t          j
        ||z  dk    ¦  «        d¦  «         ||z  }t          j        j                             |                     d¬¦  «        dd¬	¦  «        }	|	dd
…         }
|                     d
¬¦  «        }|dz
  |z  }|dz
  |z  }|
                     d¦  «        |z   }t          j        |¦  «        }|||         ||         dd…f         ||<   t          j        |                     d
¦  «        ||¦  «        }|S )as  
        This method aims at merging the token embeddings with the image hidden states into one single sequence of vectors that are fed to the transformer LM.
        The merging happens as follows:
        - The text token sequence is: `tok_1 tok_2 tok_3 <fake_token_around_image> <image> <image> ... <image> <fake_token_around_image> tok_4`.
        - We get the image hidden states for the image through the vision encoder and that hidden state, after a pixel shuffle operation, is then projected into the text embedding space.
        We thus have a sequence of image hidden states of size (1, image_seq_len, hidden_dim), where 1 is for batch_size of 1 image and hidden_dim is the hidden_dim of the LM transformer.
        - The merging happens so that we obtain the following sequence: `vector_tok_1 vector_tok_2 vector_tok_3 vector_fake_tok_around_image {sequence of image_seq_len image hidden states} vector_fake_toke_around_image vector_tok_4`. That sequence is fed to the LM.
        - To fit the format of that sequence, `input_ids`, `inputs_embeds`, `attention_mask` are all 3 adapted to insert the image hidden states.
        Nrò   ).r   r   rZ   r   zCAt least one sample has <image> tokens not divisible by patch_size.)r   r   )r�   r_   )r`   rì   rc   Útensorr"   r   ÚlongrW   rf   r   Úallr   r˜   ÚpadÚcumsumÚ	unsqueezeÚ
zeros_likeÚwhere)rQ   r#  rÝ   rù   rn   rG   Ú
image_maskÚnum_image_tokensÚblocks_per_sampleÚoffsetsÚblock_offsetÚrow_cumÚ	chunk_idxÚ	local_idxÚ	block_idxÚimage_embedsÚmerged_embedss                    r8   Úinputs_mergerzSmolVLMModel.inputs_mergerÑ  s°  € ð /Ô4Ñˆˆ:�qàÐØ&Ð*E¨$×*CÒ*CÑ*EÔ*EÝ”˜Tœ[Ô7½u¼zÐR_ÔRfÐgÑgÔgñ+ô +ò ˆJð $ FÔ+ˆJˆJà" d¤kÔ&@Ò@ˆJà%Ÿ>š>¨a˜>Ñ0Ô0ÐÝÝŒIÐ&¨Ñ3°qÒ8Ñ9Ô9ØQñ	
ô 	
ð 	
ð -°
Ñ:Ðå”(Ô%×)Ò)Ð*;×*BÒ*BÀqÐ*BÑ*IÔ*IÈ6ÐYZÐ)Ñ[Ô[ˆØ˜s ˜s”|ˆØ×#Ò#¨Ð#Ñ+Ô+ˆØ˜q‘[ ZÑ/ˆ	Ø˜q‘[ JÑ.ˆ	Ø ×*Ò*¨1Ñ-Ô-°	Ñ9ˆ	åÔ'¨Ñ6Ô6ˆØ#6°yÀÔ7LÈiÐXbÔNcÐefÐefÐefÐ7fÔ#gˆ�ZÑ åœ J×$8Ò$8¸Ñ$<Ô$<¸lÈMÑZÔZˆØÐr7   zVEncodes images into continuous embeddings that can be forwarded to the language model.rà   NrS   Úpixel_attention_maskr›   rU   c                 ó¨  ‡— ‰j         \  }}}}}‰                     | j        ¬¦  «        Š ‰j        ||z  g‰j         dd…         ¢R Ž Š‰j         dd…                              ¦   «         }	‰dk                         d¬¦  «        |	k    }
|
dxx         t          j        |
¦  «         z  cc<   ‰|
                              ¦   «         Š|€3t          j	        ˆfd	„d
D ¦   «         t          j
        ‰j        ¬¦  «        }n8 |j        ||z  g|j         dd…         ¢R Ž }||
                              ¦   «         }| j        j        j        }|                     d||¬¦  «        }|                     d||¬¦  «        }|                     d¬¦  «        dk     
                    ¦   «         } | j        d‰|ddœ|¤Ž}|j        }|                      |¦  «        }||_        |S )á4  
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
            The tensors corresponding to the input images.
        pixel_attention_mask (`torch.LongTensor`, *optional*):
            The attention mask indicating padded regions in the image.
        )r\   r   Nr   r‹   )r_   r”   éýÿÿÿrZ   r   c                 ó*   •— g | ]}‰j         |         ‘ŒS r6   )r`   )rÓ   ÚirS   s     €r8   rÔ   z3SmolVLMModel.get_image_features.<locals>.<listcomp>  s!   ø€ Ð?Ð?Ð?°�lÔ(¨Ô+Ð?Ð?Ð?r7   )r   r   r   )rX   r\   rW   )Ú	dimensionrX   Ústep)r_   r”   T)rS   rT   Úreturn_dictr6   )r`   ri   r\   rl   Únumelrf   rc   Úanyrš   ró   rô   rW   r"   rÿ   rG   Úunfoldr  rÜ   r  Úpooler_output)rQ   rS   r9  r›   rm   Ú
num_imagesrI   r  r  Únb_values_per_imageÚreal_images_indsrG   Úpatches_subgridrT   Úimage_outputsrù   Úimage_featuress    `               r8   Úget_image_featureszSmolVLMModel.get_image_featuresû  s7  ø€ ð  ?KÔ>PÑ;ˆ
�J ¨f°eØ#—’¨T¬Z�Ñ8Ô8ˆØ(�|Ô(¨°jÑ)@ÐZÀ<ÔCUÐVWÐVXÐVXÔCYÐZÐZÐZˆð +Ô0°°°Ô4×:Ò:Ñ<Ô<ÐØ(¨CÒ/×4Ò4¸Ð4ÑFÔFÐJ]Ò]Ðð 	˜ÐÐÔ¥¤	Ð*:Ñ ;Ô ;Ð;Ñ;ÐÐÑà#Ð$4Ô5×@Ò@ÑBÔBˆàÐ'Ý#(¤:Ø?Ð?Ð?Ð?°YÐ?Ñ?Ô?Ý”jØ#Ô*ð$ñ $ô $Ð Ð ð $=Ð#7Ô#<¸ZÈ*Ñ=TÐ#vÐWkÔWqÐrsÐrtÐrtÔWuÐ#vÐ#vÐ#vÐ Ø#7Ð8HÔ#I×#TÒ#TÑ#VÔ#VÐ Ø”[Ô.Ô9ˆ
Ø.×5Ò5ÀÈ
ÐYcÐ5ÑdÔdˆØ)×0Ò0¸1À:ÐT^Ð0Ñ_Ô_ˆØ /× 3Ò 3¸Ð 3Ñ AÔ AÀAÒ E×KÒKÑMÔMÐð *˜Ô)ð 
Ø%Ð<PÐ^bð
ð 
Øflð
ð 
ˆð ,Ô=Ðð ŸšÐ(;Ñ<Ô<ˆØ&4ˆÔ#àÐr7   aØ  
        Inputs fed to the model can have an arbitrary number of images. To account for this, pixel_values fed to
        the model have image padding -> (batch_size, max_num_images, 3, max_heights, max_widths) where
        max_num_images is the maximum number of images among the batch_size samples in the batch.
        Padding images are not needed beyond padding the pixel_values at the entrance of the model.
        For efficiency, we only pass through the vision_model's forward the real images by
        discarding the padding images i.e. pixel_values of size (image_batch_size, 3, height, width) where
        image_batch_size would be 7 when num_images_per_sample=[1, 3, 1, 2] and max_num_images would be 3.
        r�   rv   r(   Ú	use_cachec
           	      óŠ  — |�|j         \  }}n|�|j         \  }}}nt          d¦  «        ‚|	r|€t          | j        ¬¦  «        }|€: | j                             ¦   «         |¦  «                             |j        ¦  «        }|�|�t          d¦  «        ‚|�8|                      ||d¬¦  «        j	        }|                     |j        ¦  «        }n#|�!|                     | j
        |j        ¬¦  «        }|�|                      |||¬¦  «        } | j        d|||||	d	œ|
¤Ž}t          |j        |j        |j        |j        |¬
¦  «        S )a|  
        pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*):
            Mask to avoid performing attention on padding pixel indices.
        image_hidden_states (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
            The hidden states of the image encoder after modality projection.
        Nz5You have to specify either input_ids or inputs_embeds)r"   zMYou cannot specify both pixel_values and image_hidden_states at the same timeT)rA  rò   )r#  rÝ   rù   )rÝ   r�   rv   r(   rM  )rÜ   r(   r­   rä   rù   r6   )r`   r¤   r	   r"   r  rì   ri   rW   rL  rE  r\   r8  rø   rÜ   r(   r­   rä   )rQ   r#  r�   rv   r(   rÝ   rS   r9  rù   rM  r›   rm   Ú
seq_lengthrn   Úoutputss                  r8   r„   zSmolVLMModel.forward3  s²  € ð@ Ð Ø%.¤_Ñ"ˆJ˜
˜
ØÐ&Ø(5Ô(;Ñ%ˆJ˜
 A AåÐTÑUÔUÐUàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐ ØB˜DœO×@Ò@ÑBÔBÀ9ÑMÔM×PÒPÐQZÔQaÑbÔbˆMàÐ#Ð(;Ð(GÝÐlÑmÔmÐmàÐ#Ø"&×"9Ò"9ØÐ2Àð #:ñ #ô #äð  ð #6×"8Ò"8¸Ô9MÑ"NÔ"NÐÐØ Ð,Ø"5×"8Ò"8¸t¼zÐR_ÔRfÐ"8Ñ"gÔ"gÐàÐ*Ø ×.Ò.Ø#Ø+Ø$7ð /ñ ô ˆMð "�$”/ð 
Ø'Ø)Ø%Ø+Øð
ð 
ð ð
ð 
ˆõ .Ø%Ô7Ø#Ô3Ø!Ô/ØÔ)Ø 3ð
ñ 
ô 
ð 	
r7   r·   )	NNNNNNNNN)r)   r*   r+   r…   r   rC   rì   rï   rc   Ú
LongTensorrˆ   r8  r   r   r†   r   r   r¸   r   rL  r   r   r‡   rô   r   rø   r„   r‰   rŠ   s   @r8   r  r  °  sg  ø€ € € € € ðð ð
˜}ð ð ð ð ð ð ð 6ð 6ð 6ð4ð 4ð 4ð(ØÔ)ð(Ø:?¼,ð(Ø]bÔ]ið(ð (ð (ð (ðT Ø€^Ømðñ ô ð 9=ð2ð 2àÔ'ð2ð $Ô.°Ñ5ð2ð Ð+Ô,ð	2ð
 
Ð+Ñ	+ð2ð 2ð 2ñô ñ Ôð2ðh  ØØ€^ðð
ñ 
ô 
ð .2Ø.2Ø04Ø(,Ø26Ø15Ø8<Ø8<Ø!%ðA
ð A
àÔ# dÑ*ðA
ð œ tÑ+ðA
ð Ô&¨Ñ-ð	A
ð
  ™ðA
ð Ô(¨4Ñ/ðA
ð Ô'¨$Ñ.ðA
ð $Ô.°Ñ5ðA
ð #Ô.°Ñ5ðA
ð ˜$‘;ðA
ð Ð-Ô.ðA
ð 
Ð/Ñ	/ðA
ð A
ð A
ñ
ô 
ñ Ôñ  ÔðA
ð A
ð A
ð A
ð A
r7   r  zS
    Base class for Idefics causal language model (or autoregressive) outputs.
    c                   óâ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dS )	ÚSmolVLMCausalLMOutputWithPastaC  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
        sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder
    NÚlossÚlogitsr(   r­   rä   rù   )r)   r*   r+   r…   rT  rc   r†   r,   rU  r(   r   r­   r¸   rä   rù   r6   r7   r8   rS  rS  „  sº   € € € € € € ðð ð  &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø;?Ð˜˜uÔ0Ô1°DÑ8Ð?Ð?Ñ?Ð?Ð?r7   rS  z‡
    The SmolVLM Model with a language modeling head. It is made up a SigLIP vision encoder, with a language modeling head on top.
    c                   óÊ  ‡ — e Zd ZddiZˆ fd„Zd„ Zd„ Ze	 ddej	        dej
        dz  d	ee         d
eez  fd„¦   «         Zee	 	 	 	 	 	 	 	 	 	 	 ddej
        dz  dej        dz  dej
        dz  dedz  dej	        dz  dej	        dz  dej        dz  dej	        dz  dej
        dz  dedz  deej        z  d	ee         d
eez  fd„¦   «         ¦   «         Z	 	 	 	 	 	 	 	 	 dˆ fd„	Zˆ xZS )ÚSmolVLMForConditionalGenerationzlm_head.weightz$model.text_model.embed_tokens.weightc                 ó„  •— t          ¦   «                              |¦  «         t          |¦  «        | _        | j        j        | _        t          j        |j        j	        |j        j
        d¬¦  «        | _        |j        j
        | _
        t          j        |¦  «        | j        j        _        |                      ¦   «          d S )NFrý   )rB   rC   r  r#   r"   r   r   r§   r  rD   r  Úlm_headr
   Úfrom_model_configr  Úgeneration_configrè   rP   s     €r8   rC   z(SmolVLMForConditionalGeneration.__init__«  sš   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý! &Ñ)Ô)ˆŒ
Ø"œkÔ8ˆÔÝ”y Ô!3Ô!?ÀÔASÔA^ÐejÐkÑkÔkˆŒØ Ô,Ô7ˆŒÝ2BÔ2TÐU[Ñ2\Ô2\ˆŒ
ÔÔ/ð 	�ŠÑÔÐÐÐr7   c                 ó>   — | j         j                             ¦   «         S r·   )r#   r  rì   rë   s    r8   rì   z4SmolVLMForConditionalGeneration.get_input_embeddings¶  s   € ØŒzÔ$×9Ò9Ñ;Ô;Ð;r7   c                 óD   — | j         j                             |¦  «         d S r·   )r#   r  rï   rî   s     r8   rï   z4SmolVLMForConditionalGeneration.set_input_embeddings¹  s!   € ØŒ
Ô×2Ò2°5Ñ9Ô9Ð9Ð9Ð9r7   NrS   r9  r›   rU   c                 ó,   —  | j         j        d||dœ|¤ŽS )r;  )rS   r9  r6   )r#   rL  )rQ   rS   r9  r›   s       r8   rL  z2SmolVLMForConditionalGeneration.get_image_features¼  s5   € ð -ˆtŒzÔ,ð 
Ø%Ð<Pð
ð 
ØTZð
ð 
ð 	
r7   r   r#  r�   rv   r(   rÝ   rù   ÚlabelsrM  Úlogits_to_keepc                 óp  —  | j         d|||||||||
ddœ
|¤Ž}|d         }t          |t          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|	�  | j        d||	| j        j        j        dœ|¤Ž}t          |||j
        |j        |j        |j        ¬¦  «        S )aÔ	  
        pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*):
            Mask to avoid performing attention on padding pixel indices.
        image_hidden_states (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
            The hidden states of the image encoder after modality projection.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or `model.image_token_id`. Tokens with indices set to `model.image_token_id` are
            ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> import httpx
        >>> from io import BytesIO
        >>> import torch
        >>> from PIL import Image
        >>> from io import BytesIO

        >>> from transformers import AutoProcessor, AutoModelForImageTextToText
        >>> from transformers.image_utils import load_image

        >>> # Note that passing the image urls (instead of the actual pil images) to the processor is also possible
        >>> image1 = load_image("https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg")
        >>> image2 = load_image("https://cdn.britannica.com/59/94459-050-DBA42467/Skyline-Chicago.jpg")
        >>> image3 = load_image("https://cdn.britannica.com/68/170868-050-8DDE8263/Golden-Gate-Bridge-San-Francisco.jpg")

        >>> processor = AutoProcessor.from_pretrained("HuggingFaceTB/SmolVLM2-2.2B-Instruct")
        >>> model = AutoModelForImageTextToText.from_pretrained("HuggingFaceTB/SmolVLM2-2.2B-Instruct", dtype=torch.bfloat16, device_map="auto")

        >>> # Create inputs
        >>> messages = [
        ...     {
        ...         "role": "user",
        ...         "content": [
        ...             {"type": "video", "path": path/to/video},
        ...             {"type": "text", "text": "What is happening in this video?"},
        ...         ]
        ...     }
        ... ]

        >>> inputs = processor.apply_chat_template([messages], add_generation_prompt=True)

        >>> # Generate
        >>> generated_ids = model.generate(**inputs, max_new_tokens=256)
        >>> generated_texts = processor.batch_decode(generated_ids, skip_special_tokens=True)

        >>> print(generated_texts)
        ```T)
r#  r�   rv   r(   rÝ   rS   r9  rù   rM  rA  r   N)rU  r_  r  )rT  rU  r(   r­   rä   rù   r6   )r#   Ú
isinstancer  ÚslicerY  Úloss_functionr"   r  r  rS  r(   r­   rä   rù   )rQ   r#  r�   rv   r(   rÝ   rS   r9  rù   r_  rM  r`  r›   rP  r­   Úslice_indicesrU  rT  s                     r8   r„   z'SmolVLMForConditionalGeneration.forwardÍ  s  € ðF �$”*ð 
ØØ)Ø%Ø+Ø'Ø%Ø!5Ø 3ØØð
ð 
ð ð
ð 
ˆð   œ
ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%ð Ø f¸¼Ô9PÔ9[ðð Ø_eðð ˆDõ -ØØØ#Ô3Ø!Ô/ØÔ)Ø 'Ô ;ð
ñ 
ô 
ð 	
r7   Fc                 óp   •—  t          ¦   «         j        |f||||||||	|
dœ	|¤Ž}|€|
r|	s
d |d<   d |d<   |S )N)	r(   r�   rÝ   rS   r9  rù   r`  Úis_first_iterationrM  rS   r9  )rB   Úprepare_inputs_for_generation)rQ   r#  r(   r�   rÝ   rS   r9  rù   r`  rg  rM  r›   Úmodel_inputsrR   s                €r8   rh  z=SmolVLMForConditionalGeneration.prepare_inputs_for_generation2  s}   ø€ ð" =•u‘w”wÔ<Øð
à+Ø)Ø'Ø%Ø!5Ø 3Ø)Ø1Øð
ð 
ð ð
ð 
ˆð Ð*¨yÐ*ÐASÐ*Ø+/ˆL˜Ñ(Ø37ˆLÐ/Ñ0àÐr7   r·   )NNNNNNNNNNr   )	NNNNNNNFF)r)   r*   r+   Ú_tied_weights_keysrC   rì   rï   r   rc   r†   rQ  r   r   r¸   r   rL  r   rˆ   r   r‡   rô   r  rS  r„   rh  r‰   rŠ   s   @r8   rW  rW  £  sF  ø€ € € € € ð +Ð,RÐSÐð	ð 	ð 	ð 	ð 	ð<ð <ð <ð:ð :ð :ð ð 9=ð
ð 
àÔ'ð
ð $Ô.°Ñ5ð
ð Ð+Ô,ð	
ð
 
Ð+Ñ	+ð
ð 
ð 
ñ „^ð
ð  Øð .2Ø.2Ø04Ø(,Ø26Ø15Ø8<Ø8<Ø*.Ø!%Ø-.ða
ð a
àÔ# dÑ*ða
ð œ tÑ+ða
ð Ô&¨Ñ-ð	a
ð
  ™ða
ð Ô(¨4Ñ/ða
ð Ô'¨$Ñ.ða
ð $Ô.°Ñ5ða
ð #Ô.°Ñ5ða
ð Ô  4Ñ'ða
ð ˜$‘;ða
ð ˜eœlÑ*ða
ð Ð+Ô,ða
ð 
Ð.Ñ	.ða
ð a
ð a
ñ „^ñ Ôða
ðL ØØØØ!Ø ØØ Øð#ð #ð #ð #ð #ð #ð #ð #ð #ð #r7   rW  )rW  r!   r  rã   )r‹   )=Úcollections.abcr   Údataclassesr   rc   r   Úactivationsr   Úcache_utilsr   r	   Ú
generationr
   r   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úautor   Úconfiguration_smolvlmr   r   r!   ÚModuler:   rˆ   Úfloatrž   r&   rº   rÃ   rÐ   rã   rø   rû   r  r  rS  rW  Ú__all__r6   r7   r8   ú<module>r~     sI  ðð. %Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð à !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aØ 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø Ð Ð Ð Ð Ð Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ Eð ð
'ð 
'ð 
'ð 
'ð 
'˜_ñ 
'ô 
'ñ „ð
'ðIð Ið Ið Ið I˜bœiñ Iô Ið Iðf ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð %ð %ð %ð.8)ð 8)ð 8)ð 8)ð 8)˜RœYñ 8)ô 8)ð 8)ðvð ð ð ð �r”yñ ô ð ðð ð ð ð Ð4ñ ô ð ðD@ð @ð @ð @ð @�R”Yñ @ô @ð @ðD €ððñ ô ð
A
ð A
ð A
ð A
ð A
Ð5ñ A
ô A
ñô ð
A
ðH €ððñ ô ð
 ð@ð @ð @ð @ð @ [ñ @ô @ñ „ñô ð@ð&ð ð ð ð �r”yñ ô ð ð#ð #ð #ð #ð #�r”yñ #ô #ð #ð. €ððñ ô ð
L
ð L
ð L
ð L
ð L
Ð)ñ L
ô L
ñô ð
L
ð^ €ððñ ô ð
 ð@ð @ð @ð @ð @ Kñ @ô @ñ „ñô ð@ð2 €ððñ ô ð
mð mð mð mð mÐ&<¸oñ mô mñô ð
mð` tÐ
sÐ
s€€€r7   