§
    ‚Štj¯b  ã                   óØ  — d dl Zd dl mZ d dlmZ d dlZd dlmZ ddlm	Z
 ddlmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZmZ ddlmZ ddlmZmZmZ ddlmZmZ ddl m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z)m*Z*m+Z+m,Z,m-Z- ddl.m/Z/m0Z0 	 dCdej1        dej2        dej2        dej2        dej2        dz  de3de3e4z  fd„Z5 G d„ d e'¦  «        Z6 G d!„ d"e%¦  «        Z7 ed#¬$¦  «        e G d%„ d&e¦  «        ¦   «         ¦   «         Z8 G d'„ d(ej1        ¦  «        Z9 G d)„ d*ej1        ¦  «        Z: G d+„ d,e#¦  «        Z;ej<        e6d-œZ= G d.„ d/e¦  «        Z> G d0„ d1ej1        ¦  «        Z?e G d2„ d3e¦  «        ¦   «         Z@e G d4„ d5e@¦  «        ¦   «         ZA G d6„ d7e-¦  «        ZBdZC G d8„ d9ej1        ¦  «        ZD G d:„ d;e,¦  «        ZE G d<„ d=e+¦  «        ZF G d>„ d?e)¦  «        ZG G d@„ dAe*¦  «        ZHg dB¢ZIdS )Dé    N)ÚCallable)Ú	dataclassé   )Úinitialization)ÚACT2FN)ÚCache)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPooling)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚ	torch_int)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputsé   )ÚCLIPMLP)ÚJanusVisionAttention)ÚLlamaRMSNorm)ÚLlavaCausalLMOutputWithPastÚLlavaForConditionalGenerationÚ
LlavaModelÚLlavaModelOutputWithPastÚLlavaPreTrainedModelé   )ÚInternVLConfigÚInternVLVisionConfigç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutc                 ó‚  — |}|}	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t          j                             |
d¬¦  «        }
t          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr   r   éÿÿÿÿ©Údim)ÚpÚtrainingr   )	ÚtorchÚmatmulÚ	transposeÚnnÚ
functionalÚsoftmaxr(   r.   Ú
contiguous)r"   r#   r$   r%   r&   r'   r(   ÚkwargsÚ
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               úk/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/internvl/modular_internvl.pyÚeager_attention_forwardr<   .   s»   € ð €JØ€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆõ ”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$ó    c                   ó   — e Zd ZdS )ÚInternVLVisionRMSNormN©Ú__name__Ú
__module__Ú__qualname__© r=   r;   r?   r?   H   ó   € € € € € Ø€Dr=   r?   c                   ób   ‡ — e Zd Zdefˆ fd„Z	 ddej        dej        dz  dee         fd„Z	ˆ xZ
S )	ÚInternVLVisionAttentionÚconfigc                 ó"  •— t          ¦   «                              |¦  «         | `d| _        |j        }|rt          | j        ¦  «        nt          j        ¦   «         | _	        |rt          | j        ¦  «        nt          j        ¦   «         | _
        d S )NF)ÚsuperÚ__init__Únum_key_value_groupsÚ	is_causalÚuse_qk_normr?   Ú	embed_dimr2   ÚIdentityÚq_normÚk_norm)ÚselfrH   Úqk_normÚ	__class__s      €r;   rK   z InternVLVisionAttention.__init__M   sz   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØÐ%ð ˆŒØÔ$ˆà?FÐYÕ+¨D¬NÑ;Ô;Ð;ÍBÌKÉMÌMˆŒØ?FÐYÕ+¨D¬NÑ;Ô;Ð;ÍBÌKÉMÌMˆŒˆˆr=   NÚhidden_statesr&   r6   c                 ór  — |                      ¦   «         \  }}}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }	|                      |¦  «        }|                      |¦  «        }|                     ||| j        | j        ¦  «         	                    dd¦  «        }|                     ||| j        | j        ¦  «         	                    dd¦  «        }|	 
                    ||| j        | j        ¦  «         	                    dd¦  «        }	t          j        | j        j        t          ¦  «        }
 |
| |||	|f| j        sdn| j        | j        ddœ|¤Ž\  }}|                     ||| j        ¦  «        }|                      |¦  «        }|                      |¦  «        }||fS )Nr   r   r!   F)r(   r'   rM   )ÚsizeÚq_projÚk_projÚv_projrQ   rR   ÚreshapeÚ	num_headsÚhead_dimr1   Úviewr   Úget_interfacerH   Ú_attn_implementationr<   r.   Úattention_dropoutÚscalerO   Úprojection_layerÚprojection_dropout)rS   rV   r&   r6   Ú
batch_sizeÚseq_lenÚ_Úquery_statesr7   r8   Úattention_interfacer:   r9   Úoutputs                 r;   ÚforwardzInternVLVisionAttention.forwardX   sÁ  € ð "/×!3Ò!3Ñ!5Ô!5Ñˆ
�G˜Qà—{’{ =Ñ1Ô1ˆØ—[’[ Ñ/Ô/ˆ
Ø—{’{ =Ñ1Ô1ˆà—{’{ <Ñ0Ô0ˆØ—[’[ Ñ,Ô,ˆ
à#×+Ò+¨J¸ÀÄÐQUÔQ^Ñ_Ô_×iÒiÐjkÐmnÑoÔoˆØ×'Ò'¨
°G¸T¼^ÈTÌ]Ñ[Ô[×eÒeÐfgÐijÑkÔkˆ
Ø#×(Ò(¨°W¸d¼nÈdÌmÑ\Ô\×fÒfÐghÐjkÑlÔlˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}ÐH�C�C°$Ô2HØ”JØð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð "×)Ò)¨*°g¸t¼~ÑNÔNˆà×&Ò& {Ñ3Ô3ˆØ×(Ò(¨Ñ0Ô0ˆà�|Ð#Ð#r=   ©N)rA   rB   rC   r    rK   r/   ÚTensorr   r   rl   Ú__classcell__©rU   s   @r;   rG   rG   L   s•   ø€ € € € € ð	ZÐ3ð 	Zð 	Zð 	Zð 	Zð 	Zð 	Zð /3ð'$ð '$à”|ð'$ð œ tÑ+ð'$ð Ð+Ô,ð	'$ð '$ð '$ð '$ð '$ð '$ð '$ð '$r=   rG   z7
    Class for outputs of [`InternVLVisionModel`].
    ©Úcustom_introc                   ó   — e Zd ZdZdS )Ú$InternVLVisionModelOutputWithPoolingaF  
    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`):
        Average of the last layer hidden states of the patch tokens (excluding the *[CLS]* token) if
        *config.use_mean_pooling* is set to True. If set to False, then the final hidden state of the *[CLS]* token
        will be returned.
    N)rA   rB   rC   Ú__doc__rD   r=   r;   rt   rt   ‚   s   € € € € € ðð ð ð r=   rt   c                   óF   ‡ — e Zd ZdZˆ fd„Zdej        dej        fd„Zˆ xZS )ÚInternVLVisionPatchEmbeddingszì
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    c                 óŠ  •— t          ¦   «                              ¦   «          |j        |j        }}|j        |j        }}|d         |d         z  |d         |d         z  z  }|d         |d         z  |d         |d         z  f}|| _        || _        || _        || _        || _        t          j	        ||||¬¦  «        | _
        d S )Nr   r   )Úkernel_sizeÚstride)rJ   rK   Ú
image_sizeÚ
patch_sizeÚnum_channelsÚhidden_sizeÚnum_patchesÚpatch_shaper2   ÚConv2dÚ
projection)	rS   rH   r{   r|   r}   r~   r   r€   rU   s	           €r;   rK   z&InternVLVisionPatchEmbeddings.__init__˜   sÆ   ø€ Ý‰Œ×ÒÑÔÐØ!'Ô!2°FÔ4E�Jˆ
Ø$*Ô$7¸Ô9K�kˆà! !”}¨
°1¬Ñ5¸*ÀQ¼-È:ÐVWÌ=Ñ:XÑYˆØ! !”}¨
°1¬Ñ5°zÀ!´}È
ÐSTÌÑ7UÐVˆØ$ˆŒØ$ˆŒØ(ˆÔØ&ˆÔØ&ˆÔåœ) L°+È:Ð^hÐiÑiÔiˆŒˆˆr=   Úpixel_valuesÚreturnc                 ó  — |j         \  }}}}|| j        k    rt          d¦  «        ‚|                      |                     | j        j        j        ¦  «        ¦  «        }|                     d¦  «                             dd¦  «        }|S )NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.r   r   )	Úshaper}   Ú
ValueErrorr‚   ÚtoÚweightÚdtypeÚflattenr1   )rS   rƒ   rf   r}   ÚheightÚwidthÚ
embeddingss          r;   rl   z%InternVLVisionPatchEmbeddings.forward§   s†   € Ø2>Ô2DÑ/ˆ
�L &¨%Ø˜4Ô,Ò,Ð,ÝØwñô ð ð —_’_ \§_¢_°T´_Ô5KÔ5QÑ%RÔ%RÑSÔSˆ
Ø×'Ò'¨Ñ*Ô*×4Ò4°Q¸Ñ:Ô:ˆ
àÐr=   )	rA   rB   rC   ru   rK   r/   rn   rl   ro   rp   s   @r;   rw   rw   ‘   sm   ø€ € € € € ðð ðjð jð jð jð jð
 E¤Lð 
°U´\ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r=   rw   c                   ó”   ‡ — e Zd ZdZdeddfˆ fd„Zdej        dededej        fd	„Z		 dd
ej        dej
        dz  dej        fd„Zˆ xZS )ÚInternVLVisionEmbeddingszc
    Construct the CLS token, position and patch embeddings. Optionally, also the mask token.

    rH   r„   Nc                 óÆ  •— t          ¦   «                              ¦   «          t          j        t	          j        dd|j        ¦  «        ¦  «        | _        |j        r3t          j        t	          j        dd|j        ¦  «        ¦  «        | _	        nd | _	        t          |¦  «        | _        |j        | _        t          |j        t          j        j        ¦  «        r|j        n|j        |j        f| _        | j        j        }|j        r6t          j        t	          j        d|dz   |j        ¦  «        ¦  «        | _        nd | _        t          j        |j        ¦  «        | _        d S )Nr   )rJ   rK   r2   Ú	Parameterr/   Úzerosr~   Ú	cls_tokenÚuse_mask_tokenÚ
mask_tokenrw   Úpatch_embeddingsr|   Ú
isinstancer{   ÚcollectionsÚabcÚIterabler   Ú use_absolute_position_embeddingsÚposition_embeddingsÚDropoutÚhidden_dropout_probr(   )rS   rH   r   rU   s      €r;   rK   z!InternVLVisionEmbeddings.__init__¼   s(  ø€ Ý‰Œ×ÒÑÔÐåœ¥e¤k°!°Q¸Ô8JÑ&KÔ&KÑLÔLˆŒØÔ ð 	#Ý œl­5¬;°q¸!¸VÔ=OÑ+PÔ+PÑQÔQˆDŒOˆOà"ˆDŒOÝ =¸fÑ EÔ EˆÔØ Ô+ˆŒõ ˜&Ô+­[¬_Ô-EÑFÔFð8ˆFÔÐàÔ# VÔ%6Ð7ð 	Œð
 Ô+Ô7ˆØÔ2ð 	,Ý')¤|µE´KÀÀ;ÐQRÁ?ÐTZÔTfÑ4gÔ4gÑ'hÔ'hˆDÔ$Ð$à'+ˆDÔ$Ý”z &Ô"<Ñ=Ô=ˆŒˆˆr=   rŽ   rŒ   r�   c                 ó¬  — |j         d         dz
  }| j        j         d         dz
  }t          j                             ¦   «         s||k    r||k    r| j        S | j        dd…dd…f         }| j        dd…dd…f         }|j         d         }|| j        d         z  }	|| j        d         z  }
t          |dz  ¦  «        }|                     d|||¦  «        }|                     dddd¦  «        }t          j
                             ||	|
fdd	¬
¦  «        }|                     dddd¦  «                             dd|¦  «        }t          j        ||fd¬¦  «        S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   Nr*   r   ç      à?r   r   ÚbicubicF)rX   ÚmodeÚalign_cornersr+   )r†   r�   r/   ÚjitÚ
is_tracingr|   r   r\   Úpermuter2   r3   Úinterpolater_   Úcat)rS   rŽ   rŒ   r�   r   Únum_positionsÚclass_pos_embedÚpatch_pos_embedr,   Ú
new_heightÚ	new_widthÚsqrt_num_positionss               r;   Úinterpolate_pos_encodingz1InternVLVisionEmbeddings.interpolate_pos_encodingÒ   s|  € ð !Ô& qÔ)¨AÑ-ˆØÔ0Ô6°qÔ9¸AÑ=ˆõ Œy×#Ò#Ñ%Ô%ð 	,¨+¸Ò*FÐ*FÈ6ÐUZÊ?È?ØÔ+Ð+àÔ2°1°1°1°b°q°b°5Ô9ˆØÔ2°1°1°1°a°b°b°5Ô9ˆàÔ˜rÔ"ˆà˜tœ¨qÔ1Ñ1ˆ
Ø˜Tœ_¨QÔ/Ñ/ˆ	å& }°cÑ'9Ñ:Ô:ÐØ)×1Ò1°!Ð5GÐI[Ð]`ÑaÔaˆØ)×1Ò1°!°Q¸¸1Ñ=Ô=ˆåœ-×3Ò3ØØ˜iÐ(ØØð	 4ñ 
ô 
ˆð *×1Ò1°!°Q¸¸1Ñ=Ô=×BÒBÀ1ÀbÈ#ÑNÔNˆåŒy˜/¨?Ð;ÀÐCÑCÔCÐCr=   rƒ   Úbool_masked_posc                 óô  — |j         \  }}}}|                      |¦  «        }|                     ¦   «         \  }}}|�R| j                             ||d¦  «        }	|                     d¦  «                             |	¦  «        }
|d|
z
  z  |	|
z  z   }| j                             |dd¦  «        }t          j	        ||fd¬¦  «        }| j
        �||                      |||¦  «        z   }|                      |¦  «        }|S )Nr*   r   r+   )r†   r—   rX   r–   ÚexpandÚ	unsqueezeÚtype_asr”   r/   r©   r�   r°   r(   )rS   rƒ   r±   rh   rŒ   r�   rŽ   rf   rg   Úmask_tokensÚwÚ
cls_tokenss               r;   rl   z InternVLVisionEmbeddings.forwardú   s  € ð
 +Ô0Ñˆˆ1ˆf�eØ×*Ò*¨<Ñ8Ô8ˆ
Ø!+§¢Ñ!2Ô!2Ñˆ
�G˜QàÐ&Øœ/×0Ò0°¸WÀbÑIÔIˆKà×)Ò)¨"Ñ-Ô-×5Ò5°kÑBÔBˆAØ# q¨1¡uÑ-°¸a±Ñ?ˆJà”^×*Ò*¨:°r¸2Ñ>Ô>ˆ
Ý”Y 
¨JÐ7¸QÐ?Ñ?Ô?ˆ
àÔ#Ð/Ø# d×&CÒ&CÀJÐPVÐX]Ñ&^Ô&^Ñ^ˆJà—\’\ *Ñ-Ô-ˆ
àÐr=   rm   )rA   rB   rC   ru   r    rK   r/   rn   Úintr°   Ú
BoolTensorrl   ro   rp   s   @r;   r�   r�   ¶   sÜ   ø€ € € € € ðð ð
>Ð3ð >¸ð >ð >ð >ð >ð >ð >ð,&D°5´<ð &DÈð &DÐUXð &DÐ]bÔ]ið &Dð &Dð &Dð &DðV 48ðð à”lðð Ô)¨DÑ0ðð 
Œð	ð ð ð ð ð ð ð r=   r�   c                   ó   — e Zd ZdS )ÚInternVLVisionMLPNr@   rD   r=   r;   r¼   r¼     rE   r=   r¼   )Ú
layer_normÚrms_normc                   ó†   ‡ — e Zd ZdZdeddfˆ fd„Zdej        deej                 eej        ej        f         z  fd„Z	ˆ xZ
S )ÚInternVLVisionLayerz?This corresponds to the Block class in the timm implementation.rH   r„   Nc                 óˆ  •— t          ¦   «                              ¦   «          |j        | _        d| _        t	          |¦  «        | _        t          |¦  «        | _        t          |j	                 |j
        |j        ¬¦  «        | _        t          |j	                 |j
        |j        ¬¦  «        | _        |j        }t          j        |t#          j        |j
        ¦  «        z  d¬¦  «        | _        t          j        |t#          j        |j
        ¦  «        z  d¬¦  «        | _        t          j        |j        ¦  «        | _        d S )Nr   ©ÚepsT)Úrequires_grad)rJ   rK   Úchunk_size_feed_forwardÚseq_len_dimrG   Ú	attentionr¼   ÚmlpÚNORM2FNÚ	norm_typer~   Úlayer_norm_epsÚlayernorm_beforeÚlayernorm_afterÚlayer_scale_init_valuer2   r’   r/   ÚonesÚlambda_1Úlambda_2rž   rŸ   r(   )rS   rH   Úinit_valuesrU   s      €r;   rK   zInternVLVisionLayer.__init__  s  ø€ Ý‰Œ×ÒÑÔÐØ'-Ô'EˆÔ$ØˆÔÝ0°Ñ8Ô8ˆŒÝ$ VÑ,Ô,ˆŒå '¨Ô(8Ô 9¸&Ô:LÐRXÔRgÐ hÑ hÔ hˆÔÝ& vÔ'7Ô8¸Ô9KÐQWÔQfÐgÑgÔgˆÔàÔ3ˆÝœ [µ5´:¸fÔ>PÑ3QÔ3QÑ%QÐaeÐfÑfÔfˆŒÝœ [µ5´:¸fÔ>PÑ3QÔ3QÑ%QÐaeÐfÑfÔfˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr=   rV   c                 ó$  — |                       |                      |¦  «        ¦  «        \  }}| j        |z  }||z   }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }| j        �
| j        |z  }||z   }|S rm   )rÇ   rÌ   rÐ   rÍ   rÈ   r(   rÑ   )rS   rV   Úattention_outputrh   Úlayer_outputs        r;   rl   zInternVLVisionLayer.forward-  s§   € ð #ŸnšnØ×!Ò! -Ñ0Ô0ñ
ô 
ÑÐ˜!ð  œ=Ð+;Ñ;Ðð )¨=Ñ8ˆð ×+Ò+¨MÑ:Ô:ˆà—x’x Ñ-Ô-ˆØ—|’| LÑ1Ô1ˆàŒ=Ð$Øœ=¨<Ñ7ˆLð $ mÑ3ˆàÐr=   )rA   rB   rC   ru   r    rK   r/   rn   Útuplerl   ro   rp   s   @r;   rÀ   rÀ     s”   ø€ € € € € ØIÐIð>Ð3ð >¸ð >ð >ð >ð >ð >ð >ðà”|ðð 
ˆuŒ|Ô	˜u U¤\°5´<Ð%?Ô@Ñ	@ðð ð ð ð ð ð ð r=   rÀ   c                   óH   ‡ — e Zd Zdeddfˆ fd„Zdej        deez  fd„Z	ˆ xZ
S )ÚInternVLVisionEncoderrH   r„   Nc                 óÔ   •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS rD   )rÀ   )Ú.0ÚirH   s     €r;   ú
<listcomp>z2InternVLVisionEncoder.__init__.<locals>.<listcomp>M  s"   ø€ Ð#iÐ#iÐ#iÀAÕ$7¸Ñ$?Ô$?Ð#iÐ#iÐ#ir=   F)	rJ   rK   rH   r2   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayerÚgradient_checkpointing©rS   rH   rU   s    `€r;   rK   zInternVLVisionEncoder.__init__J  s`   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”]Ð#iÐ#iÐ#iÐ#iÍÈvÔOgÑIhÔIhÐ#iÑ#iÔ#iÑjÔjˆŒ
Ø&+ˆÔ#Ð#Ð#r=   rV   c                 óL   — | j         D ]} ||¦  «        }Œt          |¬¦  «        S )N)Úlast_hidden_state)rá   r
   )rS   rV   Úlayer_modules      r;   rl   zInternVLVisionEncoder.forwardP  s@   € ð !œJð 	8ð 	8ˆLØ(˜L¨Ñ7Ô7ˆMˆMåØ+ð
ñ 
ô 
ð 	
r=   )rA   rB   rC   r    rK   r/   rn   rÖ   r
   rl   ro   rp   s   @r;   rØ   rØ   I  sy   ø€ € € € € ð,Ð3ð ,¸ð ,ð ,ð ,ð ,ð ,ð ,ð	
à”|ð	
ð 
�Ñ	 ð	
ð 	
ð 	
ð 	
ð 	
ð 	
ð 	
ð 	
r=   rØ   c                   ó„   ‡ — e Zd ZU eed<   dZdZdZdZdgZ	dZ
dZdZdZeedœZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )	ÚInternVLVisionPreTrainedModelrH   Úinternvl_visionrƒ   )ÚimageÚvideoTrÀ   )rV   Ú
attentionsc                 óì  •— t          ¦   «                              |¦  «         t          |t          ¦  «        r]t	          j        |j        ¦  «         |j        �t	          j        |j        ¦  «         |j        �t	          j        |j        ¦  «         dS dS t          |t          ¦  «        rJt	          j
        |j        | j        j        ¦  «         t	          j
        |j        | j        j        ¦  «         dS dS )zInitialize the weightsN)rJ   Ú_init_weightsr˜   r�   ÚinitÚzeros_r”   r–   r�   rÀ   Ú	constant_rÐ   rH   rÎ   rÑ   )rS   r"   rU   s     €r;   rî   z+InternVLVisionPreTrainedModel._init_weightsn  sâ   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ6Ñ7Ô7ð 	PÝŒK˜Ô(Ñ)Ô)Ð)ØÔ Ð,Ý”˜FÔ-Ñ.Ô.Ð.ØÔ)Ð5Ý”˜FÔ6Ñ7Ô7Ð7Ð7Ð7ð 6Ð5å˜Õ 3Ñ4Ô4ð 	PÝŒN˜6œ?¨D¬KÔ,NÑOÔOÐOÝŒN˜6œ?¨D¬KÔ,NÑOÔOÐOÐOÐOð	Pð 	Pr=   )rA   rB   rC   r    Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_attention_backendrÀ   rG   Ú_can_record_outputsr/   Úno_gradrî   ro   rp   s   @r;   rè   rè   \  s¯   ø€ € € € € € à Ð Ð Ñ Ø)ÐØ$€OØ)ÐØ&*Ð#Ø.Ð/ÐØ€NØÐØÐØ"&Ðð -Ø-ðð Ðð
 €U„]�_„_ðPð Pð Pð Pñ „_ðPð Pð Pð Pð Pr=   rè   c                   óª   ‡ — e Zd Zdeddfˆ fd„Zd„ Ze ed¬¦  «        e	 dde	j
        d	e	j        dz  deez  fd
„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚInternVLVisionModelrH   r„   Nc                 óN  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        |j        rt          j	        ¦   «         nt          j
        |j        |j        ¬¦  «        | _        |                      ¦   «          d S )NrÂ   )rJ   rK   rH   r�   rŽ   rØ   ÚencoderÚuse_mean_poolingr2   rP   Ú	LayerNormr~   rË   Ú	layernormÚ	post_initrã   s     €r;   rK   zInternVLVisionModel.__init__  sŒ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒå2°6Ñ:Ô:ˆŒÝ,¨VÑ4Ô4ˆŒð $Ô4Ðu�BŒK‰MŒMˆM½"¼,ÀvÔGYÐ_eÔ_tÐ:uÑ:uÔ:uð 	Œð
 	�ŠÑÔÐÐÐr=   c                 ó   — | j         j        S rm   )rŽ   r—   )rS   s    r;   Úget_input_embeddingsz(InternVLVisionModel.get_input_embeddings�  s   € ØŒÔ/Ð/r=   F)Útie_last_hidden_statesrƒ   r±   c                 óÌ   — |                       ||¬¦  «        }|                      |¦  «        }|d         }|                      |¦  «        }t          ||j        |j        ¬¦  «        S )zË
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        )r±   r   )rå   rV   rì   )rŽ   r  r  rt   rV   rì   )rS   rƒ   r±   r6   Úembedding_outputÚencoder_outputsÚsequence_outputs          r;   rl   zInternVLVisionModel.forward�  sm   € ð  Ÿ?š?¨<È˜?ÑYÔYÐàŸ,š,Ð'7Ñ8Ô8ˆØ)¨!Ô,ˆØŸ.š.¨Ñ9Ô9ˆå3Ø-Ø)Ô7Ø&Ô1ð
ñ 
ô 
ð 	
r=   rm   )rA   rB   rC   r    rK   r  r   r   r   r/   rn   rº   rÖ   rt   rl   ro   rp   s   @r;   rÿ   rÿ   }  sÎ   ø€ € € € € ðÐ3ð ¸ð ð ð ð ð ð ð0ð 0ð 0ð  Ø€_¨EÐ2Ñ2Ô2ØàUYð
ð 
Ø!œLð
Ø;@Ô;KÈdÑ;Rð
à	Ð5Ñ	5ð
ð 
ð 
ñ „^ñ 3Ô2ñ  Ôð
ð 
ð 
ð 
ð 
r=   rÿ   c                   ó   — e Zd ZdZdS )ÚInternVLPreTrainedModel)rê   Útextrë   N)rA   rB   rC   rõ   rD   r=   r;   r  r  §  s   € € € € € Ø1ÐÐÐr=   r  c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚInternVLMultiModalProjectorrH   c                 óà  •— t          ¦   «                              ¦   «          t          j        |j        j        t          d|j        z  ¦  «        dz  z  ¦  «        | _        t          j	        |j        j        t          d|j        z  ¦  «        dz  z  |j
        j        ¦  «        | _        t          |j                 | _        t          j	        |j
        j        |j
        j        ¦  «        | _        d S )Nr   r   )rJ   rK   r2   r  Úvision_configr~   r¹   Údownsample_ratior½   ÚLinearÚtext_configÚlinear_1r   Úprojector_hidden_actÚactÚlinear_2rã   s     €r;   rK   z$InternVLMultiModalProjector.__init__¯  sÀ   ø€ Ý‰Œ×ÒÑÔÐÝœ, vÔ';Ô'GÍ#ÈaÐRXÔRiÑNiÑJjÔJjÐnoÑJoÑ'oÑpÔpˆŒÝœ	ØÔ Ô,­s°1°vÔ7NÑ3NÑ/OÔ/OÐSTÑ/TÑTÐV\ÔVhÔVtñ
ô 
ˆŒõ ˜&Ô5Ô6ˆŒÝœ	 &Ô"4Ô"@À&ÔBTÔB`ÑaÔaˆŒˆˆr=   c                 ó®   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rm   )r½   r  r  r  )rS   Úimage_featuresrV   s      r;   rl   z#InternVLMultiModalProjector.forward¸  sL   € ØŸš¨Ñ7Ô7ˆØŸš mÑ4Ô4ˆØŸš Ñ/Ô/ˆØŸš mÑ4Ô4ˆØÐr=   )rA   rB   rC   r   rK   rl   ro   rp   s   @r;   r  r  ®  sZ   ø€ € € € € ðb˜~ð bð bð bð bð bð bðð ð ð ð ð ð r=   r  c                   ó   — e Zd ZdS )ÚInternVLModelOutputWithPastNr@   rD   r=   r;   r  r  À  rE   r=   r  c                   óÖ  — e Zd Zddej        defd„Zee e	d¬¦  «        	 	 ddej
        d	eee         z  ee         z  dz  d
edz  dee         deez  f
d„¦   «         ¦   «         ¦   «         Zee		 	 	 	 	 	 	 	 ddej        dz  dej
        dz  dej        dz  dej        dz  dedz  dej
        dz  d	eee         z  ee         z  dz  d
edz  dee         deez  fd„¦   «         ¦   «         ZdS )ÚInternVLModelr¡   Úvision_featuresÚscale_factorc           
      ó(  — |                      ¦   «         \  }}}}||z  dk    s	||z  dk    rt          d¦  «        ‚|                     ||t          ||z  ¦  «        t          ||z  ¦  «        ¦  «        }|                     dddd¦  «                             ¦   «         }|                     |t          ||z  ¦  «        t          ||z  ¦  «        t          ||dz  z  ¦  «        ¦  «        }|                     dddd¦  «                             ¦   «         }|S )a&  Perform pixel shuffle downsampling on vision features.

        Args:
            vision_features (`torch.Tensor`):
                Input tensor of shape (batch_size, width, height, channels).
            scale_factor (`float`, *optional*, defaults to `0.5`):
                Factor by which to downsample. Default is 0.5, which halves the dimensions.

        Returns:
            vision_features (`torch.Tensor`):
                Downsampled tensor of shape (batch_size, height*scale_factor, width*scale_factor, channels/(scale_factor^2)).
        r   zKHeight and width must be divisible by scale_factor for proper downsampling.r   r   r   )rX   r‡   r_   r¹   r§   r5   )rS   r!  r"  rf   r�   rŒ   Úchannelss          r;   Úpixel_shufflezInternVLModel.pixel_shuffleÅ  s)  € ð />×.BÒ.BÑ.DÔ.DÑ+ˆ
�E˜6 8à�LÑ  AÒ%Ð%¨°Ñ)=ÀÒ)BÐ)BÝÐjÑkÔkÐkð *×.Ò.Ø˜�s 6¨LÑ#8Ñ9Ô9½3¸xÈ,Ñ?VÑ;WÔ;Wñ
ô 
ˆð *×1Ò1°!°Q¸¸1Ñ=Ô=×HÒHÑJÔJˆð *×.Ò.Ø�˜F \Ñ1Ñ2Ô2µC¸ÀÑ8LÑ4MÔ4MÍsÐS[Ð_kÐmnÑ_nÑSoÑOpÔOpñ
ô 
ˆð
 *×1Ò1°!°Q¸¸1Ñ=Ô=×HÒHÑJÔJˆàÐr=   zWObtains image last hidden states from the vision tower and apply multimodal projection.rq   Nrƒ   Úvision_feature_layerÚvision_feature_select_strategyr6   r„   c                 ó   — |                      | j        ¬¦  «        }| j        j        }|dk    rd|d<    | j        d|ddœ|¤Ž}|dk    r|j        }n|j        |         }|dk    r|dd…dd…dd…f         }|j        d         }t          |d	z  ¦  «        }	|j        d
         }
| 	                    |
|	|	d¦  «        }|  
                    ||¬¦  «        }| 	                    |
d|j        d         ¦  «        }|                      |¦  «        }||_        |S )a!  
        pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`)
            The tensors corresponding to the input images.
        vision_feature_layer (`int` or `list[int]`):
            Layer index or list of layer indices to extract features from.
        )rŠ   r*   TÚoutput_hidden_states)rƒ   Úreturn_dictÚdefaultNr   r¡   r   )r"  rD   )rˆ   rŠ   rH   r  Úvision_towerrå   rV   r†   r¹   r\   r%  Úmulti_modal_projectorÚpooler_output)rS   rƒ   r&  r'  r6   r  Úvision_outputsr!  r$  Úfeature_sizerf   s              r;   Úget_image_featuresz InternVLModel.get_image_featuresè  sN  € ð$ $—’¨T¬Z�Ñ8Ô8ˆàœ;Ô7ÐØ 2Ò%Ð%Ø-1ˆFÐ)Ñ*Ø*˜Ô*Ða¸ÐRVÐaÐaÐZ`ÐaÐaˆØ 2Ò%Ð%Ø,Ô>ˆOˆOà,Ô:Ð;OÔPˆOØ)¨YÒ6Ð6Ø-¨a¨a¨a°°°°Q°Q°Q¨hÔ7ˆOð #Ô(¨Ô+ˆÝ˜8 S™=Ñ)Ô)ˆØ$Ô*¨1Ô-ˆ
ð *×1Ò1°*¸lÈLÐZ\Ñ]Ô]ˆð ×,Ò,¨_ÐK[Ð,Ñ\Ô\ˆð *×1Ò1°*¸bÀ/ÔBWÐXZÔB[Ñ\Ô\ˆð ×4Ò4°_ÑEÔEˆØ'6ˆÔ$àÐr=   Ú	input_idsr&   Úposition_idsÚpast_key_valuesÚinputs_embedsc	                 óÈ  — |d u |d uz  rt          d¦  «        ‚|€ |                      ¦   «         |¦  «        }|�l|                      |||d¬¦  «        j        }
|
                     |j        |j        ¦  «        }
|                      |||
¬¦  «        }|                     ||
¦  «        } | j	        d||||dœ|	¤Ž}t          |j        |j        |j        |j        |�|
nd ¬¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embedsT)rƒ   r&  r'  r*  )r5  r  )r&   r3  r4  r5  )rå   r4  rV   rì   Úimage_hidden_statesrD   )r‡   r  r1  r.  rˆ   ÚdevicerŠ   Úget_placeholder_maskÚmasked_scatterÚlanguage_modelr  rå   r4  rV   rì   )rS   r2  rƒ   r&   r3  r4  r5  r&  r'  r6   r  Úspecial_image_maskÚoutputss                r;   rl   zInternVLModel.forward  sI  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø7˜D×5Ò5Ñ7Ô7¸	ÑBÔBˆMàÐ#Ø!×4Ò4Ø)Ø%9Ø/MØ ð	 5ñ ô ô
 ð ð ,×.Ò.¨}Ô/CÀ]ÔEXÑYÔYˆNØ!%×!:Ò!:Ø¨À~ð ";ñ "ô "Ðð *×8Ò8Ð9KÈ^Ñ\Ô\ˆMà%�$Ô%ð 
Ø)Ø%Ø+Ø'ð	
ð 
ð
 ð
ð 
ˆõ +Ø%Ô7Ø#Ô3Ø!Ô/ØÔ)Ø2>Ð2J  ÐPTð
ñ 
ô 
ð 	
r=   )r¡   )NN)NNNNNNNN)rA   rB   rC   r/   rn   Úfloatr%  r   r   r   ÚFloatTensorr¹   ÚlistÚstrr   r   rÖ   r   r1  Ú
LongTensorr   r  rl   rD   r=   r;   r   r   Ä  sñ  € € € € € ð!ð !¨U¬\ð !Èð !ð !ð !ð !ðF  ØØ€^Ønðñ ô ð DHØ59ð	,ð ,àÔ'ð,ð " D¨¤I™o°°S´	Ñ9¸DÑ@ð,ð ),¨d©
ð	,ð
 Ð+Ô,ð,ð 
Ð+Ñ	+ð,ð ,ð ,ñô ñ Ôñ  Ôð
,ð\ Øð .2Ø15Ø.2Ø04Ø(,Ø26ØCGØ59ð-
ð -
àÔ# dÑ*ð-
ð Ô'¨$Ñ.ð-
ð œ tÑ+ð	-
ð
 Ô&¨Ñ-ð-
ð  ™ð-
ð Ô(¨4Ñ/ð-
ð " D¨¤I™o°°S´	Ñ9¸DÑ@ð-
ð ),¨d©
ð-
ð Ð+Ô,ð-
ð 
Ð,Ñ	,ð-
ð -
ð -
ñ „^ñ Ôð-
ð -
ð -
r=   r   c                   ó   — e Zd ZdS )ÚInternVLCausalLMOutputWithPastNr@   rD   r=   r;   rD  rD  M  rE   r=   rD  c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )Ú InternVLForConditionalGenerationc                  ó:   •—  t          ¦   «         j        di | ¤Ž dS )ac  
        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoProcessor, AutoModelForImageTextToText

        >>> torch_device = "cuda"
        >>> processor = AutoProcessor.from_pretrained("OpenGVLab/InternVL3-1B-hf")
        >>> model = AutoModelForImageTextToText.from_pretrained(
        ...     "OpenGVLab/InternVL3-1B-hf", dtype=torch.bfloat16, device_map=torch_device
        ... )

        >>> messages = [
        ...     {
        ...         "role": "user",
        ...         "content": [
        ...             {
        ...                 "type": "image",
        ...                 "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg",
        ...             },
        ...             {
        ...                 "type": "image",
        ...                 "url": "https://thumbs.dreamstime.com/b/golden-gate-bridge-san-francisco-purple-flowers-california-echium-candicans-36805947.jpg",
        ...             },
        ...             {"type": "text", "text": "These images depict two different landmarks. Can you identify them?"},
        ...         ],
        ...     },
        ... ]

        >>> inputs = processor.apply_chat_template(messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt").to(torch_device)
        >>> generate_ids = model.generate(**inputs, max_new_tokens=200)
        >>> print(processor.decode(generate_ids[0, inputs["input_ids"].shape[1] :], skip_special_tokens=True))
        The images depict the Statue of Liberty and the Golden Gate Bridge.
        ```NrD   )rJ   rl   )Úsuper_kwargsrU   s    €r;   rl   z(InternVLForConditionalGeneration.forwardR  s(   ø€ ðH 	�‰ŒŒÐ'Ð'˜,Ð'Ð'Ð'Ð'Ð'r=   )rA   rB   rC   rl   ro   rp   s   @r;   rF  rF  Q  s8   ø€ € € € € ð$(ð $(ð $(ð $(ð $(ð $(ð $(ð $(ð $(r=   rF  )rè   rÿ   r  r   rF  )r!   )JÚcollections.abcr™   r   Údataclassesr   r/   Útorch.nnr2   Ú r   rï   Úactivationsr   Úcache_utilsr   Úmodeling_layersr	   Úmodeling_outputsr
   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   r   Úutils.output_capturingr   Úclip.modeling_clipr   Újanus.modeling_janusr   Úllama.modeling_llamar   Úllava.modeling_llavar   r   r   r   r   Úconfiguration_internvlr   r    ÚModulern   r>  r¹   r<   r?   rG   rt   rw   r�   r¼   r  rÉ   rÀ   rØ   rè   rÿ   r  ÚINTERNVL_INPUTS_DOCSTRINGr  r  r   rD  rF  Ú__all__rD   r=   r;   ú<module>r^     sv  ðð  Ð Ð Ð Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø  Ð  Ð  Ð  Ð  Ð  Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BØ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø (Ð (Ð (Ð (Ð (Ð (Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø /Ð /Ð /Ð /Ð /Ð /ðð ð ð ð ð ð ð ð ð ð ð ð ð ð IÐ HÐ HÐ HÐ HÐ HÐ HÐ Hð ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð �S‰[ð%ð %ð %ð %ð4	ð 	ð 	ð 	ð 	˜Lñ 	ô 	ð 	ð3$ð 3$ð 3$ð 3$ð 3$Ð2ñ 3$ô 3$ð 3$ðl €ððñ ô ð
 ðð ð ð ð Ð+Eñ ô ñ „ñô ðð ð  ð  ð  ð   B¤Iñ  ô  ð  ðJ[ð [ð [ð [ð [˜rœyñ [ô [ð [ð|	ð 	ð 	ð 	ð 	˜ñ 	ô 	ð 	ð œÐ3HÐ
IÐ
I€ð+ð +ð +ð +ð +Ð4ñ +ô +ð +ð\
ð 
ð 
ð 
ð 
˜BœIñ 
ô 
ð 
ð& ðPð Pð Pð Pð P Oñ Pô Pñ „ðPð@ ð&
ð &
ð &
ð &
ð &
Ð7ñ &
ô &
ñ „ð&
ðR2ð 2ð 2ð 2ð 2Ð2ñ 2ô 2ð 2ð !Ð ðð ð ð ð  "¤)ñ ô ð ð$	ð 	ð 	ð 	ð 	Ð":ñ 	ô 	ð 	ðF
ð F
ð F
ð F
ð F
�Jñ F
ô F
ð F
ðR	ð 	ð 	ð 	ð 	Ð%@ñ 	ô 	ð 	ð%(ð %(ð %(ð %(ð %(Ð'Dñ %(ô %(ð %(ðPð ð €€€r=   