§
    ‚Štj\  ã                   ó(  — d Z ddlZddlmZmZ ddlZddlmZ ddlmZ	 ddl
mZ ddlmZ dd	lmZ dd
lmZmZmZ ddlmZmZ ddlmZ ddlmZmZmZmZ ddlmZm Z  ddl!m"Z" ddl#m$Z$  ej%        e&¦  «        Z' G d„ dej(        ¦  «        Z) G d„ dej(        ¦  «        Z*	 	 d4dej(        dej+        dej+        dej+        dej+        dz  de,dz  de,dee         fd„Z- G d „ d!ej(        ¦  «        Z. G d"„ d#ej(        ¦  «        Z/ G d$„ d%e¦  «        Z0 G d&„ d'ej(        ¦  «        Z1e G d(„ d)e¦  «        ¦   «         Z2e G d*„ d+e2¦  «        ¦   «         Z3 ed,¬-¦  «         G d.„ d/e2¦  «        ¦   «         Z4 ed0¬-¦  «         G d1„ d2e2¦  «        ¦   «         Z5g d3¢Z6dS )5zPyTorch ViT model.é    N)ÚCallableÚIterable)Únné   )Úinitialization)ÚACT2FN)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPoolingÚImageClassifierOutputÚMaskedImageModelingOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚloggingÚ	torch_int)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputsé   )Ú	ViTConfigc                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚViTPatchEmbeddingszì
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    Úconfigc                 ó¢  •— t          ¦   «                              ¦   «          |j        }|j        }t	          |t
          ¦  «        r|n||f}t	          |t
          ¦  «        r|n||f}|d         |d         z  |d         |d         z  z  | _        || _        || _        |j        | _        t          j	        |j        |j
        ||¬¦  «        | _        d S )Nr   r   )Úkernel_sizeÚstride)ÚsuperÚ__init__Ú
image_sizeÚ
patch_sizeÚ
isinstancer   Únum_patchesÚnum_channelsr   ÚConv2dÚhidden_sizeÚ
projection)Úselfr   r"   r#   Ú	__class__s       €úb/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/vit/modeling_vit.pyr!   zViTPatchEmbeddings.__init__1   sÎ   ø€ Ý‰Œ×ÒÑÔÐØÔ&ˆ
ØÔ&ˆ
Ý#-¨j½(Ñ#CÔ#CÐa�Z�ZÈ*ÐV`ÐIaˆ
Ý#-¨j½(Ñ#CÔ#CÐa�Z�ZÈ*ÐV`ÐIaˆ
à& qœM¨Z¸¬]Ñ:¸zÈ!¼}ÐPZÐ[\ÔP]Ñ?]Ñ^ˆÔØ$ˆŒØ$ˆŒØ"Ô/ˆÔÝœ) FÔ$7¸Ô9KÐYcÐlvÐwÑwÔwˆŒˆˆó    Úpixel_valuesÚreturnc                 óà   — |j         d         }|| j        k    rt          d| j        › d|› d�¦  «        ‚|                      |¦  «                             d¦  «                             dd¦  «        S )Nr   zoMake sure that the channel dimension of the pixel values match with the one set in the configuration. Expected z	 but got ú.é   )Úshaper&   Ú
ValueErrorr)   ÚflattenÚ	transpose)r*   r.   r&   s      r,   ÚforwardzViTPatchEmbeddings.forward>   sŽ   € Ø#Ô)¨!Ô,ˆØ˜4Ô,Ò,Ð,ÝðIØ!Ô.ðIð IØ9EðIð Ið Iñô ð ð �Š˜|Ñ,Ô,×4Ò4°QÑ7Ô7×AÒAÀ!ÀQÑGÔGÐGr-   )
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r!   ÚtorchÚTensorr7   Ú__classcell__©r+   s   @r,   r   r   *   s…   ø€ € € € € ðð ðx˜yð xð xð xð xð xð xðH E¤Lð H°U´\ð Hð Hð Hð Hð Hð Hð Hð Hr-   r   c            	       óœ   ‡ — e Zd ZdZddedefˆ fd„Zdej        de	de	d	ej        fd
„Z
	 	 ddej        dej        dz  ded	ej        fd„Zˆ xZS )ÚViTEmbeddingszb
    Construct the CLS token, position and patch embeddings. Optionally, also the mask token.
    Fr   Úuse_mask_tokenc                 ó8  •— t          ¦   «                              ¦   «          t          j        t	          j        dd|j        ¦  «        ¦  «        | _        |r-t          j        t	          j        dd|j        ¦  «        ¦  «        nd | _	        t          |¦  «        | _        | j        j        }t          j        t	          j        d|dz   |j        ¦  «        ¦  «        | _        t          j        |j        ¦  «        | _        |j        | _        | j        j        | _        d S )Nr   )r    r!   r   Ú	Parameterr<   Úrandnr(   Ú	cls_tokenÚzerosÚ
mask_tokenr   Úpatch_embeddingsr%   Úposition_embeddingsÚDropoutÚhidden_dropout_probÚdropoutr#   r"   )r*   r   rB   r%   r+   s       €r,   r!   zViTEmbeddings.__init__M   sÚ   ø€ Ý‰Œ×ÒÑÔÐåœ¥e¤k°!°Q¸Ô8JÑ&KÔ&KÑLÔLˆŒØQ_Ði�"œ,¥u¤{°1°a¸Ô9KÑ'LÔ'LÑMÔMÐMÐeiˆŒÝ 2°6Ñ :Ô :ˆÔØÔ+Ô7ˆÝ#%¤<µ´¸A¸{ÈQ¹ÐPVÔPbÑ0cÔ0cÑ#dÔ#dˆÔ Ý”z &Ô"<Ñ=Ô=ˆŒØ Ô+ˆŒØÔ/Ô:ˆŒˆˆr-   Ú
embeddingsÚheightÚwidthr/   c                 ó”  — |j         d         dz
  }| j        j         d         dz
  }t          j                             ¦   «         s||k    r||k    r| j        S | j        dd…dd…f         }| j        dd…dd…f         }|j         d         }|| j        z  }	|| j        z  }
t          |dz  ¦  «        }|                     d|||¦  «        }|                     dddd¦  «        }t          j
                             ||	|
fdd	¬
¦  «        }|                     dddd¦  «                             dd|¦  «        }t          j        ||fd¬¦  «        S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   Néÿÿÿÿç      à?r   r   r2   ÚbicubicF)ÚsizeÚmodeÚalign_corners©Údim)r3   rJ   r<   ÚjitÚ
is_tracingr#   r   ÚreshapeÚpermuter   Ú
functionalÚinterpolateÚviewÚcat)r*   rN   rO   rP   r%   Únum_positionsÚclass_pos_embedÚpatch_pos_embedrY   Ú
new_heightÚ	new_widthÚsqrt_num_positionss               r,   Úinterpolate_pos_encodingz&ViTEmbeddings.interpolate_pos_encodingY   sr  € ð !Ô& qÔ)¨AÑ-ˆØÔ0Ô6°qÔ9¸AÑ=ˆõ Œy×#Ò#Ñ%Ô%ð 	,¨+¸Ò*FÐ*FÈ6ÐUZÊ?È?ØÔ+Ð+àÔ2°1°1°1°b°q°b°5Ô9ˆØÔ2°1°1°1°a°b°b°5Ô9ˆàÔ˜rÔ"ˆà˜tœÑ.ˆ
Ø˜Tœ_Ñ,ˆ	å& }°cÑ'9Ñ:Ô:ÐØ)×1Ò1°!Ð5GÐI[Ð]`ÑaÔaˆØ)×1Ò1°!°Q¸¸1Ñ=Ô=ˆåœ-×3Ò3ØØ˜iÐ(ØØð	 4ñ 
ô 
ˆð *×1Ò1°!°Q¸¸1Ñ=Ô=×BÒBÀ1ÀbÈ#ÑNÔNˆåŒy˜/¨?Ð;ÀÐCÑCÔCÐCr-   Nr.   Úbool_masked_posrh   c                 ó’  — |j         \  }}}}|                      |¦  «        }|�_|j         d         }	| j                             ||	d¦  «        }
|                     d¦  «                             |
¦  «        }|d|z
  z  |
|z  z   }| j                             |dd¦  «        }t          j        ||fd¬¦  «        }|r||  	                    |||¦  «        z   }n^|| j
        d         k    s|| j
        d         k    r2t          d|› d|› d| j
        d         › d| j
        d         › d	�	¦  «        ‚|| j        z   }|                      |¦  «        }|S )
Nr   rR   g      ð?rX   r   zInput image size (Ú*z) doesn't match model (z).)r3   rI   rH   ÚexpandÚ	unsqueezeÚtype_asrF   r<   ra   rh   r"   r4   rJ   rM   )r*   r.   ri   rh   Ú
batch_sizer&   rO   rP   rN   Ú
seq_lengthÚmask_tokensÚmaskÚ
cls_tokenss                r,   r7   zViTEmbeddings.forward�   s˜  € ð 3?Ô2DÑ/ˆ
�L &¨%Ø×*Ò*¨<Ñ8Ô8ˆ
àÐ&Ø#Ô)¨!Ô,ˆJØœ/×0Ò0°¸ZÈÑLÔLˆKà"×,Ò,¨RÑ0Ô0×8Ò8¸ÑEÔEˆDØ# s¨T¡zÑ2°[À4Ñ5GÑGˆJð ”^×*Ò*¨:°r¸2Ñ>Ô>ˆ
Ý”Y 
¨JÐ7¸QÐ?Ñ?Ô?ˆ
à#ð 	?Ø# d×&CÒ&CÀJÐPVÐX]Ñ&^Ô&^Ñ^ˆJˆJà˜œ¨Ô+Ò+Ð+¨u¸¼ÈÔ8JÒ/JÐ/JÝ ðE¨ð Eð E°%ð Eð EØœ¨Ô+ðEð EØ.2¬o¸aÔ.@ðEð Eð Eñô ð ð $ dÔ&>Ñ>ˆJà—\’\ *Ñ-Ô-ˆ
àÐr-   )F)NF)r8   r9   r:   r;   r   Úboolr!   r<   r=   Úintrh   Ú
BoolTensorr7   r>   r?   s   @r,   rA   rA   H   sí   ø€ € € € € ðð ð
;ð 
;˜yð 
;¸$ð 
;ð 
;ð 
;ð 
;ð 
;ð 
;ð&D°5´<ð &DÈð &DÐUXð &DÐ]bÔ]ið &Dð &Dð &Dð &DðV 48Ø).ð	 ð  à”lð ð Ô)¨DÑ0ð ð #'ð	 ð
 
Œð ð  ð  ð  ð  ð  ð  ð  r-   rA   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingrM   Úkwargsc                 óô  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |dt          j        ¬¦  «                             |j	        ¦  «        }t          j         
                    ||| j        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «                             ¦   «         }	|	|fS )NrR   ç      à¿r2   r   )rY   Údtype)ÚpÚtrainingr   )rU   r<   Úmatmulr6   r   r^   ÚsoftmaxÚfloat32Útor�   rM   rƒ   Ú
contiguous)
rx   ry   rz   r{   r|   r}   rM   r~   Úattn_weightsÚattn_outputs
             r,   Úeager_attention_forwardr‹   ¤   sÞ   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r-   c                   óŠ   ‡ — e Zd Zdefˆ fd„Z	 d	dej        dej        dz  dee         de	ej        ej        f         fd„Z
ˆ xZS )
ÚViTAttentionr   c                 ó†  •— t          ¦   «                              ¦   «          || _        |j        | _        t	          |d|j        |j        z  ¦  «        | _        |j        | _        | j        dz  | _	        d| _
        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        | j        z  |j        d¬¦  «        | _        d S )NÚhead_dimr€   F)ÚbiasT)r    r!   r   Únum_attention_headsÚgetattrr(   r�   Úattention_probs_dropout_probÚattention_dropoutr}   Ú	is_causalr   ÚLinearÚqkv_biasÚq_projÚk_projÚv_projÚo_proj©r*   r   r+   s     €r,   r!   zViTAttention.__init__Á   s  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ#)Ô#=ˆÔ Ý ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ!'Ô!DˆÔØ”} dÑ*ˆŒØˆŒå”i Ô 2°FÔ4NÐQUÔQ^Ñ4^ÐekÔetÐuÑuÔuˆŒÝ”i Ô 2°FÔ4NÐQUÔQ^Ñ4^ÐekÔetÐuÑuÔuˆŒÝ”i Ô 2°FÔ4NÐQUÔQ^Ñ4^ÐekÔetÐuÑuÔuˆŒÝ”i Ô :¸T¼]Ñ JÈFÔL^ÐeiÐjÑjÔjˆŒˆˆr-   NÚhidden_statesr|   r~   r/   c                 ó¬  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }t          j        | j	        j
        t          ¦  «        }	 |	| ||||f| j        sdn| j        | j        dœ|¤Ž\  }
} |
j        g |¢d‘R Ž                      ¦   «         }
|                      |
¦  «        }
|
|fS )NrR   r   r2   rw   )rM   r}   )r3   r�   r˜   r`   r6   r™   rš   r   Úget_interfacer   Ú_attn_implementationr‹   rƒ   r”   r}   r\   rˆ   r›   )r*   r�   r|   r~   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚattention_interfacerŠ   r‰   s               r,   r7   zViTAttention.forwardÏ   sw  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆà˜LÐ(Ð(r-   ©N)r8   r9   r:   r   r!   r<   r=   r   r   Útupler7   r>   r?   s   @r,   r�   r�   À   s¬   ø€ € € € € ðk˜yð kð kð kð kð kð kð" /3ð)ð )à”|ð)ð œ tÑ+ð)ð Ð+Ô,ð	)ð
 
ˆuŒ|˜Uœ\Ð)Ô	*ð)ð )ð )ð )ð )ð )ð )ð )r-   r�   c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚViTMLPr   c                 ó  •— t          ¦   «                              ¦   «          || _        t          |j                 | _        t          j        |j        |j	        ¦  «        | _
        t          j        |j	        |j        ¦  «        | _        d S r§   )r    r!   r   r   Ú
hidden_actÚactivation_fnr   r–   r(   Úintermediate_sizeÚfc1Úfc2rœ   s     €r,   r!   zViTMLP.__init__ò   sf   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ# FÔ$5Ô6ˆÔÝ”9˜VÔ/°Ô1IÑJÔJˆŒÝ”9˜VÔ5°vÔ7IÑJÔJˆŒˆˆr-   r�   r/   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r§   )r¯   r­   r°   )r*   r�   s     r,   r7   zViTMLP.forwardù   s=   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆàÐr-   ©	r8   r9   r:   r   r!   r<   r=   r7   r>   r?   s   @r,   rª   rª   ñ   sq   ø€ € € € € ðK˜yð Kð Kð Kð Kð Kð Kð U¤\ð °e´lð ð ð ð ð ð ð ð r-   rª   c            	       óp   ‡ — e Zd Zdefˆ fd„Z	 d	dej        dej        dz  dee         dej        fd„Z	ˆ xZ
S )
ÚViTLayerr   c                 óh  •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          j        |j        |j        ¬¦  «        | _        t	          j        |j        |j        ¬¦  «        | _	        t          |¦  «        | _        t	          j        |j        ¦  «        | _        d S )N©Úeps)r    r!   r�   Ú	attentionr   Ú	LayerNormr(   Úlayer_norm_epsÚlayernorm_beforeÚlayernorm_afterrª   ÚmlprK   rL   rM   rœ   s     €r,   r!   zViTLayer.__init__  s‡   ø€ Ý‰Œ×ÒÑÔÐÝ% fÑ-Ô-ˆŒÝ "¤¨VÔ-?ÀVÔEZÐ [Ñ [Ô [ˆÔÝ!œ|¨FÔ,>ÀFÔDYÐZÑZÔZˆÔÝ˜&‘>”>ˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr-   Nr�   r|   r~   r/   c                 ó  — |}|                       |¦  «        } | j        ||fi |¤Ž\  }}|                      |¦  «        }||z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z   }|S r§   )r»   r¸   rM   r¼   r½   )r*   r�   r|   r~   ÚresidualÚ_s         r,   r7   zViTLayer.forward
  s£   € ð !ˆØ×-Ò-¨mÑ<Ô<ˆØ)˜4œ>¨-¸ÐRÐRÈ6ÐRÐRÑˆ�qØŸš ]Ñ3Ô3ˆØ%¨Ñ0ˆð !ˆØ×,Ò,¨]Ñ;Ô;ˆØŸš Ñ/Ô/ˆØŸš ]Ñ3Ô3ˆØ%¨Ñ0ˆàÐr-   r§   )r8   r9   r:   r   r!   r<   r=   r   r   r7   r>   r?   s   @r,   r´   r´     s™   ø€ € € € € ð>˜yð >ð >ð >ð >ð >ð >ð /3ðð à”|ðð œ tÑ+ðð Ð+Ô,ð	ð
 
Œðð ð ð ð ð ð ð r-   r´   c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )Ú	ViTPoolerr   c                 ó¾   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j                 | _	        d S r§   )
r    r!   r   r–   r(   Úpooler_output_sizeÚdenser   Ú
pooler_actÚ
activationrœ   s     €r,   r!   zViTPooler.__init__"  sE   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3LÑMÔMˆŒ
Ý  Ô!2Ô3ˆŒˆˆr-   r�   r/   c                 ór   — |d d …df         }|                       |¦  «        }|                      |¦  «        }|S )Nr   )rÅ   rÇ   )r*   r�   Úfirst_token_tensorÚpooled_outputs       r,   r7   zViTPooler.forward'  s@   € ð +¨1¨1¨1¨a¨4Ô0ÐØŸ
š
Ð#5Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐr-   r²   r?   s   @r,   rÂ   rÂ   !  sj   ø€ € € € € ð4˜yð 4ð 4ð 4ð 4ð 4ð 4ð
 U¤\ð °e´lð ð ð ð ð ð ð ð r-   rÂ   c                   óŽ   ‡ — e Zd ZU eed<   dZdZdZdZddgZ	dZ
dZdZdZdZeedœZd	Z ej        ¦   «         ˆ fd
„¦   «         Zˆ xZS )ÚViTPreTrainedModelr   Úvitr.   )ÚimageTrA   r´   )r�   Ú
attentionsrI   c                 ób  •— t          ¦   «                              |¦  «         t          |t          ¦  «        ru|j        �&t          j        |j        d| j        j        ¬¦  «         t          j        |j	        d| j        j        ¬¦  «         |j
        �t          j        |j
        ¦  «         dS dS dS )zInitialize the weightsNrw   )ÚmeanÚstd)r    Ú_init_weightsr$   rA   rJ   ÚinitÚtrunc_normal_r   Úinitializer_rangerF   rH   Úzeros_)r*   rx   r+   s     €r,   rÓ   z ViTPreTrainedModel._init_weightsC  s¬   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�mÑ,Ô,ð 	/ØÔ)Ð5ÝÔ" 6Ô#=ÀCÈTÌ[ÔMjÐkÑkÔkÐkÝÔ˜vÔ/°c¸t¼{Ô?\Ð]Ñ]Ô]Ð]ØÔ Ð,Ý”˜FÔ-Ñ.Ô.Ð.Ð.Ð.ð	/ð 	/ð -Ð,r-   )r8   r9   r:   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_attention_backendÚ_can_compile_fullgraphr´   r�   Ú_can_record_outputsÚ_input_embed_layerr<   Úno_gradrÓ   r>   r?   s   @r,   rÌ   rÌ   0  s²   ø€ € € € € € àÐÐÑØÐØ$€OØ!ÐØ&*Ð#Ø(¨*Ð5ÐØ€NØÐØÐØ"&ÐØ!Ðà!Ø"ðð Ðð ,Ðà€U„]�_„_ð/ð /ð /ð /ñ „_ð/ð /ð /ð /ð /r-   rÌ   c                   óÞ   ‡ — e Zd Zddededefˆ fd„Ze ed¬¦  «        e	 	 	 	 dd	e	j
        dz  d
e	j        dz  dedz  de	j
        dz  dee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚViTModelTFr   Úadd_pooling_layerrB   c                 ó–  •‡— t          ¦   «                              ‰¦  «         ‰| _        t          ‰|¬¦  «        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _	        t          j
        ‰j        ‰j        ¬¦  «        | _        |rt          ‰¦  «        nd| _        |                      ¦   «          dS )zû
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        use_mask_token (`bool`, *optional*, defaults to `False`):
            Whether to use a mask token for masked image modeling.
        )rB   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS © )r´   )Ú.0rÀ   r   s     €r,   ú
<listcomp>z%ViTModel.__init__.<locals>.<listcomp>[  s!   ø€ Ð$_Ð$_Ð$_¸!¥X¨fÑ%5Ô%5Ð$_Ð$_Ð$_r-   r¶   N)r    r!   r   rA   rN   r   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayersr¹   r(   rº   Ú	layernormrÂ   ÚpoolerÚ	post_init)r*   r   rè   rB   r+   s    `  €r,   r!   zViTModel.__init__Q  sµ   øø€ õ 	‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ'¨¸~ÐNÑNÔNˆŒÝ”mÐ$_Ð$_Ð$_Ð$_½uÀVÔE]Ñ?^Ô?^Ð$_Ñ$_Ô$_Ñ`Ô`ˆŒÝœ fÔ&8¸fÔ>SÐTÑTÔTˆŒØ+<ÐF•i Ñ'Ô'Ð'À$ˆŒà�ŠÑÔÐÐÐr-   )Útie_last_hidden_statesNr.   ri   rh   r|   r~   r/   c                 óŠ  — | j         j        j        j        j        }|j        |k    r|                     |¦  «        }|                       |||¬¦  «        }t          | j        ||¬¦  «        }|}| j        D ]}	 |	||fi |¤Ž}Œ|  	                    |¦  «        }
| j
        �|  
                    |
¦  «        nd}t          |
|¬¦  «        S )zË
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        )ri   rh   )r   Úinputs_embedsr|   N)Úlast_hidden_stateÚpooler_output)rN   rI   r)   Úweightr�   r‡   r	   r   rñ   rò   ró   r   )r*   r.   ri   rh   r|   r~   Úexpected_dtypeÚembedding_outputr�   ÚlayerÚsequence_outputrÊ   s               r,   r7   zViTModel.forwarda  sð   € ð  œÔ9ÔDÔKÔQˆØÔ Ò/Ð/Ø'Ÿ?š?¨>Ñ:Ô:ˆLàŸ?š?Ø¨/ÐTlð +ñ 
ô 
Ðõ 3Ø”;Ø*Ø)ð
ñ 
ô 
ˆð
 )ˆØ”[ð 	Kð 	KˆEØ!˜E -°ÐJÐJÀ6ÐJÐJˆMˆMàŸ.š.¨Ñ7Ô7ˆØ8<¼Ð8O˜Ÿš OÑ4Ô4Ð4ÐUYˆå)¸OÐ[hÐiÑiÔiÐir-   )TF©NNNN)r8   r9   r:   r   rt   r!   r   r   r   r<   r=   rv   r   r   r   r7   r>   r?   s   @r,   rç   rç   O  s  ø€ € € € € ðð ˜yð ¸Tð ÐZ^ð ð ð ð ð ð ð   Ø€_¨EÐ2Ñ2Ô2Øð -1Ø37Ø04Ø.2ð jð  jà”l TÑ)ð jð Ô)¨DÑ0ð jð #'¨¡+ð	 jð
 œ tÑ+ð jð Ð+Ô,ð jð 
$ð jð  jð  jñ „^ñ 3Ô2ñ  Ôð jð  jð  jð  jð  jr-   rç   ac  
    ViT Model with a decoder on top for masked image modeling, as proposed in [SimMIM](https://huggingface.co/papers/2111.09886).

    <Tip>

    Note that we provide a script to pre-train this model on custom data in our [examples
    directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining).

    </Tip>
    )Úcustom_introc                   ó°   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 ddej        dz  dej	        dz  de
dz  dej        dz  dee         d	efd
„¦   «         ¦   «         Zˆ xZS )ÚViTForMaskedImageModelingr   c                 óV  •— t          ¦   «                              |¦  «         t          |dd¬¦  «        | _        t	          j        t	          j        |j        |j        dz  |j	        z  d¬¦  «        t	          j
        |j        ¦  «        ¦  «        | _        |                      ¦   «          d S )NFT)rè   rB   r2   r   )Úin_channelsÚout_channelsr   )r    r!   rç   rÍ   r   Ú
Sequentialr'   r(   Úencoder_strider&   ÚPixelShuffleÚdecoderrô   rœ   s     €r,   r!   z"ViTForMaskedImageModeling.__init__”  sž   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å˜F°eÈDÐQÑQÔQˆŒå”}ÝŒIØ"Ô.Ø#Ô2°AÑ5¸Ô8KÑKØðñ ô õ
 ŒO˜FÔ1Ñ2Ô2ñ
ô 
ˆŒð 	�ŠÑÔÐÐÐr-   Nr.   ri   rh   r|   r~   r/   c                 óÌ  — |�D| j         j        | j         j        k    r*t          d| j         j        › d| j         j        › d�¦  «        ‚ | j        |f|||dœ|¤Ž}|j        }|dd…dd…f         }|j        \  }}	}
t          j        |	dz  ¦  «        x}}| 	                    dd	d¦  «         
                    ||
||¦  «        }|                      |¦  «        }d}|�ñ| j         j        | j         j        z  }| 
                    d
||¦  «        }|                     | j         j        d¦  «                             | j         j        d	¦  «                             d¦  «                             ¦   «         }t           j                             ||d¬¦  «        }||z                       ¦   «         |                     ¦   «         dz   z  | j         j        z  }t+          |||j        |j        ¬¦  «        S )a|  
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

        Examples:
        ```python
        >>> from transformers import AutoImageProcessor, ViTForMaskedImageModeling
        >>> import torch
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> image_processor = AutoImageProcessor.from_pretrained("google/vit-base-patch16-224-in21k")
        >>> model = ViTForMaskedImageModeling.from_pretrained("google/vit-base-patch16-224-in21k")

        >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
        >>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
        >>> # create random boolean mask of shape (batch_size, num_patches)
        >>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

        >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
        >>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction
        >>> list(reconstructed_pixel_values.shape)
        [1, 3, 224, 224]
        ```Nz³When `bool_masked_pos` is provided, `patch_size` must be equal to `encoder_stride` to ensure that the reconstructed image has the same dimensions as the input. Got `patch_size` = z and `encoder_stride` = r1   )ri   rh   r|   r   rS   r   r2   rR   Únone)Ú	reductiongñhãˆµøä>)ÚlossÚreconstructionr�   rÏ   )r   r#   r  r4   rÍ   rø   r3   ÚmathÚfloorr]   r\   r	  r"   Úrepeat_interleaverm   rˆ   r   r^   Úl1_lossÚsumr&   r   r�   rÏ   )r*   r.   ri   rh   r|   r~   Úoutputsrþ   ro   Úsequence_lengthr&   rO   rP   Úreconstructed_pixel_valuesÚmasked_im_lossrU   rr   Úreconstruction_losss                     r,   r7   z!ViTForMaskedImageModeling.forward¥  s!  € ðP Ð&¨D¬KÔ,BÀdÄkÔF`Ò,`Ð,`Ýðtà&*¤kÔ&<ðtð tàVZÔVaÔVpðtð tð tñô ð ð /7¨d¬hØð/
à+Ø%=Ø)ð	/
ð /
ð
 ð/
ð /
ˆð "Ô3ˆð *¨!¨!¨!¨Q¨R¨R¨%Ô0ˆØ4CÔ4IÑ1ˆ
�O \Ýœ O°SÑ$8Ñ9Ô9Ð9ˆ�Ø)×1Ò1°!°Q¸Ñ:Ô:×BÒBÀ:È|Ð]cÐejÑkÔkˆð &*§\¢\°/Ñ%BÔ%BÐ"àˆØÐ&Ø”;Ô)¨T¬[Ô-CÑCˆDØ-×5Ò5°b¸$ÀÑEÔEˆOà×1Ò1°$´+Ô2HÈ!ÑLÔLß"Ò" 4¤;Ô#9¸1Ñ=Ô=ß’˜1‘”ß’‘”ð	 õ #%¤-×"7Ò"7¸ÐF`ÐlrÐ"7Ñ"sÔ"sÐØ1°DÑ8×=Ò=Ñ?Ô?À4Ç8Â8Á:Ä:ÐPTÑCTÑUÐX\ÔXcÔXpÑpˆNå(ØØ5Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r-   rÿ   )r8   r9   r:   r   r!   r   r   r<   r=   rv   rt   r   r   r   r7   r>   r?   s   @r,   r  r  ‡  së   ø€ € € € € ð˜yð ð ð ð ð ð ð" Øð -1Ø37Ø04Ø.2ðR
ð R
à”l TÑ)ðR
ð Ô)¨DÑ0ðR
ð #'¨¡+ð	R
ð
 œ tÑ+ðR
ð Ð+Ô,ðR
ð 
#ðR
ð R
ð R
ñ „^ñ ÔðR
ð R
ð R
ð R
ð R
r-   r  aà  
    ViT Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.

    <Tip>

        Note that it's possible to fine-tune ViT on higher resolution images than the ones it has been trained on, by
        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
        position embeddings to the higher resolution.

    </Tip>
    c                   ó°   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 ddej        dz  dej        dz  de	dz  dej        dz  de
e         d	efd
„¦   «         ¦   «         Zˆ xZS )ÚViTForImageClassificationr   c                 ó:  •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        |j        dk    rt          j        |j        |j        ¦  «        nt          j        ¦   «         | _	        |  
                    ¦   «          d S )NF)rè   r   )r    r!   Ú
num_labelsrç   rÍ   r   r–   r(   ÚIdentityÚ
classifierrô   rœ   s     €r,   r!   z"ViTForImageClassification.__init__  s‹   ø€ Ý‰Œ×Ò˜Ñ Ô Ð à Ô+ˆŒÝ˜F°eÐ<Ñ<Ô<ˆŒð OUÔN_ÐbcÒNcÐNc�"œ) FÔ$6¸Ô8IÑJÔJÐJÕikÔitÑivÔivˆŒð 	�ŠÑÔÐÐÐr-   Nr.   Úlabelsrh   r|   r~   r/   c                 óæ   —  | j         |f||dœ|¤Ž}|j        }|dd…ddd…f         }|                      |¦  «        }	d}
|� | j        ||	| j        fi |¤Ž}
t          |
|	|j        |j        ¬¦  «        S )aŠ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        )rh   r|   Nr   )r  Úlogitsr�   rÏ   )rÍ   rø   r  Úloss_functionr   r   r�   rÏ   )r*   r.   r  rh   r|   r~   r  rþ   rÊ   r!  r  s              r,   r7   z!ViTForImageClassification.forward  s»   € ð" /7¨d¬hØð/
à%=Ø)ð/
ð /
ð ð	/
ð /
ˆð "Ô3ˆØ'¨¨¨¨1¨a¨a¨a¨Ô0ˆØ—’ Ñ/Ô/ˆàˆØÐØ%�4Ô% f¨f°d´kÐLÐLÀVÐLÐLˆDå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r-   rÿ   )r8   r9   r:   r   r!   r   r   r<   r=   rt   r   r   r   r7   r>   r?   s   @r,   r  r  ü  sÛ   ø€ € € € € ð
˜yð 
ð 
ð 
ð 
ð 
ð 
ð Øð -1Ø&*Ø04Ø.2ð#
ð #
à”l TÑ)ð#
ð ”˜tÑ#ð#
ð #'¨¡+ð	#
ð
 œ tÑ+ð#
ð Ð+Ô,ð#
ð 
ð#
ð #
ð #
ñ „^ñ Ôð#
ð #
ð #
ð #
ð #
r-   r  )r  r  rç   rÌ   )Nrw   )7r;   r  Úcollections.abcr   r   r<   r   Ú r   rÔ   Úactivationsr   Úmasking_utilsr	   Úmodeling_layersr
   Úmodeling_outputsr   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_vitr   Ú
get_loggerr8   ÚloggerÚModuler   rA   r=   Úfloatr‹   r�   rª   r´   rÂ   rÌ   rç   r  r  Ú__all__rë   r-   r,   ú<module>r4     s6  ðð Ð à €€€Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð
 GÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø (Ð (Ð (Ð (Ð (Ð (ð 
ˆÔ	˜HÑ	%Ô	%€ðHð Hð Hð Hð H˜œñ Hô Hð Hð<Yð Yð Yð Yð Y�B”Iñ Yô Yð YðD !Øð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð8.)ð .)ð .)ð .)ð .)�2”9ñ .)ô .)ð .)ðbð ð ð ð ˆRŒYñ ô ð ð ð ð ð ð Ð)ñ ô ð ð@ð ð ð ð �”	ñ ô ð ð ð/ð /ð /ð /ð /˜ñ /ô /ñ „ð/ð< ð4jð 4jð 4jð 4jð 4jÐ!ñ 4jô 4jñ „ð4jðn €ð	ðñ ô ðf
ð f
ð f
ð f
ð f
Ð 2ñ f
ô f
ñô ðf
ðR €ððñ ô ð2
ð 2
ð 2
ð 2
ð 2
Ð 2ñ 2
ô 2
ñô ð2
ðj gÐ
fÐ
f€€€r-   