§
    ‚Štj¤G  ã                   ó´  — d dl mZmZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZ ddlmZmZ dd	lmZmZ dd
lmZ ddlmZmZmZ ddlmZmZ ddlmZ ddl m!Z!  G d„ dej"        ¦  «        Z# G d„ dej"        ¦  «        Z$	 	 d/dej"        dej%        dej%        dej%        dej%        dz  de&dz  de&dee         fd„Z' G d„ dej"        ¦  «        Z( G d „ d!ej"        ¦  «        Z) G d"„ d#e¦  «        Z*e G d$„ d%e¦  «        ¦   «         Z+ G d&„ d'ej"        ¦  «        Z,e G d(„ d)e+¦  «        ¦   «         Z- ed*¬+¦  «         G d,„ d-e+¦  «        ¦   «         Z.g d.¢Z/dS )0é    )ÚCallableÚIterableNé   )Úinitialization)ÚACT2FN)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPoolingÚImageClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚ	torch_int)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputsé   )ÚIJepaConfigc                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚIJepaPatchEmbeddingszì
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    Úconfigc                 ó¢  •— t          ¦   «                              ¦   «          |j        }|j        }t	          |t
          ¦  «        r|n||f}t	          |t
          ¦  «        r|n||f}|d         |d         z  |d         |d         z  z  | _        || _        || _        |j        | _        t          j	        |j        |j
        ||¬¦  «        | _        d S )Nr   r   )Úkernel_sizeÚstride)ÚsuperÚ__init__Ú
image_sizeÚ
patch_sizeÚ
isinstancer   Únum_patchesÚnum_channelsÚnnÚConv2dÚhidden_sizeÚ
projection)Úselfr   r   r    Ú	__class__s       €úf/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/ijepa/modeling_ijepa.pyr   zIJepaPatchEmbeddings.__init__    sÎ   ø€ Ý‰Œ×ÒÑÔÐØÔ&ˆ
ØÔ&ˆ
Ý#-¨j½(Ñ#CÔ#CÐa�Z�ZÈ*ÐV`ÐIaˆ
Ý#-¨j½(Ñ#CÔ#CÐa�Z�ZÈ*ÐV`ÐIaˆ
à& qœM¨Z¸¬]Ñ:¸zÈ!¼}ÐPZÐ[\ÔP]Ñ?]Ñ^ˆÔØ$ˆŒØ$ˆŒØ"Ô/ˆÔÝœ) FÔ$7¸Ô9KÐYcÐlvÐwÑwÔwˆŒˆˆó    Úpixel_valuesÚreturnc                 óà   — |j         d         }|| j        k    rt          d| j        › d|› d�¦  «        ‚|                      |¦  «                             d¦  «                             dd¦  «        S )Nr   zoMake sure that the channel dimension of the pixel values match with the one set in the configuration. Expected z	 but got ú.é   )Úshaper#   Ú
ValueErrorr'   ÚflattenÚ	transpose)r(   r,   r#   s      r*   ÚforwardzIJepaPatchEmbeddings.forward-   sŽ   € Ø#Ô)¨!Ô,ˆØ˜4Ô,Ò,Ð,ÝðIØ!Ô.ðIð IØ9EðIð Ið Iñô ð ð �Š˜|Ñ,Ô,×4Ò4°QÑ7Ô7×AÒAÀ!ÀQÑGÔGÐGr+   )
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   ÚtorchÚTensorr5   Ú__classcell__©r)   s   @r*   r   r      s…   ø€ € € € € ðð ðx˜{ð xð xð xð xð xð xðH E¤Lð H°U´\ð Hð Hð Hð Hð Hð Hð Hð Hr+   r   c            	       ó    ‡ — e Zd ZdZddededdfˆ fd„Zdej        d	e	d
e	dej        fd„Z
	 	 ddej        dej        dz  dedej        fd„Zˆ xZS )ÚIJepaEmbeddingszb
    Construct the CLS token, position and patch embeddings. Optionally, also the mask token.
    Fr   Úuse_mask_tokenr-   Nc                 óÎ  •— t          ¦   «                              ¦   «          |r-t          j        t	          j        dd|j        ¦  «        ¦  «        nd | _        t          |¦  «        | _	        | j	        j
        }t          j        t	          j        d||j        ¦  «        ¦  «        | _        t          j        |j        ¦  «        | _        |j        | _        | j	        j        | _        d S )Nr   )r   r   r$   Ú	Parameterr:   Úzerosr&   Ú
mask_tokenr   Úpatch_embeddingsr"   ÚrandnÚposition_embeddingsÚDropoutÚhidden_dropout_probÚdropoutr    r   )r(   r   r@   r"   r)   s       €r*   r   zIJepaEmbeddings.__init__<   s³   ø€ Ý‰Œ×ÒÑÔÐØQ_Ði�"œ,¥u¤{°1°a¸Ô9KÑ'LÔ'LÑMÔMÐMÐeiˆŒÝ 4°VÑ <Ô <ˆÔØÔ+Ô7ˆÝ#%¤<µ´¸A¸{ÈFÔL^Ñ0_Ô0_Ñ#`Ô#`ˆÔ Ý”z &Ô"<Ñ=Ô=ˆŒØ Ô+ˆŒØÔ/Ô:ˆŒˆˆr+   Ú
embeddingsÚheightÚwidthc                 ó  — |j         d         }| j        j         d         }t          j                             ¦   «         s||k    r||k    r| j        S | j        }|j         d         }|| j        z  }|| j        z  }	t          |dz  ¦  «        }
|                     d|
|
|¦  «        }|                     dddd¦  «        }t          j
                             |||	fdd¬	¦  «        }|                     dddd¦  «                             dd|¦  «        }|S )
a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   éÿÿÿÿg      à?r   r   r0   ÚbicubicF)ÚsizeÚmodeÚalign_corners)r1   rG   r:   ÚjitÚ
is_tracingr    r   ÚreshapeÚpermuter$   Ú
functionalÚinterpolateÚview)r(   rK   rL   rM   r"   Únum_positionsÚpatch_pos_embedÚdimÚ
new_heightÚ	new_widthÚsqrt_num_positionss              r*   Úinterpolate_pos_encodingz(IJepaEmbeddings.interpolate_pos_encodingF   s*  € ð !Ô& qÔ)ˆØÔ0Ô6°qÔ9ˆõ Œy×#Ò#Ñ%Ô%ð 	,¨+¸Ò*FÐ*FÈ6ÐUZÊ?È?ØÔ+Ð+àÔ2ˆàÔ˜rÔ"ˆà˜tœÑ.ˆ
Ø˜Tœ_Ñ,ˆ	å& }°cÑ'9Ñ:Ô:ÐØ)×1Ò1°!Ð5GÐI[Ð]`ÑaÔaˆØ)×1Ò1°!°Q¸¸1Ñ=Ô=ˆåœ-×3Ò3ØØ˜iÐ(ØØð	 4ñ 
ô 
ˆð *×1Ò1°!°Q¸¸1Ñ=Ô=×BÒBÀ1ÀbÈ#ÑNÔNˆàÐr+   r,   Úbool_masked_posra   c                 ó*  — |j         \  }}}}|                      |¦  «        }|�_|j         d         }	| j                             ||	d¦  «        }
|                     d¦  «                             |
¦  «        }|d|z
  z  |
|z  z   }|r||                      |||¦  «        z   }n^|| j        d         k    s|| j        d         k    r2t          d|› d|› d| j        d         › d| j        d         › d�	¦  «        ‚|| j	        z   }|  
                    |¦  «        }|S )	Nr   rO   g      ð?r   zInput image size (Ú*z) doesn't match model (z).)r1   rE   rD   ÚexpandÚ	unsqueezeÚtype_asra   r   r2   rG   rJ   )r(   r,   rb   ra   Ú
batch_sizeÚ_rL   rM   rK   Ú
seq_lengthÚmask_tokensÚmasks               r*   r5   zIJepaEmbeddings.forwardm   sf  € ð (4Ô'9Ñ$ˆ
�A�v˜uØ×*Ò*¨<Ñ8Ô8ˆ
àÐ&Ø#Ô)¨!Ô,ˆJØœ/×0Ò0°¸ZÈÑLÔLˆKà"×,Ò,¨RÑ0Ô0×8Ò8¸ÑEÔEˆDØ# s¨T¡zÑ2°[À4Ñ5GÑGˆJð $ð 	?Ø# d×&CÒ&CÀJÐPVÐX]Ñ&^Ô&^Ñ^ˆJˆJà˜œ¨Ô+Ò+Ð+¨u¸¼ÈÔ8JÒ/JÐ/JÝ ðE¨ð Eð E°%ð Eð EØœ¨Ô+ðEð EØ.2¬o¸aÔ.@ðEð Eð Eñô ð ð $ dÔ&>Ñ>ˆJà—\’\ *Ñ-Ô-ˆ
àÐr+   )F)NF)r6   r7   r8   r9   r   Úboolr   r:   r;   Úintra   Ú
BoolTensorr5   r<   r=   s   @r*   r?   r?   7   sì   ø€ € € € € ðð ð;ð ;˜{ð ;¸Dð ;ÈTð ;ð ;ð ;ð ;ð ;ð ;ð%°5´<ð %Èð %ÐUXð %Ð]bÔ]ið %ð %ð %ð %ðT 48Ø).ð	ð à”lðð Ô)¨DÑ0ðð #'ð	ð
 
Œðð ð ð ð ð ð ð r+   r?   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingrJ   Úkwargsc                 óô  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |dt          j        ¬¦  «                             |j	        ¦  «        }t          j         
                    ||| j        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «                             ¦   «         }	|	|fS )NrO   ç      à¿r0   r   )r]   Údtype)ÚpÚtrainingr   )rQ   r:   Úmatmulr4   r$   rX   ÚsoftmaxÚfloat32Útorz   rJ   r|   Ú
contiguous)
rq   rr   rs   rt   ru   rv   rJ   rw   Úattn_weightsÚattn_outputs
             r*   Úeager_attention_forwardr„   �   sÞ   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r+   c                   óŠ   ‡ — e Zd Zdefˆ fd„Z	 d	dej        dej        dz  dee         de	ej        ej        f         fd„Z
ˆ xZS )
ÚIJepaAttentionr   c                 ó†  •— t          ¦   «                              ¦   «          || _        |j        | _        t	          |d|j        |j        z  ¦  «        | _        |j        | _        | j        dz  | _	        d| _
        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        | j        z  |j        d¬¦  «        | _        d S )NÚhead_dimry   F)ÚbiasT)r   r   r   Únum_attention_headsÚgetattrr&   rˆ   Úattention_probs_dropout_probÚattention_dropoutrv   Ú	is_causalr$   ÚLinearÚqkv_biasÚq_projÚk_projÚv_projÚo_proj©r(   r   r)   s     €r*   r   zIJepaAttention.__init__ª   s  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ#)Ô#=ˆÔ Ý ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ!'Ô!DˆÔØ”} dÑ*ˆŒØˆŒå”i Ô 2°FÔ4NÐQUÔQ^Ñ4^ÐekÔetÐuÑuÔuˆŒÝ”i Ô 2°FÔ4NÐQUÔQ^Ñ4^ÐekÔetÐuÑuÔuˆŒÝ”i Ô 2°FÔ4NÐQUÔQ^Ñ4^ÐekÔetÐuÑuÔuˆŒÝ”i Ô :¸T¼]Ñ JÈFÔL^ÐeiÐjÑjÔjˆŒˆˆr+   NÚhidden_statesru   rw   r-   c                 ó¬  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }t          j        | j	        j
        t          ¦  «        }	 |	| ||||f| j        sdn| j        | j        dœ|¤Ž\  }
} |
j        g |¢d‘R Ž                      ¦   «         }
|                      |
¦  «        }
|
|fS )NrO   r   r0   rp   )rJ   rv   )r1   rˆ   r‘   rZ   r4   r’   r“   r   Úget_interfacer   Ú_attn_implementationr„   r|   r�   rv   rV   r�   r”   )r(   r–   ru   rw   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚattention_interfacerƒ   r‚   s               r*   r5   zIJepaAttention.forward¸   sw  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆà˜LÐ(Ð(r+   ©N)r6   r7   r8   r   r   r:   r;   r   r   Útupler5   r<   r=   s   @r*   r†   r†   ©   s¬   ø€ € € € € ðk˜{ð kð kð kð kð kð kð" /3ð)ð )à”|ð)ð œ tÑ+ð)ð Ð+Ô,ð	)ð
 
ˆuŒ|˜Uœ\Ð)Ô	*ð)ð )ð )ð )ð )ð )ð )ð )r+   r†   c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚIJepaMLPr   c                 ó  •— t          ¦   «                              ¦   «          || _        t          |j                 | _        t          j        |j        |j	        ¦  «        | _
        t          j        |j	        |j        ¦  «        | _        d S r    )r   r   r   r   Ú
hidden_actÚactivation_fnr$   r�   r&   Úintermediate_sizeÚfc1Úfc2r•   s     €r*   r   zIJepaMLP.__init__Û   sf   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ# FÔ$5Ô6ˆÔÝ”9˜VÔ/°Ô1IÑJÔJˆŒÝ”9˜VÔ5°vÔ7IÑJÔJˆŒˆˆr+   r–   r-   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r    )r¨   r¦   r©   )r(   r–   s     r*   r5   zIJepaMLP.forwardâ   s=   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆàÐr+   ©	r6   r7   r8   r   r   r:   r;   r5   r<   r=   s   @r*   r£   r£   Ú   sq   ø€ € € € € ðK˜{ð Kð Kð Kð Kð Kð Kð U¤\ð °e´lð ð ð ð ð ð ð ð r+   r£   c            	       óp   ‡ — e Zd Zdefˆ fd„Z	 d	dej        dej        dz  dee         dej        fd„Z	ˆ xZ
S )
Ú
IJepaLayerr   c                 óh  •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          j        |j        |j        ¬¦  «        | _        t	          j        |j        |j        ¬¦  «        | _	        t          |¦  «        | _        t	          j        |j        ¦  «        | _        d S )N©Úeps)r   r   r†   Ú	attentionr$   Ú	LayerNormr&   Úlayer_norm_epsÚlayernorm_beforeÚlayernorm_afterr£   ÚmlprH   rI   rJ   r•   s     €r*   r   zIJepaLayer.__init__ë   s‰   ø€ Ý‰Œ×ÒÑÔÐÝ'¨Ñ/Ô/ˆŒÝ "¤¨VÔ-?ÀVÔEZÐ [Ñ [Ô [ˆÔÝ!œ|¨FÔ,>ÀFÔDYÐZÑZÔZˆÔÝ˜FÑ#Ô#ˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr+   Nr–   ru   rw   r-   c                 ó  — |}|                       |¦  «        } | j        ||fi |¤Ž\  }}|                      |¦  «        }||z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z   }|S r    )r´   r±   rJ   rµ   r¶   )r(   r–   ru   rw   Úresidualri   s         r*   r5   zIJepaLayer.forwardó   s£   € ð !ˆØ×-Ò-¨mÑ<Ô<ˆØ)˜4œ>¨-¸ÐRÐRÈ6ÐRÐRÑˆ�qØŸš ]Ñ3Ô3ˆØ%¨Ñ0ˆð !ˆØ×,Ò,¨]Ñ;Ô;ˆØŸš Ñ/Ô/ˆØŸš ]Ñ3Ô3ˆØ%¨Ñ0ˆàÐr+   r    )r6   r7   r8   r   r   r:   r;   r   r   r5   r<   r=   s   @r*   r­   r­   ê   s™   ø€ € € € € ð>˜{ð >ð >ð >ð >ð >ð >ð /3ðð à”|ðð œ tÑ+ðð Ð+Ô,ð	ð
 
Œðð ð ð ð ð ð ð r+   r­   c                   óŽ   ‡ — e Zd ZU eed<   dZdZdZdZddgZ	dZ
dZdZdZdZeedœZd	Z ej        ¦   «         ˆ fd
„¦   «         Zˆ xZS )ÚIJepaPreTrainedModelr   Úijepar,   )ÚimageTr?   r­   )r–   Ú
attentionsrE   c                 óè  •— t          ¦   «                              |¦  «         t          |t          j        t          j        f¦  «        rJt          j        |j        d| j	        j
        ¬¦  «         |j        �t          j        |j        ¦  «         dS dS t          |t          ¦  «        rHt          j        |j        d| j	        j
        ¬¦  «         |j        �t          j        |j        ¦  «         dS dS dS )zInitialize the weightsrp   )ÚmeanÚstdN)r   Ú_init_weightsr!   r$   r�   r%   ÚinitÚtrunc_normal_Úweightr   Úinitializer_ranger‰   Úzeros_r?   rG   rD   )r(   rq   r)   s     €r*   rÁ   z"IJepaPreTrainedModel._init_weights  sã   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�rœy­"¬)Ð4Ñ5Ô5ð 	/ÝÔ˜vœ}°3¸D¼KÔ<YÐZÑZÔZÐZØŒ{Ð&Ý”˜FœKÑ(Ô(Ð(Ð(Ð(ð 'Ð&å˜¥Ñ0Ô0ð 	/ÝÔ˜vÔ9ÀÈÌÔIfÐgÑgÔgÐgØÔ Ð,Ý”˜FÔ-Ñ.Ô.Ð.Ð.Ð.ð	/ð 	/à,Ð,r+   )r6   r7   r8   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_attention_backendÚ_can_compile_fullgraphr­   r†   Ú_can_record_outputsÚ_input_embed_layerr:   Úno_gradrÁ   r<   r=   s   @r*   rº   rº   
  s²   ø€ € € € € € àÐÐÑØÐØ$€OØ!ÐØ&*Ð#Ø*¨LÐ9ÐØ€NØÐØÐØ"&ÐØ!Ðà#Ø$ðð Ðð ,Ðà€U„]�_„_ð
/ð 
/ð 
/ð 
/ñ „_ð
/ð 
/ð 
/ð 
/ð 
/r+   rº   c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚIJepaPoolerr   c                 ó¾   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j                 | _	        d S r    )
r   r   r$   r�   r&   Úpooler_output_sizeÚdenser   Ú
pooler_actÚ
activationr•   s     €r*   r   zIJepaPooler.__init__,  sE   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3LÑMÔMˆŒ
Ý  Ô!2Ô3ˆŒˆˆr+   r–   r-   c                 ór   — |d d …df         }|                       |¦  «        }|                      |¦  «        }|S )Nr   )rÙ   rÛ   )r(   r–   Úfirst_token_tensorÚpooled_outputs       r*   r5   zIJepaPooler.forward1  s@   € ð +¨1¨1¨1¨a¨4Ô0ÐØŸ
š
Ð#5Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐr+   r«   r=   s   @r*   rÖ   rÖ   +  sj   ø€ € € € € ð4˜{ð 4ð 4ð 4ð 4ð 4ð 4ð
 U¤\ð °e´lð ð ð ð ð ð ð ð r+   rÖ   c                   óÞ   ‡ — e Zd Zddededefˆ fd„Ze ed¬¦  «        e	 	 	 	 dde	j
        dz  d	e	j        dz  d
edz  de	j
        dz  dee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú
IJepaModelFr   Úadd_pooling_layerr@   c                 ó–  •‡— t          ¦   «                              ‰¦  «         ‰| _        t          ‰|¬¦  «        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _	        t          j
        ‰j        ‰j        ¬¦  «        | _        |rt          ‰¦  «        nd| _        |                      ¦   «          dS )zû
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        use_mask_token (`bool`, *optional*, defaults to `False`):
            Whether to use a mask token for masked image modeling.
        )r@   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS © )r­   )Ú.0ri   r   s     €r*   ú
<listcomp>z'IJepaModel.__init__.<locals>.<listcomp>F  s!   ø€ Ð$aÐ$aÐ$a¸A¥Z°Ñ%7Ô%7Ð$aÐ$aÐ$ar+   r¯   N)r   r   r   r?   rK   r$   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayersr²   r&   r³   Ú	layernormrÖ   ÚpoolerÚ	post_init)r(   r   rá   r@   r)   s    `  €r*   r   zIJepaModel.__init__<  sµ   øø€ õ 	‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ)¨&ÀÐPÑPÔPˆŒÝ”mÐ$aÐ$aÐ$aÐ$aÅÀvÔG_ÑA`ÔA`Ð$aÑ$aÔ$aÑbÔbˆŒÝœ fÔ&8¸fÔ>SÐTÑTÔTˆŒØ->ÐH•k &Ñ)Ô)Ð)ÀDˆŒà�ŠÑÔÐÐÐr+   )Útie_last_hidden_statesNr,   rb   ra   ru   rw   r-   c                 óŠ  — | j         j        j        j        j        }|j        |k    r|                     |¦  «        }|                       |||¬¦  «        }t          | j        ||¬¦  «        }|}| j        D ]}	 |	||fi |¤Ž}Œ|  	                    |¦  «        }
| j
        �|  
                    |
¦  «        nd}t          |
|¬¦  «        S )zË
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        )rb   ra   )r   Úinputs_embedsru   N)Úlast_hidden_stateÚpooler_output)rK   rE   r'   rÄ   rz   r€   r   r   rê   rë   rì   r
   )r(   r,   rb   ra   ru   rw   Úexpected_dtypeÚembedding_outputr–   ÚlayerÚsequence_outputrÞ   s               r*   r5   zIJepaModel.forwardL  sð   € ð  œÔ9ÔDÔKÔQˆØÔ Ò/Ð/Ø'Ÿ?š?¨>Ñ:Ô:ˆLàŸ?š?Ø¨/ÐTlð +ñ 
ô 
Ðõ 3Ø”;Ø*Ø)ð
ñ 
ô 
ˆð
 )ˆØ”[ð 	Kð 	KˆEØ!˜E -°ÐJÐJÀ6ÐJÐJˆMˆMàŸ.š.¨Ñ7Ô7ˆØ8<¼Ð8O˜Ÿš OÑ4Ô4Ð4ÐUYˆå)¸OÐ[hÐiÑiÔiÐir+   )FF)NNNN)r6   r7   r8   r   rm   r   r   r   r   r:   r;   ro   r   r   r
   r5   r<   r=   s   @r*   rà   rà   :  s  ø€ € € € € ðð ˜{ð ¸tð Ð]að ð ð ð ð ð ð   Ø€_¨EÐ2Ñ2Ô2Øð -1Ø37Ø04Ø.2ð jð  jà”l TÑ)ð jð Ô)¨DÑ0ð jð #'¨¡+ð	 jð
 œ tÑ+ð jð Ð+Ô,ð jð 
$ð jð  jð  jñ „^ñ 3Ô2ñ  Ôð jð  jð  jð  jð  jr+   rà   aÒ  
    IJepa Model transformer with an image classification head on top (a linear layer on top of the final hidden states)
    e.g. for ImageNet.

    <Tip>

        Note that it's possible to fine-tune IJepa on higher resolution images than the ones it has been trained on, by
        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
        position embeddings to the higher resolution.

    </Tip>
    )Úcustom_introc                   óš   ‡ — e Zd Zdefˆ fd„Zee	 	 	 d
dej        dz  dej        dz  de	dz  de
e         def
d	„¦   «         ¦   «         Zˆ xZS )ÚIJepaForImageClassificationr   c                 ó:  •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        |j        dk    rt          j        |j        |j        ¦  «        nt          j        ¦   «         | _	        |  
                    ¦   «          d S )NF)rá   r   )r   r   Ú
num_labelsrà   r»   r$   r�   r&   ÚIdentityÚ
classifierrí   r•   s     €r*   r   z$IJepaForImageClassification.__init__�  s‹   ø€ Ý‰Œ×Ò˜Ñ Ô Ð à Ô+ˆŒÝ ¸%Ð@Ñ@Ô@ˆŒ
ð OUÔN_ÐbcÒNcÐNc�"œ) FÔ$6¸Ô8IÑJÔJÐJÕikÔitÑivÔivˆŒð 	�ŠÑÔÐÐÐr+   Nr,   Úlabelsra   rw   r-   c                 óî   —  | j         |fd|i|¤Ž}|j        }|                      |                     d¬¦  «        ¦  «        }d}|� | j        ||| j        fi |¤Ž}t          |||j        |j        ¬¦  «        S )aŠ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        ra   r   )r]   N)ÚlossÚlogitsr–   r½   )	r»   rñ   rý   r¿   Úloss_functionr   r   r–   r½   )	r(   r,   rþ   ra   rw   Úoutputsrö   r  r   s	            r*   r5   z#IJepaForImageClassification.forward�  s±   € ð  /9¨d¬jØð/
ð /
à%=ð/
ð ð/
ð /
ˆð
 "Ô3ˆØ—’ ×!5Ò!5¸!Ð!5Ñ!<Ô!<Ñ=Ô=ˆàˆØÐØ%�4Ô% f¨f°d´kÐLÐLÀVÐLÐLˆDå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r+   )NNN)r6   r7   r8   r   r   r   r   r:   r;   rm   r   r   r   r5   r<   r=   s   @r*   rù   rù   r  sÇ   ø€ € € € € ð
˜{ð 
ð 
ð 
ð 
ð 
ð 
ð Øð -1Ø&*Ø04ð	
ð 
à”l TÑ)ð
ð ”˜tÑ#ð
ð #'¨¡+ð	
ð
 Ð+Ô,ð
ð 
ð
ð 
ð 
ñ „^ñ Ôð
ð 
ð 
ð 
ð 
r+   rù   )rº   rà   rù   )Nrp   )0Úcollections.abcr   r   r:   Útorch.nnr$   Ú r   rÂ   Úactivationsr   Úmasking_utilsr   Úmodeling_layersr	   Úmodeling_outputsr
   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_ijepar   ÚModuler   r?   r;   Úfloatr„   r†   r£   r­   rº   rÖ   rà   rù   Ú__all__rä   r+   r*   ú<module>r     s˜  ðð /Ð .Ð .Ð .Ð .Ð .Ð .Ð .à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø QÐ QÐ QÐ QÐ QÐ QÐ QÐ QØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BØ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,ðHð Hð Hð Hð H˜2œ9ñ Hô Hð Hð<Sð Sð Sð Sð S�b”iñ Sô Sð Sðx !Øð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð8.)ð .)ð .)ð .)ð .)�R”Yñ .)ô .)ð .)ðbð ð ð ð ˆrŒyñ ô ð ð ð ð ð ð Ð+ñ ô ð ð@ ð/ð /ð /ð /ð /˜?ñ /ô /ñ „ð/ð@ð ð ð ð �"”)ñ ô ð ð ð4jð 4jð 4jð 4jð 4jÐ%ñ 4jô 4jñ „ð4jðn €ððñ ô ð.
ð .
ð .
ð .
ð .
Ð"6ñ .
ô .
ñô ð.
ðb PÐ
OÐ
O€€€r+   