§
    ‚Štj3¢  ã                   óÀ  — d dl mZ d dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZ dd	lmZmZ dd
lmZmZ ddlmZ ddlmZmZmZmZmZ ddlmZ ddlmZmZ ddl m!Z!  ej"        e#¦  «        Z$ ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z% ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z& G d„ dej'        ¦  «        Z( G d„ dej'        ¦  «        Z)	 dJdej'        dej*        dej*        d ej*        d!ej*        dz  d"e+d#e+fd$„Z,d%„ Z- G d&„ d'ej'        ¦  «        Z. G d(„ d)ej'        ¦  «        Z/ G d*„ d+ej'        ¦  «        Z0 G d,„ d-e¦  «        Z1 G d.„ d/ej'        ¦  «        Z2d0ej*        d1e3ej*                 d2ej*        fd3„Z4 G d4„ d5ej'        ¦  «        Z5 G d6„ d7ej'        ¦  «        Z6 G d8„ d9ej'        ¦  «        Z7 G d:„ d;ej'        ¦  «        Z8 G d<„ d=e¦  «        Z9 G d>„ d?e¦  «        Z: G d@„ dAej'        ¦  «        Z;e G dB„ dCe¦  «        ¦   «         Z<e G dD„ dEe<¦  «        ¦   «         Z= edF¬¦  «         G dG„ dHe<¦  «        ¦   «         Z>g dI¢Z?dS )Ké    )ÚCallable)Ú	dataclassN)Únné   )Úinitialization)ÚACT2FN)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚImageClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚVJEPA2ConfigzO
    VJEPA Predictor outputs that also contains the masked encoder outputs
    )Úcustom_introc                   óÀ   — e Zd ZU dZej        ed<   dZej        dz  ed<   dZe	ej        df         dz  ed<   dZ
e	ej        df         dz  ed<   dZej        dz  ed<   dS )	Ú$VJEPA2WithMaskedInputPredictorOutputaê  
    masked_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*, returned when `context_mask` is provided which is applied on VJEPA2Encoder outputs):
        The masked hidden state of the model.
    target_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*, returned when `target_mask` is provided which is applied on VJEPA2Encoder outputs):
        The target hidden state of the model.
    Úlast_hidden_stateNÚmasked_hidden_state.Úhidden_statesÚ
attentionsÚtarget_hidden_state)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚtorchÚFloatTensorÚ__annotations__r   r   Útupler   r    © ó    úh/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/vjepa2/modeling_vjepa2.pyr   r   #   s£   € € € € € € ðð ð Ô(Ð(Ð(Ñ(Ø48Ð˜Ô*¨TÑ1Ð8Ð8Ñ8Ø:>€M�5˜Ô*¨CÐ/Ô0°4Ñ7Ð>Ð>Ñ>Ø7;€J��eÔ'¨Ð,Ô-°Ñ4Ð;Ð;Ñ;Ø48Ð˜Ô*¨TÑ1Ð8Ð8Ñ8Ð8Ð8r*   r   zs
    VJEPA outputs that also contains the masked encoder outputs
    Optionally contains the predictor outputs
    c                   óÆ   ‡ — e Zd ZU dZej        ed<   dZej        dz  ed<   dZe	ej        df         dz  ed<   dZ
e	ej        df         dz  ed<   dZedz  ed<   ˆ fd	„Zˆ xZS )
Ú VJEPA2WithMaskedInputModelOutputaq  
    masked_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*, returned when `context_mask` is provided which is applied on VJEPA2Encoder outputs):
        The masked hidden state of the model.
    predictor_output (`VJEPA2WithMaskedInputPredictorOutput`, *optional*):
        The output from the Predictor module.
    r   Nr   .r   r   Úpredictor_outputc                 óì   •— t          t          ¦   «                              ¦   «         ¦  «        }t          |d         t          ¦  «        r|d                              ¦   «         |d<   t          |¦  «        S )Néÿÿÿÿ)ÚlistÚsuperÚto_tupleÚ
isinstancer   r(   )ÚselfÚoutputÚ	__class__s     €r+   r3   z)VJEPA2WithMaskedInputModelOutput.to_tupleM   s\   ø€ Ý•e‘g”g×&Ò&Ñ(Ô(Ñ)Ô)ˆÝ�f˜R”jÕ"FÑGÔGð 	/Ø œ×,Ò,Ñ.Ô.ˆF�2‰JÝ�V‰}Œ}Ðr*   )r!   r"   r#   r$   r%   r&   r'   r   r   r(   r   r.   r   r3   Ú__classcell__©r7   s   @r+   r-   r-   8   sÉ   ø€ € € € € € ðð ð Ô(Ð(Ð(Ñ(Ø48Ð˜Ô*¨TÑ1Ð8Ð8Ñ8Ø:>€M�5˜Ô*¨CÐ/Ô0°4Ñ7Ð>Ð>Ñ>Ø7;€J��eÔ'¨Ð,Ô-°Ñ4Ð;Ð;Ñ;ØDHÐÐ:¸TÑAÐHÐHÑHðð ð ð ð ð ð ð ð r*   r-   c                   ój   ‡ — e Zd ZdZ	 d
dedefˆ fd„Zed„ ¦   «         Zde	j
        de	j
        fd	„Zˆ xZS )ÚVJEPA2PatchEmbeddings3Dz"
    Image to Patch Embedding
    é   ÚconfigÚhidden_sizec                 ó  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        t          j        |j        ||j        |j        |j        f|j        |j        |j        f¬¦  «        | _        d S )N)Úin_channelsÚout_channelsÚkernel_sizeÚstride)	r2   Ú__init__Ú
patch_sizeÚtubelet_sizer>   r   ÚConv3dÚin_chansÚproj©r5   r=   r>   r7   s      €r+   rD   z VJEPA2PatchEmbeddings3D.__init__Y   s‚   ø€ õ
 	‰Œ×ÒÑÔÐØ Ô+ˆŒØ"Ô/ˆÔØ&ˆÔå”IØœØ$ØÔ,¨fÔ.?ÀÔARÐSØÔ'¨Ô):¸FÔ<MÐNð	
ñ 
ô 
ˆŒ	ˆ	ˆ	r*   c                 ó`   — | j         | j        z  | j        | j        z  z  | j        | j        z  z  S ©N©Úframes_per_cliprF   Ú	crop_sizerE   ©r=   s    r+   Únum_patchesz#VJEPA2PatchEmbeddings3D.num_patchesj   s=   € ð Ô# vÔ':Ñ:ØÔ 6Ô#4Ñ4ñ6àÔ 6Ô#4Ñ4ñ6ð	
r*   Úpixel_values_videosÚreturnc                 ó~   — |                       |¦  «                             d¦  «                             dd¦  «        }|S )Né   r   )rI   ÚflattenÚ	transpose)r5   rR   Úxs      r+   ÚforwardzVJEPA2PatchEmbeddings3D.forwardr   s7   € Ø�IŠIÐ)Ñ*Ô*×2Ò2°1Ñ5Ô5×?Ò?ÀÀ1ÑEÔEˆØˆr*   ©r<   )r!   r"   r#   r$   r   ÚintrD   ÚstaticmethodrQ   r%   ÚTensorrY   r8   r9   s   @r+   r;   r;   T   s¤   ø€ € € € € ðð ð  ð
ð 
àð
ð ð
ð 
ð 
ð 
ð 
ð 
ð" ð
ð 
ñ „\ð
ð¨5¬<ð ¸E¼Lð ð ð ð ð ð ð ð r*   r;   c                   óR   ‡ — e Zd ZdZd	dedefˆ fd„Zdej        dej        fd„Z	ˆ xZ
S )
ÚVJEPA2Embeddingsú>
    Construct mask token, position and patch embeddings.
    r<   r=   r>   c                 óÊ   •— t          ¦   «                              ¦   «          || _        || _        t	          ||¬¦  «        | _        | j        j        | _        |j        | _        d S )N©r>   )r2   rD   r=   r>   r;   Úpatch_embeddingsrQ   rE   rJ   s      €r+   rD   zVJEPA2Embeddings.__init__|   sY   ø€ Ý‰Œ×ÒÑÔÐàˆŒØ&ˆÔÝ 7¸ÈKÐ XÑ XÔ XˆÔàÔ0Ô<ˆÔØ Ô+ˆŒˆˆr*   rR   rS   c                 ó:  — |j         d         }|                     ddddd¦  «        }|| j        j        k     r#|                     dd| j        j        dd¦  «        }| j        j        j        j        }| 	                    |¬¦  «        }|                      |¦  «        }|S )Nr   r   rU   r   é   )Údtype)
ÚshapeÚpermuter=   rF   Úrepeatrc   rI   Úweightrf   Úto)r5   rR   Ú
num_framesÚtarget_dtypeÚ
embeddingss        r+   rY   zVJEPA2Embeddings.forward†   s¤   € Ø(Ô.¨qÔ1ˆ
ð 2×9Ò9¸!¸QÀÀ1ÀaÑHÔHÐð ˜œÔ0Ò0Ð0Ø"5×"<Ò"<¸QÀÀ4Ä;ÔC[Ð]^Ð`aÑ"bÔ"bÐàÔ,Ô1Ô8Ô>ˆØ1×4Ò4¸<Ð4ÑHÔHÐØ×*Ò*Ð+>Ñ?Ô?ˆ
àÐr*   rZ   )r!   r"   r#   r$   r   r[   rD   r%   r]   rY   r8   r9   s   @r+   r_   r_   w   s€   ø€ € € € € ðð ð,ð ,˜|ð ,¸#ð ,ð ,ð ,ð ,ð ,ð ,ð¨5¬<ð ¸E¼Lð ð ð ð ð ð ð ð r*   r_   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutc                 ó²  — t          j        ||                     dd¦  «        ¦  «        |z  }t          j                             |dt           j        ¬¦  «                             |j        ¦  «        }t          j         	                    ||| j
        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «                             ¦   «         }	|	|fS )Nr0   éþÿÿÿ)Údimrf   )ÚpÚtrainingr   rU   )r%   ÚmatmulrW   r   Ú
functionalÚsoftmaxÚfloat32rk   rf   rv   r{   Ú
contiguous)
rp   rq   rr   rs   rt   ru   rv   ÚkwargsÚattn_weightsÚattn_outputs
             r+   Úeager_attention_forwardr„   š   s·   € õ ”<  s§}¢}°R¸Ñ'<Ô'<Ñ=Ô=ÀÑG€Lõ ”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€Lõ ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r*   c                 óL  — |                       ¦   «         \  }}}}t          j        |dz  | j        | j        ¬¦  «        }||dz  z  }dd|z  z  }|                     d¦  «        |z  }|                     ¦   «         }|                     ¦   «         }	|                     dddd¦  «        }|	                     dddd¦  «        }	|  	                    dd¦  «        }
|
 
                    d¬	¦  «        \  }}t          j        | |fd¬	¦  «        }
|
                     d
¦  «        }
| |	z  |
|z  z   S )NrU   ©rf   Údeviceg       @g      ð?i'  r0   r   )r0   rU   ©ry   rx   )Úsizer%   Úarangerf   r‡   Ú	unsqueezeÚsinÚcosri   Ú	unflattenÚunbindÚstackrV   )rX   ÚposÚBÚ	num_headsÚNÚDÚomegaÚfreqÚemb_sinÚemb_cosÚyÚy1Úy2s                r+   Úrotate_queries_or_keysr�   ´   s  € ØŸš™œÑ€A€y�!�Qõ
 ŒL˜˜a™ q¤w°q´xÐ@Ñ@Ô@€EØ	ˆQ�‰WÑ€EØ�%˜‘,Ñ€EØ�=Š=˜ÑÔ˜uÑ$€Dð �hŠh‰jŒj€GØ�hŠh‰jŒj€Gà�nŠn˜Q  1 aÑ(Ô(€GØ�nŠn˜Q  1 aÑ(Ô(€Gð 	
�Š�B˜Ñ Ô €AØ�XŠX˜"ˆXÑÔ�F€BˆåŒ�b�S˜"�I 2Ð&Ñ&Ô&€AØ	�	Š	�"‰Œ€AØ�‰K˜A ™KÑ(Ð(r*   c                   ó”   ‡ — e Zd Z	 	 ddededefˆ fd„Zd„ Zd„ Zdd
„Zd„ Z		 dde
j        d	z  dee
j        e
j        f         fd„Zˆ xZS )ÚVJEPA2RopeAttentionr<   é   r=   r>   Únum_attention_headsc                 óä  •— t          ¦   «                              ¦   «          || _        || _        || _        ||z  dk    rt          d|f› d|› d�¦  «        ‚t          ||z  ¦  «        | _        | j        | j        z  | _        t          j
        || j        |j        ¬¦  «        | _        t          j
        || j        |j        ¬¦  «        | _        t          j
        || j        |j        ¬¦  «        | _        t          j
        ||¦  «        | _        |j        | _        t          j        | j        ¦  «        | _        | j        j        | j        j        z  | _        | j        j        | j        j        z  | _        t          d| j        dz  dz  z  ¦  «        | _        t          d| j        dz  dz  z  ¦  «        | _        t          d| j        dz  dz  z  ¦  «        | _        | j        dz  | _        d	| _        d S )
Nr   zThe hidden size z4 is not a multiple of the number of attention heads ú.©ÚbiasrU   r   ç      à¿F)r2   rD   r=   r>   r¡   Ú
ValueErrorr[   Úattention_head_sizeÚall_head_sizer   ÚLinearÚqkv_biasrq   rr   rs   rI   Úattention_probs_dropout_probÚdropout_probÚDropoutrv   rO   rE   Ú	grid_sizerN   rF   Ú
grid_depthÚd_dimÚh_dimÚw_dimru   Ú	is_causal)r5   r=   r>   r¡   r7   s       €r+   rD   zVJEPA2RopeAttention.__init__Ð   s×  ø€ õ 	‰Œ×ÒÑÔÐØˆŒØ&ˆÔØ#6ˆÔ ØÐ,Ñ,°Ò1Ð1Ýð0 K >ð 0ð 0Ø,ð0ð 0ð 0ñô ð õ
 $' {Ð5HÑ'HÑ#IÔ#IˆÔ Ø!Ô5¸Ô8PÑPˆÔå”Y˜{¨DÔ,>ÀVÄ_ÐUÑUÔUˆŒ
Ý”9˜[¨$Ô*<À6Ä?ÐSÑSÔSˆŒÝ”Y˜{¨DÔ,>ÀVÄ_ÐUÑUÔUˆŒ
å”I˜k¨;Ñ7Ô7ˆŒ	Ø"Ô?ˆÔÝ”z $Ô"3Ñ4Ô4ˆŒàœÔ.°$´+Ô2HÑHˆŒØœ+Ô5¸¼Ô9QÑQˆŒå˜˜tÔ7¸1Ñ<ÀÑBÑCÑDÔDˆŒ
Ý˜˜tÔ7¸1Ñ<ÀÑBÑCÑDÔDˆŒ
Ý˜˜tÔ7¸1Ñ<ÀÑBÑCÑDÔDˆŒ
àÔ/°Ñ5ˆŒØˆŒˆˆr*   c                 óD   — t          | j        | j        z  ¦  «        }||z  S rL   )r[   r¯   )r5   ÚidsÚtokens_per_frames      r+   Ú_get_frame_posz"VJEPA2RopeAttention._get_frame_posõ   s%   € Ý˜tœ~°´Ñ>Ñ?Ô?ÐØÐ&Ñ&Ð&r*   c                 óŒ   — t          | j        | j        z  ¦  «        }|                      |¦  «        }|||z  z
  }| j        }||z  S rL   )r[   r¯   r¸   )r5   r¶   r·   Ú	frame_idsÚtokens_per_rows        r+   Ú_get_height_posz#VJEPA2RopeAttention._get_height_posù   sN   € å˜tœ~°´Ñ>Ñ?Ô?ÐØ×'Ò'¨Ñ,Ô,ˆ	ØÐ$ yÑ0Ñ0ˆàœˆØ�nÑ$Ð$r*   Nc                 óŠ  — |j         }|                     d¦  «        }|�0|                     d¦  «                             d| j        d¦  «        }nt          j        ||¬¦  «        }t          | j        | j        z  ¦  «        }|  	                    |¦  «        }| j        }|  
                    |¦  «        }	|||z  z
  ||	z  z
  }
||	|
fS )Nr   ©r‡   )r‡   r‰   r‹   ri   r¡   r%   rŠ   r[   r¯   r¸   r¼   )r5   rX   Úmasksr‡   Ú
token_sizer¶   r·   rº   r»   Ú
height_idsÚ	width_idss              r+   Úget_position_idsz$VJEPA2RopeAttention.get_position_ids  sÉ   € Ø”ˆØ—V’V˜A‘Y”Yˆ
ð ÐØ—/’/ !Ñ$Ô$×+Ò+¨A¨tÔ/GÈÑKÔKˆCˆCå”,˜z°&Ð9Ñ9Ô9ˆCå˜tœ~°´Ñ>Ñ?Ô?ÐØ×'Ò'¨Ñ,Ô,ˆ	àœˆØ×)Ò)¨#Ñ.Ô.ˆ
ð Ð+¨iÑ7Ñ7¸>ÈJÑ;VÑVˆ	Ø˜* iÐ/Ð/r*   c                 óº  — |\  }}}d}t          |d||| j        z   …f         |¬¦  «        }|| j        z  }t          |d||| j        z   …f         |¬¦  «        }|| j        z  }t          |d||| j        z   …f         |¬¦  «        }	|| j        z  }|| j        k     r'|d|d …f         }
t          j        |||	|
gd¬¦  «        }nt          j        |||	gd¬¦  «        }|S )Nr   .)r‘   r0   rˆ   )r�   r±   r²   r³   r¨   r%   Úcat)r5   ÚqkÚpos_idsÚd_maskÚh_maskÚw_maskÚsÚqkdÚqkhÚqkwÚqkrs              r+   Úapply_rotary_embeddingsz+VJEPA2RopeAttention.apply_rotary_embeddings  s  € Ø!(Ñˆ�˜ØˆÝ$ R¨¨Q°°T´Z±Ð-?Ð(?Ô%@ÀfÐMÑMÔMˆØ	ˆTŒZ‰ˆÝ$ R¨¨Q°°T´Z±Ð-?Ð(?Ô%@ÀfÐMÑMÔMˆØ	ˆTŒZ‰ˆÝ$ R¨¨Q°°T´Z±Ð-?Ð(?Ô%@ÀfÐMÑMÔMˆØ	ˆTŒZ‰ˆàˆtÔ'Ò'Ð'Ø�S˜!˜"˜"�W”+ˆCÝ”˜C  c¨3Ð/°RÐ8Ñ8Ô8ˆBˆBå”˜C  c˜?°Ð3Ñ3Ô3ˆBØˆ	r*   Úposition_maskrS   c           
      óp  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|                      ||¬¦  «        }|                      ||¦  «        }|                      ||¦  «        }t          j
        | j        j        t          ¦  «        }	 |	| |||d | j        | j        | j        sdn| j        ¬¦  «        \  }
}|
                     ¦   «         d d…         | j        fz   }|                      |
                     |¦  «        ¦  «        }
|
|fS )Nr0   r   rU   )r¿   ro   ©r´   ru   rv   rx   )rg   r¨   rq   ÚviewrW   rr   rs   rÃ   rÐ   r   Úget_interfacer=   Ú_attn_implementationr„   r´   ru   r{   r­   r‰   r©   rI   Úreshape)r5   r   rÑ   Úinput_shapeÚhidden_shapeÚquery_layerÚ	key_layerÚvalue_layerrÇ   Úattention_interfaceÚcontext_layerÚattention_probsÚnew_context_layer_shapes                r+   rY   zVJEPA2RopeAttention.forward(  s¹  € ð
 $Ô)¨#¨2¨#Ô.ˆØC˜ÐC bÐC¨$Ô*BÐCÐCˆØ—j’j Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆØ—H’H˜]Ñ+Ô+×0Ò0°Ñ>Ô>×HÒHÈÈAÑNÔNˆ	Ø—j’j Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆà×'Ò'¨¸]Ð'ÑKÔKˆØ×0Ò0°¸GÑDÔDˆ	Ø×2Ò2°;ÀÑHÔHˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð *=Ð)<ØØØØØØ”nØ”LØ#œ}ÐC�C�C°$Ô2Cð	*
ñ 	*
ô 	*
Ñ&ˆ�ð #0×"4Ò"4Ñ"6Ô"6°s¸°sÔ";¸tÔ?QÐ>SÑ"SÐØŸ	š	 -×"7Ò"7Ð8OÑ"PÔ"PÑQÔQˆà˜oÐ-Ð-r*   )r<   r    rL   )r!   r"   r#   r   r[   rD   r¸   r¼   rÃ   rÐ   r%   r]   r(   rY   r8   r9   s   @r+   rŸ   rŸ   Ï   sï   ø€ € € € € ð  Ø#%ð	#ð #àð#ð ð#ð !ð	#ð #ð #ð #ð #ð #ðJ'ð 'ð 'ð%ð %ð %ð0ð 0ð 0ð 0ð*ð ð ð( .2ð!.ð !.ð ”| dÑ*ð!.ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð	!.ð !.ð !.ð !.ð !.ð !.ð !.ð !.r*   rŸ   c                   óR   ‡ — e Zd Zd
dededefˆ fd„Zdej        dej        fd	„Z	ˆ xZ
S )Ú	VJEPA2MLPr<   ç      @r=   r>   Ú	mlp_ratioc                 ó  •— t          ¦   «                              ¦   «          |x}}t          ||z  ¦  «        }t          j        ||d¬¦  «        | _        t          |j                 | _        t          j        ||d¬¦  «        | _	        d S ©NTr¤   )
r2   rD   r[   r   rª   Úfc1r   Ú
hidden_actÚ
activationÚfc2)r5   r=   r>   rä   Úin_featuresÚout_featuresÚhidden_featuresr7   s          €r+   rD   zVJEPA2MLP.__init__M  sx   ø€ Ý‰Œ×ÒÑÔÐØ%0Ð0ˆ�lÝ˜k¨IÑ5Ñ6Ô6ˆÝ”9˜[¨/ÀÐEÑEÔEˆŒÝ  Ô!2Ô3ˆŒÝ”9˜_¨lÀÐFÑFÔFˆŒˆˆr*   Úhidden_staterS   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rL   )rç   ré   rê   )r5   rî   s     r+   rY   zVJEPA2MLP.forwardU  s;   € Ø—x’x Ñ-Ô-ˆØ—’ |Ñ4Ô4ˆØ—x’x Ñ-Ô-ˆØÐr*   )r<   rã   )r!   r"   r#   r   r[   ÚfloatrD   r%   r]   rY   r8   r9   s   @r+   râ   râ   L  sˆ   ø€ € € € € ðGð G˜|ð G¸#ð GÐQVð Gð Gð Gð Gð Gð Gð E¤Lð °U´\ð ð ð ð ð ð ð ð r*   râ   c                   ó^   ‡ — e Zd ZdZd
deddfˆ fd„Zdej        dej        fd„Zde	fd	„Z
ˆ xZS )ÚVJEPA2DropPathzÏStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    ro   Ú	drop_probrS   Nc                 óV   •— t          ¦   «                              ¦   «          || _        d S rL   )r2   rD   ró   )r5   ró   r7   s     €r+   rD   zVJEPA2DropPath.__init__d  s$   ø€ Ý‰Œ×ÒÑÔÐØ"ˆŒˆˆr*   r   c                 ó  — | j         dk    s| j        s|S d| j         z
  }|j        d         fd|j        dz
  z  z   }t	          j        ||j        |j        ¬¦  «        }t	          j        ||z   ¦  «        }| 	                    |¦  «        |z  S )Nro   r   r   ©r   r†   )
ró   r{   rg   Úndimr%   Úrandrf   r‡   ÚfloorÚdiv)r5   r   Ú	keep_probrg   Úrandom_tensors        r+   rY   zVJEPA2DropPath.forwardh  s“   € ØŒ>˜SÒ Ð ¨¬Ð Ø Ð Ø˜œÑ&ˆ	ØÔ$ QÔ'Ð)¨D°MÔ4FÈÑ4JÑ,KÑKˆÝœ
 5°Ô0CÈMÔL`ÐaÑaÔaˆÝœ M°IÑ$=Ñ>Ô>ˆØ× Ò  Ñ+Ô+¨mÑ;Ð;r*   c                 ó   — d| j         › �S )Nzp=)ró   ©r5   s    r+   Ú
extra_reprzVJEPA2DropPath.extra_reprq  s   € Ø$�D”NÐ$Ð$Ð$r*   ©ro   )r!   r"   r#   r$   rð   rD   r%   r]   rY   Ústrrÿ   r8   r9   s   @r+   rò   rò   ]  s›   ø€ € € € € ðð ð#ð # %ð #°$ð #ð #ð #ð #ð #ð #ð< U¤\ð <°e´lð <ð <ð <ð <ð%˜Cð %ð %ð %ð %ð %ð %ð %ð %r*   rò   c                   óž   ‡ — e Zd ZdZ	 	 	 	 ddededed	ed
ef
ˆ fd„Z	 ddej	        dej	        dz  de
e         deej	        df         fd„Zˆ xZS )ÚVJEPA2LayerzCThis corresponds to the Block class in the original implementation.ro   r<   r    rã   r=   Údrop_path_rater>   r¡   rä   c                 ó¾  •— t          ¦   «                              ¦   «          || _        || _        || _        || _        t          j        ||j        ¬¦  «        | _	        t          |||¦  «        | _        |j        dk    rt          |¦  «        nt          j        ¦   «         | _        t          j        ||j        ¬¦  «        | _        t#          |||¬¦  «        | _        d S )N©Úepsro   )r>   rä   )r2   rD   r=   r>   r¡   rä   r   Ú	LayerNormÚlayer_norm_epsÚnorm1rŸ   Ú	attentionr  rò   ÚIdentityÚ	drop_pathÚnorm2râ   Úmlp)r5   r=   r  r>   r¡   rä   r7   s         €r+   rD   zVJEPA2Layer.__init__x  sÅ   ø€ õ 	‰Œ×ÒÑÔÐØˆŒØ&ˆÔØ#6ˆÔ Ø"ˆŒå”\ +°6Ô3HÐIÑIÔIˆŒ
Ý,¨V°[ÐBUÑVÔVˆŒØ;AÔ;PÐSVÒ;VÐ;V�¨Ñ7Ô7Ð7Õ\^Ô\gÑ\iÔ\iˆŒÝ”\ +°6Ô3HÐIÑIÔIˆŒ
Ý˜V°È	ÐRÑRÔRˆŒˆˆr*   Nr   rÑ   r�   rS   .c                 ó$  — |}|                       |¦  «        }|                      ||¬¦  «        \  }}|                      |¦  «        |z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        |z   }||fS )N)rÑ   )r
  r  r  r  r  )r5   r   rÑ   r�   ÚresidualÚattention_outputr‚   s          r+   rY   zVJEPA2Layer.forwardŒ  s£   € ð !ˆØŸ
š
 =Ñ1Ô1ˆØ)-¯ªØØ'ð *8ñ *
ô *
Ñ&Ð˜,ð ŸšÐ'7Ñ8Ô8¸8ÑCˆð !ˆØŸ
š
 =Ñ1Ô1ˆØŸš Ñ/Ô/ˆØŸš }Ñ5Ô5¸Ñ@ˆð ˜lÐ*Ð*r*   )ro   r<   r    rã   rL   )r!   r"   r#   r$   r   rð   r[   rD   r%   r]   r   r   r(   rY   r8   r9   s   @r+   r  r  u  sñ   ø€ € € € € ØMÐMð
 !$ØØ#%ØðSð SàðSð ðSð ð	Sð
 !ðSð ðSð Sð Sð Sð Sð Sð. .2ð+ð +à”|ð+ð ”| dÑ*ð+ð Ð+Ô,ð	+ð
 
ˆuŒ|˜SÐ Ô	!ð+ð +ð +ð +ð +ð +ð +ð +r*   r  c                   óX   ‡ — e Zd Zdefˆ fd„Z	 ddej        dz  dee         de	fd„Z
ˆ xZS )	ÚVJEPA2Encoderr=   c                 ó˜  •‡‡— t          ¦   «                              ¦   «          ‰| _        t          ‰‰j        ¬¦  «        | _        ˆfd„t          ‰j        ¦  «        D ¦   «         Št          j	        ˆˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _
        t          j        ‰j        ‰j        ¬¦  «        | _        d| _        d S )Nrb   c                 óT   •— g | ]$}‰j         d k    r‰j        |z  ‰j         d z
  z  nd‘Œ%S ©r   ro   )Únum_hidden_layersr  ©Ú.0Úir=   s     €r+   ú
<listcomp>z*VJEPA2Encoder.__init__.<locals>.<listcomp>«  sT   ø€ ð 
ð 
ð 
àð LRÔKcÐfgÒKgÐKgˆVÔ" QÑ&¨&Ô*BÀQÑ*FÑGÐGÐmpð
ð 
ð 
r*   c           	      ób   •— g | ]+}t          ‰‰|         ‰j        ‰j        ‰j        ¬ ¦  «        ‘Œ,S ©)r  r>   r¡   rä   )r  r>   r¡   rä   ©r  r  r=   Údrop_path_ratess     €€r+   r  z*VJEPA2Encoder.__init__.<locals>.<listcomp>°  sW   ø€ ð 	ð 	ð 	ð õ ØØ#2°1Ô#5Ø &Ô 2Ø(.Ô(BØ$Ô.ðñ ô ð	ð 	ð 	r*   r  F)r2   rD   r=   r_   r>   rn   Úranger  r   Ú
ModuleListÚlayerr  r	  Ú	layernormÚgradient_checkpointing©r5   r=   r   r7   s    `@€r+   rD   zVJEPA2Encoder.__init__¦  så   øøø€ Ý‰Œ×ÒÑÔÐØˆŒå*¨6¸vÔ?QÐRÑRÔRˆŒð
ð 
ð 
ð 
å˜6Ô3Ñ4Ô4ð
ñ 
ô 
ˆõ ”]ð	ð 	ð 	ð 	ð 	õ ˜vÔ7Ñ8Ô8ð	ñ 	ô 	ñ
ô 
ˆŒ
õ œ fÔ&8¸fÔ>SÐTÑTÔTˆŒØ&+ˆÔ#Ð#Ð#r*   NrR   r�   rS   c                 óÎ   — |                       |¦  «        }t          | j        ¦  «        D ]\  }} ||d fi |¤Ž}|d         }Œ|                      |¦  «        }t	          |¬¦  «        S )Nr   ©r   )rn   Ú	enumerater#  r$  r
   )r5   rR   r�   r   r  Úlayer_moduleÚlayer_outputss          r+   rY   zVJEPA2Encoder.forward¾  sƒ   € ð
 ŸšÐ(;Ñ<Ô<ˆå(¨¬Ñ4Ô4ð 	-ð 	-‰OˆAˆ|Ø(˜L¨¸ÐGÐGÀÐGÐGˆMØ)¨!Ô,ˆMˆMàŸš }Ñ5Ô5ˆåØ+ð
ñ 
ô 
ð 	
r*   rL   )r!   r"   r#   r   rD   r%   r]   r   r   r
   rY   r8   r9   s   @r+   r  r  ¥  s‹   ø€ € € € € ð,˜|ð ,ð ,ð ,ð ,ð ,ð ,ð4 48ð
ð 
à"œ\¨DÑ0ð
ð Ð+Ô,ð
ð 
ð	
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r*   r  Útensorr¿   rS   c                 ó   — g }|D ]t}|                      | j        ¦  «        }|                     d¦  «                             dd|                      d¦  «        ¦  «        }|t          j        | d|¬¦  «        gz  }Œut          j        |d¬¦  «        S )zþ
    Args:
        tensor (`torch.Tensor`):
            Tensor of shape [batch_size, num_patches, feature_dim]
        masks (`List[torch.Tensor]`):
            List of tensors of shape [batch_size, num_patches] containing indices of patches to keep
    r0   r   ©ry   Úindexr   rˆ   )rk   r‡   r‹   ri   r‰   r%   ÚgatherrÅ   )r,  r¿   Úall_masked_tensorsÚmaskÚ	mask_keeps        r+   Úapply_masksr4  Ð  s“   € ð ÐØð Mð MˆØ�wŠw�v”}Ñ%Ô%ˆØ—N’N 2Ñ&Ô&×-Ò-¨a°°F·K²KÀ±O´OÑDÔDˆ	Ø�uœ|¨F¸ÀÐKÑKÔKÐLÑLÐÐåŒ9Ð'¨QÐ/Ñ/Ô/Ð/r*   c                   ó¸   ‡ — e Zd ZdZdefˆ fd„Zed„ ¦   «         Z	 ddej	        de
ej	                 de
ej	                 d	ed
eej	        ej	        f         f
d„Zˆ xZS )ÚVJEPA2PredictorEmbeddingsr`   r=   c                 ór  •— t          ¦   «                              ¦   «          || _        t          j        |j        |j        ¦  «        | _        d| _        |j	        | _
        |j        | _        t          j        t          j        | j        dd|j        ¦  «        ¦  «        | _        |j        | _        || _        d S )Nr   r   )r2   rD   r=   r   rª   r>   Úpred_hidden_sizeÚpredictor_embeddingsÚnum_mask_tokensÚpred_zero_init_mask_tokensÚzero_init_mask_tokensÚpred_num_mask_tokensÚ	Parameterr%   ÚzerosÚmask_tokensrE   ©r5   r=   r7   s     €r+   rD   z"VJEPA2PredictorEmbeddings.__init__æ  s–   ø€ Ý‰Œ×ÒÑÔÐàˆŒÝ$&¤I¨fÔ.@À&ÔBYÑ$ZÔ$ZˆÔ!Ø ˆÔØ%+Ô%FˆÔ"Ø%Ô:ˆÔÝœ<­¬°DÔ4HÈ!ÈQÐPVÔPgÑ(hÔ(hÑiÔiˆÔà Ô+ˆŒØˆŒˆˆr*   c                 ó´   — | j         dk    r/| j         | j        z  | j        | j        z  z  | j        | j        z  z  S | j        | j        z  | j        | j        z  z  S )Nr   rM   rP   s    r+   rQ   z%VJEPA2PredictorEmbeddings.num_patchesó  sm   € àÔ! AÒ%Ð%àÔ'¨6Ô+>Ñ>ØÔ# vÔ'8Ñ8ñ:àÔ# vÔ'8Ñ8ñ:ðð Ô$¨Ô(9Ñ9¸fÔ>NÐRXÔRcÑ>cÑdÐdr*   r   r   Úcontext_maskÚtarget_maskÚ
mask_indexrS   c                 ó  — |                      d¦  «        }|                      |¦  «        }|| j        z  }| j        |         }|d                              ¦   «         dz   }|                     ||d¦  «        }t          ||¦  «        }|                     t          |¦  «        dd¦  «        }t          j	        ||gd¬¦  «        }	t          j	        |d¬¦  «        }
t          j	        |d¬¦  «        }t          j	        |
|gd¬¦  «        }|	|fS )zø
        hidden_states : encoder outputs (context)
        context_mask: tokens of the context (outputs from the encoder)
        target_mask: tokens to predict
        mask_index: index of the target mask to choose (useful for multiclip?)
        r   r   rˆ   )
r‰   r9  r:  r@  Úmaxri   r4  Úlenr%   rÅ   )r5   r   rC  rD  rE  r’   ÚcontextÚtargetÚmax_patch_numrn   ÚcmÚtmr¿   s                r+   rY   z!VJEPA2PredictorEmbeddings.forwardþ  s  € ð ×Ò˜qÑ!Ô!ˆØ×+Ò+¨MÑ:Ô:ˆð   $Ô"6Ñ6ˆ
ØÔ! *Ô-ˆð $ Aœ×*Ò*Ñ,Ô,¨qÑ0ˆØ—’˜q -°Ñ3Ô3ˆÝ˜V [Ñ1Ô1ˆð —.’.¥ \Ñ!2Ô!2°A°qÑ9Ô9ˆÝ”Y ¨Ð0°aÐ8Ñ8Ô8ˆ
õ ŒY�|¨Ð+Ñ+Ô+ˆÝŒY�{¨Ð*Ñ*Ô*ˆÝ”	˜2˜r˜(¨Ð*Ñ*Ô*ˆà˜5Ð Ð r*   rö   )r!   r"   r#   r$   r   rD   r\   rQ   r%   r]   r1   r[   r(   rY   r8   r9   s   @r+   r6  r6  á  sØ   ø€ € € € € ðð ð˜|ð ð ð ð ð ð ð ðeð eñ „\ðeð ð&!ð &!à”|ð&!ð ˜5œ<Ô(ð&!ð ˜%œ,Ô'ð	&!ð
 ð&!ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð&!ð &!ð &!ð &!ð &!ð &!ð &!ð &!r*   r6  c            
       óŽ   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zdej        de	ej                 de	ej                 de
e         d	ef
d
„Zˆ xZS )ÚVJEPA2Predictorr=   c                 óÖ  •‡‡— t          ¦   «                              ¦   «          ‰| _        d| _        t	          ‰¦  «        | _        ˆfd„t          ‰j        ¦  «        D ¦   «         Št          j	        ˆˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _
        t          j        ‰j        ‰j        ¬¦  «        | _        t          j        ‰j        ‰j        d¬¦  «        | _        d S )NFc                 óT   •— g | ]$}‰j         d k    r‰j        |z  ‰j         d z
  z  nd‘Œ%S r  )Úpred_num_hidden_layersr  r  s     €r+   r  z,VJEPA2Predictor.__init__.<locals>.<listcomp>-  sV   ø€ ð 
ð 
ð 
ð ð Ô0°1Ò4Ð4ð Ô%¨Ñ)¨VÔ-JÈQÑ-NÑOÐOàð	
ð 
ð 
r*   c           	      ób   •— g | ]+}t          ‰‰|         ‰j        ‰j        ‰j        ¬ ¦  «        ‘Œ,S r  )r  r8  Úpred_num_attention_headsÚpred_mlp_ratior  s     €€r+   r  z,VJEPA2Predictor.__init__.<locals>.<listcomp>6  sW   ø€ ð 	ð 	ð 	ð õ ØØ#2°1Ô#5Ø &Ô 7Ø(.Ô(GØ$Ô3ðñ ô ð	ð 	ð 	r*   r  Tr¤   )r2   rD   r=   r%  r6  rn   r!  rR  r   r"  r#  r  r8  r	  r$  rª   r>   rI   r&  s    `@€r+   rD   zVJEPA2Predictor.__init__(  sü   øøø€ Ý‰Œ×ÒÑÔÐØˆŒØ&+ˆÔ#Ý3°FÑ;Ô;ˆŒð
ð 
ð 
ð 
õ ˜6Ô8Ñ9Ô9ð
ñ 
ô 
ˆõ ”]ð	ð 	ð 	ð 	ð 	õ ˜vÔ<Ñ=Ô=ð	ñ 	ô 	ñ
ô 
ˆŒ
õ œ fÔ&=À6ÔCXÐYÑYÔYˆŒÝ”I˜fÔ5°vÔ7IÐPTÐUÑUÔUˆŒ	ˆ	ˆ	r*   c                 óH  — |                      |j        ¦  «        }t          j        |d|¬¦  «        }|                      |j        ¦  «        }|                     d¦  «                             dd|                     d¦  «        ¦  «        }t          j        |d|¬¦  «        }||fS )Nr   r.  r0   )rk   r‡   r%   r0  r‹   Úexpandr‰   )r5   r   Úposition_masksÚargsortÚhidden_states_argsorts        r+   Úsort_tokenszVJEPA2Predictor.sort_tokensD  s™   € à—*’*˜^Ô2Ñ3Ô3ˆÝœ n¸!À7ÐKÑKÔKˆð —*’*˜]Ô1Ñ2Ô2ˆØ '× 1Ò 1°"Ñ 5Ô 5× <Ò <¸RÀÀ]×EWÒEWÐXZÑE[ÔE[Ñ \Ô \ÐÝœ ]¸ÐAVÐWÑWÔWˆà˜nÐ,Ð,r*   c                 ó  — |                      |j        ¦  «        }t          j        |d¬¦  «        }|                     d¦  «                             dd|                     d¦  «        ¦  «        }t          j        |d|¬¦  «        }|S )Nr   rˆ   r0   r.  )rk   r‡   r%   rY  r‹   rW  r‰   r0  )r5   r   rY  Úreverse_argsorts       r+   Úunsort_tokenszVJEPA2Predictor.unsort_tokensP  sz   € Ø—*’*˜]Ô1Ñ2Ô2ˆÝœ-¨°QÐ7Ñ7Ô7ˆØ)×3Ò3°BÑ7Ô7×>Ò>¸rÀ2À}×GYÒGYÐZ\ÑG]ÔG]Ñ^Ô^ˆÝœ ]¸ÀÐQÑQÔQˆØÐr*   Úencoder_hidden_statesrC  rD  r�   rS   c                 óà  — t          ||¦  «        }|j        \  }}}|                      |||¦  «        \  }}	t          j        |	d¬¦  «        }
|                      ||	|
¦  «        \  }}	t          | j        ¦  «        D ]\  }} |||	fi |¤Ž}|d         }Œ|                      |¦  «        }|  	                    ||
¦  «        }|d d …|d …f         }|  
                    |¦  «        }t          |¬¦  «        S )Nr   rˆ   r   r(  )r4  rg   rn   r%   rY  r[  r)  r#  r$  r^  rI   r
   )r5   r_  rC  rD  r�   Ú_ÚN_ctxtr•   r   rX  rY  r  r*  r+  s                 r+   rY   zVJEPA2Predictor.forwardW  s  € õ !,Ð,AÀ<Ñ PÔ PÐØ,Ô2‰ˆˆ6�1Ø(,¯ªÐ8MÈ|Ð]hÑ(iÔ(iÑ%ˆ�~õ ”- °AÐ6Ñ6Ô6ˆØ(,×(8Ò(8¸ÈÐX_Ñ(`Ô(`Ñ%ˆ�~å(¨¬Ñ4Ô4ð 	-ð 	-‰OˆAˆ|Ø(˜L¨¸ÐQÐQÈ&ÐQÐQˆMØ)¨!Ô,ˆMˆMàŸš }Ñ5Ô5ˆà×*Ò*¨=¸'ÑBÔBˆØ% a a a¨¨¨ jÔ1ˆàŸ	š	 -Ñ0Ô0ˆåØ+ð
ñ 
ô 
ð 	
r*   )r!   r"   r#   r   rD   r[  r^  r%   r]   r1   r   r   r
   rY   r8   r9   s   @r+   rO  rO  '  sÃ   ø€ € € € € ðV˜|ð Vð Vð Vð Vð Vð Vð8
-ð 
-ð 
-ðð ð ð
à$œ|ð
ð ˜5œ<Ô(ð
ð ˜%œ,Ô'ð	
ð
 Ð+Ô,ð
ð 
ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r*   rO  c            	       ó~   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej        dej        dz  deej        ej        f         fd„Z	ˆ xZ
S )
ÚVJEPA2PoolerSelfAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr=   c                 ó‚  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        | j        | j        z  | _        | j        | j        z  | j        k    r t          d| j        › d| j        › d�¦  «        ‚| j        dz  | _	        |j
        | _        d| _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        d S ©Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).r¦   F)r2   rD   r=   r>   Ú	embed_dimr¡   r“   Úhead_dimr§   ÚscaleÚattention_dropoutrv   r´   r   rª   Úk_projÚv_projÚq_projÚout_projrA  s     €r+   rD   z"VJEPA2PoolerSelfAttention.__init__{  s  ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØÔ3ˆŒØœ¨$¬.Ñ8ˆŒØŒ=˜4œ>Ñ)¨T¬^Ò;Ð;Ýð'ÈdÌnð 'ð 'Ø”Nð'ð 'ð 'ñô ð ð ”] DÑ(ˆŒ
ØÔ/ˆŒØˆŒå”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝœ	 $¤.°$´.ÑAÔAˆŒˆˆr*   Nr   rt   rS   c           
      ó¼  — |j         dd…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }t          j        | j	        j
        t          ¦  «        } || ||||| j        | j        | j        sdn| j        ¬¦  «        \  }	}
 |	j        g |¢d‘R Ž                      ¦   «         }	|                      |	¦  «        }	|	|
fS )ú#Input shape: Batch x Time x ChannelNr0   r   rU   ro   rÓ   )rg   rh  rm  rÔ   rW   rk  rl  r   rÕ   r=   rÖ   r„   r´   ri  r{   rv   r×   r€   rn  )r5   r   rt   rØ   rÙ   ÚqueriesÚkeysÚvaluesrÝ   rƒ   r‚   s              r+   rY   z!VJEPA2PoolerSelfAttention.forward�  sg  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆØ—+’+˜mÑ,Ô,×1Ò1°,Ñ?Ô?×IÒIÈ!ÈQÑOÔOˆØ�{Š{˜=Ñ)Ô)×.Ò.¨|Ñ<Ô<×FÒFÀqÈ!ÑLÔLˆØ—’˜]Ñ+Ô+×0Ò0°Ñ>Ô>×HÒHÈÈAÑNÔNˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØØ”nØ”JØ#œ}Ð>�C�C°$´,ð	%
ñ 	%
ô 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—m’m KÑ0Ô0ˆà˜LÐ(Ð(r*   rL   ©r!   r"   r#   r$   r   rD   r%   r]   r(   rY   r8   r9   s   @r+   rd  rd  x  s¢   ø€ € € € € ØGÐGðB˜|ð Bð Bð Bð Bð Bð Bð. /3ð)ð )à”|ð)ð œ tÑ+ð)ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð	)ð )ð )ð )ð )ð )ð )ð )r*   rd  c                   óš   ‡ — e Zd ZdZdefˆ fd„Z	 ddej        dej        dej        dej        dz  d	eej        ej        f         f
d
„Z	ˆ xZ
S )ÚVJEPA2PoolerCrossAttentionz_It's different from other cross-attention layers, doesn't have output projection layer (o_proj)r=   c                 ó:  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        | j        | j        z  | _        | j        | j        z  | j        k    r t          d| j        › d| j        › d�¦  «        ‚| j        dz  | _	        |j
        | _        d| _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        d S rf  )r2   rD   r=   r>   rg  r¡   r“   rh  r§   ri  rj  rv   r´   r   rª   rk  rl  rm  rA  s     €r+   rD   z#VJEPA2PoolerCrossAttention.__init__µ  sý   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØÔ3ˆŒØœ¨$¬.Ñ8ˆŒØŒ=˜4œ>Ñ)¨T¬^Ò;Ð;Ýð'ÈdÌnð 'ð 'Ø”Nð'ð 'ð 'ñô ð ð ”] DÑ(ˆŒ
ØÔ/ˆŒØˆŒå”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒˆˆr*   Nrq  rr  rs  rt   rS   c           
      óò  — |j         \  }}}|j         d         }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                     ||| j        | j        ¦  «                             dd¦  «        }|                     ||| j        | j        ¦  «                             dd¦  «        }|                     ||| j        | j        ¦  «                             dd¦  «        }t          j	        | j
        j        t          ¦  «        }	 |	| ||||| j        | j        | j        sdn| j        ¬¦  «        \  }
}|
                     |||¦  «                             ¦   «         }
|
|fS )rp  r   rU   ro   rÓ   )rg   rm  rk  rl  rÔ   r“   rh  rW   r   rÕ   r=   rÖ   r„   r´   ri  r{   rv   r×   r€   )r5   rq  rr  rs  rt   Ú
batch_sizeÚq_seq_lengthrg  Úkv_seq_lengthrÝ   rƒ   r‚   s               r+   rY   z"VJEPA2PoolerCrossAttention.forwardÈ  sl  € ð /6¬mÑ+ˆ
�L )Øœ
 1œˆà—+’+˜gÑ&Ô&ˆØ�{Š{˜4Ñ Ô ˆØ—’˜VÑ$Ô$ˆà—,’,˜z¨<¸¼ÈÌÑWÔW×aÒaÐbcÐefÑgÔgˆØ�yŠy˜ ]°D´NÀDÄMÑRÔR×\Ò\Ð]^Ð`aÑbÔbˆØ—’˜Z¨¸¼ÈÌÑVÔV×`Ò`ÐabÐdeÑfÔfˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØØ”nØ”JØ#œ}Ð>�C�C°$´,ð	%
ñ 	%
ô 	%
Ñ!ˆ�\ð "×)Ò)¨*°lÀIÑNÔN×YÒYÑ[Ô[ˆà˜LÐ(Ð(r*   rL   rt  r9   s   @r+   rv  rv  °  sº   ø€ € € € € ØiÐið@˜|ð @ð @ð @ð @ð @ð @ð0 /3ð%)ð %)à”ð%)ð Œlð%)ð ”ð	%)ð
 œ tÑ+ð%)ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð%)ð %)ð %)ð %)ð %)ð %)ð %)ð %)r*   rv  c                   óp   ‡ — e Zd Zdefˆ fd„Zdej        dej        deej        ej        f         fd„Zˆ xZ	S )ÚVJEPA2PoolerSelfAttentionLayerr=   c                 ó:  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¬¦  «        | _        t          |¦  «        | _        t          j        |j        |j        ¬¦  «        | _	        t          ||j        ¬¦  «        | _        d S ©Nr  rb   )r2   rD   r   r  r>   r	  Úlayer_norm1rd  Ú	self_attnÚlayer_norm2râ   r  rA  s     €r+   rD   z'VJEPA2PoolerSelfAttentionLayer.__init__ò  s}   ø€ Ý‰Œ×ÒÑÔÐÝœ<¨Ô(:ÀÔ@UÐVÑVÔVˆÔÝ2°6Ñ:Ô:ˆŒÝœ<¨Ô(:ÀÔ@UÐVÑVÔVˆÔÝ˜V°Ô1CÐDÑDÔDˆŒˆˆr*   r   rt   rS   c                 óØ   — |}|                       |¦  «        }|                      ||¬¦  «        \  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }||fS )aR  
        Args:
            hidden_states (`torch.FloatTensor`):
                Input to the layer of shape `(batch, seq_len, embed_dim)`.
            attention_mask (`torch.FloatTensor`):
                Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values.
        )r   rt   )r€  r�  r‚  r  )r5   r   rt   r  r‚   s        r+   rY   z&VJEPA2PoolerSelfAttentionLayer.forwardù  s‰   € ð !ˆØ×(Ò(¨Ñ7Ô7ˆØ&*§n¢nØ'Ø)ð '5ñ '
ô '
Ñ#ˆ�|ð ! =Ñ0ˆà ˆØ×(Ò(¨Ñ7Ô7ˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆà˜lÐ*Ð*r*   ©
r!   r"   r#   r   rD   r%   r]   r(   rY   r8   r9   s   @r+   r}  r}  ñ  s�   ø€ € € € € ðE˜|ð Eð Eð Eð Eð Eð Eð+à”|ð+ð œð+ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð	+ð +ð +ð +ð +ð +ð +ð +r*   r}  c                   óˆ   ‡ — e Zd Zdefˆ fd„Z	 d	dej        dej        dej        dz  deej        ej        f         fd„Zˆ xZ	S )
ÚVJEPA2PoolerCrossAttentionLayerr=   c                 ó:  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¬¦  «        | _        t          |¦  «        | _        t          j        |j        |j        ¬¦  «        | _	        t          ||j        ¬¦  «        | _        d S r  )r2   rD   r   r  r>   r	  r€  rv  Ú
cross_attnr‚  râ   r  rA  s     €r+   rD   z(VJEPA2PoolerCrossAttentionLayer.__init__  s}   ø€ Ý‰Œ×ÒÑÔÐÝœ<¨Ô(:ÀÔ@UÐVÑVÔVˆÔÝ4°VÑ<Ô<ˆŒÝœ<¨Ô(:ÀÔ@UÐVÑVÔVˆÔÝ˜V°Ô1CÐDÑDÔDˆŒˆˆr*   Nrq  rî   rt   rS   c                 óÞ   — |}|                       |¦  «        }|                      ||||¬¦  «        ^}}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|g|¢R S )N©rt   )r€  rˆ  r‚  r  )r5   rq  rî   rt   r  r‚   s         r+   rY   z'VJEPA2PoolerCrossAttentionLayer.forward  s—   € ð ˆØ×'Ò'¨Ñ5Ô5ˆØ&*§o¢oØØØØ)ð	 '6ñ '
ô '
Ð#ˆ�|ð   ,Ñ.ˆð  ˆØ×'Ò'¨Ñ5Ô5ˆØ—x’x Ñ-Ô-ˆØ ,Ñ.ˆàÐ*˜lÐ*Ð*Ð*r*   rL   r„  r9   s   @r+   r†  r†    s¨   ø€ € € € € ðE˜|ð Eð Eð Eð Eð Eð Eð /3ð	+ð +à”ð+ð ”lð+ð œ tÑ+ð	+ð
 
ˆuŒ|˜Uœ\Ð)Ô	*ð+ð +ð +ð +ð +ð +ð +ð +r*   r†  c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚVJEPA2AttentivePoolerzAttentive Poolerr=   c                 óD  •‡— t          ¦   «                              ¦   «          t          j        t	          j        dd‰j        ¦  «        ¦  «        | _        t          ‰¦  «        | _	        t          j
        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d S )Nr   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r)   )r}  )r  ra  r=   s     €r+   r  z2VJEPA2AttentivePooler.__init__.<locals>.<listcomp>?  s"   ø€ Ð]Ð]Ð]¸Õ+¨FÑ3Ô3Ð]Ð]Ð]r*   )r2   rD   r   r>  r%   r?  r>   Úquery_tokensr†  Úcross_attention_layerr"  r!  Únum_pooler_layersÚself_attention_layersrA  s    `€r+   rD   zVJEPA2AttentivePooler.__init__:  s‰   øø€ Ý‰Œ×ÒÑÔÐÝœL­¬°Q¸¸6Ô;MÑ)NÔ)NÑOÔOˆÔÝ%DÀVÑ%LÔ%LˆÔ"Ý%'¤]Ø]Ð]Ð]Ð]½UÀ6ÔC[Ñ=\Ô=\Ð]Ñ]Ô]ñ&
ô &
ˆÔ"Ð"Ð"r*   rî   rS   c                 óì   — | j         D ]} ||d ¬¦  «        d         }Œ| j                             |j        d         dd¦  «        }|                      ||¦  «        d         }|                     d¦  «        S )NrŠ  r   r   )r’  r�  ri   rg   r�  Úsqueeze)r5   rî   r#  rq  s       r+   rY   zVJEPA2AttentivePooler.forwardB  s�   € ØÔ/ð 	Gð 	GˆEØ ˜5 ¸dÐCÑCÔCÀAÔFˆLˆLØÔ#×*Ò*¨<Ô+=¸aÔ+@À!ÀQÑGÔGˆØ×1Ò1°'¸<ÑHÔHÈÔKˆØ×#Ò# AÑ&Ô&Ð&r*   )
r!   r"   r#   r$   r   rD   r%   r]   rY   r8   r9   s   @r+   rŒ  rŒ  7  sp   ø€ € € € € ØÐð
˜|ð 
ð 
ð 
ð 
ð 
ð 
ð' E¤Lð '°U´\ð 'ð 'ð 'ð 'ð 'ð 'ð 'ð 'r*   rŒ  c                   ó¬   ‡ — e Zd ZU eed<   dZdZdZdZg d¢Z	dZ
dZ eed¬¦  «         eed	d¬
¦  «        dœZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚVJEPA2PreTrainedModelr=   Úvjepa2rR   ÚvideoT)r  r}  r†  r6  zencoder.layer)Ú
layer_namer   )r/  r™  )r   r   c                 ó¶  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          ¦  «        rÑt          j        |j        |¬¦  «         t          |j
        d¦  «        D ]W\  }}||dz  z  }t          j        |j        j        j        |¬¦  «         t          j        |j        j        j        |¬¦  «         ŒX|t!          |j
        ¦  «        dz   dz  z  }t          j        |j        j        j        j        |¬¦  «         dS t	          |t$          ¦  «        r?|j        rt          j        |j        ¦  «         dS t          j        |j        |¬¦  «         dS t	          |t,          j        t,          j        t,          j        f¦  «        r=t          j        |j        |¬¦  «         |j        �t          j        |j        ¦  «         dS dS dS )zInitialize the weights)Ústdr   g      à?N)r2   Ú_init_weightsr=   Úinitializer_ranger4   rŒ  ÚinitÚtrunc_normal_r�  r)  r’  r�  rn  rj   r  rê   rH  r�  r6  r<  Úzeros_r@  r   rª   ÚConv2drG   r¥   )r5   rp   Úinit_stdr  r#  r›  r7   s         €r+   rœ  z#VJEPA2PreTrainedModel._init_weights^  sÇ  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%à”;Ô0ˆÝ�fÕ3Ñ4Ô4ð 	)ÝÔ˜vÔ2¸ÐAÑAÔAÐAÝ% fÔ&BÀAÑFÔFð Bð B‘��5Ø ! S¡&Ñ)�ÝÔ" 5¤?Ô#;Ô#BÈÐLÑLÔLÐLÝÔ" 5¤9¤=Ô#7¸SÐAÑAÔAÐAÐAØ�c &Ô">Ñ?Ô?À!ÑCÈÑKÑKˆCÝÔ˜vÔ;Ô?ÔCÔJÐPSÐTÑTÔTÐTÐTÐTÝ˜Õ 9Ñ:Ô:ð 	)ØÔ+ð EÝ”˜FÔ.Ñ/Ô/Ð/Ð/Ð/åÔ" 6Ô#5¸8ÐDÑDÔDÐDÐDÐDÝ˜¥¤­B¬Iµr´yÐ AÑBÔBð 	)ÝÔ˜vœ}°(Ð;Ñ;Ô;Ð;ØŒ{Ð&Ý”˜FœKÑ(Ô(Ð(Ð(Ð(ð	)ð 	)à&Ð&r*   )r!   r"   r#   r   r'   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_sdpaÚ_supports_flash_attnr   r  rŸ   Ú_can_record_outputsr%   Úno_gradrœ  r8   r9   s   @r+   r–  r–  J  sÄ   ø€ € € € € € àÐÐÑØ ÐØ+€OØÐØ&*Ð#ðð ð Ðð €NØÐà'˜¨ÀÐPÑPÔPØ$�nÐ%8ÀÈoÐ^Ñ^Ô^ðð Ðð
 €U„]�_„_ð)ð )ð )ð )ñ „_ð)ð )ð )ð )ð )r*   r–  c                   ó   ‡ — e Zd Zdefˆ fd„Zdefd„Ze ed¬¦  «        e		 	 	 dde
j        d	ee
j                 dz  d
ee
j                 dz  dedee         defd„¦   «         ¦   «         ¦   «         Zde
j        fd„Zˆ xZS )ÚVJEPA2Modelr=   c                 óÐ   •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S rL   )r2   rD   r=   r  ÚencoderrO  Ú	predictorÚ	post_initrA  s     €r+   rD   zVJEPA2Model.__init__y  sX   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒå$ VÑ,Ô,ˆŒÝ(¨Ñ0Ô0ˆŒð 	�ŠÑÔÐÐÐr*   rS   c                 ó$   — | j         j        j        S rL   )r¯  rn   rc   rþ   s    r+   Úget_input_embeddingsz VJEPA2Model.get_input_embeddingsƒ  s   € ØŒ|Ô&Ô7Ð7r*   F)Útie_last_hidden_statesNrR   rC  rD  Úskip_predictorr�   c                 ó¤  — |€t          d¦  «        ‚ | j        d
d|i|¤Ž}|j        }|€´|€²|                     d¦  «        }|                     d¦  «        }	t	          j        |	|j        ¬¦  «                             d¦  «                             |df¦  «        g}t	          j        |	|j        ¬¦  «                             d¦  «                             |df¦  «        g}|sB | j	        d
|||dœ|¤Ž}
t          |
j        t          ||¦  «        |
j        |
j        ¬¦  «        }nd}t          |t          ||¦  «        |j        |j        |¬	¦  «        }|S )az  
        context_mask (`torch.Tensor` with shape `[batch_size, patch_size, 1]`, *optional*):
            The mask position ids indicating which encoder output patches are going to be exposed to the predictor.
            By default, this mask is created as torch.arange(N).unsqueeze(0).repeat(B,1), indicating full context
            available to the predictor.
        target_mask (`torch.Tensor` with shape `[batch_size, patch_size, 1]`, *optional*):
            The mask position ids indicating which encoder output patches are going to be used as a prediction target
            for the predictor. By default, this mask is created as torch.arange(N).unsqueeze(0).repeat(B,1), indicating
            that the predictor should predict all encoder patches.
        skip_predictor (bool):
            flag to skip the predictor forward, useful if you just need the encoder outputs
        Nz'You have to specify pixel_values_videosrR   r   r   r¾   )r_  rC  rD  )r   r    r   r   )r   r   r   r   r.   r)   )r§   r¯  r   r‰   r%   rŠ   r‡   r‹   ri   r°  r   r4  r   r   r-   )r5   rR   rC  rD  rµ  r�   Úencoder_outputsÚsequence_outputr’   r”   Úpredictor_outputsr.   Úencoder_outputs                r+   rY   zVJEPA2Model.forward†  s°  € ð. Ð&ÝÐFÑGÔGÐGà+7¨4¬<ð ,
ð ,
Ø 3ð,
àð,
ð ,
ˆð *Ô;ˆàÐ KÐ$7Ø#×(Ò(¨Ñ+Ô+ˆAØ×$Ò$ QÑ'Ô'ˆAÝ!œL¨Ð3FÔ3MÐNÑNÔN×XÒXÐYZÑ[Ô[×bÒbÐdeÐghÐciÑjÔjÐkˆLÝ œ<¨Ð2EÔ2LÐMÑMÔM×WÒWÐXYÑZÔZ×aÒaÐcdÐfgÐbhÑiÔiÐjˆKàð 	$Ø1?°´ð 2Ø&5Ø)Ø'ð2ð 2ð ð	2ð 2Ðõ  DØ"3Ô"EÝ$/°ÀÑ$MÔ$MØ/Ô=Ø,Ô7ð	 ñ  ô  ÐÐð  $Ðå9Ø-Ý +¨O¸\Ñ JÔ JØ)Ô7Ø&Ô1Ø-ð
ñ 
ô 
ˆð Ðr*   c                 ó>   — |                       |d¬¦  «        }|j        S )NT)rµ  )rY   r   )r5   rR   rº  s      r+   Úget_vision_featureszVJEPA2Model.get_vision_featuresÆ  s!   € ØŸšÐ&9È$˜ÑOÔOˆØÔ/Ð/r*   )NNF)r!   r"   r#   r   rD   r;   r³  r   r   r   r%   r]   r1   Úboolr   r   r-   rY   r¼  r8   r9   s   @r+   r­  r­  w  s+  ø€ € € € € ð˜|ð ð ð ð ð ð ð8Ð&=ð 8ð 8ð 8ð 8ð  Ø€_¨EÐ2Ñ2Ô2Øð 37Ø15Ø$ð;ð ;à"œ\ð;ð ˜5œ<Ô(¨4Ñ/ð;ð ˜%œ,Ô'¨$Ñ.ð	;ð
 ð;ð Ð+Ô,ð;ð 
*ð;ð ;ð ;ñ „^ñ 3Ô2ñ  Ôð;ðz0¸%¼,ð 0ð 0ð 0ð 0ð 0ð 0ð 0ð 0r*   r­  z}
    V-JEPA 2 Model transformer with a video classification head on top (a linear layer on top of the attentive pooler).
    c                   óŒ   ‡ — e Zd Zdefˆ fd„Zee	 d	dej        dej        dz  de	e
         deez  fd„¦   «         ¦   «         Zˆ xZS )
ÚVJEPA2ForVideoClassificationr=   c                 ó&  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          |¦  «        | _        t          j        |j	        |j        d¬¦  «        | _
        |                      ¦   «          d S ræ   )r2   rD   Ú
num_labelsr­  r—  rŒ  Úpoolerr   rª   r>   Ú
classifierr±  rA  s     €r+   rD   z%VJEPA2ForVideoClassification.__init__Ñ  s|   ø€ Ý‰Œ×Ò˜Ñ Ô Ð à Ô+ˆŒÝ! &Ñ)Ô)ˆŒõ ,¨FÑ3Ô3ˆŒÝœ) FÔ$6¸Ô8IÐPTÐUÑUÔUˆŒð 	�ŠÑÔÐÐÐr*   NrR   Úlabelsr�   rS   c                 ó   —  | j         d|ddœ|¤Ž}|j        }|                      |¦  «        }|                      |¦  «        }d}|�|                      ||| j        ¬¦  «        }t          |||j        |j        ¬¦  «        S )ag  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

        Examples:

        ```python
        >>> import torch
        >>> import numpy as np
        >>> from transformers import AutoVideoProcessor, VJEPA2ForVideoClassification

        >>> device = "cuda"

        >>> video_processor = AutoVideoProcessor.from_pretrained("facebook/vjepa2-vitl-fpc16-256-ssv2")
        >>> model = VJEPA2ForVideoClassification.from_pretrained("facebook/vjepa2-vitl-fpc16-256-ssv2").to(device)

        >>> video = np.ones((64, 256, 256, 3))  # 64 frames, 256x256 RGB
        >>> inputs = video_processor(video, return_tensors="pt").to(device)

        >>> # For inference
        >>> with torch.no_grad():
        ...     outputs = model(**inputs)
        >>> logits = outputs.logits

        >>> predicted_label = logits.argmax(-1).item()
        >>> print(model.config.id2label[predicted_label])

        >>> # For training
        >>> labels = torch.ones(1, dtype=torch.long, device=device)
        >>> loss = model(**inputs, labels=labels).loss

        ```T)rR   rµ  N)Úpooled_logitsrÄ  r=   )ÚlossÚlogitsr   r   r)   )	r—  r   rÂ  rÃ  Úloss_functionr=   r   r   r   )	r5   rR   rÄ  r�   Úoutputsr   Úpooler_outputrÈ  rÇ  s	            r+   rY   z$VJEPA2ForVideoClassification.forwardÞ  s²   € ðV �$”+ð 
Ø 3Øð
ð 
ð ð
ð 
ˆð $Ô5ÐØŸšÐ$5Ñ6Ô6ˆØ—’ Ñ/Ô/ˆàˆØÐØ×%Ò%°FÀ6ÐRVÔR]Ð%Ñ^Ô^ˆDå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r*   rL   )r!   r"   r#   r   rD   r   r   r%   r]   r   r   r(   r   rY   r8   r9   s   @r+   r¿  r¿  Ë  s´   ø€ € € € € ð˜|ð ð ð ð ð ð ð Øð '+ð<
ð <
à"œ\ð<
ð ”˜tÑ#ð<
ð Ð+Ô,ð	<
ð
 
Ð&Ñ	&ð<
ð <
ð <
ñ „^ñ Ôð<
ð <
ð <
ð <
ð <
r*   r¿  )r­  r–  r¿  r   )@Úcollections.abcr   Údataclassesr   r%   r   Ú r   rž  Úactivationsr   Úmodeling_layersr	   Úmodeling_outputsr
   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úconfiguration_vjepa2r   Ú
get_loggerr!   Úloggerr   r-   ÚModuler;   r_   r]   rð   r„   r�   rŸ   râ   rò   r  r  r1   r4  r6  rO  rd  rv  r}  r†  rŒ  r–  r­  r¿  Ú__all__r)   r*   r+   ú<module>rÜ     s
  ðð %Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ .Ð .Ð .Ð .Ð .Ð .ð 
ˆÔ	˜HÑ	%Ô	%€ð €ððñ ô ð
 ð9ð 9ð 9ð 9ð 9¨;ñ 9ô 9ñ „ñô ð9ð €ððñ ô ð ðð ð ð ð  {ñ ô ñ „ñô ðð* ð  ð  ð  ð  ˜bœiñ  ô  ð  ðFð ð ð ð �r”yñ ô ð ðT ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð %ð %ð %ð4)ð )ð )ð6z.ð z.ð z.ð z.ð z.˜"œ)ñ z.ô z.ð z.ðzð ð ð ð �”	ñ ô ð ð"%ð %ð %ð %ð %�R”Yñ %ô %ð %ð0-+ð -+ð -+ð -+ð -+Ð,ñ -+ô -+ð -+ð`(
ð (
ð (
ð (
ð (
�B”Iñ (
ô (
ð (
ðV0˜œð 0¨T°%´,Ô-?ð 0ÀEÄLð 0ð 0ð 0ð 0ð"C!ð C!ð C!ð C!ð C! ¤	ñ C!ô C!ð C!ðLN
ð N
ð N
ð N
ð N
�b”iñ N
ô N
ð N
ðb5)ð 5)ð 5)ð 5)ð 5) ¤	ñ 5)ô 5)ð 5)ðp=)ð =)ð =)ð =)ð =) ¤ñ =)ô =)ð =)ðB!+ð !+ð !+ð !+ð !+Ð%?ñ !+ô !+ð !+ðH+ð +ð +ð +ð +Ð&@ñ +ô +ð +ðD'ð 'ð 'ð 'ð '˜BœIñ 'ô 'ð 'ð& ð))ð ))ð ))ð ))ð ))˜Oñ ))ô ))ñ „ð))ðX ðP0ð P0ð P0ð P0ð P0Ð'ñ P0ô P0ñ „ðP0ðf €ððñ ô ð
L
ð L
ð L
ð L
ð L
Ð#8ñ L
ô L
ñô ð
L
ð^ SÐ
RÐ
R€€€r*   