§
    ‚ŠtjkW  ã                   ó  — d dl mZ d dlZd dlmZ ddlmZ ddlm	Z	 ddl
mZ ddlmZmZ ddlmZmZ dd	lmZ dd
lmZmZmZ ddlmZ ddlmZ ddlmZ  G d„ dej        ¦  «        Z G d„ dej        ¦  «        Z  G d„ dej        ¦  «        Z!	 d5dej        dej"        dej"        dej"        dej"        dz  de#de#dee         fd„Z$d„ Z%d ej"        d!e&d"ej"        fd#„Z'd$ej"        d%ej"        d&ej"        d'ej"        d"e(ej"        ej"        f         f
d(„Z) G d)„ d*ej        ¦  «        Z* G d+„ d,e¦  «        Z+ G d-„ d.ej        ¦  «        Z,e G d/„ d0e¦  «        ¦   «         Z- ed1¬2¦  «         G d3„ d4e-¦  «        ¦   «         Z.d0d4gZ/dS )6é    )ÚCallableNé   )Úinitialization)ÚACT2FN)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPooling)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚ	torch_int)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚMLCDVisionConfigc                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚMLCDMLPc                 ó  •— t          ¦   «                              ¦   «          || _        t          |j                 | _        t          j        |j        |j	        ¦  «        | _
        t          j        |j	        |j        ¦  «        | _        d S ©N)ÚsuperÚ__init__Úconfigr   Ú
hidden_actÚactivation_fnÚnnÚLinearÚhidden_sizeÚintermediate_sizeÚfc1Úfc2©Úselfr   Ú	__class__s     €úd/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/mlcd/modeling_mlcd.pyr   zMLCDMLP.__init__&   sf   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ# FÔ$5Ô6ˆÔÝ”9˜VÔ/°Ô1IÑJÔJˆŒÝ”9˜VÔ5°vÔ7IÑJÔJˆŒˆˆó    Úhidden_statesÚreturnc                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r   )r!   r   r"   )r$   r(   s     r&   ÚforwardzMLCDMLP.forward-   s=   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆØÐr'   )Ú__name__Ú
__module__Ú__qualname__r   ÚtorchÚTensorr+   Ú__classcell__©r%   s   @r&   r   r   %   sc   ø€ € € € € ðKð Kð Kð Kð Kð U¤\ð °e´lð ð ð ð ð ð ð ð r'   r   c                   óh   ‡ — e Zd ZU ej        ed<   d
dededdfˆ fd„Zdej        dej        fd	„Z	ˆ xZ
S )ÚMLCDRotaryEmbeddingÚinv_freqç     ˆÃ@ÚdimÚthetar)   Nc                 óê   •— t          ¦   «                              ¦   «          || _        || _        d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                      d|d¬¦  «         d S )Nç      ð?r   é   ©Údtyper5   F©Ú
persistent)r   r   r7   r8   r/   ÚarangeÚfloatÚregister_buffer)r$   r7   r8   r5   r%   s       €r&   r   zMLCDRotaryEmbedding.__init__7   sr   ø€ Ý‰Œ×ÒÑÔÐØˆŒØˆŒ
Ø˜%¥E¤L°°C¸Å%Ä+Ð$NÑ$NÔ$NÐQTÑ$TÑUÑVˆØ×Ò˜Z¨¸eÐÑDÔDÐDÐDÐDr'   Úposition_idsc                 ób   — |                      d¦  «        | j        z                       d¦  «        S )Néÿÿÿÿr   )Ú	unsqueezer5   Úflatten)r$   rC   s     r&   r+   zMLCDRotaryEmbedding.forward>   s+   € Ø×&Ò& rÑ*Ô*¨T¬]Ñ:×CÒCÀAÑFÔFÐFr'   )r6   )r,   r-   r.   r/   r0   Ú__annotations__ÚintrA   r   r+   r1   r2   s   @r&   r4   r4   4   s¡   ø€ € € € € € ØŒlÐÐÑðEð E˜Cð E¨ð E¸Dð Eð Eð Eð Eð Eð EðG E¤Lð G°U´\ð Gð Gð Gð Gð Gð Gð Gð Gr'   r4   c                   ót   ‡ — e Zd Zdefˆ fd„Zdej        dededej        fd„Zdej	        dej        fd	„Z
ˆ xZS )
ÚMLCDVisionEmbeddingsr   c                 ó2  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        t          j        t          j
        | j        ¦  «        ¦  «        | _        t          j        |j        | j        | j        | j        d¬¦  «        | _        | j        | j        z  dz  | _        | j        dz   | _        |                      dt          j        | j        ¦  «                             d¦  «        d¬¦  «         d S )NF)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚbiasr;   r   rC   ©r   rE   r>   )r   r   r   r   Ú	embed_dimÚ
image_sizeÚ
patch_sizer   Ú	Parameterr/   ÚrandnÚclass_embeddingÚConv2dÚnum_channelsÚpatch_embeddingÚnum_patchesÚnum_positionsrB   r@   Úexpandr#   s     €r&   r   zMLCDVisionEmbeddings.__init__C   só   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØ Ô+ˆŒØ Ô+ˆŒå!œ|­E¬K¸¼Ñ,GÔ,GÑHÔHˆÔå!œyØÔ+ØœØœØ”?Øð 
ñ  
ô  
ˆÔð !œO¨t¬Ñ>À1ÑDˆÔØ!Ô-°Ñ1ˆÔØ×Ò˜^­U¬\¸$Ô:LÑ-MÔ-M×-TÒ-TÐU\Ñ-]Ô-]ÐjoÐÑpÔpÐpÐpÐpr'   Ú
embeddingsÚheightÚwidthr)   c                 óÚ  — |j         d         dz
  }| j        j                             d¦  «        }|j         d         dz
  }t          j                             ¦   «         s&||k    r ||k    r|                      | j        ¦  «        S |dd…dd…f         }|dd…dd…f         }|j         d         }	|| j        z  }
|| j        z  }t          |dz  ¦  «        }| 
                    d|||	¦  «        }|                     dddd¦  «        }t          j                             ||
|fdd	¬
¦  «        }|                     dddd¦  «                             dd|	¦  «        }t	          j        ||fd¬¦  «        S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   r   NrE   g      à?r   r;   ÚbicubicF)ÚsizeÚmodeÚalign_corners©r7   )ÚshapeÚposition_embeddingÚweightrF   r/   ÚjitÚ
is_tracingrC   rU   r   ÚreshapeÚpermuter   Ú
functionalÚinterpolateÚviewÚcat)r$   r_   r`   ra   r\   ri   r]   Úclass_pos_embedÚpatch_pos_embedr7   Ú
new_heightÚ	new_widthÚsqrt_num_positionss                r&   Úinterpolate_pos_encodingz-MLCDVisionEmbeddings.interpolate_pos_encodingX   s‘  € ð !Ô& qÔ)¨AÑ-ˆØ!Ô4Ô;×EÒEÀaÑHÔHÐØ*Ô0°Ô3°aÑ7ˆõ Œy×#Ò#Ñ%Ô%ð 	>¨+¸Ò*FÐ*FÈ6ÐUZÊ?È?Ø×*Ò*¨4Ô+<Ñ=Ô=Ð=à,¨Q¨Q¨Q°°°¨UÔ3ˆØ,¨Q¨Q¨Q°°°¨UÔ3ˆàÔ˜rÔ"ˆà˜tœÑ.ˆ
Ø˜Tœ_Ñ,ˆ	å& }°cÑ'9Ñ:Ô:ÐØ)×1Ò1°!Ð5GÐI[Ð]`ÑaÔaˆØ)×1Ò1°!°Q¸¸1Ñ=Ô=ˆåœ-×3Ò3ØØ˜iÐ(ØØð	 4ñ 
ô 
ˆð *×1Ò1°!°Q¸¸1Ñ=Ô=×BÒBÀ1ÀbÈ#ÑNÔNˆåŒy˜/¨?Ð;ÀÐCÑCÔCÐCr'   Úpixel_valuesc                 óN  — |j         d         }| j        j        j        }|                      |                     |¬¦  «        ¦  «        }|                     d¦  «                             dd¦  «        }| j                             |dd¦  «        }t          j
        ||gd¬¦  «        }|S )Nr   r<   r;   r   rE   rg   )rh   r[   rj   r=   ÚtorG   Ú	transposerX   r^   r/   rr   )r$   ry   Ú
batch_sizeÚtarget_dtypeÚpatch_embedsÚclass_embedsr_   s          r&   r+   zMLCDVisionEmbeddings.forward�   sš   € Ø!Ô'¨Ô*ˆ
ØÔ+Ô2Ô8ˆà×+Ò+¨L¯OªOÀ,¨OÑ,OÔ,OÑPÔPˆØ#×+Ò+¨AÑ.Ô.×8Ò8¸¸AÑ>Ô>ˆàÔ+×2Ò2°:¸qÀ"ÑEÔEˆÝ”Y ¨lÐ;ÀÐCÑCÔCˆ
àÐr'   )r,   r-   r.   r   r   r/   r0   rI   rx   ÚFloatTensorr+   r1   r2   s   @r&   rK   rK   B   s³   ø€ € € € € ðqÐ/ð qð qð qð qð qð qð*'D°5´<ð 'DÈð 'DÐUXð 'DÐ]bÔ]ið 'Dð 'Dð 'Dð 'DðR
 EÔ$5ð 
¸%¼,ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r'   rK   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr;   r   rE   )r7   r=   )ÚpÚtrainingr   )Ú	repeat_kvÚnum_key_value_groupsr/   Úmatmulr|   r   ro   ÚsoftmaxÚfloat32r{   r=   r‰   r�   Ú
contiguous)rƒ   r„   r…   r†   r‡   rˆ   r‰   rŠ   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r&   Úeager_attention_forwardr˜   Ž   sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r'   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..NrE   r;   rg   )rh   r/   rr   )ÚxÚx1Úx2s      r&   Úrotate_halfr�   §   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r'   r(   Ún_repr)   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)rh   r^   rm   )r(   rž   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         r&   rŽ   rŽ   ®   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr'   ÚqÚkÚcosÚsinc                 óÆ  — | j         }|j         }|                      ¦   «         |                     ¦   «         }} |                     d¦  «                             ¦   «         |                     d¦  «                             ¦   «         }}| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }|                     |¦  «        }|                     |¦  «        }||fS )Néþÿÿÿ)r=   rA   rF   r�   r{   )r¤   r¥   r¦   r§   Úorig_q_dtypeÚorig_k_dtypeÚq_embedÚk_embeds           r&   Úapply_rotary_pos_emb_visionr®   º   sÃ   € ð ”7€LØ”7€LØ�7Š7‰9Œ9�a—g’g‘i”i€q€AØ�}Š}˜RÑ Ô ×&Ò&Ñ(Ô(¨#¯-ª-¸Ñ*;Ô*;×*AÒ*AÑ*CÔ*Cˆ€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�jŠj˜Ñ&Ô&€GØ�jŠj˜Ñ&Ô&€GØ�GÐÐr'   c                   ó¼   ‡ — e Zd ZdZdefˆ fd„Z	 ddej        deej        ej        f         dej        dz  de	e
         d	eej        ej        dz  f         f
d
„Zˆ xZS )ÚMLCDAttentionzÿMulti-headed attention with RoPE. Refer to papers:
    - Attention is all you need:
        https://huggingface.co/papers/1706.03762
    - RoFormer: Enhanced Transformer with Rotary Position Embedding:
        https://huggingface.co/papers/2104.09864
    r   c                 ó*  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        | j        | j        z  | _        | j        dz  | _        |j	        | _
        d| _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        |j        | _        d S )Nç      à¿F)r   r   r   r   rS   Únum_attention_headsÚ	num_headsr£   ÚscaleÚattention_dropoutr‰   Ú	is_causalr   r   Úk_projÚv_projÚq_projÚout_projr�   r#   s     €r&   r   zMLCDAttention.__init__Ð   sÐ   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØÔ3ˆŒØœ¨$¬.Ñ8ˆŒØ”] DÑ(ˆŒ
ØÔ/ˆŒØˆŒå”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝœ	 $¤.°$´.ÑAÔAˆŒØ$*Ô$?ˆÔ!Ð!Ð!r'   Nr(   Úposition_embeddingsr‡   rŠ   r)   c                 óê  — |j         dd…         \  }}|                      |¦  «                             ||| j        | j        f¦  «        }|                      |¦  «                             ||| j        | j        f¦  «        }|                      |¦  «                             ||| j        | j        f¦  «        }	|d                              d¦  «                             ¦   «         }
|d                              d¦  «                             ¦   «         }t          |||
|¦  «        \  }}| 
                    dddd¦  «                             ¦   «         }| 
                    dddd¦  «                             ¦   «         }|	 
                    dddd¦  «                             ¦   «         }	t          j        | j        j        t           ¦  «        } || |||	|f| j        sdn| j        | j        | j        dœ|¤Ž\  }}| 
                    dddd¦  «                             ¦   «         }|                     ||d¦  «        }|                      |¦  «        }| 
                    ddd¦  «                             ¦   «         }||fS )	z#Input shape: Batch x Time x ChannelNrE   r   r   r;   r   r‚   )r‰   rˆ   r·   )rh   rº   rm   r´   r£   r¸   r¹   rF   rA   r®   rn   r“   r
   Úget_interfacer   Ú_attn_implementationr˜   r�   r‰   rµ   r·   rq   r»   )r$   r(   r¼   r‡   rŠ   r}   Ú
seq_lengthÚquery_statesr”   r•   r¦   r§   Úattention_interfacer—   r–   s                  r&   r+   zMLCDAttention.forwardà   st  € ð "/Ô!4°S°b°SÔ!9Ñˆ
�Jð —{’{ =Ñ1Ô1×9Ò9¸:ÀzÐSWÔSaÐcgÔcpÐ:qÑrÔrˆØ—[’[ Ñ/Ô/×7Ò7¸ÀZÐQUÔQ_ÐaeÔanÐ8oÑpÔpˆ
Ø—{’{ =Ñ1Ô1×9Ò9¸:ÀzÐSWÔSaÐcgÔcpÐ:qÑrÔrˆð " !Ô$×.Ò.¨qÑ1Ô1×7Ò7Ñ9Ô9ˆØ! !Ô$×.Ò.¨qÑ1Ô1×7Ò7Ñ9Ô9ˆÝ#>¸|ÈZÐY\Ð^aÑ#bÔ#bÑ ˆ�jð $×+Ò+¨A¨q°!°QÑ7Ô7×BÒBÑDÔDˆØ×'Ò'¨¨1¨a°Ñ3Ô3×>Ò>Ñ@Ô@ˆ
Ø#×+Ò+¨A¨q°!°QÑ7Ô7×BÒBÑDÔDˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}Ð>�C�C°$´,Ø”JØ”nð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð "×)Ò)¨!¨Q°°1Ñ5Ô5×@Ò@ÑBÔBˆØ!×&Ò& z°:¸rÑBÔBˆØ—m’m KÑ0Ô0ˆØ!×)Ò)¨!¨Q°Ñ2Ô2×=Ò=Ñ?Ô?ˆØ˜LÐ(Ð(r'   r   )r,   r-   r.   Ú__doc__r   r   r/   r0   Útupler   r   r+   r1   r2   s   @r&   r°   r°   È   sÔ   ø€ € € € € ðð ð@Ð/ð @ð @ð @ð @ð @ð @ð( /3ð	-)ð -)à”|ð-)ð # 5¤<°´Ð#=Ô>ð-)ð œ tÑ+ð	-)ð
 Ð+Ô,ð-)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð-)ð -)ð -)ð -)ð -)ð -)ð -)ð -)r'   r°   c                   ó¤   ‡ — e Zd Zdefˆ fd„Z	 d
dej        deej        ej        f         dej        dz  dee	         deej
                 f
d	„Zˆ xZS )ÚMLCDEncoderLayerr   c                 óD  •— t          ¦   «                              ¦   «          |j        | _        t	          |¦  «        | _        t          j        | j        |j        ¬¦  «        | _	        t          |¦  «        | _        t          j        | j        |j        ¬¦  «        | _        d S )N©Úeps)r   r   r   rS   r°   Ú	self_attnr   Ú	LayerNormÚlayer_norm_epsÚlayer_norm1r   ÚmlpÚlayer_norm2r#   s     €r&   r   zMLCDEncoderLayer.__init__  s}   ø€ Ý‰Œ×ÒÑÔÐØÔ+ˆŒÝ& vÑ.Ô.ˆŒÝœ<¨¬¸FÔ<QÐRÑRÔRˆÔÝ˜6‘?”?ˆŒÝœ<¨¬¸FÔ<QÐRÑRÔRˆÔÐÐr'   Nr(   r¼   r‡   rŠ   r)   c                 óÈ   — |}|                       |¦  «        } | j        d|||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )a¸  
        Args:
            hidden_states (`torch.FloatTensor`):
                Input to the layer of shape `(batch, seq_len, embed_dim)`.
                Represents the hidden states from the previous layer or the input embeddings.
            position_embeddings (`tuple[torch.Tensor, torch.Tensor]`):
                A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`.
                Represents absolute positional embeddings for the query and key in the attention mechanism.
            attention_mask (`torch.FloatTensor`):
                Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values.
        )r(   r¼   r‡   © )rÍ   rÊ   rÏ   rÎ   )r$   r(   r¼   r‡   rŠ   ÚresidualÚ_s          r&   r+   zMLCDEncoderLayer.forward  s–   € ð$ !ˆà×(Ò(¨Ñ7Ô7ˆØ)˜4œ>ð 
Ø'Ø 3Ø)ð
ð 
ð ð	
ð 
Ñˆ�qð ! =Ñ0ˆà ˆØ×(Ò(¨Ñ7Ô7ˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆàÐr'   r   )r,   r-   r.   r   r   r/   r0   rÄ   r   r   r�   r+   r1   r2   s   @r&   rÆ   rÆ     s¿   ø€ € € € € ðSÐ/ð Sð Sð Sð Sð Sð Sð /3ð	"ð "à”|ð"ð # 5¤<°´Ð#=Ô>ð"ð œ tÑ+ð	"ð
 Ð+Ô,ð"ð 
ˆuÔ Ô	!ð"ð "ð "ð "ð "ð "ð "ð "r'   rÆ   c                   ó˜   ‡ — e Zd ZdZdefˆ fd„Z	 ddej        deej	        ej	        f         dej	        dz  de
e         d	eez  f
d
„Zˆ xZS )ÚMLCDEncoderz³
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`MLCDEncoderLayer`].

    Args:
        config: MLCDVisionConfig
    r   c                 óÔ   •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        dS )z3Overwrite dummy `MLCDConfig` to `MLCDVisionConfig`.c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS rÑ   )rÆ   )Ú.0rÓ   r   s     €r&   ú
<listcomp>z(MLCDEncoder.__init__.<locals>.<listcomp>K  s"   ø€ Ð$gÐ$gÐ$gÀ!Õ%5°fÑ%=Ô%=Ð$gÐ$gÐ$gr'   FN)	r   r   r   r   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayersÚgradient_checkpointingr#   s    `€r&   r   zMLCDEncoder.__init__G  s`   øø€ å‰Œ×ÒÑÔÐØˆŒÝ”mÐ$gÐ$gÐ$gÐ$gÅuÈVÔMeÑGfÔGfÐ$gÑ$gÔ$gÑhÔhˆŒØ&+ˆÔ#Ð#Ð#r'   NÚinputs_embedsr¼   r‡   rŠ   r)   c                 óP   — |}| j         D ]} ||||fi |¤Ž}Œt          |¬¦  «        S )a=  
        Args:
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            position_embeddings (`tuple[torch.Tensor, torch.Tensor]`):
                A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`.
                Represents absolute positional embeddings for the query and key in the attention mechanism.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.
                [What are attention masks?](../glossary#attention-mask)
        )Úlast_hidden_state)rÝ   r   )r$   rß   r¼   r‡   rŠ   r(   Úencoder_layers          r&   r+   zMLCDEncoder.forwardN  sa   € ð, &ˆØ!œ[ð 	ð 	ˆMØ)˜MØØ#Øðð ð ð	ð ˆMˆMõ Ø+ð
ñ 
ô 
ð 	
r'   r   )r,   r-   r.   rÃ   r   r   r/   r�   rÄ   r0   r   r   r   r+   r1   r2   s   @r&   rÕ   rÕ   >  sÀ   ø€ € € € € ðð ð,Ð/ð ,ð ,ð ,ð ,ð ,ð ,ð /3ð	!
ð !
àÔ(ð!
ð # 5¤<°´Ð#=Ô>ð!
ð œ tÑ+ð	!
ð
 Ð+Ô,ð!
ð 
�Ñ	 ð!
ð !
ð !
ð !
ð !
ð !
ð !
ð !
r'   rÕ   c                   ó€   ‡ — e Zd ZU eed<   dZdgZdZdZdZ	dZ
dZdZeedœZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚMLCDPreTrainedModelr   Úvision_modelrÆ   TF)r(   Ú
attentionsc                 ó`  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          ¦  «        r±| j        j        }t          j        |j        d|j	        dz  |z  ¬¦  «         t          j        |j
        j        |j        j        |z  ¬¦  «         t          j        |j        t          j        |j        j        d         ¦  «                             d¦  «        ¦  «         dS t	          |t&          ¦  «        r»| j        j        }|j	        dz  d|j        j        z  dz  z  |z  }|j	        dz  |z  }t          j        |j        j        |¬¦  «         t          j        |j        j        |¬¦  «         t          j        |j        j        |¬¦  «         t          j        |j        j        |¬¦  «         dS t	          |t2          ¦  «        rˆ| j        j        }|j        j        dz  d|j        j        z  dz  z  |z  }d|j        j        z  dz  |z  }t          j        |j        j        |¬¦  «         t          j        |j        j        |¬¦  «         dS t	          |t:          ¦  «        rL| j        j        }|j        j        |j        j        z  dz  dz  |z  }t          j        |j        d|¬¦  «         dS t	          |t@          ¦  «        rVd|j!        t          j        d	|j"        dt          j#        ¬
¦  «        |j"        z  z  z  }t          j        |j$        |¦  «         dS dS )zInitialize the weightsr‚   r²   )ÚmeanÚstd)ré   rE   rR   r;   r:   r   r<   N)%r   Ú_init_weightsr   Úinitializer_factorÚ
isinstancerK   ÚinitÚnormal_rX   rS   r[   rj   Úinitializer_rangeÚcopy_rC   r/   r@   rh   r^   r°   rÜ   rº   r¸   r¹   r»   r   r   r!   r"   ÚMLCDVisionModelr³   Úclass_pos_embr4   r8   r7   rA   r5   )	r$   rƒ   ÚfactorÚin_proj_stdÚout_proj_stdÚfc_stdÚpos_emb_stdr5   r%   s	           €r&   rê   z!MLCDPreTrainedModel._init_weights‚  sü  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ø”Ô/ˆÝ�fÕ2Ñ3Ô3ð 	2Ø”[Ô3ˆFÝŒL˜Ô/°c¸vÔ?OÐQUÑ?UÐX^Ñ?^Ð_Ñ_Ô_Ð_ÝŒL˜Ô/Ô6¸F¼MÔ<[Ð^dÑ<dÐeÑeÔeÐeÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÑhÔhÐhÐhÐhÝ˜¥Ñ.Ô.ð 	2Ø”[Ô3ˆFØ!Ô+¨TÑ1°q¸6¼=Ô;ZÑ7ZÐ_cÑ6cÑdÐgmÑmˆKØ"Ô,¨dÑ2°fÑ<ˆLÝŒL˜œÔ-°;Ð?Ñ?Ô?Ð?ÝŒL˜œÔ-°;Ð?Ñ?Ô?Ð?ÝŒL˜œÔ-°;Ð?Ñ?Ô?Ð?ÝŒL˜œÔ/°\ÐBÑBÔBÐBÐBÐBÝ˜¥Ñ(Ô(ð 	2Ø”[Ô3ˆFØ!œ=Ô4°dÑ:ÀÀFÄMÔDcÑ@cÐhlÑ?lÑmÐpvÑvˆKØ˜&œ-Ô3Ñ3¸Ñ<¸vÑEˆFÝŒL˜œÔ*°Ð7Ñ7Ô7Ð7ÝŒL˜œÔ*°Ð<Ñ<Ô<Ð<Ð<Ð<Ý˜¥Ñ0Ô0ð 	2Ø”[Ô3ˆFØ!œ=Ô4¸¼Ô8YÑYÐ]^Ñ^ÐcgÑgÐjpÑpˆKÝŒL˜Ô-°C¸[ÐIÑIÔIÐIÐIÐIÝ˜Õ 3Ñ4Ô4ð 	2Ø˜fœl­u¬|¸A¸v¼zÈ1ÕTYÔT_Ð/`Ñ/`Ô/`ÐciÔcmÑ/mÑnÑoˆHÝŒJ�v”¨Ñ1Ô1Ð1Ð1Ð1ð	2ð 	2r'   )r,   r-   r.   r   rH   Úbase_model_prefixÚ_no_split_modulesÚsupports_gradient_checkpointingÚaccepts_loss_kwargsÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendrÆ   r°   Ú_can_record_outputsr/   Úno_gradrê   r1   r2   s   @r&   rä   rä   r  sŸ   ø€ € € € € € àÐÐÑØ&ÐØ+Ð,ÐØ&*Ð#ØÐØÐØ€NØÐØ"&Ðà)Ø#ðð Ðð
 €U„]�_„_ð2ð 2ð 2ð 2ñ „_ð2ð 2ð 2ð 2ð 2r'   rä   zN
    The vision model from M_L_C_D without any head or projection on top.
    )Úcustom_introc                   óº   ‡ — e Zd ZU eed<   dZdZdZdefˆ fd„Ze	 e
d¬¦  «        e	 ddej        dz  d	ee         d
eez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )rñ   r   ry   )Úimager[   c                 ó  •— t          ¦   «                              |¦  «         |j        }t          |¦  «        | _        t          j        ||j        ¬¦  «        | _        t          |¦  «        | _
        t          j        ||j        ¬¦  «        | _        t          |j        |j        z  dz  ¦  «        | _        t          j        t!          j        d|j        |j        z  dz  ¦  «        ¦  «        | _        |                      ¦   «          d S )NrÈ   r;   r   )r   r   r   rK   r_   r   rË   rÌ   Úpre_layrnormrÕ   ÚencoderÚpost_layernormr4   r³   Úvision_rotary_embeddingrV   r/   rW   rò   Ú	post_init)r$   r   rS   r%   s      €r&   r   zMLCDVisionModel.__init__®  sÜ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØÔ&ˆ	å.¨vÑ6Ô6ˆŒÝœL¨¸Ô8MÐNÑNÔNˆÔÝ" 6Ñ*Ô*ˆŒÝ œl¨9¸&Ô:OÐPÑPÔPˆÔÝ':¸6Ô;MÐQWÔQkÑ;kÐopÑ;pÑ'qÔ'qˆÔ$Ýœ\­%¬+°a¸Ô9KÈvÔOiÑ9iÐmnÑ9nÑ*oÔ*oÑpÔpˆÔØ�ŠÑÔÐÐÐr'   F)Útie_last_hidden_statesNrŠ   r)   c                 óÞ  — |€t          d¦  «        ‚|j        d         | j        j        z  }|j        d         | j        j        z  }t	          j        ||j        ¬¦  «                             d¦  «                             d|¦  «        }t	          j        ||j        ¬¦  «                             d¦  «                             |d¦  «        }t	          j	        | 
                    ¦   «         | 
                    ¦   «         gd¬¦  «        }|                      |¦  «        }t	          j        | j        |gd¬¦  «        }t	          j        ||fd¬¦  «        }	|	                     ¦   «         |	                     ¦   «         f}
|                      |¦  «        }|                      |¦  «        } | j        d||
d	œ|¤Ž}|d         }|dd…ddd…f         }|                      |¦  «        }t)          ||¬
¦  «        S )aû  
        Example:

        ```python
        >>> import httpx
        >>> from io import BytesIO
        >>> from PIL import Image
        >>> from transformers import AutoProcessor, MLCDVisionModel
        >>> model = MLCDVisionModel.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448")
        >>> processor = AutoProcessor.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))
        >>> inputs = processor(images=image, return_tensors="pt")

        >>> with torch.no_grad():
        ...     outputs = model(**inputs, output_attentions=True)

        >>> features = outputs.last_hidden_state
        >>> print(f"Extracted features shape: {features.shape}")
        >>> print(f"Number of attention layers: {len(outputs.attentions)}")
        >>> print(f"Attention shape: {outputs.attentions[0].shape}")
        ```Nz You have to specify pixel_valuesr©   rE   )Údevicer   r   rg   )rß   r¼   )rá   Úpooler_outputrÑ   )Ú
ValueErrorrh   r   rU   r/   r@   r  rF   r^   ÚstackrG   r	  rr   rò   r¦   r§   r_   r  r  r  r	   )r$   ry   rŠ   Únum_patches_heightÚnum_patches_widthÚhpos_idsÚwpos_idsÚpos_idsÚrotary_pos_embÚembr¼   r(   Úencoder_outputsrá   Úpooled_outputs                  r&   r+   zMLCDVisionModel.forwardº  s   € ð@ ÐÝÐ?Ñ@Ô@Ð@à)Ô/°Ô3°t´{Ô7MÑMÐØ(Ô.¨rÔ2°d´kÔ6LÑLÐåŒLÐ+°LÔ4GÐHÑHÔH×RÒRÐSTÑUÔU×\Ò\Ð]_ÐarÑsÔsð 	õ ŒLÐ*°<Ô3FÐGÑGÔG×QÒQÐRSÑTÔT×[Ò[Ð\nÐprÑsÔsð 	õ ”+˜x×/Ò/Ñ1Ô1°8×3CÒ3CÑ3EÔ3EÐFÈBÐOÑOÔOˆØ×5Ò5°gÑ>Ô>ˆÝœ DÔ$6¸Ð#GÈQÐOÑOÔOˆÝŒi˜¨Ð8¸bÐAÑAÔAˆØ"Ÿwšw™yœy¨#¯'ª'©)¬)Ð4ÐàŸš¨Ñ5Ô5ˆØ×)Ò)¨-Ñ8Ô8ˆà&˜$œ,ð 
Ø'Ø 3ð
ð 
ð ð
ð 
ˆð ,¨AÔ.ÐØ)¨!¨!¨!¨Q°°°¨'Ô2ˆØ×+Ò+¨MÑ:Ô:ˆå)Ø/Ø'ð
ñ 
ô 
ð 	
r'   r   )r,   r-   r.   r   rH   Úmain_input_nameÚinput_modalitiesÚ_input_embed_layerr   r   r   r   r/   r�   r   r   rÄ   r	   r+   r1   r2   s   @r&   rñ   rñ   £  sã   ø€ € € € € € ð ÐÐÑØ$€OØ!ÐØ*Ðð
Ð/ð 
ð 
ð 
ð 
ð 
ð 
ð  Ø€_¨EÐ2Ñ2Ô2Øð 26ð>
ð >
àÔ'¨$Ñ.ð>
ð Ð+Ô,ð>
ð 
Ð+Ñ	+ð	>
ð >
ð >
ñ „^ñ 3Ô2ñ  Ôð>
ð >
ð >
ð >
ð >
r'   rñ   )r‚   )0Úcollections.abcr   r/   Útorch.nnr   Ú r   rí   Úactivationsr   Úmodeling_layersr   Úmodeling_outputsr   r	   Úmodeling_utilsr
   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.output_capturingr   Úconfiguration_mlcdr   ÚModuler   r4   rK   r0   rA   r˜   r�   rI   rŽ   rÄ   r®   r°   rÆ   rÕ   rä   rñ   Ú__all__rÑ   r'   r&   ú<module>r+     sØ  ðð( %Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BØ 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0ðð ð ð ð ˆbŒiñ ô ð ðGð Gð Gð Gð G˜"œ)ñ Gô Gð GðIð Ið Ið Ið I˜2œ9ñ Iô Ið Iðf ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð2(ð (ð (ð	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	UðØ„|ðØœðØ+0¬<ðØ>C¼lðà
ˆ5Œ<˜œÐ%Ô&ðð ð ð ðE)ð E)ð E)ð E)ð E)�B”Iñ E)ô E)ð E)ðP+ð +ð +ð +ð +Ð1ñ +ô +ð +ð\1
ð 1
ð 1
ð 1
ð 1
�"”)ñ 1
ô 1
ð 1
ðh ð-2ð -2ð -2ð -2ð -2˜/ñ -2ô -2ñ „ð-2ð` €ððñ ô ð
S
ð S
ð S
ð S
ð S
Ð)ñ S
ô S
ñô ð
S
ðl !Ð"3Ð
4€€€r'   