§
    ‚ŠtjP\  ã                   ó®  — d Z ddlZddlmZ ddlZddlmZ ddlmZm	Z	m
Z
 ddlmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZ  ej        e¦  «        Z ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z G d„ dej        ¦  «        Z G d„ dej        ¦  «        Z G d„ dej        ¦  «        Z G d„ dej        ¦  «        Z  G d„ dej        ¦  «        Z! G d„ dej        ¦  «        Z" G d„ dej        ¦  «        Z# G d„ d ej        ¦  «        Z$ G d!„ d"ej        ¦  «        Z% G d#„ d$ej        ¦  «        Z& G d%„ d&ej        ¦  «        Z' G d'„ d(ej        ¦  «        Z( G d)„ d*ej        ¦  «        Z) G d+„ d,ej        ¦  «        Z*e G d-„ d.e¦  «        ¦   «         Z+e G d/„ d0e+¦  «        ¦   «         Z, ed1¬¦  «         G d2„ d3e+¦  «        ¦   «         Z-g d4¢Z.dS )5zPyTorch CvT model.é    N)Ú	dataclass)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)Ú$ImageClassifierOutputWithNoAttentionÚModelOutput)ÚPreTrainedModel)Úauto_docstringÚloggingé   )Ú	CvtConfigzV
    Base class for model's outputs, with potential hidden states and attentions.
    )Úcustom_introc                   ó~   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej        df         dz  ed<   dS )ÚBaseModelOutputWithCLSTokenz£
    cls_token_value (`torch.FloatTensor` of shape `(batch_size, 1, hidden_size)`):
        Classification token at the output of the last layer of the model.
    NÚlast_hidden_stateÚcls_token_value.Úhidden_states)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚFloatTensorÚ__annotations__r   r   Útuple© ó    úb/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/cvt/modeling_cvt.pyr   r   !   sq   € € € € € € ðð ð
 37Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø04€O�UÔ&¨Ñ-Ð4Ð4Ñ4Ø:>€M�5˜Ô*¨CÐ/Ô0°4Ñ7Ð>Ð>Ñ>Ð>Ð>r    r   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚCvtEmbeddingsz'
    Construct the CvT embeddings.
    c                 ó¬   •— t          ¦   «                              ¦   «          t          |||||¬¦  «        | _        t	          j        |¦  «        | _        d S )N)Ú
patch_sizeÚnum_channelsÚ	embed_dimÚstrideÚpadding)ÚsuperÚ__init__ÚCvtConvEmbeddingsÚconvolution_embeddingsr   ÚDropoutÚdropout)Úselfr%   r&   r'   r(   r)   Údropout_rateÚ	__class__s          €r!   r+   zCvtEmbeddings.__init__7   sT   ø€ Ý‰Œ×ÒÑÔÐÝ&7Ø!°È	ÐZ`Ðjqð'
ñ '
ô '
ˆÔ#õ ”z ,Ñ/Ô/ˆŒˆˆr    c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S ©N)r-   r/   )r0   Úpixel_valuesÚhidden_states      r!   ÚforwardzCvtEmbeddings.forward>   s,   € Ø×2Ò2°<Ñ@Ô@ˆØ—|’| LÑ1Ô1ˆØÐr    ©r   r   r   r   r+   r7   Ú__classcell__©r2   s   @r!   r#   r#   2   sQ   ø€ € € € € ðð ð0ð 0ð 0ð 0ð 0ðð ð ð ð ð ð r    r#   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )r,   z"
    Image to Conv Embedding.
    c                 ó  •— t          ¦   «                              ¦   «          t          |t          j        j        ¦  «        r|n||f}|| _        t          j        |||||¬¦  «        | _	        t          j
        |¦  «        | _        d S )N)Úkernel_sizer(   r)   )r*   r+   Ú
isinstanceÚcollectionsÚabcÚIterabler%   r   ÚConv2dÚ
projectionÚ	LayerNormÚnormalization)r0   r%   r&   r'   r(   r)   r2   s         €r!   r+   zCvtConvEmbeddings.__init__I   sz   ø€ Ý‰Œ×ÒÑÔÐÝ#-¨j½+¼/Ô:RÑ#SÔ#SÐq�Z�ZÐZdÐfpÐYqˆ
Ø$ˆŒÝœ) L°)ÈÐ\bÐlsÐtÑtÔtˆŒÝœ\¨)Ñ4Ô4ˆÔÐÐr    c                 ó<  — |                       |¦  «        }|j        \  }}}}||z  }|                     |||¦  «                             ddd¦  «        }| j        r|                      |¦  «        }|                     ddd¦  «                             ||||¦  «        }|S ©Nr   é   r   )rC   ÚshapeÚviewÚpermuterE   )r0   r5   Ú
batch_sizer&   ÚheightÚwidthÚhidden_sizes          r!   r7   zCvtConvEmbeddings.forwardP   sª   € Ø—’ |Ñ4Ô4ˆØ2>Ô2DÑ/ˆ
�L &¨%Ø˜u‘nˆà#×(Ò(¨°\À;ÑOÔO×WÒWÐXYÐ[\Ð^_Ñ`Ô`ˆØÔð 	<Ø×-Ò-¨lÑ;Ô;ˆLà#×+Ò+¨A¨q°!Ñ4Ô4×9Ò9¸*ÀlÐTZÐ\aÑbÔbˆØÐr    r8   r:   s   @r!   r,   r,   D   sQ   ø€ € € € € ðð ð5ð 5ð 5ð 5ð 5ð
ð 
ð 
ð 
ð 
ð 
ð 
r    r,   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚCvtSelfAttentionConvProjectionc           	      óº   •— t          ¦   «                              ¦   «          t          j        |||||d|¬¦  «        | _        t          j        |¦  «        | _        d S )NF)r=   r)   r(   ÚbiasÚgroups)r*   r+   r   rB   ÚconvolutionÚBatchNorm2drE   )r0   r'   r=   r)   r(   r2   s        €r!   r+   z'CvtSelfAttentionConvProjection.__init__^   sa   ø€ Ý‰Œ×ÒÑÔÐÝœ9ØØØ#ØØØØð
ñ 
ô 
ˆÔõ  œ^¨IÑ6Ô6ˆÔÐÐr    c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r4   )rU   rE   ©r0   r6   s     r!   r7   z&CvtSelfAttentionConvProjection.forwardk   s.   € Ø×'Ò'¨Ñ5Ô5ˆØ×)Ò)¨,Ñ7Ô7ˆØÐr    ©r   r   r   r+   r7   r9   r:   s   @r!   rQ   rQ   ]   sG   ø€ € € € € ð7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð r    rQ   c                   ó   — e Zd Zd„ ZdS )Ú CvtSelfAttentionLinearProjectionc                 ó€   — |j         \  }}}}||z  }|                     |||¦  «                             ddd¦  «        }|S rG   )rI   rJ   rK   )r0   r6   rL   r&   rM   rN   rO   s          r!   r7   z(CvtSelfAttentionLinearProjection.forwardr   sN   € Ø2>Ô2DÑ/ˆ
�L &¨%Ø˜u‘nˆà#×(Ò(¨°\À;ÑOÔO×WÒWÐXYÐ[\Ð^_Ñ`Ô`ˆØÐr    N)r   r   r   r7   r   r    r!   r[   r[   q   s#   € € € € € ðð ð ð ð r    r[   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚCvtSelfAttentionProjectionÚdw_bnc                 ó¨   •— t          ¦   «                              ¦   «          |dk    rt          ||||¦  «        | _        t	          ¦   «         | _        d S )Nr_   )r*   r+   rQ   Úconvolution_projectionr[   Úlinear_projection)r0   r'   r=   r)   r(   Úprojection_methodr2   s         €r!   r+   z#CvtSelfAttentionProjection.__init__{   sQ   ø€ Ý‰Œ×ÒÑÔÐØ Ò'Ð'Ý*HÈÐT_ÐahÐjpÑ*qÔ*qˆDÔ'Ý!AÑ!CÔ!CˆÔÐÐr    c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r4   )ra   rb   rX   s     r!   r7   z"CvtSelfAttentionProjection.forward�   s.   € Ø×2Ò2°<Ñ@Ô@ˆØ×-Ò-¨lÑ;Ô;ˆØÐr    )r_   rY   r:   s   @r!   r^   r^   z   sR   ø€ € € € € ðDð Dð Dð Dð Dð Dðð ð ð ð ð ð r    r^   c                   ó.   ‡ — e Zd Z	 dˆ fd„	Zd„ Zd„ Zˆ xZS )ÚCvtSelfAttentionTc                 ó  •— t          ¦   «                              ¦   «          |dz  | _        || _        || _        || _        t          |||||dk    rdn|¬¦  «        | _        t          |||||¬¦  «        | _        t          |||||¬¦  «        | _	        t          j        |||	¬¦  «        | _        t          j        |||	¬¦  «        | _        t          j        |||	¬¦  «        | _        t          j        |
¦  «        | _        d S )Ng      à¿ÚavgÚlinear)rc   )rS   )r*   r+   ÚscaleÚwith_cls_tokenr'   Ú	num_headsr^   Úconvolution_projection_queryÚconvolution_projection_keyÚconvolution_projection_valuer   ÚLinearÚprojection_queryÚprojection_keyÚprojection_valuer.   r/   )r0   rl   r'   r=   Ú	padding_qÚ
padding_kvÚstride_qÚ	stride_kvÚqkv_projection_methodÚqkv_biasÚattention_drop_raterk   Úkwargsr2   s                €r!   r+   zCvtSelfAttention.__init__ˆ   s   ø€ õ 	‰Œ×ÒÑÔÐØ ‘_ˆŒ
Ø,ˆÔØ"ˆŒØ"ˆŒå,FØØØØØ*?À5Ò*HÐ*H˜h˜hÐNcð-
ñ -
ô -
ˆÔ)õ +EØ�{ J°	ÐMbð+
ñ +
ô +
ˆÔ'õ -GØ�{ J°	ÐMbð-
ñ -
ô -
ˆÔ)õ !#¤	¨)°YÀXÐ NÑ NÔ NˆÔÝ œi¨	°9À8ÐLÑLÔLˆÔÝ "¤	¨)°YÀXÐ NÑ NÔ NˆÔå”zÐ"5Ñ6Ô6ˆŒˆˆr    c                 óœ   — |j         \  }}}| j        | j        z  }|                     ||| j        |¦  «                             dddd¦  «        S )Nr   rH   r   r   )rI   r'   rl   rJ   rK   )r0   r6   rL   rO   Ú_Úhead_dims         r!   Ú"rearrange_for_multi_head_attentionz3CvtSelfAttention.rearrange_for_multi_head_attention±   sS   € Ø%1Ô%7Ñ"ˆ
�K Ø”> T¤^Ñ3ˆà× Ò  ¨[¸$¼.È(ÑSÔS×[Ò[Ð\]Ð_`ÐbcÐefÑgÔgÐgr    c                 ór  — | j         rt          j        |d||z  gd¦  «        \  }}|j        \  }}}|                     ddd¦  «                             ||||¦  «        }|                      |¦  «        }|                      |¦  «        }	|                      |¦  «        }
| j         rHt          j	        ||	fd¬¦  «        }	t          j	        ||fd¬¦  «        }t          j	        ||
fd¬¦  «        }
| j
        | j        z  }|                      |                      |	¦  «        ¦  «        }	|                      |                      |¦  «        ¦  «        }|                      |                      |
¦  «        ¦  «        }
t          j        d|	|g¦  «        | j        z  }t          j        j                             |d¬¦  «        }|                      |¦  «        }t          j        d||
g¦  «        }|j        \  }}}}|                     dddd¦  «                             ¦   «                              ||| j        |z  ¦  «        }|S )	Nr   r   rH   ©Údimzbhlk,bhtk->bhltéÿÿÿÿzbhlt,bhtv->bhlvr   )rk   r   ÚsplitrI   rK   rJ   rn   rm   ro   Úcatr'   rl   r   rq   rr   rs   Úeinsumrj   r   Ú
functionalÚsoftmaxr/   Ú
contiguous)r0   r6   rM   rN   Ú	cls_tokenrL   rO   r&   ÚkeyÚqueryÚvaluer~   Úattention_scoreÚattention_probsÚcontextr}   s                   r!   r7   zCvtSelfAttention.forward·   s"  € ØÔð 	XÝ&+¤k°,ÀÀFÈUÁNÐ@SÐUVÑ&WÔ&WÑ#ˆI�|Ø0<Ô0BÑ-ˆ
�K à#×+Ò+¨A¨q°!Ñ4Ô4×9Ò9¸*ÀlÐTZÐ\aÑbÔbˆà×-Ò-¨lÑ;Ô;ˆØ×1Ò1°,Ñ?Ô?ˆØ×1Ò1°,Ñ?Ô?ˆàÔð 	9Ý”I˜y¨%Ð0°aÐ8Ñ8Ô8ˆEÝ”)˜Y¨Ð,°!Ð4Ñ4Ô4ˆCÝ”I˜y¨%Ð0°aÐ8Ñ8Ô8ˆEà”> T¤^Ñ3ˆà×7Ò7¸×8MÒ8MÈeÑ8TÔ8TÑUÔUˆØ×5Ò5°d×6IÒ6IÈ#Ñ6NÔ6NÑOÔOˆØ×7Ò7¸×8MÒ8MÈeÑ8TÔ8TÑUÔUˆåœ,Ð'8¸5À#¸,ÑGÔGÈ$Ì*ÑTˆÝœ(Ô-×5Ò5°oÈ2Ð5ÑNÔNˆØŸ,š, Ñ7Ô7ˆå”,Ð0°?ÀEÐ2JÑKÔKˆà&œ}Ñˆˆ1ˆk˜1Ø—/’/ ! Q¨¨1Ñ-Ô-×8Ò8Ñ:Ô:×?Ò?À
ÈKÐY]ÔYgÐjrÑYrÑsÔsˆØˆr    ©T)r   r   r   r+   r   r7   r9   r:   s   @r!   rf   rf   ‡   sd   ø€ € € € € ð ð'7ð '7ð '7ð '7ð '7ð '7ðRhð hð hðð ð ð ð ð ð r    rf   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚCvtSelfOutputz 
    The residual connection is defined in CvtLayer instead of here (as is the case with other models), due to the
    layernorm applied before each block.
    c                 ó®   •— t          ¦   «                              ¦   «          t          j        ||¦  «        | _        t          j        |¦  «        | _        d S r4   )r*   r+   r   rp   Údenser.   r/   )r0   r'   Ú	drop_rater2   s      €r!   r+   zCvtSelfOutput.__init__Þ   sA   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜y¨)Ñ4Ô4ˆŒ
Ý”z )Ñ,Ô,ˆŒˆˆr    c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r4   ©r•   r/   ©r0   r6   Úinput_tensors      r!   r7   zCvtSelfOutput.forwardã   s*   € Ø—z’z ,Ñ/Ô/ˆØ—|’| LÑ1Ô1ˆØÐr    r8   r:   s   @r!   r“   r“   Ø   sQ   ø€ € € € € ðð ð
-ð -ð -ð -ð -ð
ð ð ð ð ð ð r    r“   c                   ó(   ‡ — e Zd Z	 dˆ fd„	Zd„ Zˆ xZS )ÚCvtAttentionTc                 ó®   •— t          ¦   «                              ¦   «          t          |||||||||	|
|¦  «        | _        t	          ||¦  «        | _        d S r4   )r*   r+   rf   Ú	attentionr“   Úoutput)r0   rl   r'   r=   rt   ru   rv   rw   rx   ry   rz   r–   rk   r2   s                €r!   r+   zCvtAttention.__init__ê   sd   ø€ õ 	‰Œ×ÒÑÔÐÝ)ØØØØØØØØ!ØØØñ
ô 
ˆŒõ $ I¨yÑ9Ô9ˆŒˆˆr    c                 ó`   — |                       |||¦  «        }|                      ||¦  «        }|S r4   )rž   rŸ   )r0   r6   rM   rN   Úself_outputÚattention_outputs         r!   r7   zCvtAttention.forward	  s1   € Ø—n’n \°6¸5ÑAÔAˆØŸ;š; {°LÑAÔAÐØÐr    r‘   rY   r:   s   @r!   rœ   rœ   é   sQ   ø€ € € € € ð ð:ð :ð :ð :ð :ð :ð> ð  ð  ð  ð  ð  ð  r    rœ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚCvtIntermediatec                 óÌ   •— t          ¦   «                              ¦   «          t          j        |t	          ||z  ¦  «        ¦  «        | _        t          j        ¦   «         | _        d S r4   )r*   r+   r   rp   Úintr•   ÚGELUÚ
activation)r0   r'   Ú	mlp_ratior2   s      €r!   r+   zCvtIntermediate.__init__  sJ   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜y­#¨i¸)Ñ.CÑ*DÔ*DÑEÔEˆŒ
Ýœ'™)œ)ˆŒˆˆr    c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r4   )r•   r¨   rX   s     r!   r7   zCvtIntermediate.forward  s*   € Ø—z’z ,Ñ/Ô/ˆØ—’ |Ñ4Ô4ˆØÐr    rY   r:   s   @r!   r¤   r¤     sG   ø€ € € € € ð$ð $ð $ð $ð $ð
ð ð ð ð ð ð r    r¤   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú	CvtOutputc                 óÎ   •— t          ¦   «                              ¦   «          t          j        t	          ||z  ¦  «        |¦  «        | _        t          j        |¦  «        | _        d S r4   )r*   r+   r   rp   r¦   r•   r.   r/   )r0   r'   r©   r–   r2   s       €r!   r+   zCvtOutput.__init__  sN   ø€ Ý‰Œ×ÒÑÔÐÝ”Y�s 9¨yÑ#8Ñ9Ô9¸9ÑEÔEˆŒ
Ý”z )Ñ,Ô,ˆŒˆˆr    c                 ód   — |                       |¦  «        }|                      |¦  «        }||z   }|S r4   r˜   r™   s      r!   r7   zCvtOutput.forward!  s4   € Ø—z’z ,Ñ/Ô/ˆØ—|’| LÑ1Ô1ˆØ# lÑ2ˆØÐr    rY   r:   s   @r!   r¬   r¬     sG   ø€ € € € € ð-ð -ð -ð -ð -ð
ð ð ð ð ð ð r    r¬   c                   ó^   ‡ — e Zd ZdZd
deddfˆ fd„Zdej        dej        fd„Zde	fd	„Z
ˆ xZS )ÚCvtDropPathzÏStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    ç        Ú	drop_probÚreturnNc                 óV   •— t          ¦   «                              ¦   «          || _        d S r4   )r*   r+   r²   )r0   r²   r2   s     €r!   r+   zCvtDropPath.__init__0  s$   ø€ Ý‰Œ×ÒÑÔÐØ"ˆŒˆˆr    r   c                 ó  — | j         dk    s| j        s|S d| j         z
  }|j        d         fd|j        dz
  z  z   }t	          j        ||j        |j        ¬¦  «        }t	          j        ||z   ¦  «        }| 	                    |¦  «        |z  S )Nr±   r   r   )r   )ÚdtypeÚdevice)
r²   ÚtrainingrI   Úndimr   Úrandr¶   r·   ÚfloorÚdiv)r0   r   Ú	keep_probrI   Úrandom_tensors        r!   r7   zCvtDropPath.forward4  s“   € ØŒ>˜SÒ Ð ¨¬Ð Ø Ð Ø˜œÑ&ˆ	ØÔ$ QÔ'Ð)¨D°MÔ4FÈÑ4JÑ,KÑKˆÝœ
 5°Ô0CÈMÔL`ÐaÑaÔaˆÝœ M°IÑ$=Ñ>Ô>ˆØ× Ò  Ñ+Ô+¨mÑ;Ð;r    c                 ó   — d| j         › �S )Nzp=)r²   )r0   s    r!   Ú
extra_reprzCvtDropPath.extra_repr=  s   € Ø$�D”NÐ$Ð$Ð$r    )r±   )r   r   r   r   Úfloatr+   r   ÚTensorr7   ÚstrrÀ   r9   r:   s   @r!   r°   r°   )  s›   ø€ € € € € ðð ð#ð # %ð #°$ð #ð #ð #ð #ð #ð #ð< U¤\ð <°e´lð <ð <ð <ð <ð%˜Cð %ð %ð %ð %ð %ð %ð %ð %r    r°   c                   ó,   ‡ — e Zd ZdZ	 dˆ fd„	Zd„ Zˆ xZS )ÚCvtLayerzb
    CvtLayer composed by attention layers, normalization and multi-layer perceptrons (mlps).
    Tc                 óš  •— t          ¦   «                              ¦   «          t          |||||||||	|
||¦  «        | _        t	          ||¦  «        | _        t          |||¦  «        | _        |dk    rt          |¦  «        nt          j
        ¦   «         | _        t          j        |¦  «        | _        t          j        |¦  «        | _        d S )Nr±   )r*   r+   rœ   rž   r¤   Úintermediater¬   rŸ   r°   r   ÚIdentityÚ	drop_pathrD   Úlayernorm_beforeÚlayernorm_after)r0   rl   r'   r=   rt   ru   rv   rw   rx   ry   rz   r–   r©   Údrop_path_raterk   r2   s                  €r!   r+   zCvtLayer.__init__F  sÉ   ø€ õ" 	‰Œ×ÒÑÔÐÝ%ØØØØØØØØ!ØØØØñ
ô 
ˆŒõ ,¨I°yÑAÔAˆÔÝ 	¨9°iÑ@Ô@ˆŒØ8FÈÒ8LÐ8L� ^Ñ4Ô4Ð4ÕRTÔR]ÑR_ÔR_ˆŒÝ "¤¨YÑ 7Ô 7ˆÔÝ!œ|¨IÑ6Ô6ˆÔÐÐr    c                 ó<  — |                       |                      |¦  «        ||¦  «        }|}|                      |¦  «        }||z   }|                      |¦  «        }|                      |¦  «        }|                      ||¦  «        }|                      |¦  «        }|S r4   )rž   rÊ   rÉ   rË   rÇ   rŸ   )r0   r6   rM   rN   Úself_attention_outputr¢   Úlayer_outputs          r!   r7   zCvtLayer.forwardm  s¨   € Ø $§¢Ø×!Ò! ,Ñ/Ô/ØØñ!
ô !
Ðð
 1ÐØŸ>š>Ð*:Ñ;Ô;Ðð (¨,Ñ6ˆð ×+Ò+¨LÑ9Ô9ˆØ×(Ò(¨Ñ6Ô6ˆð —{’{ <°Ñ>Ô>ˆØ—~’~ lÑ3Ô3ˆØÐr    r‘   r8   r:   s   @r!   rÅ   rÅ   A  s\   ø€ € € € € ðð ð& ð%7ð %7ð %7ð %7ð %7ð %7ðNð ð ð ð ð ð r    rÅ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚCvtStagec           	      ó&  •‡ ‡‡— t          ¦   «                              ¦   «          ‰‰ _        |‰ _        ‰ j        j        ‰ j                 r=t          j        t          j        dd‰ j        j	        d         ¦  «        ¦  «        ‰ _        t          ‰j        ‰ j                 ‰j        ‰ j                 ‰ j        dk    r‰j        n‰j	        ‰ j        dz
           ‰j	        ‰ j                 ‰j        ‰ j                 ‰j        ‰ j                 ¬¦  «        ‰ _        d„ t          j        d‰j        ‰ j                 ‰j        |         d¬¦  «        D ¦   «         Št          j        ˆˆˆ fd„t+          ‰j        ‰ j                 ¦  «        D ¦   «         Ž ‰ _        d S )	Nr   rƒ   r   )r%   r(   r&   r'   r)   r1   c                 ó6   — g | ]}|                      ¦   «         ‘ŒS r   )Úitem)Ú.0Úxs     r!   ú
<listcomp>z%CvtStage.__init__.<locals>.<listcomp>”  s-   € ð 
ð 
ð 
ØˆA�FŠF‰HŒHð
ð 
ð 
r    Úcpu)r·   c                 ó   •— g | ]ú}t          ‰j        ‰j                 ‰j        ‰j                 ‰j        ‰j                 ‰j        ‰j                 ‰j        ‰j                 ‰j        ‰j                 ‰j        ‰j                 ‰j	        ‰j                 ‰j
        ‰j                 ‰j        ‰j                 ‰j        ‰j                 ‰‰j                 ‰j        ‰j                 ‰j        ‰j                 ¬ ¦  «        ‘ŒûS ))rl   r'   r=   rt   ru   rw   rv   rx   ry   rz   r–   rÌ   r©   rk   )rÅ   rl   Ústager'   Ú
kernel_qkvrt   ru   rw   rv   rx   ry   rz   r–   r©   rŠ   )rÕ   r}   ÚconfigÚdrop_path_ratesr0   s     €€€r!   r×   z%CvtStage.__init__.<locals>.<listcomp>™  së   ø€ ð ð ð ð" õ! Ø$Ô.¨t¬zÔ:Ø$Ô.¨t¬zÔ:Ø &Ô 1°$´*Ô =Ø$Ô.¨t¬zÔ:Ø%Ô0°´Ô<Ø$Ô.¨t¬zÔ:Ø#œ_¨T¬ZÔ8Ø*0Ô*FÀtÄzÔ*RØ#œ_¨T¬ZÔ8Ø(.Ô(BÀ4Ä:Ô(NØ$Ô.¨t¬zÔ:Ø#2°4´:Ô#>Ø$Ô.¨t¬zÔ:Ø#)Ô#3°D´JÔ#?ðñ ô ðð ð r    )r*   r+   rÜ   rÚ   rŠ   r   Ú	Parameterr   Úrandnr'   r#   Úpatch_sizesÚpatch_strider&   Úpatch_paddingr–   Ú	embeddingÚlinspacerÌ   ÚdepthÚ
SequentialÚrangeÚlayers)r0   rÜ   rÚ   rÝ   r2   s   `` @€r!   r+   zCvtStage.__init__„  s„  øøøø€ Ý‰Œ×ÒÑÔÐØˆŒØˆŒ
ØŒ;Ô  ¤Ô,ð 	XÝœ\­%¬+°a¸¸D¼KÔ<QÐRTÔ<UÑ*VÔ*VÑWÔWˆDŒNå&ØÔ)¨$¬*Ô5ØÔ& t¤zÔ2Ø04´
¸a²°˜Ô,Ð,ÀVÔEUÐVZÔV`ÐcdÑVdÔEeØÔ& t¤zÔ2ØÔ(¨¬Ô4ØÔ)¨$¬*Ô5ð
ñ 
ô 
ˆŒð
ð 
Ý#œn¨Q°Ô0EÀdÄjÔ0QÐSYÔS_Ð`eÔSfÐotÐuÑuÔuð
ñ 
ô 
ˆõ ”mðð ð ð ð ð õ" ˜vœ|¨D¬JÔ7Ñ8Ô8ð#ñ ô ð
ˆŒˆˆr    c                 ó:  — d }|                       |¦  «        }|j        \  }}}}|                     ||||z  ¦  «                             ddd¦  «        }| j        j        | j                 r4| j                             |dd¦  «        }t          j	        ||fd¬¦  «        }| j
        D ]} ||||¦  «        }|}Œ| j        j        | j                 rt          j        |d||z  gd¦  «        \  }}|                     ddd¦  «                             ||||¦  «        }||fS )Nr   rH   r   rƒ   r�   )rã   rI   rJ   rK   rÜ   rŠ   rÚ   Úexpandr   r…   rè   r„   )	r0   r6   rŠ   rL   r&   rM   rN   ÚlayerÚlayer_outputss	            r!   r7   zCvtStage.forward®  s:  € Øˆ	Ø—~’~ lÑ3Ô3ˆØ2>Ô2DÑ/ˆ
�L &¨%à#×(Ò(¨°\À6ÈEÁ>ÑRÔR×ZÒZÐ[\Ð^_ÐabÑcÔcˆØŒ;Ô  ¤Ô,ð 	GØœ×-Ò-¨j¸"¸bÑAÔAˆIÝ œ9 i°Ð%>ÀAÐFÑFÔFˆLà”[ð 	)ð 	)ˆEØ!˜E ,°¸Ñ>Ô>ˆMØ(ˆLˆLàŒ;Ô  ¤Ô,ð 	XÝ&+¤k°,ÀÀFÈUÁNÐ@SÐUVÑ&WÔ&WÑ#ˆI�|Ø#×+Ò+¨A¨q°!Ñ4Ô4×9Ò9¸*ÀlÐTZÐ\aÑbÔbˆØ˜YÐ&Ð&r    rY   r:   s   @r!   rÑ   rÑ   ƒ  sH   ø€ € € € € ð(
ð (
ð (
ð (
ð (
ðT'ð 'ð 'ð 'ð 'ð 'ð 'r    rÑ   c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )Ú
CvtEncoderc                 ó   •— t          ¦   «                              ¦   «          || _        t          j        g ¦  «        | _        t          t          |j        ¦  «        ¦  «        D ]*}| j         	                    t          ||¦  «        ¦  «         Œ+d S r4   )r*   r+   rÜ   r   Ú
ModuleListÚstagesrç   Úlenrå   ÚappendrÑ   )r0   rÜ   Ú	stage_idxr2   s      €r!   r+   zCvtEncoder.__init__Ã  s   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”m BÑ'Ô'ˆŒÝ�s 6¤<Ñ0Ô0Ñ1Ô1ð 	<ð 	<ˆIØŒK×Ò�x¨°	Ñ:Ô:Ñ;Ô;Ð;Ð;ð	<ð 	<r    FTc                 óÖ   — |rdnd }|}d }t          | j        ¦  «        D ]\  }} ||¦  «        \  }}|r||fz   }Œ|st          d„ |||fD ¦   «         ¦  «        S t          |||¬¦  «        S )Nr   c              3   ó   K  — | ]}|®|V — Œ	d S r4   r   )rÕ   Úvs     r!   ú	<genexpr>z%CvtEncoder.forward.<locals>.<genexpr>Õ  s(   è è € ÐbÐb˜qÐTUÐTa˜ÐTaÐTaÐTaÐTaÐbÐbr    ©r   r   r   )Ú	enumeraterñ   r   r   )	r0   r5   Úoutput_hidden_statesÚreturn_dictÚall_hidden_statesr6   rŠ   r}   Ústage_modules	            r!   r7   zCvtEncoder.forwardÊ  s´   € Ø"6Ð@˜B˜B¸DÐØ#ˆàˆ	Ý!*¨4¬;Ñ!7Ô!7ð 	Hð 	HÑˆA�Ø&2 l°<Ñ&@Ô&@Ñ#ˆL˜)Ø#ð HØ$5¸¸Ñ$GÐ!øàð 	cÝÐbÐb \°9Ð>OÐ$PÐbÑbÔbÑbÔbÐbå*Ø*Ø%Ø+ð
ñ 
ô 
ð 	
r    )FTrY   r:   s   @r!   rî   rî   Â  sL   ø€ € € € € ð<ð <ð <ð <ð <ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r    rî   c                   ób   ‡ — e Zd ZU eed<   dZdZdgZ ej	        ¦   «         ˆ fd„¦   «         Z
ˆ xZS )ÚCvtPreTrainedModelrÜ   Úcvtr5   rÅ   c                 óÖ  •— t          ¦   «                              |¦  «         t          |t          j        t          j        f¦  «        rJt          j        |j        d| j	        j
        ¬¦  «         |j        �t          j        |j        ¦  «         dS dS t          |t          ¦  «        r?| j	        j        |j                 r*t          j        |j        d| j	        j
        ¬¦  «         dS dS dS )zInitialize the weightsr±   )ÚmeanÚstdN)r*   Ú_init_weightsr>   r   rp   rB   ÚinitÚtrunc_normal_ÚweightrÜ   Úinitializer_rangerS   Úzeros_rÑ   rŠ   rÚ   )r0   Úmoduler2   s     €r!   r  z CvtPreTrainedModel._init_weightså  sæ   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�rœy­"¬)Ð4Ñ5Ô5ð 	bÝÔ˜vœ}°3¸D¼KÔ<YÐZÑZÔZÐZØŒ{Ð&Ý”˜FœKÑ(Ô(Ð(Ð(Ð(ð 'Ð&å˜¥Ñ)Ô)ð 	bØŒ{Ô$ V¤\Ô2ð bÝÔ" 6Ô#3¸#À4Ä;ÔC`ÐaÑaÔaÐaÐaÐað	bð 	bðbð br    )r   r   r   r   r   Úbase_model_prefixÚmain_input_nameÚ_no_split_modulesr   Úno_gradr  r9   r:   s   @r!   r   r   Þ  st   ø€ € € € € € àÐÐÑØÐØ$€OØ#˜Ðà€U„]�_„_ð	bð 	bð 	bð 	bñ „_ð	bð 	bð 	bð 	bð 	br    r   c                   ór   ‡ — e Zd Zd	ˆ fd„	Ze	 	 	 d
dej        dz  dedz  dedz  dee	z  fd„¦   «         Z
ˆ xZS )ÚCvtModelTc                 ó¨   •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        |                      ¦   «          dS )zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        N)r*   r+   rÜ   rî   ÚencoderÚ	post_init)r0   rÜ   Úadd_pooling_layerr2   s      €r!   r+   zCvtModel.__init__ô  sI   ø€ õ
 	‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ! &Ñ)Ô)ˆŒØ�ŠÑÔÐÐÐr    Nr5   rû   rü   r³   c                 óü   — |�|n| j         j        }|�|n| j         j        }|€t          d¦  «        ‚|                      |||¬¦  «        }|d         }|s|f|dd …         z   S t          ||j        |j        ¬¦  «        S )Nz You have to specify pixel_values©rû   rü   r   r   rù   )rÜ   rû   rü   Ú
ValueErrorr  r   r   r   )r0   r5   rû   rü   r{   Úencoder_outputsÚsequence_outputs          r!   r7   zCvtModel.forwardþ  s¸   € ð %9Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàÐÝÐ?Ñ@Ô@Ð@àŸ,š,ØØ!5Ø#ð 'ñ 
ô 
ˆð
 *¨!Ô,ˆàð 	<Ø#Ð%¨¸¸¸Ô(;Ñ;Ð;å*Ø-Ø+Ô;Ø)Ô7ð
ñ 
ô 
ð 	
r    r‘   )NNN)r   r   r   r+   r   r   rÂ   Úboolr   r   r7   r9   r:   s   @r!   r  r  ò  s¨   ø€ € € € € ðð ð ð ð ð ð ð -1Ø,0Ø#'ð	
ð 
à”l TÑ)ð
ð # T™kð
ð ˜D‘[ð	
ð 
Ð,Ñ	,ð
ð 
ð 
ñ „^ð
ð 
ð 
ð 
ð 
r    r  z¤
    Cvt Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.
    c                   ó†   ‡ — e Zd Zˆ fd„Ze	 	 	 	 d	dej        dz  dej        dz  dedz  dedz  dee	z  f
d„¦   «         Z
ˆ xZS )
ÚCvtForImageClassificationc                 óŽ  •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        t          j        |j        d         ¦  «        | _        |j        dk    r%t          j	        |j        d         |j        ¦  «        nt          j
        ¦   «         | _        |                      ¦   «          d S )NF)r  rƒ   r   )r*   r+   Ú
num_labelsr  r  r   rD   r'   Ú	layernormrp   rÈ   Ú
classifierr  )r0   rÜ   r2   s     €r!   r+   z"CvtForImageClassification.__init__&  s¬   ø€ Ý‰Œ×Ò˜Ñ Ô Ð à Ô+ˆŒÝ˜F°eÐ<Ñ<Ô<ˆŒÝœ fÔ&6°rÔ&:Ñ;Ô;ˆŒð CIÔBSÐVWÒBWÐBW�BŒI�fÔ& rÔ*¨FÔ,=Ñ>Ô>Ð>Õ]_Ô]hÑ]jÔ]jð 	Œð
 	�ŠÑÔÐÐÐr    Nr5   Úlabelsrû   rü   r³   c                 ó   — |�|n| j         j        }|                      |||¬¦  «        }|d         }|d         }| j         j        d         r|                      |¦  «        }nP|j        \  }	}
}}|                     |	|
||z  ¦  «                             ddd¦  «        }|                      |¦  «        }|                     d¬¦  «        }|  	                    |¦  «        }d}|��n| j         j
        €p| j         j        dk    rd| j         _
        nS| j         j        dk    r7|j        t          j        k    s|j        t          j        k    rd	| j         _
        nd
| j         _
        | j         j
        dk    r\t!          ¦   «         }| j         j        dk    r1 ||                     ¦   «         |                     ¦   «         ¦  «        }n“ |||¦  «        }n†| j         j
        d	k    rLt%          ¦   «         } ||                     d| j         j        ¦  «        |                     d¦  «        ¦  «        }n*| j         j
        d
k    rt'          ¦   «         } |||¦  «        }|s|f|dd…         z   }|�|f|z   n|S t)          |||j        ¬¦  «        S )aŠ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr  r   r   rƒ   rH   r�   Ú
regressionÚsingle_label_classificationÚmulti_label_classification)ÚlossÚlogitsr   )rÜ   rü   r  rŠ   r   rI   rJ   rK   r  r!  Úproblem_typer  r¶   r   Úlongr¦   r   Úsqueezer   r   r
   r   )r0   r5   r"  rû   rü   r{   Úoutputsr  rŠ   rL   r&   rM   rN   Úsequence_output_meanr(  r'  Úloss_fctrŸ   s                     r!   r7   z!CvtForImageClassification.forward4  s‘  € ð &1Ð%<�k�kÀ$Ä+ÔBYˆØ—(’(ØØ!5Ø#ð ñ 
ô 
ˆð " !œ*ˆØ˜A”Jˆ	ØŒ;Ô  Ô$ð 	>Ø"Ÿnšn¨YÑ7Ô7ˆOˆOà6EÔ6KÑ3ˆJ˜ f¨eà-×2Ò2°:¸|ÈVÐV[É^Ñ\Ô\×dÒdÐefÐhiÐklÑmÔmˆOØ"Ÿnšn¨_Ñ=Ô=ˆOà.×3Ò3¸Ð3Ñ:Ô:ÐØ—’Ð!5Ñ6Ô6ˆàˆØÑØŒ{Ô'Ð/Ø”;Ô)¨QÒ.Ð.Ø/;�D”KÔ,Ð,Ø”[Ô+¨aÒ/Ð/°V´\ÅUÄZÒ5OÐ5OÐSYÔS_ÕchÔclÒSlÐSlØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”;Ô)¨QÒ.Ð.Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ô0FÑ GÔ GÈÏÊÐUWÉÌÑYÔY��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�àð 	FØ�Y ¨¨¨¤Ñ,ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå3¸ÀfÐ\cÔ\qÐrÑrÔrÐrr    )NNNN)r   r   r   r+   r   r   rÂ   r  r   r
   r7   r9   r:   s   @r!   r  r    sÅ   ø€ € € € € ðð ð ð ð ð ð -1Ø&*Ø,0Ø#'ð=sð =sà”l TÑ)ð=sð ”˜tÑ#ð=sð # T™kð	=sð
 ˜D‘[ð=sð 
Ð5Ñ	5ð=sð =sð =sñ „^ð=sð =sð =sð =sð =sr    r  )r  r  r   )/r   Úcollections.abcr?   Údataclassesr   r   r   Útorch.nnr   r   r   Ú r	   r  Úmodeling_outputsr
   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_cvtr   Ú
get_loggerr   Úloggerr   ÚModuler#   r,   rQ   r[   r^   rf   r“   rœ   r¤   r¬   r°   rÅ   rÑ   rî   r   r  r  Ú__all__r   r    r!   ú<module>r;     s–  ðð Ð à Ð Ð Ð Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø QÐ QÐ QÐ QÐ QÐ QÐ QÐ QØ -Ð -Ð -Ð -Ð -Ð -Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø (Ð (Ð (Ð (Ð (Ð (ð 
ˆÔ	˜HÑ	%Ô	%€ð €ððñ ô ð
 ð?ð ?ð ?ð ?ð ? +ñ ?ô ?ñ „ñô ð?ðð ð ð ð �B”Iñ ô ð ð$ð ð ð ð ˜œ	ñ ô ð ð2ð ð ð ð  R¤Yñ ô ð ð(ð ð ð ð  r¤yñ ô ð ð
ð 
ð 
ð 
ð 
 ¤ñ 
ô 
ð 
ðNð Nð Nð Nð N�r”yñ Nô Nð Nðbð ð ð ð �B”Iñ ô ð ð"# ð # ð # ð # ð # �2”9ñ # ô # ð # ðL	ð 	ð 	ð 	ð 	�b”iñ 	ô 	ð 	ð
ð 
ð 
ð 
ð 
�”	ñ 
ô 
ð 
ð%ð %ð %ð %ð %�"”)ñ %ô %ð %ð0?ð ?ð ?ð ?ð ?ˆrŒyñ ?ô ?ð ?ðD<'ð <'ð <'ð <'ð <'ˆrŒyñ <'ô <'ð <'ð~
ð 
ð 
ð 
ð 
�”ñ 
ô 
ð 
ð8 ðbð bð bð bð b˜ñ bô bñ „ðbð& ð)
ð )
ð )
ð )
ð )
Ð!ñ )
ô )
ñ „ð)
ðX €ððñ ô ðMsð Msð Msð Msð MsÐ 2ñ Msô Msñô ðMsð` JÐ
IÐ
I€€€r    