§
    ‚ŠtjÐV  ã                   óŒ  — d Z ddlZddlZddlmZ ddlZddlmc mZ	 ddlmZ ddl
mZ ddlmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZ  ej        e¦  «        Z G d„ dej        ¦  «        Z G d„ dej        ¦  «        Z G d„ dej        ¦  «        Z G d„ dej        ¦  «        Z  G d„ dej        ¦  «        Z! G d„ dej        ¦  «        Z" G d„ dej        ¦  «        Z# G d„ dej        ¦  «        Z$e G d„ de¦  «        ¦   «         Z%e G d„ d e%¦  «        ¦   «         Z& ed!¬"¦  «         G d#„ d$e%¦  «        ¦   «         Z'g d%¢Z(dS )&zPyTorch PVT model.é    N)ÚIterable)Únné   )Úinitialization)ÚACT2FN)ÚBaseModelOutputÚImageClassifierOutput)ÚPreTrainedModel)Úauto_docstringÚloggingé   )Ú	PvtConfigc                   óÊ   ‡ — e Zd ZdZ	 ddedeee         z  deee         z  dededed	efˆ fd
„Zde	j
        dedede	j
        fd„Zde	j
        dee	j
        eef         fd„Zˆ xZS )ÚPvtPatchEmbeddingszì
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    FÚconfigÚ
image_sizeÚ
patch_sizeÚstrideÚnum_channelsÚhidden_sizeÚ	cls_tokenc                 óê  •— t          ¦   «                              ¦   «          || _        t          |t          j        j        ¦  «        r|n||f}t          |t          j        j        ¦  «        r|n||f}|d         |d         z  |d         |d         z  z  }|| _        || _        || _	        || _
        t          j        t          j        d|r|dz   n||¦  «        ¦  «        | _        |r(t          j        t          j        dd|¦  «        ¦  «        nd | _        t          j        ||||¬¦  «        | _        t          j        ||j        ¬¦  «        | _        t          j        |j        ¬¦  «        | _        d S )Nr   r   ©Úkernel_sizer   ©Úeps)Úp)ÚsuperÚ__init__r   Ú
isinstanceÚcollectionsÚabcr   r   r   r   Únum_patchesr   Ú	ParameterÚtorchÚrandnÚposition_embeddingsÚzerosr   ÚConv2dÚ
projectionÚ	LayerNormÚlayer_norm_epsÚ
layer_normÚDropoutÚhidden_dropout_probÚdropout)
Úselfr   r   r   r   r   r   r   r#   Ú	__class__s
            €úb/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/pvt/modeling_pvt.pyr   zPvtPatchEmbeddings.__init__,   s\  ø€ õ 	‰Œ×ÒÑÔÐØˆŒÝ#-¨j½+¼/Ô:RÑ#SÔ#SÐq�Z�ZÐZdÐfpÐYqˆ
Ý#-¨j½+¼/Ô:RÑ#SÔ#SÐq�Z�ZÐZdÐfpÐYqˆ
Ø! !”}¨
°1¬Ñ5¸*ÀQ¼-È:ÐVWÌ=Ñ:XÑYˆØ$ˆŒØ$ˆŒØ(ˆÔØ&ˆÔå#%¤<ÝŒK˜¨iÐH˜;¨™?˜?¸[È+ÑVÔVñ$
ô $
ˆÔ ð JSÐ\�œ¥e¤k°!°Q¸Ñ&DÔ&DÑEÔEÐEÐX\ˆŒÝœ) L°+È6ÐZdÐeÑeÔeˆŒÝœ, {¸Ô8MÐNÑNÔNˆŒÝ”z FÔ$>Ð?Ñ?Ô?ˆŒˆˆó    Ú
embeddingsÚheightÚwidthÚreturnc                 ó€  — ||z  }t           j                             ¦   «         s$|| j        j        | j        j        z  k    r| j        S |                     d||d¦  «                             dddd¦  «        }t          j	        |||fd¬¦  «        }|                     dd||z  ¦  «                             ddd¦  «        }|S )Nr   éÿÿÿÿr   r   é   Úbilinear)ÚsizeÚmode)
r%   ÚjitÚ
is_tracingr   r   r'   ÚreshapeÚpermuteÚFÚinterpolate)r1   r5   r6   r7   r#   Úinterpolated_embeddingss         r3   Úinterpolate_pos_encodingz+PvtPatchEmbeddings.interpolate_pos_encodingH   sÈ   € Ø˜u‘nˆõ Œy×#Ò#Ñ%Ô%ð 	,¨+¸¼Ô9OÐRVÔR]ÔRhÑ9hÒ*hÐ*hØÔ+Ð+Ø×'Ò'¨¨6°5¸"Ñ=Ô=×EÒEÀaÈÈAÈqÑQÔQˆ
Ý"#¤-°
À&È%ÀÐWaÐ"bÑ"bÔ"bÐØ"9×"AÒ"AÀ!ÀRÈÐRWÉÑ"XÔ"X×"`Ò"`ÐabÐdeÐghÑ"iÔ"iÐØ&Ð&r4   Úpixel_valuesc                 ó–  — |j         \  }}}}|| j        k    rt          d¦  «        ‚|                      |¦  «        }|j         �^ }}}|                     d¦  «                             dd¦  «        }|                      |¦  «        }| j        �†| j                             |dd¦  «        }	t          j
        |	|fd¬¦  «        }|                      | j        d d …dd …f         ||¦  «        }
t          j
        | j        d d …d d…f         |
fd¬¦  «        }
n|                      | j        ||¦  «        }
|                      ||
z   ¦  «        }|||fS )NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.r;   r   r:   ©Údim)Úshaper   Ú
ValueErrorr*   ÚflattenÚ	transposer-   r   Úexpandr%   ÚcatrF   r'   r0   )r1   rG   Ú
batch_sizer   r6   r7   Úpatch_embedÚ_r5   r   r'   s              r3   ÚforwardzPvtPatchEmbeddings.forwardS   so  € Ø2>Ô2DÑ/ˆ
�L &¨%Ø˜4Ô,Ò,Ð,ÝØwñô ð ð —o’o lÑ3Ô3ˆØ'Ô-ÑˆˆF�EØ!×)Ò)¨!Ñ,Ô,×6Ò6°q¸!Ñ<Ô<ˆØ—_’_ [Ñ1Ô1ˆ
ØŒ>Ð%Øœ×-Ò-¨j¸"¸bÑAÔAˆIÝœ I¨zÐ#:ÀÐBÑBÔBˆJØ"&×"?Ò"?ÀÔ@XÐYZÐYZÐYZÐ\]Ð\^Ð\^ÐY^Ô@_ÐagÐinÑ"oÔ"oÐÝ"'¤)¨TÔ-EÀaÀaÀaÈÈ!ÈÀeÔ-LÐNaÐ,bÐhiÐ"jÑ"jÔ"jÐÐà"&×"?Ò"?ÀÔ@XÐZ`ÐbgÑ"hÔ"hÐØ—\’\ *Ð/BÑ"BÑCÔCˆ
à˜6 5Ð(Ð(r4   ©F)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úintr   Úboolr   r%   ÚTensorrF   ÚtuplerT   Ú__classcell__©r2   s   @r3   r   r   %   s  ø€ € € € € ðð ð  ð@ð @àð@ð ˜( 3œ-Ñ'ð@ð ˜( 3œ-Ñ'ð	@ð
 ð@ð ð@ð ð@ð ð@ð @ð @ð @ð @ð @ð8	'°5´<ð 	'Èð 	'ÐUXð 	'Ð]bÔ]ið 	'ð 	'ð 	'ð 	'ð) E¤Lð )°U¸5¼<ÈÈcÐ;QÔ5Rð )ð )ð )ð )ð )ð )ð )ð )r4   r   c                   óL   ‡ — e Zd Zdedefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚPvtSelfOutputr   r   c                 ó¸   •— t          ¦   «                              ¦   «          t          j        ||¦  «        | _        t          j        |j        ¦  «        | _        d S ©N)r   r   r   ÚLinearÚdenser.   r/   r0   )r1   r   r   r2   s      €r3   r   zPvtSelfOutput.__init__j   sD   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜{¨KÑ8Ô8ˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒˆˆr4   Úhidden_statesr8   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rc   )re   r0   ©r1   rf   s     r3   rT   zPvtSelfOutput.forwardo   s*   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØÐr4   ©
rV   rW   rX   r   rZ   r   r%   r\   rT   r^   r_   s   @r3   ra   ra   i   sq   ø€ € € € € ð>˜yð >°sð >ð >ð >ð >ð >ð >ð
 U¤\ð °e´lð ð ð ð ð ð ð ð r4   ra   c                   óŽ   ‡ — e Zd ZdZdedededefˆ fd„Zdedej	        fd	„Z
	 ddej	        dedededeej	                 f
d„Zˆ xZS )ÚPvtEfficientSelfAttentionzxEfficient self-attention mechanism with reduction of the sequence [PvT paper](https://huggingface.co/papers/2102.12122).r   r   Únum_attention_headsÚsequences_reduction_ratioc                 ó
  •— t          ¦   «                              ¦   «          || _        || _        | j        | j        z  dk    r t	          d| j        › d| j        › d�¦  «        ‚t          | j        | j        z  ¦  «        | _        | j        | j        z  | _        t          j	        | j        | j        |j
        ¬¦  «        | _        t          j	        | j        | j        |j
        ¬¦  «        | _        t          j	        | j        | j        |j
        ¬¦  «        | _        t          j        |j        ¦  «        | _        || _        |dk    r?t          j        ||||¬¦  «        | _        t          j        ||j        ¬¦  «        | _        d S d S )	Nr   zThe hidden size (z6) is not a multiple of the number of attention heads (ú))Úbiasr   r   r   )r   r   r   rl   rL   rZ   Úattention_head_sizeÚall_head_sizer   rd   Úqkv_biasÚqueryÚkeyÚvaluer.   Úattention_probs_dropout_probr0   rm   r)   Úsequence_reductionr+   r,   r-   ©r1   r   r   rl   rm   r2   s        €r3   r   z"PvtEfficientSelfAttention.__init__x   s~  ø€ õ 	‰Œ×ÒÑÔÐØ&ˆÔØ#6ˆÔ àÔ˜dÔ6Ñ6¸!Ò;Ð;Ýð6 DÔ$4ð 6ð 6ØÔ2ð6ð 6ð 6ñô ð õ
 $' tÔ'7¸$Ô:RÑ'RÑ#SÔ#SˆÔ Ø!Ô5¸Ô8PÑPˆÔå”Y˜tÔ/°Ô1CÈ&Ì/ÐZÑZÔZˆŒ
Ý”9˜TÔ-¨tÔ/AÈÌÐXÑXÔXˆŒÝ”Y˜tÔ/°Ô1CÈ&Ì/ÐZÑZÔZˆŒ
å”z &Ô"EÑFÔFˆŒà)BˆÔ&Ø$ qÒ(Ð(Ý&(¤iØ˜[Ð6OÐXqð'ñ 'ô 'ˆDÔ#õ !œl¨;¸FÔ<QÐRÑRÔRˆDŒOˆOˆOð	 )Ð(r4   rf   r8   c                 ó²   — |                      ¦   «         d d…         | j        | j        fz   }|                     |¦  «        }|                     dddd¦  «        S )Nr:   r   r;   r   r   )r=   rl   rq   ÚviewrB   )r1   rf   Ú	new_shapes      r3   Útranspose_for_scoresz.PvtEfficientSelfAttention.transpose_for_scores•   sY   € Ø!×&Ò&Ñ(Ô(¨¨"¨Ô-°Ô1IÈ4ÔKcÐ0dÑdˆ	Ø%×*Ò*¨9Ñ5Ô5ˆØ×$Ò$ Q¨¨1¨aÑ0Ô0Ð0r4   Fr6   r7   Úoutput_attentionsc                 ó"  — |                       |                      |¦  «        ¦  «        }| j        dk    rŽ|j        \  }}}|                     ddd¦  «                             ||||¦  «        }|                      |¦  «        }|                     ||d¦  «                             ddd¦  «        }|                      |¦  «        }|                       |                      |¦  «        ¦  «        }	|                       |  	                    |¦  «        ¦  «        }
t          j        ||	                     dd¦  «        ¦  «        }|t          j        | j        ¦  «        z  }t           j                             |d¬¦  «        }|                      |¦  «        }t          j        ||
¦  «        }|                     dddd¦  «                             ¦   «         }|                     ¦   «         d d…         | j        fz   }|                     |¦  «        }|r||fn|f}|S )Nr   r   r;   r:   éþÿÿÿrI   r   )r}   rt   rm   rK   rB   rA   rx   r-   ru   rv   r%   ÚmatmulrN   ÚmathÚsqrtrq   r   Ú
functionalÚsoftmaxr0   Ú
contiguousr=   rr   r{   )r1   rf   r6   r7   r~   Úquery_layerrQ   Úseq_lenr   Ú	key_layerÚvalue_layerÚattention_scoresÚattention_probsÚcontext_layerÚnew_context_layer_shapeÚoutputss                   r3   rT   z!PvtEfficientSelfAttention.forwardš   só  € ð ×/Ò/°·
²
¸=Ñ0IÔ0IÑJÔJˆàÔ)¨AÒ-Ð-Ø0=Ô0CÑ-ˆJ˜ à)×1Ò1°!°Q¸Ñ:Ô:×BÒBÀ:È|Ð]cÐejÑkÔkˆMà ×3Ò3°MÑBÔBˆMà)×1Ò1°*¸lÈBÑOÔO×WÒWÐXYÐ[\Ð^_Ñ`Ô`ˆMØ ŸOšO¨MÑ:Ô:ˆMà×-Ò-¨d¯hªh°}Ñ.EÔ.EÑFÔFˆ	Ø×/Ò/°·
²
¸=Ñ0IÔ0IÑJÔJˆõ !œ<¨°Y×5HÒ5HÈÈRÑ5PÔ5PÑQÔQÐà+­d¬i¸Ô8PÑ.QÔ.QÑQÐõ œ-×/Ò/Ð0@ÀbÐ/ÑIÔIˆð Ÿ,š, Ñ7Ô7ˆåœ _°kÑBÔBˆà%×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔFˆØ"/×"4Ò"4Ñ"6Ô"6°s¸°sÔ";¸tÔ?QÐ>SÑ"SÐØ%×*Ò*Ð+BÑCÔCˆà6GÐ]�= /Ð2Ð2ÈmÐM]ˆàˆr4   rU   )rV   rW   rX   rY   r   rZ   Úfloatr   r%   r\   r}   r[   r]   rT   r^   r_   s   @r3   rk   rk   u   sî   ø€ € € € € ð Cð  CðSØðSØ.1ðSØHKðSØhmðSð Sð Sð Sð Sð Sð:1°#ð 1¸%¼,ð 1ð 1ð 1ð 1ð #(ð*ð *à”|ð*ð ð*ð ð	*ð
  ð*ð 
ˆuŒ|Ô	ð*ð *ð *ð *ð *ð *ð *ð *r4   rk   c                   óp   ‡ — e Zd Zdedededefˆ fd„Z	 ddej        ded	ed
e	de
ej                 f
d„Zˆ xZS )ÚPvtAttentionr   r   rl   rm   c                 ó¤   •— t          ¦   «                              ¦   «          t          ||||¬¦  «        | _        t	          ||¬¦  «        | _        d S )N)r   rl   rm   )r   )r   r   rk   r1   ra   Úoutputry   s        €r3   r   zPvtAttention.__init__È   sW   ø€ õ 	‰Œ×ÒÑÔÐÝ-ØØ#Ø 3Ø&?ð	
ñ 
ô 
ˆŒ	õ $ F¸ÐDÑDÔDˆŒˆˆr4   Frf   r6   r7   r~   r8   c                 óˆ   — |                       ||||¦  «        }|                      |d         ¦  «        }|f|dd …         z   }|S )Nr   r   )r1   r”   )r1   rf   r6   r7   r~   Úself_outputsÚattention_outputr�   s           r3   rT   zPvtAttention.forwardÔ   sM   € ð —y’y °¸Ð?PÑQÔQˆàŸ;š; |°A¤Ñ7Ô7ÐØ#Ð%¨°Q°R°RÔ(8Ñ8ˆØˆr4   rU   )rV   rW   rX   r   rZ   r�   r   r%   r\   r[   r]   rT   r^   r_   s   @r3   r’   r’   Ç   s¸   ø€ € € € € ð
EØð
EØ.1ð
EØHKð
EØhmð
Eð 
Eð 
Eð 
Eð 
Eð 
Eð _dðð Ø"œ\ðØ36ðØ?BðØW[ðà	ˆuŒ|Ô	ðð ð ð ð ð ð ð r4   r’   c            
       óf   ‡ — e Zd Z	 	 d
dedededz  dedz  fˆ fd„Zdej        dej        fd	„Zˆ xZ	S )ÚPvtFFNNr   Úin_featuresÚhidden_featuresÚout_featuresc                 ót  •— t          ¦   «                              ¦   «          |�|n|}t          j        ||¦  «        | _        t          |j        t          ¦  «        rt          |j                 | _	        n|j        | _	        t          j        ||¦  «        | _
        t          j        |j        ¦  «        | _        d S rc   )r   r   r   rd   Údense1r    Ú
hidden_actÚstrr   Úintermediate_act_fnÚdense2r.   r/   r0   )r1   r   rš   r›   rœ   r2   s        €r3   r   zPvtFFN.__init__ß   sš   ø€ õ 	‰Œ×ÒÑÔÐØ'3Ð'?�|�|À[ˆÝ”i ¨_Ñ=Ô=ˆŒÝ�fÔ'­Ñ-Ô-ð 	9Ý'-¨fÔ.?Ô'@ˆDÔ$Ð$à'-Ô'8ˆDÔ$Ý”i °Ñ>Ô>ˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr4   rf   r8   c                 óØ   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rc   )rž   r¡   r0   r¢   rh   s     r3   rT   zPvtFFN.forwardð   s_   € ØŸš MÑ2Ô2ˆØ×0Ò0°Ñ?Ô?ˆØŸš ]Ñ3Ô3ˆØŸš MÑ2Ô2ˆØŸš ]Ñ3Ô3ˆØÐr4   )NNri   r_   s   @r3   r™   r™   Þ   sž   ø€ € € € € ð
 '+Ø#'ð>ð >àð>ð ð>ð ˜t™ð	>ð
 ˜D‘jð>ð >ð >ð >ð >ð >ð" U¤\ð °e´lð ð ð ð ð ð ð ð r4   r™   c                   ó^   ‡ — e Zd ZdZd
deddfˆ fd„Zdej        dej        fd„Zde	fd	„Z
ˆ xZS )ÚPvtDropPathzÏStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    ç        Ú	drop_probr8   Nc                 óV   •— t          ¦   «                              ¦   «          || _        d S rc   )r   r   r§   )r1   r§   r2   s     €r3   r   zPvtDropPath.__init__  s$   ø€ Ý‰Œ×ÒÑÔÐØ"ˆŒˆˆr4   rf   c                 ó  — | j         dk    s| j        s|S d| j         z
  }|j        d         fd|j        dz
  z  z   }t	          j        ||j        |j        ¬¦  «        }t	          j        ||z   ¦  «        }| 	                    |¦  «        |z  S )Nr¦   r   r   )r   )ÚdtypeÚdevice)
r§   ÚtrainingrK   Úndimr%   Úrandrª   r«   ÚfloorÚdiv)r1   rf   Ú	keep_probrK   Úrandom_tensors        r3   rT   zPvtDropPath.forward  s“   € ØŒ>˜SÒ Ð ¨¬Ð Ø Ð Ø˜œÑ&ˆ	ØÔ$ QÔ'Ð)¨D°MÔ4FÈÑ4JÑ,KÑKˆÝœ
 5°Ô0CÈMÔL`ÐaÑaÔaˆÝœ M°IÑ$=Ñ>Ô>ˆØ× Ò  Ñ+Ô+¨mÑ;Ð;r4   c                 ó   — d| j         › �S )Nzp=)r§   )r1   s    r3   Ú
extra_reprzPvtDropPath.extra_repr  s   € Ø$�D”NÐ$Ð$Ð$r4   )r¦   )rV   rW   rX   rY   r�   r   r%   r\   rT   r    r´   r^   r_   s   @r3   r¥   r¥   ú   s›   ø€ € € € € ðð ð#ð # %ð #°$ð #ð #ð #ð #ð #ð #ð< U¤\ð <°e´lð <ð <ð <ð <ð%˜Cð %ð %ð %ð %ð %ð %ð %ð %r4   r¥   c                   ó\   ‡ — e Zd Zdedededededefˆ fd„Zdd	ej        d
edede	fd„Z
ˆ xZS )ÚPvtLayerr   r   rl   Ú	drop_pathrm   Ú	mlp_ratioc                 ó¤  •— t          ¦   «                              ¦   «          t          j        ||j        ¬¦  «        | _        t          ||||¬¦  «        | _        |dk    rt          |¦  «        nt          j	        ¦   «         | _
        t          j        ||j        ¬¦  «        | _        t          ||z  ¦  «        }t          |||¬¦  «        | _        d S )Nr   )r   r   rl   rm   r¦   )r   rš   r›   )r   r   r   r+   r,   Úlayer_norm_1r’   Ú	attentionr¥   ÚIdentityr·   Úlayer_norm_2rZ   r™   Úmlp)	r1   r   r   rl   r·   rm   r¸   Úmlp_hidden_sizer2   s	           €r3   r   zPvtLayer.__init__  sÂ   ø€ õ 	‰Œ×ÒÑÔÐÝœL¨¸&Ô:OÐPÑPÔPˆÔÝ%ØØ#Ø 3Ø&?ð	
ñ 
ô 
ˆŒð 4=¸s²?°?� YÑ/Ô/Ð/ÍÌÉÌˆŒÝœL¨¸&Ô:OÐPÑPÔPˆÔÝ˜k¨IÑ5Ñ6Ô6ˆÝ °[ÐRaÐbÑbÔbˆŒˆˆr4   Frf   r6   r7   r~   c                 óF  — |                       |                      |¦  «        |||¬¦  «        }|d         }|dd …         }|                      |¦  «        }||z   }|                      |                      |¦  «        ¦  «        }|                      |¦  «        }||z   }	|	f|z   }|S )N)rf   r6   r7   r~   r   r   )r»   rº   r·   r¾   r½   )
r1   rf   r6   r7   r~   Úself_attention_outputsr—   r�   Ú
mlp_outputÚlayer_outputs
             r3   rT   zPvtLayer.forward)  s¸   € Ø!%§¢Ø×+Ò+¨MÑ:Ô:ØØØ/ð	 "0ñ "
ô "
Ðð 2°!Ô4ÐØ(¨¨¨Ô,ˆàŸ>š>Ð*:Ñ;Ô;ÐØ(¨=Ñ8ˆà—X’X˜d×/Ò/°Ñ>Ô>Ñ?Ô?ˆ
à—^’^ JÑ/Ô/ˆ
Ø$ zÑ1ˆà�/ GÑ+ˆàˆr4   rU   )rV   rW   rX   r   rZ   r�   r   r%   r\   r[   rT   r^   r_   s   @r3   r¶   r¶     s»   ø€ € € € € ðcàðcð ðcð !ð	cð
 ðcð $)ðcð ðcð cð cð cð cð cð,ð  U¤\ð ¸3ð Àsð Ð_cð ð ð ð ð ð ð ð r4   r¶   c                   ój   ‡ — e Zd Zdefˆ fd„Z	 	 	 ddej        dedz  dedz  d	edz  d
ee	z  f
d„Z
ˆ xZS )Ú
PvtEncoderr   c                 ó”  •— t          ¦   «                              ¦   «          || _        t          j        d|j        t          |j        ¦  «        d¬¦  «                             ¦   «         }g }t          |j
        ¦  «        D ]“}|                     t          ||dk    r|j        n| j        j        d|dz   z  z  |j        |         |j        |         |dk    r|j        n|j        |dz
           |j        |         ||j
        dz
  k    ¬¦  «        ¦  «         Œ”t%          j        |¦  «        | _        g }d}t          |j
        ¦  «        D ]¾}g }|dk    r||j        |dz
           z  }t          |j        |         ¦  «        D ]_}|                     t+          ||j        |         |j        |         |||z            |j        |         |j        |         ¬¦  «        ¦  «         Œ`|                     t%          j        |¦  «        ¦  «         Œ¿t%          j        |¦  «        | _        t%          j        |j        d         |j        ¬	¦  «        | _        d S )
Nr   Úcpu)r«   r;   r   )r   r   r   r   r   r   r   )r   r   rl   r·   rm   r¸   r:   r   )r   r   r   r%   ÚlinspaceÚdrop_path_rateÚsumÚdepthsÚtolistÚrangeÚnum_encoder_blocksÚappendr   r   Úpatch_sizesÚstridesr   Úhidden_sizesr   Ú
ModuleListÚpatch_embeddingsr¶   rl   Úsequence_reduction_ratiosÚ
mlp_ratiosÚblockr+   r,   r-   )
r1   r   Údrop_path_decaysr5   ÚiÚblocksÚcurÚlayersÚjr2   s
            €r3   r   zPvtEncoder.__init__A  sa  ø€ Ý‰Œ×ÒÑÔÐØˆŒõ !œ>¨!¨VÔ-BÅCÈÌÑDVÔDVÐ_dÐeÑeÔe×lÒlÑnÔnÐð ˆ
å�vÔ0Ñ1Ô1ð 	ð 	ˆAØ×ÒÝ"Ø!Ø45¸²F°F˜vÔ0Ð0ÀÄÔ@VÐ[\ÐabÐefÑafÑ[gÑ@hØ%Ô1°!Ô4Ø!œ>¨!Ô,Ø89¸Qº¸ Ô!4Ð!4ÀFÔDWÐXYÐ\]ÑX]ÔD^Ø &Ô 3°AÔ 6Ø 6Ô#<¸qÑ#@Ò@ðñ ô ñ
ô 
ð 
ð 
õ !#¤¨jÑ 9Ô 9ˆÔð ˆØˆÝ�vÔ0Ñ1Ô1ð 	1ð 	1ˆAàˆFØ�AŠvˆvØ�v”} Q¨¡UÔ+Ñ+�Ý˜6œ=¨Ô+Ñ,Ô,ð 
ð 
�Ø—’ÝØ%Ø$*Ô$7¸Ô$:Ø,2Ô,FÀqÔ,IØ"2°3¸±7Ô";Ø28Ô2RÐSTÔ2UØ"(Ô"3°AÔ"6ðñ ô ñ	ô 	ð 	ð 	ð �MŠM�"œ-¨Ñ/Ô/Ñ0Ô0Ð0Ð0å”] 6Ñ*Ô*ˆŒ
õ œ, vÔ':¸2Ô'>ÀFÔDYÐZÑZÔZˆŒˆˆr4   FTrG   r~   NÚoutput_hidden_statesÚreturn_dictr8   c                 ól  — |rdnd }|rdnd }|j         d         }t          | j        ¦  «        }|}	t          t	          | j        | j        ¦  «        ¦  «        D ]‘\  }
\  }} ||	¦  «        \  }	}}|D ].} ||	|||¦  «        }|d         }	|r||d         fz   }|r||	fz   }Œ/|
|dz
  k    r@|	                     |||d¦  «                             dddd¦  «                             ¦   «         }	Œ’|  	                    |	¦  «        }	|r||	fz   }|st          d„ |	||fD ¦   «         ¦  «        S t          |	||¬¦  «        S )	N© r   r   r:   r   r;   c              3   ó   K  — | ]}|®|V — Œ	d S rc   rá   )Ú.0Úvs     r3   ú	<genexpr>z%PvtEncoder.forward.<locals>.<genexpr>‘  s(   è è € ÐmÐm˜qÐ_`Ð_l˜Ð_lÐ_lÐ_lÐ_lÐmÐmr4   ©Úlast_hidden_staterf   Ú
attentions)rK   Úlenr×   Ú	enumerateÚziprÔ   rA   rB   r†   r-   r]   r   )r1   rG   r~   rÞ   rß   Úall_hidden_statesÚall_self_attentionsrQ   Ú
num_blocksrf   ÚidxÚembedding_layerÚblock_layerr6   r7   r×   Úlayer_outputss                    r3   rT   zPvtEncoder.forwards  sµ  € ð #7Ð@˜B˜B¸DÐØ$5Ð?˜b˜b¸4Ðà!Ô'¨Ô*ˆ
Ý˜œ‘_”_ˆ
Ø$ˆÝ3<½SÀÔAVÐX\ÔXbÑ=cÔ=cÑ3dÔ3dð 	vð 	vÑ/ˆCÑ/�/ ;à+:¨?¸=Ñ+IÔ+IÑ(ˆM˜6 5à$ð Mð M�Ø %  m°V¸UÐDUÑ VÔ V�Ø -¨aÔ 0�Ø$ð TØ*=ÀÈqÔAQÐ@SÑ*SÐ'Ø'ð MØ(9¸]Ð<LÑ(LÐ%øØ�j 1‘nÒ$Ð$Ø -× 5Ò 5°jÀ&È%ÐQSÑ TÔ T× \Ò \Ð]^Ð`aÐcdÐfgÑ hÔ h× sÒ sÑ uÔ u�øØŸš¨Ñ6Ô6ˆØð 	EØ 1°]Ð4DÑ DÐØð 	nÝÐmÐm ]Ð4EÐGZÐ$[ÐmÑmÔmÑmÔmÐmÝØ+Ø+Ø*ð
ñ 
ô 
ð 	
r4   )FFT)rV   rW   rX   r   r   r%   ÚFloatTensorr[   r]   r   rT   r^   r_   s   @r3   rÅ   rÅ   @  s´   ø€ € € € € ð0[˜yð 0[ð 0[ð 0[ð 0[ð 0[ð 0[ðj */Ø,1Ø#'ð#
ð #
àÔ'ð#
ð   $™;ð#
ð # T™kð	#
ð
 ˜D‘[ð#
ð 
�Ñ	 ð#
ð #
ð #
ð #
ð #
ð #
ð #
ð #
r4   rÅ   c                   óx   ‡ — e Zd ZU eed<   dZdZdZg Z e	j
        ¦   «         dej        ddfˆ fd„¦   «         Zˆ xZS )	ÚPvtPreTrainedModelr   ÚpvtrG   )ÚimageÚmoduler8   Nc                 óÞ  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          j        t
          j        f¦  «        r@t          j	        |j
        d|¬¦  «         |j        �t          j        |j        ¦  «         dS dS t	          |t          ¦  «        rAt          j	        |j        d|¬¦  «         |j        � t          j	        |j        d|¬¦  «         dS dS dS )zInitialize the weightsr¦   )ÚmeanÚstdN)r   Ú_init_weightsr   Úinitializer_ranger    r   rd   r)   ÚinitÚtrunc_normal_Úweightrp   Úzeros_r   r'   r   )r1   rø   rû   r2   s      €r3   rü   z PvtPreTrainedModel._init_weights¡  sð   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%ØŒkÔ+ˆÝ�f�rœy­"¬)Ð4Ñ5Ô5ð 	HÝÔ˜vœ}°3¸CÐ@Ñ@Ô@Ð@ØŒ{Ð&Ý”˜FœKÑ(Ô(Ð(Ð(Ð(ð 'Ð&å˜Õ 2Ñ3Ô3ð 	HÝÔ˜vÔ9ÀÈÐMÑMÔMÐMØÔÐ+ÝÔ" 6Ô#3¸#À3ÐGÑGÔGÐGÐGÐGð	Hð 	Hà+Ð+r4   )rV   rW   rX   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚ_no_split_modulesr%   Úno_gradr   ÚModulerü   r^   r_   s   @r3   rõ   rõ   ™  s�   ø€ € € € € € àÐÐÑØÐØ$€OØ!ÐØÐà€U„]�_„_ðH B¤Ið H°$ð Hð Hð Hð Hð Hñ „_ðHð Hð Hð Hð Hr4   rõ   c                   óz   ‡ — e Zd Zdefˆ fd„Ze	 	 	 d
dej        dedz  dedz  dedz  de	e
z  f
d	„¦   «         Zˆ xZS )ÚPvtModelr   c                 ó¨   •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        |                      ¦   «          d S rc   )r   r   r   rÅ   ÚencoderÚ	post_init©r1   r   r2   s     €r3   r   zPvtModel.__init__²  sK   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒõ " &Ñ)Ô)ˆŒð 	�ŠÑÔÐÐÐr4   NrG   r~   rÞ   rß   r8   c                 óü   — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|                      ||||¬¦  «        }|d         }|s|f|dd …         z   S t          ||j        |j        ¬¦  «        S )N©rG   r~   rÞ   rß   r   r   ræ   )r   r~   rÞ   rß   r  r   rf   rè   )r1   rG   r~   rÞ   rß   ÚkwargsÚencoder_outputsÚsequence_outputs           r3   rT   zPvtModel.forward¼  s¿   € ð 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàŸ,š,Ø%Ø/Ø!5Ø#ð	 'ñ 
ô 
ˆð *¨!Ô,ˆàð 	<Ø#Ð%¨¸¸¸Ô(;Ñ;Ð;åØ-Ø)Ô7Ø&Ô1ð
ñ 
ô 
ð 	
r4   )NNN)rV   rW   rX   r   r   r   r%   ró   r[   r]   r   rT   r^   r_   s   @r3   r
  r
  °  s¸   ø€ € € € € ð˜yð ð ð ð ð ð ð ð *.Ø,0Ø#'ð
ð 
àÔ'ð
ð   $™;ð
ð # T™kð	
ð
 ˜D‘[ð
ð 
�Ñ	 ð
ð 
ð 
ñ „^ð
ð 
ð 
ð 
ð 
r4   r
  z¤
    Pvt Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.
    )Úcustom_introc                   óš   ‡ — e Zd Zdeddfˆ fd„Ze	 	 	 	 ddej        dz  dej        dz  dedz  dedz  d	edz  de	e
z  fd
„¦   «         Zˆ xZS )ÚPvtForImageClassificationr   r8   Nc                 óB  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        |j        dk    r%t          j        |j        d         |j        ¦  «        nt          j        ¦   «         | _	        |  
                    ¦   «          d S )Nr   r:   )r   r   Ú
num_labelsr
  rö   r   rd   rÒ   r¼   Ú
classifierr  r  s     €r3   r   z"PvtForImageClassification.__init__ä  sŽ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð à Ô+ˆŒÝ˜FÑ#Ô#ˆŒð FLÔEVÐYZÒEZÐEZ�BŒI�fÔ)¨"Ô-¨vÔ/@ÑAÔAÐAÕ`bÔ`kÑ`mÔ`mð 	Œð
 	�ŠÑÔÐÐÐr4   rG   Úlabelsr~   rÞ   rß   c                 óV  — |�|n| j         j        }|                      ||||¬¦  «        }|d         }|                      |dd…ddd…f         ¦  «        }	d}
|�|                      ||	| j         ¦  «        }
|s|	f|dd…         z   }|
�|
f|z   n|S t          |
|	|j        |j        ¬¦  «        S )aŠ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr  r   r   )ÚlossÚlogitsrf   rè   )r   rß   rö   r  Úloss_functionr	   rf   rè   )r1   rG   r  r~   rÞ   rß   r  r�   r  r  r  r”   s               r3   rT   z!PvtForImageClassification.forwardò  së   € ð  &1Ð%<�k�kÀ$Ä+ÔBYˆà—(’(Ø%Ø/Ø!5Ø#ð	 ñ 
ô 
ˆð " !œ*ˆà—’ °°°°A°q°q°q°Ô!9Ñ:Ô:ˆàˆØÐØ×%Ò% f¨f°d´kÑBÔBˆDàð 	FØ�Y ¨¨¨¤Ñ,ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r4   )NNNN)rV   rW   rX   r   r   r   r%   r\   r[   r]   r	   rT   r^   r_   s   @r3   r  r  Ý  sØ   ø€ € € € € ð˜yð ¨Tð ð ð ð ð ð ð ð '+Ø)-Ø,0Ø#'ð)
ð )
à”l TÑ)ð)
ð ”˜tÑ#ð)
ð   $™;ð	)
ð
 # T™kð)
ð ˜D‘[ð)
ð 
Ð&Ñ	&ð)
ð )
ð )
ñ „^ð)
ð )
ð )
ð )
ð )
r4   r  )r  r
  rõ   ))rY   r!   r‚   Úcollections.abcr   r%   Útorch.nn.functionalr   r„   rC   Ú r   rþ   Úactivationsr   Úmodeling_outputsr   r	   Úmodeling_utilsr
   Úutilsr   r   Úconfiguration_pvtr   Ú
get_loggerrV   Úloggerr  r   ra   rk   r’   r™   r¥   r¶   rÅ   rõ   r
  r  Ú__all__rá   r4   r3   ú<module>r*     sK  ðð  Ð à Ð Ð Ð Ø €€€Ø $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ -Ð -Ð -Ð -Ð -Ð -Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø (Ð (Ð (Ð (Ð (Ð (ð 
ˆÔ	˜HÑ	%Ô	%€ðA)ð A)ð A)ð A)ð A)˜œñ A)ô A)ð A)ðH	ð 	ð 	ð 	ð 	�B”Iñ 	ô 	ð 	ðOð Oð Oð Oð O ¤	ñ Oô Oð Oðdð ð ð ð �2”9ñ ô ð ð.ð ð ð ð ˆRŒYñ ô ð ð8%ð %ð %ð %ð %�"”)ñ %ô %ð %ð0+ð +ð +ð +ð +ˆrŒyñ +ô +ð +ð\V
ð V
ð V
ð V
ð V
�”ñ V
ô V
ð V
ðr ðHð Hð Hð Hð H˜ñ Hô Hñ „ðHð, ð)
ð )
ð )
ð )
ð )
Ð!ñ )
ô )
ñ „ð)
ðX €ððñ ô ð9
ð 9
ð 9
ð 9
ð 9
Ð 2ñ 9
ô 9
ñô ð9
ðx JÐ
IÐ
I€€€r4   