§
    ‚ŠtjÕ‡  ã            	       óê  — d Z ddlZddlZddlmZ ddlmZ ddlmZ ddl	m
Z
 ddlmZ dd	lmZmZmZmZ dd
lmZ ddlmZmZmZ ddlmZ  ej        e¦  «        Zd>dedededz  defd„Z G d„ dej        ¦  «        Z  G d„ dej        ¦  «        Z! G d„ dej        ¦  «        Z" G d„ dej        ¦  «        Z# G d„ dej        ¦  «        Z$ G d„ dej        ¦  «        Z% G d „ d!ej        ¦  «        Z& G d"„ d#ej        ¦  «        Z' G d$„ d%ej        ¦  «        Z( G d&„ d'ej        ¦  «        Z) G d(„ d)e¦  «        Z* G d*„ d+ej        ¦  «        Z+e G d,„ d-e¦  «        ¦   «         Z,e G d.„ d/e,¦  «        ¦   «         Z- ed0¬1¦  «         G d2„ d3e,¦  «        ¦   «         Z. G d4„ d5ej        ¦  «        Z/ G d6„ d7ej        ¦  «        Z0 G d8„ d9ej        ¦  «        Z1 ed:¬1¦  «         G d;„ d<e,¦  «        ¦   «         Z2g d=¢Z3dS )?zPyTorch MobileViT model.é    N)Únn)ÚCrossEntropyLossé   )Úinitialization)ÚACT2FN)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithNoAttentionÚ(BaseModelOutputWithPoolingAndNoAttentionÚ$ImageClassifierOutputWithNoAttentionÚSemanticSegmenterOutput)ÚPreTrainedModel)Úauto_docstringÚloggingÚ	torch_inté   )ÚMobileViTConfigé   ÚvalueÚdivisorÚ	min_valueÚreturnc                 ó–   — |€|}t          |t          | |dz  z   ¦  «        |z  |z  ¦  «        }|d| z  k     r||z  }t          |¦  «        S )zU
    Ensure that all layers have a channel count that is divisible by `divisor`.
    Né   gÍÌÌÌÌÌì?)ÚmaxÚint)r   r   r   Ú	new_values       ún/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/mobilevit/modeling_mobilevit.pyÚmake_divisibler   )   s^   € ð ÐØˆ	Ý�I�s 5¨7°Q©;Ñ#6Ñ7Ô7¸7ÑBÀWÑLÑMÔM€Ià�3˜‘;ÒÐØ�WÑˆ	Ýˆy‰>Œ>Ðó    c                   ó„   ‡ — e Zd Z	 	 	 	 	 	 ddededededed	ed
edededeez  ddfˆ fd„Zdej	        dej	        fd„Z
ˆ xZS )ÚMobileViTConvLayerr   FTÚconfigÚin_channelsÚout_channelsÚkernel_sizeÚstrideÚgroupsÚbiasÚdilationÚuse_normalizationÚuse_activationr   Nc                 ón  •— t          ¦   «                              ¦   «          t          |dz
  dz  ¦  «        |z  }||z  dk    rt          d|› d|› d�¦  «        ‚||z  dk    rt          d|› d|› d�¦  «        ‚t	          j        ||||||||d¬	¦	  «	        | _        |	rt	          j        |d
ddd¬¦  «        | _        nd | _        |
rjt          |
t          ¦  «        rt          |
         | _        d S t          |j        t          ¦  «        rt          |j                 | _        d S |j        | _        d S d | _        d S )Nr   r   r   zInput channels (z) are not divisible by z groups.zOutput channels (Úzeros)	r#   r$   r%   r&   Úpaddingr)   r'   r(   Úpadding_modegñhãˆµøä>gš™™™™™¹?T)Únum_featuresÚepsÚmomentumÚaffineÚtrack_running_stats)ÚsuperÚ__init__r   Ú
ValueErrorr   ÚConv2dÚconvolutionÚBatchNorm2dÚnormalizationÚ
isinstanceÚstrr   Ú
activationÚ
hidden_act)Úselfr"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r.   Ú	__class__s               €r   r6   zMobileViTConvLayer.__init__7   sz  ø€ õ 	‰Œ×ÒÑÔÐÝ�{ Q‘¨!Ñ+Ñ,Ô,¨xÑ7ˆà˜Ñ 1Ò$Ð$ÝÐd°ÐdÐdÐTZÐdÐdÐdÑeÔeÐeØ˜&Ñ  AÒ%Ð%ÝÐf°ÐfÐfÐV\ÐfÐfÐfÑgÔgÐgåœ9Ø#Ø%Ø#ØØØØØØ ð

ñ 

ô 

ˆÔð ð 		&Ý!#¤Ø)ØØØØ$(ð"ñ "ô "ˆDÔÐð "&ˆDÔàð 	#Ý˜.­#Ñ.Ô.ð 4Ý"(¨Ô"8�”��Ý˜FÔ-­sÑ3Ô3ð 4Ý"(¨Ô):Ô";�”��à"(Ô"3�”��à"ˆDŒOˆOˆOr   Úfeaturesc                 ó    — |                       |¦  «        }| j        �|                      |¦  «        }| j        �|                      |¦  «        }|S ©N)r9   r;   r>   )r@   rB   s     r   ÚforwardzMobileViTConvLayer.forwardm   sO   € Ø×#Ò# HÑ-Ô-ˆØÔÐ)Ø×)Ò)¨(Ñ3Ô3ˆHØŒ?Ð&Ø—’ xÑ0Ô0ˆHØˆr   )r   r   Fr   TT)Ú__name__Ú
__module__Ú__qualname__r   r   Úboolr=   r6   ÚtorchÚTensorrE   Ú__classcell__©rA   s   @r   r!   r!   6   sí   ø€ € € € € ð ØØØØ"&Ø%)ð4#ð 4#àð4#ð ð4#ð ð	4#ð
 ð4#ð ð4#ð ð4#ð ð4#ð ð4#ð  ð4#ð ˜s™
ð4#ð 
ð4#ð 4#ð 4#ð 4#ð 4#ð 4#ðl ¤ð °´ð ð ð ð ð ð ð ð r   r!   c                   ód   ‡ — e Zd ZdZ	 ddedededededd	fˆ fd
„Zdej        dej        fd„Z	ˆ xZ
S )ÚMobileViTInvertedResidualzY
    Inverted residual block (MobileNetv2): https://huggingface.co/papers/1801.04381
    r   r"   r#   r$   r&   r)   r   Nc           	      ó”  •— t          ¦   «                              ¦   «          t          t          t	          ||j        z  ¦  «        ¦  «        d¦  «        }|dvrt          d|› d�¦  «        ‚|dk    o||k    | _        t          |||d¬¦  «        | _	        t          |||d|||¬¦  «        | _
        t          |||dd	¬
¦  «        | _        d S )Nr   )r   r   zInvalid stride ú.r   ©r#   r$   r%   r   )r#   r$   r%   r&   r'   r)   F©r#   r$   r%   r+   )r5   r6   r   r   ÚroundÚexpand_ratior7   Úuse_residualr!   Ú
expand_1x1Úconv_3x3Ú
reduce_1x1)r@   r"   r#   r$   r&   r)   Úexpanded_channelsrA   s          €r   r6   z"MobileViTInvertedResidual.__init__{   sø   ø€ õ 	‰Œ×ÒÑÔÐÝ*­3­u°[À6ÔCVÑ5VÑ/WÔ/WÑ+XÔ+XÐZ[Ñ\Ô\Ðà˜ÐÐÝÐ8¨vÐ8Ð8Ð8Ñ9Ô9Ð9à# qš[ÐK¨{¸lÒ/JˆÔå,Ø Ð:KÐYZð
ñ 
ô 
ˆŒõ +ØØ)Ø*ØØØ$Øð
ñ 
ô 
ˆŒõ -ØØ)Ø%ØØ ð
ñ 
ô 
ˆŒˆˆr   rB   c                 ó    — |}|                       |¦  «        }|                      |¦  «        }|                      |¦  «        }| j        r||z   n|S rD   )rW   rX   rY   rV   )r@   rB   Úresiduals      r   rE   z!MobileViTInvertedResidual.forwardœ   sR   € Øˆà—?’? 8Ñ,Ô,ˆØ—=’= Ñ*Ô*ˆØ—?’? 8Ñ,Ô,ˆà&*Ô&7ÐEˆx˜(Ñ"Ð"¸XÐEr   ©r   )rF   rG   rH   Ú__doc__r   r   r6   rJ   rK   rE   rL   rM   s   @r   rO   rO   v   s´   ø€ € € € € ðð ð
 jkð
ð 
Ø%ð
Ø47ð
ØGJð
ØTWð
Øcfð
à	ð
ð 
ð 
ð 
ð 
ð 
ðBF ¤ð F°´ð Fð Fð Fð Fð Fð Fð Fð Fr   rO   c                   ó`   ‡ — e Zd Z	 ddedededededdfˆ fd	„Zd
ej        dej        fd„Zˆ xZ	S )ÚMobileViTMobileNetLayerr   r"   r#   r$   r&   Ú
num_stagesr   Nc                 ó
  •— t          ¦   «                              ¦   «          t          j        ¦   «         | _        t          |¦  «        D ]9}t          ||||dk    r|nd¬¦  «        }| j                             |¦  «         |}Œ:d S )Nr   r   )r#   r$   r&   )r5   r6   r   Ú
ModuleListÚlayerÚrangerO   Úappend)	r@   r"   r#   r$   r&   ra   Úird   rA   s	           €r   r6   z MobileViTMobileNetLayer.__init__§   s”   ø€ õ 	‰Œ×ÒÑÔÐå”]‘_”_ˆŒ
Ý�zÑ"Ô"ð 	'ð 	'ˆAÝ-ØØ'Ø)Ø!" a¢ �v�v¨Qð	ñ ô ˆEð ŒJ×Ò˜eÑ$Ô$Ð$Ø&ˆKˆKð	'ð 	'r   rB   c                 ó0   — | j         D ]} ||¦  «        }Œ|S rD   ©rd   )r@   rB   Úlayer_modules      r   rE   zMobileViTMobileNetLayer.forward·   s)   € Ø œJð 	.ð 	.ˆLØ#�| HÑ-Ô-ˆHˆHØˆr   )r   r   ©
rF   rG   rH   r   r   r6   rJ   rK   rE   rL   rM   s   @r   r`   r`   ¦   s›   ø€ € € € € àopð'ð 'Ø%ð'Ø47ð'ØGJð'ØTWð'Øilð'à	ð'ð 'ð 'ð 'ð 'ð 'ð  ¤ð °´ð ð ð ð ð ð ð ð r   r`   c                   óP   ‡ — e Zd Zdededdfˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚMobileViTSelfAttentionr"   Úhidden_sizer   Nc                 ó2  •— t          ¦   «                              ¦   «          ||j        z  dk    rt          d|› d|j        › d�¦  «        ‚|j        | _        t	          ||j        z  ¦  «        | _        | j        | j        z  | _        t          j        || j        |j	        ¬¦  «        | _
        t          j        || j        |j	        ¬¦  «        | _        t          j        || j        |j	        ¬¦  «        | _        t          j        |j        ¦  «        | _        d S )Nr   zThe hidden size z4 is not a multiple of the number of attention heads rQ   )r(   )r5   r6   Únum_attention_headsr7   r   Úattention_head_sizeÚall_head_sizer   ÚLinearÚqkv_biasÚqueryÚkeyr   ÚDropoutÚattention_probs_dropout_probÚdropout©r@   r"   rn   rA   s      €r   r6   zMobileViTSelfAttention.__init__¾   s  ø€ Ý‰Œ×ÒÑÔÐà˜Ô3Ñ3°qÒ8Ð8Ýð7 ;ð 7ð 7ØÔ3ð7ð 7ð 7ñô ð ð
 $*Ô#=ˆÔ Ý#& {°VÔ5OÑ'OÑ#PÔ#PˆÔ Ø!Ô5¸Ô8PÑPˆÔå”Y˜{¨DÔ,>ÀVÄ_ÐUÑUÔUˆŒ
Ý”9˜[¨$Ô*<À6Ä?ÐSÑSÔSˆŒÝ”Y˜{¨DÔ,>ÀVÄ_ÐUÑUÔUˆŒ
å”z &Ô"EÑFÔFˆŒˆˆr   Úhidden_statesc                 óz  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }t          j        ||                     dd¦  «        ¦  «        }|t          j
        | j        ¦  «        z  }t          j                             |d¬¦  «        }|                      |¦  «        }t          j        ||¦  «        }	|	                     dddd¦  «                             ¦   «         }	|	                     ¦   «         d d…         | j        fz   }
 |	j        |
Ž }	|	S )Néÿÿÿÿr   r   éþÿÿÿ©Údimr   r   )Úshaperq   ru   ÚviewÚ	transposerv   r   rJ   ÚmatmulÚmathÚsqrtr   Ú
functionalÚsoftmaxry   ÚpermuteÚ
contiguousÚsizerr   )r@   r{   Úinput_shapeÚhidden_shapeÚquery_layerÚ	key_layerÚvalue_layerÚattention_scoresÚattention_probsÚcontext_layerÚnew_context_layer_shapes              r   rE   zMobileViTSelfAttention.forwardÑ   sš  € Ø#Ô)¨#¨2¨#Ô.ˆØC˜ÐC bÐC¨$Ô*BÐCÐCˆØ—j’j Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆØ—H’H˜]Ñ+Ô+×0Ò0°Ñ>Ô>×HÒHÈÈAÑNÔNˆ	Ø—j’j Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆõ !œ<¨°Y×5HÒ5HÈÈRÑ5PÔ5PÑQÔQÐØ+­d¬i¸Ô8PÑ.QÔ.QÑQÐõ œ-×/Ò/Ð0@ÀbÐ/ÑIÔIˆð Ÿ,š, Ñ7Ô7ˆåœ _°kÑBÔBˆà%×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔFˆØ"/×"4Ò"4Ñ"6Ô"6°s¸°sÔ";¸tÔ?QÐ>SÑ"SÐØ*˜Ô*Ð,CÐDˆØÐr   rk   rM   s   @r   rm   rm   ½   s�   ø€ € € € € ðG˜ð G¸Sð GÀTð Gð Gð Gð Gð Gð Gð& U¤\ð °e´lð ð ð ð ð ð ð ð r   rm   c                   óP   ‡ — e Zd Zdededdfˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚMobileViTSelfOutputr"   rn   r   Nc                 ó¸   •— t          ¦   «                              ¦   «          t          j        ||¦  «        | _        t          j        |j        ¦  «        | _        d S rD   ©r5   r6   r   rs   Údenserw   Úhidden_dropout_probry   rz   s      €r   r6   zMobileViTSelfOutput.__init__ì   sD   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜{¨KÑ8Ô8ˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒˆˆr   r{   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rD   ©r™   ry   ©r@   r{   s     r   rE   zMobileViTSelfOutput.forwardñ   s*   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØÐr   rk   rM   s   @r   r–   r–   ë   sx   ø€ € € € € ð>˜ð >¸Sð >ÀTð >ð >ð >ð >ð >ð >ð
 U¤\ð °e´lð ð ð ð ð ð ð ð r   r–   c                   óP   ‡ — e Zd Zdededdfˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚMobileViTAttentionr"   rn   r   Nc                 óœ   •— t          ¦   «                              ¦   «          t          ||¦  «        | _        t	          ||¦  «        | _        d S rD   )r5   r6   rm   Ú	attentionr–   Úoutputrz   s      €r   r6   zMobileViTAttention.__init__ø   s?   ø€ Ý‰Œ×ÒÑÔÐÝ/°¸ÑDÔDˆŒÝ)¨&°+Ñ>Ô>ˆŒˆˆr   r{   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rD   )r¡   r¢   )r@   r{   Úself_outputsÚattention_outputs       r   rE   zMobileViTAttention.forwardý   s+   € Ø—~’~ mÑ4Ô4ˆØŸ;š; |Ñ4Ô4ÐØÐr   rk   rM   s   @r   rŸ   rŸ   ÷   sx   ø€ € € € € ð?˜ð ?¸Sð ?ÀTð ?ð ?ð ?ð ?ð ?ð ?ð
  U¤\ð  °e´lð  ð  ð  ð  ð  ð  ð  ð  r   rŸ   c                   óT   ‡ — e Zd Zdedededdfˆ fd„Zdej        dej        fd„Zˆ xZ	S )	ÚMobileViTIntermediater"   rn   Úintermediate_sizer   Nc                 óú   •— t          ¦   «                              ¦   «          t          j        ||¦  «        | _        t          |j        t          ¦  «        rt          |j                 | _	        d S |j        | _	        d S rD   )
r5   r6   r   rs   r™   r<   r?   r=   r   Úintermediate_act_fn©r@   r"   rn   r¨   rA   s       €r   r6   zMobileViTIntermediate.__init__  si   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜{Ð,=Ñ>Ô>ˆŒ
Ý�fÔ'­Ñ-Ô-ð 	9Ý'-¨fÔ.?Ô'@ˆDÔ$Ð$Ð$à'-Ô'8ˆDÔ$Ð$Ð$r   r{   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rD   )r™   rª   r�   s     r   rE   zMobileViTIntermediate.forward  s,   € ØŸ
š
 =Ñ1Ô1ˆØ×0Ò0°Ñ?Ô?ˆØÐr   rk   rM   s   @r   r§   r§     s�   ø€ € € € € ð9˜ð 9¸Sð 9ÐUXð 9Ð]að 9ð 9ð 9ð 9ð 9ð 9ð U¤\ð °e´lð ð ð ð ð ð ð ð r   r§   c                   ób   ‡ — e Zd Zdedededdfˆ fd„Zdej        dej        dej        fd	„Zˆ xZ	S )
ÚMobileViTOutputr"   rn   r¨   r   Nc                 ó¸   •— t          ¦   «                              ¦   «          t          j        ||¦  «        | _        t          j        |j        ¦  «        | _        d S rD   r˜   r«   s       €r   r6   zMobileViTOutput.__init__  sE   ø€ Ý‰Œ×ÒÑÔÐÝ”YÐ0°+Ñ>Ô>ˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒˆˆr   r{   Úinput_tensorc                 ód   — |                       |¦  «        }|                      |¦  «        }||z   }|S rD   rœ   )r@   r{   r°   s      r   rE   zMobileViTOutput.forward  s4   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØ%¨Ñ4ˆØÐr   rk   rM   s   @r   r®   r®     sŒ   ø€ € € € € ð>˜ð >¸Sð >ÐUXð >Ð]að >ð >ð >ð >ð >ð >ð
 U¤\ð ÀÄð ÐRWÔR^ð ð ð ð ð ð ð ð r   r®   c                   óT   ‡ — e Zd Zdedededdfˆ fd„Zdej        dej        fd„Zˆ xZ	S )	ÚMobileViTTransformerLayerr"   rn   r¨   r   Nc                 óJ  •— t          ¦   «                              ¦   «          t          ||¦  «        | _        t	          |||¦  «        | _        t          |||¦  «        | _        t          j	        ||j
        ¬¦  «        | _        t          j	        ||j
        ¬¦  «        | _        d S )N©r1   )r5   r6   rŸ   r¡   r§   Úintermediater®   r¢   r   Ú	LayerNormÚlayer_norm_epsÚlayernorm_beforeÚlayernorm_afterr«   s       €r   r6   z"MobileViTTransformerLayer.__init__   s�   ø€ Ý‰Œ×ÒÑÔÐÝ+¨F°KÑ@Ô@ˆŒÝ1°&¸+ÐGXÑYÔYˆÔÝ% f¨kÐ;LÑMÔMˆŒÝ "¤¨[¸fÔ>SÐ TÑ TÔ TˆÔÝ!œ|¨K¸VÔ=RÐSÑSÔSˆÔÐÐr   r{   c                 óà   — |                       |                      |¦  «        ¦  «        }||z   }|                      |¦  «        }|                      |¦  «        }|                      ||¦  «        }|S rD   )r¡   r¹   rº   r¶   r¢   )r@   r{   r¥   Úlayer_outputs       r   rE   z!MobileViTTransformerLayer.forward(  sk   € ØŸ>š>¨$×*?Ò*?ÀÑ*NÔ*NÑOÔOÐØ(¨=Ñ8ˆà×+Ò+¨MÑ:Ô:ˆØ×(Ò(¨Ñ6Ô6ˆØ—{’{ <°Ñ?Ô?ˆØÐr   rk   rM   s   @r   r³   r³     s‹   ø€ € € € € ðT˜ð T¸Sð TÐUXð TÐ]að Tð Tð Tð Tð Tð Tð U¤\ð °e´lð ð ð ð ð ð ð ð r   r³   c                   óT   ‡ — e Zd Zdedededdfˆ fd„Zdej        dej        fd„Zˆ xZ	S )	ÚMobileViTTransformerr"   rn   ra   r   Nc           	      ó  •— t          ¦   «                              ¦   «          t          j        ¦   «         | _        t          |¦  «        D ]C}t          ||t          ||j        z  ¦  «        ¬¦  «        }| j         	                    |¦  «         ŒDd S )N)rn   r¨   )
r5   r6   r   rc   rd   re   r³   r   Ú	mlp_ratiorf   )r@   r"   rn   ra   Ú_Útransformer_layerrA   s         €r   r6   zMobileViTTransformer.__init__3  s‘   ø€ Ý‰Œ×ÒÑÔÐå”]‘_”_ˆŒ
Ý�zÑ"Ô"ð 	1ð 	1ˆAÝ 9ØØ'Ý"% k°FÔ4DÑ&DÑ"EÔ"Eð!ñ !ô !Ðð
 ŒJ×ÒÐ/Ñ0Ô0Ð0Ð0ð	1ð 	1r   r{   c                 ó0   — | j         D ]} ||¦  «        }Œ|S rD   ri   )r@   r{   rj   s      r   rE   zMobileViTTransformer.forward?  s*   € Ø œJð 	8ð 	8ˆLØ(˜L¨Ñ7Ô7ˆMˆMØÐr   rk   rM   s   @r   r¾   r¾   2  s€   ø€ € € € € ð
1˜ð 
1¸Sð 
1Ècð 
1ÐVZð 
1ð 
1ð 
1ð 
1ð 
1ð 
1ð U¤\ð °e´lð ð ð ð ð ð ð ð r   r¾   c                   óÈ   ‡ — e Zd ZdZ	 ddedededededed	ed
dfˆ fd„Zdej        d
e	ej        e
f         fd„Zdej        de
d
ej        fd„Zdej        d
ej        fd„Zˆ xZS )ÚMobileViTLayerzC
    MobileViT block: https://huggingface.co/papers/2110.02178
    r   r"   r#   r$   r&   rn   ra   r)   r   Nc                 ó<  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |dk    r/t          ||||dk    r|nd|dk    r|dz  nd¬¦  «        | _        |}nd | _        t          ||||j        ¬¦  «        | _	        t          |||ddd¬¦  «        | _
        t          |||¬¦  «        | _        t          j        ||j        ¬¦  «        | _        t          |||d¬¦  «        | _        t          |d|z  ||j        ¬¦  «        | _        d S )	Nr   r   )r#   r$   r&   r)   rR   F)r#   r$   r%   r*   r+   )rn   ra   rµ   )r5   r6   Ú
patch_sizeÚpatch_widthÚpatch_heightrO   Údownsampling_layerr!   Úconv_kernel_sizeÚconv_kxkÚconv_1x1r¾   Útransformerr   r·   r¸   Ú	layernormÚconv_projectionÚfusion)	r@   r"   r#   r$   r&   rn   ra   r)   rA   s	           €r   r6   zMobileViTLayer.__init__J  sg  ø€ õ 	‰Œ×ÒÑÔÐØ!Ô,ˆÔØ"Ô-ˆÔà�QŠ;ˆ;Ý&?ØØ'Ø)Ø!)¨Q¢ �v�v°AØ*2°Qª,¨,˜ Q™˜¸Að'ñ 'ô 'ˆDÔ#ð 'ˆKˆKà&*ˆDÔ#å*ØØ#Ø$ØÔ/ð	
ñ 
ô 
ˆŒõ +ØØ#Ø$ØØ#Ø ð
ñ 
ô 
ˆŒõ 0ØØ#Ø!ð
ñ 
ô 
ˆÔõ œ k°vÔ7LÐMÑMÔMˆŒå1Ø ¸+ÐSTð 
ñ  
ô  
ˆÔõ )Ø  K¡¸kÐW]ÔWnð
ñ 
ô 
ˆŒˆˆr   rB   c                 óª  — | j         | j        }}t          ||z  ¦  «        }|j        \  }}}}t          j                             ¦   «         r't          t	          j        ||z  ¦  «        |z  ¦  «        n&t          t          j        ||z  ¦  «        |z  ¦  «        }	t          j                             ¦   «         r't          t	          j        ||z  ¦  «        |z  ¦  «        n&t          t          j        ||z  ¦  «        |z  ¦  «        }
d}|
|k    s|	|k    r't          j                             ||	|
fdd¬¦  «        }d}|
|z  }|	|z  }||z  }|                     ||z  |z  |||¦  «        }|                     dd¦  «        }|                     ||||¦  «        }|                     dd¦  «        }|                     ||z  |d¦  «        }||f||||||d	œ}||fS )
NFÚbilinear©r‹   ÚmodeÚalign_cornersTr   r   r   r}   )Ú	orig_sizeÚ
batch_sizeÚchannelsÚinterpolateÚnum_patchesÚnum_patches_widthÚnum_patches_height)rÈ   rÉ   r   r�   rJ   ÚjitÚ
is_tracingr   Úceilr…   r   r‡   rÚ   Úreshaperƒ   )r@   rB   rÈ   rÉ   Ú
patch_arearØ   rÙ   Úorig_heightÚ
orig_widthÚ
new_heightÚ	new_widthrÚ   Únum_patch_widthÚnum_patch_heightrÛ   ÚpatchesÚ	info_dicts                    r   Ú	unfoldingzMobileViTLayer.unfolding„  s  € Ø$(Ô$4°dÔ6G�\ˆÝ˜ |Ñ3Ñ4Ô4ˆ
à8@¼Ñ5ˆ
�H˜k¨:õ Œy×#Ò#Ñ%Ô%ðK�I•e”j ¨|Ñ!;Ñ<Ô<¸|ÑKÑLÔLÐLå•T”Y˜{¨\Ñ9Ñ:Ô:¸\ÑIÑJÔJð 	õ Œy×#Ò#Ñ%Ô%ðH�I•e”j ¨kÑ!9Ñ:Ô:¸[ÑHÑIÔIÐIå•T”Y˜z¨KÑ7Ñ8Ô8¸;ÑFÑGÔGð 	ð ˆØ˜
Ò"Ð" j°KÒ&?Ð&?å”}×0Ò0Ø 
¨IÐ6¸ZÐW\ð 1ñ ô ˆHð ˆKð $ {Ñ2ˆØ%¨Ñ5ÐØ&¨Ñ8ˆð ×"Ò"Ø˜Ñ!Ð$4Ñ4°lÀOÐU`ñ
ô 
ˆð ×#Ò# A qÑ)Ô)ˆØ—/’/ *¨h¸ÀZÑPÔPˆØ×#Ò# A qÑ)Ô)ˆØ—/’/ *¨zÑ"9¸;ÈÑKÔKˆð & zÐ2Ø$Ø Ø&Ø&Ø!0Ø"2ð
ð 
ˆ	ð ˜	Ð!Ð!r   ré   rê   c                 ó  — | j         | j        }}t          ||z  ¦  «        }|d         }|d         }|d         }|d         }	|d         }
|                     ¦   «                              |||d¦  «        }|                     dd¦  «        }|                     ||z  |	z  |
||¦  «        }|                     dd	¦  «        }|                     |||	|z  |
|z  ¦  «        }|d
         r)t          j         	                    ||d         dd¬¦  «        }|S )NrØ   rÙ   rÛ   rÝ   rÜ   r}   r   r   r   rÚ   r×   rÓ   FrÔ   )
rÈ   rÉ   r   rŠ   r‚   rƒ   rá   r   r‡   rÚ   )r@   ré   rê   rÈ   rÉ   râ   rØ   rÙ   rÛ   rè   rç   rB   s               r   ÚfoldingzMobileViTLayer.folding·  sA  € Ø$(Ô$4°dÔ6G�\ˆÝ˜ |Ñ3Ñ4Ô4ˆ
à˜|Ô,ˆ
Ø˜ZÔ(ˆØ Ô.ˆØ$Ð%9Ô:ÐØ#Ð$7Ô8ˆð ×%Ò%Ñ'Ô'×,Ò,¨Z¸À[ÐRTÑUÔUˆØ×%Ò% a¨Ñ+Ô+ˆØ×#Ò#Ø˜Ñ!Ð$4Ñ4°oÀ|ÐU`ñ
ô 
ˆð ×%Ò% a¨Ñ+Ô+ˆØ×#Ò#Ø˜Ð"2°\Ñ"AÀ?ÐU`ÑC`ñ
ô 
ˆð �]Ô#ð 	Ý”}×0Ò0Ø˜y¨Ô5¸JÐV[ð 1ñ ô ˆHð ˆr   c                 óÆ  — | j         r|                       |¦  «        }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        \  }}|                      |¦  «        }|                      |¦  «        }|                      ||¦  «        }|                      |¦  «        }|                      t          j
        ||fd¬¦  «        ¦  «        }|S ©Nr   r   )rÊ   rÌ   rÍ   rë   rÎ   rÏ   rí   rÐ   rÑ   rJ   Úcat)r@   rB   r\   ré   rê   s        r   rE   zMobileViTLayer.forwardÔ  sÙ   € àÔ"ð 	9Ø×.Ò.¨xÑ8Ô8ˆHàˆð —=’= Ñ*Ô*ˆØ—=’= Ñ*Ô*ˆð "Ÿ^š^¨HÑ5Ô5Ñˆ�ð ×"Ò" 7Ñ+Ô+ˆØ—.’. Ñ)Ô)ˆð —<’< ¨Ñ3Ô3ˆà×'Ò'¨Ñ1Ô1ˆØ—;’;�uœy¨(°HÐ)=À1ÐEÑEÔEÑFÔFˆØˆr   r]   )rF   rG   rH   r^   r   r   r6   rJ   rK   ÚtupleÚdictrë   rí   rE   rL   rM   s   @r   rÅ   rÅ   E  s$  ø€ € € € € ðð ð ð8
ð 8
àð8
ð ð8
ð ð	8
ð
 ð8
ð ð8
ð ð8
ð ð8
ð 
ð8
ð 8
ð 8
ð 8
ð 8
ð 8
ðt1" %¤,ð 1"°5¸¼ÀtÐ9KÔ3Lð 1"ð 1"ð 1"ð 1"ðf˜uœ|ð ¸ð ÀÄð ð ð ð ð: ¤ð °´ð ð ð ð ð ð ð ð r   rÅ   c            
       óV   ‡ — e Zd Zdeddfˆ fd„Z	 	 ddej        ded	edee	z  fd
„Z
ˆ xZS )ÚMobileViTEncoderr"   r   Nc           	      óê  •— t          ¦   «                              ¦   «          || _        t          j        ¦   «         | _        d| _        dx}}|j        dk    rd}d}n|j        dk    rd}d}t          ||j	        d         |j	        d         dd¬¦  «        }| j         
                    |¦  «         t          ||j	        d         |j	        d         dd	¬¦  «        }| j         
                    |¦  «         t          ||j	        d         |j	        d	         d|j        d         d¬
¦  «        }| j         
                    |¦  «         |r|dz  }t          ||j	        d	         |j	        d         d|j        d         d|¬¦  «        }| j         
                    |¦  «         |r|dz  }t          ||j	        d         |j	        d         d|j        d         d	|¬¦  «        }	| j         
                    |	¦  «         d S )NFr   Té   r   r   )r#   r$   r&   ra   r   r   )r#   r$   r&   rn   ra   é   )r#   r$   r&   rn   ra   r)   é   )r5   r6   r"   r   rc   rd   Úgradient_checkpointingÚoutput_strider`   Úneck_hidden_sizesrf   rÅ   Úhidden_sizes)r@   r"   Údilate_layer_4Údilate_layer_5r)   Úlayer_1Úlayer_2Úlayer_3Úlayer_4Úlayer_5rA   s             €r   r6   zMobileViTEncoder.__init__ï  s9  ø€ Ý‰Œ×ÒÑÔÐØˆŒå”]‘_”_ˆŒ
Ø&+ˆÔ#ð +0Ð/ˆ˜ØÔ 1Ò$Ð$Ø!ˆNØ!ˆNˆNØÔ! RÒ'Ð'Ø!ˆNàˆå)ØØÔ0°Ô3ØÔ1°!Ô4ØØð
ñ 
ô 
ˆð 	Œ
×Ò˜'Ñ"Ô"Ð"å)ØØÔ0°Ô3ØÔ1°!Ô4ØØð
ñ 
ô 
ˆð 	Œ
×Ò˜'Ñ"Ô"Ð"å ØØÔ0°Ô3ØÔ1°!Ô4ØØÔ+¨AÔ.Øð
ñ 
ô 
ˆð 	Œ
×Ò˜'Ñ"Ô"Ð"àð 	Ø˜‰MˆHå ØØÔ0°Ô3ØÔ1°!Ô4ØØÔ+¨AÔ.ØØð
ñ 
ô 
ˆð 	Œ
×Ò˜'Ñ"Ô"Ð"àð 	Ø˜‰MˆHå ØØÔ0°Ô3ØÔ1°!Ô4ØØÔ+¨AÔ.ØØð
ñ 
ô 
ˆð 	Œ
×Ò˜'Ñ"Ô"Ð"Ð"Ð"r   FTr{   Úoutput_hidden_statesÚreturn_dictc                 óÄ   — |rdnd }t          | j        ¦  «        D ]\  }} ||¦  «        }|r||fz   }Œ|st          d„ ||fD ¦   «         ¦  «        S t          ||¬¦  «        S )N© c              3   ó   K  — | ]}|®|V — Œ	d S rD   r  )Ú.0Úvs     r   ú	<genexpr>z+MobileViTEncoder.forward.<locals>.<genexpr>H  s"   è è € ÐXÐX˜qÈ!È-˜È-È-È-È-ÐXÐXr   )Úlast_hidden_stater{   )Ú	enumeraterd   rñ   r	   )r@   r{   r  r  Úall_hidden_statesrg   rj   s          r   rE   zMobileViTEncoder.forward9  s™   € ð #7Ð@˜B˜B¸DÐå(¨¬Ñ4Ô4ð 	Ið 	I‰OˆAˆ|Ø(˜L¨Ñ7Ô7ˆMà#ð IØ$5¸Ð8HÑ$HÐ!øàð 	YÝÐXÐX ]Ð4EÐ$FÐXÑXÔXÑXÔXÐXå-ÀÐ]nÐoÑoÔoÐor   )FT)rF   rG   rH   r   r6   rJ   rK   rI   rñ   r	   rE   rL   rM   s   @r   rô   rô   î  s°   ø€ € € € € ðH#˜ð H#°4ð H#ð H#ð H#ð H#ð H#ð H#ðZ &+Ø ð	pð pà”|ðpð #ðpð ð	pð
 
Ð/Ñ	/ðpð pð pð pð pð pð pð pr   rô   c                   ó~   ‡ — e Zd ZU eed<   dZdZdZdZdgZ	 e
j        ¦   «         dej        dd	fˆ fd
„¦   «         Zˆ xZS )ÚMobileViTPreTrainedModelr"   Ú	mobilevitÚpixel_values)ÚimageTrÅ   Úmoduler   Nc                 óÊ  •— t          ¦   «                              |¦  «         t          |t          j        ¦  «        r¤t          j        |j        d| j        j	        ¬¦  «         |j
        �t          j        |j
        ¦  «         t          |dd¦  «        �Ot          j        |j        ¦  «         t          j        |j        ¦  «         t          j        |j        ¦  «         dS dS dS )zInitialize the weightsg        )ÚmeanÚstdNÚrunning_mean)r5   Ú_init_weightsr<   r   r:   ÚinitÚnormal_Úweightr"   Úinitializer_ranger(   Úzeros_Úgetattrr  Úones_Úrunning_varÚnum_batches_tracked)r@   r  rA   s     €r   r  z&MobileViTPreTrainedModel._init_weightsV  sÍ   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�bœnÑ-Ô-ð 	8ÝŒL˜œ¨S°d´kÔ6SÐTÑTÔTÐTØŒ{Ð&Ý”˜FœKÑ(Ô(Ð(Ý�v˜~¨tÑ4Ô4Ð@Ý”˜FÔ/Ñ0Ô0Ð0Ý”
˜6Ô-Ñ.Ô.Ð.Ý”˜FÔ6Ñ7Ô7Ð7Ð7Ð7ð	8ð 	8ð AÐ@r   )rF   rG   rH   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesrJ   Úno_gradr   ÚModuler  rL   rM   s   @r   r  r  M  s�   ø€ € € € € € àÐÐÑØ#ÐØ$€OØ!ÐØ&*Ð#Ø)Ð*Ðà€U„]�_„_ð
8 B¤Ið 
8°$ð 
8ð 
8ð 
8ð 
8ð 
8ñ „_ð
8ð 
8ð 
8ð 
8ð 
8r   r  c                   ó|   ‡ — e Zd Zddedefˆ fd„Ze	 	 	 ddej        dz  dedz  dedz  d	e	e
z  fd
„¦   «         Zˆ xZS )ÚMobileViTModelTr"   Úexpand_outputc                 ór  •— t          ¦   «                              |¦  «         || _        || _        t	          ||j        |j        d         dd¬¦  «        | _        t          |¦  «        | _	        | j        r.t	          ||j        d         |j        d         d¬¦  «        | _
        |                      ¦   «          d	S )
aE  
        expand_output (`bool`, *optional*, defaults to `True`):
            Whether to expand the output of the model using a 1x1 convolution. If `True`, the model will apply an additional
            1x1 convolution to expand the output channels from `config.neck_hidden_sizes[5]` to `config.neck_hidden_sizes[6]`.
        r   r   r   )r#   r$   r%   r&   rø   é   r   rR   N)r5   r6   r"   r-  r!   Únum_channelsrû   Ú	conv_stemrô   ÚencoderÚconv_1x1_expÚ	post_init)r@   r"   r-  rA   s      €r   r6   zMobileViTModel.__init__f  sÃ   ø€ õ 	‰Œ×Ò˜Ñ Ô Ð ØˆŒØ*ˆÔå+ØØÔ+ØÔ1°!Ô4ØØð
ñ 
ô 
ˆŒõ (¨Ñ/Ô/ˆŒàÔð 	Ý 2ØØ"Ô4°QÔ7Ø#Ô5°aÔ8Øð	!ñ !ô !ˆDÔð 	�ŠÑÔÐÐÐr   Nr  r  r  r   c                 ó¨  — |�|n| j         j        }|�|n| j         j        }|€t          d¦  «        ‚|                      |¦  «        }|                      |||¬¦  «        }| j        r5|                      |d         ¦  «        }t          j	        |ddgd¬¦  «        }n
|d         }d }|s|�||fn|f}	|	|dd …         z   S t          |||j        ¬	¦  «        S )
Nz You have to specify pixel_values©r  r  r   r~   r}   F)r€   Úkeepdimr   )r  Úpooler_outputr{   )r"   r  r  r7   r1  r2  r-  r3  rJ   r  r
   r{   )
r@   r  r  r  ÚkwargsÚembedding_outputÚencoder_outputsr  Úpooled_outputr¢   s
             r   rE   zMobileViTModel.forward…  s'  € ð %9Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàÐÝÐ?Ñ@Ô@Ð@àŸ>š>¨,Ñ7Ô7ÐàŸ,š,ØØ!5Ø#ð 'ñ 
ô 
ˆð Ôð 	!Ø $× 1Ò 1°/À!Ô2DÑ EÔ EÐõ "œJÐ'8¸rÀ2¸hÐPUÐVÑVÔVˆMˆMà /°Ô 2ÐØ ˆMàð 	0Ø;HÐ;TÐ'¨Ð7Ð7Ð[lÐZnˆFØ˜O¨A¨B¨BÔ/Ñ/Ð/å7Ø/Ø'Ø)Ô7ð
ñ 
ô 
ð 	
r   )T)NNN)rF   rG   rH   r   rI   r6   r   rJ   rK   rñ   r
   rE   rL   rM   s   @r   r,  r,  d  s»   ø€ € € € € ðð ˜ð ¸tð ð ð ð ð ð ð> ð -1Ø,0Ø#'ð	(
ð (
à”l TÑ)ð(
ð # T™kð(
ð ˜D‘[ð	(
ð 
Ð9Ñ	9ð(
ð (
ð (
ñ „^ð(
ð (
ð (
ð (
ð (
r   r,  z‰
    MobileViT model with an image classification head on top (a linear layer on top of the pooled features), e.g. for
    ImageNet.
    )Úcustom_introc                   ó�   ‡ — e Zd Zdeddfˆ fd„Ze	 	 	 	 d
dej        dz  dedz  dej        dz  dedz  de	e
z  f
d	„¦   «         Zˆ xZS )ÚMobileViTForImageClassificationr"   r   Nc                 ó‚  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        d¬¦  «        | _        |j        dk    r%t          j	        |j
        d         |j        ¦  «        nt          j        ¦   «         | _        |                      ¦   «          d S )NT)Úinplacer   r}   )r5   r6   Ú
num_labelsr,  r  r   rw   Úclassifier_dropout_probry   rs   rû   ÚIdentityÚ
classifierr4  ©r@   r"   rA   s     €r   r6   z(MobileViTForImageClassification.__init__¸  s¥   ø€ Ý‰Œ×Ò˜Ñ Ô Ð à Ô+ˆŒÝ'¨Ñ/Ô/ˆŒõ ”z &Ô"@È$ÐOÑOÔOˆŒàJPÔJ[Ð^_ÒJ_ÐJ_�BŒI�fÔ.¨rÔ2°FÔ4EÑFÔFÐFÕegÔepÑerÔerð 	Œð
 	�ŠÑÔÐÐÐr   r  r  Úlabelsr  c                 óf  — |�|n| j         j        }|                      |||¬¦  «        }|r|j        n|d         }|                      |                      |¦  «        ¦  «        }d}	|�|                      ||| j         ¦  «        }	|s|f|dd…         z   }
|	�|	f|
z   n|
S t          |	||j        ¬¦  «        S )aŠ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss). If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr6  r   r   )ÚlossÚlogitsr{   )	r"   r  r  r8  rE  ry   Úloss_functionr   r{   )r@   r  r  rG  r  r9  Úoutputsr<  rJ  rI  r¢   s              r   rE   z'MobileViTForImageClassification.forwardÇ  sÞ   € ð &1Ð%<�k�kÀ$Ä+ÔBYˆà—.’. ÐDXÐfq�.ÑrÔrˆà1<ÐL˜Ô-Ð-À'È!Ä*ˆà—’ §¢¨mÑ!<Ô!<Ñ=Ô=ˆàˆØÐØ×%Ò% f¨f°d´kÑBÔBˆDàð 	FØ�Y ¨¨¨¤Ñ,ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå3ØØØ!Ô/ð
ñ 
ô 
ð 	
r   ©NNNN)rF   rG   rH   r   r6   r   rJ   rK   rI   rñ   r   rE   rL   rM   s   @r   r?  r?  ±  sÊ   ø€ € € € € ð˜ð °4ð ð ð ð ð ð ð ð -1Ø,0Ø&*Ø#'ð"
ð "
à”l TÑ)ð"
ð # T™kð"
ð ”˜tÑ#ð	"
ð
 ˜D‘[ð"
ð 
Ð5Ñ	5ð"
ð "
ð "
ñ „^ð"
ð "
ð "
ð "
ð "
r   r?  c                   óT   ‡ — e Zd Zdedededdfˆ fd„Zdej        dej        fd„Zˆ xZ	S )	ÚMobileViTASPPPoolingr"   r#   r$   r   Nc           	      ó²   •— t          ¦   «                              ¦   «          t          j        d¬¦  «        | _        t          |||dddd¬¦  «        | _        d S )Nr   )Úoutput_sizeTÚrelu)r#   r$   r%   r&   r*   r+   )r5   r6   r   ÚAdaptiveAvgPool2dÚglobal_poolr!   rÍ   )r@   r"   r#   r$   rA   s       €r   r6   zMobileViTASPPPooling.__init__î  s^   ø€ Ý‰Œ×ÒÑÔÐåÔ/¸AÐ>Ñ>Ô>ˆÔå*ØØ#Ø%ØØØ"Ø!ð
ñ 
ô 
ˆŒˆˆr   rB   c                 ó¾   — |j         dd …         }|                      |¦  «        }|                      |¦  «        }t          j                             ||dd¬¦  «        }|S )Nr~   rÓ   FrÔ   )r�   rT  rÍ   r   r‡   rÚ   )r@   rB   Úspatial_sizes      r   rE   zMobileViTASPPPooling.forwardý  sZ   € Ø”~ b c cÔ*ˆØ×#Ò# HÑ-Ô-ˆØ—=’= Ñ*Ô*ˆÝ”=×,Ò,¨X¸LÈzÐinÐ,ÑoÔoˆØˆr   rk   rM   s   @r   rO  rO  í  s�   ø€ € € € € ð
˜ð 
¸Sð 
ÐPSð 
ÐX\ð 
ð 
ð 
ð 
ð 
ð 
ð ¤ð °´ð ð ð ð ð ð ð ð r   rO  c                   óP   ‡ — e Zd ZdZdeddfˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚMobileViTASPPzƒ
    ASPP module defined in DeepLab papers: https://huggingface.co/papers/1606.00915, https://huggingface.co/papers/1706.05587
    r"   r   Nc                 óv  •‡‡‡— t          ¦   «                              ¦   «          ‰j        d         Š‰j        Št	          ‰j        ¦  «        dk    rt          d¦  «        ‚t          j        ¦   «         | _	        t          ‰‰‰dd¬¦  «        }| j	                             |¦  «         | j	                             ˆˆˆfd„‰j        D ¦   «         ¦  «         t          ‰‰‰¦  «        }| j	                             |¦  «         t          ‰d‰z  ‰dd¬¦  «        | _        t          j        ‰j        ¬	¦  «        | _        d S )
Nr~   r   z"Expected 3 values for atrous_ratesr   rR  rS   c           
      ó:   •— g | ]}t          ‰‰‰d |d¬¦  «        ‘ŒS )r   rR  )r#   r$   r%   r)   r+   )r!   )r	  Úrater"   r#   r$   s     €€€r   ú
<listcomp>z*MobileViTASPP.__init__.<locals>.<listcomp>  sL   ø€ ð 
ð 
ð 
ð õ #ØØ +Ø!-Ø !Ø!Ø#)ðñ ô ð
ð 
ð 
r   rø   )Úp)r5   r6   rû   Úaspp_out_channelsÚlenÚatrous_ratesr7   r   rc   Úconvsr!   rf   ÚextendrO  Úprojectrw   Úaspp_dropout_probry   )r@   r"   Úin_projectionÚ
pool_layerr#   r$   rA   s    `  @@€r   r6   zMobileViTASPP.__init__
  s[  øøøø€ Ý‰Œ×ÒÑÔÐàÔ.¨rÔ2ˆØÔ/ˆåˆvÔ"Ñ#Ô# qÒ(Ð(ÝÐAÑBÔBÐBå”]‘_”_ˆŒ
å*ØØ#Ø%ØØ!ð
ñ 
ô 
ˆð 	Œ
×Ò˜-Ñ(Ô(Ð(àŒ
×Òð
ð 
ð 
ð 
ð 
ð 
ð #Ô/ð
ñ 
ô 
ñ	
ô 	
ð 	
õ *¨&°+¸|ÑLÔLˆ
ØŒ
×Ò˜*Ñ%Ô%Ð%å)Ø  LÑ 0¸|ÐYZÐkqð
ñ 
ô 
ˆŒõ ”z FÔ$<Ð=Ñ=Ô=ˆŒˆˆr   rB   c                 óÚ   — g }| j         D ] }|                      ||¦  «        ¦  «         Œ!t          j        |d¬¦  «        }|                      |¦  «        }|                      |¦  «        }|S rï   )ra  rf   rJ   rð   rc  ry   )r@   rB   ÚpyramidÚconvÚpooled_featuress        r   rE   zMobileViTASPP.forward5  sq   € ØˆØ”Jð 	+ð 	+ˆDØ�NŠN˜4˜4 ™>œ>Ñ*Ô*Ð*Ð*Ý”)˜G¨Ð+Ñ+Ô+ˆàŸ,š, wÑ/Ô/ˆØŸ,š, Ñ7Ô7ˆØÐr   ©
rF   rG   rH   r^   r   r6   rJ   rK   rE   rL   rM   s   @r   rX  rX    s|   ø€ € € € € ðð ð)>˜ð )>°4ð )>ð )>ð )>ð )>ð )>ð )>ðV ¤ð °´ð ð ð ð ð ð ð ð r   rX  c                   óP   ‡ — e Zd ZdZdeddfˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚMobileViTDeepLabV3zJ
    DeepLabv3 architecture: https://huggingface.co/papers/1706.05587
    r"   r   Nc           	      óö   •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          j        |j        ¦  «        | _        t          ||j	        |j
        dddd¬¦  «        | _        d S )Nr   FT)r#   r$   r%   r*   r+   r(   )r5   r6   rX  Úasppr   Ú	Dropout2drC  ry   r!   r^  rB  rE  rF  s     €r   r6   zMobileViTDeepLabV3.__init__E  sq   ø€ Ý‰Œ×ÒÑÔÐÝ! &Ñ)Ô)ˆŒ	å”| FÔ$BÑCÔCˆŒå,ØØÔ0ØÔ*ØØ#Ø Øð
ñ 
ô 
ˆŒˆˆr   r{   c                 ó�   — |                       |d         ¦  «        }|                      |¦  «        }|                      |¦  «        }|S )Nr}   )ro  ry   rE  )r@   r{   rB   s      r   rE   zMobileViTDeepLabV3.forwardU  s?   € Ø—9’9˜]¨2Ô.Ñ/Ô/ˆØ—<’< Ñ)Ô)ˆØ—?’? 8Ñ,Ô,ˆØˆr   rk  rM   s   @r   rm  rm  @  s{   ø€ € € € € ðð ð
˜ð 
°4ð 
ð 
ð 
ð 
ð 
ð 
ð  U¤\ð °e´lð ð ð ð ð ð ð ð r   rm  zX
    MobileViT model with a semantic segmentation head on top, e.g. for Pascal VOC.
    c                   ó�   ‡ — e Zd Zdeddfˆ fd„Ze	 	 	 	 d
dej        dz  dej        dz  dedz  dedz  de	e
z  f
d	„¦   «         Zˆ xZS )Ú MobileViTForSemanticSegmentationr"   r   Nc                 óÞ   •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S )NF)r-  )r5   r6   rB  r,  r  rm  Úsegmentation_headr4  rF  s     €r   r6   z)MobileViTForSemanticSegmentation.__init__b  sa   ø€ Ý‰Œ×Ò˜Ñ Ô Ð à Ô+ˆŒÝ'¨¸eÐDÑDÔDˆŒÝ!3°FÑ!;Ô!;ˆÔð 	�ŠÑÔÐÐÐr   r  rG  r  r  c                 óB  — |�|n| j         j        }|�|n| j         j        }|�| j         j        dk    rt	          d¦  «        ‚|                      |d|¬¦  «        }|r|j        n|d         }|                      |¦  «        }d}	|�Vt          j	         
                    ||j        dd…         dd¬	¦  «        }
t          | j         j        ¬
¦  «        } ||
|¦  «        }	|s)|r|f|dd…         z   }n|f|dd…         z   }|	�|	f|z   n|S t          |	||r|j        ndd¬¦  «        S )aÌ  
        labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*):
            Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy).

        Examples:

        ```python
        >>> import httpx
        >>> from io import BytesIO
        >>> import torch
        >>> from PIL import Image
        >>> from transformers import AutoImageProcessor, MobileViTForSemanticSegmentation

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> image_processor = AutoImageProcessor.from_pretrained("apple/deeplabv3-mobilevit-small")
        >>> model = MobileViTForSemanticSegmentation.from_pretrained("apple/deeplabv3-mobilevit-small")

        >>> inputs = image_processor(images=image, return_tensors="pt")

        >>> with torch.no_grad():
        ...     outputs = model(**inputs)

        >>> # logits are of shape (batch_size, num_labels, height, width)
        >>> logits = outputs.logits
        ```Nr   z/The number of labels should be greater than oneTr6  r~   rÓ   FrÔ   )Úignore_indexr   )rI  rJ  r{   Ú
attentions)r"   r  r  rB  r7   r  r{   ru  r   r‡   rÚ   r�   r   Úsemantic_loss_ignore_indexr   )r@   r  rG  r  r  r9  rL  Úencoder_hidden_statesrJ  rI  Úupsampled_logitsÚloss_fctr¢   s                r   rE   z(MobileViTForSemanticSegmentation.forwardl  s�  € ðN %9Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàÐ $¤+Ô"8¸AÒ"=Ð"=ÝÐNÑOÔOÐOà—.’.ØØ!%Ø#ð !ñ 
ô 
ˆð :EÐ T Ô 5Ð 5È'ÐRSÌ*Ðà×'Ò'Ð(=Ñ>Ô>ˆàˆØÐå!œ}×8Ò8Ø˜Vœ\¨"¨#¨#Ô.°ZÈuð  9ñ  ô  Ðõ (°T´[Ô5[Ð\Ñ\Ô\ˆHØ�8Ð,¨fÑ5Ô5ˆDàð 	FØ#ð 1Ø ˜ W¨Q¨R¨R¤[Ñ0��à ˜ W¨Q¨R¨R¤[Ñ0�Ø)-Ð)9�T�G˜fÑ$Ð$¸vÐEå&ØØØ3GÐQ˜'Ô/Ð/ÈTØð	
ñ 
ô 
ð 	
r   rM  )rF   rG   rH   r   r6   r   rJ   rK   rI   rñ   r   rE   rL   rM   s   @r   rs  rs  \  sØ   ø€ € € € € ð˜ð °4ð ð ð ð ð ð ð ð -1Ø&*Ø,0Ø#'ðL
ð L
à”l TÑ)ðL
ð ”˜tÑ#ðL
ð # T™kð	L
ð
 ˜D‘[ðL
ð 
Ð(Ñ	(ðL
ð L
ð L
ñ „^ðL
ð L
ð L
ð L
ð L
r   rs  )r?  rs  r,  r  )r   N)4r^   r…   rJ   r   Útorch.nnr   Ú r   r  Úactivationsr   Úmodeling_layersr   Úmodeling_outputsr	   r
   r   r   Úmodeling_utilsr   Úutilsr   r   r   Úconfiguration_mobilevitr   Ú
get_loggerrF   Úloggerr   r   r*  r!   rO   r`   rm   r–   rŸ   r§   r®   r³   r¾   rÅ   rô   r  r,  r?  rO  rX  rm  rs  Ú__all__r  r   r   ú<module>rˆ     s5  ðð  Ð à €€€à €€€Ø Ð Ð Ð Ð Ð Ø %Ð %Ð %Ð %Ð %Ð %à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð .Ð -Ð -Ð -Ð -Ð -Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4ð 
ˆÔ	˜HÑ	%Ô	%€ð
ð 
˜#ð 
¨ð 
¸CÀ$¹Jð 
ÐRUð 
ð 
ð 
ð 
ð=ð =ð =ð =ð =˜œñ =ô =ð =ð@-Fð -Fð -Fð -Fð -F ¤	ñ -Fô -Fð -Fð`ð ð ð ð ˜bœiñ ô ð ð.+ð +ð +ð +ð +˜RœYñ +ô +ð +ð\	ð 	ð 	ð 	ð 	˜"œ)ñ 	ô 	ð 	ð	 ð 	 ð 	 ð 	 ð 	 ˜œñ 	 ô 	 ð 	 ðð ð ð ð ˜BœIñ ô ð ð
ð 
ð 
ð 
ð 
�b”iñ 
ô 
ð 
ðð ð ð ð  ¤	ñ ô ð ð&ð ð ð ð ˜2œ9ñ ô ð ð&fð fð fð fð fÐ/ñ fô fð fðR\pð \pð \pð \pð \p�r”yñ \pô \pð \pð~ ð8ð 8ð 8ð 8ð 8˜ñ 8ô 8ñ „ð8ð, ðI
ð I
ð I
ð I
ð I
Ð-ñ I
ô I
ñ „ðI
ðX €ððñ ô ð3
ð 3
ð 3
ð 3
ð 3
Ð&>ñ 3
ô 3
ñô ð3
ðlð ð ð ð ˜2œ9ñ ô ð ð08ð 8ð 8ð 8ð 8�B”Iñ 8ô 8ð 8ðvð ð ð ð ˜œñ ô ð ð8 €ððñ ô ð
X
ð X
ð X
ð X
ð X
Ð'?ñ X
ô X
ñô ð
X
ðvð ð €€€r   