§
    ‚Štj÷l  ã                   óx  — d Z ddlZddlZddlZddlZddlmZmZ ddlm	Z
 ddlmZ ddlmZmZ ddlmZmZmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ  ej        e¦  «        Z d5de!e!e"f         fd„Z# G d„ dej$        ¦  «        Z% G d„ dej&        ¦  «        Z' G d„ dej(        ¦  «        Z) G d„ dej*        ¦  «        Z+ G d„ dej(        ¦  «        Z, G d„ dej(        ¦  «        Z-d6d„Z. G d„ d ej(        ¦  «        Z/ G d!„ d"ej(        ¦  «        Z0 G d#„ d$ej(        ¦  «        Z1 G d%„ d&ej(        ¦  «        Z2 G d'„ d(ej(        ¦  «        Z3e G d)„ d*e¦  «        ¦   «         Z4e G d+„ d,e4¦  «        ¦   «         Z5 ed-¬.¦  «         G d/„ d0e4¦  «        ¦   «         Z6 ed1¬.¦  «         G d2„ d3ee4¦  «        ¦   «         Z7g d4¢Z8dS )7z9PyTorch BiT model. Also supports backbone for ViT hybrid.é    N)ÚTensorÚnné   )Úinitialization)ÚACT2FN)ÚBackboneMixinÚfilter_output_hidden_states)ÚBackboneOutputÚBaseModelOutputWithNoAttentionÚ(BaseModelOutputWithPoolingAndNoAttentionÚ$ImageClassifierOutputWithNoAttention)ÚPreTrainedModel)Úauto_docstringÚlogging)Úcan_return_tupleé   )Ú	BitConfigé   Úreturnc                 ó&  — d}| €|dz
  ||dz
  z  z   dz  } | |fS t          | t          ¦  «        r`|                      ¦   «         } | dk    r,|dk    r!||dz
  z  dz  dk    r|dz
  ||dz
  z  z   dz  } nd} d}n| dk    rd} n|dz
  ||dz
  z  z   dz  } | |fS )	al  
    Utility function to get the tuple padding value given the kernel_size and padding.

    Args:
        padding (Union[`str`, `int`], *optional*):
            Padding value, can be either `"same"`, `"valid"`. If a different value is provided the default padding from
            PyTorch is used.
        kernel_size (`int`, *optional*, defaults to 7):
            Kernel size of the convolution layers.
        stride (`int`, *optional*, defaults to 1):
            Stride value of the convolution layers.
        dilation (`int`, *optional*, defaults to 1):
            Dilation value of the convolution layers.
    FNr   é   Úsamer   TÚvalid)Ú
isinstanceÚstrÚlower)ÚpaddingÚkernel_sizeÚstrideÚdilationÚdynamics        úb/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/bit/modeling_bit.pyÚget_padding_valuer#   )   sè   € ð €GØ€Ø˜Q‘J (¨k¸A©oÑ">Ñ>À1ÑDˆØ˜ÐÐå�'�3ÑÔð Ià—-’-‘/”/ˆØ�fÒÐà˜Š{ˆ{ ¨K¸!©OÑ <ÀÑAÀQÒFÐFà" Q™J¨(°kÀA±oÑ*FÑFÈ1ÑL��ð �Ø��Ø˜ÒÐàˆGˆGð  ™
 h°+À±/Ñ&BÑBÀqÑHˆGØ�GÐÐó    c                   ó6   ‡ — e Zd ZdZ	 	 	 	 	 	 dˆ fd„	Zd„ Zˆ xZS )	ÚWeightStandardizedConv2dzÐConv2d with Weight Standardization. Used for ViT Hybrid model.

    Paper: [Micro-Batch Training with Batch-Channel Normalization and Weight
    Standardization](https://huggingface.co/papers/1903.10520)
    r   ÚSAMEFç�íµ ÷Æ°>c
           
      óÔ   •— t          ||||¬¦  «        \  }}
t          ¦   «                              ||||||||¬¦  «         |
rt          |||¦  «        | _        nd | _        |	| _        d S )N)r   r    )r   r   r    ÚgroupsÚbias)r#   ÚsuperÚ__init__ÚDynamicPad2dÚpadÚeps)ÚselfÚ
in_channelÚout_channelsr   r   r   r    r*   r+   r0   Ú
is_dynamicÚ	__class__s              €r"   r-   z!WeightStandardizedConv2d.__init__Y   s‹   ø€ õ 0°¸ÈVÐ^fÐgÑgÔgÑˆ�Ý‰Œ×ÒØØØØØØØØð 	ñ 		
ô 		
ð 		
ð ð 	Ý# K°¸ÑBÔBˆDŒHˆHàˆDŒHØˆŒˆˆr$   c           	      ó|  — | j         �|                       |¦  «        }t          j                             | j                             d| j        d¦  «        d d dd| j        ¬¦  «                             | j        ¦  «        }t          j         	                    ||| j
        | j        | j        | j        | j        ¦  «        }|S )Nr   éÿÿÿÿTç        )ÚtrainingÚmomentumr0   )r/   r   Ú
functionalÚ
batch_normÚweightÚreshaper3   r0   Ú
reshape_asÚconv2dr+   r   r   r    r*   )r1   Úhidden_stater=   s      r"   Úforwardz WeightStandardizedConv2d.forwardv   s¯   € ØŒ8ÐØŸ8š8 LÑ1Ô1ˆLÝ”×)Ò)ØŒK×Ò  4Ô#4°bÑ9Ô9¸4ÀÐPTÐ_bÐhlÔhpð *ñ 
ô 
ç
Š*�T”[Ñ
!Ô
!ð 	õ ”}×+Ò+Ø˜& $¤)¨T¬[¸$¼,ÈÌÐW[ÔWbñ
ô 
ˆð Ðr$   )r   r'   r   r   Fr(   ©Ú__name__Ú
__module__Ú__qualname__Ú__doc__r-   rB   Ú__classcell__©r5   s   @r"   r&   r&   R   sj   ø€ € € € € ðð ð ØØØØØðð ð ð ð ð ð:	ð 	ð 	ð 	ð 	ð 	ð 	r$   r&   c                   ó*   ‡ — e Zd ZdZdˆ fd„	Zd„ Zˆ xZS )ÚBitGroupNormActivationzQ
    A module that combines group normalization with an activation function.
    çñhãˆµøä>Tc                 óÂ   •— t          ¦   «                              |j        |||¬¦  «         |rt          |j                 | _        d S t          j        ¦   «         | _        d S )N)r0   Úaffine)r,   r-   Ú
num_groupsr   Ú
hidden_actÚ
activationr   ÚIdentity)r1   ÚconfigÚnum_channelsr0   rN   Úapply_activationr5   s         €r"   r-   zBitGroupNormActivation.__init__‡   sS   ø€ Ý‰Œ×Ò˜Ô*¨L¸cÈ&ÐÑQÔQÐQØð 	,Ý$ VÔ%6Ô7ˆDŒOˆOˆOå œk™mœmˆDŒOˆOˆOr$   c                 óž   — t           j                             || j        | j        | j        | j        ¦  «        }|                      |¦  «        }|S ©N)r   r;   Ú
group_normrO   r=   r+   r0   rQ   )r1   rA   s     r"   rB   zBitGroupNormActivation.forwardŽ   sB   € Ý”}×/Ò/°¸d¼oÈtÌ{Ð\`Ô\eÐgkÔgoÑpÔpˆØ—’ |Ñ4Ô4ˆØÐr$   )rL   TTrC   rI   s   @r"   rK   rK   ‚   sV   ø€ € € € € ðð ð,ð ,ð ,ð ,ð ,ð ,ðð ð ð ð ð ð r$   rK   c                   ó*   ‡ — e Zd ZdZdˆ fd„	Zd„ Zˆ xZS )r.   zŒ
    A module that wraps dynamic padding of any input, given the parameters of the convolutional layer and the input
    hidden states.
    r   c                 ó*  •— t          ¦   «                              ¦   «          t          |t          ¦  «        r||f}t          |t          ¦  «        r||f}t          |t          ¦  «        r||f}|| _        || _        || _        || _        d„ }|| _        d S )Nc                 óv   — t          t          j        | |z  ¦  «        dz
  |z  |dz
  |z  z   dz   | z
  d¦  «        S )Nr   r   )ÚmaxÚmathÚceil)Úxr   r   r    s       r"   Úcompute_paddingz.DynamicPad2d.__init__.<locals>.compute_padding«   sF   € Ý�œ	 ! f¡*Ñ-Ô-°Ñ1°VÑ;¸{ÈQ¹ÐRZÑ>ZÑZÐ]^Ñ^ÐabÑbÐdeÑfÔfÐfr$   )	r,   r-   r   Úintr   r   r    Úvaluer`   )r1   r   r   r    rb   r`   r5   s         €r"   r-   zDynamicPad2d.__init__š   s¬   ø€ Ý‰Œ×ÒÑÔÐå�k¥3Ñ'Ô'ð 	5Ø&¨Ð4ˆKå�f�cÑ"Ô"ð 	&Ø˜fÐ%ˆFå�h¥Ñ$Ô$ð 	,Ø  (Ð+ˆHà&ˆÔØˆŒØ ˆŒØˆŒ
ð	gð 	gð 	gð  /ˆÔÐÐr$   c           	      óº  — |                      ¦   «         dd …         \  }}|                      || j        d         | j        d         | j        d         ¦  «        }|                      || j        d         | j        d         | j        d         ¦  «        }|dk    s|dk    r=t
          j                             ||dz  ||dz  z
  |dz  ||dz  z
  g| j        ¬¦  «        }|S )Néþÿÿÿr   r   r   )rb   )	Úsizer`   r   r   r    r   r;   r/   rb   )r1   ÚinputÚinput_heightÚinput_widthÚpadding_heightÚpadding_widths         r"   rB   zDynamicPad2d.forward°   sù   € à$)§J¢J¡L¤L°°°Ô$5Ñ!ˆ�kð ×-Ò-¨l¸DÔ<LÈQÔ<OÐQUÔQ\Ð]^ÔQ_ÐaeÔanÐopÔaqÑrÔrˆØ×,Ò,¨[¸$Ô:JÈ1Ô:MÈtÌ{Ð[\Ì~Ð_cÔ_lÐmnÔ_oÑpÔpˆð ˜AÒÐ °Ò!2Ð!2Ý”M×%Ò%Øà! QÑ&Ø! M°QÑ$6Ñ6Ø" aÑ'Ø" ^°qÑ%8Ñ8ð	ð ”jð &ñ 	ô 	ˆEð ˆr$   )r   rC   rI   s   @r"   r.   r.   ”   sV   ø€ € € € € ðð ð
/ð /ð /ð /ð /ð /ð,ð ð ð ð ð ð r$   r.   c                   ó8   ‡ — e Zd Z	 	 	 	 	 	 d
defˆ fd„Zd	„ Zˆ xZS )ÚBitMaxPool2dNr   F©r   r   r   Tr   c                 ó–  •— t          |t          j        j        ¦  «        r|n||f}t          |t          j        j        ¦  «        r|n||f}t          |t          j        j        ¦  «        r|n||f}t	          ¦   «                              |||||¦  «         |rt          ||||¦  «        | _        d S t          j	        ¦   «         | _        d S rW   )
r   ÚcollectionsÚabcÚIterabler,   r-   r.   r/   r   rR   )	r1   r   r   r    Ú	ceil_moder   Úpadding_valueÚuse_dynamic_paddingr5   s	           €r"   r-   zBitMaxPool2d.__init__È   sÂ   ø€ õ &0°½[¼_Ô=UÑ%VÔ%VÐv�k�kÐ]hÐjuÐ\vˆÝ% f­k¬oÔ.FÑGÔGÐ]��ÈfÐV\ÐM]ˆÝ)¨(µK´OÔ4LÑMÔMÐg�8�8ÐT\Ð^fÐSgˆÝ‰Œ×Ò˜ f¨g°xÀÑKÔKÐKØð 	%Ý# K°¸À=ÑQÔQˆDŒHˆHˆHå”{‘}”}ˆDŒHˆHˆHr$   c                 ó¦   — |                       |¦  «        }t          j                             || j        | j        | j        | j        | j        ¦  «        S rW   )	r/   r   r;   Ú
max_pool2dr   r   r   r    rr   )r1   Úhidden_statess     r"   rB   zBitMaxPool2d.forwardÛ   sG   € ØŸš Ñ/Ô/ˆÝŒ}×'Ò'Ø˜4Ô+¨T¬[¸$¼,ÈÌÐW[ÔWeñ
ô 
ð 	
r$   )Nr   Frm   r   T)rD   rE   rF   ra   r-   rB   rH   rI   s   @r"   rl   rl   Ç   sm   ø€ € € € € ð ØØØØØ ð%ð %àð%ð %ð %ð %ð %ð %ð&
ð 
ð 
ð 
ð 
ð 
ð 
r$   rl   c                   ó8   ‡ — e Zd ZdZdefˆ fd„Zdedefd„Zˆ xZS )ÚBitEmbeddingszL
    BiT Embeddings (stem) composed of a single aggressive convolution.
    rS   c                 ó  •— t          ¦   «                              ¦   «          t          |j        |j        ddd|j        ¬¦  «        | _        t          dd|j        ¬¦  «        | _	        |j        �6|j         
                    ¦   «         dk    rt          j        ¦   «         | _        nt          j        dd	¬
¦  «        | _        |j        dk    rt!          ||j        ¬¦  «        | _        nt          j        ¦   «         | _        |j        | _        d S )Nr   r   ç:Œ0âŽyE>)r   r   r0   r   r   )r   r   rt   r'   )r   r   r   r   r8   )r   rb   Úpreactivation©rT   )r,   r-   r&   rT   Úembedding_sizeÚglobal_paddingÚconvolutionrl   Úembedding_dynamic_paddingÚpoolerÚupperr   rR   r/   ÚConstantPad2dÚ
layer_typerK   Únorm©r1   rS   r5   s     €r"   r-   zBitEmbeddings.__init__ç   s÷   ø€ Ý‰Œ×ÒÑÔÐå3ØÔØÔ!ØØØØÔ)ð
ñ 
ô 
ˆÔõ #¨q¸ÐPVÔPpÐqÑqÔqˆŒð Ô Ð,°Ô1F×1LÒ1LÑ1NÔ1NÐRXÒ1XÐ1XÝ”{‘}”}ˆDŒHˆHåÔ'°ÀCÐHÑHÔHˆDŒHàÔ Ò/Ð/Ý.¨vÀFÔDYÐZÑZÔZˆDŒIˆIåœ™œˆDŒIà"Ô/ˆÔÐÐr$   Úpixel_valuesr   c                 óü   — |j         d         }|| j        k    rt          d¦  «        ‚|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S )Nr   zeMake sure that the channel dimension of the pixel values match with the one set in the configuration.)ÚshaperT   Ú
ValueErrorr€   r/   r†   r‚   )r1   rˆ   rT   Ú	embeddings       r"   rB   zBitEmbeddings.forward  s€   € Ø#Ô)¨!Ô,ˆØ˜4Ô,Ò,Ð,ÝØwñô ð ð ×$Ò$ \Ñ2Ô2ˆ	à—H’H˜YÑ'Ô'ˆ	à—I’I˜iÑ(Ô(ˆ	à—K’K 	Ñ*Ô*ˆ	àÐr$   )	rD   rE   rF   rG   r   r-   r   rB   rH   rI   s   @r"   ry   ry   â   sp   ø€ € € € € ðð ð0˜yð 0ð 0ð 0ð 0ð 0ð 0ð6 Fð ¨vð ð ð ð ð ð ð ð r$   ry   c                   ó^   ‡ — e Zd ZdZd
deddfˆ fd„Zdej        dej        fd„Zde	fd	„Z
ˆ xZS )ÚBitDropPathzÏStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    r8   Ú	drop_probr   Nc                 óV   •— t          ¦   «                              ¦   «          || _        d S rW   )r,   r-   r�   )r1   r�   r5   s     €r"   r-   zBitDropPath.__init__  s$   ø€ Ý‰Œ×ÒÑÔÐØ"ˆŒˆˆr$   rw   c                 ó  — | j         dk    s| j        s|S d| j         z
  }|j        d         fd|j        dz
  z  z   }t	          j        ||j        |j        ¬¦  «        }t	          j        ||z   ¦  «        }| 	                    |¦  «        |z  S )Nr8   r   r   )r   )ÚdtypeÚdevice)
r�   r9   rŠ   ÚndimÚtorchÚrandr’   r“   ÚfloorÚdiv)r1   rw   Ú	keep_probrŠ   Úrandom_tensors        r"   rB   zBitDropPath.forward   s“   € ØŒ>˜SÒ Ð ¨¬Ð Ø Ð Ø˜œÑ&ˆ	ØÔ$ QÔ'Ð)¨D°MÔ4FÈÑ4JÑ,KÑKˆÝœ
 5°Ô0CÈMÔL`ÐaÑaÔaˆÝœ M°IÑ$=Ñ>Ô>ˆØ× Ò  Ñ+Ô+¨mÑ;Ð;r$   c                 ó   — d| j         › �S )Nzp=)r�   )r1   s    r"   Ú
extra_reprzBitDropPath.extra_repr)  s   € Ø$�D”NÐ$Ð$Ð$r$   )r8   )rD   rE   rF   rG   Úfloatr-   r•   r   rB   r   rœ   rH   rI   s   @r"   rŽ   rŽ     s›   ø€ € € € € ðð ð#ð # %ð #°$ð #ð #ð #ð #ð #ð #ð< U¤\ð <°e´lð <ð <ð <ð <ð%˜Cð %ð %ð %ð %ð %ð %ð %ð %r$   rŽ   é   c                 óx   — |}t          |t          | |dz  z   ¦  «        |z  |z  ¦  «        }|d| z  k     r||z  }|S )Nr   gÍÌÌÌÌÌì?)r\   ra   )rb   ÚdivisorÚ	min_valueÚ	new_values       r"   Úmake_divr£   -  sP   € Ø€IÝ�I�s 5¨7°Q©;Ñ#6Ñ7Ô7¸7ÑBÀWÑLÑMÔM€IØ�3˜‘;ÒÐØ�WÑˆ	ØÐr$   c                   ó:   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 d	ˆ fd„	Zd„ Zˆ xZS )
ÚBitPreActivationBottleneckLayera  Pre-activation (v2) bottleneck block.
    Follows the implementation of "Identity Mappings in Deep Residual Networks":
    https://github.com/KaimingHe/resnet-1k-layers/blob/master/resnet-pre-act.lua

    Except it puts the stride on 3x3 conv when available.
    Nç      Ð?r   r8   Fc           	      óT  •— t          ¦   «                              ¦   «          |p|}|p|}t          ||z  ¦  «        }|
rt          ||||d¬¦  «        | _        nd | _        t          ||¦  «        | _        t          ||dd|j        ¬¦  «        | _	        t          ||¬¦  «        | _
        t          ||d||d|j        ¬¦  «        | _        t          ||¦  «        | _        t          ||dd|j        ¬¦  «        | _        |	d	k    rt          |	¦  «        nt          j        ¦   «         | _        d S )
NT©r   Úpreactr   r{   ©r0   r   r}   r   )r   r*   r0   r   r   )r,   r-   r£   ÚBitDownsampleConvÚ
downsamplerK   Únorm1r&   r   Úconv1Únorm2Úconv2Únorm3Úconv3rŽ   r   rR   Ú	drop_path)r1   rS   Úin_channelsr3   Úbottle_ratior   r    Úfirst_dilationr*   Údrop_path_rateÚis_first_layerÚmid_channelsr5   s               €r"   r-   z(BitPreActivationBottleneckLayer.__init__=  sD  ø€ õ 	‰Œ×ÒÑÔÐà'Ð3¨8ˆà#Ð2 {ˆÝ ¨|Ñ ;Ñ<Ô<ˆàð 		#Ý/ØØØØØðñ ô ˆDŒOˆOð #ˆDŒOå+¨F°KÑ@Ô@ˆŒ
Ý-¨k¸<ÈÐPTÐ^dÔ^sÐtÑtÔtˆŒ
å+¨FÀÐNÑNÔNˆŒ
Ý-Ø˜,¨°&ÀÈTÐ[aÔ[pð
ñ 
ô 
ˆŒ
õ ,¨F°LÑAÔAˆŒ
Ý-¨l¸LÈ!ÐQUÐ_eÔ_tÐuÑuÔuˆŒ
à8FÈÒ8JÐ8J� ^Ñ4Ô4Ð4ÕPRÔP[ÑP]ÔP]ˆŒˆˆr$   c                 óf  — |                       |¦  «        }|}| j        �|                      |¦  «        }|                      |¦  «        }|                      |                      |¦  «        ¦  «        }|                      |                      |¦  «        ¦  «        }|                      |¦  «        }||z   S rW   )r­   r¬   r®   r°   r¯   r²   r±   r³   )r1   rw   Úhidden_states_preactÚshortcuts       r"   rB   z'BitPreActivationBottleneckLayer.forwardi  sŸ   € Ø#Ÿzšz¨-Ñ8Ô8Ðð !ˆØŒ?Ð&Ø—’Ð';Ñ<Ô<ˆHð Ÿ
š
Ð#7Ñ8Ô8ˆØŸ
š
 4§:¢:¨mÑ#<Ô#<Ñ=Ô=ˆØŸ
š
 4§:¢:¨mÑ#<Ô#<Ñ=Ô=ˆØŸš }Ñ5Ô5ˆØ˜xÑ'Ð'r$   ©Nr¦   r   r   Nr   r8   FrC   rI   s   @r"   r¥   r¥   5  sw   ø€ € € € € ðð ð ØØØØØØØð*^ð *^ð *^ð *^ð *^ð *^ðX(ð (ð (ð (ð (ð (ð (r$   r¥   c                   ó:   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 d	ˆ fd„	Zd„ Zˆ xZS )
ÚBitBottleneckLayerz\Non Pre-activation bottleneck block, equivalent to V1.5/V1b bottleneck. Used for ViT Hybrid.Nr¦   r   r8   Fc           
      óŠ  •— t          ¦   «                              ¦   «          |p|}|p|}t          ||z  ¦  «        }|
rt          ||||d¬¦  «        | _        nd | _        t          ||dd|j        ¬¦  «        | _        t          ||¬¦  «        | _	        t          ||d|||d|j        ¬¦  «        | _
        t          ||¬¦  «        | _        t          ||dd|j        ¬¦  «        | _        t          ||d¬	¦  «        | _        |	d
k    rt          |	¦  «        nt          j        ¦   «         | _        t$          |j                 | _        d S )NFr¨   r   r{   rª   r}   r   )r   r    r*   r0   r   ©rT   rU   r   )r,   r-   r£   r«   r¬   r&   r   r®   rK   r­   r°   r¯   r²   r±   rŽ   r   rR   r³   r   rP   rQ   )r1   rS   r´   r3   rµ   r   r    r¶   r*   r·   r¸   Úmid_chsr5   s               €r"   r-   zBitBottleneckLayer.__init__|  sb  ø€ õ 	‰Œ×ÒÑÔÐØ'Ð3¨8ˆà#Ð2 {ˆÝ˜<¨,Ñ6Ñ7Ô7ˆàð 		#Ý/ØØØØØðñ ô ˆDŒOˆOð #ˆDŒOå-¨k¸7ÀAÈ4ÐY_ÔYnÐoÑoÔoˆŒ
Ý+¨FÀÐIÑIÔIˆŒ
Ý-ØØØØØ#ØØØÔ)ð	
ñ 	
ô 	
ˆŒ
õ ,¨FÀÐIÑIÔIˆŒ
Ý-¨g°|ÀQÈDÐZ`ÔZoÐpÑpÔpˆŒ
Ý+¨FÀÐ`eÐfÑfÔfˆŒ
Ø8FÈÒ8JÐ8J� ^Ñ4Ô4Ð4ÕPRÔP[ÑP]ÔP]ˆŒå  Ô!2Ô3ˆŒˆˆr$   c                 ó˜  — |}| j         �|                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|S rW   )	r¬   r®   r­   r°   r¯   r²   r±   r³   rQ   )r1   rw   r¼   s      r"   rB   zBitBottleneckLayer.forward­  sµ   € à ˆØŒ?Ð&Ø—’ }Ñ5Ô5ˆHð Ÿ
š
 =Ñ1Ô1ˆØŸ
š
 =Ñ1Ô1ˆàŸ
š
 =Ñ1Ô1ˆØŸ
š
 =Ñ1Ô1ˆàŸ
š
 =Ñ1Ô1ˆØŸ
š
 =Ñ1Ô1ˆàŸš }Ñ5Ô5ˆØŸš¨¸Ñ(@ÑAÔAˆØÐr$   r½   rC   rI   s   @r"   r¿   r¿   y  sm   ø€ € € € € ØfÐfð ØØØØØØØð/4ð /4ð /4ð /4ð /4ð /4ðbð ð ð ð ð ð r$   r¿   c                   ó*   ‡ — e Zd Z	 	 dˆ fd„	Zd„ Zˆ xZS )r«   r   Tc                 óÞ   •— t          ¦   «                              ¦   «          t          ||d|d|j        ¬¦  «        | _        |rt          j        ¦   «         nt          ||d¬¦  «        | _        d S )Nr   r{   )r   r0   r   FrÁ   )	r,   r-   r&   r   Úconvr   rR   rK   r†   )r1   rS   r´   r3   r   r©   r5   s         €r"   r-   zBitDownsampleConv.__init__Ã  st   ø€ õ 	‰Œ×ÒÑÔÐÝ,Ø˜ q°¸TÈ6ÔK`ð
ñ 
ô 
ˆŒ	ð
 ðc�BŒK‰MŒMˆMå'¨¸\Ð\aÐbÑbÔbð 	Œ	ˆ	ˆ	r$   c                 óR   — |                       |                      |¦  «        ¦  «        S rW   )r†   rÆ   )r1   r_   s     r"   rB   zBitDownsampleConv.forwardÕ  s   € Ø�yŠy˜Ÿš 1™œÑ&Ô&Ð&r$   )r   T)rD   rE   rF   r-   rB   rH   rI   s   @r"   r«   r«   Â  sT   ø€ € € € € ð Øð
ð 
ð 
ð 
ð 
ð 
ð$'ð 'ð 'ð 'ð 'ð 'ð 'r$   r«   c                   ó>   ‡ — e Zd ZdZ	 	 d	ˆ fd„	Zd„ Zdedefd„Zˆ xZS )
ÚBitStagez7
    A ResNet v2 stage composed by stacked layers.
    r¦   Nc	                 ó–  •— t          ¦   «                              ¦   «          |dv rdnd}	|j        dk    rt          }
nt          }
|}t          j        ¦   «         | _        t          |¦  «        D ][}|  	                    |||¦  «        \  }}}| j         
                    t          |¦  «         |
|||||||	||¬¦	  «	        ¦  «         |}|}	Œ\d S )N)r   r   r   r   Ú
bottleneck)r   r    rµ   r¶   r·   r¸   )r,   r-   r…   r¿   r¥   r   Ú
SequentialÚlayersÚrangeÚ_get_updated_hyperparametersÚ
add_moduler   )r1   rS   r´   r3   r   r    Údepthrµ   Úlayer_dropoutr¶   Ú	layer_clsÚprev_chsÚ	layer_idxr·   r¸   r5   s                  €r"   r-   zBitStage.__init__Þ  s   ø€ õ 	‰Œ×ÒÑÔÐà&¨&Ð0Ð0˜˜°aˆð Ô Ò,Ð,Ý*ˆIˆIå7ˆIàˆÝ”m‘o”oˆŒÝ˜u™œð 	&ð 	&ˆIà59×5VÒ5VØ˜6 =ñ6ô 6Ñ2ˆF�N Nð ŒK×"Ò"Ý�I‘”Ø�	ØØØ Ø!Ø%Ø!-Ø#1Ø#1Ø#1ð
ñ 
ô 
ñô ð ð $ˆHØ%ˆNˆNð+	&ð 	&r$   c                 óB   — |r	||         }nd}|dk    rd}|dk    }|||fS )zt
        Get the new hyper-parameters with respect to the previous ones and the index of the current layer.
        r8   r   r   © )r1   rÕ   r   rÒ   r·   r¸   s         r"   rÏ   z%BitStage._get_updated_hyperparameters  sA   € ð ð 	!Ø*¨9Ô5ˆNˆNà ˆNà˜Š>ˆ>ØˆFà" ašˆà�~ ~Ð5Ð5r$   rf   r   c                 óT   — |}t          | j        ¦  «        D ]\  }} ||¦  «        }Œ|S rW   )Ú	enumeraterÍ   )r1   rf   rA   Ú_Úlayers        r"   rB   zBitStage.forward  s;   € ØˆÝ! $¤+Ñ.Ô.ð 	/ð 	/‰HˆAˆuØ ˜5 Ñ.Ô.ˆLˆLØÐr$   )r¦   N)	rD   rE   rF   rG   r-   rÏ   r   rB   rH   rI   s   @r"   rÉ   rÉ   Ù  s�   ø€ € € € € ðð ð Øð,&ð ,&ð ,&ð ,&ð ,&ð ,&ð\6ð 6ð 6ð ˜Vð ¨ð ð ð ð ð ð ð ð r$   rÉ   c            	       óF   ‡ — e Zd Zdefˆ fd„Zd„ Z	 ddededed	efd
„Z	ˆ xZ
S )Ú
BitEncoderrS   c           
      óz  •— t          ¦   «                              ¦   «          t          j        g ¦  «        | _        |j        }d}d}d„ t          j        t          j	        d|j
        t          |j        ¦  «        ¦  «        ¦  «                             |j        ¦  «        D ¦   «         }t          t          |j        |j        |¦  «        ¦  «        D ]k\  }\  }}}	|                      |||||¦  «        \  }
}}t%          |||
||||	¬¦  «        }|
}||z  }| j                             t)          |¦  «        |¦  «         Œld S )Né   r   c                 ó6   — g | ]}|                      ¦   «         ‘ŒS r×   )Útolist)Ú.0r_   s     r"   ú
<listcomp>z'BitEncoder.__init__.<locals>.<listcomp>.  s0   € ð 
ð 
ð 
àð �HŠH‰JŒJð
ð 
ð 
r$   r   )r   r    rÑ   rÒ   )r,   r-   r   Ú
ModuleListÚstagesr~   r•   r   ÚnpÚlinspacer·   ÚsumÚdepthsÚsplitrÙ   ÚzipÚhidden_sizesrÏ   rÉ   rÐ   r   )r1   rS   rÔ   Úcurrent_strider    Úlayer_dropoutsÚ	stage_idxÚcurrent_depthÚcurrent_hidden_sizerÒ   r3   r   Ústager5   s                €r"   r-   zBitEncoder.__init__$  sd  ø€ Ý‰Œ×ÒÑÔÐÝ”m BÑ'Ô'ˆŒàÔ(ˆð ˆØˆð
ð 
å”\¥"¤+¨a°Ô1FÍÈFÌMÑHZÔHZÑ"[Ô"[Ñ\Ô\×bÒbÐciÔcpÑqÔqð
ñ 
ô 
ˆõ
 OXÝ�”˜vÔ2°NÑCÔCñO
ô O
ð 	:ð 	:ÑJˆIÑJ˜Ð':¸Mð .2×-NÒ-NØ˜>Ð+>ÀÈ&ñ.ô .Ñ*ˆL˜& (õ ØØØØØ!Ø#Ø+ðñ ô ˆEð $ˆHØ˜fÑ$ˆNàŒK×"Ò"¥3 y¡>¤>°5Ñ9Ô9Ð9Ð9ð+	:ð 	:r$   c                 ór   — t          ||j        z  ¦  «        }|dk    rdnd}||j        k    r||z  }d}|||fS )Nr   r   r   )r£   Úwidth_factorÚoutput_stride)r1   rï   rí   rñ   r    rS   r3   r   s           r"   rÏ   z'BitEncoder._get_updated_hyperparametersJ  sS   € ÝÐ 3°fÔ6IÑ IÑJÔJˆØ 1’n�n��¨!ˆØ˜VÔ1Ò1Ð1Ø˜ÑˆHØˆFØ˜V XÐ-Ð-r$   FTrA   Úoutput_hidden_statesÚreturn_dictr   c                 ó´   — |rdnd }| j         D ]}|r||fz   } ||¦  «        }Œ|r||fz   }|st          d„ ||fD ¦   «         ¦  «        S t          ||¬¦  «        S )Nr×   c              3   ó   K  — | ]}|®|V — Œ	d S rW   r×   )râ   Úvs     r"   ú	<genexpr>z%BitEncoder.forward.<locals>.<genexpr>a  s"   è è € ÐSÐS˜qÀQÀ]˜À]À]À]À]ÐSÐSr$   )Úlast_hidden_staterw   )rå   Útupler   )r1   rA   rö   r÷   rw   Ústage_modules         r"   rB   zBitEncoder.forwardR  s¥   € ð 3Ð<˜˜¸ˆà œKð 	6ð 	6ˆLØ#ð @Ø -°°Ñ ?�à'˜<¨Ñ5Ô5ˆLˆLàð 	<Ø)¨\¨OÑ;ˆMàð 	TÝÐSÐS \°=Ð$AÐSÑSÔSÑSÔSÐSå-Ø*Ø'ð
ñ 
ô 
ð 	
r$   )FT)rD   rE   rF   r   r-   rÏ   r   Úboolr   rB   rH   rI   s   @r"   rÝ   rÝ   #  s”   ø€ € € € € ð$:˜yð $:ð $:ð $:ð $:ð $:ð $:ðL.ð .ð .ð ]að
ð 
Ø"ð
Ø:>ð
ØUYð
à	'ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r$   rÝ   c                   óf   ‡ — e Zd ZU eed<   dZdZdZdgZ e	j
        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚBitPreTrainedModelrS   Úbit)Úimagerˆ   ry   c                 ó,  •— t          ¦   «                              |¦  «         t          |t          j        ¦  «        rt          j        |j        dd¬¦  «         d S t          |t          j        ¦  «        r�t          j	        |j        t          j        d¦  «        ¬¦  «         |j        �kt          j        j                             |j        ¦  «        \  }}|dk    rdt          j        |¦  «        z  nd}t          j        |j        | |¦  «         d S d S d S )NÚfan_outÚrelu)ÚmodeÚnonlinearityé   )Úar   r   )r,   Ú_init_weightsr   r   ÚConv2dÚinitÚkaiming_normal_r=   ÚLinearÚkaiming_uniform_r]   Úsqrtr+   r•   Ú_calculate_fan_in_and_fan_outÚuniform_)r1   ÚmoduleÚfan_inrÚ   Úboundr5   s        €r"   r  z BitPreTrainedModel._init_weightsq  s÷   ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�bœiÑ(Ô(ð 	:ÝÔ  ¤°YÈVÐTÑTÔTÐTÐTÐTå˜¥¤	Ñ*Ô*ð 	:ÝÔ! &¤-µ4´9¸Q±<´<Ð@Ñ@Ô@Ð@ØŒ{Ð&Ý!œHœM×GÒGÈÌÑVÔV‘	�˜Ø17¸!²°˜�DœI fÑ-Ô-Ñ-Ð-À�Ý”˜fœk¨E¨6°5Ñ9Ô9Ð9Ð9Ð9ð	:ð 	:à&Ð&r$   )rD   rE   rF   r   Ú__annotations__Úbase_model_prefixÚinput_modalitiesÚmain_input_nameÚ_no_split_modulesr•   Úno_gradr  rH   rI   s   @r"   r  r  i  sr   ø€ € € € € € àÐÐÑØÐØ!ÐØ$€OØ(Ð)Ðà€U„]�_„_ð
:ð 
:ð 
:ð 
:ñ „_ð
:ð 
:ð 
:ð 
:ð 
:r$   r  c            
       óX   ‡ — e Zd Zˆ fd„Ze	 	 ddededz  dedz  defd„¦   «         Zˆ xZ	S )	ÚBitModelc                 ó€  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        |j        dk    rt          ||j	        d         ¬¦  «        nt          j        ¦   «         | _        t          j        d¦  «        | _        |                      ¦   «          d S )Nr|   r7   r}   )r   r   )r,   r-   rS   ry   ÚembedderrÝ   Úencoderr…   rK   rì   r   rR   r†   ÚAdaptiveAvgPool2dr‚   Ú	post_initr‡   s     €r"   r-   zBitModel.__init__�  s§   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒå% fÑ-Ô-ˆŒå! &Ñ)Ô)ˆŒð Ô  OÒ3Ð3õ # 6¸Ô8KÈBÔ8OÐPÑPÔPÐPå”‘”ð 	Œ	õ Ô*¨6Ñ2Ô2ˆŒà�ŠÑÔÐÐÐr$   Nrˆ   rö   r÷   r   c                 óP  — |�|n| j         j        }|�|n| j         j        }|                      |¦  «        }|                      |||¬¦  «        }|d         }|                      |¦  «        }|                      |¦  «        }|s||f|dd …         z   S t          |||j        ¬¦  «        S )N©rö   r÷   r   r   )rü   Úpooler_outputrw   )	rS   rö   r÷   r   r!  r†   r‚   r   rw   )	r1   rˆ   rö   r÷   ÚkwargsÚembedding_outputÚencoder_outputsrü   Úpooled_outputs	            r"   rB   zBitModel.forward’  sÛ   € ð %9Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàŸ=š=¨Ñ6Ô6ÐàŸ,š,ØÐ3GÐU`ð 'ñ 
ô 
ˆð ,¨AÔ.Ðà ŸIšIÐ&7Ñ8Ô8ÐàŸšÐ$5Ñ6Ô6ˆàð 	LØ% }Ð5¸ÈÈÈÔ8KÑKÐKå7Ø/Ø'Ø)Ô7ð
ñ 
ô 
ð 	
r$   ©NN)
rD   rE   rF   r-   r   r   rÿ   r   rB   rH   rI   s   @r"   r  r    s“   ø€ € € € € ðð ð ð ð ð" ð -1Ø#'ð	
ð 
àð
ð # T™kð
ð ˜D‘[ð	
ð 
2ð
ð 
ð 
ñ „^ð
ð 
ð 
ð 
ð 
r$   r  zƒ
    BiT Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for
    ImageNet.
    )Úcustom_introc                   ó€   ‡ — e Zd Zˆ fd„Ze	 	 	 	 d	dej        dz  dej        dz  dedz  dedz  de	f
d„¦   «         Z
ˆ xZS )
ÚBitForImageClassificationc                 óŠ  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        t          j        ¦   «         |j        dk    r%t          j        |j	        d         |j        ¦  «        nt          j
        ¦   «         ¦  «        | _        |                      ¦   «          d S )Nr   r7   )r,   r-   Ú
num_labelsr  r  r   rÌ   ÚFlattenr  rì   rR   Ú
classifierr#  r‡   s     €r"   r-   z"BitForImageClassification.__init__¼  sŸ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒÝ˜FÑ#Ô#ˆŒåœ-ÝŒJ‰LŒLØEKÔEVÐYZÒEZÐEZ�BŒI�fÔ)¨"Ô-¨vÔ/@ÑAÔAÐAÕ`bÔ`kÑ`mÔ`mñ
ô 
ˆŒð
 	�ŠÑÔÐÐÐr$   Nrˆ   Úlabelsrö   r÷   r   c                 ó@  — |�|n| j         j        }|                      |||¬¦  «        }|r|j        n|d         }|                      |¦  «        }d}	|�|                      ||| j         ¦  «        }	|s|f|dd…         z   }
|	�|	f|
z   n|
S t          |	||j        ¬¦  «        S )a0  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr%  r   r   )ÚlossÚlogitsrw   )rS   r÷   r  r&  r2  Úloss_functionr   rw   )r1   rˆ   r3  rö   r÷   r'  Úoutputsr*  r6  r5  Úoutputs              r"   rB   z!BitForImageClassification.forwardÈ  sÈ   € ð &1Ð%<�k�kÀ$Ä+ÔBYˆà—(’(˜<Ð>RÐ`k�(ÑlÔlˆà1<ÐL˜Ô-Ð-À'È!Ä*ˆà—’ Ñ/Ô/ˆàˆàÐØ×%Ò% f¨f°d´kÑBÔBˆDàð 	DØ�Y ¨¨¨¤Ñ,ˆFØ'+Ð'7�D�7˜VÑ#Ð#¸VÐCå3¸ÀfÐ\cÔ\qÐrÑrÔrÐrr$   )NNNN)rD   rE   rF   r-   r   r•   ÚFloatTensorÚ
LongTensorrÿ   r   rB   rH   rI   s   @r"   r.  r.  µ  sÁ   ø€ € € € € ð
ð 
ð 
ð 
ð 
ð ð 26Ø*.Ø,0Ø#'ðsð sàÔ'¨$Ñ.ðsð Ô  4Ñ'ðsð # T™kð	sð
 ˜D‘[ðsð 
.ðsð sð sñ „^ðsð sð sð sð sr$   r.  zL
    BiT backbone, to be used with frameworks like DETR and MaskFormer.
    c                   ó|   ‡ — e Zd ZdZˆ fd„Zeee	 	 d	dede	dz  de	dz  de
fd„¦   «         ¦   «         ¦   «         Zˆ xZS )
ÚBitBackboneFc                 óÄ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        g|j        z   | _        |                      ¦   «          d S rW   )r,   r-   r  r  r~   rì   Únum_featuresr#  r‡   s     €r"   r-   zBitBackbone.__init__ò  sX   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å˜FÑ#Ô#ˆŒØ#Ô2Ð3°fÔ6IÑIˆÔð 	�ŠÑÔÐÐÐr$   Nrˆ   rö   r÷   r   c                 ó@  — |�|n| j         j        }|�|n| j         j        }|                      |dd¬¦  «        }|j        }d}t          | j        ¦  «        D ]\  }}	|	| j        v r|||         fz  }Œ|s|f}
|r|
|j        fz  }
|
S t          ||r|j        ndd¬¦  «        S )aŸ  
        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, AutoBackbone
        >>> import torch
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> processor = AutoImageProcessor.from_pretrained("google/bit-50")
        >>> model = AutoBackbone.from_pretrained("google/bit-50")

        >>> inputs = processor(image, return_tensors="pt")
        >>> outputs = model(**inputs)
        ```NTr%  r×   )Úfeature_mapsrw   Ú
attentions)	rS   r÷   rö   r  rw   rÙ   Ústage_namesÚout_featuresr
   )r1   rˆ   rö   r÷   r'  r8  rw   rA  Úidxrò   r9  s              r"   rB   zBitBackbone.forwardû  sî   € ð< &1Ð%<�k�kÀ$Ä+ÔBYˆà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð —(’(˜<¸dÐPT�(ÑUÔUˆàÔ-ˆàˆÝ# DÔ$4Ñ5Ô5ð 	6ð 	6‰JˆC�Ø˜Ô)Ð)Ð)Ø ¨sÔ!3Ð 5Ñ5�øàð 	Ø"�_ˆFØ#ð 3Ø˜7Ô0Ð2Ñ2�ØˆMåØ%Ø3GÐQ˜'Ô/Ð/ÈTØð
ñ 
ô 
ð 	
r$   r+  )rD   rE   rF   Úhas_attentionsr-   r   r	   r   r   rÿ   r
   rB   rH   rI   s   @r"   r=  r=  ê  s°   ø€ € € € € ð €Nðð ð ð ð ð Ø Øð -1Ø#'ð	3
ð 3
àð3
ð # T™kð3
ð ˜D‘[ð	3
ð 
ð3
ð 3
ð 3
ñ „^ñ !Ô ñ Ôð3
ð 3
ð 3
ð 3
ð 3
r$   r=  )r.  r  r  r=  )Nr   r   r   )rž   )9rG   ro   r]   Únumpyræ   r•   r   r   Ú r   r  Úactivationsr   Úbackbone_utilsr   r	   Úmodeling_outputsr
   r   r   r   Úmodeling_utilsr   Úutilsr   r   Úutils.genericr   Úconfiguration_bitr   Ú
get_loggerrD   Úloggerrý   rÿ   r#   r  r&   Ú	GroupNormrK   ÚModuler.   Ú	MaxPool2drl   ry   rŽ   r£   r¥   r¿   r«   rÉ   rÝ   r  r  r.  r=  Ú__all__r×   r$   r"   ú<module>rV     s¬  ðð @Ð ?à Ð Ð Ð Ø €€€à Ð Ð Ð Ø €€€Ø Ð Ð Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø HÐ HÐ HÐ HÐ HÐ HÐ HÐ Hðð ð ð ð ð ð ð ð ð ð ð ð .Ð -Ð -Ð -Ð -Ð -Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø -Ð -Ð -Ð -Ð -Ð -Ø (Ð (Ð (Ð (Ð (Ð (ð 
ˆÔ	˜HÑ	%Ô	%€ð&ð &ÈEÐRWÐY]ÐR]ÔL^ð &ð &ð &ð &ðR-ð -ð -ð -ð -˜rœyñ -ô -ð -ð`ð ð ð ð ˜Rœ\ñ ô ð ð$0ð 0ð 0ð 0ð 0�2”9ñ 0ô 0ð 0ðf
ð 
ð 
ð 
ð 
�2”<ñ 
ô 
ð 
ð6/ð /ð /ð /ð /�B”Iñ /ô /ð /ðf%ð %ð %ð %ð %�"”)ñ %ô %ð %ð0ð ð ð ðA(ð A(ð A(ð A(ð A( b¤iñ A(ô A(ð A(ðHFð Fð Fð Fð F˜œñ Fô Fð FðR'ð 'ð 'ð 'ð '˜œ	ñ 'ô 'ð 'ð.Gð Gð Gð Gð GˆrŒyñ Gô Gð GðTC
ð C
ð C
ð C
ð C
�”ñ C
ô C
ð C
ðL ð:ð :ð :ð :ð :˜ñ :ô :ñ „ð:ð* ð2
ð 2
ð 2
ð 2
ð 2
Ð!ñ 2
ô 2
ñ „ð2
ðj €ððñ ô ð,sð ,sð ,sð ,sð ,sÐ 2ñ ,sô ,sñô ð,sð^ €ððñ ô ð
B
ð B
ð B
ð B
ð B
�-Ð!3ñ B
ô B
ñô ð
B
ðJ YÐ
XÐ
X€€€r$   