§
    ‚ŠtjKº  ã                   ó@  — d dl mZmZ d dlmZ d dlZd dlmZmZ ddlm	Z
 ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZ ddlmZmZ ddlmZ ddlm Z  ddl!m"Z"m#Z#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddl*m+Z+  e#d¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z, G d„ dej-        ¦  «        Z. G d„ dej-        ¦  «        Z/ G d„ dej-        ¦  «        Z0	 	 dPdej-        d ej        d!ej        d"ej        d#ej        dz  d$e1dz  d%e1d&ee"         fd'„Z2 G d(„ d)ej-        ¦  «        Z3 G d*„ d+ej-        ¦  «        Z4 G d,„ d-ej-        ¦  «        Z5 G d.„ d/e¦  «        Z6e# G d0„ d1e¦  «        ¦   «         Z7e# G d2„ d3e7¦  «        ¦   «         Z8 G d4„ d5ej-        ¦  «        Z9 e#d6¬¦  «         G d7„ d8e7¦  «        ¦   «         Z: e#d9¬¦  «         G d:„ d;e7¦  «        ¦   «         Z; G d<„ d=ej-        ¦  «        Z< G d>„ d?ej-        ¦  «        Z= G d@„ dAej-        ¦  «        Z> G dB„ dCej-        ¦  «        Z? G dD„ dEej-        ¦  «        Z@ G dF„ dGej-        ¦  «        ZA G dH„ dIej-        ¦  «        ZBe# G dJ„ dKe7¦  «        ¦   «         ZC e#dL¬¦  «         G dM„ dNee7¦  «        ¦   «         ZDg dO¢ZEdS )Qé    )ÚCallableÚIterable)Ú	dataclassN)ÚTensorÚnné   )Úinitialization)ÚACT2FN)ÚBackboneMixinÚfilter_output_hidden_states)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)ÚBackboneOutputÚBaseModelOutputWithPoolingÚImageClassifierOutputÚMaskedLMOutputÚSemanticSegmenterOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Ú#compile_compatible_method_lru_cache)ÚTransformersKwargsÚauto_docstringÚ	torch_int)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputsé   )Ú
BeitConfigz-
    Class for outputs of [`BeitModel`].
    )Úcustom_introc                   ó   — e Zd ZdZdS )ÚBeitModelOutputWithPoolingaF  
    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`):
        Average of the last layer hidden states of the patch tokens (excluding the *[CLS]* token) if
        *config.use_mean_pooling* is set to True. If set to False, then the final hidden state of the *[CLS]* token
        will be returned.
    N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__© ó    úd/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/beit/modeling_beit.pyr"   r"   0   s   € € € € € ðð ð ð r(   r"   c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚBeitPatchEmbeddingszì
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    Úconfigc                 ó¢  •— t          ¦   «                              ¦   «          |j        }|j        }t	          |t
          ¦  «        r|n||f}t	          |t
          ¦  «        r|n||f}|d         |d         z  |d         |d         z  z  | _        || _        || _        |j        | _        t          j	        |j        |j
        ||¬¦  «        | _        d S )Nr   r   ©Úkernel_sizeÚstride)ÚsuperÚ__init__Ú
image_sizeÚ
patch_sizeÚ
isinstancer   Únum_patchesÚnum_channelsr   ÚConv2dÚhidden_sizeÚ
projection)Úselfr,   r3   r4   Ú	__class__s       €r)   r2   zBeitPatchEmbeddings.__init__F   sÎ   ø€ Ý‰Œ×ÒÑÔÐØÔ&ˆ
ØÔ&ˆ
Ý#-¨j½(Ñ#CÔ#CÐa�Z�ZÈ*ÐV`ÐIaˆ
Ý#-¨j½(Ñ#CÔ#CÐa�Z�ZÈ*ÐV`ÐIaˆ
à& qœM¨Z¸¬]Ñ:¸zÈ!¼}ÐPZÐ[\ÔP]Ñ?]Ñ^ˆÔØ$ˆŒØ$ˆŒØ"Ô/ˆÔÝœ) FÔ$7¸Ô9KÐYcÐlvÐwÑwÔwˆŒˆˆr(   Úpixel_valuesÚreturnc                 óà   — |j         d         }|| j        k    rt          d| j        › d|› d�¦  «        ‚|                      |¦  «                             d¦  «                             dd¦  «        S )Nr   zoMake sure that the channel dimension of the pixel values match with the one set in the configuration. Expected z	 but got ú.é   )Úshaper7   Ú
ValueErrorr:   ÚflattenÚ	transpose)r;   r=   r7   s      r)   ÚforwardzBeitPatchEmbeddings.forwardS   sŽ   € Ø#Ô)¨!Ô,ˆØ˜4Ô,Ò,Ð,ÝðIØ!Ô.ðIð IØ9EðIð Ið Iñô ð ð �Š˜|Ñ,Ô,×4Ò4°QÑ7Ô7×AÒAÀ!ÀQÑGÔGÐGr(   )
r#   r$   r%   r&   r   r2   Útorchr   rF   Ú__classcell__©r<   s   @r)   r+   r+   ?   s…   ø€ € € € € ðð ðx˜zð xð xð xð xð xð xðH E¤Lð H°U´\ð Hð Hð Hð Hð Hð Hð Hð Hr(   r+   c                   ó”   ‡ — e Zd ZdZdeddfˆ fd„Zdej        dededej        fd	„Z		 dd
ej        dej
        dz  dej        fd„Zˆ xZS )ÚBeitEmbeddingszb
    Construct the CLS token, position and patch embeddings. Optionally, also the mask token.
    r,   r>   Nc                 ó2  •— t          ¦   «                              ¦   «          t          j        t	          j        dd|j        ¦  «        ¦  «        | _        |j        r-t          j        t	          j        dd|j        ¦  «        ¦  «        nd | _	        t          |¦  «        | _        |j        | _        | j        j        }|j        r0t          j        t	          j        d|dz   |j        ¦  «        ¦  «        nd | _        t          j        |j        ¦  «        | _        d S )Nr   )r1   r2   r   Ú	ParameterrG   Úzerosr9   Ú	cls_tokenÚuse_mask_tokenÚ
mask_tokenr+   Úpatch_embeddingsr4   r6   Ú use_absolute_position_embeddingsÚposition_embeddingsÚDropoutÚhidden_dropout_probÚdropout)r;   r,   r6   r<   s      €r)   r2   zBeitEmbeddings.__init__b   sæ   ø€ Ý‰Œ×ÒÑÔÐåœ¥e¤k°!°Q¸Ô8JÑ&KÔ&KÑLÔLˆŒØQWÔQfÐp�"œ,¥u¤{°1°a¸Ô9KÑ'LÔ'LÑMÔMÐMÐlpˆŒÝ 3°FÑ ;Ô ;ˆÔØ Ô+ˆŒØÔ+Ô7ˆð Ô6ð�BŒL�œ Q¨°a©¸Ô9KÑLÔLÑMÔMÐMàð 	Ô õ
 ”z &Ô"<Ñ=Ô=ˆŒˆˆr(   Ú
embeddingsÚheightÚwidthc                 ó”  — |j         d         dz
  }| j        j         d         dz
  }t          j                             ¦   «         s||k    r||k    r| j        S | j        dd…dd…f         }| j        dd…dd…f         }|j         d         }|| j        z  }	|| j        z  }
t          |dz  ¦  «        }|                     d|||¦  «        }|                     dddd¦  «        }t          j
                             ||	|
fdd	¬
¦  «        }|                     dddd¦  «                             dd|¦  «        }t          j        ||fd¬¦  «        S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   Néÿÿÿÿg      à?r   r   rA   ÚbicubicF©ÚsizeÚmodeÚalign_corners©Údim)rB   rT   rG   ÚjitÚ
is_tracingr4   r   ÚreshapeÚpermuter   Ú
functionalÚinterpolateÚviewÚcat)r;   rX   rY   rZ   r6   Únum_positionsÚclass_pos_embedÚpatch_pos_embedrc   Ú
new_heightÚ	new_widthÚsqrt_num_positionss               r)   Úinterpolate_pos_encodingz'BeitEmbeddings.interpolate_pos_encodingq   sr  € ð !Ô& qÔ)¨AÑ-ˆØÔ0Ô6°qÔ9¸AÑ=ˆõ Œy×#Ò#Ñ%Ô%ð 	,¨+¸Ò*FÐ*FÈ6ÐUZÊ?È?ØÔ+Ð+àÔ2°1°1°1°b°q°b°5Ô9ˆØÔ2°1°1°1°a°b°b°5Ô9ˆàÔ˜rÔ"ˆà˜tœÑ.ˆ
Ø˜Tœ_Ñ,ˆ	å& }°cÑ'9Ñ:Ô:ÐØ)×1Ò1°!Ð5GÐI[Ð]`ÑaÔaˆØ)×1Ò1°!°Q¸¸1Ñ=Ô=ˆåœ-×3Ò3ØØ˜iÐ(ØØð	 4ñ 
ô 
ˆð *×1Ò1°!°Q¸¸1Ñ=Ô=×BÒBÀ1ÀbÈ#ÑNÔNˆåŒy˜/¨?Ð;ÀÐCÑCÔCÐCr(   r=   Úbool_masked_posc                 óô  — |j         \  }}}}|                      |¦  «        }|                     ¦   «         \  }}}|�R| j                             ||d¦  «        }	|                     d¦  «                             |	¦  «        }
|d|
z
  z  |	|
z  z   }| j                             |dd¦  «        }t          j	        ||fd¬¦  «        }| j
        �||                      |||¦  «        z   }|                      |¦  «        }|S ©Nr\   r   rb   )rB   rR   r_   rQ   ÚexpandÚ	unsqueezeÚtype_asrO   rG   rk   rT   rr   rW   )r;   r=   rs   Ú_rY   rZ   rX   Ú
batch_sizeÚseq_lenÚmask_tokensÚmaskÚ
cls_tokenss               r)   rF   zBeitEmbeddings.forward™   s  € ð
 +Ô0Ñˆˆ1ˆf�eØ×*Ò*¨<Ñ8Ô8ˆ
Ø!+§¢Ñ!2Ô!2Ñˆ
�G˜QàÐ&Øœ/×0Ò0°¸WÀbÑIÔIˆKà"×,Ò,¨RÑ0Ô0×8Ò8¸ÑEÔEˆDØ# q¨4¡xÑ0°;ÀÑ3EÑEˆJà”^×*Ò*¨:°r¸2Ñ>Ô>ˆ
Ý”Y 
¨JÐ7¸QÐ?Ñ?Ô?ˆ
àÔ#Ð/Ø# d×&CÒ&CÀJÐPVÐX]Ñ&^Ô&^Ñ^ˆJà—\’\ *Ñ-Ô-ˆ
àÐr(   ©N)r#   r$   r%   r&   r   r2   rG   r   Úintrr   Ú
BoolTensorrF   rH   rI   s   @r)   rK   rK   ]   sÛ   ø€ € € € € ðð ð>˜zð >¨dð >ð >ð >ð >ð >ð >ð&D°5´<ð &DÈð &DÐUXð &DÐ]bÔ]ið &Dð &Dð &Dð &DðV 48ðð à”lðð Ô)¨DÑ0ðð 
Œð	ð ð ð ð ð ð ð r(   rK   c                   ó¢   ‡ — e Zd Zdeddfˆ fd„Ze ed¬¦  «        deeef         de	j
        fd„¦   «         ¦   «         Zdd
ede	j
        fd„Zˆ xZS )ÚBeitRelativePositionBiasr,   r>   Nc                 óª  •— t          ¦   «                              ¦   «          |j        }t          |t          t
          f¦  «        s||f}|d         |j        z  |d         |j        z  f| _        d| j        d         z  dz
  d| j        d         z  dz
  z  dz   | _        t          j
        t          j        | j        |j        ¦  «        ¦  «        | _        d S ©Nr   r   rA   r   )r1   r2   r3   r5   ÚtupleÚlistr4   Úwindow_sizeÚnum_relative_distancer   rM   rG   rN   Únum_attention_headsÚrelative_position_bias_table)r;   r,   r3   r<   s      €r)   r2   z!BeitRelativePositionBias.__init__´   sÏ   ø€ Ý‰Œ×ÒÑÔÐØÔ&ˆ
Ý˜*¥u­d mÑ4Ô4ð 	2Ø$ jÐ1ˆJØ& qœM¨VÔ->Ñ>À
È1ÄÐQWÔQbÑ@bÐcˆÔØ&'¨$Ô*:¸1Ô*=Ñ&=ÀÑ&AÀaÈ$ÔJZÐ[\ÔJ]ÑF]Ð`aÑFaÑ%bÐefÑ%fˆÔ"Ý,.¬LÝŒK˜Ô2°FÔ4NÑOÔOñ-
ô -
ˆÔ)Ð)Ð)r(   é
   )Úmaxsizerˆ   c                 óT  — d| d         z  dz
  d| d         z  dz
  z  dz   }| d         | d         z  }t          j        t          j        t          j        t          j        | d         ¦  «        t          j        | d         ¦  «        d¬¦  «        ¦  «        d¬¦  «        }|dd…dd…df         |dd…ddd…f         z
                       ddd¦  «                             ¦   «         }|dd…dd…dfxx         | d         dz
  z  cc<   |dd…dd…dfxx         | d         dz
  z  cc<   |dd…dd…dfxx         d| d         z  dz
  z  cc<   t          j        |dz   fdz  |j        ¬	¦  «        }| 	                    d
¦  «        |dd…dd…f<   |dz
  |ddd…f<   |dz
  |dd…df<   |dz
  |d<   |S )z¿
        This method creates the relative position index, modified to support arbitrary window sizes,
        as introduced in [MiDaS v3.1](https://huggingface.co/papers/2307.14460).
        rA   r   r   r   Úij)Úindexing)Ú	start_dimN)r_   Údtyper\   )r   r   )
rG   rD   ÚstackÚmeshgridÚarangerg   Ú
contiguousrN   r’   Úsum)rˆ   r‰   Úwindow_areaÚcoords_flattenÚrelative_coordsÚrelative_position_indexs         r)   Ú generate_relative_position_indexz9BeitRelativePositionBias.generate_relative_position_indexÀ   s!  € ð "# [°¤^Ñ!3°aÑ!7¸AÀÈAÄÑ<NÐQRÑ<RÑ SÐVWÑ WÐØ! !”n {°1¤~Ñ5ˆõ œÝŒK�œ¥u¤|°KÀ´NÑ'CÔ'CÅUÄ\ÐR]Ð^_ÔR`ÑEaÔEaÐlpÐqÑqÔqÑrÔrØð
ñ 
ô 
ˆð *¨!¨!¨!¨Q¨Q¨Q°¨*Ô5¸ÀqÀqÀqÈ$ÐPQÐPQÐPQÀzÔ8RÑR×[Ò[Ð\]Ð_`ÐbcÑdÔd×oÒoÑqÔqˆà˜˜˜˜1˜1˜1˜a˜Ð Ð Ô  K°¤N°QÑ$6Ñ6Ð Ð Ñ Ø˜˜˜˜1˜1˜1˜a˜Ð Ð Ô  K°¤N°QÑ$6Ñ6Ð Ð Ñ Ø˜˜˜˜1˜1˜1˜a˜Ð Ð Ô  A¨°A¬Ñ$6¸Ñ$:Ñ:Ð Ð Ñ å"'¤+°KÀ!±OÐ3EÈÑ3IÐQ`ÔQfÐ"gÑ"gÔ"gÐØ*9×*=Ò*=¸bÑ*AÔ*AÐ    A B B Ñ'Ø)>ÀÑ)BÐ  1 2 2 Ñ&Ø)>ÀÑ)BÐ    A Ñ&Ø(=ÀÑ(AÐ Ñ%Ø&Ð&r(   Frr   c                 ó0  — d| j         d         z  dz
  }d| j         d         z  dz
  }d|d         z  dz
  }d|d         z  dz
  }| j        }| j        }	||z  dz   }
|d|	dz
  …         }|                     d||d¦  «                             dddd¦  «        }t
          j                             |t          |¦  «        t          |¦  «        fd¬¦  «        }|                     dddd¦  «                             |
dz
  d¦  «        }t          j
        |||	dz
  d…         g¦  «        }|                      |¦  «        }||                     d¦  «                 }|                     |d         |d         z  dz   |d         |d         z  dz   d¦  «        }|                     ddd¦  «                             ¦   «         }|rKt
          j                             |                     d¦  «        ||fdd	¬
¦  «                             d¦  «        }|                     d¦  «        S )zu
        Modification of timm.models.beit.py: Attention._get_rel_pos_bias to support arbitrary window sizes.
        rA   r   r   r   Nr\   Úbilinear)r_   r`   Fr^   )rˆ   r‹   r‰   rf   rg   r   rh   ri   r   rG   rk   rœ   rj   r–   rw   Úsqueeze)r;   rˆ   rr   Údim_sizeÚ
old_heightÚ	old_widthro   rp   Ú old_relative_position_bias_tableÚold_num_relative_distanceÚnew_num_relative_distanceÚold_sub_tableÚnew_sub_tableÚ new_relative_position_bias_tabler›   Úrelative_position_biass                   r)   rF   z BeitRelativePositionBias.forwardÜ   s`  € ð ˜Ô)¨!Ô,Ñ,¨qÑ0ˆ
Ø˜Ô(¨Ô+Ñ+¨aÑ/ˆ	à˜ QœÑ'¨!Ñ+ˆ
Ø˜ AœÑ&¨Ñ*ˆ	à+/Ô+LÐ(à$(Ô$>Ð!Ø$.°Ñ$:¸QÑ$>Ð!à8Ð9XÐ;TÐWXÑ;XÐ9XÔYˆà%×-Ò-¨a°¸JÈÑKÔK×SÒSÐTUÐWXÐZ[Ð]^Ñ_Ô_ˆÝœ×1Ò1Ø¥¨:Ñ!6Ô!6½	À)Ñ8LÔ8LÐ MÐT^ð 2ñ 
ô 
ˆð &×-Ò-¨a°°A°qÑ9Ô9×AÒAÐB[Ð^_ÑB_ÐacÑdÔdˆå+0¬9ØÐ<Ð=VÐYZÑ=ZÐ=\Ð=\Ô]Ð^ñ,
ô ,
Ð(ð #'×"GÒ"GÈÑ"TÔ"TÐØ!AÐBY×B^ÒB^Ð_aÑBbÔBbÔ!cÐð "8×!<Ò!<Ø˜ŒN˜[¨œ^Ñ+¨aÑ/°¸Q´À+ÈaÄ.Ñ1PÐSTÑ1TÐVXñ"
ô "
Ðð "8×!?Ò!?ÀÀ1ÀaÑ!HÔ!H×!SÒ!SÑ!UÔ!UÐà#ð 	Ý%'¤]×%>Ò%>Ø&×0Ò0°Ñ3Ô3Ø Ð)ØØ#ð	 &?ñ &ô &÷
 Šg�a‰jŒjð #ð &×/Ò/°Ñ2Ô2Ð2r(   )FN)r#   r$   r%   r   r2   Ústaticmethodr   r†   r€   rG   r   rœ   ÚboolrF   rH   rI   s   @r)   rƒ   rƒ   ³   sÇ   ø€ € € € € ð	
˜zð 	
¨dð 	
ð 	
ð 	
ð 	
ð 	
ð 	
ð Ø(Ð(°Ð4Ñ4Ô4ð'°e¸CÀ¸H´oð 'È%Ì,ð 'ð 'ð 'ñ 5Ô4ñ „\ð'ð4-3ð -3¸Tð -3Ð]bÔ]ið -3ð -3ð -3ð -3ð -3ð -3ð -3ð -3r(   rƒ   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingrW   Úkwargsc                 óô  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |dt          j        ¬¦  «                             |j	        ¦  «        }t          j         
                    ||| j        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «                             ¦   «         }	|	|fS )Nr\   ç      à¿rA   r   )rc   r’   )ÚpÚtrainingr   )r_   rG   ÚmatmulrE   r   rh   ÚsoftmaxÚfloat32Útor’   rW   r·   r–   )
r­   r®   r¯   r°   r±   r²   rW   r³   Úattn_weightsÚattn_outputs
             r)   Úeager_attention_forwardr¾     sÞ   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r(   c                   óŠ   ‡ — e Zd Zdefˆ fd„Z	 d	dej        dej        dz  dee         de	ej        ej        f         fd„Z
ˆ xZS )
ÚBeitAttentionr,   c                 ó\  •— t          ¦   «                              ¦   «          || _        |j        | _        t	          |d|j        |j        z  ¦  «        | _        |j        | _        | j        dz  | _	        d| _
        t          j        |j        |j        | j        z  ¦  «        | _        t          j        |j        |j        | j        z  d¬¦  «        | _        t          j        |j        |j        | j        z  ¦  «        | _        t          j        |j        | j        z  |j        ¦  «        | _        d S )NÚhead_dimrµ   F)Úbias)r1   r2   r,   rŠ   Úgetattrr9   rÂ   Úattention_probs_dropout_probÚattention_dropoutr²   Ú	is_causalr   ÚLinearÚq_projÚk_projÚv_projÚo_proj©r;   r,   r<   s     €r)   r2   zBeitAttention.__init__)  sù   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ#)Ô#=ˆÔ Ý ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ!'Ô!DˆÔØ”} dÑ*ˆŒØˆŒÝ”i Ô 2°FÔ4NÐQUÔQ^Ñ4^Ñ_Ô_ˆŒÝ”i Ô 2°FÔ4NÐQUÔQ^Ñ4^ÐejÐkÑkÔkˆŒÝ”i Ô 2°FÔ4NÐQUÔQ^Ñ4^Ñ_Ô_ˆŒÝ”i Ô :¸T¼]Ñ JÈFÔL^Ñ_Ô_ˆŒˆˆr(   NÚhidden_statesr±   r³   r>   c                 ó¬  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }t          j        | j	        j
        t          ¦  «        }	 |	| ||||f| j        sdn| j        | j        dœ|¤Ž\  }
} |
j        g |¢d‘R Ž                      ¦   «         }
|                      |
¦  «        }
|
|fS )Nr\   r   rA   r¬   )rW   r²   )rB   rÂ   rÉ   rj   rE   rÊ   rË   r   Úget_interfacer,   Ú_attn_implementationr¾   r·   rÆ   r²   rf   r–   rÌ   )r;   rÎ   r±   r³   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚattention_interfacer½   r¼   s               r)   rF   zBeitAttention.forward6  sw  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆà˜LÐ(Ð(r(   r   )r#   r$   r%   r   r2   rG   r   r   r   r†   rF   rH   rI   s   @r)   rÀ   rÀ   (  s¬   ø€ € € € € ð`˜zð `ð `ð `ð `ð `ð `ð  /3ð)ð )à”|ð)ð œ tÑ+ð)ð Ð+Ô,ð	)ð
 
ˆuŒ|˜Uœ\Ð)Ô	*ð)ð )ð )ð )ð )ð )ð )ð )r(   rÀ   c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚBeitMLPr,   c                 ó  •— t          ¦   «                              ¦   «          || _        t          |j                 | _        t          j        |j        |j	        ¦  «        | _
        t          j        |j	        |j        ¦  «        | _        d S r   )r1   r2   r,   r
   Ú
hidden_actÚactivation_fnr   rÈ   r9   Úintermediate_sizeÚfc1Úfc2rÍ   s     €r)   r2   zBeitMLP.__init__Y  sf   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ# FÔ$5Ô6ˆÔÝ”9˜VÔ/°Ô1IÑJÔJˆŒÝ”9˜VÔ5°vÔ7IÑJÔJˆŒˆˆr(   rÎ   r>   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r   )rÞ   rÜ   rß   ©r;   rÎ   s     r)   rF   zBeitMLP.forward`  s=   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆàÐr(   ©	r#   r$   r%   r   r2   rG   r   rF   rH   rI   s   @r)   rÙ   rÙ   X  sq   ø€ € € € € ðK˜zð Kð Kð Kð Kð Kð Kð U¤\ð °e´lð ð ð ð ð ð ð ð r(   rÙ   c                   ó^   ‡ — e Zd ZdZd
deddfˆ fd„Zdej        dej        fd„Zde	fd	„Z
ˆ xZS )ÚBeitDropPathzÏStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    r¬   Ú	drop_probr>   Nc                 óV   •— t          ¦   «                              ¦   «          || _        d S r   )r1   r2   rå   )r;   rå   r<   s     €r)   r2   zBeitDropPath.__init__o  s$   ø€ Ý‰Œ×ÒÑÔÐØ"ˆŒˆˆr(   rÎ   c                 ó  — | j         dk    s| j        s|S d| j         z
  }|j        d         fd|j        dz
  z  z   }t	          j        ||j        |j        ¬¦  «        }t	          j        ||z   ¦  «        }| 	                    |¦  «        |z  S )Nr¬   r   r   ©r   )r’   Údevice)
rå   r·   rB   ÚndimrG   Úrandr’   ré   ÚfloorÚdiv)r;   rÎ   Ú	keep_probrB   Úrandom_tensors        r)   rF   zBeitDropPath.forwards  s“   € ØŒ>˜SÒ Ð ¨¬Ð Ø Ð Ø˜œÑ&ˆ	ØÔ$ QÔ'Ð)¨D°MÔ4FÈÑ4JÑ,KÑKˆÝœ
 5°Ô0CÈMÔL`ÐaÑaÔaˆÝœ M°IÑ$=Ñ>Ô>ˆØ× Ò  Ñ+Ô+¨mÑ;Ð;r(   c                 ó   — d| j         › �S )Nzp=)rå   ©r;   s    r)   Ú
extra_reprzBeitDropPath.extra_repr|  s   € Ø$�D”NÐ$Ð$Ð$r(   ©r¬   )r#   r$   r%   r&   Úfloatr2   rG   r   rF   Ústrrò   rH   rI   s   @r)   rä   rä   h  s›   ø€ € € € € ðð ð#ð # %ð #°$ð #ð #ð #ð #ð #ð #ð< U¤\ð <°e´lð <ð <ð <ð <ð%˜Cð %ð %ð %ð %ð %ð %ð %ð %r(   rä   c                   óœ   ‡ — e Zd ZdZddedefˆ fd„Z	 	 	 ddej        d	ej        dz  d
e	de
eef         dz  dee         dej        fd„Zˆ xZS )Ú	BeitLayerz?This corresponds to the Block class in the timm implementation.r¬   r,   Údrop_path_ratec                 ó  •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          j        |j        |j        ¬¦  «        | _        t	          j        |j        |j        ¬¦  «        | _	        t          |¦  «        | _        t	          j        |j        ¦  «        | _        |j        | _        |dk    rt!          |¦  «        nt	          j        ¦   «         | _        |j        }|dk    r0t	          j        |t+          j        |j        ¦  «        z  d¬¦  «        nd| _        |dk    r0t	          j        |t+          j        |j        ¦  «        z  d¬¦  «        nd| _        |j        rt5          |¦  «        nd | _        d S )N©Úepsr¬   r   T)Úrequires_gradg      ð?)r1   r2   rÀ   Ú	attentionr   Ú	LayerNormr9   Úlayer_norm_epsÚlayernorm_beforeÚlayernorm_afterrÙ   ÚmlprU   rV   rW   r4   rä   ÚIdentityÚ	drop_pathÚlayer_scale_init_valuerM   rG   ÚonesÚlambda_1Úlambda_2Úuse_relative_position_biasrƒ   r©   )r;   r,   rø   Úinit_valuesr<   s       €r)   r2   zBeitLayer.__init__ƒ  sd  ø€ Ý‰Œ×ÒÑÔÐÝ& vÑ.Ô.ˆŒÝ "¤¨VÔ-?ÀVÔEZÐ [Ñ [Ô [ˆÔÝ!œ|¨FÔ,>ÀFÔDYÐZÑZÔZˆÔÝ˜6‘?”?ˆŒÝ”z &Ô"<Ñ=Ô=ˆŒØ Ô+ˆŒØ9GÈ#Ò9MÐ9M� nÑ5Ô5Ð5ÕSUÔS^ÑS`ÔS`ˆŒàÔ3ˆà^iÐlmÒ^mÐ^m�BŒL˜¥u¤z°&Ô2DÑ'EÔ'EÑEÐUYÐZÑZÔZÐZÐsvð 	Œð _jÐlmÒ^mÐ^m�BŒL˜¥u¤z°&Ô2DÑ'EÔ'EÑEÐUYÐZÑZÔZÐZÐsvð 	Œð KQÔJkÐ&uÕ&>¸vÑ&FÔ&FÐ&FÐquˆÔ#Ð#Ð#r(   NFrÎ   r±   rr   Ú
resolutionr³   r>   c                 ó&  — | j         �E|\  }}|| j        z  || j        z  f}|                       |||j        d         ¬¦  «        }	|�|	|z   n|	}|}
|                      |¦  «        } | j        |fd|i|¤Ž\  }}|                      |¦  «        }| j        |z  }|                      |¦  «        |
z   }|}
|                      |¦  «        }|  	                    |¦  «        }|                      |¦  «        }| j
        |z  }|                      |¦  «        |
z   }|S )Nr   )r    r±   )r©   r4   rB   r   rý   rW   r  r  r  r  r  )r;   rÎ   r±   rr   r  r³   rY   rZ   rˆ   r©   Úresidualry   s               r)   rF   zBeitLayer.forward–  s\  € ð Ô&Ð2Ø&‰MˆF�EØ! T¤_Ñ4°e¸t¼Ñ6NÐOˆKØ%)×%@Ò%@ØÐ5ÀÔ@SÐTUÔ@Vð &Añ &ô &Ð"ð <JÐ;UÐ&¨Ñ7Ð7Ð[qð ð
 !ˆØ×-Ò-¨mÑ<Ô<ˆØ)˜4œ>Øð
ð 
à)ð
ð ð
ð 
Ñˆ�qð
 Ÿš ]Ñ3Ô3ˆØœ¨Ñ5ˆØŸš }Ñ5Ô5¸Ñ@ˆð !ˆØ×,Ò,¨]Ñ;Ô;ˆØŸš Ñ/Ô/ˆØŸš ]Ñ3Ô3ˆØœ¨Ñ5ˆØŸš }Ñ5Ô5¸Ñ@ˆàÐr(   ró   ©NFN)r#   r$   r%   r&   r   rô   r2   rG   r   r«   r†   r€   r   r   rF   rH   rI   s   @r)   r÷   r÷   €  sÛ   ø€ € € € € ØIÐIðvð v˜zð v¸5ð vð vð vð vð vð vð, /3Ø).Ø-1ð&ð &à”|ð&ð œ tÑ+ð&ð #'ð	&ð
 ˜#˜s˜(”O dÑ*ð&ð Ð+Ô,ð&ð 
Œð&ð &ð &ð &ð &ð &ð &ð &r(   r÷   c                   ó’   ‡ — e Zd ZU eed<   dZdZdZdZdgZ	dZ
dZdZdZdZeedœZd	Zd
gZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚBeitPreTrainedModelr,   Úbeitr=   )ÚimageTr÷   F)rÎ   Ú
attentionsrR   z.*relative_position_index.*c                 óŽ  •— t          ¦   «                              |¦  «         t          |t          ¦  «        r]t	          j        |j        ¦  «         |j        �t	          j        |j        ¦  «         |j        �t	          j        |j        ¦  «         dS dS t          |t          ¦  «        rt	          j        |j
        ¦  «         dS t          |t          ¦  «        rit          |j        t          j        ¦  «        rLt	          j        |j        | j        j        ¦  «         t	          j        |j        | j        j        ¦  «         dS dS dS )zInitialize the weightsN)r1   Ú_init_weightsr5   rK   ÚinitÚzeros_rO   rQ   rT   rƒ   r‹   r÷   r  r   rM   Ú	constant_r,   r  r  )r;   r­   r<   s     €r)   r  z!BeitPreTrainedModel._init_weightsÓ  s0  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�nÑ-Ô-ð 	TÝŒK˜Ô(Ñ)Ô)Ð)ØÔ Ð,Ý”˜FÔ-Ñ.Ô.Ð.ØÔ)Ð5Ý”˜FÔ6Ñ7Ô7Ð7Ð7Ð7ð 6Ð5å˜Õ 8Ñ9Ô9ð 	TÝŒK˜Ô;Ñ<Ô<Ð<Ð<Ð<Ý˜¥	Ñ*Ô*ð 	TÝ˜&œ/­2¬<Ñ8Ô8ð TÝ”˜vœ°´Ô0RÑSÔSÐSÝ”˜vœ°´Ô0RÑSÔSÐSÐSÐSð	Tð 	TðTð Tr(   )r#   r$   r%   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_attention_backendÚ_can_compile_fullgraphr÷   rÀ   Ú_can_record_outputsÚ_input_embed_layerÚ"_keys_to_ignore_on_load_unexpectedrG   Úno_gradr  rH   rI   s   @r)   r  r  ¿  sÁ   ø€ € € € € € àÐÐÑØÐØ$€OØ!ÐØ&*Ð#Ø$˜ÐØ€NØ ÐØÐØ"&ÐØ!Ðà"Ø#ðð Ðð ,ÐØ*HÐ)IÐ&à€U„]�_„_ðTð Tð Tð Tñ „_ðTð Tð Tð Tð Tr(   r  c                   óÐ   ‡ — e Zd Zddededdfˆ fd„Ze ed¬¦  «        e	 	 	 dd	e	j
        d
e	j        dz  dede	j
        dz  dee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú	BeitModelTr,   Úadd_pooling_layerr>   Nc                 ó  •‡— t          ¦   «                              ‰¦  «         ‰| _        t          ‰¦  «        | _        ‰j        rt          ‰¦  «        nd| _        ˆfd„t          ‰j	        ¦  «        D ¦   «         }t          j        ˆfd„|D ¦   «         ¦  «        | _        ‰j        rt          j        ¦   «         nt          j        ‰j        ‰j        ¬¦  «        | _        |rt'          ‰¦  «        nd| _        |                      ¦   «          dS )zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        Nc                 óV   •— g | ]%}‰j         |z  t          ‰j        d z
  d ¦  «        z  ‘Œ&S rè   )rø   ÚmaxÚnum_hidden_layers)Ú.0Úir,   s     €r)   ú
<listcomp>z&BeitModel.__init__.<locals>.<listcomp>ó  sE   ø€ ð 
ð 
ð 
ØQRˆFÔ! AÑ%­¨FÔ,DÀqÑ,HÈ!Ñ(LÔ(LÑLð
ð 
ð 
r(   c                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS ))rø   )r÷   )r/  Úrr,   s     €r)   r1  z&BeitModel.__init__.<locals>.<listcomp>ö  s&   ø€ Ð$bÐ$bÐ$bÈQ¥Y¨vÀaÐ%HÑ%HÔ%HÐ$bÐ$bÐ$br(   rú   )r1   r2   r,   rK   rX   Ú!use_shared_relative_position_biasrƒ   Úshared_position_biasÚranger.  r   Ú
ModuleListÚlayersÚuse_mean_poolingr  rþ   r9   rÿ   Ú	layernormÚ
BeitPoolerÚpoolerÚ	post_init)r;   r,   r*  Údrop_path_ratesr<   s    `  €r)   r2   zBeitModel.__init__ç  s  øø€ õ
 	‰Œ×Ò˜Ñ Ô Ð ØˆŒå(¨Ñ0Ô0ˆŒà06Ô0XÐbÕ$ VÑ,Ô,Ð,Ð^bð 	Ô!ð
ð 
ð 
ð 
ÝV[Ð\bÔ\tÑVuÔVuð
ñ 
ô 
ˆõ ”mÐ$bÐ$bÐ$bÐ$bÐRaÐ$bÑ$bÔ$bÑcÔcˆŒð $Ô4Ðu�BŒK‰MŒMˆM½"¼,ÀvÔGYÐ_eÔ_tÐ:uÑ:uÔ:uð 	Œð ->ÐG•j Ñ(Ô(Ð(À4ˆŒð 	�ŠÑÔÐÐÐr(   F)Útie_last_hidden_statesr=   rs   rr   r±   r³   c                 óâ  — |                       ||¬¦  «        }|j        dd…         }t          | j        ||¬¦  «        }| j        �O|\  }}	|| j        j        z  |	| j        j        z  f}
|                      |
||j        d         ¬¦  «        }|�||z   n|}|}| j        D ]} ||f|||dœ|¤Ž}Œ|                      |¦  «        }| j        �|                      |¦  «        nd}t          ||¬¦  «        S )	zË
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        )rs   rA   N)r,   Úinputs_embedsr±   r   )rr   r    )r±   rr   r  )Úlast_hidden_stateÚpooler_output)
rX   rB   r   r,   r5  r4   r8  r:  r<  r"   )r;   r=   rs   rr   r±   r³   Úembedding_outputr  rY   rZ   rˆ   Úshared_relative_position_biasrÎ   ÚlayerÚsequence_outputÚpooled_outputs                   r)   rF   zBeitModel.forward   sW  € ð  Ÿ?š?¨<È˜?ÑYÔYÐØ!Ô'¨¨¨Ô+ˆ
å2Ø”;Ø*Ø)ð
ñ 
ô 
ˆð Ô$Ð0Ø&‰MˆF�EØ! T¤[Ô%;Ñ;¸UÀdÄkÔF\Ñ=\Ð]ˆKØ,0×,EÒ,EØÐ6NÐYiÔYoÐpqÔYrð -Fñ -ô -Ð)ð
 "Ð-ð .°Ñ>Ð>à2ð ð )ˆØ”[ð 	ð 	ˆEØ!˜EØðà-Ø)AØ%ð	ð ð
 ðð ˆMˆMð Ÿ.š.¨Ñ7Ô7ˆØ8<¼Ð8O˜Ÿš OÑ4Ô4Ð4ÐUYˆå)¸OÐ[hÐiÑiÔiÐir(   )Tr  )r#   r$   r%   r   r«   r2   r   r   r   rG   r   r�   r   r   r"   rF   rH   rI   s   @r)   r)  r)  å  s
  ø€ € € € € ðð ˜zð ¸dð Èdð ð ð ð ð ð ð2  Ø€_¨EÐ2Ñ2Ô2Øð 48Ø).Ø.2ð-jð -jà”lð-jð Ô)¨DÑ0ð-jð #'ð	-jð
 œ tÑ+ð-jð Ð+Ô,ð-jð 
$ð-jð -jð -jñ „^ñ 3Ô2ñ  Ôð-jð -jð -jð -jð -jr(   r)  c                   óL   ‡ — e Zd Zdeddfˆ fd„Zdej        dej        fd„Zˆ xZS )r;  r,   r>   Nc                 ó¤   •— t          ¦   «                              ¦   «          |j        r t          j        |j        |j        ¬¦  «        nd | _        d S )Nrú   )r1   r2   r9  r   rþ   r9   rÿ   r:  rÍ   s     €r)   r2   zBeitPooler.__init__4  sJ   ø€ Ý‰Œ×ÒÑÔÐàKQÔKbÐl�BŒL˜Ô+°Ô1FÐGÑGÔGÐGÐhlð 	Œˆˆr(   rÎ   c                 ó–   — | j         �7|                       |d d …dd …d d …f                              d¦  «        ¦  «        n|d d …df         S )Nr   r   )r:  Úmeanrá   s     r)   rF   zBeitPooler.forward:  sX   € àBFÄ.ÐB\ˆt�~Š~˜m¨A¨A¨A¨q¨r¨r°1°1°1¨HÔ5×:Ò:¸1Ñ=Ô=Ñ>Ô>Ð>ÐboÐpqÐpqÐpqÐstÐptÔbuÐur(   râ   rI   s   @r)   r;  r;  3  s{   ø€ € € € € ð
˜zð 
¨dð 
ð 
ð 
ð 
ð 
ð 
ðv U¤\ð v°e´lð vð vð vð vð vð vð vð vr(   r;  aÔ  
    Beit Model transformer with a 'language' modeling head on top. BEiT does masked image modeling by predicting
    visual tokens of a Vector-Quantize Variational Autoencoder (VQ-VAE), whereas other vision models like ViT and DeiT
    predict RGB pixel values. As a result, this class is incompatible with [`AutoModelForMaskedImageModeling`], so you
    will need to use [`BeitForMaskedImageModeling`] directly if you wish to do masked image modeling with BEiT.
    c                   óÐ   ‡ — e Zd Zdeddfˆ fd„Zd„ Zee	 	 	 	 	 ddej	        dz  dej
        dz  d	ej	        dz  d
edej	        dz  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚBeitForMaskedImageModelingr,   r>   Nc                 óH  •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        t          j        |j        |j        ¬¦  «        | _	        t          j
        |j        |j        ¦  «        | _        |                      ¦   «          d S )NF©r*  rú   )r1   r2   Ú
num_labelsr)  r  r   rþ   r9   rÿ   r:  rÈ   Ú
vocab_sizeÚlm_headr=  rÍ   s     €r)   r2   z#BeitForMaskedImageModeling.__init__H  sˆ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð à Ô+ˆŒÝ˜f¸Ð>Ñ>Ô>ˆŒ	õ œ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”y Ô!3°VÔ5FÑGÔGˆŒð 	�ŠÑÔÐÐÐr(   c                 ó   — d S r   r'   rñ   s    r)   Úget_output_embeddingsz0BeitForMaskedImageModeling.get_output_embeddingsU  s   € Øˆtr(   Fr=   rs   Úlabelsrr   r±   r³   c                 ó,  —  | j         |f|||dœ|¤Ž}|j        }|                      |¦  «        }|                      |dd…dd…f         ¦  «        }	d}
|�%t	          j        ¦   «         } ||	|         |¦  «        }
t          |
|	|j        |j        ¬¦  «        S )a…  
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, BeitForMaskedImageModeling
        >>> import torch
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-patch16-224-pt22k")
        >>> model = BeitForMaskedImageModeling.from_pretrained("microsoft/beit-base-patch16-224-pt22k")

        >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
        >>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
        >>> # create random boolean mask of shape (batch_size, num_patches)
        >>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

        >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
        >>> loss, logits = outputs.loss, outputs.logits
        >>> list(logits.shape)
        [1, 196, 8192]
        ```)rs   rr   r±   Nr   ©ÚlossÚlogitsrÎ   r  )	r  rB  r:  rS  r   ÚCrossEntropyLossr   rÎ   r  )r;   r=   rs   rV  rr   r±   r³   ÚoutputsrG  Úprediction_scoresÚmasked_lm_lossÚloss_fcts               r)   rF   z"BeitForMaskedImageModeling.forwardX  sÏ   € ðX �$”)Øð
à+Ø%=Ø)ð	
ð 
ð
 ð
ð 
ˆð "Ô3ˆØŸ.š.¨Ñ9Ô9ˆØ ŸLšL¨¸¸¸¸A¸B¸B¸Ô)?Ñ@Ô@ÐàˆØÐÝÔ*Ñ,Ô,ˆHØ%˜XÐ&7¸Ô&HÈ&ÑQÔQˆNåØØ$Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r(   )NNNFN)r#   r$   r%   r   r2   rU  r   r   rG   r   r�   r«   r   r   r†   r   rF   rH   rI   s   @r)   rN  rN  ?  s  ø€ € € € € ð˜zð ¨dð ð ð ð ð ð ðð ð ð Øð -1Ø37Ø&*Ø).Ø.2ð@
ð @
à”l TÑ)ð@
ð Ô)¨DÑ0ð@
ð ”˜tÑ#ð	@
ð
 #'ð@
ð œ tÑ+ð@
ð Ð+Ô,ð@
ð 
�Ñ	ð@
ð @
ð @
ñ „^ñ Ôð@
ð @
ð @
ð @
ð @
r(   rN  z¶
    Beit Model transformer with an image classification head on top (a linear layer on top of the average of the final
    hidden states of the patch tokens) e.g. for ImageNet.
    c                   óž   ‡ — e Zd Zdeddfˆ fd„Zee	 	 	 ddej        dz  dej        dz  de	d	e
e         deez  f
d
„¦   «         ¦   «         Zˆ xZS )ÚBeitForImageClassificationr,   r>   Nc                 ó:  •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        |j        dk    rt          j        |j        |j        ¦  «        nt          j        ¦   «         | _	        |  
                    ¦   «          d S )NTrP  r   )r1   r2   rQ  r)  r  r   rÈ   r9   r  Ú
classifierr=  rÍ   s     €r)   r2   z#BeitForImageClassification.__init__¤  s‹   ø€ Ý‰Œ×Ò˜Ñ Ô Ð à Ô+ˆŒÝ˜f¸Ð=Ñ=Ô=ˆŒ	ð OUÔN_ÐbcÒNcÐNc�"œ) FÔ$6¸Ô8IÑJÔJÐJÕikÔitÑivÔivˆŒð 	�ŠÑÔÐÐÐr(   Fr=   rV  rr   r³   c                 óÔ   —  | j         |fd|i|¤Ž}|j        }|                      |¦  «        }d}|�|                      ||| j        ¦  «        }t          |||j        |j        ¬¦  «        S )aŠ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        rr   NrX  )r  rC  rc  Úloss_functionr,   r   rÎ   r  )	r;   r=   rV  rr   r³   r\  rH  rZ  rY  s	            r)   rF   z"BeitForImageClassification.forward°  s™   € ð �$”)Øð
ð 
à%=ð
ð ð
ð 
ˆð  Ô-ˆà—’ Ñ/Ô/ˆàˆØÐØ×%Ò% f¨f°d´kÑBÔBˆDå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r(   ©NNF)r#   r$   r%   r   r2   r   r   rG   r   r«   r   r   r†   r   rF   rH   rI   s   @r)   ra  ra  �  sÐ   ø€ € € € € ð
˜zð 
¨dð 
ð 
ð 
ð 
ð 
ð 
ð Øð -1Ø&*Ø).ð	 
ð  
à”l TÑ)ð 
ð ”˜tÑ#ð 
ð #'ð	 
ð
 Ð+Ô,ð 
ð 
Ð&Ñ	&ð 
ð  
ð  
ñ „^ñ Ôð 
ð  
ð  
ð  
ð  
r(   ra  c                   óÀ   ‡ — e Zd Z	 	 	 	 	 	 	 ddededeeeef         z  d	ed
eeeef         z  ez  dedeeeef         z  dedefˆ fd„Zdej	        dej	        fd„Z
ˆ xZS )ÚBeitConvLayerr   r   r   FÚreluÚin_channelsÚout_channelsr/   r0   ÚpaddingrÃ   ÚdilationÚgroupsÚ
activationc
           
      ó
  •— t          ¦   «                              ¦   «          t          j        ||||||||¬¦  «        | _        t          j        |¦  «        | _        |	�t          |	         nt          j        ¦   «         | _	        d S )N)rj  rk  r/   r0   rl  rm  rn  rÃ   )
r1   r2   r   r8   ÚconvolutionÚBatchNorm2dÚnormalizationr
   r  ro  )r;   rj  rk  r/   r0   rl  rÃ   rm  rn  ro  r<   s             €r)   r2   zBeitConvLayer.__init__Ö  s€   ø€ õ 	‰Œ×ÒÑÔÐÝœ9Ø#Ø%Ø#ØØØØØð	
ñ 	
ô 	
ˆÔõ  œ^¨LÑ9Ô9ˆÔØ0:Ð0F�& Ô,Ð,ÍBÌKÉMÌMˆŒˆˆr(   rÎ   r>   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r   )rq  rs  ro  rá   s     r)   rF   zBeitConvLayer.forwardð  s?   € Ø×(Ò(¨Ñ7Ô7ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš¨Ñ6Ô6ˆØÐr(   )r   r   r   Fr   r   ri  )r#   r$   r%   r€   r†   rõ   r«   r2   rG   r   rF   rH   rI   s   @r)   rh  rh  Õ  s  ø€ € € € € ð
 ./ØØ/0ØØ*+ØØ ðZð ZàðZð ðZð ˜5  c œ?Ñ*ð	Zð
 ðZð �u˜S #˜X”Ñ&¨Ñ,ðZð ðZð ˜˜c 3˜hœÑ'ðZð ðZð ðZð Zð Zð Zð Zð Zð4 U¤\ð °e´lð ð ð ð ð ð ð ð r(   rh  c                   óh   ‡ — e Zd Zdedededdfˆ fd„Zdej        deeef         dej        fd	„Zˆ xZ	S )
ÚBeitPyramidPoolingBlockÚ
pool_scalerj  Úchannelsr>   Nc                 ó¨   •— t          ¦   «                              ¦   «          t          j        |¦  «        | _        t          ||d¬¦  «        | _        d S )Nr   ©r/   )r1   r2   r   ÚAdaptiveAvgPool2dÚpoolingrh  Úconv)r;   rw  rj  rx  r<   s       €r)   r2   z BeitPyramidPoolingBlock.__init__ø  sE   ø€ Ý‰Œ×ÒÑÔÐÝÔ+¨JÑ7Ô7ˆŒÝ! +¨xÀQÐGÑGÔGˆŒ	ˆ	ˆ	r(   Úinputr_   c                 ó    — |                       |¦  «        }|                      |¦  «        }t          j                             ||dd¬¦  «        }|S )Nrž   Fr^   )r|  r}  r   rh   ri   )r;   r~  r_   Úhidden_states       r)   rF   zBeitPyramidPoolingBlock.forwardý  sI   € Ø—|’| EÑ*Ô*ˆØ—y’y Ñ.Ô.ˆÝ”}×0Ò0°ÀDÈzÐinÐ0ÑoÔoˆØÐr(   )
r#   r$   r%   r€   r2   rG   r   r†   rF   rH   rI   s   @r)   rv  rv  ÷  s˜   ø€ € € € € ðH 3ð H°Sð HÀCð HÈDð Hð Hð Hð Hð Hð Hð
˜Uœ\ð °°s¸C°x´ð ÀUÄ\ð ð ð ð ð ð ð ð r(   rv  c                   ót   ‡ — e Zd ZdZdeedf         dededdfˆ fd„Zd	ej        de	ej                 fd
„Z
ˆ xZS )ÚBeitPyramidPoolingModuleak  
    Pyramid Pooling Module (PPM) used in PSPNet.

    Args:
        pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid
            Module.
        in_channels (int): Input channels.
        channels (int): Channels after modules, before conv_seg.

    Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
    Úpool_scales.rj  rx  r>   Nc                 óÂ   •‡‡— t          ¦   «                              ¦   «          || _        ‰| _        ‰| _        t          j        ˆˆfd„|D ¦   «         ¦  «        | _        d S )Nc                 ó4   •— g | ]}t          |‰‰¬ ¦  «        ‘ŒS ))rw  rj  rx  )rv  )r/  rw  rx  rj  s     €€r)   r1  z5BeitPyramidPoolingModule.__init__.<locals>.<listcomp>  s9   ø€ ð ð ð àõ (°:È;ÐaiÐjÑjÔjðð ð r(   )r1   r2   rƒ  rj  rx  r   r7  Úblocks)r;   rƒ  rj  rx  r<   s     ``€r)   r2   z!BeitPyramidPoolingModule.__init__  st   øøø€ Ý‰Œ×ÒÑÔÐØ&ˆÔØ&ˆÔØ ˆŒÝ”mðð ð ð ð à"-ðñ ô ñ
ô 
ˆŒˆˆr(   rÎ   c                 óf   ‡‡— ‰                      ¦   «         dd …         Šˆˆfd„| j        D ¦   «         S )NrA   c                 ó*   •— g | ]} |‰‰¬ ¦  «        ‘ŒS ))r_   r'   )r/  ÚblockrÎ   Úoriginal_sizes     €€r)   r1  z4BeitPyramidPoolingModule.forward.<locals>.<listcomp>  s(   ø€ ÐRÐRÐR¸U���m¨-Ð8Ñ8Ô8ÐRÐRÐRr(   )r_   r†  )r;   rÎ   rŠ  s    `@r)   rF   z BeitPyramidPoolingModule.forward  s=   øø€ Ø%×*Ò*Ñ,Ô,¨Q¨R¨RÔ0ˆØRÐRÐRÐRÐRÀdÄkÐRÑRÔRÐRr(   )r#   r$   r%   r&   r†   r€   r2   rG   r   r‡   rF   rH   rI   s   @r)   r‚  r‚    s¢   ø€ € € € € ð
ð 
ð

 E¨#¨s¨(¤Oð 

À#ð 

ÐQTð 

ÐY]ð 

ð 

ð 

ð 

ð 

ð 

ðS U¤\ð S°d¸5¼<Ô6Hð Sð Sð Sð Sð Sð Sð Sð Sr(   r‚  c                   óŒ   ‡ — e Zd ZdZdeddfˆ fd„Zdeej                 dej        fd„Z	deej                 dej        fd	„Z
ˆ xZS )
ÚBeitUperHeadzú
    Unified Perceptual Parsing for Scene Understanding. This head is the implementation of
    [UPerNet](https://huggingface.co/papers/1807.10221).

    Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
    r,   r>   Nc           	      ó~  •— t          ¦   «                              ¦   «          |j        | _        |j        gdz  | _        |j        | _        t          j        | j        |j        d¬¦  «        | _	        t          | j        | j        d         | j        ¦  «        | _        t          | j        d         t          | j        ¦  «        | j        z  z   | j        dd¬¦  «        | _        t          j        ¦   «         | _        t          j        ¦   «         | _        | j        d d…         D ]f}| j                             t          || j        d¬¦  «        ¦  «         | j                             t          | j        | j        dd¬¦  «        ¦  «         Œgt          t          | j        ¦  «        | j        z  | j        dd¬¦  «        | _        d S )Né   r   rz  r\   r   ©r/   rl  )r1   r2   rƒ  r9   rj  rx  r   r8   rQ  rc  r‚  Úpsp_modulesrh  ÚlenÚpsp_bottleneckr7  Úlateral_convsÚ	fpn_convsÚappendÚfpn_bottleneck)r;   r,   rj  r<   s      €r)   r2   zBeitUperHead.__init__*  sŸ  ø€ Ý‰Œ×ÒÑÔÐà!Ô-ˆÔØ"Ô.Ð/°!Ñ3ˆÔØÔ*ˆŒÝœ) D¤M°6Ô3DÐRSÐTÑTÔTˆŒõ 4ØÔØÔ˜RÔ ØŒMñ
ô 
ˆÔõ
 ,ØÔ˜RÔ ¥3 tÔ'7Ñ#8Ô#8¸4¼=Ñ#HÑHØŒMØØð	
ñ 
ô 
ˆÔõ  œ]™_œ_ˆÔÝœ™œˆŒØÔ+¨C¨R¨CÔ0ð 	ið 	iˆKØÔ×%Ò%¥m°KÀÄÐ\]Ð&^Ñ&^Ô&^Ñ_Ô_Ð_ØŒN×!Ò!¥-°´¸t¼}ÐZ[ÐefÐ"gÑ"gÔ"gÑhÔhÐhÐhå+Ý�Ô Ñ!Ô! D¤MÑ1ØŒMØØð	
ñ 
ô 
ˆÔÐÐr(   rÎ   c                 ó”   — |d         }t          j        |g|                      |¦  «        ¢d¬¦  «        }|                      |¦  «        S ru   )rG   rk   r�  r’  )r;   rÎ   r€  s      r)   Úpsp_forwardzBeitUperHead.psp_forwardL  sK   € Ø$ RÔ(ˆÝ”y ,Ð!P°×1AÒ1AÀ,Ñ1OÔ1OÐ!PÐVWÐXÑXÔXˆØ×"Ò" <Ñ0Ô0Ð0r(   Úencoder_hidden_statesc                 ó¬  — g }t          | j        |¦  «        D ]#\  }}|                      ||¦  «        ¦  «         Œ$|                     |                      |¦  «        ¦  «         t	          |¦  «        }t          |dz
  dd¦  «        D ]U}||dz
           j        dd …         }||dz
           t          j         	                    ||         |dd¬¦  «        z   ||dz
  <   ŒVg }t          |dz
  ¦  «        D ]1}|                      | j
        |         ||         ¦  «        ¦  «         Œ2|                     |d         ¦  «         t          |dz
  dd¦  «        D ]A}t          j         	                    ||         |d         j        dd …         dd¬¦  «        ||<   ŒBt          j        |d¬¦  «        }|                      |¦  «        }	|                      |	¦  «        }	|	S )	Nr   r   r\   rA   rž   Fr^   rb   )Úzipr“  r•  r˜  r‘  r6  rB   r   rh   ri   r”  rG   rk   r–  rc  )
r;   r™  ÚlateralsÚlateral_convr€  Úused_backbone_levelsr0  Ú
prev_shapeÚfpn_outsÚoutputs
             r)   rF   zBeitUperHead.forwardQ  sù  € àˆÝ*-¨dÔ.@ÐBWÑ*XÔ*Xð 	8ð 	8Ñ&ˆL˜,Ø�OŠO˜L˜L¨Ñ6Ô6Ñ7Ô7Ð7Ð7à�Š˜×(Ò(Ð)>Ñ?Ô?Ñ@Ô@Ð@õ  # 8™}œ}ÐÝÐ+¨aÑ/°°BÑ7Ô7ð 	ð 	ˆAØ! ! a¡%œÔ.¨q¨r¨rÔ2ˆJØ& q¨1¡uœoµ´×0IÒ0IØ˜” *°:ÈUð 1Jñ 1ô 1ñ ˆH�Q˜‘U‰OˆOð
 ˆÝÐ+¨aÑ/Ñ0Ô0ð 	<ð 	<ˆAØ�OŠOÐ-˜DœN¨1Ô-¨h°q¬kÑ:Ô:Ñ;Ô;Ð;Ð;à�Š˜ œÑ%Ô%Ð%åÐ+¨aÑ/°°BÑ7Ô7ð 	ð 	ˆAÝœ-×3Ò3Ø˜” (¨1¤+Ô"3°A°B°BÔ"7¸jÐX]ð 4ñ ô ˆH�Q‰KˆKõ ”9˜X¨1Ð-Ñ-Ô-ˆØ×$Ò$ XÑ.Ô.ˆØ—’ Ñ(Ô(ˆàˆr(   )r#   r$   r%   r&   r   r2   r‡   rG   r   r˜  rF   rH   rI   s   @r)   rŒ  rŒ  "  s¬   ø€ € € € € ðð ð 
˜zð  
¨dð  
ð  
ð  
ð  
ð  
ð  
ðD1¨¨e¬lÔ);ð 1ÀÄð 1ð 1ð 1ð 1ð
¨T°%´,Ô-?ð ÀEÄLð ð ð ð ð ð ð ð r(   rŒ  c                   ó‚   ‡ — e Zd ZdZ	 ddedededeeeef         z  d	d
f
ˆ fd„Zdee	j
                 d	e	j
        fd„Zˆ xZS )ÚBeitFCNHeadaÛ  
    Fully Convolution Networks for Semantic Segmentation. This head is implemented of
    [FCNNet](https://huggingface.co/papers/1411.4038>).

    Args:
        config (BeitConfig): Configuration.
        in_channels
        kernel_size (int): The kernel size for convs in the head. Default: 3.
        dilation (int): The dilation rate for convs in the head. Default: 1.


    Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
    rA   r   r   r,   Úin_indexr/   rm  r>   Nc           
      óÌ  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        |j        | _	        || _
        |dz  |z  }t          j        ¦   «         | _        | j        dk    r†| j                             t          | j        | j        |||¬¦  «        ¦  «         t!          | j        dz
  ¦  «        D ]8}| j                             t          | j        | j        |||¬¦  «        ¦  «         Œ9| j	        r-t          | j        | j        z   | j        ||dz  ¬¦  «        | _        t          j        | j        |j        d¬¦  «        | _        d S )NrA   r   )r/   rl  rm  r   r�  rz  )r1   r2   r9   rj  Úauxiliary_channelsrx  Úauxiliary_num_convsÚ	num_convsÚauxiliary_concat_inputÚconcat_inputr¤  r   r7  Úconvsr•  rh  r6  Úconv_catr8   rQ  rc  )r;   r,   r¤  r/   rm  Úconv_paddingry   r<   s          €r)   r2   zBeitFCNHead.__init__‚  sz  ø€ õ 	‰Œ×ÒÑÔÐØ!Ô-ˆÔØÔ1ˆŒØÔ3ˆŒØ"Ô9ˆÔØ ˆŒà# qÑ(¨HÑ4ˆÝ”]‘_”_ˆŒ
ØŒ>˜AÒÐØŒJ×ÒÝØÔ$ d¤mÀÐVbÐmuðñ ô ñô ð õ
 ˜4œ>¨AÑ-Ñ.Ô.ð 	ð 	�Ø”
×!Ò!Ý!ØœØœØ$/Ø ,Ø!)ðñ ô ñô ð ð ð Ôð 	Ý)ØÔ  4¤=Ñ0°$´-È[ÐbmÐqrÑbrðñ ô ˆDŒMõ œ) D¤M°6Ô3DÐRSÐTÑTÔTˆŒˆˆr(   r™  c                 óÜ   — || j                  }|}| j        D ]} ||¦  «        }Œ| j        r+|                      t	          j        ||gd¬¦  «        ¦  «        }|                      |¦  «        }|S )Nr   rb   )r¤  r«  rª  r¬  rG   rk   rc  )r;   r™  r  rÎ   r}  s        r)   rF   zBeitFCNHead.forward¥  s~   € Ø(¨¬Ô7ˆØ ˆØ”Jð 	0ð 	0ˆDØ ˜D Ñ/Ô/ˆMˆMØÔð 	WØ ŸMšM­%¬)°X¸}Ð4MÐSTÐ*UÑ*UÔ*UÑVÔVˆMØŸš¨Ñ6Ô6ˆØÐr(   )rA   r   r   )r#   r$   r%   r&   r   r€   r†   r2   r‡   rG   r   rF   rH   rI   s   @r)   r£  r£  s  sÅ   ø€ € € € € ðð ð noð!Uð !UØ ð!UØ,/ð!UØBEð!UØUXÐ[`ÐadÐfiÐaiÔ[jÑUjð!Uà	ð!Uð !Uð !Uð !Uð !Uð !UðF¨T°%´,Ô-?ð ÀEÄLð ð ð ð ð ð ð ð r(   r£  c            	       óZ   ‡ — e Zd ZdZddedededdfˆ fd„Zd	ej        dej        fd
„Zˆ xZ	S )ÚBeitFPNUpBlockuE   4x upsampling block: ConvTranspose â†’ BN â†’ GELU â†’ ConvTranspose.rA   r9   r/   r0   r>   Nc                 ó  •— t          ¦   «                              ¦   «          t          j        ||||¬¦  «        | _        t          j        |¦  «        | _        t          j        ¦   «         | _        t          j        ||||¬¦  «        | _	        d S )Nr.   )
r1   r2   r   ÚConvTranspose2dÚconv_transpose1rr  rs  ÚGELUro  Úconv_transpose2)r;   r9   r/   r0   r<   s       €r)   r2   zBeitFPNUpBlock.__init__³  sz   ø€ Ý‰Œ×ÒÑÔÐÝ!Ô1°+¸{ÐXcÐlrÐsÑsÔsˆÔÝœ^¨KÑ8Ô8ˆÔÝœ'™)œ)ˆŒÝ!Ô1°+¸{ÐXcÐlrÐsÑsÔsˆÔÐÐr(   rÎ   c                 ó®   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r   )r³  rs  ro  rµ  rá   s     r)   rF   zBeitFPNUpBlock.forwardº  sR   € Ø×,Ò,¨]Ñ;Ô;ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš¨Ñ6Ô6ˆØ×,Ò,¨]Ñ;Ô;ˆØÐr(   )rA   rA   )
r#   r$   r%   r&   r€   r2   rG   r   rF   rH   rI   s   @r)   r°  r°  °  s–   ø€ € € € € ØOÐOðtð t Cð t°cð tÀsð tÐSWð tð tð tð tð tð tð U¤\ð °e´lð ð ð ð ð ð ð ð r(   r°  c                   ól   ‡ — e Zd ZdZdefˆ fd„Zdeej        df         deej        df         fd„Z	ˆ xZ
S )ÚBeitFPNNeckz¤
    4-level feature pyramid neck for BeiT. Produces x4 upsample, x2 upsample,
    identity, and x2 downsample outputs from the four selected ViT feature maps.
    r,   c                 óþ   •— t          ¦   «                              ¦   «          t          |j        ¦  «        | _        t          j        |j        |j        dd¬¦  «        | _        t          j        dd¬¦  «        | _	        d S )NrA   r.   )
r1   r2   r°  r9   Úfpn1r   r²  Úfpn2Ú	MaxPool2dÚfpn4rÍ   s     €r)   r2   zBeitFPNNeck.__init__È  sh   ø€ Ý‰Œ×ÒÑÔÐÝ" 6Ô#5Ñ6Ô6ˆŒ	ÝÔ& vÔ'9¸6Ô;MÐ[\ÐefÐgÑgÔgˆŒ	Ý”L¨Q°qÐ9Ñ9Ô9ˆŒ	ˆ	ˆ	r(   Úfeature_maps.r>   c                 ó°   — |                       |d         ¦  «        |                      |d         ¦  «        |d         |                      |d         ¦  «        fS r…   )rº  r»  r½  )r;   r¾  s     r)   rF   zBeitFPNNeck.forwardÎ  sL   € à�IŠI�l 1”oÑ&Ô&Ø�IŠI�l 1”oÑ&Ô&Ø˜ŒOØ�IŠI�l 1”oÑ&Ô&ð	
ð 	
r(   )r#   r$   r%   r&   r   r2   r†   rG   r   rF   rH   rI   s   @r)   r¸  r¸  Â  s‰   ø€ € € € € ðð ð
:˜zð :ð :ð :ð :ð :ð :ð
 E¨%¬,¸Ð*;Ô$<ð 
ÀÀuÄ|ÐUXÐGXÔAYð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r(   r¸  c                   ó®   ‡ — e Zd Zdeddfˆ fd„Zeee	 	 	 ddej	        dz  dej	        dz  de
d	ee         deez  f
d
„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚBeitForSemanticSegmentationr,   r>   Nc                 ó˜  •— t          ¦   «                              |¦  «         |j        | _        t          |d¬¦  «        | _        t          | j        j        ¦  «        dk    rt          d¦  «        ‚t          |¦  «        | _
        t          |¦  «        | _        |j        rt          |¦  «        nd | _        |                      ¦   «          d S )NFrP  rŽ  zÇBeitForSemanticSegmentation requires config.out_indices to be a list of 4 integers, specifying which features to use from the backbone. One can use [3, 5, 7, 11] in case of a base-sized architecture.)r1   r2   rQ  r)  r  r‘  r,   Úout_indicesrC   r¸  ÚfpnrŒ  Údecode_headÚuse_auxiliary_headr£  Úauxiliary_headr=  rÍ   s     €r)   r2   z$BeitForSemanticSegmentation.__init__Ù  sÁ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð à Ô+ˆŒÝ˜f¸Ð>Ñ>Ô>ˆŒ	åˆtŒ{Ô&Ñ'Ô'¨1Ò,Ð,Ýð-ñô ð õ
 ˜vÑ&Ô&ˆŒõ (¨Ñ/Ô/ˆÔØ5;Ô5NÐX�k¨&Ñ1Ô1Ð1ÐTXˆÔð 	�ŠÑÔÐÐÐr(   Fr=   rV  rr   r³   c                 óL  ‡‡‡‡— |�| j         j        dk    rt          d¦  «        ‚ | j        |fd|i|¤Ž}|j        Š|j        \  Š}}}|| j         j        z  Š|| j         j        z  Št          ˆˆˆˆfd„| j         j        D ¦   «         ¦  «        }	|  	                    |	¦  «        }	|  
                    |	¦  «        }
d}| j        �|                      |	¦  «        }d}|�.|                      |
|| j         j        || j         j        ¬¦  «        }t          ||
|j        |j        ¬¦  «        S )aD  
        labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*):
            Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy).

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, BeitForSemanticSegmentation
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-finetuned-ade-640-640")
        >>> model = BeitForSemanticSegmentation.from_pretrained("microsoft/beit-base-finetuned-ade-640-640")

        >>> inputs = image_processor(images=image, return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> # logits are of shape (batch_size, num_labels, height, width)
        >>> logits = outputs.logits
        ```Nr   z/The number of labels should be greater than onerr   c              3   óš   •K  — | ]E}‰|d z
           dd…d d…f                               d d¦  «                             ‰d‰‰¦  «        V — ŒFdS )r   NrA   r\   )rE   rf   )r/  r0  rz   r™  Úpatch_heightÚpatch_widths     €€€€r)   ú	<genexpr>z6BeitForSemanticSegmentation.forward.<locals>.<genexpr>  sx   øè è € ð 
ð 
àð " ! a¡%Ô(¨¨¨¨A¨B¨B¨Ô/×9Ò9¸!¸QÑ?Ô?×GÒGÈ
ÐTVÐXdÐfqÑrÔrð
ð 
ð 
ð 
ð 
ð 
r(   )Úignore_indexÚauxiliary_logitsÚauxiliary_loss_weightrX  )r,   rQ  rC   r  rÎ   rB   r4   r†   rÃ  rÄ  rÅ  rÇ  re  Úsemantic_loss_ignore_indexrÏ  r   r  )r;   r=   rV  rr   r³   r\  ry   rY   rZ   r¾  rZ  rÎ  rY  rz   r™  rÊ  rË  s                @@@@r)   rF   z#BeitForSemanticSegmentation.forwardî  s‘  øøøø€ ðB Ð $¤+Ô"8¸AÒ"=Ð"=ÝÐNÑOÔOÐOØ�$”)Øð
ð 
à%=ð
ð ð
ð 
ˆð !(Ô 5ÐØ'3Ô'9Ñ$ˆ
�A�v˜uØ ¤Ô!7Ñ7ˆØ˜tœ{Ô5Ñ5ˆõ ð 
ð 
ð 
ð 
ð 
ð 
ð 
à”[Ô,ð
ñ 
ô 
ñ 
ô 
ˆð —x’x Ñ-Ô-ˆà×!Ò! ,Ñ/Ô/ˆàÐØÔÐ*Ø#×2Ò2°<Ñ@Ô@ÐàˆØÐØ×%Ò%ØØØ!œ[ÔCØ!1Ø&*¤kÔ&Gð &ñ ô ˆDõ 'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r(   rf  )r#   r$   r%   r   r2   r   r   r   rG   r   r«   r   r   r†   r   rF   rH   rI   s   @r)   rÁ  rÁ  ×  sé   ø€ € € € € ð˜zð ¨dð ð ð ð ð ð ð* Ø Øð -1Ø&*Ø).ð	G
ð G
à”l TÑ)ðG
ð ”˜tÑ#ðG
ð #'ð	G
ð
 Ð+Ô,ðG
ð 
Ð(Ñ	(ðG
ð G
ð G
ñ „^ñ !Ô ñ ÔðG
ð G
ð G
ð G
ð G
r(   rÁ  zM
    BEiT backbone, to be used with frameworks like DETR and MaskFormer.
    c            	       ón   ‡ — e Zd Zˆ fd„Zeeededee	         de
fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚBeitBackbonec                 óL  •‡— t          ¦   «                              ‰¦  «         ˆfd„t          ‰j        dz   ¦  «        D ¦   «         | _        t          ‰d¬¦  «        | _        ‰j        rt          ‰¦  «        nt          j
        ¦   «         | _        |                      ¦   «          d S )Nc                 ó   •— g | ]	}‰j         ‘Œ
S r'   )r9   )r/  ry   r,   s     €r)   r1  z)BeitBackbone.__init__.<locals>.<listcomp>D  s   ø€ Ð]Ð]Ð]°A˜VÔ/Ð]Ð]Ð]r(   r   FrP  )r1   r2   r6  r.  Únum_featuresr)  r  Úadd_fpnr¸  r   r  rÄ  r=  rÍ   s    `€r)   r2   zBeitBackbone.__init__A  s–   øø€ Ý‰Œ×Ò˜Ñ Ô Ð à]Ð]Ð]Ð]½¸vÔ?WÐZ[Ñ?[Ñ9\Ô9\Ð]Ñ]Ô]ˆÔÝ˜f¸Ð>Ñ>Ô>ˆŒ	Ø*0¬.ÐK•;˜vÑ&Ô&Ð&½b¼k¹m¼mˆŒð 	�ŠÑÔÐÐÐr(   r=   r³   r>   c                 óÐ  — |j         \  }}}}|| j        j        z  }|| j        j        z  } | j        |fi |¤Ž}	|	j        }
d}t          | j        |
¦  «        D ]_\  }}|| j        v rQ| j        j        r?|dd…dd…dd…f         }| 	                    dd¦  «        }| 
                    |d||¦  «        }||fz  }Œ`|                      |¦  «        }t          ||	j        |	j        ¬¦  «        S )a:  
        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, AutoBackbone
        >>> import torch
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-patch16-224")
        >>> model = AutoBackbone.from_pretrained(
        ...     "microsoft/beit-base-patch16-224", out_features=["stage1", "stage2", "stage3", "stage4"]
        ... )

        >>> inputs = processor(image, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> feature_maps = outputs.feature_maps
        >>> list(feature_maps[-1].shape)
        [1, 768, 14, 14]
        ```r'   Nr   rA   r\   )r¾  rÎ   r  )rB   r,   r4   r  rÎ   r›  Ústage_namesÚout_featuresÚreshape_hidden_statesrE   rf   rÄ  r   r  )r;   r=   r³   rz   ry   rY   rZ   rÊ  rË  r\  rÎ   r¾  Ústager€  s                 r)   rF   zBeitBackbone.forwardK  s&  € ð@ (4Ô'9Ñ$ˆ
�A�v˜uØ ¤Ô!7Ñ7ˆØ˜tœ{Ô5Ñ5ˆØ�$”)˜LÐ3Ð3¨FÐ3Ð3ˆàÔ-ˆØˆÝ#& tÔ'7¸Ñ#GÔ#Gð 	0ð 	0ÑˆE�<Ø˜Ô)Ð)Ð)Ø”;Ô4ð cØ#/°°°°1°2°2°q°q°q°Ô#9�LØ#/×#9Ò#9¸!¸QÑ#?Ô#?�LØ#/×#7Ò#7¸
ÀBÈÐVaÑ#bÔ#b�Là  Ñ/�øà—x’x Ñ-Ô-ˆåØ%Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r(   )r#   r$   r%   r2   r   r   r   r   r   r   r   rF   rH   rI   s   @r)   rÒ  rÒ  ;  s�   ø€ € € € € ðð ð ð ð ð Ø Øð3
àð3
ð Ð+Ô,ð3
ð 
ð	3
ð 3
ð 3
ñ „^ñ !Ô ñ Ôð3
ð 3
ð 3
ð 3
ð 3
r(   rÒ  )ra  rN  rÁ  r)  r  rÒ  )Nr¬   )FÚcollections.abcr   r   Údataclassesr   rG   r   r   Ú r	   r  Úactivationsr
   Úbackbone_utilsr   r   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r   r   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_beitr   r"   ÚModuler+   rK   rƒ   rô   r¾   rÀ   rÙ   rä   r÷   r  r)  r;  rN  ra  rh  rv  r‚  rŒ  r£  r°  r¸  rÁ  rÒ  Ú__all__r'   r(   r)   ú<module>rí     s  ðð* /Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø HÐ HÐ HÐ HÐ HÐ HÐ HÐ HØ 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð GÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø @Ð @Ð @Ð @Ð @Ð @Ø BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BØ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø *Ð *Ð *Ð *Ð *Ð *ð €ððñ ô ð
 ðð ð ð ð Ð!;ñ ô ñ „ñô ððHð Hð Hð Hð H˜"œ)ñ Hô Hð Hð<Sð Sð Sð Sð S�R”Yñ Sô Sð SðlV3ð V3ð V3ð V3ð V3˜rœyñ V3ô V3ð V3ð~ !Øð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð8-)ð -)ð -)ð -)ð -)�B”Iñ -)ô -)ð -)ð`ð ð ð ð ˆbŒiñ ô ð ð %ð %ð %ð %ð %�2”9ñ %ô %ð %ð0<ð <ð <ð <ð <Ð*ñ <ô <ð <ð~ ð"Tð "Tð "Tð "Tð "T˜/ñ "Tô "Tñ „ð"TðJ ðJjð Jjð Jjð Jjð JjÐ#ñ Jjô Jjñ „ðJjðZ	vð 	vð 	vð 	vð 	v�”ñ 	vô 	vð 	vð €ððñ ô ðS
ð S
ð S
ð S
ð S
Ð!4ñ S
ô S
ñô ðS
ðl €ððñ ô ð/
ð /
ð /
ð /
ð /
Ð!4ñ /
ô /
ñô ð/
ðdð ð ð ð �B”Iñ ô ð ðD
ð 
ð 
ð 
ð 
˜bœiñ 
ô 
ð 
ðSð Sð Sð Sð S˜rœyñ Sô Sð Sð<Nð Nð Nð Nð N�2”9ñ Nô Nð Nðb:ð :ð :ð :ð :�"”)ñ :ô :ð :ðzð ð ð ð �R”Yñ ô ð ð$
ð 
ð 
ð 
ð 
�"”)ñ 
ô 
ð 
ð* ð`
ð `
ð `
ð `
ð `
Ð"5ñ `
ô `
ñ „ð`
ðF €ððñ ô ð
A
ð A
ð A
ð A
ð A
�=Ð"5ñ A
ô A
ñô ð
A
ðHð ð €€€r(   