§
    ‚ŠtjäŽ  ã                   óZ  — d Z ddlZddlZddlmZ ddlZddlmZ ddlm	Z
 ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZ ddlmZ ddlmZ  ej        e¦  «        Z  ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z! ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z" ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z# ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z$ G d„ dej%        ¦  «        Z& G d„ d ej%        ¦  «        Z' G d!„ d"ej%        ¦  «        Z( G d#„ d$ej%        ¦  «        Z) G d%„ d&ej%        ¦  «        Z* G d'„ d(ej%        ¦  «        Z+ G d)„ d*e¦  «        Z, G d+„ d,ej%        ¦  «        Z-e G d-„ d.e¦  «        ¦   «         Z.e G d/„ d0e.¦  «        ¦   «         Z/ ed1¬¦  «         G d2„ d3e.¦  «        ¦   «         Z0 ed4¬¦  «         G d5„ d6e.¦  «        ¦   «         Z1 ed7¬¦  «         G d8„ d9ee.¦  «        ¦   «         Z2g d:¢Z3dS );zPyTorch FocalNet model.é    N)Ú	dataclass)Únné   )Úinitialization)ÚACT2FN)ÚBackboneMixinÚfilter_output_hidden_states)ÚGradientCheckpointingLayer)ÚBackboneOutput)ÚPreTrainedModel)ÚModelOutputÚauto_docstringÚlogging)Úcan_return_tupleé   )ÚFocalNetConfigzC
    FocalNet encoder's outputs, with potential hidden states.
    )Úcustom_introc                   ó†   — e Zd ZU dZdZej        dz  ed<   dZe	ej                 dz  ed<   dZ
e	ej                 dz  ed<   dS )ÚFocalNetEncoderOutputaí  
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    NÚlast_hidden_stateÚhidden_statesÚreshaped_hidden_states)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚFloatTensorÚ__annotations__r   Útupler   © ó    úl/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/focalnet/modeling_focalnet.pyr   r   %   sr   € € € € € € ðð ð 37Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø>BÐ˜E %Ô"3Ô4°tÑ;ÐBÐBÑBÐBÐBr"   r   zZ
    FocalNet model's outputs that also contains a pooling of the last hidden states.
    c                   ó¤   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej                 dz  ed<   dZe
ej                 dz  ed<   dS )ÚFocalNetModelOutputa±  
    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`, *optional*, returned when `add_pooling_layer=True` is passed):
        Average pooling of the last layer hidden-state.
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nr   Úpooler_outputr   r   )r   r   r   r   r   r   r   r   r&   r   r    r   r!   r"   r#   r%   r%   :   sŠ   € € € € € € ð	ð 	ð 37Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø.2€M�5Ô$ tÑ+Ð2Ð2Ñ2Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø>BÐ˜E %Ô"3Ô4°tÑ;ÐBÐBÑBÐBÐBr"   r%   z.
    FocalNet masked image model outputs.
    c                   ó¤   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej                 dz  ed<   dZe
ej                 dz  ed<   dS )Ú!FocalNetMaskedImageModelingOutputa  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `bool_masked_pos` is provided):
        Masked image modeling (MLM) loss.
    reconstruction (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
        Reconstructed pixel values.
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    NÚlossÚreconstructionr   r   )r   r   r   r   r)   r   r   r   r*   r   r    r   r!   r"   r#   r(   r(   R   s‰   € € € € € € ðð ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø/3€N�EÔ%¨Ñ,Ð3Ð3Ñ3Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø>BÐ˜E %Ô"3Ô4°tÑ;ÐBÐBÑBÐBÐBr"   r(   z4
    FocalNet outputs for image classification.
    c                   ó¤   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej                 dz  ed<   dZe
ej                 dz  ed<   dS )ÚFocalNetImageClassifierOutputa7  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Classification (or regression if config.num_labels==1) loss.
    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Classification (or regression if config.num_labels==1) scores (before SoftMax).
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nr)   Úlogitsr   r   )r   r   r   r   r)   r   r   r   r-   r   r    r   r!   r"   r#   r,   r,   l   s‰   € € € € € € ðð ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø>BÐ˜E %Ô"3Ô4°tÑ;ÐBÐBÑBÐBÐBr"   r,   c                   ór   ‡ — e Zd ZdZd	ˆ fd„	Z	 d
dej        dz  dej        dz  deej	                 fd„Z
ˆ xZS )ÚFocalNetEmbeddingszX
    Construct the patch embeddings and layernorm. Optionally, also the mask token.
    Fc           	      óÄ  •— t          ¦   «                              ¦   «          t          ||j        |j        |j        |j        |j        d¬¦  «        | _        | j        j	        | _
        |r-t          j        t          j        dd|j        ¦  «        ¦  «        nd | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j        ¦  «        | _        d S )NT)ÚconfigÚ
image_sizeÚ
patch_sizeÚnum_channelsÚ	embed_dimÚuse_conv_embedÚis_stemr   ©Úeps)ÚsuperÚ__init__ÚFocalNetPatchEmbeddingsr2   r3   r4   r5   r6   Úpatch_embeddingsÚ	grid_sizeÚ
patch_gridr   Ú	Parameterr   ÚzerosÚ
mask_tokenÚ	LayerNormÚlayer_norm_epsÚnormÚDropoutÚhidden_dropout_probÚdropout)Úselfr1   Úuse_mask_tokenÚ	__class__s      €r#   r;   zFocalNetEmbeddings.__init__‹   sÄ   ø€ Ý‰Œ×ÒÑÔÐå 7ØØÔ(ØÔ(ØÔ,ØÔ&Ø!Ô0Øð!
ñ !
ô !
ˆÔð Ô/Ô9ˆŒØO]Ðg�"œ,¥u¤{°1°a¸Ô9IÑ'JÔ'JÑKÔKÐKÐcgˆŒå”L Ô!1°vÔ7LÐMÑMÔMˆŒ	Ý”z &Ô"<Ñ=Ô=ˆŒˆˆr"   NÚpixel_valuesÚbool_masked_posÚreturnc                 óf  — |                       |¦  «        \  }}|                      |¦  «        }|                     ¦   «         \  }}}|�R| j                             ||d¦  «        }|                     d¦  «                             |¦  «        }	|d|	z
  z  ||	z  z   }|                      |¦  «        }||fS )Néÿÿÿÿç      ð?)r=   rE   ÚsizerB   ÚexpandÚ	unsqueezeÚtype_asrH   )
rI   rL   rM   Ú
embeddingsÚoutput_dimensionsÚ
batch_sizeÚseq_lenÚ_Úmask_tokensÚmasks
             r#   ÚforwardzFocalNetEmbeddings.forward�   s¹   € ð )-×(=Ò(=¸lÑ(KÔ(KÑ%ˆ
Ð%Ø—Y’Y˜zÑ*Ô*ˆ
Ø!+§¢Ñ!2Ô!2Ñˆ
�G˜QàÐ&Øœ/×0Ò0°¸WÀbÑIÔIˆKà"×,Ò,¨RÑ0Ô0×8Ò8¸ÑEÔEˆDØ# s¨T¡zÑ2°[À4Ñ5GÑGˆJà—\’\ *Ñ-Ô-ˆ
ØÐ,Ð,Ð,r"   )F©N)r   r   r   r   r;   r   r   Ú
BoolTensorr    ÚTensorr]   Ú__classcell__©rK   s   @r#   r/   r/   †   s—   ø€ € € € € ðð ð>ð >ð >ð >ð >ð >ð& bfð-ð -Ø!Ô-°Ñ4ð-ØGLÔGWÐZ^ÑG^ð-à	ˆuŒ|Ô	ð-ð -ð -ð -ð -ð -ð -ð -r"   r/   c                   ór   ‡ — e Zd Z	 	 	 dˆ fd„	Zd„ Zdej        dz  deej        ee	         f         fd„Z
ˆ xZS )	r<   Fc	                 ó˜  •— t          ¦   «                              ¦   «          t          |t          j        j        ¦  «        r|n||f}t          |t          j        j        ¦  «        r|n||f}|d         |d         z  |d         |d         z  z  }	|| _        || _        || _        |	| _	        |d         |d         z  |d         |d         z  f| _
        |r.|rd}
d}d}nd}
d}d}t          j        |||
||¬¦  «        | _        nt          j        ||||¬¦  «        | _        |r"t          j        ||j        ¬	¦  «        | _        d S d | _        d S )
Nr   r   é   é   é   r   )Úkernel_sizeÚstrideÚpadding)rh   ri   r8   )r:   r;   Ú
isinstanceÚcollectionsÚabcÚIterabler2   r3   r4   Únum_patchesr>   r   ÚConv2dÚ
projectionrC   rD   rE   )rI   r1   r2   r3   r4   r5   Úadd_normr6   r7   ro   rh   rj   ri   rK   s                €r#   r;   z FocalNetPatchEmbeddings.__init__¯   sq  ø€ õ 	‰Œ×ÒÑÔÐÝ#-¨j½+¼/Ô:RÑ#SÔ#SÐq�Z�ZÐZdÐfpÐYqˆ
Ý#-¨j½+¼/Ô:RÑ#SÔ#SÐq�Z�ZÐZdÐfpÐYqˆ
Ø! !”}¨
°1¬Ñ5¸*ÀQ¼-È:ÐVWÌ=Ñ:XÑYˆØ$ˆŒØ$ˆŒØ(ˆÔØ&ˆÔØ$ Qœ-¨:°a¬=Ñ8¸*ÀQ¼-È:ÐVWÌ=Ñ:XÐYˆŒàð 	làð Ø�Ø�Ø��à�Ø�Ø�Ý œiØ˜i°[ÈÐY`ðñ ô ˆDŒOˆOõ !œi¨°iÈZÐ`jÐkÑkÔkˆDŒOàð 	Ýœ Y°FÔ4IÐJÑJÔJˆDŒIˆIˆIàˆDŒIˆIˆIr"   c                 óZ  — || j         d         z  dk    r@d| j         d         || j         d         z  z
  f}t          j                             ||¦  «        }|| j         d         z  dk    rBddd| j         d         || j         d         z  z
  f}t          j                             ||¦  «        }|S )Nr   r   )r3   r   Ú
functionalÚpad)rI   rL   ÚheightÚwidthÚ
pad_valuess        r#   Ú	maybe_padz!FocalNetPatchEmbeddings.maybe_padÙ   s­   € Ø�4”? 1Ô%Ñ%¨Ò*Ð*Ø˜Tœ_¨QÔ/°%¸$¼/È!Ô:LÑ2LÑLÐMˆJÝœ=×,Ò,¨\¸:ÑFÔFˆLØ�D”O AÔ&Ñ&¨!Ò+Ð+Ø˜Q  4¤?°1Ô#5¸ÀÄÐQRÔASÑ8SÑ#SÐTˆJÝœ=×,Ò,¨\¸:ÑFÔFˆLØÐr"   rL   NrN   c                 óX  — |j         \  }}}}|| j        k    rt          d¦  «        ‚|                      |||¦  «        }|                      |¦  «        }|j         \  }}}}||f}|                     d¦  «                             dd¦  «        }| j        �|                      |¦  «        }||fS )NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.rf   r   )Úshaper4   Ú
ValueErrorry   rq   ÚflattenÚ	transposerE   )rI   rL   rZ   r4   rv   rw   rV   rW   s           r#   r]   zFocalNetPatchEmbeddings.forwardâ   sÁ   € Ø)5Ô);Ñ&ˆˆ<˜ Ø˜4Ô,Ò,Ð,ÝØwñô ð ð —~’~ l°F¸EÑBÔBˆØ—_’_ \Ñ2Ô2ˆ
Ø(Ô.Ñˆˆ1ˆf�eØ# U˜OÐØ×'Ò'¨Ñ*Ô*×4Ò4°Q¸Ñ:Ô:ˆ
àŒ9Ð ØŸš :Ñ.Ô.ˆJàÐ,Ð,Ð,r"   )FFF)r   r   r   r;   ry   r   r   r    r`   Úintr]   ra   rb   s   @r#   r<   r<   ®   s•   ø€ € € € € ð ØØð(ð (ð (ð (ð (ð (ðTð ð ð- EÔ$5¸Ñ$<ð -ÀÀuÄ|ÐUZÐ[^ÔU_ÐG_ÔA`ð -ð -ð -ð -ð -ð -ð -ð -r"   r<   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚFocalNetModulationrf   Tç        c                 óÌ  •— t          ¦   «                              ¦   «          || _        |j        |         | _        |j        |         | _        || _        |j        | _        |j	        | _	        t          j        |d|z  | j        dz   z   |¬¦  «        | _        t          j        ||dd|¬¦  «        | _        t          j        ¦   «         | _        t          j        ||¦  «        | _        t          j        |¦  «        | _        t          j        ¦   «         | _        g | _        t/          | j        ¦  «        D ]ˆ}| j        |z  | j        z   }| j                             t          j        t          j        |||d||dz  d¬¦  «        t          j        ¦   «         ¦  «        ¦  «         | j                             |¦  «         Œ‰| j        r"t          j        ||j        ¬¦  «        | _        d S d S )Nrf   r   )Úbias)rh   ri   r„   F)rh   ri   Úgroupsrj   r„   r8   )r:   r;   ÚdimÚfocal_windowsÚfocal_windowÚfocal_levelsÚfocal_levelÚfocal_factorÚ use_post_layernorm_in_modulationÚnormalize_modulatorr   ÚLinearÚprojection_inrp   Úprojection_contextÚGELUÚ
activationÚprojection_outrF   Úprojection_dropoutÚ
ModuleListÚfocal_layersÚkernel_sizesÚrangeÚappendÚ
SequentialrC   rD   Ú	layernorm)
rI   r1   Úindexr†   r‹   r„   r”   Úkrh   rK   s
            €r#   r;   zFocalNetModulation.__init__ö   sÑ  ø€ Ý‰Œ×ÒÑÔÐàˆŒØ"Ô0°Ô7ˆÔØ!Ô.¨uÔ5ˆÔØ(ˆÔØ06Ô0WˆÔ-Ø#)Ô#=ˆÔ åœY s¨A°©G°tÔ7GÈ!Ñ7KÑ,LÐSWÐXÑXÔXˆÔÝ"$¤)¨C°À!ÈAÐTXÐ"YÑ"YÔ"YˆÔåœ'™)œ)ˆŒÝ œi¨¨SÑ1Ô1ˆÔÝ"$¤*Ð-?Ñ"@Ô"@ˆÔÝœM™OœOˆÔàˆÔÝ�tÔ'Ñ(Ô(ð 
	2ð 
	2ˆAØÔ+¨aÑ/°$Ô2CÑCˆKØÔ×$Ò$Ý”Ý”IØ˜S¨kÀ!ÈCÐYdÐhiÑYiÐpuðñ ô õ ”G‘I”Iñ	ô ñô ð ð Ô×$Ò$ [Ñ1Ô1Ð1Ð1ØÔ0ð 	JÝœ\¨#°6Ô3HÐIÑIÔIˆDŒNˆNˆNð	Jð 	Jr"   c                 ó\  — |j         d         }|                      |¦  «                             dddd¦  «                             ¦   «         }t	          j        |||| j        dz   fd¦  «        \  }}}d}t          | j        ¦  «        D ]/} | j        |         |¦  «        }|||dd…||dz   …f         z  z   }Œ0|  	                    | 
                    dd¬¦  «         
                    dd¬¦  «        ¦  «        }	||	|dd…| j        d…f         z  z   }| j        r|| j        dz   z  }|                      |¦  «        }
||
z  }|                     dddd¦  «                             ¦   «         }| j        r|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S )	zˆ
        Args:
            hidden_state:
                Input features with shape of (batch_size, height, width, num_channels)
        rP   r   r   r   rf   NT)Úkeepdim)r{   r�   ÚpermuteÚ
contiguousr   ÚsplitrŠ   r˜   r–   r’   Úmeanr�   r�   rŒ   r›   r“   r”   )rI   Úhidden_stater4   ÚxÚqÚctxÚgatesÚctx_allÚlevelÚ
ctx_globalÚ	modulatorÚx_outs               r#   r]   zFocalNetModulation.forward  sÕ  € ð $Ô)¨"Ô-ˆð ×Ò˜|Ñ,Ô,×4Ò4°Q¸¸1¸aÑ@Ô@×KÒKÑMÔMˆÝœ A¨°lÀDÔDTÐWXÑDXÐ'YÐ[\Ñ]Ô]‰ˆˆ3�ð ˆÝ˜4Ô+Ñ,Ô,ð 	Bð 	BˆEØ*�$Ô# EÔ*¨3Ñ/Ô/ˆCØ  e¨A¨A¨A¨u°u¸q±yÐ/@Ð,@Ô&AÑ AÑAˆGˆGØ—_’_ S§X¢X¨a¸ XÑ%>Ô%>×%CÒ%CÀAÈtÐ%CÑ%TÔ%TÑUÔUˆ
Ø˜J¨¨q¨q¨q°$Ô2BÐ2DÐ2DÐ/DÔ)EÑEÑEˆð Ô#ð 	7Ø Ô!1°AÑ!5Ñ6ˆGð ×+Ò+¨GÑ4Ô4ˆ	Ø�I‘ˆØ—’˜a  A qÑ)Ô)×4Ò4Ñ6Ô6ˆØÔ0ð 	*Ø—N’N 5Ñ)Ô)ˆEð ×#Ò# EÑ*Ô*ˆØ×'Ò'¨Ñ.Ô.ˆØˆr"   )rf   Tr‚   ©r   r   r   r;   r]   ra   rb   s   @r#   r�   r�   õ   sS   ø€ € € € € ðJð Jð Jð Jð Jð JðB"ð "ð "ð "ð "ð "ð "r"   r�   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚFocalNetMlpNr‚   c                 ó   •— t          ¦   «                              ¦   «          |p|}|p|}t          j        ||¦  «        | _        t
          |j                 | _        t          j        ||¦  «        | _        t          j	        |¦  «        | _
        d S r^   )r:   r;   r   rŽ   Úfc1r   Ú
hidden_actr’   Úfc2rF   Údrop)rI   r1   Úin_featuresÚhidden_featuresÚout_featuresrµ   rK   s         €r#   r;   zFocalNetMlp.__init__=  sw   ø€ Ý‰Œ×ÒÑÔÐØ#Ð2 {ˆØ)Ð8¨[ˆÝ”9˜[¨/Ñ:Ô:ˆŒÝ  Ô!2Ô3ˆŒÝ”9˜_¨lÑ;Ô;ˆŒÝ”J˜tÑ$Ô$ˆŒ	ˆ	ˆ	r"   c                 óØ   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r^   )r²   r’   rµ   r´   )rI   r¤   s     r#   r]   zFocalNetMlp.forwardF  s]   € Ø—x’x Ñ-Ô-ˆØ—’ |Ñ4Ô4ˆØ—y’y Ñ.Ô.ˆØ—x’x Ñ-Ô-ˆØ—y’y Ñ.Ô.ˆØÐr"   )NNr‚   r®   rb   s   @r#   r°   r°   <  sL   ø€ € € € € ð%ð %ð %ð %ð %ð %ðð ð ð ð ð ð r"   r°   c                   ó^   ‡ — e Zd ZdZd
deddfˆ fd„Zdej        dej        fd„Zde	fd	„Z
ˆ xZS )ÚFocalNetDropPathzÏStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    r‚   Ú	drop_probrN   Nc                 óV   •— t          ¦   «                              ¦   «          || _        d S r^   )r:   r;   r¼   )rI   r¼   rK   s     €r#   r;   zFocalNetDropPath.__init__W  s$   ø€ Ý‰Œ×ÒÑÔÐØ"ˆŒˆˆr"   r   c                 ó  — | j         dk    s| j        s|S d| j         z
  }|j        d         fd|j        dz
  z  z   }t	          j        ||j        |j        ¬¦  «        }t	          j        ||z   ¦  «        }| 	                    |¦  «        |z  S )Nr‚   r   r   )r   )ÚdtypeÚdevice)
r¼   Útrainingr{   Úndimr   Úrandr¿   rÀ   ÚfloorÚdiv)rI   r   Ú	keep_probr{   Úrandom_tensors        r#   r]   zFocalNetDropPath.forward[  s“   € ØŒ>˜SÒ Ð ¨¬Ð Ø Ð Ø˜œÑ&ˆ	ØÔ$ QÔ'Ð)¨D°MÔ4FÈÑ4JÑ,KÑKˆÝœ
 5°Ô0CÈMÔL`ÐaÑaÔaˆÝœ M°IÑ$=Ñ>Ô>ˆØ× Ò  Ñ+Ô+¨mÑ;Ð;r"   c                 ó   — d| j         › �S )Nzp=)r¼   ©rI   s    r#   Ú
extra_reprzFocalNetDropPath.extra_reprd  s   € Ø$�D”NÐ$Ð$Ð$r"   ©r‚   )r   r   r   r   Úfloatr;   r   r`   r]   ÚstrrÊ   ra   rb   s   @r#   r»   r»   P  s›   ø€ € € € € ðð ð#ð # %ð #°$ð #ð #ð #ð #ð #ð #ð< U¤\ð <°e´lð <ð <ð <ð <ð%˜Cð %ð %ð %ð %ð %ð %ð %ð %r"   r»   c                   ó*   ‡ — e Zd ZdZdˆ fd„	Zd„ Zˆ xZS )ÚFocalNetLayeraƒ  Focal Modulation Network layer (block).

    Args:
        config (`FocalNetConfig`):
            Model config.
        index (`int`):
            Layer index.
        dim (`int`):
            Number of input channels.
        input_resolution (`tuple[int]`):
            Input resolution.
        drop_path (`float`, *optional*, defaults to 0.0):
            Stochastic depth rate.
    r‚   c                 ó   •— t          ¦   «                              ¦   «          || _        || _        || _        |j        | _        |j        | _        t          j	        ||j
        ¬¦  «        | _        t          |||| j        ¬¦  «        | _        |dk    rt          |¦  «        nt          j        ¦   «         | _        t          j	        ||j
        ¬¦  «        | _        t%          ||j        z  ¦  «        }t)          |||| j        ¬¦  «        | _        d| _        d| _        |j        rlt          j        |j        t7          j        |¦  «        z  d¬¦  «        | _        t          j        |j        t7          j        |¦  «        z  d¬¦  «        | _        d S d S )Nr8   )r1   rœ   r†   r”   r‚   )r1   r¶   r·   rµ   rQ   T)Úrequires_grad)r:   r;   r1   r†   Úinput_resolutionrG   rµ   Úuse_post_layernormr   rC   rD   Únorm1r�   Ú
modulationr»   ÚIdentityÚ	drop_pathÚnorm2r   Ú	mlp_ratior°   ÚmlpÚgamma_1Úgamma_2Úuse_layerscaler@   Úlayerscale_valuer   Úones)rI   r1   rœ   r†   rÒ   r×   Úmlp_hidden_dimrK   s          €r#   r;   zFocalNetLayer.__init__x  sm  ø€ Ý‰Œ×ÒÑÔÐàˆŒð ˆŒØ 0ˆÔð Ô.ˆŒ	Ø"(Ô";ˆÔå”\ #¨6Ô+@ÐAÑAÔAˆŒ
Ý,ØØØØ#œyð	
ñ 
ô 
ˆŒð 9BÀCº¸Õ)¨)Ñ4Ô4Ð4ÍRÌ[É]Ì]ˆŒÝ”\ #¨6Ô+@ÐAÑAÔAˆŒ
Ý˜S 6Ô#3Ñ3Ñ4Ô4ˆÝ f¸#È~ÐdhÔdmÐnÑnÔnˆŒàˆŒØˆŒØÔ ð 	gÝœ<¨Ô(?Å%Ä*ÈSÁ/Ä/Ñ(QÐaeÐfÑfÔfˆDŒLÝœ<¨Ô(?Å%Ä*ÈSÁ/Ä/Ñ(QÐaeÐfÑfÔfˆDŒLˆLˆLð	gð 	gr"   c           	      óV  — |\  }}|j         \  }}}|}| j        r|n|                      |¦  «        }|                     ||||¦  «        }|                      |¦  «                             |||z  |¦  «        }| j        s|n|                      |¦  «        }||                      | j        |z  ¦  «        z   }||                      | j        | j        r(|                      |  	                    |¦  «        ¦  «        n'|  	                    |                      |¦  «        ¦  «        z  ¦  «        z   }|S r^   )
r{   rÓ   rÔ   ÚviewrÕ   r×   rÛ   rÜ   rØ   rÚ   )	rI   r¤   Úinput_dimensionsrv   rw   rX   rZ   r4   Úshortcuts	            r#   r]   zFocalNetLayer.forward˜  s0  € Ø(‰ˆ�Ø&2Ô&8Ñ#ˆ
�A�|Øˆð (,Ô'>Ð\�|�|ÀDÇJÂJÈ|ÑD\ÔD\ˆØ#×(Ò(¨°V¸UÀLÑQÔQˆØ—’ |Ñ4Ô4×9Ò9¸*ÀfÈuÁnÐVbÑcÔcˆØ+/Ô+BÐ`�|�|ÈÏ
Ê
ÐS_ÑH`ÔH`ˆð   $§.¢.°´ÀÑ1LÑ"MÔ"MÑMˆØ# d§n¢nØŒLØ59Ô5LÐtˆt�zŠz˜$Ÿ(š( <Ñ0Ô0Ñ1Ô1Ð1ÐRV×RZÒRZÐ[_×[eÒ[eÐfrÑ[sÔ[sÑRtÔRtñvñ'
ô '
ñ 
ˆð
 Ðr"   rË   )r   r   r   r   r;   r]   ra   rb   s   @r#   rÏ   rÏ   h  s]   ø€ € € € € ðð ðgð gð gð gð gð gð@ð ð ð ð ð ð r"   rÏ   c                   ób   ‡ — e Zd Zˆ fd„Zdej        deeef         deej                 fd„Zˆ xZ	S )ÚFocalNetStagec           
      ó  •‡‡‡‡‡	— t          ¦   «                              ¦   «          ‰| _        t          ‰j        ¦  «        | _        ˆfd„t          | j        ¦  «        D ¦   «         }|‰         Š‰| j        dz
  k     r|‰dz            nd }‰| j        dz
  k     rt          nd }d„ t          j	        d‰j
        t          ‰j        ¦  «        d¬¦  «        D ¦   «         }|t          ‰j        d ‰…         ¦  «        t          ‰j        d ‰dz   …         ¦  «        …         Š	t          j        ˆˆˆ	ˆˆfd„t          ‰j        ‰         ¦  «        D ¦   «         ¦  «        | _        |� |‰‰d‰|d	‰j        d
¬¦  «        | _        nd | _        d
| _        d S )Nc                 ó*   •— g | ]}‰j         d |z  z  ‘ŒS )rf   )r5   )Ú.0Úir1   s     €r#   ú
<listcomp>z*FocalNetStage.__init__.<locals>.<listcomp>´  s%   ø€ ÐOÐOÐO°1�VÔ%¨¨A©Ñ.ÐOÐOÐOr"   r   c                 ó6   — g | ]}|                      ¦   «         ‘ŒS r!   )Úitem)ré   r¥   s     r#   rë   z*FocalNetStage.__init__.<locals>.<listcomp>º  s    € ÐlÐlÐl˜Aˆq�vŠv‰xŒxÐlÐlÐlr"   r   Úcpu)rÀ   c                 ór   •— g | ]3}t          ‰‰‰‰t          ‰t          ¦  «        r‰|         n‰¬ ¦  «        ‘Œ4S ))r1   rœ   r†   rÒ   r×   )rÏ   rk   Úlist)ré   rê   r1   r†   r×   rœ   rÒ   s     €€€€€r#   rë   z*FocalNetStage.__init__.<locals>.<listcomp>¾  s`   ø€ ð 	ð 	ð 	ð õ Ø!ØØØ%5Ý.8¸ÅDÑ.IÔ.IÐX˜i¨œl˜lÈyðñ ô ð	ð 	ð 	r"   rf   TF)r1   r2   r3   r4   r5   rr   r6   r7   )r:   r;   r1   ÚlenÚdepthsÚ
num_stagesr˜   r<   r   ÚlinspaceÚdrop_path_rateÚsumr   r•   Úlayersr6   Ú
downsampleÚpointing)rI   r1   rœ   rÒ   r5   Úout_dimrø   Údprr†   r×   rK   s    ```    @@€r#   r;   zFocalNetStage.__init__®  sÄ  øøøøøø€ Ý‰Œ×ÒÑÔÐàˆŒÝ˜fœmÑ,Ô,ˆŒàOÐOÐOÐO½¸d¼oÑ8NÔ8NÐOÑOÔOˆ	Ø˜ÔˆØ+0°4´?ÀQÑ3FÒ+FÐ+F�)˜E A™IÔ&Ð&ÈTˆØ16¸¼È1Ñ9LÒ1LÐ1LÕ,Ð,ÐSWˆ
ð mÐl¥¤°°6Ô3HÍ#ÈfÌmÑJ\ÔJ\ÐejÐ!kÑ!kÔ!kÐlÑlÔlˆØ�˜FœM¨&¨5¨&Ô1Ñ2Ô2µS¸¼À{ÈÐQRÉÀ{Ô9SÑ5TÔ5TÐTÔUˆ	å”mð	ð 	ð 	ð 	ð 	ð 	ð 	ð 	õ ˜vœ}¨UÔ3Ñ4Ô4ð	ñ 	ô 	ñ
ô 
ˆŒð Ð!Ø(˜jØØ+ØØ Ø!ØØ%Ô4Øð	ñ 	ô 	ˆDŒOˆOð #ˆDŒOàˆŒˆˆr"   r   rã   rN   c                 ó  — |\  }}| j         D ]} |||¦  «        }Œ|}| j        �U|\  }}|                     dd¦  «                             |j        d         d||¦  «        }|                      |¦  «        \  }}n||||f}|||f}|S )Nr   rf   r   rP   )r÷   rø   r~   Úreshaper{   )	rI   r   rã   rv   rw   Úlayer_moduleÚ!hidden_states_before_downsamplingrW   Ústage_outputss	            r#   r]   zFocalNetStage.forwardÚ  sÀ   € Ø(‰ˆ�Ø œKð 	Jð 	JˆLØ(˜L¨Ð8HÑIÔIˆMˆMà,9Ð)ØŒ?Ð&Ø,‰MˆF�EØ)×3Ò3°A°qÑ9Ô9×AÒAØ1Ô7¸Ô:¸BÀÈñô ˆMð 04¯ª¸}Ñ/MÔ/MÑ,ˆMÐ,Ð,ð "(¨°¸Ð >Ðà&Ð(IÐK\Ð]ˆàÐr"   )
r   r   r   r;   r   r`   r    r   r]   ra   rb   s   @r#   ræ   ræ   ­  sw   ø€ € € € € ð*ð *ð *ð *ð *ðX U¤\ð ÀUÈ3ÐPSÈ8Ä_ð ÐY^Ð_dÔ_kÔYlð ð ð ð ð ð ð ð r"   ræ   c                   óx   ‡ — e Zd Zˆ fd„Z	 	 	 ddej        deeef         dedz  dedz  d	edz  d
ee	z  fd„Z
ˆ xZS )ÚFocalNetEncoderc                 ó
  •‡‡— t          ¦   «                              ¦   «          t          ‰j        ¦  «        | _        ‰| _        t          j        ˆˆfd„t          | j        ¦  «        D ¦   «         ¦  «        | _	        d| _
        d S )Nc           
      óh   •— g | ].}t          ‰|‰d          d|z  z  ‰d         d|z  z  f¬¦  «        ‘Œ/S )r   rf   r   )r1   rœ   rÒ   )ræ   )ré   Úi_layerr1   r>   s     €€r#   rë   z,FocalNetEncoder.__init__.<locals>.<listcomp>ö  se   ø€ ð ð ð ð õ Ø!Ø!Ø&/°¤l°q¸'±zÑ&BÀIÈaÄLÐUVÐX_ÑU_ÑD`Ð%aðñ ô ðð ð r"   F)r:   r;   rñ   rò   ró   r1   r   r•   r˜   ÚstagesÚgradient_checkpointing)rI   r1   r>   rK   s    ``€r#   r;   zFocalNetEncoder.__init__ð  s�   øøø€ Ý‰Œ×ÒÑÔÐÝ˜fœmÑ,Ô,ˆŒØˆŒå”mðð ð ð ð õ  % T¤_Ñ5Ô5ðñ ô ñ	
ô 	
ˆŒð ',ˆÔ#Ð#Ð#r"   FTr   rã   Úoutput_hidden_statesNÚ(output_hidden_states_before_downsamplingÚreturn_dictrN   c                 óÀ  — |rdnd }|rdnd }|r?|j         \  }}	}
 |j        |g|¢|
‘R Ž }|                     dddd¦  «        }||fz  }||fz  }t          | j        ¦  «        D ]Î\  }} |||¦  «        }|d         }|d         }|d         }|d         |d         f}|rP|rN|j         \  }}	}
 |j        |g|d         |d         f¢|
‘R Ž }|                     dddd¦  «        }||fz  }||fz  }Œ‹|rA|s?|j         \  }}	}
 |j        |g|¢|
‘R Ž }|                     dddd¦  «        }||fz  }||fz  }ŒÏ|st          d„ ||fD ¦   «         ¦  «        S t          |||¬	¦  «        S )
Nr!   r   r   r   rf   éþÿÿÿrP   c              3   ó   K  — | ]}|®|V — Œ	d S r^   r!   )ré   Úvs     r#   ú	<genexpr>z*FocalNetEncoder.forward.<locals>.<genexpr>1  s"   è è € ÐXÐX˜qÈ!È-˜È-È-È-È-ÐXÐXr"   )r   r   r   )r{   râ   r    Ú	enumerater  r    r   )rI   r   rã   r  r	  r
  Úall_hidden_statesÚall_reshaped_hidden_statesrX   rZ   Úhidden_sizeÚreshaped_hidden_staterê   Ústage_moduler   rÿ   rW   s                    r#   r]   zFocalNetEncoder.forward  s`  € ð #7Ð@˜B˜B¸DÐØ+?Ð%I R RÀTÐ"àð 	CØ)6Ô)<Ñ&ˆJ˜˜;à$6 MÔ$6°zÐ$bÐDTÐ$bÐVaÐ$bÐ$bÐ$bÐ!Ø$9×$AÒ$AÀ!ÀQÈÈ1Ñ$MÔ$MÐ!Ø -Ð!1Ñ1ÐØ&Ð+@Ð*BÑBÐ&å(¨¬Ñ5Ô5ð 	Gð 	G‰OˆAˆ|Ø(˜L¨Ð8HÑIÔIˆMà)¨!Ô,ˆMØ0=¸aÔ0@Ð-Ø -¨aÔ 0Ðà 1°"Ô 5Ð7HÈÔ7LÐMÐà#ð GÐ(Pð GØ-NÔ-TÑ*�
˜A˜{ð )OÐ(IÔ(NØð)Ø"3°AÔ"6Ð8IÈ!Ô8LÐ!Mð)ØOZð)ð )ð )Ð%ð )>×(EÒ(EÀaÈÈAÈqÑ(QÔ(QÐ%Ø!Ð&GÐ%IÑIÐ!Ø*Ð/DÐ.FÑFÐ*Ð*Ø%ð GÐ.Vð GØ-:Ô-@Ñ*�
˜A˜{à(:¨Ô(:¸:Ð(fÐHXÐ(fÐZeÐ(fÐ(fÐ(fÐ%Ø(=×(EÒ(EÀaÈÈAÈqÑ(QÔ(QÐ%Ø! mÐ%5Ñ5Ð!Ø*Ð/DÐ.FÑFÐ*øàð 	YÝÐXÐX ]Ð4EÐ$FÐXÑXÔXÑXÔXÐXå$Ø+Ø+Ø#=ð
ñ 
ô 
ð 	
r"   )FFT)r   r   r   r;   r   r`   r    r   Úboolr   r]   ra   rb   s   @r#   r  r  ï  s²   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð, -2Ø@EØ#'ð5
ð 5
à”|ð5
ð    S œ/ð5
ð # T™kð	5
ð
 37¸±+ð5
ð ˜D‘[ð5
ð 
Ð&Ñ	&ð5
ð 5
ð 5
ð 5
ð 5
ð 5
ð 5
ð 5
r"   r  c                   óf   ‡ — e Zd ZU eed<   dZdZdZdgZ e	j
        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚFocalNetPreTrainedModelr1   ÚfocalnetrL   Træ   c                 ó–  •— t          ¦   «                              |¦  «         t          |t          ¦  «        r$|j        �t          j        |j        ¦  «         dS dS t          |t          ¦  «        rV| j        j	        rLt          j
        |j        | j        j        ¦  «         t          j
        |j        | j        j        ¦  «         dS dS dS )zInitialize the weightsN)r:   Ú_init_weightsrk   r/   rB   ÚinitÚzeros_rÏ   r1   rÝ   Ú	constant_rÛ   rÞ   rÜ   )rI   ÚmodulerK   s     €r#   r  z%FocalNetPreTrainedModel._init_weightsB  sÌ   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ0Ñ1Ô1ð 	MØÔ Ð,Ý”˜FÔ-Ñ.Ô.Ð.Ð.Ð.ð -Ð,å˜¥Ñ.Ô.ð 	MØŒ{Ô)ð MÝ”˜vœ~¨t¬{Ô/KÑLÔLÐLÝ”˜vœ~¨t¬{Ô/KÑLÔLÐLÐLÐLð	Mð 	MðMð Mr"   )r   r   r   r   r   Úbase_model_prefixÚmain_input_nameÚsupports_gradient_checkpointingÚ_no_split_modulesr   Úno_gradr  ra   rb   s   @r#   r  r  :  s{   ø€ € € € € € àÐÐÑØ"ÐØ$€OØ&*Ð#Ø(Ð)Ðà€U„]�_„_ð	Mð 	Mð 	Mð 	Mñ „_ð	Mð 	Mð 	Mð 	Mð 	Mr"   r  c                   óŽ   ‡ — e Zd Zdˆ fd„	Zd„ Ze	 	 	 	 ddej        dz  dej        dz  de	dz  d	e	dz  d
e
ez  f
d„¦   «         Zˆ xZS )ÚFocalNetModelTFc                 óî  •— t          ¦   «                              |¦  «         || _        t          |j        ¦  «        | _        t          |j        d| j        dz
  z  z  ¦  «        | _        t          ||¬¦  «        | _
        t          || j
        j        ¦  «        | _        t          j        | j        |j        ¬¦  «        | _        |rt          j        d¦  «        nd| _        |                      ¦   «          dS )zû
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        use_mask_token (`bool`, *optional*, defaults to `False`):
            Whether to use a mask token for masked image modeling.
        rf   r   )rJ   r8   N)r:   r;   r1   rñ   rò   ró   r   r5   Únum_featuresr/   rV   r  r?   Úencoderr   rC   rD   r›   ÚAdaptiveAvgPool1dÚpoolerÚ	post_init)rI   r1   Úadd_pooling_layerrJ   rK   s       €r#   r;   zFocalNetModel.__init__Q  sÓ   ø€ õ 	‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ˜fœmÑ,Ô,ˆŒÝ Ô 0°1¸¼È1Ñ9LÑ3MÑ MÑNÔNˆÔå,¨VÀNÐSÑSÔSˆŒÝ& v¨t¬Ô/IÑJÔJˆŒåœ dÔ&7¸VÔ=RÐSÑSÔSˆŒØ1BÐL•bÔ*¨1Ñ-Ô-Ð-ÈˆŒð 	�ŠÑÔÐÐÐr"   c                 ó   — | j         j        S r^   )rV   r=   rÉ   s    r#   Úget_input_embeddingsz"FocalNetModel.get_input_embeddingsf  s   € ØŒÔ/Ð/r"   NrL   rM   r  r
  rN   c                 óò  — |�|n| j         j        }|�|n| j         j        }|€t          d¦  «        ‚|                      ||¬¦  «        \  }}|                      ||||¬¦  «        }|d         }	|                      |	¦  «        }	d}
| j        �>|                      |	                     dd¦  «        ¦  «        }
t          j
        |
d¦  «        }
|s|	|
f|dd…         z   }|S t          |	|
|j        |j        ¬¦  «        S )	z¿
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        Nz You have to specify pixel_values)rM   ©r  r
  r   r   rf   )r   r&   r   r   )r1   r  r
  r|   rV   r)  r›   r+  r~   r   r}   r%   r   r   )rI   rL   rM   r  r
  ÚkwargsÚembedding_outputrã   Úencoder_outputsÚsequence_outputÚpooled_outputÚoutputs               r#   r]   zFocalNetModel.forwardi  s1  € ð %9Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàÐÝÐ?Ñ@Ô@Ð@à-1¯_ª_¸\Ð[j¨_Ñ-kÔ-kÑ*ÐÐ*àŸ,š,ØØØ!5Ø#ð	 'ñ 
ô 
ˆð *¨!Ô,ˆØŸ.š.¨Ñ9Ô9ˆàˆØŒ;Ð"Ø ŸKšK¨×(AÒ(AÀ!ÀQÑ(GÔ(GÑHÔHˆMÝ!œM¨-¸Ñ;Ô;ˆMàð 	Ø% }Ð5¸ÈÈÈÔ8KÑKˆFàˆMå"Ø-Ø'Ø)Ô7Ø#2Ô#Ið	
ñ 
ô 
ð 	
r"   )TF©NNNN)r   r   r   r;   r/  r   r   r   r_   r  r    r%   r]   ra   rb   s   @r#   r&  r&  O  sÍ   ø€ € € € € ðð ð ð ð ð ð*0ð 0ð 0ð ð 26Ø37Ø,0Ø#'ð/
ð /
àÔ'¨$Ñ.ð/
ð Ô)¨DÑ0ð/
ð # T™kð	/
ð
 ˜D‘[ð/
ð 
Ð$Ñ	$ð/
ð /
ð /
ñ „^ð/
ð /
ð /
ð /
ð /
r"   r&  a‰  
    FocalNet Model with a decoder on top for masked image modeling.

    This follows the same implementation as in [SimMIM](https://huggingface.co/papers/2111.09886).

    <Tip>

    Note that we provide a script to pre-train this model on custom data in our [examples
    directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining).

    </Tip>
    c                   ó†   ‡ — e Zd Zˆ fd„Ze	 	 	 	 d	dej        dz  dej        dz  dedz  dedz  de	e
z  f
d„¦   «         Zˆ xZS )
ÚFocalNetForMaskedImageModelingc                 óÂ  •— t          ¦   «                              |¦  «         t          |dd¬¦  «        | _        t	          |j        ¦  «        | _        t          |j        d| j        dz
  z  z  ¦  «        }t          j
        t          j        ||j        dz  |j        z  d¬¦  «        t          j        |j        ¦  «        ¦  «        | _        |                      ¦   «          d S )NFT)r-  rJ   rf   r   )Úin_channelsÚout_channelsrh   )r:   r;   r&  r  rñ   rò   ró   r   r5   r   rš   rp   Úencoder_strider4   ÚPixelShuffleÚdecoderr,  )rI   r1   r(  rK   s      €r#   r;   z'FocalNetForMaskedImageModeling.__init__«  sÎ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å% fÀÐVZÐ[Ñ[Ô[ˆŒå˜fœmÑ,Ô,ˆŒÝ˜6Ô+¨a°D´OÀaÑ4GÑ.HÑHÑIÔIˆÝ”}ÝŒIØ(°vÔ7LÈaÑ7OÐRXÔReÑ7eÐstðñ ô õ ŒO˜FÔ1Ñ2Ô2ñ	
ô 
ˆŒð 	�ŠÑÔÐÐÐr"   NrL   rM   r  r
  rN   c                 óŠ  — |�|n| j         j        }|                      ||||¬¦  «        }|d         }|                     dd¦  «        }|j        \  }}	}
t          j        |
dz  ¦  «        x}}|                     ||	||¦  «        }|                      |¦  «        }d}|�ñ| j         j	        | j         j
        z  }|                     d||¦  «        }|                     | j         j
        d¦  «                             | j         j
        d¦  «                             d¦  «                             ¦   «         }t          j                             ||d¬	¦  «        }||z                       ¦   «         |                     ¦   «         d
z   z  | j         j        z  }|s|f|dd…         z   }|�|f|z   n|S t'          |||j        |j        ¬¦  «        S )a�  
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

        Examples:
        ```python
        >>> from transformers import AutoImageProcessor, FocalNetConfig, FocalNetForMaskedImageModeling
        >>> import torch
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/focalnet-base-simmim-window6-192")
        >>> config = FocalNetConfig()
        >>> model = FocalNetForMaskedImageModeling(config)

        >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
        >>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
        >>> # create random boolean mask of shape (batch_size, num_patches)
        >>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

        >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
        >>> loss, reconstructed_pixel_values = outputs.loss, outputs.logits
        >>> list(reconstructed_pixel_values.shape)
        [1, 3, 192, 192]
        ```N)rM   r  r
  r   r   rf   g      à?rP   Únone)Ú	reductiongñhãˆµøä>)r)   r*   r   r   )r1   r
  r  r~   r{   ÚmathrÄ   rý   r@  r2   r3   Úrepeat_interleaverT   r¡   r   rt   Úl1_lossrö   r4   r(   r   r   )rI   rL   rM   r  r
  r2  Úoutputsr5  rX   r4   Úsequence_lengthrv   rw   Úreconstructed_pixel_valuesÚmasked_im_lossrR   r\   Úreconstruction_lossr7  s                      r#   r]   z&FocalNetForMaskedImageModeling.forward¼  sä  € ðN &1Ð%<�k�kÀ$Ä+ÔBYˆà—-’-ØØ+Ø!5Ø#ð	  ñ 
ô 
ˆð " !œ*ˆà)×3Ò3°A°qÑ9Ô9ˆØ4CÔ4IÑ1ˆ
�L /Ýœ O°SÑ$8Ñ9Ô9Ð9ˆ�Ø)×1Ò1°*¸lÈFÐTYÑZÔZˆð &*§\¢\°/Ñ%BÔ%BÐ"àˆØÐ&Ø”;Ô)¨T¬[Ô-CÑCˆDØ-×5Ò5°b¸$ÀÑEÔEˆOà×1Ò1°$´+Ô2HÈ!ÑLÔLß"Ò" 4¤;Ô#9¸1Ñ=Ô=ß’˜1‘”ß’‘”ð	 õ #%¤-×"7Ò"7¸ÐF`ÐlrÐ"7Ñ"sÔ"sÐØ1°DÑ8×=Ò=Ñ?Ô?À4Ç8Â8Á:Ä:ÐPTÑCTÑUÐX\ÔXcÔXpÑpˆNàð 	ZØ0Ð2°W¸Q¸R¸R´[Ñ@ˆFØ3AÐ3M�^Ð%¨Ñ.Ð.ÐSYÐYå0ØØ5Ø!Ô/Ø#*Ô#Að	
ñ 
ô 
ð 	
r"   r8  )r   r   r   r;   r   r   r   r_   r  r    r(   r]   ra   rb   s   @r#   r:  r:  œ  sÇ   ø€ € € € € ðð ð ð ð ð" ð 26Ø37Ø,0Ø#'ðO
ð O
àÔ'¨$Ñ.ðO
ð Ô)¨DÑ0ðO
ð # T™kð	O
ð
 ˜D‘[ðO
ð 
Ð2Ñ	2ðO
ð O
ð O
ñ „^ðO
ð O
ð O
ð O
ð O
r"   r:  z…
    FocalNet Model with an image classification head on top (a linear layer on top of the pooled output) e.g. for
    ImageNet.
    c                   ó†   ‡ — e Zd Zˆ fd„Ze	 	 	 	 d	dej        dz  dej        dz  dedz  dedz  de	e
z  f
d„¦   «         Zˆ xZS )
ÚFocalNetForImageClassificationc                 ó@  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        |j        dk    r$t          j        | j        j        |j        ¦  «        nt          j        ¦   «         | _	        |  
                    ¦   «          d S )Nr   )r:   r;   Ú
num_labelsr&  r  r   rŽ   r(  rÖ   Ú
classifierr,  ©rI   r1   rK   s     €r#   r;   z'FocalNetForImageClassification.__init__  s‹   ø€ Ý‰Œ×Ò˜Ñ Ô Ð à Ô+ˆŒÝ% fÑ-Ô-ˆŒð IOÔHYÐ\]ÒH]ÐH]�BŒI�d”mÔ0°&Ô2CÑDÔDÐDÕceÔcnÑcpÔcpð 	Œð
 	�ŠÑÔÐÐÐr"   NrL   Úlabelsr  r
  rN   c                 ó:  — |�|n| j         j        }|                      |||¬¦  «        }|d         }|                      |¦  «        }d}	|�|                      ||| j         ¦  «        }	|s|f|dd…         z   }
|	�|	f|
z   n|
S t          |	||j        |j        ¬¦  «        S )aŠ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr1  r   rf   )r)   r-   r   r   )r1   r
  r  rP  Úloss_functionr,   r   r   )rI   rL   rR  r  r
  r2  rG  r6  r-   r)   r7  s              r#   r]   z&FocalNetForImageClassification.forward%  sÕ   € ð &1Ð%<�k�kÀ$Ä+ÔBYˆà—-’-ØØ!5Ø#ð  ñ 
ô 
ˆð   œ
ˆà—’ Ñ/Ô/ˆàˆØÐØ×%Ò% f¨f°d´kÑBÔBˆDàð 	FØ�Y ¨¨¨¤Ñ,ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå,ØØØ!Ô/Ø#*Ô#Að	
ñ 
ô 
ð 	
r"   r8  )r   r   r   r;   r   r   r   Ú
LongTensorr  r    r,   r]   ra   rb   s   @r#   rM  rM    s¹   ø€ € € € € ðð ð ð ð ð ð 26Ø*.Ø,0Ø#'ð'
ð '
àÔ'¨$Ñ.ð'
ð Ô  4Ñ'ð'
ð # T™kð	'
ð
 ˜D‘[ð'
ð 
Ð.Ñ	.ð'
ð '
ð '
ñ „^ð'
ð '
ð '
ð '
ð '
r"   rM  zG
    FocalNet backbone, to be used with frameworks like X-Decoder.
    c                   óŒ   ‡ — e Zd ZdZdefˆ fd„Zeee	 	 d
de	j
        dedz  dedz  defd	„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚFocalNetBackboneFr1   c                 óÄ   •— t          ¦   «                              |¦  «         |j        g|j        z   | _        t          |¦  «        | _        |                      ¦   «          d S r^   )r:   r;   r5   Úhidden_sizesr(  r&  r  r,  rQ  s     €r#   r;   zFocalNetBackbone.__init__X  sX   ø€ Ý‰Œ×Ò˜Ñ Ô Ð à#Ô-Ð.°Ô1DÑDˆÔÝ% fÑ-Ô-ˆŒð 	�ŠÑÔÐÐÐr"   NrL   r  r
  rN   c                 ó@  — |�|n| j         j        }|�|n| j         j        }|                      |dd¬¦  «        }|j        }d}t          | j        ¦  «        D ]\  }}	|	| j        v r|||         fz  }Œ|s|f}
|r|
|j        fz  }
|
S t          ||r|j        ndd¬¦  «        S )a»  
        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, AutoBackbone
        >>> import torch
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> processor = AutoImageProcessor.from_pretrained("microsoft/focalnet-tiny-lrf")
        >>> model = AutoBackbone.from_pretrained("microsoft/focalnet-tiny-lrf")

        >>> inputs = processor(image, return_tensors="pt")
        >>> outputs = model(**inputs)
        ```NTr1  r!   )Úfeature_mapsr   Ú
attentions)
r1   r
  r  r  r   r  Ústage_namesr¸   r   r   )rI   rL   r  r
  r2  rG  r   r[  ÚidxÚstager7  s              r#   r]   zFocalNetBackbone.forwarda  sî   € ð< &1Ð%<�k�kÀ$Ä+ÔBYˆà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð —-’- À4ÐUY�-ÑZÔZˆàÔ6ˆàˆÝ# DÔ$4Ñ5Ô5ð 	6ð 	6‰JˆC�Ø˜Ô)Ð)Ð)Ø ¨sÔ!3Ð 5Ñ5�øàð 	Ø"�_ˆFØ#ð 3Ø˜7Ô0Ð2Ñ2�ØˆMåØ%Ø3GÐQ˜'Ô/Ð/ÈTØð
ñ 
ô 
ð 	
r"   )NN)r   r   r   Úhas_attentionsr   r;   r   r	   r   r   r`   r  r   r]   ra   rb   s   @r#   rW  rW  P  s¾   ø€ € € € € ð €Nð˜~ð ð ð ð ð ð ð Ø Øð -1Ø#'ð	3
ð 3
à”lð3
ð # T™kð3
ð ˜D‘[ð	3
ð 
ð3
ð 3
ð 3
ñ „^ñ !Ô ñ Ôð3
ð 3
ð 3
ð 3
ð 3
r"   rW  )rM  r:  rW  r&  r  )4r   Úcollections.abcrl   rD  Údataclassesr   r   r   Ú r   r  Úactivationsr   Úbackbone_utilsr   r	   Úmodeling_layersr
   Úmodeling_outputsr   Úmodeling_utilsr   Úutilsr   r   r   Úutils.genericr   Úconfiguration_focalnetr   Ú
get_loggerr   Úloggerr   r%   r(   r,   ÚModuler/   r<   r�   r°   r»   rÏ   ræ   r  r  r&  r:  rM  rW  Ú__all__r!   r"   r#   ú<module>rp     sÀ  ðð Ð à Ð Ð Ð Ø €€€Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø HÐ HÐ HÐ HÐ HÐ HÐ HÐ HØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø .Ð .Ð .Ð .Ð .Ð .Ø -Ð -Ð -Ð -Ð -Ð -Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø -Ð -Ð -Ð -Ð -Ð -Ø 2Ð 2Ð 2Ð 2Ð 2Ð 2ð 
ˆÔ	˜HÑ	%Ô	%€ð €ððñ ô ð
 ðCð Cð Cð Cð C˜Kñ Cô Cñ „ñô ðCð €ððñ ô ð
 ðCð Cð Cð Cð C˜+ñ Cô Cñ „ñô ðCð$ €ððñ ô ð
 ðCð Cð Cð Cð C¨ñ Cô Cñ „ñô ðCð( €ððñ ô ð
 ðCð Cð Cð Cð C Kñ Cô Cñ „ñô ðCð(%-ð %-ð %-ð %-ð %-˜œñ %-ô %-ð %-ðPD-ð D-ð D-ð D-ð D-˜bœiñ D-ô D-ð D-ðNDð Dð Dð Dð D˜œñ Dô Dð DðNð ð ð ð �"”)ñ ô ð ð(%ð %ð %ð %ð %�r”yñ %ô %ð %ð0Bð Bð Bð Bð B�B”Iñ Bô Bð BðJ?ð ?ð ?ð ?ð ?Ð.ñ ?ô ?ð ?ðDH
ð H
ð H
ð H
ð H
�b”iñ H
ô H
ð H
ðV ðMð Mð Mð Mð M˜oñ Mô Mñ „ðMð( ðI
ð I
ð I
ð I
ð I
Ð+ñ I
ô I
ñ „ðI
ðX €ððñ ô ðb
ð b
ð b
ð b
ð b
Ð%<ñ b
ô b
ñô ðb
ðJ €ððñ ô ð8
ð 8
ð 8
ð 8
ð 8
Ð%<ñ 8
ô 8
ñô ð8
ðv €ððñ ô ð
B
ð B
ð B
ð B
ð B
�}Ð&=ñ B
ô B
ñô ð
B
ðJð ð €€€r"   