§
    ‚ŠtjÐÚ  ã                   ó  — d Z ddlZddlZddlmZ ddlZddlmZmZ ddl	m
Z ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZmZ ddlmZ ddlmZ  ej         e!¦  «        Z" ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z# ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z$ ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z% ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z&d„ Z'd„ Z( G d„ d ej)        ¦  «        Z* G d!„ d"ej)        ¦  «        Z+ G d#„ d$ej)        ¦  «        Z, G d%„ d&ej)        ¦  «        Z- G d'„ d(ej)        ¦  «        Z. G d)„ d*ej)        ¦  «        Z/ G d+„ d,ej)        ¦  «        Z0 G d-„ d.ej)        ¦  «        Z1 G d/„ d0ej)        ¦  «        Z2 G d1„ d2ej)        ¦  «        Z3 G d3„ d4e¦  «        Z4 G d5„ d6ej)        ¦  «        Z5e G d7„ d8e¦  «        ¦   «         Z6e G d9„ d:e6¦  «        ¦   «         Z7 ed;¬¦  «         G d<„ d=e6¦  «        ¦   «         Z8 ed>¬¦  «         G d?„ d@e6¦  «        ¦   «         Z9 edA¬¦  «         G dB„ dCee6¦  «        ¦   «         Z:g dD¢Z;dS )Ez!PyTorch Swinv2 Transformer model.é    N)Ú	dataclass)ÚTensorÚnné   )Úinitialization)ÚACT2FN)ÚBackboneMixinÚfilter_output_hidden_states)ÚGradientCheckpointingLayer)ÚBackboneOutput)ÚPreTrainedModel)ÚModelOutputÚauto_docstringÚloggingÚ	torch_int)Úcan_return_tupleé   )ÚSwinv2ConfigzP
    Swinv2 encoder's outputs, with potential hidden states and attentions.
    )Úcustom_introc                   ó¼   — e Zd ZU dZdZej        dz  ed<   dZe	ej        df         dz  ed<   dZ
e	ej        df         dz  ed<   dZe	ej        df         dz  ed<   dS )ÚSwinv2EncoderOutputaí  
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    NÚlast_hidden_state.Úhidden_statesÚ
attentionsÚreshaped_hidden_states)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚFloatTensorÚ__annotations__r   Útupler   r   © ó    úh/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/swinv2/modeling_swinv2.pyr   r   (   sž   € € € € € € ðð ð 37Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø:>€M�5˜Ô*¨CÐ/Ô0°4Ñ7Ð>Ð>Ñ>Ø7;€J��eÔ'¨Ð,Ô-°Ñ4Ð;Ð;Ñ;ØCGÐ˜E %Ô"3°SÐ"8Ô9¸DÑ@ÐGÐGÑGÐGÐGr%   r   zX
    Swinv2 model's outputs that also contains a pooling of the last hidden states.
    c                   óÚ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej        df         dz  ed<   dZe
ej        df         dz  ed<   dZe
ej        df         dz  ed<   dS )	ÚSwinv2ModelOutputa±  
    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`, *optional*, returned when `add_pooling_layer=True` is passed):
        Average pooling of the last layer hidden-state.
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nr   Úpooler_output.r   r   r   )r   r   r   r   r   r    r!   r"   r)   r   r#   r   r   r$   r%   r&   r(   r(   ?   s¶   € € € € € € ð	ð 	ð 37Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø.2€M�5Ô$ tÑ+Ð2Ð2Ñ2Ø:>€M�5˜Ô*¨CÐ/Ô0°4Ñ7Ð>Ð>Ñ>Ø7;€J��eÔ'¨Ð,Ô-°Ñ4Ð;Ð;Ñ;ØCGÐ˜E %Ô"3°SÐ"8Ô9¸DÑ@ÐGÐGÑGÐGÐGr%   r(   z,
    Swinv2 masked image model outputs.
    c                   óÚ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej        df         dz  ed<   dZe
ej        df         dz  ed<   dZe
ej        df         dz  ed<   dS )	ÚSwinv2MaskedImageModelingOutputa  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `bool_masked_pos` is provided):
        Masked image modeling (MLM) loss.
    reconstruction (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
        Reconstructed pixel values.
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    NÚlossÚreconstruction.r   r   r   )r   r   r   r   r,   r    r!   r"   r-   r   r#   r   r   r$   r%   r&   r+   r+   Y   sµ   € € € € € € ðð ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø/3€N�EÔ%¨Ñ,Ð3Ð3Ñ3Ø:>€M�5˜Ô*¨CÐ/Ô0°4Ñ7Ð>Ð>Ñ>Ø7;€J��eÔ'¨Ð,Ô-°Ñ4Ð;Ð;Ñ;ØCGÐ˜E %Ô"3°SÐ"8Ô9¸DÑ@ÐGÐGÑGÐGÐGr%   r+   z2
    Swinv2 outputs for image classification.
    c                   óÚ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej        df         dz  ed<   dZe
ej        df         dz  ed<   dZe
ej        df         dz  ed<   dS )	ÚSwinv2ImageClassifierOutputa7  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Classification (or regression if config.num_labels==1) loss.
    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Classification (or regression if config.num_labels==1) scores (before SoftMax).
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nr,   Úlogits.r   r   r   )r   r   r   r   r,   r    r!   r"   r0   r   r#   r   r   r$   r%   r&   r/   r/   u   sµ   € € € € € € ðð ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø:>€M�5˜Ô*¨CÐ/Ô0°4Ñ7Ð>Ð>Ñ>Ø7;€J��eÔ'¨Ð,Ô-°Ñ4Ð;Ð;Ñ;ØCGÐ˜E %Ô"3°SÐ"8Ô9¸DÑ@ÐGÐGÑGÐGÐGr%   r/   c                 óÚ   — | j         \  }}}}|                      |||z  |||z  ||¦  «        } |                      dd¦  «                             ¦   «                              d|||¦  «        }|S )z2
    Partitions the given input into windows.
    é   r   éÿÿÿÿ©ÚshapeÚviewÚ	transposeÚ
contiguous)Úinput_featureÚwindow_sizeÚ
batch_sizeÚheightÚwidthÚnum_channelsÚwindowss          r&   Úwindow_partitionr@   ’   s   € ð /<Ô.AÑ+€J�˜˜|Ø!×&Ò&Ø�F˜kÑ)¨;¸ÀÑ8LÈkÐ[gñô €Mð ×%Ò% a¨Ñ+Ô+×6Ò6Ñ8Ô8×=Ò=¸bÀ+È{Ð\hÑiÔi€GØ€Nr%   c                 óÜ   — | j         d         }|                      d||z  ||z  |||¦  «        } |                      dd¦  «                             ¦   «                              d|||¦  «        } | S )z?
    Merges windows to produce higher resolution features.
    r3   r2   r   r4   )r?   r:   r<   r=   r>   s        r&   Úwindow_reverserB   Ÿ   sq   € ð ”= Ô$€LØ�lŠl˜2˜v¨Ñ4°e¸{Ñ6JÈKÐYdÐfrÑsÔs€GØ×Ò  1Ñ%Ô%×0Ò0Ñ2Ô2×7Ò7¸¸FÀEÈ<ÑXÔX€GØ€Nr%   c            
       ó¤   ‡ — e Zd ZdZdˆ fd„	Zdej        dededej        fd„Z	 	 dd
ej	        d	z  dej
        d	z  dedeej                 fd„Zˆ xZS )ÚSwinv2EmbeddingszW
    Construct the patch and position embeddings. Optionally, also the mask token.
    Fc                 ó<  •— t          ¦   «                              ¦   «          t          |¦  «        | _        | j        j        }| j        j        | _        |r-t          j        t          j
        dd|j        ¦  «        ¦  «        nd | _        |j        r6t          j        t          j
        d|dz   |j        ¦  «        ¦  «        | _        nd | _        t          j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        |j        | _        || _        d S )Nr   )ÚsuperÚ__init__ÚSwinv2PatchEmbeddingsÚpatch_embeddingsÚnum_patchesÚ	grid_sizeÚ
patch_gridr   Ú	Parameterr    ÚzerosÚ	embed_dimÚ
mask_tokenÚuse_absolute_embeddingsÚposition_embeddingsÚ	LayerNormÚnormÚDropoutÚhidden_dropout_probÚdropoutÚ
patch_sizeÚconfig)ÚselfrY   Úuse_mask_tokenrJ   Ú	__class__s       €r&   rG   zSwinv2Embeddings.__init__¯   sê   ø€ Ý‰Œ×ÒÑÔÐå 5°fÑ =Ô =ˆÔØÔ+Ô7ˆØÔ/Ô9ˆŒØO]Ðg�"œ,¥u¤{°1°a¸Ô9IÑ'JÔ'JÑKÔKÐKÐcgˆŒàÔ)ð 	,Ý')¤|µE´KÀÀ;ÐQRÁ?ÐTZÔTdÑ4eÔ4eÑ'fÔ'fˆDÔ$Ð$à'+ˆDÔ$å”L Ô!1Ñ2Ô2ˆŒ	Ý”z &Ô"<Ñ=Ô=ˆŒØ Ô+ˆŒØˆŒˆˆr%   Ú
embeddingsr<   r=   Úreturnc                 ó”  — |j         d         dz
  }| j        j         d         dz
  }t          j                             ¦   «         s||k    r||k    r| j        S | j        dd…dd…f         }| j        dd…dd…f         }|j         d         }|| j        z  }	|| j        z  }
t          |dz  ¦  «        }|                     d|||¦  «        }|                     dddd¦  «        }t          j
                             ||	|
fdd	¬
¦  «        }|                     dddd¦  «                             dd|¦  «        }t          j        ||fd¬¦  «        S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   Nr3   ç      à?r   r   r2   ÚbicubicF)ÚsizeÚmodeÚalign_corners©Údim)r5   rR   r    ÚjitÚ
is_tracingrX   r   ÚreshapeÚpermuter   Ú
functionalÚinterpolater6   Úcat)rZ   r]   r<   r=   rJ   Únum_positionsÚclass_pos_embedÚpatch_pos_embedrf   Ú
new_heightÚ	new_widthÚsqrt_num_positionss               r&   Úinterpolate_pos_encodingz)Swinv2Embeddings.interpolate_pos_encodingÂ   sr  € ð !Ô& qÔ)¨AÑ-ˆØÔ0Ô6°qÔ9¸AÑ=ˆõ Œy×#Ò#Ñ%Ô%ð 	,¨+¸Ò*FÐ*FÈ6ÐUZÊ?È?ØÔ+Ð+àÔ2°1°1°1°b°q°b°5Ô9ˆØÔ2°1°1°1°a°b°b°5Ô9ˆàÔ˜rÔ"ˆà˜tœÑ.ˆ
Ø˜Tœ_Ñ,ˆ	å& }°cÑ'9Ñ:Ô:ÐØ)×1Ò1°!Ð5GÐI[Ð]`ÑaÔaˆØ)×1Ò1°!°Q¸¸1Ñ=Ô=ˆåœ-×3Ò3ØØ˜iÐ(ØØð	 4ñ 
ô 
ˆð *×1Ò1°!°Q¸¸1Ñ=Ô=×BÒBÀ1ÀbÈ#ÑNÔNˆåŒy˜/¨?Ð;ÀÐCÑCÔCÐCr%   NÚpixel_valuesÚbool_masked_posrt   c                 óÚ  — |j         \  }}}}|                      |¦  «        \  }}	|                      |¦  «        }|                     ¦   «         \  }
}}|�R| j                             |
|d¦  «        }|                     d¦  «                             |¦  «        }|d|z
  z  ||z  z   }| j        �'|r||  	                    |||¦  «        z   }n
|| j        z   }|  
                    |¦  «        }||	fS )Nr3   ç      ð?)r5   rI   rT   rb   rP   ÚexpandÚ	unsqueezeÚtype_asrR   rt   rW   )rZ   ru   rv   rt   Ú_r>   r<   r=   r]   Úoutput_dimensionsr;   Úseq_lenÚmask_tokensÚmasks                 r&   ÚforwardzSwinv2Embeddings.forwardê   s	  € ð *6Ô);Ñ&ˆˆ<˜ Ø(,×(=Ò(=¸lÑ(KÔ(KÑ%ˆ
Ð%Ø—Y’Y˜zÑ*Ô*ˆ
Ø!+§¢Ñ!2Ô!2Ñˆ
�G˜QàÐ&Øœ/×0Ò0°¸WÀbÑIÔIˆKà"×,Ò,¨RÑ0Ô0×8Ò8¸ÑEÔEˆDØ# s¨T¡zÑ2°[À4Ñ5GÑGˆJàÔ#Ð/Ø'ð CØ'¨$×*GÒ*GÈ
ÐTZÐ\aÑ*bÔ*bÑb�
�
à'¨$Ô*BÑB�
à—\’\ *Ñ-Ô-ˆ
àÐ,Ð,Ð,r%   ©F©NF)r   r   r   r   rG   r    r   Úintrt   r!   Ú
BoolTensorÚboolr#   r�   Ú__classcell__©r\   s   @r&   rD   rD   ª   så   ø€ € € € € ðð ðð ð ð ð ð ð&&D°5´<ð &DÈð &DÐUXð &DÐ]bÔ]ið &Dð &Dð &Dð &DðV 48Ø).ð	-ð -àÔ'¨$Ñ.ð-ð Ô)¨DÑ0ð-ð #'ð	-ð
 
ˆuŒ|Ô	ð-ð -ð -ð -ð -ð -ð -ð -r%   rD   c                   ón   ‡ — e Zd ZdZˆ fd„Zd„ Zdej        dz  deej	        ee
         f         fd„Zˆ xZS )rH   zì
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    c                 óþ  •— t          ¦   «                              ¦   «          |j        |j        }}|j        |j        }}t          |t          j        j	        ¦  «        r|n||f}t          |t          j        j	        ¦  «        r|n||f}|d         |d         z  |d         |d         z  z  }|| _        || _
        |d         |d         z  |d         |d         z  f| _        t          j        ||||¬¦  «        | _        d S )Nr   r   )Úkernel_sizeÚstride)rF   rG   Ú
image_sizerX   r>   rO   Ú
isinstanceÚcollectionsÚabcÚIterablerJ   rK   r   ÚConv2dÚ
projection)rZ   rY   r�   rX   r>   Úhidden_sizerJ   r\   s          €r&   rG   zSwinv2PatchEmbeddings.__init__  sù   ø€ Ý‰Œ×ÒÑÔÐØ!'Ô!2°FÔ4E�Jˆ
Ø$*Ô$7¸Ô9I�kˆÝ#-¨j½+¼/Ô:RÑ#SÔ#SÐq�Z�ZÐZdÐfpÐYqˆ
Ý#-¨j½+¼/Ô:RÑ#SÔ#SÐq�Z�ZÐZdÐfpÐYqˆ
Ø! !”}¨
°1¬Ñ5¸*ÀQ¼-È:ÐVWÌ=Ñ:XÑYˆØ$ˆŒØ&ˆÔØ$ Qœ-¨:°a¬=Ñ8¸*ÀQ¼-È:ÐVWÌ=Ñ:XÐYˆŒåœ) L°+È:Ð^hÐiÑiÔiˆŒˆˆr%   c                 óZ  — || j         d         z  dk    r@d| j         d         || j         d         z  z
  f}t          j                             ||¦  «        }|| j         d         z  dk    rBddd| j         d         || j         d         z  z
  f}t          j                             ||¦  «        }|S )z9Pad pixel_values to be divisible by patch_size if needed.r   r   )rX   r   rk   Úpad)rZ   ru   r<   r=   Ú
pad_valuess        r&   Ú	maybe_padzSwinv2PatchEmbeddings.maybe_pad  s­   € à�4”? 1Ô%Ñ%¨Ò*Ð*Ø˜Tœ_¨QÔ/°%¸$¼/È!Ô:LÑ2LÑLÐMˆJÝœ=×,Ò,¨\¸:ÑFÔFˆLØ�D”O AÔ&Ñ&¨!Ò+Ð+Ø˜Q  4¤?°1Ô#5¸ÀÄÐQRÔASÑ8SÑ#SÐTˆJÝœ=×,Ò,¨\¸:ÑFÔFˆLØÐr%   ru   Nr^   c                 óì   — |j         \  }}}}|                      |||¦  «        }|                      |¦  «        }|j         \  }}}}||f}|                     d¦  «                             dd¦  «        }||fS )Nr2   r   )r5   r˜   r“   Úflattenr7   )rZ   ru   r|   r>   r<   r=   r]   r}   s           r&   r�   zSwinv2PatchEmbeddings.forward%  sƒ   € Ø)5Ô);Ñ&ˆˆ<˜ à—~’~ l°F¸EÑBÔBˆØ—_’_ \Ñ2Ô2ˆ
Ø(Ô.Ñˆˆ1ˆf�eØ# U˜OÐØ×'Ò'¨Ñ*Ô*×4Ò4°Q¸Ñ:Ô:ˆ
àÐ,Ð,Ð,r%   )r   r   r   r   rG   r˜   r    r!   r#   r   r„   r�   r‡   rˆ   s   @r&   rH   rH     s“   ø€ € € € € ðð ðjð jð jð jð jðð ð ð	- EÔ$5¸Ñ$<ð 	-ÀÀuÄ|ÐUZÐ[^ÔU_ÐG_ÔA`ð 	-ð 	-ð 	-ð 	-ð 	-ð 	-ð 	-ð 	-r%   rH   c            	       ó–   ‡ — e Zd ZdZej        fdee         dedej        ddfˆ fd„Z	d„ Z
d	ej        d
eeef         dej        fd„Zˆ xZS )ÚSwinv2PatchMerginga'  
    Patch Merging Layer.

    Args:
        input_resolution (`tuple[int]`):
            Resolution of input feature.
        dim (`int`):
            Number of input channels.
        norm_layer (`nn.Module`, *optional*, defaults to `nn.LayerNorm`):
            Normalization layer class.
    Úinput_resolutionrf   Ú
norm_layerr^   Nc                 óÎ   •— t          ¦   «                              ¦   «          || _        || _        t	          j        d|z  d|z  d¬¦  «        | _         |d|z  ¦  «        | _        d S )Né   r2   F©Úbias)rF   rG   r�   rf   r   ÚLinearÚ	reductionrT   )rZ   r�   rf   rž   r\   s       €r&   rG   zSwinv2PatchMerging.__init__>  sa   ø€ Ý‰Œ×ÒÑÔÐØ 0ˆÔØˆŒÝœ 1 s¡7¨A°©G¸%Ð@Ñ@Ô@ˆŒØ�J˜q 3™wÑ'Ô'ˆŒ	ˆ	ˆ	r%   c                 óŠ   — |dz  dk    p|dz  dk    }|r.ddd|dz  d|dz  f}t           j                             ||¦  «        }|S )Nr2   r   r   )r   rk   r–   )rZ   r9   r<   r=   Ú
should_padr—   s         r&   r˜   zSwinv2PatchMerging.maybe_padE  s\   € Ø˜q‘j A’oÐ:¨5°1©9¸ª>ˆ
Øð 	IØ˜Q  5¨1¡9¨a°¸!±Ð<ˆJÝœM×-Ò-¨m¸ZÑHÔHˆMàÐr%   r9   Úinput_dimensionsc                 óî  — |\  }}|j         \  }}}|                     ||||¦  «        }|                      |||¦  «        }|d d …dd d…dd d…d d …f         }|d d …dd d…dd d…d d …f         }	|d d …dd d…dd d…d d …f         }
|d d …dd d…dd d…d d …f         }t          j        ||	|
|gd¦  «        }|                     |dd|z  ¦  «        }|                      |¦  «        }|                      |¦  «        }|S )Nr   r2   r   r3   r    )r5   r6   r˜   r    rm   r¤   rT   )rZ   r9   r§   r<   r=   r;   rf   r>   Úinput_feature_0Úinput_feature_1Úinput_feature_2Úinput_feature_3s               r&   r�   zSwinv2PatchMerging.forwardM  sD  € Ø(‰ˆ�à(5Ô(;Ñ%ˆ
�C˜à%×*Ò*¨:°v¸uÀlÑSÔSˆàŸš }°f¸eÑDÔDˆà'¨¨¨¨1¨4¨a¨4°°°A°°q°q°qÐ(8Ô9ˆà'¨¨¨¨1¨4¨a¨4°°°A°°q°q°qÐ(8Ô9ˆà'¨¨¨¨1¨4¨a¨4°°°A°°q°q°qÐ(8Ô9ˆà'¨¨¨¨1¨4¨a¨4°°°A°°q°q°qÐ(8Ô9ˆåœ	 ?°OÀ_ÐVeÐ"fÐhjÑkÔkˆØ%×*Ò*¨:°r¸1¸|Ñ;KÑLÔLˆàŸš }Ñ5Ô5ˆØŸ	š	 -Ñ0Ô0ˆàÐr%   )r   r   r   r   r   rS   r#   r„   ÚModulerG   r˜   r    r   r�   r‡   rˆ   s   @r&   rœ   rœ   1  sÀ   ø€ € € € € ð
ð 
ð XZÔWcð (ð (¨¨s¬ð (¸#ð (È2Ì9ð (Ðhlð (ð (ð (ð (ð (ð (ðð ð ð U¤\ð ÀUÈ3ÐPSÈ8Ä_ð ÐY^ÔYeð ð ð ð ð ð ð ð r%   rœ   c            
       ó€   ‡ — e Zd Zddgfˆ fd„	Z	 	 ddej        dej        dz  dedz  deej                 fd	„Z	d
„ Z
ˆ xZS )ÚSwinv2SelfAttentionr   c           
      óD  •— t          ¦   «                              ¦   «          ||z  dk    rt          d|› d|› d�¦  «        ‚|| _        t	          ||z  ¦  «        | _        | j        | j        z  | _        t          |t          j	        j
        ¦  «        r|n||f| _        || _        t          j        t          j        dt          j        |ddf¦  «        z  ¦  «        ¦  «        | _        t          j        t          j        ddd	¬
¦  «        t          j        d	¬¦  «        t          j        d|d¬
¦  «        ¦  «        | _        |                      ¦   «         \  }}|                      d|d¬¦  «         |                      d|d¬¦  «         t          j        | j        | j        |j        ¬
¦  «        | _        t          j        | j        | j        d¬
¦  «        | _        t          j        | j        | j        |j        ¬
¦  «        | _        t          j        |j        ¦  «        | _        d S )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads (ú)é
   r   r2   i   Tr¡   )ÚinplaceFÚrelative_coords_table)Ú
persistentÚrelative_position_index) rF   rG   Ú
ValueErrorÚnum_attention_headsr„   Úattention_head_sizeÚall_head_sizerŽ   r�   r�   r‘   r:   Úpretrained_window_sizer   rM   r    ÚlogÚonesÚlogit_scaleÚ
Sequentialr£   ÚReLUÚcontinuous_position_bias_mlpÚcreate_coords_table_and_indexÚregister_bufferÚqkv_biasÚqueryÚkeyÚvaluerU   Úattention_probs_dropout_probrW   )	rZ   rY   rf   Ú	num_headsr:   r»   r´   r¶   r\   s	           €r&   rG   zSwinv2SelfAttention.__init__h  s  ø€ Ý‰Œ×ÒÑÔÐØ�‰?˜aÒÐÝØk CÐkÐkÐ_hÐkÐkÐkñô ð ð $-ˆÔ Ý#& s¨Y¡Ñ#7Ô#7ˆÔ Ø!Ô5¸Ô8PÑPˆÔå% kµ;´?Ô3KÑLÔLÐlˆKˆKÐS^Ð`kÐRlð 	Ôð '=ˆÔ#Ýœ<­¬	°"µu´zÀ9ÈaÐQRÐBSÑ7TÔ7TÑ2TÑ(UÔ(UÑVÔVˆÔå,.¬MÝŒI�a˜ 4Ð(Ñ(Ô(­"¬'¸$Ð*?Ñ*?Ô*?ÅÄÈ3ÐPYÐ`eÐAfÑAfÔAfñ-
ô -
ˆÔ)ð :>×9[Ò9[Ñ9]Ô9]Ñ6ÐÐ6Ø×ÒÐ4Ð6KÐX]ÐÑ^Ô^Ð^Ø×ÒÐ6Ð8OÐ\aÐÑbÔbÐbå”Y˜tÔ1°4Ô3EÈFÌOÐ\Ñ\Ô\ˆŒ
Ý”9˜TÔ/°Ô1CÈ%ÐPÑPÔPˆŒÝ”Y˜tÔ1°4Ô3EÈFÌOÐ\Ñ\Ô\ˆŒ
Ý”z &Ô"EÑFÔFˆŒˆˆr%   NFr   Úattention_maskÚoutput_attentionsr^   c                 óž  — |j         \  }}}|                      |¦  «                             |d| j        | j        ¦  «                             dd¦  «        }|                      |¦  «                             |d| j        | j        ¦  «                             dd¦  «        }|                      |¦  «                             |d| j        | j        ¦  «                             dd¦  «        }	t          j	         
                    |d¬¦  «        t          j	         
                    |d¬¦  «                             dd¦  «        z  }
t          j        | j        t          j        d¦  «        ¬¦  «                             ¦   «         }|
|z  }
|                      | j        ¦  «                             d| j        ¦  «        }|| j                             d¦  «                                      | j        d         | j        d         z  | j        d         | j        d         z  d¦  «        }|                     ddd¦  «                             ¦   «         }d	t          j        |¦  «        z  }|
|                     d¦  «        z   }
|�Ÿ|j         d         }|
                     ||z  || j        ||¦  «        |                     d¦  «                             d¦  «        z   }
|
|                     d¦  «                             d¦  «        z   }
|
                     d| j        ||¦  «        }
t          j	                             |
d¬¦  «        }|                      |¦  «        }t          j        ||	¦  «        }|                     dddd
¦  «                             ¦   «         }|                     ¦   «         d d…         | j        fz   }|                     |¦  «        }|r||fn|f}|S )Nr3   r   r2   re   éþÿÿÿg      Y@)Úmaxr   é   r   )r5   rÅ   r6   r¸   r¹   r7   rÆ   rÇ   r   rk   Ú	normalizer    Úclampr¾   Úmathr¼   ÚexprÁ   r´   r¶   r:   rj   r8   Úsigmoidrz   ÚsoftmaxrW   Úmatmulrb   rº   )rZ   r   rÊ   rË   r;   rf   r>   Úquery_layerÚ	key_layerÚvalue_layerÚattention_scoresr¾   Úrelative_position_bias_tableÚrelative_position_biasÚ
mask_shapeÚattention_probsÚcontext_layerÚnew_context_layer_shapeÚoutputss                      r&   r�   zSwinv2SelfAttention.forward…  s¢  € ð )6Ô(;Ñ%ˆ
�C˜à�JŠJ�}Ñ%Ô%ßŠT�*˜b $Ô":¸DÔ<TÑUÔUßŠY�q˜!‰_Œ_ð 	ð �HŠH�]Ñ#Ô#ßŠT�*˜b $Ô":¸DÔ<TÑUÔUßŠY�q˜!‰_Œ_ð 	ð �JŠJ�}Ñ%Ô%ßŠT�*˜b $Ô":¸DÔ<TÑUÔUßŠY�q˜!‰_Œ_ð 	õ œ=×2Ò2°;ÀBÐ2ÑGÔGÍ"Ì-×JaÒJaØ˜2ð Kbñ K
ô K
ç
Š)�B˜Ñ
Ô
ñÐõ ”k $Ô"2½¼ÀÑ8LÔ8LÐMÑMÔM×QÒQÑSÔSˆØ+¨kÑ9ÐØ'+×'HÒ'HÈÔIcÑ'dÔ'd×'iÒ'iØ�Ô(ñ(
ô (
Ð$ð ">¸dÔ>Z×>_Ò>_Ð`bÑ>cÔ>cÔ!d×!iÒ!iØÔ˜QÔ $Ô"2°1Ô"5Ñ5°tÔ7GÈÔ7JÈTÔM]Ð^_ÔM`Ñ7`Ðbdñ"
ô "
Ðð "8×!?Ò!?ÀÀ1ÀaÑ!HÔ!H×!SÒ!SÑ!UÔ!UÐØ!#¥e¤mÐ4JÑ&KÔ&KÑ!KÐØ+Ð.D×.NÒ.NÈqÑ.QÔ.QÑQÐàÐ%à'Ô-¨aÔ0ˆJØ/×4Ò4Ø˜jÑ(¨*°dÔ6NÐPSÐUXñ ô  à×(Ò(¨Ñ+Ô+×5Ò5°aÑ8Ô8ñ 9Ðð  0°.×2JÒ2JÈ1Ñ2MÔ2M×2WÒ2WÐXYÑ2ZÔ2ZÑZÐØ/×4Ò4°R¸Ô9QÐSVÐX[Ñ\Ô\Ðõ œ-×/Ò/Ð0@ÀbÐ/ÑIÔIˆð Ÿ,š, Ñ7Ô7ˆõ œ _°kÑBÔBˆØ%×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔFˆØ"/×"4Ò"4Ñ"6Ô"6°s¸°sÔ";¸tÔ?QÐ>SÑ"SÐØ%×*Ò*Ð+BÑCÔCˆà6GÐ]�= /Ð2Ð2ÈmÐM]ˆàˆr%   c                 óP  — t          j        | j        d         dz
   | j        d         t           j        ¬¦  «                             ¦   «         }t          j        | j        d         dz
   | j        d         t           j        ¬¦  «                             ¦   «         }t          j        t          j        ||gd¬¦  «        ¦  «                             ddd¦  «                             ¦   «          	                    d¦  «        }| j
        d         dk    rQ|d d …d d …d d …dfxx         | j
        d         dz
  z  cc<   |d d …d d …d d …dfxx         | j
        d         dz
  z  cc<   na| j        d         dk    rP|d d …d d …d d …dfxx         | j        d         dz
  z  cc<   |d d …d d …d d …dfxx         | j        d         dz
  z  cc<   |dz  }t          j        |¦  «        t          j        t          j        |¦  «        dz   ¦  «        z  t          j        d¦  «        z  }|                     t!          | j                             ¦   «         ¦  «        j        ¦  «        }t          j        | j        d         ¦  «        }t          j        | j        d         ¦  «        }t          j        t          j        ||gd¬¦  «        ¦  «        }t          j        |d¦  «        }|d d …d d …d f         |d d …d d d …f         z
  }|                     ddd¦  «                             ¦   «         }|d d …d d …dfxx         | j        d         dz
  z  cc<   |d d …d d …dfxx         | j        d         dz
  z  cc<   |d d …d d …dfxx         d| j        d         z  dz
  z  cc<   |                     d	¦  «        }	||	fS )
Nr   r   ©ÚdtypeÚij)Úindexingr2   é   rx   r3   )r    Úaranger:   Úint64ÚfloatÚstackÚmeshgridrj   r8   rz   r»   ÚsignÚlog2ÚabsrÒ   ÚtoÚnextrÁ   Ú
parametersrä   rš   Úsum)
rZ   Úrelative_coords_hÚrelative_coords_wr´   Úcoords_hÚcoords_wÚcoordsÚcoords_flattenÚrelative_coordsr¶   s
             r&   rÂ   z1Swinv2SelfAttention.create_coords_table_and_indexÉ  sò  € å!œL¨4Ô+;¸AÔ+>ÀÑ+BÐ)CÀTÔEUÐVWÔEXÕ`eÔ`kÐlÑlÔl×rÒrÑtÔtÐÝ!œL¨4Ô+;¸AÔ+>ÀÑ+BÐ)CÀTÔEUÐVWÔEXÕ`eÔ`kÐlÑlÔl×rÒrÑtÔtÐåŒK�œÐ(9Ð;LÐ'MÐX\Ð]Ñ]Ô]Ñ^Ô^ßŠW�Q˜˜1ÑÔßŠZ‰\Œ\ßŠY�q‰\Œ\ð	 	ð Ô& qÔ)¨AÒ-Ð-Ø! ! ! ! Q Q Q¨¨¨¨1 *Ð-Ð-Ô-°Ô1LÈQÔ1OÐRSÑ1SÑSÐ-Ð-Ñ-Ø! ! ! ! Q Q Q¨¨¨¨1 *Ð-Ð-Ô-°Ô1LÈQÔ1OÐRSÑ1SÑSÐ-Ð-Ñ-Ð-ØÔ˜aÔ  1Ò$Ð$Ø! ! ! ! Q Q Q¨¨¨¨1 *Ð-Ð-Ô-°Ô1AÀ!Ô1DÀqÑ1HÑHÐ-Ð-Ñ-Ø! ! ! ! Q Q Q¨¨¨¨1 *Ð-Ð-Ô-°Ô1AÀ!Ô1DÀqÑ1HÑHÐ-Ð-Ñ-Ø Ñ"ÐåŒJÐ,Ñ-Ô-µ´
½5¼9ÐEZÑ;[Ô;[Ð^aÑ;aÑ0bÔ0bÑbÕeiÔenÐopÑeqÔeqÑqð 	ð !6× 8Ò 8½¸dÔ>_×>jÒ>jÑ>lÔ>lÑ9mÔ9mÔ9sÑ tÔ tÐõ ”< Ô 0°Ô 3Ñ4Ô4ˆÝ”< Ô 0°Ô 3Ñ4Ô4ˆÝ”�Uœ^¨X°xÐ,@È4ÐPÑPÔPÑQÔQˆÝœ v¨qÑ1Ô1ˆØ(¨¨¨¨A¨A¨A¨t¨Ô4°~ÀaÀaÀaÈÈqÈqÈqÀjÔ7QÑQˆØ)×1Ò1°!°Q¸Ñ:Ô:×EÒEÑGÔGˆØ˜˜˜˜1˜1˜1˜a˜Ð Ð Ô  DÔ$4°QÔ$7¸!Ñ$;Ñ;Ð Ð Ñ Ø˜˜˜˜1˜1˜1˜a˜Ð Ð Ô  DÔ$4°QÔ$7¸!Ñ$;Ñ;Ð Ð Ñ Ø˜˜˜˜1˜1˜1˜a˜Ð Ð Ô  A¨Ô(8¸Ô(;Ñ$;¸aÑ$?Ñ?Ð Ð Ñ Ø"1×"5Ò"5°bÑ"9Ô"9Ðà$Ð&=Ð=Ð=r%   rƒ   )r   r   r   rG   r    r   r!   r†   r#   r�   rÂ   r‡   rˆ   s   @r&   r¯   r¯   g  sÂ   ø€ € € € € ØTUÐWXÐSYð Gð Gð Gð Gð Gð Gð@ 48Ø).ð	Bð Bà”|ðBð Ô)¨DÑ0ðBð   $™;ð	Bð
 
ˆuŒ|Ô	ðBð Bð Bð BðH#>ð #>ð #>ð #>ð #>ð #>ð #>r%   r¯   c                   óP   ‡ — e Zd Zˆ fd„Zdej        dej        dej        fd„Zˆ xZS )ÚSwinv2SelfOutputc                 ó¸   •— t          ¦   «                              ¦   «          t          j        ||¦  «        | _        t          j        |j        ¦  «        | _        d S ©N)rF   rG   r   r£   ÚdenserU   rÈ   rW   ©rZ   rY   rf   r\   s      €r&   rG   zSwinv2SelfOutput.__init__ñ  sD   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜s CÑ(Ô(ˆŒ
Ý”z &Ô"EÑFÔFˆŒˆˆr%   r   Úinput_tensorr^   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rþ   ©rÿ   rW   )rZ   r   r  s      r&   r�   zSwinv2SelfOutput.forwardö  s*   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆàÐr%   ©r   r   r   rG   r    r   r�   r‡   rˆ   s   @r&   rü   rü   ð  sn   ø€ € € € € ðGð Gð Gð Gð Gð
 U¤\ð ÀÄð ÐRWÔR^ð ð ð ð ð ð ð ð r%   rü   c            
       ót   ‡ — e Zd Zd
ˆ fd„	Z	 	 ddej        dej        dz  dedz  deej                 fd	„Z	ˆ xZ
S )ÚSwinv2Attentionr   c           
      óê   •— t          ¦   «                              ¦   «          t          ||||t          |t          j        j        ¦  «        r|n||f¬¦  «        | _        t          ||¦  «        | _	        d S )N©rY   rf   rÉ   r:   r»   )
rF   rG   r¯   rŽ   r�   r�   r‘   rZ   rü   Úoutput)rZ   rY   rf   rÉ   r:   r»   r\   s         €r&   rG   zSwinv2Attention.__init__þ  sz   ø€ Ý‰Œ×ÒÑÔÐÝ'ØØØØ#åÐ0µ+´/Ô2JÑKÔKð$BÐ#9Ð#9à(Ð*@ÐAð
ñ 
ô 
ˆŒ	õ ' v¨sÑ3Ô3ˆŒˆˆr%   NFr   rÊ   rË   r^   c                 óˆ   — |                       |||¦  «        }|                      |d         |¦  «        }|f|dd …         z   }|S )Nr   r   )rZ   r	  )rZ   r   rÊ   rË   Úself_outputsÚattention_outputrá   s          r&   r�   zSwinv2Attention.forward  sM   € ð —y’y °Ð@QÑRÔRˆØŸ;š; |°A¤¸ÑFÔFÐØ#Ð%¨°Q°R°RÔ(8Ñ8ˆØˆr%   ©r   rƒ   )r   r   r   rG   r    r   r!   r†   r#   r�   r‡   rˆ   s   @r&   r  r  ý  s™   ø€ € € € € ð4ð 4ð 4ð 4ð 4ð 4ð  48Ø).ð		ð 	à”|ð	ð Ô)¨DÑ0ð	ð   $™;ð		ð
 
ˆuŒ|Ô	ð	ð 	ð 	ð 	ð 	ð 	ð 	ð 	r%   r  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚSwinv2Intermediatec                 ó$  •— t          ¦   «                              ¦   «          t          j        |t	          |j        |z  ¦  «        ¦  «        | _        t          |j        t          ¦  «        rt          |j                 | _        d S |j        | _        d S rþ   )rF   rG   r   r£   r„   Ú	mlp_ratiorÿ   rŽ   Ú
hidden_actÚstrr   Úintermediate_act_fnr   s      €r&   rG   zSwinv2Intermediate.__init__  sx   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜s¥C¨Ô(8¸3Ñ(>Ñ$?Ô$?Ñ@Ô@ˆŒ
Ý�fÔ'­Ñ-Ô-ð 	9Ý'-¨fÔ.?Ô'@ˆDÔ$Ð$Ð$à'-Ô'8ˆDÔ$Ð$Ð$r%   r   r^   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rþ   )rÿ   r  ©rZ   r   s     r&   r�   zSwinv2Intermediate.forward!  s,   € ØŸ
š
 =Ñ1Ô1ˆØ×0Ò0°Ñ?Ô?ˆØÐr%   r  rˆ   s   @r&   r  r    s^   ø€ € € € € ð9ð 9ð 9ð 9ð 9ð U¤\ð °e´lð ð ð ð ð ð ð ð r%   r  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚSwinv2Outputc                 óâ   •— t          ¦   «                              ¦   «          t          j        t	          |j        |z  ¦  «        |¦  «        | _        t          j        |j        ¦  «        | _	        d S rþ   )
rF   rG   r   r£   r„   r  rÿ   rU   rV   rW   r   s      €r&   rG   zSwinv2Output.__init__)  sT   ø€ Ý‰Œ×ÒÑÔÐÝ”Y�s 6Ô#3°cÑ#9Ñ:Ô:¸CÑ@Ô@ˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒˆˆr%   r   r^   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rþ   r  r  s     r&   r�   zSwinv2Output.forward.  s*   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØÐr%   r  rˆ   s   @r&   r  r  (  s^   ø€ € € € € ð>ð >ð >ð >ð >ð
 U¤\ð °e´lð ð ð ð ð ð ð ð r%   r  c                   ó^   ‡ — e Zd ZdZd
deddfˆ fd„Zdej        dej        fd„Zde	fd	„Z
ˆ xZS )ÚSwinv2DropPathzÏStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    ç        Ú	drop_probr^   Nc                 óV   •— t          ¦   «                              ¦   «          || _        d S rþ   )rF   rG   r  )rZ   r  r\   s     €r&   rG   zSwinv2DropPath.__init__<  s$   ø€ Ý‰Œ×ÒÑÔÐØ"ˆŒˆˆr%   r   c                 ó  — | j         dk    s| j        s|S d| j         z
  }|j        d         fd|j        dz
  z  z   }t	          j        ||j        |j        ¬¦  «        }t	          j        ||z   ¦  «        }| 	                    |¦  «        |z  S )Nr  r   r   )r   )rä   Údevice)
r  Útrainingr5   Úndimr    Úrandrä   r!  ÚfloorÚdiv)rZ   r   Ú	keep_probr5   Úrandom_tensors        r&   r�   zSwinv2DropPath.forward@  s“   € ØŒ>˜SÒ Ð ¨¬Ð Ø Ð Ø˜œÑ&ˆ	ØÔ$ QÔ'Ð)¨D°MÔ4FÈÑ4JÑ,KÑKˆÝœ
 5°Ô0CÈMÔL`ÐaÑaÔaˆÝœ M°IÑ$=Ñ>Ô>ˆØ× Ò  Ñ+Ô+¨mÑ;Ð;r%   c                 ó   — d| j         › �S )Nzp=)r  ©rZ   s    r&   Ú
extra_reprzSwinv2DropPath.extra_reprI  s   € Ø$�D”NÐ$Ð$Ð$r%   )r  )r   r   r   r   rê   rG   r    r   r�   r  r+  r‡   rˆ   s   @r&   r  r  5  s›   ø€ € € € € ðð ð#ð # %ð #°$ð #ð #ð #ð #ð #ð #ð< U¤\ð <°e´lð <ð <ð <ð <ð%˜Cð %ð %ð %ð %ð %ð %ð %ð %r%   r  c                   óÊ   ‡ — e Zd Z	 dˆ fd„	Zdeeeef         eeef         f         fd„Zd„ Zd„ Z	 dd	e	j
        d
eeef         dedz  dee	j
        e	j
        f         fd„Zˆ xZS )ÚSwinv2Layerr  r   c           
      óˆ  •— t          ¦   «                              ¦   «          || _        |                      |j        |j        f||f¦  «        \  }}|d         | _        |d         | _        t          |||| j        t          |t          j	        j
        ¦  «        r|n||f¬¦  «        | _        t          j        ||j        ¬¦  «        | _        |dk    rt!          |¦  «        nt          j        ¦   «         | _        t'          ||¦  «        | _        t+          ||¦  «        | _        t          j        ||j        ¬¦  «        | _        d S )Nr   r  ©Úepsr  )rF   rG   r�   Ú_compute_window_shiftr:   Ú
shift_sizer  rŽ   r�   r�   r‘   Ú	attentionr   rS   Úlayer_norm_epsÚlayernorm_beforer  ÚIdentityÚ	drop_pathr  Úintermediater  r	  Úlayernorm_after)
rZ   rY   rf   r�   rÉ   Údrop_path_rater2  r»   r:   r\   s
            €r&   rG   zSwinv2Layer.__init__N  s?  ø€ õ 	‰Œ×ÒÑÔÐØ 0ˆÔØ"&×"<Ò"<ØÔ Ô!3Ð4°zÀ:Ð6Nñ#
ô #
Ñˆ�Zð ' qœ>ˆÔØ$ Qœ-ˆŒÝ(ØØØØÔ(åÐ0µ+´/Ô2JÑKÔKð$BÐ#9Ð#9à(Ð*@ÐAð
ñ 
ô 
ˆŒõ !#¤¨S°fÔ6KÐ LÑ LÔ LˆÔØ;IÈCÒ;OÐ;O�¨Ñ7Ô7Ð7ÕUWÔU`ÑUbÔUbˆŒÝ.¨v°sÑ;Ô;ˆÔÝ" 6¨3Ñ/Ô/ˆŒÝ!œ|¨C°VÔ5JÐKÑKÔKˆÔÐÐr%   r^   c                 óˆ   — d„ t          | j        |¦  «        D ¦   «         }d„ t          | j        ||¦  «        D ¦   «         }||fS )Nc                 ó4   — g | ]\  }}t          ||¦  «        ‘ŒS r$   )Úmin)Ú.0ÚrÚws      r&   ú
<listcomp>z5Swinv2Layer._compute_window_shift.<locals>.<listcomp>h  s$   € Ð\Ð\Ð\¡T Q¨•s˜1˜a‘y”yÐ\Ð\Ð\r%   c                 ó*   — g | ]\  }}}||k    rd n|‘ŒS r  r$   )r>  r?  r@  Úss       r&   rA  z5Swinv2Layer._compute_window_shift.<locals>.<listcomp>i  s*   € ÐsÐsÐs©W¨Q°°1˜1 š6˜6�a�a qÐsÐsÐsr%   )Úzipr�   )rZ   Útarget_window_sizeÚtarget_shift_sizer:   r2  s        r&   r1  z!Swinv2Layer._compute_window_shiftg  sR   € Ø\Ð\­S°Ô1FÐHZÑ-[Ô-[Ð\Ñ\Ô\ˆØsÐs½¸DÔ<QÐS^Ð`qÑ8rÔ8rÐsÑsÔsˆ
Ø˜JÐ&Ð&r%   c                 ó  — | j         dk    rdS t          j        |¦  «        }t          j        |¦  «        }||| j        z
  k                         ¦   «         ||| j         z
  k                         ¦   «         z   }||| j        z
  k                         ¦   «         ||| j         z
  k                         ¦   «         z   }|ddd…ddf         dz  |dddd…df         z                        |¦  «        }t          || j        ¦  «        }	|	                     d| j        | j        z  ¦  «        }	|	                     d¦  «        |	                     d¦  «        z
  }
|
 	                    |
dk    d¦  «         	                    |
dk    d¦  «        }
|
S )	uµ  Build the cyclic-shift attention mask for shifted-window MSA; returns None when shift_size is 0.

        Each (h, w) position belongs to one of 9 cyclic-shift regions (3 along each axis), encoded
        as ``h_region * 3 + w_region``. Regions per axis:
        - 0: indices ``[0, axis - window_size)``
        - 1: indices ``[axis - window_size, axis - shift_size)``
        - 2: indices ``[axis - shift_size, axis)``
        Implementation note: a single arithmetic pass on `torch.arange` (two comparisons +
        broadcast add) replaces the original 9-iteration nested-Python-loop slice-assignment â€”
        fully vectorised, no per-cell host-side scatter, no GPUâ†”host sync.
        r   Nr   r3   r   r2   g      YÀr  )
r2  r    rè   r:   Úlongrð   r@   r6   rz   Úmasked_fill)rZ   r<   r=   rä   Úh_idxÚw_idxÚh_regionÚw_regionÚimg_maskÚmask_windowsÚ	attn_masks              r&   Úget_attn_maskzSwinv2Layer.get_attn_maskl  s~  € ð Œ?˜aÒÐØ�4Ý”˜VÑ$Ô$ˆÝ”˜UÑ#Ô#ˆØ˜V dÔ&6Ñ6Ò6×<Ò<Ñ>Ô>À%È6ÐTXÔTcÑKcÒBc×AiÒAiÑAkÔAkÑkˆØ˜U TÔ%5Ñ5Ò5×;Ò;Ñ=Ô=ÀÈ%ÐRVÔRaÑJaÒAa×@gÒ@gÑ@iÔ@iÑiˆØ˜T 1 1 1 d¨DÐ0Ô1°AÑ5¸ÀÀtÈQÈQÈQÐPTÐATÔ8UÑU×YÒYÐZ_Ñ`Ô`ˆÝ'¨°$Ô2BÑCÔCˆØ#×(Ò(¨¨TÔ-=ÀÔ@PÑ-PÑQÔQˆØ ×*Ò*¨1Ñ-Ô-°×0FÒ0FÀqÑ0IÔ0IÑIˆ	Ø×)Ò)¨)°qª.¸&ÑAÔA×MÒMÈiÐ[\ÊnÐ^aÑbÔbˆ	ØÐr%   c                 óÂ   — | j         || j         z  z
  | j         z  }| j         || j         z  z
  | j         z  }ddd|d|f}t          j                             ||¦  «        }||fS ©Nr   )r:   r   rk   r–   )rZ   r   r<   r=   Ú	pad_rightÚ
pad_bottomr—   s          r&   r˜   zSwinv2Layer.maybe_pad…  sp   € ØÔ%¨°Ô0@Ñ(@Ñ@ÀDÔDTÑTˆ	ØÔ&¨°$Ô2BÑ)BÑBÀdÔFVÑVˆ
Ø˜˜A˜y¨!¨ZÐ8ˆ
Ýœ×)Ò)¨-¸ÑDÔDˆØ˜jÐ(Ð(r%   Fr   r§   rË   Nc                 óô  — |\  }}|                      ¦   «         \  }}}|}	|                     ||||¦  «        }|                      |||¦  «        \  }}
|j        \  }}}}| j        dk    r&t          j        || j         | j         fd¬¦  «        }n|}t          || j        ¦  «        }|                     d| j        | j        z  |¦  «        }|  	                    |||j
        ¬¦  «        }|�|                     |j        ¦  «        }|                      |||¬¦  «        }|d         }|                     d| j        | j        |¦  «        }t          || j        ||¦  «        }| j        dk    r$t          j        || j        | j        fd¬¦  «        }n|}|
d         dk    p|
d         dk    }|r&|d d …d |…d |…d d …f                              ¦   «         }|                     |||z  |¦  «        }|                      |¦  «        }|	|                      |¦  «        z   }|                      |¦  «        }|                      |¦  «        }||                      |                      |¦  «        ¦  «        z   }|r
||d	         fn|f}|S )
Nr   )r   r2   )ÚshiftsÚdimsr3   rã   )rË   r   é   r   )rb   r6   r˜   r5   r2  r    Úrollr@   r:   rQ  rä   rð   r!  r3  rB   r8   r5  r7  r8  r	  r9  )rZ   r   r§   rË   r<   r=   r;   r|   ÚchannelsÚshortcutr—   Ú
height_padÚ	width_padÚshifted_hidden_statesÚhidden_states_windowsrP  Úattention_outputsr  Úattention_windowsÚshifted_windowsÚ
was_paddedÚlayer_outputÚlayer_outputss                          r&   r�   zSwinv2Layer.forwardŒ  sÂ  € ð )‰ˆ�Ø"/×"4Ò"4Ñ"6Ô"6Ñˆ
�A�xØ ˆð &×*Ò*¨:°v¸uÀhÑOÔOˆØ$(§N¢N°=À&È%Ñ$PÔ$PÑ!ˆ�zØ&3Ô&9Ñ#ˆˆ:�y !àŒ?˜QÒÐÝ$)¤J¨}ÀtÄÐFVÐY]ÔYhÐXhÐEiÐpvÐ$wÑ$wÔ$wÐ!Ð!à$1Ð!õ !1Ð1FÈÔHXÑ YÔ YÐØ 5× :Ò :¸2¸tÔ?OÐRVÔRbÑ?bÐdlÑ mÔ mÐØ×&Ò& z°9ÀMÔDWÐ&ÑXÔXˆ	ØÐ Ø!ŸšÐ%:Ô%AÑBÔBˆIà ŸNšNÐ+@À)Ð_p˜NÑqÔqÐà,¨QÔ/Ðà,×1Ò1°"°dÔ6FÈÔHXÐZbÑcÔcÐÝ(Ð):¸DÔ<LÈjÐZcÑdÔdˆð Œ?˜QÒÐÝ %¤
¨?ÀDÄOÐUYÔUdÐCeÐlrÐ sÑ sÔ sÐÐà /Ðà ”] QÒ&Ð;¨*°Q¬-¸!Ò*;ˆ
Øð 	VØ 1°!°!°!°W°f°W¸f¸u¸fÀaÀaÀaÐ2GÔ H× SÒ SÑ UÔ UÐà-×2Ò2°:¸vÈ¹~ÈxÑXÔXÐØ×-Ò-Ð.?Ñ@Ô@ˆØ  4§>¢>°-Ñ#@Ô#@Ñ@ˆà×(Ò(¨Ñ7Ô7ˆØ—{’{ <Ñ0Ô0ˆØ$ t§~¢~°d×6JÒ6JÈ<Ñ6XÔ6XÑ'YÔ'YÑYˆà@QÐf˜Ð'8¸Ô';Ð<Ð<ÐXdÐWfˆØÐr%   )r  r   r   r‚   )r   r   r   rG   r#   r„   r1  rQ  r˜   r    r   r†   r�   r‡   rˆ   s   @r&   r-  r-  M  s  ø€ € € € € àqrðLð Lð Lð Lð Lð Lð2'ÈeÐTYÐZ]Ð_bÐZbÔTcÐejÐknÐpsÐksÔetÐTtÔNuð 'ð 'ð 'ð 'ð
ð ð ð2)ð )ð )ð */ð	5ð 5à”|ð5ð    S œ/ð5ð   $™;ð	5ð
 
ˆuŒ|˜Uœ\Ð)Ô	*ð5ð 5ð 5ð 5ð 5ð 5ð 5ð 5r%   r-  c            
       ót   ‡ — e Zd Z	 d
ˆ fd„	Z	 ddej        deeef         dedz  deej                 fd	„Z	ˆ xZ
S )ÚSwinv2Stager   c	           
      ó¢  •— t          ¦   «                              ¦   «          || _        || _        g }	t	          |¦  «        D ]F}
t          ||||||
         |
dz  dk    rdn	|j        dz  |¬¦  «        }|	                     |¦  «         ŒGt          j	        |	¦  «        | _
        |� |||t          j        ¬¦  «        | _        nd | _        d| _        d S )Nr2   r   )rY   rf   r�   rÉ   r:  r2  r»   )rf   rž   F)rF   rG   rY   rf   Úranger-  r:   Úappendr   Ú
ModuleListÚblocksrS   Ú
downsampleÚpointing)rZ   rY   rf   r�   ÚdepthrÉ   r7  rn  r»   rm  ÚiÚblockr\   s               €r&   rG   zSwinv2Stage.__init__Å  sç   ø€ õ 	‰Œ×ÒÑÔÐØˆŒØˆŒØˆÝ�u‘”ð 
	!ð 
	!ˆAÝØØØ!1Ø#Ø(¨œ|Ø!" Q¡¨!¢ ˜1˜1°&Ô2DÈÑ2IØ'=ðñ ô ˆEð �MŠM˜%Ñ Ô Ð Ð Ý”m FÑ+Ô+ˆŒð Ð!Ø(˜jÐ)9¸sÍrÌ|Ð\Ñ\Ô\ˆDŒOˆOà"ˆDŒOàˆŒˆˆr%   Fr   r§   rË   Nr^   c                 ó  — |\  }}t          | j        ¦  «        D ]\  }} ||||¦  «        }|d         }Œ|}	| j        �-|dz   dz  |dz   dz  }}
|||
|f}|                      |	|¦  «        }n||||f}||	|f}|r||dd …         z  }|S )Nr   r   r2   )Ú	enumeraterm  rn  )rZ   r   r§   rË   r<   r=   rq  Úlayer_modulerf  Ú!hidden_states_before_downsamplingÚheight_downsampledÚwidth_downsampledr}   Ústage_outputss                 r&   r�   zSwinv2Stage.forwardá  så   € ð )‰ˆ�Ý(¨¬Ñ5Ô5ð 	-ð 	-‰OˆAˆ|Ø(˜LØØ Ø!ñô ˆMð *¨!Ô,ˆMˆMà,9Ð)ØŒ?Ð&Ø5;¸a±ZÀAÑ4EÈÐPQÉ	ÐVWÑGWÐ 1ÐØ!'¨Ð0BÐDUÐ VÐØ ŸOšOÐ,MÐO_Ñ`Ô`ˆMˆMà!'¨°¸Ð >Ðà&Ð(IÐK\Ð]ˆàð 	/Ø˜]¨1¨2¨2Ô.Ñ.ˆMØÐr%   r  r‚   )r   r   r   rG   r    r   r#   r„   r†   r�   r‡   rˆ   s   @r&   rh  rh  Ä  sš   ø€ € € € € àmnðð ð ð ð ð ð@ */ð	ð à”|ðð    S œ/ðð   $™;ð	ð
 
ˆuŒ|Ô	ðð ð ð ð ð ð ð r%   rh  c                   ó†   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej        deeef         dedz  d	edz  d
edz  dedz  dee	z  fd„Z
ˆ xZS )ÚSwinv2Encoder©r   r   r   r   c                 ó.  •— t          ¦   «                              ¦   «          t          |j        ¦  «        | _        || _        | j        j        �|j        }d„ t          j        d|j	        t          |j        ¦  «        d¬¦  «        D ¦   «         }g }t          | j        ¦  «        D ]Ð}t          |t          |j        d|z  z  ¦  «        |d         d|z  z  |d         d|z  z  f|j        |         |j        |         |t          |j        d |…         ¦  «        t          |j        d |dz   …         ¦  «        …         || j        dz
  k     rt           nd ||         ¬¦  «        }|                     |¦  «         ŒÑt%          j        |¦  «        | _        d| _        d S )	Nc                 ó6   — g | ]}|                      ¦   «         ‘ŒS r$   )Úitem)r>  Úxs     r&   rA  z*Swinv2Encoder.__init__.<locals>.<listcomp>  s    € ÐlÐlÐl˜Aˆq�vŠv‰xŒxÐlÐlÐlr%   r   Úcpu)r!  r2   r   )rY   rf   r�   rp  rÉ   r7  rn  r»   F)rF   rG   ÚlenÚdepthsÚ
num_layersrY   Úpretrained_window_sizesr    Úlinspacer:  ró   rj  rh  r„   rO   rÉ   rœ   rk  r   rl  ÚlayersÚgradient_checkpointing)	rZ   rY   rK   r…  Údprr‡  Úi_layerÚstager\   s	           €r&   rG   zSwinv2Encoder.__init__  s’  ø€ Ý‰Œ×ÒÑÔÐÝ˜fœmÑ,Ô,ˆŒØˆŒØŒ;Ô.Ð:Ø&,Ô&DÐ#ØlÐl¥¤°°6Ô3HÍ#ÈfÌmÑJ\ÔJ\ÐejÐ!kÑ!kÔ!kÐlÑlÔlˆàˆÝ˜Tœ_Ñ-Ô-ð 	!ð 	!ˆGÝØÝ˜Ô(¨1¨g©:Ñ5Ñ6Ô6Ø"+¨A¤,°1°g±:Ñ">À	È!ÄÐQRÐT[ÑQ[Ñ@\Ð!]Ø”m GÔ,Ø Ô*¨7Ô3Ø�c &¤-°°°Ô"9Ñ:Ô:½SÀÄÈ}ÐQXÐ[\ÑQ\È}ÔA]Ñ=^Ô=^Ð^Ô_Ø29¸D¼OÈaÑ<OÒ2OÐ2OÕ-Ð-ÐVZØ'>¸wÔ'Gð	ñ 	ô 	ˆEð �MŠM˜%Ñ Ô Ð Ð Ý”m FÑ+Ô+ˆŒà&+ˆÔ#Ð#Ð#r%   FTr   r§   rË   NÚoutput_hidden_statesÚ(output_hidden_states_before_downsamplingÚreturn_dictr^   c                 óò  — |rdnd }|rdnd }|rdnd }	|r?|j         \  }
}} |j        |
g|¢|‘R Ž }|                     dddd¦  «        }||fz  }||fz  }t          | j        ¦  «        D ]Þ\  }} ||||¦  «        }|d         }|d         }|d         }|d         |d         f}|rP|rN|j         \  }
}} |j        |
g|d         |d         f¢|‘R Ž }|                     dddd¦  «        }||fz  }||fz  }nC|rA|s?|j         \  }
}} |j        |
g|¢|‘R Ž }|                     dddd¦  «        }||fz  }||fz  }|r|	|dd …         z  }	Œß|st          d„ |||	|fD ¦   «         ¦  «        S t          |||	|¬	¦  «        S )
Nr$   r   r   r   r2   rÍ   r3   c              3   ó   K  — | ]}|®|V — Œ	d S rþ   r$   )r>  Úvs     r&   ú	<genexpr>z(Swinv2Encoder.forward.<locals>.<genexpr>R  s0   è è € ð ð àØ�=ð à �=�=�=ðð r%   )r   r   r   r   )r5   r6   rj   rt  r‡  r#   r   )rZ   r   r§   rË   rŒ  r�  rŽ  Úall_hidden_statesÚall_reshaped_hidden_statesÚall_self_attentionsr;   r|   r”   Úreshaped_hidden_staterq  ru  rf  rv  r}   s                      r&   r�   zSwinv2Encoder.forward  s®  € ð #7Ð@˜B˜B¸DÐØ+?Ð%I R RÀTÐ"Ø$5Ð?˜b˜b¸4Ðàð 	CØ)6Ô)<Ñ&ˆJ˜˜;à$6 MÔ$6°zÐ$bÐDTÐ$bÐVaÐ$bÐ$bÐ$bÐ!Ø$9×$AÒ$AÀ!ÀQÈÈ1Ñ$MÔ$MÐ!Ø -Ð!1Ñ1ÐØ&Ð+@Ð*BÑBÐ&å(¨¬Ñ5Ô5ð  	9ð  	9‰OˆAˆ|Ø(˜LØØ Ø!ñô ˆMð *¨!Ô,ˆMØ0=¸aÔ0@Ð-Ø -¨aÔ 0Ðà 1°"Ô 5Ð7HÈÔ7LÐMÐà#ð GÐ(Pð GØ-NÔ-TÑ*�
˜A˜{ð )OÐ(IÔ(NØð)Ø"3°AÔ"6Ð8IÈ!Ô8LÐ!Mð)ØOZð)ð )ð )Ð%ð )>×(EÒ(EÀaÈÈAÈqÑ(QÔ(QÐ%Ø!Ð&GÐ%IÑIÐ!Ø*Ð/DÐ.FÑFÐ*Ð*Ø%ð GÐ.Vð GØ-:Ô-@Ñ*�
˜A˜{à(:¨Ô(:¸:Ð(fÐHXÐ(fÐZeÐ(fÐ(fÐ(fÐ%Ø(=×(EÒ(EÀaÈÈAÈqÑ(QÔ(QÐ%Ø! mÐ%5Ñ5Ð!Ø*Ð/DÐ.FÑFÐ*à ð 9Ø# }°Q°R°RÔ'8Ñ8Ð#øàð 	Ýð ð à'Ð):Ð<OÐQkÐlðñ ô ñ ô ð õ #Ø+Ø+Ø*Ø#=ð	
ñ 
ô 
ð 	
r%   )r|  )FFFT)r   r   r   rG   r    r   r#   r„   r†   r   r�   r‡   rˆ   s   @r&   r{  r{     sØ   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð ,ð: */Ø,1Ø@EØ#'ðC
ð C
à”|ðC
ð    S œ/ðC
ð   $™;ð	C
ð
 # T™kðC
ð 37¸±+ðC
ð ˜D‘[ðC
ð 
Ð$Ñ	$ðC
ð C
ð C
ð C
ð C
ð C
ð C
ð C
r%   r{  c                   ój   ‡ — e Zd ZU eed<   dZdZdZdZdgZ	 e
j        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚSwinv2PreTrainedModelrY   Úswinv2ru   )ÚimageTrh  c                 ó  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rD|j        �t          j        |j        ¦  «         |j        �t          j        |j        ¦  «         dS dS t          |t          ¦  «        ryt          j	        |j
        t          j        d¦  «        ¦  «         |                     ¦   «         \  }}t          j        |j        |¦  «         t          j        |j        |¦  «         dS dS )zInitialize the weightsNr²   )rF   Ú_init_weightsrŽ   rD   rP   ÚinitÚzeros_rR   r¯   Ú	constant_r¾   rÒ   r¼   rÂ   Úcopy_r´   r¶   )rZ   Úmoduler´   r¶   r\   s       €r&   rœ  z#Swinv2PreTrainedModel._init_weightsi  sþ   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ.Ñ/Ô/ð 		PØÔ Ð,Ý”˜FÔ-Ñ.Ô.Ð.ØÔ)Ð5Ý”˜FÔ6Ñ7Ô7Ð7Ð7Ð7ð 6Ð5å˜Õ 3Ñ4Ô4ð 	PÝŒN˜6Ô-­t¬x¸©|¬|Ñ<Ô<Ð<Ø=C×=aÒ=aÑ=cÔ=cÑ:Ð!Ð#:ÝŒJ�vÔ3Ð5JÑKÔKÐKÝŒJ�vÔ5Ð7NÑOÔOÐOÐOÐOð		Pð 	Pr%   )r   r   r   r   r"   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesr    Úno_gradrœ  r‡   rˆ   s   @r&   r˜  r˜  `  s€   ø€ € € € € € àÐÐÑØ ÐØ$€OØ!ÐØ&*Ð#Ø&˜Ðà€U„]�_„_ðPð Pð Pð Pñ „_ðPð Pð Pð Pð Pr%   r˜  c                   ó    ‡ — e Zd Zdˆ fd„	Zd„ Ze	 	 	 	 	 	 ddej        dz  dej        dz  de	dz  d	e	dz  d
e	de	dz  de
ez  fd„¦   «         Zˆ xZS )ÚSwinv2ModelTFc                 óî  •— t          ¦   «                              |¦  «         || _        t          |j        ¦  «        | _        t          |j        d| j        dz
  z  z  ¦  «        | _        t          ||¬¦  «        | _
        t          || j
        j        ¦  «        | _        t          j        | j        |j        ¬¦  «        | _        |rt          j        d¦  «        nd| _        |                      ¦   «          dS )a  
        add_pooling_layer (`bool`, *optional*, defaults to `True`):
            Whether or not to apply pooling layer.
        use_mask_token (`bool`, *optional*, defaults to `False`):
            Whether or not to create and apply mask tokens in the embedding layer.
        r2   r   )r[   r/  N)rF   rG   rY   r‚  rƒ  r„  r„   rO   Únum_featuresrD   r]   r{  rL   Úencoderr   rS   r4  Ú	layernormÚAdaptiveAvgPool1dÚpoolerÚ	post_init)rZ   rY   Úadd_pooling_layerr[   r\   s       €r&   rG   zSwinv2Model.__init__|  sÓ   ø€ õ 	‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ˜fœmÑ,Ô,ˆŒÝ Ô 0°1¸¼È1Ñ9LÑ3MÑ MÑNÔNˆÔå*¨6À.ÐQÑQÔQˆŒÝ$ V¨T¬_Ô-GÑHÔHˆŒåœ dÔ&7¸VÔ=RÐSÑSÔSˆŒØ1BÐL•bÔ*¨1Ñ-Ô-Ð-ÈˆŒð 	�ŠÑÔÐÐÐr%   c                 ó   — | j         j        S rþ   ©r]   rI   r*  s    r&   Úget_input_embeddingsz Swinv2Model.get_input_embeddings‘  ó   € ØŒÔ/Ð/r%   Nru   rv   rË   rŒ  rt   rŽ  r^   c                 ó"  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|€t	          d¦  «        ‚|                      |||¬¦  «        \  }}	|                      ||	|||¬¦  «        }
|
d         }|                      |¦  «        }d}| j        �>|                      | 	                    dd¦  «        ¦  «        }t          j        |d¦  «        }|s||f|
dd…         z   }|S t          |||
j        |
j        |
j        ¬¦  «        S )	zË
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        Nz You have to specify pixel_values)rv   rt   )rË   rŒ  rŽ  r   r   r2   )r   r)   r   r   r   )rY   rË   rŒ  rŽ  r·   r]   r¬  r­  r¯  r7   r    rš   r(   r   r   r   )rZ   ru   rv   rË   rŒ  rt   rŽ  ÚkwargsÚembedding_outputr§   Úencoder_outputsÚsequence_outputÚpooled_outputr	  s                 r&   r�   zSwinv2Model.forward”  s_  € ð 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàÐÝÐ?Ñ@Ô@Ð@à-1¯_ª_Ø¨/ÐTlð .=ñ .
ô .
Ñ*ÐÐ*ð Ÿ,š,ØØØ/Ø!5Ø#ð 'ñ 
ô 
ˆð *¨!Ô,ˆØŸ.š.¨Ñ9Ô9ˆàˆØŒ;Ð"Ø ŸKšK¨×(AÒ(AÀ!ÀQÑ(GÔ(GÑHÔHˆMÝ!œM¨-¸Ñ;Ô;ˆMàð 	Ø% }Ð5¸ÈÈÈÔ8KÑKˆFàˆMå Ø-Ø'Ø)Ô7Ø&Ô1Ø#2Ô#Ið
ñ 
ô 
ð 	
r%   )TF©NNNNFN)r   r   r   rG   r´  r   r    r!   r…   r†   r#   r(   r�   r‡   rˆ   s   @r&   r©  r©  y  së   ø€ € € € € ðð ð ð ð ð ð*0ð 0ð 0ð ð 26Ø37Ø)-Ø,0Ø).Ø#'ð6
ð 6
àÔ'¨$Ñ.ð6
ð Ô)¨DÑ0ð6
ð   $™;ð	6
ð
 # T™kð6
ð #'ð6
ð ˜D‘[ð6
ð 
Ð"Ñ	"ð6
ð 6
ð 6
ñ „^ð6
ð 6
ð 6
ð 6
ð 6
r%   r©  a~  
        Swinv2 Model with a decoder on top for masked image modeling, as proposed in
    [SimMIM](https://huggingface.co/papers/2111.09886).

        <Tip>

        Note that we provide a script to pre-train this model on custom data in our [examples
        directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining).

        </Tip>
    c                   ó˜   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 ddej        dz  dej        dz  dedz  dedz  ded	edz  d
e	e
z  fd„¦   «         Zˆ xZS )ÚSwinv2ForMaskedImageModelingc                 ó�  •— t          ¦   «                              |¦  «         t          |dd¬¦  «        | _        t	          |j        d|j        dz
  z  z  ¦  «        }t          j        t          j	        ||j
        dz  |j        z  d¬¦  «        t          j        |j
        ¦  «        ¦  «        | _        |                      ¦   «          d S )NFT)r±  r[   r2   r   )Úin_channelsÚout_channelsr‹   )rF   rG   r©  r™  r„   rO   r„  r   r¿   r’   Úencoder_strider>   ÚPixelShuffleÚdecoderr°  )rZ   rY   r«  r\   s      €r&   rG   z%Swinv2ForMaskedImageModeling.__init__Ý  s¾   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å! &¸EÐRVÐWÑWÔWˆŒå˜6Ô+¨a°FÔ4EÈÑ4IÑ.JÑJÑKÔKˆÝ”}ÝŒIØ(°vÔ7LÈaÑ7OÐRXÔReÑ7eÐstðñ ô õ ŒO˜FÔ1Ñ2Ô2ñ	
ô 
ˆŒð 	�ŠÑÔÐÐÐr%   NFru   rv   rË   rŒ  rt   rŽ  r^   c                 óš  — |�|n| j         j        }|                      ||||||¬¦  «        }|d         }	|	                     dd¦  «        }	|	j        \  }
}}t          j        |dz  ¦  «        x}}|	                     |
|||¦  «        }	|                      |	¦  «        }d}|�ñ| j         j	        | j         j
        z  }|                     d||¦  «        }|                     | j         j
        d¦  «                             | j         j
        d¦  «                             d¦  «                             ¦   «         }t          j                             ||d¬	¦  «        }||z                       ¦   «         |                     ¦   «         d
z   z  | j         j        z  }|s|f|dd…         z   }|�|f|z   n|S t'          |||j        |j        |j        ¬¦  «        S )a�  
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

        Examples:
        ```python
        >>> from transformers import AutoImageProcessor, Swinv2ForMaskedImageModeling
        >>> import torch
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/swinv2-tiny-patch4-window8-256")
        >>> model = Swinv2ForMaskedImageModeling.from_pretrained("microsoft/swinv2-tiny-patch4-window8-256")

        >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
        >>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
        >>> # create random boolean mask of shape (batch_size, num_patches)
        >>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

        >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
        >>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction
        >>> list(reconstructed_pixel_values.shape)
        [1, 3, 256, 256]
        ```N)rv   rË   rŒ  rt   rŽ  r   r   r2   r`   r3   Únone)r¤   gñhãˆµøä>)r,   r-   r   r   r   )rY   rŽ  r™  r7   r5   rÒ   r%  ri   rÄ  r�   rX   Úrepeat_interleaverz   r8   r   rk   Úl1_lossró   r>   r+   r   r   r   )rZ   ru   rv   rË   rŒ  rt   rŽ  r·  rá   rº  r;   r>   Úsequence_lengthr<   r=   Úreconstructed_pixel_valuesÚmasked_im_lossrb   r€   Úreconstruction_lossr	  s                        r&   r�   z$Swinv2ForMaskedImageModeling.forwardí  sð  € ðP &1Ð%<�k�kÀ$Ä+ÔBYˆà—+’+ØØ+Ø/Ø!5Ø%=Ø#ð ñ 
ô 
ˆð " !œ*ˆà)×3Ò3°A°qÑ9Ô9ˆØ4CÔ4IÑ1ˆ
�L /Ýœ O°SÑ$8Ñ9Ô9Ð9ˆ�Ø)×1Ò1°*¸lÈFÐTYÑZÔZˆð &*§\¢\°/Ñ%BÔ%BÐ"àˆØÐ&Ø”;Ô)¨T¬[Ô-CÑCˆDØ-×5Ò5°b¸$ÀÑEÔEˆOà×1Ò1°$´+Ô2HÈ!ÑLÔLß"Ò" 4¤;Ô#9¸1Ñ=Ô=ß’˜1‘”ß’‘”ð	 õ #%¤-×"7Ò"7¸ÐF`ÐlrÐ"7Ñ"sÔ"sÐØ1°DÑ8×=Ò=Ñ?Ô?À4Ç8Â8Á:Ä:ÐPTÑCTÑUÐX\ÔXcÔXpÑpˆNàð 	ZØ0Ð2°W¸Q¸R¸R´[Ñ@ˆFØ3AÐ3M�^Ð%¨Ñ.Ð.ÐSYÐYå.ØØ5Ø!Ô/ØÔ)Ø#*Ô#Að
ñ 
ô 
ð 	
r%   r¼  )r   r   r   rG   r   r    r!   r…   r†   r#   r+   r�   r‡   rˆ   s   @r&   r¾  r¾  Î  sç   ø€ € € € € ðð ð ð ð ð  ð 26Ø37Ø)-Ø,0Ø).Ø#'ðS
ð S
àÔ'¨$Ñ.ðS
ð Ô)¨DÑ0ðS
ð   $™;ð	S
ð
 # T™kðS
ð #'ðS
ð ˜D‘[ðS
ð 
Ð0Ñ	0ðS
ð S
ð S
ñ „^ðS
ð S
ð S
ð S
ð S
r%   r¾  aæ  
    Swinv2 Model transformer with an image classification head on top (a linear layer on top of the final hidden state
    of the [CLS] token) e.g. for ImageNet.

    <Tip>

        Note that it's possible to fine-tune SwinV2 on higher resolution images than the ones it has been trained on, by
        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
        position embeddings to the higher resolution.

    </Tip>
    c                   ó˜   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 ddej        dz  dej        dz  dedz  dedz  ded	edz  d
e	e
z  fd„¦   «         Zˆ xZS )ÚSwinv2ForImageClassificationc                 ó@  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        |j        dk    r$t          j        | j        j        |j        ¦  «        nt          j        ¦   «         | _	        |  
                    ¦   «          d S rS  )rF   rG   Ú
num_labelsr©  r™  r   r£   r«  r6  Ú
classifierr°  ©rZ   rY   r\   s     €r&   rG   z%Swinv2ForImageClassification.__init__T  s‹   ø€ Ý‰Œ×Ò˜Ñ Ô Ð à Ô+ˆŒÝ! &Ñ)Ô)ˆŒð GMÔFWÐZ[ÒF[ÐF[�BŒI�d”kÔ.°Ô0AÑBÔBÐBÕacÔalÑanÔanð 	Œð
 	�ŠÑÔÐÐÐr%   NFru   ÚlabelsrË   rŒ  rt   rŽ  r^   c                 óJ  — |�|n| j         j        }|                      |||||¬¦  «        }|d         }	|                      |	¦  «        }
d}|�|                      ||
| j         ¦  «        }|s|
f|dd…         z   }|�|f|z   n|S t          ||
|j        |j        |j        ¬¦  «        S )aŠ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N)rË   rŒ  rt   rŽ  r   r2   )r,   r0   r   r   r   )	rY   rŽ  r™  rÑ  Úloss_functionr/   r   r   r   )rZ   ru   rÓ  rË   rŒ  rt   rŽ  r·  rá   r»  r0   r,   r	  s                r&   r�   z$Swinv2ForImageClassification.forwardb  sá   € ð" &1Ð%<�k�kÀ$Ä+ÔBYˆà—+’+ØØ/Ø!5Ø%=Ø#ð ñ 
ô 
ˆð   œ
ˆà—’ Ñ/Ô/ˆàˆØÐØ×%Ò% f¨f°d´kÑBÔBˆDàð 	FØ�Y ¨¨¨¤Ñ,ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå*ØØØ!Ô/ØÔ)Ø#*Ô#Að
ñ 
ô 
ð 	
r%   r¼  )r   r   r   rG   r   r    r!   Ú
LongTensorr†   r#   r/   r�   r‡   rˆ   s   @r&   rÎ  rÎ  D  s×   ø€ € € € € ð ð ð ð ð ð ð 26Ø*.Ø)-Ø,0Ø).Ø#'ð,
ð ,
àÔ'¨$Ñ.ð,
ð Ô  4Ñ'ð,
ð   $™;ð	,
ð
 # T™kð,
ð #'ð,
ð ˜D‘[ð,
ð 
Ð,Ñ	,ð,
ð ,
ð ,
ñ „^ð,
ð ,
ð ,
ð ,
ð ,
r%   rÎ  zO
    Swinv2 backbone, to be used with frameworks like DETR and MaskFormer.
    c                   óŠ   ‡ — e Zd Zˆ fd„Zd„ Zeee	 	 	 d
dede	dz  de	dz  de	dz  de
f
d	„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚSwinv2Backbonec                 óP  •‡— t          ¦   «                              ‰¦  «         ‰j        gˆfd„t          t	          ‰j        ¦  «        ¦  «        D ¦   «         z   | _        t          ‰¦  «        | _        t          ‰| j        j
        ¦  «        | _        |                      ¦   «          d S )Nc                 óD   •— g | ]}t          ‰j        d |z  z  ¦  «        ‘ŒS )r2   )r„   rO   )r>  rq  rY   s     €r&   rA  z+Swinv2Backbone.__init__.<locals>.<listcomp>›  s.   ø€ Ð1rÐ1rÐ1rÐSTµ#°fÔ6FÈÈAÉÑ6MÑ2NÔ2NÐ1rÐ1rÐ1rr%   )rF   rG   rO   rj  r‚  rƒ  r«  rD   r]   r{  rL   r¬  r°  rÒ  s    `€r&   rG   zSwinv2Backbone.__init__˜  s—   øø€ Ý‰Œ×Ò˜Ñ Ô Ð à#Ô-Ð.Ð1rÐ1rÐ1rÐ1rÕX]Õ^aÐbhÔboÑ^pÔ^pÑXqÔXqÐ1rÑ1rÔ1rÑrˆÔÝ*¨6Ñ2Ô2ˆŒÝ$ V¨T¬_Ô-GÑHÔHˆŒð 	�ŠÑÔÐÐÐr%   c                 ó   — | j         j        S rþ   r³  r*  s    r&   r´  z#Swinv2Backbone.get_input_embeddings¢  rµ  r%   Nru   rË   rŒ  rŽ  r^   c                 óÈ  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|                      |¦  «        \  }}|                      |||dd|¬¦  «        }|r|j        n|d         }	d}
t          | j        |	¦  «        D ]\  }}|| j	        v r|
|fz  }
Œ|s!|
f}|r||d         fz  }|r||d         fz  }|S t          |
|r|j        nd|j        ¬¦  «        S )	aœ  
        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, AutoBackbone
        >>> import torch
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> processor = AutoImageProcessor.from_pretrained("microsoft/swinv2-tiny-patch4-window8-256")
        >>> model = AutoBackbone.from_pretrained(
        ...     "microsoft/swinv2-tiny-patch4-window8-256", out_features=["stage1", "stage2", "stage3", "stage4"]
        ... )

        >>> inputs = processor(image, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> feature_maps = outputs.feature_maps
        >>> list(feature_maps[-1].shape)
        [1, 2048, 7, 7]
        ```NT)rË   rŒ  r�  rŽ  r3   r$   r   r2   )Úfeature_mapsr   r   )rY   rŽ  rŒ  rË   r]   r¬  r   rD  Ústage_namesÚout_featuresr   r   r   )rZ   ru   rË   rŒ  rŽ  r·  r¸  r§   rá   r   rÝ  r‹  Úhidden_stater	  s                 r&   r�   zSwinv2Backbone.forward¥  s\  € ðJ &1Ð%<�k�kÀ$Ä+ÔBYˆà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà-1¯_ª_¸\Ñ-JÔ-JÑ*ÐÐ*à—,’,ØØØ/Ø!%Ø59Ø#ð ñ 
ô 
ˆð ;FÐV˜Ô6Ð6È7ÐSUÌ;ˆàˆÝ#& tÔ'7¸Ñ#GÔ#Gð 	0ð 	0ÑˆE�<Ø˜Ô)Ð)Ð)Ø  Ñ/�øàð 	Ø"�_ˆFØ#ð (Ø˜7 1œ:˜-Ñ'�Ø ð (Ø˜7 1œ:˜-Ñ'�ØˆMåØ%Ø3GÐQ˜'Ô/Ð/ÈTØÔ)ð
ñ 
ô 
ð 	
r%   )NNN)r   r   r   rG   r´  r   r
   r   r   r†   r   r�   r‡   rˆ   s   @r&   rØ  rØ  ’  s×   ø€ € € € € ðð ð ð ð ð0ð 0ð 0ð Ø Øð *.Ø,0Ø#'ðF
ð F
àðF
ð   $™;ðF
ð # T™kð	F
ð
 ˜D‘[ðF
ð 
ðF
ð F
ð F
ñ „^ñ !Ô ñ ÔðF
ð F
ð F
ð F
ð F
r%   rØ  )rÎ  r¾  r©  r˜  rØ  )<r   Úcollections.abcr�   rÒ   Údataclassesr   r    r   r   Ú r   r�  Úactivationsr   Úbackbone_utilsr	   r
   Úmodeling_layersr   Úmodeling_outputsr   Úmodeling_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úconfiguration_swinv2r   Ú
get_loggerr   Úloggerr   r(   r+   r/   r@   rB   r­   rD   rH   rœ   r¯   rü   r  r  r  r  r-  rh  r{  r˜  r©  r¾  rÎ  rØ  Ú__all__r$   r%   r&   ú<module>rï     s›  ðð (Ð 'à Ð Ð Ð Ø €€€Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø HÐ HÐ HÐ HÐ HÐ HÐ HÐ HØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø .Ð .Ð .Ð .Ð .Ð .Ø -Ð -Ð -Ð -Ð -Ð -Ø DÐ DÐ DÐ DÐ DÐ DÐ DÐ DÐ DÐ DÐ DÐ DØ -Ð -Ð -Ð -Ð -Ð -Ø .Ð .Ð .Ð .Ð .Ð .ð 
ˆÔ	˜HÑ	%Ô	%€ð €ððñ ô ð
 ðHð Hð Hð Hð H˜+ñ Hô Hñ „ñô ðHð  €ððñ ô ð
 ðHð Hð Hð Hð H˜ñ Hô Hñ „ñô ðHð& €ððñ ô ð
 ðHð Hð Hð Hð H kñ Hô Hñ „ñô ðHð* €ððñ ô ð
 ðHð Hð Hð Hð H +ñ Hô Hñ „ñô ðHð,	ð 	ð 	ðð ð ðY-ð Y-ð Y-ð Y-ð Y-�r”yñ Y-ô Y-ð Y-ðz'-ð '-ð '-ð '-ð '-˜BœIñ '-ô '-ð '-ðT3ð 3ð 3ð 3ð 3˜œñ 3ô 3ð 3ðlE>ð E>ð E>ð E>ð E>˜"œ)ñ E>ô E>ð E>ðR
ð 
ð 
ð 
ð 
�r”yñ 
ô 
ð 
ðð ð ð ð �b”iñ ô ð ð6ð ð ð ð ˜œñ ô ð ð 	ð 	ð 	ð 	ð 	�2”9ñ 	ô 	ð 	ð%ð %ð %ð %ð %�R”Yñ %ô %ð %ð0tð tð tð tð t�"”)ñ tô tð tðn9ð 9ð 9ð 9ð 9Ð,ñ 9ô 9ð 9ðx]
ð ]
ð ]
ð ]
ð ]
�B”Iñ ]
ô ]
ð ]
ð@ ðPð Pð Pð Pð P˜Oñ Pô Pñ „ðPð0 ðP
ð P
ð P
ð P
ð P
Ð'ñ P
ô P
ñ „ðP
ðf €ð
ðñ ô ðe
ð e
ð e
ð e
ð e
Ð#8ñ e
ô e
ñô ðe
ðP €ððñ ô ð<
ð <
ð <
ð <
ð <
Ð#8ñ <
ô <
ñô ð<
ð~ €ððñ ô ð
W
ð W
ð W
ð W
ð W
�]Ð$9ñ W
ô W
ñô ð
W
ðtð ð €€€r%   