§
    ‚ŠtjÕã  ã            
       ó€  — d Z ddlZddlmZ ddlZddlmZ ddlmZ ddl	m
Z
 ddlmZmZ dd	lmZ dd
lmZmZmZmZmZ ddlmZ ddlmZmZmZ ddlmZ ddlmZ  ej         e!¦  «        Z" ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z# ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z$ ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z% ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z& G d„ dej'        ¦  «        Z( G d„ d ej'        ¦  «        Z) G d!„ d"ej'        ¦  «        Z* G d#„ d$ej'        ¦  «        Z+ G d%„ d&ej'        ¦  «        Z, G d'„ d(ej'        ¦  «        Z- G d)„ d*e¦  «        Z.d+ej/        d,e0e1         d-e0e1         d.ej/        fd/„Z2 G d0„ d1ej'        ¦  «        Z3d+ej/        d2e4e1e1f         d3e4e1e1f         d4e0e0e1                  d.ej/        f
d5„Z5e G d6„ d7e¦  «        ¦   «         Z6 G d8„ d9ej'        ¦  «        Z7e G d:„ d;e6¦  «        ¦   «         Z8 G d<„ d=ej'        ¦  «        Z9 G d>„ d?ej'        ¦  «        Z: ed@¬¦  «         G dA„ dBe6¦  «        ¦   «         Z; edC¬¦  «         G dD„ dEe6¦  «        ¦   «         Z< edF¬¦  «         G dG„ dHee6¦  «        ¦   «         Z=g dI¢Z>dS )JzPyTorch Hiera model.é    N)Ú	dataclass)Únné   )Úinitialization)ÚACT2FN)ÚBackboneMixinÚfilter_output_hidden_states)ÚGradientCheckpointingLayer)ÚBackboneOutputÚBaseModelOutputÚBaseModelOutputWithPoolingÚImageClassifierOutputÚModelOutput)ÚPreTrainedModel)Úauto_docstringÚloggingÚ	torch_int)Úcan_return_tupleé   )ÚHieraConfigzO
    Hiera encoder's outputs, with potential hidden states and attentions.
    )Úcustom_introc                   ó¼   — e Zd ZU dZdZej        dz  ed<   dZe	ej        df         dz  ed<   dZ
e	ej        df         dz  ed<   dZe	ej        df         dz  ed<   dS )ÚHieraEncoderOutputa.  
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, height, width, hidden_size)`. These are the reshaped and re-rolled hidden states of the model.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    NÚlast_hidden_state.Úhidden_statesÚ
attentionsÚreshaped_hidden_states)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚFloatTensorÚ__annotations__r   Útupler   r   © ó    úf/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/hiera/modeling_hiera.pyr   r   *   sž   € € € € € € ðð ð 37Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø:>€M�5˜Ô*¨CÐ/Ô0°4Ñ7Ð>Ð>Ñ>Ø7;€J��eÔ'¨Ð,Ô-°Ñ4Ð;Ð;Ñ;ØCGÐ˜E %Ô"3°SÐ"8Ô9¸DÑ@ÐGÐGÑGÐGÐGr'   r   zW
    Hiera model's outputs that also contains a pooling of the last hidden states.
    c                   ó  — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej
        dz  ed<   dZej        dz  ed<   dZeej        df         dz  ed<   dZeej        df         dz  ed	<   dZeej        df         dz  ed
<   dS )ÚHieraModelOutputa1  
    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`, *optional*, returned when `add_pooling_layer=True` is passed):
        Average pooling of the last layer hidden-state.
    bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, sequence_length)`):
        Tensor indicating which patches are masked (0) and which are not (1).
    ids_restore (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
        Tensor containing the original index of the (shuffled) masked patches.
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, height, width, hidden_size)`. These are the reshaped and re-rolled hidden states of the model.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nr   Úpooler_outputÚbool_masked_posÚids_restore.r   r   r   )r   r   r    r!   r   r"   r#   r$   r+   r,   Ú
BoolTensorr-   Ú
LongTensorr   r%   r   r   r&   r'   r(   r*   r*   @   sæ   € € € € € € ðð ð 37Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø.2€M�5Ô$ tÑ+Ð2Ð2Ñ2Ø/3€O�UÔ%¨Ñ,Ð3Ð3Ñ3Ø+/€K�Ô! DÑ(Ð/Ð/Ñ/Ø:>€M�5˜Ô*¨CÐ/Ô0°4Ñ7Ð>Ð>Ñ>Ø7;€J��eÔ'¨Ð,Ô-°Ñ4Ð;Ð;Ñ;ØCGÐ˜E %Ô"3°SÐ"8Ô9¸DÑ@ÐGÐGÑGÐGÐGr'   r*   z-
    Hiera image classification outputs.
    c                   óB   — e Zd ZU dZdZeej        df         dz  ed<   dS )Ú!HieraForImageClassificationOutputaÎ  
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, `optional`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, height, width, hidden_size)`. These are the reshaped and re-rolled hidden states of the model.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    N.r   )	r   r   r    r!   r   r%   r"   r#   r$   r&   r'   r(   r1   r1   _   sC   € € € € € € ðð ð DHÐ˜E %Ô"3°SÐ"8Ô9¸DÑ@ÐGÐGÑGÐGÐGr'   r1   z_
    Class for HieraForPreTraining's outputs, with potential hidden states and attentions.
    c                   ó
  — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej
        dz  ed<   dZej        dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed	<   dS )
ÚHieraForPreTrainingOutputa  
    loss (`torch.FloatTensor` of shape `(1,)`):
        Pixel reconstruction loss.
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, patch_size ** 2 * num_channels)`):
        Pixel reconstruction logits.
    bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, sequence_length)`):
        Tensor indicating which patches are masked (0) and which are not (1).
    ids_restore (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
        Tensor containing the original index of the (shuffled) masked patches.
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, height, width, hidden_size)`. Hidden-states of the model at the output of each layer
        plus the initial embedding outputs reshaped to include the spatial dimensions.
    NÚlossÚlogitsr,   r-   r   r   r   )r   r   r    r!   r4   r"   r#   r$   r5   r,   r.   r-   r/   r   r%   r   r   r&   r'   r(   r3   r3   r   sÖ   € € € € € € ðð ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø/3€O�UÔ%¨Ñ,Ð3Ð3Ñ3Ø+/€K�Ô! DÑ(Ð/Ð/Ñ/Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø>BÐ˜E %Ô"3Ô4°tÑ;ÐBÐBÑBÐBÐBr'   r3   c                   ó*  ‡ — e Zd ZdZddefˆ fd„Z	 ddej        dej        dz  dej	        fd	„Z
	 ddej        d
ej        dz  deej        ej        f         fd„Z	 ddej        d
ej        dz  deej	        ej        dz  ej        dz  f         fd„Zˆ xZS )ÚHieraPatchEmbeddingszì
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    FÚis_maec                 ó8  •— t          ¦   «                              ¦   «          t          |j        ¦  «        | _        | j        dk    rt          d| j        › d�¦  «        ‚|j        | _        |j        dd …         | _        d„ t          |j        |j	        ¦  «        D ¦   «         | _
        d„ t          | j
        |j        ¦  «        D ¦   «         | _        |j        | _        || _        t          j        | j        |j        |j        |j	        |j        ¬¦  «        | _        d S )Né   zAThe number of dimensions of the input image should be 2, but got ú.éþÿÿÿc                 ó   — g | ]
\  }}||z  ‘ŒS r&   r&   ©Ú.0ÚiÚss      r(   ú
<listcomp>z1HieraPatchEmbeddings.__init__.<locals>.<listcomp>¡   s    € Ð$dÐ$dÐ$d±°°1 Q¨!¡VÐ$dÐ$dÐ$dr'   c                 ó   — g | ]
\  }}||z  ‘ŒS r&   r&   r>   s      r(   rB   z1HieraPatchEmbeddings.__init__.<locals>.<listcomp>¢   s    € Ð"nÐ"nÐ"n©d¨a° 1¨¡6Ð"nÐ"nÐ"nr'   )Úkernel_sizeÚstrideÚpadding)ÚsuperÚ__init__ÚlenÚ
patch_sizeÚspatial_dimsÚ
ValueErrorÚnum_channelsÚ
image_sizeÚzipÚpatch_strideÚtokens_spatial_shapeÚmasked_unit_sizeÚmask_spatial_shapeÚ
mask_ratior8   r   ÚConv2dÚ	embed_dimÚpatch_paddingÚ
projection)ÚselfÚconfigr8   Ú	__class__s      €r(   rH   zHieraPatchEmbeddings.__init__˜   s  ø€ Ý‰Œ×ÒÑÔÐõ   Ô 1Ñ2Ô2ˆÔØÔ Ò!Ð!ÝÐuÐaeÔarÐuÐuÐuÑvÔvÐvØ"Ô/ˆÔØ Ô+¨B¨C¨CÔ0ˆŒØ$dÐ$d½¸FÔ<MÈvÔObÑ8cÔ8cÐ$dÑ$dÔ$dˆÔ!Ø"nÐ"nµc¸$Ô:SÐU[ÔUlÑ6mÔ6mÐ"nÑ"nÔ"nˆÔØ Ô+ˆŒØˆŒÝœ)ØÔØÔØÔ)ØÔ&ØÔ(ð
ñ 
ô 
ˆŒˆˆr'   NÚpixel_valuesr,   Úreturnc                 ó"  — |€|                       |¦  «        S |j        dd…         } |j        |j        d         dg| j        ¢R Ž }t          j                             |                     ¦   «         |¬¦  «        }|                       ||z  ¦  «        S )z‰Zero-out the masked regions of the input before conv.
        Prevents leakage of masked regions when using overlapping kernels.
        Nr:   r   r   )Úsize)rX   ÚshapeÚviewrS   r   Ú
functionalÚinterpolateÚfloat)rY   r\   r,   Útarget_sizes       r(   Úmasked_convz HieraPatchEmbeddings.masked_conv­   s’   € ð Ð"Ø—?’? <Ñ0Ô0Ð0à"Ô(¨¨¨Ô,ˆà.˜/Ô.¨|Ô/AÀ!Ô/DÀaÐbÈ$ÔJaÐbÐbÐbˆåœ-×3Ò3°O×4IÒ4IÑ4KÔ4KÐR]Ð3Ñ^Ô^ˆà�Š˜|¨oÑ=Ñ>Ô>Ð>r'   Únoisec                 óð  — |j         d         }t          j        | j        ¦  «        }t	          |d| j        z
  z  ¦  «        }|€t          j        |||j        ¬¦  «        }t          j	        |d¬¦  «        }t          j	        |d¬¦  «         
                    |j        ¦  «        }t          j        ||g|j        ¬¦  «        }d|dd…d|…f<   t          j        |d|¬¦  «                             ¦   «         }||fS )aÊ  
        Perform per-sample random masking by per-sample shuffling. Per-sample shuffling is done by argsort random
        noise.

        Args:
            pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`)
            noise (`torch.FloatTensor` of shape `(batch_size, num_mask_units)`, *optional*) which is
                mainly used for testing purposes to control randomness and maintain the reproducibility
        r   r   N©Údevice©Údim)rl   Úindex)r`   ÚmathÚprodrS   ÚintrT   r"   Úrandrj   ÚargsortÚtoÚzerosÚgatherÚbool)	rY   r\   rg   Ú
batch_sizeÚnum_windowsÚlen_keepÚids_shuffler-   r,   s	            r(   Úrandom_maskingz#HieraPatchEmbeddings.random_masking¾   sö   € ð "Ô'¨Ô*ˆ
å”i Ô 7Ñ8Ô8ˆÝ�{ a¨$¬/Ñ&9Ñ:Ñ;Ô;ˆàˆ=Ý”J˜z¨;¸|Ô?RÐSÑSÔSˆEõ ”m E¨qÐ1Ñ1Ô1ˆå”m K°QÐ7Ñ7Ô7×:Ò:¸<Ô;NÑOÔOˆõ  œ+ z°;Ð&?ÈÔH[Ð\Ñ\Ô\ˆØ()ˆ˜˜˜˜9˜H˜9˜Ñ%åœ, ¸AÀ[ÐQÑQÔQ×VÒVÑXÔXˆà Ð+Ð+r'   c                 óÐ   — | j         r|                      ||¬¦  «        nd\  }}|                      ||¦  «        }|                     d¦  «                             dd¦  «        }|||fS )N©rg   )NNr:   r   )r8   r{   rf   ÚflattenÚ	transpose)rY   r\   rg   r,   r-   Ú
embeddingss         r(   ÚforwardzHieraPatchEmbeddings.forwardà   sw   € ð ?C¼kÐ[ˆD×Ò °EÐÑ:Ô:Ð:È|ñ 	'ˆ˜+ð ×%Ò% l°OÑDÔDˆ
Ø×'Ò'¨Ñ*Ô*×4Ò4°Q¸Ñ:Ô:ˆ
à˜?¨KÐ7Ð7r'   ©F©N)r   r   r    r!   rv   rH   r"   r#   r.   ÚTensorrf   r%   r/   r{   r�   Ú__classcell__©r[   s   @r(   r7   r7   ‘   sP  ø€ € € € € ðð ð
ð 
 tð 
ð 
ð 
ð 
ð 
ð 
ð, [_ð?ð ?Ø!Ô-ð?Ø@EÔ@PÐSWÑ@Wð?à	Œð?ð ?ð ?ð ?ð$ RVð ,ð  ,Ø!Ô-ð ,Ø6;Ô6GÈ$Ñ6Nð ,à	ˆuÔ Ô!1Ð1Ô	2ð ,ð  ,ð  ,ð  ,ðJ +/ð8ð 8àÔ'ð8ð Ô  4Ñ'ð8ð 
ˆuŒ|˜UÔ-°Ñ4°eÔ6FÈÑ6MÐMÔ	Nð	8ð 8ð 8ð 8ð 8ð 8ð 8ð 8r'   r7   c                   ó  ‡ — e Zd ZdZddededdfˆ fd„Zdej        d	ej        d
e	de	dej        f
d„Z
dej        d
e	de	dedej        f
d„Z	 	 ddej        dej        dz  dedeej        ej        dz  ej        dz  f         fd„Zˆ xZS )ÚHieraEmbeddingsz2
    Construct position and patch embeddings.
    FrZ   r8   r]   Nc                 óÊ  •— t          ¦   «                              ¦   «          |j        | _        d„ t          |j        |j        ¦  «        D ¦   «         }d„ t          ||j        ¦  «        D ¦   «         | _        t          j        |¦  «        | _	        || _
        t          ||¬¦  «        | _        t          j        t          j        d| j	        |j        ¦  «        ¦  «        | _        d S )Nc                 ó   — g | ]
\  }}||z  ‘ŒS r&   r&   r>   s      r(   rB   z,HieraEmbeddings.__init__.<locals>.<listcomp>÷   ó    € Ð_Ð_Ð_©4¨1¨a  Q¡Ð_Ð_Ð_r'   c                 ó   — g | ]
\  }}||z  ‘ŒS r&   r&   r>   s      r(   rB   z,HieraEmbeddings.__init__.<locals>.<listcomp>ø   s    € Ð"iÐ"iÐ"i©d¨a° 1¨¡6Ð"iÐ"iÐ"ir'   ©r8   r   )rG   rH   rP   rO   rN   rR   rS   rn   ro   Ú
num_tokensr8   r7   Úpatch_embeddingsr   Ú	Parameterr"   rt   rV   Úposition_embeddings)rY   rZ   r8   rQ   r[   s       €r(   rH   zHieraEmbeddings.__init__ô   sÇ   ø€ Ý‰Œ×ÒÑÔÐØ"Ô/ˆÔØ_Ð_µ3°vÔ7HÈ&ÔJ]Ñ3^Ô3^Ð_Ñ_Ô_ÐØ"iÐ"iµcÐ:NÐPVÔPgÑ6hÔ6hÐ"iÑ"iÔ"iˆÔÝœ)Ð$8Ñ9Ô9ˆŒØˆŒå 4°VÀFÐ KÑ KÔ KˆÔå#%¤<µ´¸A¸t¼ÐPVÔP`Ñ0aÔ0aÑ#bÔ#bˆÔ Ð Ð r'   r€   Ú
pos_embedsÚheightÚwidthc                 ó  — |j         d         }|j         d         }t          j                             ¦   «         s||k    r||k    r|S |j         d         }|| j        d         z  }|| j        d         z  }	t          |dz  ¦  «        }
|                     d|
|
|¦  «        }|                     dddd¦  «        }t          j	         
                    |||	fdd¬	¦  «        }|                     dddd¦  «                             dd|¦  «        }|S )
a2  
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing, no class embeddings, and different patch strides.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   éÿÿÿÿr   ç      à?r   r:   ÚbicubicF)r_   ÚmodeÚalign_corners)r`   r"   ÚjitÚ
is_tracingrP   r   ÚreshapeÚpermuter   rb   rc   ra   )rY   r€   r’   r“   r”   Únum_patchesÚnum_positionsrl   Ú
new_heightÚ	new_widthÚsqrt_num_positionss              r(   Úinterpolate_pos_encodingz(HieraEmbeddings.interpolate_pos_encoding   s(  € ð !Ô& qÔ)ˆØ"Ô(¨Ô+ˆõ Œy×#Ò#Ñ%Ô%ð 	¨+¸Ò*FÐ*FÈ6ÐUZÊ?È?ØÐàÔ˜rÔ"ˆà˜tÔ0°Ô3Ñ3ˆ
Ø˜TÔ.¨qÔ1Ñ1ˆ	å& }°cÑ'9Ñ:Ô:ÐØ×'Ò'¨Ð+=Ð?QÐSVÑWÔWˆ
Ø×'Ò'¨¨1¨a°Ñ3Ô3ˆ
å”]×.Ò.ØØ˜iÐ(ØØð	 /ñ 
ô 
ˆ
ð  ×'Ò'¨¨1¨a°Ñ3Ô3×8Ò8¸¸BÀÑDÔDˆ
ØÐr'   r¤   c                 óN   — |r|                       || j        ||¦  «        n| j        S rƒ   )r¤   r‘   )rY   r€   r“   r”   r¤   s        r(   Úget_position_embeddingz&HieraEmbeddings.get_position_embedding&  s5   € ð
 (ð*ˆD×)Ò)¨*°dÔ6NÐPVÐX]Ñ^Ô^Ð^àÔ)ð	
r'   r\   rg   c                 óœ   — |j         dd …         \  }}|                      ||¬¦  «        \  }}}||                      ||||¦  «        z   }|||fS )Nr<   r}   )r`   r�   r¦   )	rY   r\   rg   r¤   r“   r”   r€   r,   r-   s	            r(   r�   zHieraEmbeddings.forward/  sf   € ð %Ô*¨2¨3¨3Ô/‰ˆ�Ø37×3HÒ3HÈÐ]bÐ3HÑ3cÔ3cÑ0ˆ
�O [Ø $×"=Ò"=¸jÈ&ÐRWÐYqÑ"rÔ"rÑrˆ
Ø˜?¨KÐ7Ð7r'   r‚   )NF)r   r   r    r!   r   rv   rH   r"   r„   rp   r¤   r#   r¦   r%   r.   r/   r�   r…   r†   s   @r(   rˆ   rˆ   ï   s_  ø€ € € € € ðð ð
cð 
c˜{ð 
c°Dð 
cÀTð 
cð 
cð 
cð 
cð 
cð 
cð$Øœ,ð$Ø49´Lð$ØJMð$ØVYð$à	Œð$ð $ð $ð $ðL
Øœ,ð
Ø03ð
Ø<?ð
Ø[_ð
à	Ô	ð
ð 
ð 
ð 
ð +/Ø).ð		8ð 	8àÔ'ð	8ð Ô  4Ñ'ð	8ð #'ð		8ð
 
ˆuŒ|˜UÔ-°Ñ4°eÔ6FÈÑ6MÐMÔ	Nð	8ð 	8ð 	8ð 	8ð 	8ð 	8ð 	8ð 	8r'   rˆ   c                   ó”   ‡ — e Zd ZdZ	 	 	 ddedededed	ed
eddfˆ fd„Z	 ddej        dede	ej        ej        dz  f         fd„Z
ˆ xZS )ÚHieraMaskUnitAttentionz¼
    Computes either Mask Unit or Global Attention. Also is able to perform query pooling.

    Note: this assumes the tokens have already been flattened and unrolled into mask units.
    r   r   FÚhidden_sizeÚhidden_size_outputÚ	num_headsÚquery_strideÚwindow_sizeÚuse_mask_unit_attnr]   Nc                 ó.  •— t          ¦   «                              ¦   «          || _        || _        || _        ||z  | _        | j        dz  | _        t          j        |d|z  ¦  «        | _	        t          j        ||¦  «        | _
        || _        || _        d S )Ng      à¿r   )rG   rH   r¬   r­   r«   Úhead_dimÚscaler   ÚLinearÚqkvÚprojr®   r¯   )rY   rª   r«   r¬   r­   r®   r¯   r[   s          €r(   rH   zHieraMaskUnitAttention.__init__B  s�   ø€ õ 	‰Œ×ÒÑÔÐØ"ˆŒØ(ˆÔØ"4ˆÔà*¨iÑ7ˆŒØ”m¨Ñ,ˆŒ
å”9˜[¨!Ð.@Ñ*@ÑAÔAˆŒÝ”IÐ0Ð2DÑEÔEˆŒ	à&ˆÔØ"4ˆÔÐÐr'   r   Úoutput_attentionsc                 óÚ  — |j         \  }}}d}| j        r|| j        | j        z  z  }|                      |¦  «        }|                     |d|d| j        | j        ¦  «        }|                     dddddd¦  «        }| 	                    d¦  «        \  }}	}
| j        dk    rD| 
                    || j        || j        d| j        ¦  «        }|                     d¬¦  «        j        }|| j        z  |	                     dd	¦  «        z  }|                     d¬¦  «        }||
z  }|                     dd¦  «                             |d| j        ¦  «        }|                      |¦  «        }|r||fn|d
fS )z3Input should be of shape [batch, tokens, channels].r   r–   r   r   é   r:   é   rk   r<   N)r`   r¯   r­   r®   r´   r�   r¬   r±   rž   Úunbindra   ÚmaxÚvaluesr²   r   Úsoftmaxr«   rµ   )rY   r   r¶   rw   Úseq_lenÚ_rx   r´   ÚqueryÚkeyÚvalueÚattn_weightsÚattn_outputs                r(   r�   zHieraMaskUnitAttention.forwardY  sx  € ð "/Ô!4Ñˆ
�G˜QàˆØÔ"ð 	LØ! dÔ&7¸$Ô:JÑ&JÑKˆKà�hŠh�}Ñ%Ô%ˆØ�kŠk˜* b¨+°q¸$¼.È$Ì-ÑXÔXˆØ�kŠk˜!˜Q  1 a¨Ñ+Ô+ˆàŸJšJ q™MœMÑˆˆs�EàÔ˜qÒ Ð à—J’J˜z¨4¬>¸;ÈÔHYÐ[]Ð_cÔ_lÑmÔmˆEØ—I’I !�IÑ$Ô$Ô+ˆEà ¤
Ñ*¨c¯mªm¸BÀÑ.CÔ.CÑCˆØ#×+Ò+°Ð+Ñ3Ô3ˆà" UÑ*ˆØ!×+Ò+¨A¨qÑ1Ô1×9Ò9¸*ÀbÈ$ÔJaÑbÔbˆØ—i’i Ñ,Ô,ˆà.?ÐX�˜\Ð*Ð*ÀkÐSWÐEXÐXr'   )r   r   Fr‚   )r   r   r    r!   rp   rv   rH   r"   r„   r%   r�   r…   r†   s   @r(   r©   r©   ;  sö   ø€ € € € € ðð ð ØØ#(ð5ð 5àð5ð  ð5ð ð	5ð
 ð5ð ð5ð !ð5ð 
ð5ð 5ð 5ð 5ð 5ð 5ð4 #(ðYð Yà”|ðYð  ðYð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð	Yð Yð Yð Yð Yð Yð Yð Yr'   r©   c                   óL   ‡ — e Zd Zdeddfˆ fd„Zdej        dej        fd„Zˆ xZS )ÚHieraMlprl   r]   Nc                 ó2  •— t          ¦   «                              ¦   «          t          |j                 | _        t          j        |t          ||j        z  ¦  «        ¦  «        | _	        t          j        t          ||j        z  ¦  «        |¦  «        | _
        d S rƒ   )rG   rH   r   Ú
hidden_actÚactivation_fnr   r³   rp   Ú	mlp_ratioÚfc1Úfc2)rY   rZ   rl   r[   s      €r(   rH   zHieraMlp.__init__{  ss   ø€ Ý‰Œ×ÒÑÔÐÝ# FÔ$5Ô6ˆÔÝ”9˜S¥# c¨FÔ,<Ñ&<Ñ"=Ô"=Ñ>Ô>ˆŒÝ”9�S  vÔ'7Ñ!7Ñ8Ô8¸#Ñ>Ô>ˆŒˆˆr'   r   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rƒ   )rË   rÉ   rÌ   )rY   r   s     r(   r�   zHieraMlp.forward�  s=   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆØÐr'   )	r   r   r    rp   rH   r"   r„   r�   r…   r†   s   @r(   rÆ   rÆ   z  sq   ø€ € € € € ð? Cð ?¨Dð ?ð ?ð ?ð ?ð ?ð ?ð U¤\ð °e´lð ð ð ð ð ð ð ð r'   rÆ   c                   ó^   ‡ — e Zd ZdZd
deddfˆ fd„Zdej        dej        fd„Zde	fd	„Z
ˆ xZS )ÚHieraDropPathzÏStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    ç        Ú	drop_probr]   Nc                 óV   •— t          ¦   «                              ¦   «          || _        d S rƒ   )rG   rH   rÑ   )rY   rÑ   r[   s     €r(   rH   zHieraDropPath.__init__�  s$   ø€ Ý‰Œ×ÒÑÔÐØ"ˆŒˆˆr'   r   c                 ó  — | j         dk    s| j        s|S d| j         z
  }|j        d         fd|j        dz
  z  z   }t	          j        ||j        |j        ¬¦  «        }t	          j        ||z   ¦  «        }| 	                    |¦  «        |z  S )NrÐ   r   r   ©r   )Údtyperj   )
rÑ   Útrainingr`   Úndimr"   rq   rÕ   rj   ÚfloorÚdiv)rY   r   Ú	keep_probr`   Úrandom_tensors        r(   r�   zHieraDropPath.forward”  s“   € ØŒ>˜SÒ Ð ¨¬Ð Ø Ð Ø˜œÑ&ˆ	ØÔ$ QÔ'Ð)¨D°MÔ4FÈÑ4JÑ,KÑKˆÝœ
 5°Ô0CÈMÔL`ÐaÑaÔaˆÝœ M°IÑ$=Ñ>Ô>ˆØ× Ò  Ñ+Ô+¨mÑ;Ð;r'   c                 ó   — d| j         › �S )Nzp=)rÑ   ©rY   s    r(   Ú
extra_reprzHieraDropPath.extra_repr�  s   € Ø$�D”NÐ$Ð$Ð$r'   )rÐ   )r   r   r    r!   rd   rH   r"   r„   r�   ÚstrrÞ   r…   r†   s   @r(   rÏ   rÏ   ‰  s›   ø€ € € € € ðð ð#ð # %ð #°$ð #ð #ð #ð #ð #ð #ð< U¤\ð <°e´lð <ð <ð <ð <ð%˜Cð %ð %ð %ð %ð %ð %ð %ð %r'   rÏ   c                   ó–   ‡ — e Zd Z	 	 	 	 ddedededed	ed
ededdfˆ fd„Z	 ddej        dede	ej        ej        dz  f         fd„Z
ˆ xZS )Ú
HieraLayerrÐ   r   r   Frª   r«   r¬   Ú	drop_pathr­   r®   r¯   r]   Nc	                 óî  •— t          ¦   «                              ¦   «          || _        || _        || _        t          j        ||j        ¬¦  «        | _        t          ||||||¬¦  «        | _
        t          j        ||j        ¬¦  «        | _        t          ||¦  «        | _        |dk    rt          |¦  «        nt          j        ¦   «         | _        ||k    rt          j        ||¦  «        | _        d S d S )N©Úeps)rª   r«   r¬   r­   r®   r¯   r   )rG   rH   rª   r«   r­   r   Ú	LayerNormÚlayer_norm_epsÚlayernorm_beforer©   ÚattnÚlayernorm_afterrÆ   ÚmlprÏ   ÚIdentityrâ   r³   rµ   )
rY   rZ   rª   r«   r¬   râ   r­   r®   r¯   r[   s
            €r(   rH   zHieraLayer.__init__¢  sò   ø€ õ 	‰Œ×ÒÑÔÐà&ˆÔØ"4ˆÔØ(ˆÔå "¤¨[¸fÔ>SÐ TÑ TÔ TˆÔÝ*Ø#Ø1ØØ%Ø#Ø1ð
ñ 
ô 
ˆŒ	õ  "œ|Ð,>ÀFÔDYÐZÑZÔZˆÔÝ˜FÐ$6Ñ7Ô7ˆŒà5>À²]°]� yÑ1Ô1Ð1ÍÌÉÌˆŒØÐ,Ò,Ð,Ýœ	 +Ð/AÑBÔBˆDŒIˆIˆIð -Ð,r'   r   r¶   c                 óö  — |j         \  }}}|                      |¦  «        }| j        | j        k    rP|                      |¦  «        }|                     || j        d| j        ¦  «                             d¬¦  «        j        }|  	                    ||¬¦  «        \  }}||  
                    |¦  «        z   }|}|                      |¦  «        }|                      |¦  «        }||  
                    |¦  «        z   }||fS )Nr–   r   rk   ©r¶   )r`   rè   rª   r«   rµ   ra   r­   r»   r¼   ré   râ   rê   rë   )	rY   r   r¶   rw   r¾   r¿   Úhidden_states_normrÃ   Úresiduals	            r(   r�   zHieraLayer.forwardÄ  s  € ð
 "/Ô!4Ñˆ
�G˜Qà!×2Ò2°=ÑAÔAÐØÔ˜tÔ6Ò6Ð6Ø ŸIšIÐ&8Ñ9Ô9ˆMð ×"Ò" :¨tÔ/@À"ÀdÔF]Ñ^Ô^×bÒbÐghÐbÑiÔiÔpð ð .2¯YªYÐ7IÐ]n¨YÑ-oÔ-oÑ*Ð	˜\Ø%¨¯ªÐ7IÑ(JÔ(JÑJˆà ˆØ×,Ò,¨]Ñ;Ô;ˆØŸš Ñ/Ô/ˆØ  4§>¢>°-Ñ#@Ô#@Ñ@ˆà˜|Ð,Ð,r'   )rÐ   r   r   Fr‚   )r   r   r    rp   rd   rv   rH   r"   r„   r%   r�   r…   r†   s   @r(   rá   rá   ¡  sÿ   ø€ € € € € ð ØØØ#(ð Cð  Cð ð Cð  ð	 Cð
 ð Cð ð Cð ð Cð ð Cð !ð Cð 
ð Cð  Cð  Cð  Cð  Cð  CðJ #(ð-ð -à”|ð-ð  ð-ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð	-ð -ð -ð -ð -ð -ð -ð -r'   rá   c                   ó¶   ‡ — e Zd Z	 ddededededee         dee         ded	ed
edz  ddfˆ fd„Z	 ddej	        dede
ej	        ej	        dz  f         fd„Zˆ xZS )Ú
HieraStageNÚdepthrª   r«   r¬   râ   r­   r®   r¯   Ú	stage_numr]   c                 ó  •‡‡‡‡‡‡‡‡	‡— t          ¦   «                              ¦   «          dŠ|
�‰j        |
dk    r|
dz
  nd         Št          j        ˆˆˆˆˆˆˆˆ	ˆf	d„t          |¦  «        D ¦   «         ¦  «        | _        d S )NFr   r   c                 óv   •	— g | ]5}t          ‰|d k    r‰n‰‰‰‰|         ‰|         ‰
‰	p‰o|d k    ¬¦  «        ‘Œ6S )r   )rZ   rª   r«   r¬   râ   r­   r®   r¯   )rá   )r?   r@   rZ   râ   rª   r«   r¬   Ú$previous_stage_used_masked_attentionr­   r¯   r®   s     €€€€€€€€€r(   rB   z'HieraStage.__init__.<locals>.<listcomp>õ  sy   ø€ ð ð ð ð õ Ø!Ø/0°Aªv¨v  Ð;MØ'9Ø'Ø'¨œlØ!-¨a¤Ø +Ø'9Ð'nÐ>bÐ>mÐghÐlmÒgmð	ñ 	ô 	ðð ð r'   )rG   rH   Úmasked_unit_attentionr   Ú
ModuleListÚrangeÚlayers)rY   rZ   ró   rª   r«   r¬   râ   r­   r®   r¯   rô   r÷   r[   s    ` ``````` @€r(   rH   zHieraStage.__init__ß  sÆ   øøøøøøøøøø€ õ 	‰Œ×ÒÑÔÐð
 05Ð,ØÐ Ø39Ô3OÐajÐmnÒanÐanÐPYÐ\]ÑP]ÐP]ÐtuÔ3vÐ0Ý”mðð ð ð ð ð ð ð ð ð ð ð õ ˜u™œðñ ô ñ
ô 
ˆŒˆˆr'   Fr   r¶   c                 ó^   — t          | j        ¦  «        D ]\  }} |||¬¦  «        \  }}Œ||fS )Nrî   )Ú	enumeraterû   )rY   r   r¶   r@   Úlayer_modulerÃ   s         r(   r�   zHieraStage.forward  sJ   € õ  )¨¬Ñ5Ô5ð 	mð 	m‰OˆAˆ|Ø,8¨L¸ÐZkÐ,lÑ,lÔ,lÑ)ˆ]˜L˜Là˜lÐ*Ð*r'   rƒ   r‚   )r   r   r    rp   Úlistrd   rv   rH   r"   r„   r%   r�   r…   r†   s   @r(   rò   rò   Þ  s  ø€ € € € € ð !%ð#
ð #
ð ð#
ð ð	#
ð
  ð#
ð ð#
ð ˜”;ð#
ð ˜3”ið#
ð ð#
ð !ð#
ð ˜‘:ð#
ð 
ð#
ð #
ð #
ð #
ð #
ð #
ðL FKð+ð +Ø"œ\ð+Ø>Bð+à	ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð+ð +ð +ð +ð +ð +ð +ð +r'   rò   r   r`   Úmask_unit_shaper]   c                 óæ   — | j         d         | j         d         }}d„ t          ||¦  «        D ¦   «         } | j        |g|¢|¢|‘R Ž } |                      dddddd¦  «        }  | j        |g|¢|‘R Ž } | S )	a]  
    Restore spatial organization by undoing windowed organization of mask units.

    Args:
        hidden_states (`torch.Tensor`): The hidden states tensor of shape `[batch_size, num_mask_unit_height*num_mask_unit_width, hidden_size]`.
        shape (`list[int]`): The original shape of the hidden states tensor before windowing.
        mask_unit_shape (`list[int]`): The shape of the mask units used for windowing.

    Returns:
        torch.Tensor: The restored hidden states tensor of shape [batch_size, num_mask_unit_height*mask_unit_height, num_mask_unit_width*mask_unit_width, hidden_size].
    r   r–   c                 ó   — g | ]
\  }}||z  ‘ŒS r&   r&   )r?   rA   Úmus      r(   rB   z"undo_windowing.<locals>.<listcomp>  s    € ÐGÐGÐG¡% ! R�a˜2‘gÐGÐGÐGr'   r   r   r:   r¸   r¹   )r`   rO   ra   rž   r�   )r   r`   r   rw   rª   Únum_mask_unitss         r(   Úundo_windowingr    s¦   € ð ,Ô1°!Ô4°mÔ6IÈ"Ô6M�€Jð HÐG­3¨u°oÑ+FÔ+FÐGÑGÔG€NØ&�MÔ& zÐb°NÐbÀ_ÐbÐVaÐbÐbÐb€Mð "×)Ò)¨!¨Q°°1°a¸Ñ;Ô;€MØ)�MÔ)¨*ÐJ°uÐJ¸kÐJÐJÐJ€MàÐr'   c                   ó²   ‡ — e Zd Zdeddfˆ fd„Z	 ddej        dedej        dz  dej        fd„Z		 	 	 	 ddej        dej        dz  de
de
de
deez  fd„Zˆ xZS )ÚHieraEncoderrZ   r]   Nc                 óH  •‡‡— t          ¦   «                              ¦   «          t          ‰j        ¦  «        }d„ t	          j        d‰j        |d¬¦  «        D ¦   «         }t	          j        ‰j        d¬¦  «                             d¦  «         	                    ¦   «         }|d ‰j
        …         Šˆˆfd„t          |¦  «        D ¦   «         }t          j        ¦   «         | _        ‰j        }dg|z   }t!          j        ‰j        ¦  «        }t!          j        ‰j        ¦  «        }	t)          ‰j        ¦  «        D ]±\  }
}t+          ‰j        ‰j        |
z  z  ¦  «        }t/          ‰|||‰j        |
         |||
         ||
dz            …         |||
         ||
dz            …         t+          ||	|
 z  z  ¦  «        ‰j        |
         |
¬¦
  «
        }|}| j                             |¦  «         Œ²d„ t7          ‰j        ‰j        ¦  «        D ¦   «         }‰j        gt=          ‰j        d d	…         ¦  «        z  }i | _        t          t=          ‰j        ¦  «        ¦  «        D ]B}
||f| j        |
<   |
‰j
        k     r)d
„ t7          |‰j        ¦  «        D ¦   «         }|dd …         }ŒCd| _         d S )Nc                 ó6   — g | ]}|                      ¦   «         ‘ŒS r&   )Úitem)r?   Úxs     r(   rB   z)HieraEncoder.__init__.<locals>.<listcomp>,  s    € ÐeÐeÐe˜Aˆq�vŠv‰xŒxÐeÐeÐer'   r   Úcpuri   c                 óN   •— g | ]!}|‰v rt          j        ‰j        ¦  «        nd ‘Œ"S rÔ   )rn   ro   r­   )r?   r@   rZ   Úquery_pool_layers     €€r(   rB   z)HieraEncoder.__init__.<locals>.<listcomp>0  s8   ø€ ÐtÐtÐtÐ\]¸1Ð@PÐ;PÐ;P�œ 6Ô#6Ñ7Ô7Ð7ÐVWÐtÐtÐtr'   r   )
rZ   ró   rª   r«   r¬   râ   r­   r®   r¯   rô   c                 ó   — g | ]
\  }}||z  ‘ŒS r&   r&   r>   s      r(   rB   z)HieraEncoder.__init__.<locals>.<listcomp>N  s    € ÐUÐUÐU¡  A�a˜1‘fÐUÐUÐUr'   r–   c                 ó   — g | ]
\  }}||z  ‘ŒS r&   r&   r>   s      r(   rB   z)HieraEncoder.__init__.<locals>.<listcomp>U  s    € ÐVÐVÐV©¨¨A˜a 1™fÐVÐVÐVr'   F)!rG   rH   ÚsumÚdepthsr"   ÚlinspaceÚdrop_path_rateÚtensorÚcumsumÚtolistÚnum_query_poolrú   r   rù   ÚstagesrV   rn   ro   rR   r­   rý   rp   Úembed_dim_multiplierrò   r¬   rø   ÚappendrO   rN   rP   rI   ÚscheduleÚgradient_checkpointing)rY   rZ   Útotal_depthÚdprÚcumulative_depthsÚquery_stridesrª   Ú
stage_endsÚmasked_unit_areaÚquery_stride_areaÚ	idx_stageró   r«   ÚstageÚ
stage_sizeÚunroll_scheduler  r[   s    `              @€r(   rH   zHieraEncoder.__init__(  s¼  øøø€ Ý‰Œ×ÒÑÔÐÝ˜&œ-Ñ(Ô(ˆàeÐe¥¤°°6Ô3HÈ+Ð^cÐ!dÑ!dÔ!dÐeÑeÔeˆå!œL¨¬¸uÐEÑEÔE×LÒLÈQÑOÔO×VÒVÑXÔXÐØ,Ð-D¨vÔ/DÐ-DÔEÐØtÐtÐtÐtÐtÕafÐgrÑasÔasÐtÑtÔtˆõ ”m‘o”oˆŒØÔ&ˆØ�SÐ,Ñ,ˆ
Ýœ9 VÔ%<Ñ=Ô=ÐÝ œI fÔ&9Ñ:Ô:ÐÝ )¨&¬-Ñ 8Ô 8ð 	&ð 	&ÑˆI�uÝ!$ VÔ%5¸Ô8SÐU^Ñ8^Ñ%^Ñ!_Ô!_ÐåØØØ'Ø#5Ø Ô*¨9Ô5Ø˜j¨Ô3°jÀÈQÁÔ6OÐOÔPØ*¨:°iÔ+@À:ÈiÐZ[ÉmÔC\Ð+\Ô]ÝÐ 0Ð3DÀyÀjÑ3PÑ PÑQÔQØ#)Ô#?À	Ô#JØ#ðñ ô ˆEð -ˆKØŒK×Ò˜uÑ%Ô%Ð%Ð%ð
 VÐU­¨VÔ->ÀÔ@SÑ)TÔ)TÐUÑUÔUˆ
Ø!Ô.Ð/µ#°f´mÀCÀRÀCÔ6HÑ2IÔ2IÑIˆàˆŒÝ�s 6¤=Ñ1Ô1Ñ2Ô2ð 	6ð 	6ˆIØ'6¸
Ð'BˆDŒM˜)Ñ$Ø˜6Ô0Ò0Ð0ØVÐVµ°ZÀÔATÑ1UÔ1UÐVÑVÔV�
Ø"1°!°"°"Ô"5�øà&+ˆÔ#Ð#Ð#r'   r   Ú	stage_idxr,   c           
      óÐ  — | j         |         \  }}|j        \  }}}t          |¦  «        }	dg|	z  }
|D ]Œ} |j        |g|¢|t	          j        |¦  «        z  ‘|
¢|‘R Ž }|                     ddddddd¦  «        }t          |	¦  «        D ]}|
|xx         ||         z  cc<   Œ |j        |dg|
¢|‘R Ž }|j        d         }Œ� |j        ||g|
¢|‘R Ž }|�|S t          |||
¦  «        }|S )	a\  
        Roll the given tensor back up to spatial order assuming it's from the given block.

        If no bool_masked_pos is provided returns:
            - [batch_size, height, width, hidden_size]
        If a bool_masked_pos is provided returns:
            - [batch_size, num_mask_units, mask_unit_height, mask_unit_width, hidden_size]
        r   r   r   r¸   r:   r¹   é   r–   )
r  r`   rI   ra   rn   ro   rž   rú   r�   r  )rY   r   r)  r,   r  r_   rw   r¾   rª   Únum_dimr   Ústridesr@   s                r(   ÚrerollzHieraEncoder.rerollZ  sj  € ð œ yÔ1‰ˆ�$Ø+8Ô+>Ñ(ˆ
�G˜[å�d‘)”)ˆØ˜# ™-ˆàð 	-ð 	-ˆGà.˜MÔ.ØðØ$ðØ&-µ´¸7Ñ1CÔ1CÑ&CðØFUðØWbðð ð ˆMð *×1Ò1°!°Q¸¸1¸aÀÀAÑFÔFˆMõ ˜7‘^”^ð 1ð 1�Ø Ð"Ð"Ô" g¨a¤jÑ0Ð"Ð"Ñ"Ð"Ø1˜MÔ1°*¸bÐ`À?Ð`ÐT_Ð`Ð`Ð`ˆMØ#Ô)¨!Ô,ˆGˆGð +˜Ô*¨:°wÐ^ÀÐ^ÐR]Ð^Ð^Ð^ˆð Ð&Ø Ð õ ' }°d¸OÑLÔLˆàÐr'   FTr¶   Úoutput_hidden_statesÚreturn_dictc                 óš  — |rdnd }|rdnd }|rdnd }|r$||fz   }|                       |d|¬¦  «        }	||	fz   }t          | j        ¦  «        D ]M\  }
} |||¦  «        }|d         }|r||d         fz   }|r$||fz   }|                       ||
|¬¦  «        }	||	fz   }ŒN|st          d„ ||||fD ¦   «         ¦  «        S t	          ||||¬¦  «        S )Nr&   r   )r)  r,   r   c              3   ó   K  — | ]}|®|V — Œ	d S rƒ   r&   )r?   Úvs     r(   ú	<genexpr>z'HieraEncoder.forward.<locals>.<genexpr>§  s0   è è € ð ð àØ�=ð à �=�=�=ðð r'   )r   r   r   r   )r.  rý   r  r%   r   )rY   r   r,   r¶   r/  r0  Úall_hidden_statesÚall_reshaped_hidden_statesÚall_self_attentionsr   r@   Ústage_moduleÚlayer_outputss                r(   r�   zHieraEncoder.forwardˆ  sn  € ð #7Ð@˜B˜B¸DÐØ+?Ð%I R RÀTÐ"Ø$5Ð?˜b˜b¸4Ðàð 	`Ø 1°]Ð4DÑ DÐØ%)§[¢[°È!Ð]l [Ñ%mÔ%mÐ"Ø)CÐG]ÐF_Ñ)_Ð&å(¨¬Ñ5Ô5ð 	dð 	d‰OˆAˆ|Ø(˜L¨Ð8IÑJÔJˆMà)¨!Ô,ˆMà ð PØ&9¸]È1Ô=MÐ<OÑ&OÐ#à#ð dØ$5¸Ð8HÑ$HÐ!Ø)-¯ª°]ÈaÐap¨Ñ)qÔ)qÐ&Ø-GÐKaÐJcÑ-cÐ*øàð 	Ýð ð à'Ð):Ð<OÐQkÐlðñ ô ñ ô ð õ
 "Ø+Ø+Ø*Ø#=ð	
ñ 
ô 
ð 	
r'   rƒ   )NFFT)r   r   r    r   rH   r"   r„   rp   r.   r.  rv   r%   r   r�   r…   r†   s   @r(   r  r  '  s
  ø€ € € € € ð0,˜{ð 0,¨tð 0,ð 0,ð 0,ð 0,ð 0,ð 0,ðf gkð,ð ,Ø"œ\ð,Ø69ð,ØLQÔL\Ð_cÑLcð,à	Œð,ð ,ð ,ð ,ðb 48Ø"'Ø%*Ø ð)
ð )
à”|ð)
ð Ô)¨DÑ0ð)
ð  ð	)
ð
 #ð)
ð ð)
ð 
�Ñ	 ð)
ð )
ð )
ð )
ð )
ð )
ð )
ð )
r'   r  Úimage_shaperP   r  c           	      óÖ  — | j         \  }}}d„ t          ||¦  «        D ¦   «         }|} | j        |g|z   |gz   Ž } |D �]}	d„ t          ||	¦  «        D ¦   «         }d„ t          ||	¦  «        D ¦   «         }
|g|
z   |gz   }
|                      |
¦  «        } t          |
¦  «        }dgt	          t          d|dz
  d¦  «        ¦  «        z   t	          t          d|dz
  d¦  «        ¦  «        z   |dz
  gz   }|                      |¦  «        } |                      dt          |	¦  «        ¦  «        } |t          j	        |	¦  «        z  }�Œ|  
                    dt          j	        |¦  «        |¦  «        } | S )a¨  
    Reorders the tokens such that patches are contiguous in memory.
    E.g., given [batch_size, (height, width), hidden_size] and stride of (stride, stride), this will re-order the tokens as
    [batch_size, (stride, stride, height // stride, width // stride), hidden_size]

    This allows operations like Max2d to be computed as x.view(batch_size, stride*stride, -1, hidden_size).max(dim=1).
    Not only is this faster, but it also makes it easy to support inputs of arbitrary
    dimensions in addition to patch-wise sparsity.

    Performing this operation multiple times in sequence puts entire windows as contiguous
    in memory. For instance, if you applied the stride (2, 2) 3 times, entire windows of
    size 8x8 would be contiguous in memory, allowing operations like mask unit attention
    computed easily and efficiently, while also allowing max to be applied sequentially.

    Note: This means that intermediate values of the model are not in height x width order, so they
    need to be re-rolled if you want to use the intermediate values as a height x width feature map.
    The last block of the network is fine though, since by then the strides are all consumed.
    c                 ó   — g | ]
\  }}||z  ‘ŒS r&   r&   r>   s      r(   rB   zunroll.<locals>.<listcomp>Ë  s    € Ð>Ð>Ð>‘t�q˜!ˆA�‰FÐ>Ð>Ð>r'   c                 ó   — g | ]
\  }}||z  ‘ŒS r&   r&   r>   s      r(   rB   zunroll.<locals>.<listcomp>Õ  s    € ÐFÐFÐF¡4 1 a˜˜Q™ÐFÐFÐFr'   c                 ó   — g | ]	}|D ]}|‘ŒŒ
S r&   r&   )r?   Úpairr
  s      r(   rB   zunroll.<locals>.<listcomp>×  s%   € ÐRÐRÐR˜dÈTÐRÐRÀT�TÐRÐRÐRÐRr'   r   r:   r   r–   )r`   rO   ra   rI   rÿ   rú   rž   r~   rn   ro   r�   )r   r:  rP   r  rw   r¿   rª   r_   Úcurrent_sizer-  Ú	new_shapeÚnum_dimsrž   s                r(   ÚunrollrC  ´  sŽ  € ð* "/Ô!4Ñ€J��;à>Ð>�s ;°Ñ=Ô=Ð>Ñ>Ô>€Dà€LØ&�MÔ&¨*¨¸Ñ)DÈÀ}Ñ)TÐV€Màð )ñ )ˆð
 GÐF­3¨|¸WÑ+EÔ+EÐFÑFÔFˆàRÐR¥c¨,¸Ñ&@Ô&@ÐRÑRÔRˆ	à�L 9Ñ,°¨}Ñ<ˆ	Ø%×*Ò*¨9Ñ5Ô5ˆõ �y‘>”>ˆØ�#��U 1 h°¡l°AÑ6Ô6Ñ7Ô7Ñ7½$½uÀQÈÐSTÉÐVWÑ?XÔ?XÑ:YÔ:YÑYÐ]eÐhiÑ]iÐ\jÑjˆØ%×-Ò-¨gÑ6Ô6ˆð &×-Ò-¨aµ°W±´Ñ>Ô>ˆØ•d”i Ñ(Ô(Ñ(ˆ
‰
à!×)Ò)¨"­d¬i¸©o¬o¸{ÑKÔK€MØÐr'   c                   óf   ‡ — e Zd ZU eed<   dZdZdZdZ e	j
        ¦   «         d	ˆ fd„¦   «         Zˆ xZS )
ÚHieraPreTrainedModelrZ   Úhierar\   )ÚimageTr]   Nc                 óö  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          ¦  «        rt          j        |j        |¬¦  «         dS t	          |t          ¦  «        r8t          j        |j
        |¬¦  «         t          j        |j        |¬¦  «         dS t	          |t          j        t          j        t          j        f¦  «        r@t          j        |j        |¬¦  «         |j        �t          j        |j        |¦  «         dS dS t	          |t          j        ¦  «        r@t          j        |j        |¦  «         t          j        |j        | j        j        ¦  «         dS dS )zInitialize the weights)ÚstdN)rG   Ú_init_weightsrZ   Úinitializer_rangeÚ
isinstancerˆ   ÚinitÚtrunc_normal_r‘   ÚHieraDecoderÚ
mask_tokenÚdecoder_position_embeddingsr   r³   ÚConv1drU   ÚweightÚbiasÚ	constant_ræ   Úlayer_norm_init)rY   ÚmodulerI  r[   s      €r(   rJ  z"HieraPreTrainedModel._init_weightsò  s\  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%ØŒkÔ+ˆå�f�oÑ.Ô.ð 	GÝÔ˜vÔ9¸sÐCÑCÔCÐCÐCÐCå˜¥Ñ-Ô-ð 	GÝÔ˜vÔ0°cÐ:Ñ:Ô:Ð:ÝÔ˜vÔAÀsÐKÑKÔKÐKÐKÐKå˜¥¤­B¬Iµr´yÐ AÑBÔBð 	GÝÔ˜vœ}°#Ð6Ñ6Ô6Ð6ØŒ{Ð&Ý”˜vœ{¨CÑ0Ô0Ð0Ð0Ð0ð 'Ð&õ ˜¥¤Ñ-Ô-ð 	GÝŒN˜6œ;¨Ñ,Ô,Ð,ÝŒN˜6œ=¨$¬+Ô*EÑFÔFÐFÐFÐFð	Gð 	Gr'   )r]   N)r   r   r    r   r$   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingr"   Úno_gradrJ  r…   r†   s   @r(   rE  rE  ê  s~   ø€ € € € € € àÐÐÑØÐØ$€OØ!ÐØ&*Ð#à€U„]�_„_ðGð Gð Gð Gð Gñ „_ðGð Gð Gð Gð Gr'   rE  c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚHieraPoolerrZ   c                 ó"  •— t          ¦   «                              ¦   «          t          |j        |j        t          |j        ¦  «        dz
  z  z  ¦  «        }t          j        ||j	        ¬¦  «        | _
        t          j        d¦  «        | _        d S )Nr   rä   )rG   rH   rp   rV   r  rI   r  r   ræ   rç   Ú	layernormÚAdaptiveAvgPool1dÚpooler)rY   rZ   Únum_featuresr[   s      €r(   rH   zHieraPooler.__init__
  sw   ø€ Ý‰Œ×ÒÑÔÐÝ˜6Ô+¨fÔ.IÍcÐRXÔR_ÑN`ÔN`ÐcdÑNdÑ.eÑeÑfÔfˆÝœ l¸Ô8MÐNÑNÔNˆŒÝÔ*¨1Ñ-Ô-ˆŒˆˆr'   r   r]   c                 ó°   — |                      dd¦  «        }|                      |¦  «        }t          j        |d¦  «        }|                      |¦  «        }|S )Nr   r:   )r   rb  r"   r~   r`  )rY   r   Úpooled_outputs      r(   r�   zHieraPooler.forward  sP   € Ø%×/Ò/°°1Ñ5Ô5ˆØŸš MÑ2Ô2ˆÝœ m°QÑ7Ô7ˆØŸš }Ñ5Ô5ˆØÐr'   )	r   r   r    r   rH   r"   r„   r�   r…   r†   s   @r(   r^  r^  	  sj   ø€ € € € € ð.˜{ð .ð .ð .ð .ð .ð .ð U¤\ð °e´lð ð ð ð ð ð ð ð r'   r^  c                   óº   ‡ — e Zd Zddededefˆ fd„Zdefd„Ze	 	 	 	 	 	 dd
e	j
        d	z  de	j        d	z  ded	z  ded	z  ded	z  ded	z  deez  fd„¦   «         Zˆ xZS )Ú
HieraModelTFrZ   Úadd_pooling_layerr8   c                 ó¼  •— t          ¦   «                              |¦  «         t          |j        |j        t          |j        ¦  «        dz
  z  z  ¦  «        | _        t          ||¬¦  «        | _	        t          |¦  «        | _        |j        gt          |j        dd…         ¦  «        z  | _        |rt          |¦  «        nd| _        |                      ¦   «          dS )zó
        add_pooling_layer (`bool`, *optional*, defaults to `True`):
            Whether or not to apply pooling layer.
        is_mae (`bool`, *optional*, defaults to `False`):
            Whether or not to run the model on MAE mode.
        r   r�   Nr–   )rG   rH   rp   rV   r  rI   r  rc  rˆ   r€   r  Úencoderr­   r(  r^  rb  Ú	post_init)rY   rZ   rh  r8   r[   s       €r(   rH   zHieraModel.__init__  sÊ   ø€ õ 	‰Œ×Ò˜Ñ Ô Ð Ý Ô 0°6Ô3NÕSVÐW]ÔWdÑSeÔSeÐhiÑSiÑ3jÑ jÑkÔkˆÔå)¨&¸Ð@Ñ@Ô@ˆŒÝ# FÑ+Ô+ˆŒà &Ô 3Ð4µs¸6¼=ÈÈ"ÈÔ;MÑ7NÔ7NÑNˆÔà->ÐH•k &Ñ)Ô)Ð)ÀDˆŒð 	�ŠÑÔÐÐÐr'   r]   c                 ó   — | j         j        S rƒ   ©r€   r�   rÝ   s    r(   Úget_input_embeddingszHieraModel.get_input_embeddings.  ó   € ØŒÔ/Ð/r'   Nr\   rg   r¶   r/  r¤   r0  c           	      ó,  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|€t	          d¦  «        ‚|                      |||¬¦  «        \  }}	}
|j        d         |j        d         f}t          ||| j         j        | j	        ¬¦  «        }|	�rt          j        | j         j        ¦  «        }|j        \  }}}|	                     d¦  «                             d||¦  «        }||         }|                     |d|¦  «        }|                      ||	|||¬¦  «        }|d	         }d}| j        �|                      |¦  «        }|s!|�||fn|f}|	�||	|
fz   n|}||dd…         z   S t%          |||	|
|j        |j        |j        ¬
¦  «        S )zÆ
        noise (`torch.FloatTensor` of shape `(batch_size, num_mask_units)`, *optional*):
            Mainly used for testing purposes to control randomness and maintain the reproducibility
        Nz You have to specify pixel_values)r¤   rg   r<   r–   )r:  rP   r  r   )r,   r¶   r/  r0  r   )r   r+   r,   r-   r   r   r   )rZ   r¶   r/  r0  rL   r€   r`   rC  rP   r(  rn   ro   rR   Ú	unsqueezeÚtilera   rj  rb  r*   r   r   r   )rY   r\   rg   r¶   r/  r¤   r0  ÚkwargsÚembedding_outputr,   r-   r:  r   Úmask_unit_arearw   r¿   rª   Ú	positionsÚencoder_outputsÚsequence_outputre  Úhead_outputss                         r(   r�   zHieraModel.forward1  s  € ð 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàÐÝÐ?Ñ@Ô@Ð@à9=¿ºØÐ3KÐSXð :Iñ :
ô :
Ñ6Ð˜/¨;ð $Ô)¨"Ô-¨|Ô/AÀ"Ô/EÐFˆÝØØ#ØœÔ1ØÔ)ð	
ñ 
ô 
ˆð Ð&Ý!œY t¤{Ô'CÑDÔDˆNØ)6Ô)<Ñ&ˆJ˜˜;Ø'×1Ò1°"Ñ5Ô5×:Ò:¸1¸nÈkÑZÔZˆIØ)¨)Ô4ˆMØ)×.Ò.¨z¸2¸{ÑKÔKˆMàŸ,š,ØØ+Ø/Ø!5Ø#ð 'ñ 
ô 
ˆð *¨!Ô,ˆØˆØŒ;Ð"Ø ŸKšK¨Ñ8Ô8ˆMàð 	6Ø?LÐ?X˜O¨]Ð;Ð;Ð_nÐ^pˆLàAPÐA\� °Ð=Ñ=Ð=Ðbnð ð   /°!°"°"Ô"5Ñ5Ð5åØ-Ø'Ø+Ø#Ø)Ô7Ø&Ô1Ø#2Ô#Ið
ñ 
ô 
ð 	
r'   )TF©NNNNNN)r   r   r    r   rv   rH   r7   rn  r   r"   r„   r#   r%   r   r�   r…   r†   s   @r(   rg  rg    s&  ø€ € € € € ðð ˜{ð ¸tð ÐTXð ð ð ð ð ð ð(0Ð&:ð 0ð 0ð 0ð 0ð ð -1Ø*.Ø)-Ø,0Ø04Ø#'ðF
ð F
à”l TÑ)ðF
ð Ô  4Ñ'ðF
ð   $™;ð	F
ð
 # T™kðF
ð #'¨¡+ðF
ð ˜D‘[ðF
ð 
Ð+Ñ	+ðF
ð F
ð F
ñ „^ðF
ð F
ð F
ð F
ð F
r'   rg  c                   óx   ‡ — e Zd Zdefˆ fd„Z	 d	dej        dej        dede	ej        ej        f         fd„Z
ˆ xZS )
rO  rZ   c                 óv  •‡— t          ¦   «                              ¦   «          t          ‰j        ‰j        t          ‰j        ¦  «        dz
  z  z  ¦  «        }d„ t          ‰j        ‰j	        ¦  «        D ¦   «         }ˆfd„t          |‰j
        ¦  «        D ¦   «         | _        ˆfd„t          ‰j        ‰j
        ¦  «        D ¦   «         | _        t          j        |‰j        ¦  «        | _        t          j        t'          j        dd‰j        ¦  «        ¦  «        | _        t          j        t'          j        dt-          j        | j        ¦  «        ‰j        ¦  «        ¦  «        | _        t3          ‰‰j        ‰j        ‰j        ‰j        ddg‰j        z  dg‰j        z  d¬¦	  «	        | _        t          j        ‰j        ‰j        ¬	¦  «        | _        ‰j	        d
         ‰j
        d
         ‰j         z  z  | _!        | j!        t          ‰j
        ¦  «        z  ‰j"        z  }t          j        ‰j        |¦  «        | _#        d S )Nr   c                 ó   — g | ]
\  }}||z  ‘ŒS r&   r&   r>   s      r(   rB   z)HieraDecoder.__init__.<locals>.<listcomp>  r‹   r'   c                 ó0   •— g | ]\  }}||‰j         z  z  ‘ŒS r&   ©r  ©r?   r@   rA   rZ   s      €r(   rB   z)HieraDecoder.__init__.<locals>.<listcomp>€  s7   ø€ ð +
ð +
ð +
Ù26°!°QˆA��vÔ,Ñ-Ñ-ð+
ð +
ð +
r'   c                 ó0   •— g | ]\  }}||‰j         z  z  ‘ŒS r&   r  r€  s      €r(   rB   z)HieraDecoder.__init__.<locals>.<listcomp>ƒ  ó7   ø€ ð .
ð .
ð .
Ù26°!°QˆA��vÔ,Ñ-Ñ-ð.
ð .
ð .
r'   FrÐ   r   )	rZ   rª   r«   r¬   ró   r¯   râ   r­   r®   rä   r–   )$rG   rH   rp   rV   r  rI   r  rO   rN   rP   r­   Útokens_spatial_shape_finalrR   Úmask_unit_spatial_shape_finalr   r³   Údecoder_hidden_sizeÚdecoder_embeddingsr�   r"   rt   rP  rn   ro   rQ  rò   Údecoder_num_headsÚdecoder_depthÚdecoder_blockræ   rç   Údecoder_normr  Úpred_striderM   Údecoder_pred)rY   rZ   rc  rQ   Úpred_dimr[   s    `   €r(   rH   zHieraDecoder.__init__|  s+  øø€ Ý‰Œ×ÒÑÔÐÝ˜6Ô+¨fÔ.IÍcÐRXÔR_ÑN`ÔN`ÐcdÑNdÑ.eÑeÑfÔfˆØ_Ð_µ3°vÔ7HÈ&ÔJ]Ñ3^Ô3^Ð_Ñ_Ô_Ðð+
ð +
ð +
ð +
Ý:=Ð>RÐTZÔTgÑ:hÔ:hð+
ñ +
ô +
ˆÔ'ð.
ð .
ð .
ð .
Ý:=¸fÔ>UÐW]ÔWjÑ:kÔ:kð.
ñ .
ô .
ˆÔ*õ #%¤)¨L¸&Ô:TÑ"UÔ"UˆÔåœ,¥u¤{°1°a¸Ô9SÑ'TÔ'TÑUÔUˆŒå+-¬<ÝŒK˜�4œ9 TÔ%DÑEÔEÀvÔGaÑbÔbñ,
ô ,
ˆÔ(õ (ØØÔ2Ø%Ô9ØÔ.ØÔ&Ø$Ø�e˜fÔ2Ñ2Ø˜˜vÔ3Ñ3Øð

ñ 

ô 

ˆÔõ œL¨Ô)CÈÔI^Ð_Ñ_Ô_ˆÔð "Ô.¨rÔ2°fÔ6IÈ"Ô6MÐQWÔQfÑ6fÑgˆÔØÔ$­¨FÔ,?Ñ(@Ô(@Ñ@ÀFÔDWÑWˆåœI fÔ&@À(ÑKÔKˆÔÐÐr'   FÚencoder_hidden_statesr,   r¶   r]   c           	      ó¢  — |                       |¦  «        }|j        dd …         \  }}}|j        \  }}	t          j        ||	||||j        |j        ¬¦  «        }
| j                             ddddd¦  «        }|                     ||	ddd¦  «        }| 	                    dd|||¦  «        }| 
                    ¦   «         |
|<   d|                     ¦   «         z
  |z  |                     ¦   «         |
z  z   }
t          |
| j        | j        ¦  «        }t          |ddd…f         | j        | j        ¦  «        }|                     |j        d         d|j        d         ¦  «        }|                     |j        d         d¦  «        }|| j        z   }|                      ||¬¦  «        \  }}|                      |¦  «        }|                      |¦  «        }||fS )Nr:   )rj   rÕ   r   r–   .r   rî   )r†  r`   r"   rt   rj   rÕ   rP  ra   r�   Úexpandr~   rd   r  rƒ  r„  rQ  r‰  rŠ  rŒ  )rY   rŽ  r,   r¶   r   Úmask_unit_heightÚmask_unit_widthr…  rw   r  Údecoder_hidden_statesÚmask_tokensrÃ   s                r(   r�   zHieraDecoder.forward£  s  € ð ×/Ò/Ð0EÑFÔFˆð BOÔATÐUVÐUWÐUWÔAXÑ>Ð˜/Ð+>Ø%4Ô%:Ñ"ˆ
�Nå %¤ØØØØØØ Ô'ØÔ%ð!
ñ !
ô !
Ðð ”o×*Ò*¨1¨a°°A°rÑ:Ô:ˆØ)×1Ò1°*¸nÈaÐQRÐTUÑVÔVˆØ)×0Ò0°°RÐ9IÈ?Ð\oÑpÔpˆØ1>×1FÒ1FÑ1HÔ1HÐ˜oÑ.à�×%Ò%Ñ'Ô'Ñ'Øñ!à)×/Ò/Ñ1Ô1Ð4IÑIñ!JÐõ
 'Ø!ØÔ+ØÔ.ñ
ô 
ˆõ
 )Ø˜C  1 ˜HÔ%ØÔ+ØÔ.ñ
ô 
ˆð &×-Ò-¨mÔ.AÀ!Ô.DÀbÈ-ÔJ]Ð^`ÔJaÑbÔbˆØ)×.Ò.¨}Ô/BÀ1Ô/EÀrÑJÔJˆð &¨Ô(HÑHˆð '+×&8Ò&8¸ÐZkÐ&8Ñ&lÔ&lÑ#ˆ�|Ø×)Ò)¨-Ñ8Ô8ˆð ×)Ò)¨-Ñ8Ô8ˆà˜oÐ-Ð-r'   r‚   )r   r   r    r   rH   r"   r„   r.   rv   r%   r�   r…   r†   s   @r(   rO  rO  {  s¤   ø€ € € € € ð%L˜{ð %Lð %Lð %Lð %Lð %Lð %LðV #(ð	;.ð ;.à$œ|ð;.ð Ô)ð;.ð  ð	;.ð
 
ˆuŒ|˜UÔ-Ð-Ô	.ð;.ð ;.ð ;.ð ;.ð ;.ð ;.ð ;.ð ;.r'   rO  c                   ó†   ‡ — e Zd Zdefˆ fd„Zdej        dej        dej        fd„Z	de
ej                 dej        fd„Zˆ xZS )	ÚHieraMultiScaleHeadrZ   c           	      óÎ  •‡— t          ¦   «                              ¦   «          ˆfd„t          ‰j        ‰j        ¦  «        D ¦   «         | _        ˆfd„t          t          ‰j        ¦  «        ¦  «        D ¦   «         | _	        ‰j        }t          j        ¦   «         | _        t          ‰j        ¦  «        D ]†}d„ t          || j        ¦  «        D ¦   «         }d„ t          |‰j        ¦  «        D ¦   «         }| j                             t          j        | j	        |         | j	        d         ||¬¦  «        ¦  «         Œ‡| j                             t          j        ¦   «         ¦  «         d S )Nc                 ó0   •— g | ]\  }}||‰j         z  z  ‘ŒS r&   r  r€  s      €r(   rB   z0HieraMultiScaleHead.__init__.<locals>.<listcomp>ä  r‚  r'   c                 óN   •— g | ]!}t          ‰j        ‰j        |z  z  ¦  «        ‘Œ"S r&   ©rp   rV   r  ©r?   r@   rZ   s     €r(   rB   z0HieraMultiScaleHead.__init__.<locals>.<listcomp>ç  s>   ø€ ð !
ð !
ð !
ØGH�C�Ô  6Ô#>ÀÑ#AÑAÑBÔBð!
ð !
ð !
r'   c                 ó   — g | ]
\  }}||z  ‘ŒS r&   r&   r>   s      r(   rB   z0HieraMultiScaleHead.__init__.<locals>.<listcomp>î  s    € ÐkÐkÐk¡  A�a˜1‘fÐkÐkÐkr'   c                 ó   — g | ]
\  }}||z  ‘ŒS r&   r&   r>   s      r(   rB   z0HieraMultiScaleHead.__init__.<locals>.<listcomp>ï  s    € Ð'nÐ'nÐ'n±4°1°a¨¨Q©Ð'nÐ'nÐ'nr'   r–   )rD   rE   )rG   rH   rO   rR   r­   r„  rú   rI   r  Ústage_dimensionsr   rù   Úmulti_scale_fusion_headsr  r  rU   rì   )rY   rZ   Úcurrent_masked_unit_sizeÚidxÚkernelr[   s    `   €r(   rH   zHieraMultiScaleHead.__init__â  s‚  øø€ Ý‰Œ×ÒÑÔÐð.
ð .
ð .
ð .
Ý:=¸fÔ>UÐW]ÔWjÑ:kÔ:kð.
ñ .
ô .
ˆÔ*ð!
ð !
ð !
ð !
ÝLQÕRUÐV\ÔVcÑRdÔRdÑLeÔLeð!
ñ !
ô !
ˆÔð $*Ô#:Ð Ý(*¬©¬ˆÔ%å˜Ô.Ñ/Ô/ð 
	ð 
	ˆCØkÐk­Ð-EÀtÔGiÑ)jÔ)jÐkÑkÔkˆFØ'nÐ'n½3Ð?WÐY_ÔYlÑ;mÔ;mÐ'nÑ'nÔ'nÐ$ØÔ)×0Ò0Ý”	ØÔ)¨#Ô.ØÔ)¨"Ô-Ø &Ø!ð	ñ ô ñô ð ð ð 	Ô%×,Ò,­R¬[©]¬]Ñ;Ô;Ð;Ð;Ð;r'   Úheadr   r]   c                 ó\  — t          |t          j        ¦  «        r|S |j        \  }}}}}|                     ||z  |||¦  «        }|                     dddd¦  «        } ||¦  «        }|                     dddd¦  «        }|j        dd …         \  }}	}|                     ||||	|¦  «        }|S )Nr   r   r   r:   )rL  r   rì   r`   r�   rž   )
rY   r£  r   rw   r  r‘  r’  rª   Úmask_unit_height_finalÚmask_unit_width_finals
             r(   Úapply_fusion_headz%HieraMultiScaleHead.apply_fusion_headú  sã   € Ý�d�BœKÑ(Ô(ð 	!Ø Ð àUbÔUhÑRˆ
�NÐ$4°oÀ{ð &×-Ò-Ø˜Ñ'Ð)9¸?ÈKñ
ô 
ˆð &×-Ò-¨a°°A°qÑ9Ô9ˆØ˜˜]Ñ+Ô+ˆð &×-Ò-¨a°°A°qÑ9Ô9ˆØERÔEXÐYZÐY[ÐY[ÔE\ÑBÐÐ 5°{Ø%×-Ò-Ø˜Ð(>Ð@UÐWbñ
ô 
ˆð Ðr'   Úfeature_mapsc                 ór   — d}t          | j        |¦  «        D ]\  }}||                      ||¦  «        z   }Œ|S )NrÐ   )rO   rŸ  r§  )rY   r¨  r   r£  Úfeature_maps        r(   r�   zHieraMultiScaleHead.forward  sL   € àˆÝ!$ TÔ%BÀLÑ!QÔ!Qð 	Vð 	VÑˆD�+Ø)¨D×,BÒ,BÀ4ÈÑ,UÔ,UÑUˆMˆMàÐr'   )r   r   r    r   rH   r   ÚModuler"   r„   r§  rÿ   r�   r…   r†   s   @r(   r–  r–  á  s    ø€ € € € € ð<˜{ð <ð <ð <ð <ð <ð <ð0 b¤ið ÀÄð ÐQVÔQ]ð ð ð ð ð, D¨¬Ô$6ð ¸5¼<ð ð ð ð ð ð ð ð r'   r–  a;  
    The Hiera Model transformer with the decoder on top for self-supervised pre-training.

    <Tip>

    Note that we provide a script to pre-train this model on custom data in our [examples
    directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining).

    </Tip>
    c                   ó  ‡ — e Zd Zdeddfˆ fd„Zdej        dej        dej        fd„Zdej        dej        dej        fd	„Z	e
	 	 	 	 	 	 ddej        dz  d
ej        dz  dedz  dedz  dedz  dedz  deez  fd„¦   «         Zˆ xZS )ÚHieraForPreTrainingrZ   r]   Nc                 óf  •— t          ¦   «                              |¦  «         t          |dd¬¦  «        | _        t	          j        | j        j        |j        ¬¦  «        | _        t          |¦  «        | _
        t          |¦  «        | _        | j        j        | _        |                      ¦   «          d S )NFT©rh  r8   rä   )rG   rH   rg  rF  r   ræ   rc  rç   Úencoder_normr–  Úmultiscale_fusionrO  Údecoderr‹  rk  ©rY   rZ   r[   s     €r(   rH   zHieraForPreTraining.__init__&  s”   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å ¸%ÈÐMÑMÔMˆŒ
ÝœL¨¬Ô)@ÀfÔF[Ð\Ñ\Ô\ˆÔå!4°VÑ!<Ô!<ˆÔå# FÑ+Ô+ˆŒØœ<Ô3ˆÔð 	�ŠÑÔÐÐÐr'   r\   r,   c                 óŽ  — |                      dddd¦  «        }| j        }|                     d||¦  «                             d||¦  «        }|                     dd¦  «                             d¦  «        }||         }| j        j        r<|                     dd¬¦  «        }|                     dd¬¦  «        }||z
  |dz   d	z  z  }|S )
Nr   r:   r   r   r–   T)rl   Úkeepdimg�íµ ÷Æ°>r—   )rž   r‹  Úunfoldr~   rZ   Únormalize_pixel_lossÚmeanÚvar)rY   r\   r,   r_   Úlabelr¸  r¹  s          r(   Úget_pixel_label_2dz&HieraForPreTraining.get_pixel_label_2d4  sÊ   € à#×+Ò+¨A¨q°!°QÑ7Ô7ˆàÔˆØ×#Ò# A t¨TÑ2Ô2×9Ò9¸!¸TÀ4ÑHÔHˆØ—’˜a Ñ#Ô#×+Ò+¨AÑ.Ô.ˆØ�oÔ&ˆØŒ;Ô+ð 	;Ø—:’: "¨d�:Ñ3Ô3ˆDØ—)’) ¨D�)Ñ1Ô1ˆCØ˜T‘\ c¨F¡l°sÑ%:Ñ:ˆEàˆr'   r5   c                 ó€   — | }|                       ||¦  «        }||         }||z
  dz  }|                     ¦   «         }|S )Nr:   )r»  r¸  )rY   r\   r5   r,   rº  r4   s         r(   Úforward_lossz HieraForPreTraining.forward_lossC  sI   € à*Ð*ˆØ×'Ò'¨°oÑFÔFˆà˜Ô(ˆØ˜‘ 1Ñ$ˆØ�yŠy‰{Œ{ˆàˆr'   rg   r¶   r/  r¤   r0  c           	      óœ  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|                      |||d||¬¦  «        }|d         }	|d         }
|d         }|	d| j        j         j        dz   …         |	d         fz   }	|                      |	¦  «        }|                      |¦  «        }|                      ||
|¬¦  «        \  }}
|  	                    |||
¦  «        }|s9||
|f}|r||d         fz   }|r||d	         fz   }|r||d         fz   }|�|f|z   n|S t          |||
||r|j        nd|j        |r|j        nd¬
¦  «        S )a   
        noise (`torch.FloatTensor` of shape `(batch_size, num_mask_units)`, *optional*):
            Mainly used for testing purposes to control randomness and maintain the reproducibility

        Examples:
        ```python
        >>> from transformers import AutoImageProcessor, HieraForPreTraining
        >>> import torch
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> image_processor = AutoImageProcessor.from_pretrained("facebook/hiera-tiny-224-mae-hf")
        >>> model = HieraForPreTraining.from_pretrained("facebook/hiera-tiny-224-mae-hf")

        >>> inputs = image_processor(images=image, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> logits = outputs.logits
        >>> loss = outputs.loss
        >>> print(list(logits.shape))
        [1, 196, 768]
        ```NT)rg   r¶   r/  r¤   r0  r–   r   r:   )r,   r¶   r   r¸   )r4   r5   r,   r-   r   r   r   )rZ   r0  r¶   r/  rF  r  r±  r°  r²  r½  r3   r   r   r   )rY   r\   rg   r¶   r/  r¤   r0  rs  Úoutputsr¨  r,   Úids_to_restoreÚfused_hidden_statesr5   r4   Úoutputs                   r(   r�   zHieraForPreTraining.forwardN  sä  € ðL &1Ð%<�k�kÀ$Ä+ÔBYˆØ1BÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð —*’*ØØØ/Ø!%Ø%=Ø#ð ñ 
ô 
ˆð ˜r”{ˆØ! !œ*ˆØ  œˆà# A¨¬
Ô(9Ô(HÈ1Ñ(LÐ$LÔMÐQ]Ð^`ÔQaÐPcÑcˆØ"×4Ò4°\ÑBÔBÐØ"×/Ò/Ð0CÑDÔDÐð #'§,¢,ØØ+Ø/ð #/ñ #
ô #
Ñˆ�ð × Ò  ¨v°ÑGÔGˆàð 	FØ˜o¨~Ð>ˆFØ#ð 0Ø 7¨1¤: -Ñ/�Ø ð 0Ø 7¨1¤: -Ñ/�Ø#ð 1Ø 7¨2¤; .Ñ0�Ø)-Ð)9�T�G˜fÑ$Ð$¸vÐEå(ØØØ+Ø&Ø3GÐQ˜'Ô/Ð/ÈTØÔ)ØEYÐ#c 7Ô#AÐ#AÐ_cð
ñ 
ô 
ð 	
r'   rz  )r   r   r    r   rH   r"   r„   r.   r»  r½  r   r#   rv   r%   r3   r�   r…   r†   s   @r(   r­  r­    s`  ø€ € € € € ð˜{ð ¨tð ð ð ð ð ð ð¨u¬|ð ÈeÔN^ð ÐchÔcoð ð ð ð ð	¨¬ð 	¸u¼|ð 	Ð^cÔ^nð 	ð 	ð 	ð 	ð ð -1Ø*.Ø)-Ø,0Ø04Ø#'ðW
ð W
à”l TÑ)ðW
ð Ô  4Ñ'ðW
ð   $™;ð	W
ð
 # T™kðW
ð #'¨¡+ðW
ð ˜D‘[ðW
ð 
Ð*Ñ	*ðW
ð W
ð W
ñ „^ðW
ð W
ð W
ð W
ð W
r'   r­  aæ  
    Hiera Model transformer with an image classification head on top (a linear layer on top of the final hidden state with
    average pooling) e.g. for ImageNet.

    <Tip>

        Note that it's possible to fine-tune Hiera on higher resolution images than the ones it has been trained on, by
        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
        position embeddings to the higher resolution.

    </Tip>
    c                   ó’   ‡ — e Zd Zdeddfˆ fd„Ze	 	 	 	 	 ddej        dz  dedz  dedz  dedz  d	edz  de	e
z  fd
„¦   «         Zˆ xZS )ÚHieraForImageClassificationrZ   r]   Nc                 óF  •— t          ¦   «                              |¦  «         |j        | _        t          |dd¬¦  «        | _        |j        dk    r$t          j        | j        j        |j        ¦  «        nt          j        ¦   «         | _	        |  
                    ¦   «          d S )NTFr¯  r   )rG   rH   Ú
num_labelsrg  rF  r   r³   rc  rì   Ú
classifierrk  r³  s     €r(   rH   z$HieraForImageClassification.__init__¸  s’   ø€ Ý‰Œ×Ò˜Ñ Ô Ð à Ô+ˆŒÝ ¸$ÀuÐMÑMÔMˆŒ
ð FLÔEVÐYZÒEZÐEZ�BŒI�d”jÔ-¨vÔ/@ÑAÔAÐAÕ`bÔ`kÑ`mÔ`mð 	Œð
 	�ŠÑÔÐÐÐr'   Úlabelsr¶   r/  r¤   r0  c                 óŠ  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|                      |||||¬¦  «        }|d         }	|                      |	¦  «        }
d}|�|                      ||
| j         ¦  «        }|s|
f|dd…         z   }|�|f|z   n|S t          ||
|j        |j	        |j
        ¬¦  «        S )aŠ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N)r¶   r/  r¤   r0  r   r:   )r4   r5   r   r   r   )rZ   r0  r¶   r/  rF  rÇ  Úloss_functionr1   r   r   r   )rY   r\   rÈ  r¶   r/  r¤   r0  rs  r¿  re  r5   r4   rÂ  s                r(   r�   z#HieraForImageClassification.forwardÆ  s  € ð" &1Ð%<�k�kÀ$Ä+ÔBYˆØ1BÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð —*’*ØØ/Ø!5Ø%=Ø#ð ñ 
ô 
ˆð   œ
ˆà—’ Ñ/Ô/ˆàˆØÐØ×%Ò% f¨f°d´kÑBÔBˆDàð 	FØ�Y ¨¨¨¤Ñ,ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå0ØØØ!Ô/ØÔ)Ø#*Ô#Að
ñ 
ô 
ð 	
r'   )NNNNN)r   r   r    r   rH   r   r"   r„   rv   r%   r1   r�   r…   r†   s   @r(   rÄ  rÄ  ©  sÚ   ø€ € € € € ð˜{ð ¨tð ð ð ð ð ð ð ð '+Ø)-Ø,0Ø04Ø#'ð0
ð 0
ð ”˜tÑ#ð0
ð   $™;ð	0
ð
 # T™kð0
ð #'¨¡+ð0
ð ˜D‘[ð0
ð 
Ð2Ñ	2ð0
ð 0
ð 0
ñ „^ð0
ð 0
ð 0
ð 0
ð 0
r'   rÄ  zN
    Hiera backbone, to be used with frameworks like DETR and MaskFormer.
    c                   óŠ   ‡ — e Zd Zdefˆ fd„Zd„ Zee	 	 	 ddej	        de
dz  de
dz  de
dz  d	ef
d
„¦   «         ¦   «         Zˆ xZS )ÚHieraBackbonerZ   c                 óâ  •‡— t          ¦   «                              ‰¦  «         ‰j        gˆfd„t          t	          ‰j        ¦  «        ¦  «        D ¦   «         z   | _        t          ‰d¬¦  «        | _        t          ‰¦  «        | _
        i }t          | j        | j        ¦  «        D ]\  }}t          j        |¦  «        ||<   Œt          j        |¦  «        | _        |                      ¦   «          d S )Nc                 óN   •— g | ]!}t          ‰j        ‰j        |z  z  ¦  «        ‘Œ"S r&   rš  r›  s     €r(   rB   z*HieraBackbone.__init__.<locals>.<listcomp>  s>   ø€ ð 2
ð 2
ð 2
ØGH�C�Ô  6Ô#>ÀÑ#AÑAÑBÔBð2
ð 2
ð 2
r'   Fr�   )rG   rH   rV   rú   rI   r  rc  rˆ   r€   r  rj  rO   Úout_featuresÚchannelsr   ræ   Ú
ModuleDictÚhidden_states_normsrk  )rY   rZ   rÒ  r&  rM   r[   s    `   €r(   rH   zHieraBackbone.__init__   sÿ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð à#Ô-Ð.ð 2
ð 2
ð 2
ð 2
ÝLQÕRUÐV\ÔVcÑRdÔRdÑLeÔLeð2
ñ 2
ô 2
ñ 
ˆÔõ *¨&¸Ð?Ñ?Ô?ˆŒÝ# FÑ+Ô+ˆŒð !ÐÝ#& tÔ'8¸$¼-Ñ#HÔ#Hð 	Dð 	DÑˆE�<Ý)+¬°lÑ)CÔ)CÐ Ñ&Ð&Ý#%¤=Ð1DÑ#EÔ#EˆÔ ð 	�ŠÑÔÐÐÐr'   c                 ó   — | j         j        S rƒ   rm  rÝ   s    r(   rn  z"HieraBackbone.get_input_embeddings  ro  r'   Nr\   r/  r¶   r0  r]   c                 ó¼  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|                      |¦  «        \  }}}|                      ||d|¬¦  «        }|d         }	d}
t          | j        |	¦  «        D ]’\  }}|| j        v r„|j	        \  }}}}| 
                    |||z  |¦  «        } | j        |         |¦  «        }| 
                    ||||¦  «        }|                     dddd	¦  «                             ¦   «         }|
|fz  }
Œ“|s!|
f}|r||d         fz  }|r||d	         fz  }|S t          |
|r|d         nd|r|d	         nd¬
¦  «        S )a�  
        Returns:

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, AutoBackbone
        >>> import torch
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> processor = AutoImageProcessor.from_pretrained("facebook/hiera-tiny-224-hf")
        >>> model = AutoBackbone.from_pretrained(
        ...     "facebook/hiera-tiny-224-hf", out_features=["stage1", "stage2", "stage3", "stage4"]
        ... )

        >>> inputs = processor(image, return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> feature_maps = outputs.feature_maps
        >>> list(feature_maps[-1].shape)
        [1, 768, 7, 7]
        ```NT)r¶   r/  r0  r–   r&   r   r   r   r:   )r¨  r   r   )rZ   r0  r/  r¶   r€   rj  rO   Ústage_namesrÏ  r`   ra   rÒ  rž   Ú
contiguousr   )rY   r\   r/  r¶   r0  rs  rt  r¿   r¿  r   r¨  r&  Úhidden_staterw   r“   r”   rM   rÂ  s                     r(   r�   zHieraBackbone.forward  sÓ  € ðJ &1Ð%<�k�kÀ$Ä+ÔBYˆà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà!%§¢°Ñ!>Ô!>ÑÐ˜!˜Qà—,’,ØØ/Ø!%Ø#ð	 ñ 
ô 
ˆð   œˆàˆÝ#& tÔ'7¸Ñ#GÔ#Gð 	0ð 	0ÑˆE�<Ø˜Ô)Ð)Ð)Ø:FÔ:LÑ7�
˜F E¨<Ø+×0Ò0°¸VÀe¹^È\ÑZÔZ�Ø>˜tÔ7¸Ô>¸|ÑLÔL�Ø+×0Ò0°¸VÀUÈLÑYÔY�Ø+×3Ò3°A°q¸!¸QÑ?Ô?×JÒJÑLÔL�Ø  Ñ/�øàð 	Ø"�_ˆFØ#ð (Ø˜7 1œ:˜-Ñ'�Ø ð (Ø˜7 1œ:˜-Ñ'�ØˆMåØ%Ø(<ÐF˜' !œ*˜*À$Ø%6Ð@�w˜q”z�z¸Dð
ñ 
ô 
ð 	
r'   )NNN)r   r   r    r   rH   rn  r   r	   r"   r„   rv   r   r�   r…   r†   s   @r(   rÌ  rÌ  ú  sÛ   ø€ € € € € ð˜{ð ð ð ð ð ð ð$0ð 0ð 0ð Ø ð -1Ø)-Ø#'ðJ
ð J
à”lðJ
ð # T™kðJ
ð   $™;ð	J
ð
 ˜D‘[ðJ
ð 
ðJ
ð J
ð J
ñ !Ô ñ ÔðJ
ð J
ð J
ð J
ð J
r'   rÌ  )rÄ  r­  rÌ  rg  rE  )?r!   rn   Údataclassesr   r"   r   Ú r   rM  Úactivationsr   Úbackbone_utilsr   r	   Úmodeling_layersr
   Úmodeling_outputsr   r   r   r   r   Úmodeling_utilsr   Úutilsr   r   r   Úutils.genericr   Úconfiguration_hierar   Ú
get_loggerr   Úloggerr   r*   r1   r3   r«  r7   rˆ   r©   rÆ   rÏ   rá   rò   r„   rÿ   rp   r  r  r%   rC  rE  r^  rg  rO  r–  r­  rÄ  rÌ  Ú__all__r&   r'   r(   ú<module>rå     s  ðð Ð à €€€Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø HÐ HÐ HÐ HÐ HÐ HÐ HÐ HØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð .Ð -Ð -Ð -Ð -Ð -Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø -Ð -Ð -Ð -Ð -Ð -Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,ð 
ˆÔ	˜HÑ	%Ô	%€ð €ððñ ô ð
 ðHð Hð Hð Hð H˜ñ Hô Hñ „ñô ðHð  €ððñ ô ð
 ðHð Hð Hð Hð H�{ñ Hô Hñ „ñô ðHð2 €ððñ ô ð
 ð
Hð 
Hð 
Hð 
Hð 
HÐ(=ñ 
Hô 
Hñ „ñô ð
Hð €ððñ ô ð
 ðCð Cð Cð Cð C ñ Cô Cñ „ñô ðCð2[8ð [8ð [8ð [8ð [8˜2œ9ñ [8ô [8ð [8ð|I8ð I8ð I8ð I8ð I8�b”iñ I8ô I8ð I8ðX<Yð <Yð <Yð <Yð <Y˜RœYñ <Yô <Yð <Yð~ð ð ð ð ˆrŒyñ ô ð ð%ð %ð %ð %ð %�B”Iñ %ô %ð %ð0:-ð :-ð :-ð :-ð :-�”ñ :-ô :-ð :-ðz,+ð ,+ð ,+ð ,+ð ,+Ð+ñ ,+ô ,+ð ,+ð^ %¤,ð °t¸C´yð ÐSWÐX[ÔS\ð ÐafÔamð ð ð ð ð4J
ð J
ð J
ð J
ð J
�2”9ñ J
ô J
ð J
ðZ3Ø”<ð3Ø.3°C¸°H¬oð3ØMRÐSVÐX[ÐS[Ì_ð3ØhlÐmqÐruÔmvÔhwð3à
„\ð3ð 3ð 3ð 3ðl ðGð Gð Gð Gð G˜?ñ Gô Gñ „ðGð<ð ð ð ð �"”)ñ ô ð ð ð_
ð _
ð _
ð _
ð _
Ð%ñ _
ô _
ñ „ð_
ðDc.ð c.ð c.ð c.ð c.�2”9ñ c.ô c.ð c.ðL5ð 5ð 5ð 5ð 5˜"œ)ñ 5ô 5ð 5ðp €ð	ðñ ô ðA
ð A
ð A
ð A
ð A
Ð.ñ A
ô A
ñô ðA
ðH €ððñ ô ð@
ð @
ð @
ð @
ð @
Ð"6ñ @
ô @
ñô ð@
ðF €ððñ ô ð
b
ð b
ð b
ð b
ð b
�MÐ#7ñ b
ô b
ñô ð
b
ðJ xÐ
wÐ
w€€€r'   