§
    ‚ŠtjÓ  ã                   óœ  — d Z ddlZddlmZ ddlmZ ddlZddl	Z	ddl	m
Z
 ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZ ddlmZmZmZmZm Z m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z'm(Z(m)Z)  e j*        e+¦  «        Z,de	j-        de	j-        fd„Z.de	j-        de	j-        fd„Z/de	j-        de0fd„Z1dJde	j-        de2de3de0de	j-        f
d„Z4dKd„Z5d „ Z6 G d!„ d"e
j7        ¦  «        Z8 G d#„ d$e
j7        ¦  «        Z9 G d%„ d&e
j7        ¦  «        Z:ee G d'„ d(e¦  «        ¦   «         ¦   «         Z; G d)„ d*e
j7        ¦  «        Z< G d+„ d,e
j7        ¦  «        Z= G d-„ d.e
j7        ¦  «        Z> G d/„ d0e
j7        ¦  «        Z? G d1„ d2e
j7        ¦  «        Z@ G d3„ d4e@¦  «        ZA G d5„ d6e
j7        ¦  «        ZB G d7„ d8e¦  «        ZCe G d9„ d:e¦  «        ¦   «         ZD G d;„ d<e
j7        ¦  «        ZE G d=„ d>e
j7        ¦  «        ZF G d?„ d@eD¦  «        ZG G dA„ dBeD¦  «        ZH G dC„ dDe
j7        ¦  «        ZI G dE„ dFeD¦  «        ZJe G dG„ dHeD¦  «        ¦   «         ZKg dI¢ZLdS )LzPyTorch GroupViT model.é    N)Ú	dataclass)ÚAny)Únné   )Úinitialization)ÚACT2FN)Úcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPooling)ÚPreTrainedModel)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚ	torch_int)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚGroupViTConfigÚGroupViTTextConfigÚGroupViTVisionConfigÚlogitsÚreturnc                 óŽ   — t           j                             | t          j        t          | ¦  «        | j        ¬¦  «        ¦  «        S )N©Údevice)r   Ú
functionalÚcross_entropyÚtorchÚarangeÚlenr   )r   s    úl/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/groupvit/modeling_groupvit.pyÚcontrastive_lossr&   *   s3   € ÝŒ=×&Ò& v­u¬|½CÀ¹K¼KÐPVÔP]Ð/^Ñ/^Ô/^Ñ_Ô_Ð_ó    Ú
similarityc                 óX   — t          | ¦  «        }t          | j        ¦  «        }||z   dz  S )Ng       @)r&   ÚT)r(   Úcaption_lossÚ
image_losss      r%   Úimage_text_contrastive_lossr-   /   s.   € Ý# JÑ/Ô/€LÝ! *¤,Ñ/Ô/€JØ˜:Ñ%¨Ñ,Ð,r'   Údimc                 ó  — |                       |¦  «        }|                     |d¬¦  «        d         }t          j        | t          j        ¬¦  «                             ||d¦  «        }||                     ¦   «         z
  |z   }|S )NT©Úkeepdimr   ©Úmemory_formatç      ð?)ÚsoftmaxÚmaxr"   Ú
zeros_likeÚlegacy_contiguous_formatÚscatter_Údetach)r   r.   Úy_softÚindexÚy_hardÚrets         r%   Úhard_softmaxr?   5   sv   € Ø�^Š^˜CÑ Ô €Fà�JŠJ�s DˆJÑ)Ô)¨!Ô,€EÝÔ˜fµEÔ4RÐSÑSÔS×\Ò\Ð]`ÐbgÐilÑmÔm€FØ
�6—=’=‘?”?Ñ
" VÑ
+€Cà€Jr'   FéÿÿÿÿÚtauÚhardc                 ó  — t           j        j                             t          j        d| j        | j        ¬¦  «        t          j        d| j        | j        ¬¦  «        ¦  «        }|                     | j        ¦  «        }| |z   |z  }| 	                    |¦  «        }|rm| 
                    |d¬¦  «        d         }t          j        | t           j        ¬¦  «                             ||d¦  «        }||                     ¦   «         z
  |z   }	n|}	|	S )Nç        )r   Údtyper4   Tr0   r   r2   )r"   ÚdistributionsÚgumbelÚGumbelÚtensorr   rE   ÚsampleÚshaper5   r6   r7   r8   r9   r:   )
r   rA   rB   r.   Úgumbel_distÚgumbelsr;   r<   r=   r>   s
             r%   Úgumbel_softmaxrN   ?   sø   € åÔ%Ô,×3Ò3ÝŒ�S ¤°f´lÐCÑCÔCÝŒ�S ¤°f´lÐCÑCÔCñô €Kð × Ò  ¤Ñ.Ô.€Gà˜Ñ 3Ñ&€GØ�_Š_˜SÑ!Ô!€Fàð à—
’
˜3¨�
Ñ-Ô-¨aÔ0ˆÝÔ! &½Ô8VÐWÑWÔW×`Ò`ÐadÐfkÐmpÑqÔqˆØ�v—}’}‘”Ñ&¨Ñ/ˆˆð ˆØ€Jr'   c                 ó¾  — ||z  | j         d         z  dz  }||k    r5t          t          j        ||z  ¦  «        ¦  «        }| j         d         |z  }n4t          t          j        ||z  ¦  «        ¦  «        }| j         d         |z  }| j         d         }| j         d         }|                      ||||¦  «        } t
          j                             | ||fd|¬¦  «        } | S )a¾  
    Args:
        attentions (`torch.Tensor`): attention map of shape [batch_size, groups, feat_height*feat_width]
        height (`int`): height of the output attention map
        width (`int`): width of the output attention map
        align_corners (`bool`, *optional*): the `align_corner` argument for `nn.functional.interpolate`.

    Returns:
        `torch.Tensor`: resized attention map of shape [batch_size, groups, height, width]
    é   ç      à?r   r   Úbilinear©ÚsizeÚmodeÚalign_corners)rK   ÚintÚnpÚroundÚreshaper   r    Úinterpolate)	Ú
attentionsÚheightÚwidthrV   ÚscaleÚ
feat_widthÚfeat_heightÚ
batch_sizeÚgroupss	            r%   Úresize_attention_maprd   U   sè   € ð �e‰^˜zÔ/°Ô2Ñ2°sÑ:€EØ�‚~€~Ý�œ %¨%¡-Ñ0Ô0Ñ1Ô1ˆ
Ø Ô& qÔ)¨ZÑ7ˆˆå�"œ( 6¨E¡>Ñ2Ô2Ñ3Ô3ˆØÔ% aÔ(¨KÑ7ˆ
àÔ! !Ô$€JØÔ˜aÔ €Fà×#Ò# J°¸ÀZÑPÔP€JÝ”×*Ò*Ø˜& %˜¨zÈð +ñ ô €Jð Ðr'   c           	      óx  — g }t          j        ¦   «         5  d}| D ]~}|                     ddd¦  «                             ¦   «         }|€|}n||z  }t	          |                     ddd¦  «                             ¦   «         g|¢R Ž }|                     |¦  «         Œ	 ddd¦  «         n# 1 swxY w Y   |d         }|S )a1  
    Args:
        attentions (`tuple(torch.FloatTensor)`: tuple of attention maps returned by `GroupViTVisionTransformer`
        hw_shape (`tuple(int)`): height and width of the output attention map
    Returns:
        `torch.Tensor`: the attention map of shape [batch_size, groups, height, width]
    Nr   rP   r   r@   )r"   Úno_gradÚpermuteÚ
contiguousrd   Úappend)r\   Úhw_shapeÚ	attn_mapsÚprev_attn_masksÚ
attn_masksÚcur_attn_mapÚfinal_groupings          r%   Úget_grouping_from_attentionsrp   s   s  € ð €IÝ	Œ‰Œð +ð +ØˆØ$ð 		+ð 		+ˆJà#×+Ò+¨A¨q°!Ñ4Ô4×?Ò?ÑAÔAˆJØÐ&Ø",��à"1°JÑ">�å/°×0GÒ0GÈÈ1ÈaÑ0PÔ0P×0[Ò0[Ñ0]Ô0]ÐiÐ`hÐiÐiÐiˆLØ×Ò˜\Ñ*Ô*Ð*Ð*ð		+ð+ð +ð +ñ +ô +ð +ð +ð +ð +ð +ð +øøøð +ð +ð +ð +ð ˜r”]€NàÐs   –BB'Â'B+Â.B+c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚGroupViTCrossAttentionLayerÚconfigc                 ó,  •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          j        |j        |j        ¬¦  «        | _        t          |¦  «        | _
        t	          j        |j        |j        ¬¦  «        | _        d S ©N©Úeps)ÚsuperÚ__init__ÚGroupViTAttentionÚattnr   Ú	LayerNormÚhidden_sizeÚlayer_norm_epsÚnorm2ÚGroupViTMLPÚmlpÚ	norm_post©Úselfrs   Ú	__class__s     €r%   ry   z$GroupViTCrossAttentionLayer.__init__‘   ss   ø€ Ý‰Œ×ÒÑÔÐÝ% fÑ-Ô-ˆŒ	Ý”\ &Ô"4¸&Ô:OÐPÑPÔPˆŒ
Ý˜vÑ&Ô&ˆŒÝœ fÔ&8¸fÔ>SÐTÑTÔTˆŒˆˆr'   c                 óÊ   — |}||                       ||¬¦  «        d         z   }||                      |                      |¦  «        ¦  «        z   }|                      |¦  «        }|S )N)Úencoder_hidden_statesr   )r{   r�   r   r‚   )r„   ÚqueryÚkeyÚxs       r%   Úforwardz#GroupViTCrossAttentionLayer.forward˜   s\   € ØˆØ�—	’	˜%°s�	Ñ;Ô;¸AÔ>Ñ>ˆØ�—’˜Ÿš A™œÑ'Ô'Ñ'ˆØ�NŠN˜1ÑÔˆØˆr'   )Ú__name__Ú
__module__Ú__qualname__r   ry   r‹   Ú__classcell__©r…   s   @r%   rr   rr   �   s[   ø€ € € € € ðUÐ3ð Uð Uð Uð Uð Uð Uðð ð ð ð ð ð r'   rr   c                   ó2   ‡ — e Zd Zdefˆ fd„Zdd„Zd„ Zˆ xZS )ÚGroupViTAssignAttentionrs   c                 óž  •— t          ¦   «                              ¦   «          |j        dz  | _        t	          j        |j        |j        ¦  «        | _        t	          j        |j        |j        ¦  «        | _        t	          j        |j        |j        ¦  «        | _        t	          j        |j        |j        ¦  «        | _	        |j
        | _
        d S )Nç      à¿)rx   ry   r}   r_   r   ÚLinearÚq_projÚk_projÚv_projÚprojÚ
assign_epsrƒ   s     €r%   ry   z GroupViTAssignAttention.__init__¡   sš   ø€ Ý‰Œ×ÒÑÔÐØÔ'¨Ñ-ˆŒ
å”i Ô 2°FÔ4FÑGÔGˆŒÝ”i Ô 2°FÔ4FÑGÔGˆŒÝ”i Ô 2°FÔ4FÑGÔGˆŒÝ”I˜fÔ0°&Ô2DÑEÔEˆŒ	Ø Ô+ˆŒˆˆr'   Tc                 ó¨   — |r| j         rt          |d|¬¦  «        }n5|rt          |d¬¦  «        }n!t          j                             |d¬¦  «        }|S )Néþÿÿÿ)r.   rB   ©r.   )ÚtrainingrN   r?   r   r    r5   )r„   r{   rG   rB   s       r%   Úget_attnz GroupViTAssignAttention.get_attn«   sd   € Øð 	;�d”mð 	;Ý! $¨B°TÐ:Ñ:Ô:ˆDˆDàð ;Ý# D¨bÐ1Ñ1Ô1��å”}×,Ò,¨T°rÐ,Ñ:Ô:�àˆr'   c                 ó   — |}|                       |¦  «        }|                      |¦  «        }|                      |¦  «        }||                     dd¦  «        z  | j        z  }|                      |¦  «        }|                      |dd¬¦  «        }||                     dd¬¦  «        | j        z   z  }||z  }|                      |¦  «        }||fS )Nrœ   r@   F)rG   rB   T©r.   r1   )	r–   r—   r˜   Ú	transposer_   rŸ   Úsumrš   r™   )r„   rˆ   r‰   ÚvalueÚraw_attnr{   Ú	soft_attnÚouts           r%   r‹   zGroupViTAssignAttention.forward¶   sÊ   € Øˆà—’˜EÑ"Ô"ˆð �kŠk˜#ÑÔˆð —’˜EÑ"Ô"ˆð ˜CŸMšM¨"¨bÑ1Ô1Ñ1°T´ZÑ?ˆà�}Š}˜XÑ&Ô&ˆØ—M’M (°5¸u�MÑEÔEˆ	à�t—x’x B°�xÑ5Ô5¸¼ÑGÑHˆà�U‰lˆà�iŠi˜‰nŒnˆà�Iˆ~Ðr'   )TT)rŒ   r�   rŽ   r   ry   rŸ   r‹   r�   r�   s   @r%   r’   r’       sh   ø€ € € € € ð,Ð3ð ,ð ,ð ,ð ,ð ,ð ,ð	ð 	ð 	ð 	ðð ð ð ð ð ð r'   r’   c                   ó0   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zˆ xZS )ÚGroupViTTokenAssignrs   c                 óÔ  •‡— t          ¦   «                              ¦   «          || _        t          j        ‰j        ‰j        ¬¦  «        | _        t          ‰j	        t          j        j        ¦  «        r‰j	        n‰j	        ‰j	        f}ˆfd„|D ¦   «         \  }}t          ‰|||¦  «        | _        t          j        ‰j        ‰j        ¬¦  «        | _        t          j        ‰j        ‰j        ¬¦  «        | _        t#          ‰¦  «        | _        t'          ‰¦  «        | _        t          j        ‰j        ‰j        ¬¦  «        | _        t-          ‰‰j        |‰j        ¦  «        | _        d S )Nrv   c                 ó>   •— g | ]}t          |‰j        z  ¦  «        ‘ŒS © )rW   r}   )Ú.0rŠ   rs   s     €r%   ú
<listcomp>z0GroupViTTokenAssign.__init__.<locals>.<listcomp>Û   s)   ø€ Ð#ZÐ#ZÐ#ZÀA¥C¨¨FÔ,>Ñ(>Ñ$?Ô$?Ð#ZÐ#ZÐ#Zr'   )rx   ry   Únum_output_groupr   r|   r}   r~   Únorm_tokensÚ
isinstanceÚassign_mlp_ratioÚcollectionsÚabcÚIterableÚGroupViTMixerMLPÚ	mlp_interÚnorm_post_tokensÚnorm_xrr   Úpre_assign_attnr’   ÚassignÚ
norm_new_xr€   Úmlp_channels)r„   rs   Únum_group_tokenr¯   r²   Ú
tokens_dimÚchannels_dimr…   s    `     €r%   ry   zGroupViTTokenAssign.__init__Ñ   sF  øø€ Ý‰Œ×ÒÑÔÐØ 0ˆÔåœ<¨Ô(:ÀÔ@UÐVÑVÔVˆÔõ ˜&Ô1µ;´?Ô3KÑLÔLðDˆFÔ#Ð#àÔ)¨6Ô+BÐCð 	ð
 $[Ð#ZÐ#ZÐ#ZÐIYÐ#ZÑ#ZÔ#ZÑ ˆ
�LÝ)¨&°/À:ÐO_Ñ`Ô`ˆŒÝ "¤¨VÔ-?ÀVÔEZÐ [Ñ [Ô [ˆÔå”l 6Ô#5¸6Ô;PÐQÑQÔQˆŒÝ:¸6ÑBÔBˆÔå-¨fÑ5Ô5ˆŒÝœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ'¨°Ô0BÀLÐRXÔRdÑeÔeˆÔÐÐr'   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S )zæ
        Args:
            group_tokens (torch.Tensor): group tokens, [batch_size, num_group_tokens, channels]

        Returns:
            projected_group_tokens (torch.Tensor): [batch_size, num_output_groups, channels]
        )r·   r¸   )r„   Úgroup_tokensÚprojected_group_tokenss      r%   Úproject_group_tokenz'GroupViTTokenAssign.project_group_tokenæ   s1   € ð "&§¢°Ñ!=Ô!=ÐØ!%×!6Ò!6Ð7MÑ!NÔ!NÐØ%Ð%r'   c                 óF  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      ||¦  «        }|                      ||¦  «        \  }}||z  }||                      |                      |¦  «        ¦  «        z   }||fS )zà
        Args:
            image_tokens (`torch.Tensor`): image tokens, of shape [batch_size, input_length, channels]
            group_tokens (`torch.Tensor`): group tokens, [batch_size, num_group_tokens, channels]
        )r°   r¹   rÄ   rº   r»   r½   r¼   )r„   Úimage_tokensrÂ   rÃ   Únew_image_tokensÚ	attentions         r%   r‹   zGroupViTTokenAssign.forwardó   sª   € ð ×'Ò'¨Ñ5Ô5ˆØ—{’{ <Ñ0Ô0ˆà!%×!9Ò!9¸,Ñ!GÔ!GÐØ!%×!5Ò!5Ð6LÈlÑ![Ô![ÐØ&*§k¢kÐ2HÈ,Ñ&WÔ&WÑ#Ð˜)ØÐ2Ñ2Ðà+¨d×.?Ò.?ÀÇÂÐP`Ñ@aÔ@aÑ.bÔ.bÑbÐà Ð*Ð*r'   )rŒ   r�   rŽ   r   ry   rÄ   r‹   r�   r�   s   @r%   r©   r©   Ð   sj   ø€ € € € € ðfÐ3ð fð fð fð fð fð fð*&ð &ð &ð+ð +ð +ð +ð +ð +ð +r'   r©   c                   óü   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dZ
ej        dz  ed<   dZej        dz  ed<   dZej        dz  ed<   dZeed	<   dZeed
<   dee         fd„ZdS )ÚGroupViTModelOutputaí  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
        Contrastive loss for image-text similarity.
    logits_per_image (`torch.FloatTensor` of shape `(image_batch_size, text_batch_size)`):
        The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the image-text
        similarity scores.
    logits_per_text (`torch.FloatTensor` of shape `(text_batch_size, image_batch_size)`):
        The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the text-image
        similarity scores.
    segmentation_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels, logits_height, logits_width)`):
        Classification scores for each pixel.

        <Tip warning={true}>

        The logits returned do not necessarily have the same size as the `pixel_values` passed as inputs. This is
        to avoid doing two interpolations and lose some quality when a user needs to resize the logits to the
        original image size as post-processing. You should always check your logits shape and resize as needed.

        </Tip>
    text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):
        The text embeddings obtained by applying the projection layer to the pooled output of
        [`GroupViTTextModel`].
    image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):
        The image embeddings obtained by applying the projection layer to the pooled output of
        [`GroupViTVisionModel`].
    text_model_output (`BaseModelOutputWithPooling`):
        The output of the [`GroupViTTextModel`].
    vision_model_output (`BaseModelOutputWithPooling`):
        The output of the [`GroupViTVisionModel`].
    NÚlossÚlogits_per_imageÚlogits_per_textÚsegmentation_logitsÚtext_embedsÚimage_embedsÚtext_model_outputÚvision_model_outputr   c                 ó^   ‡ — t          ˆ fd„‰                      ¦   «         D ¦   «         ¦  «        S )Nc              3   ót   •K  — | ]2}|d vr‰|         n!t          ‰|¦  «                             ¦   «         V — Œ3dS ))rÑ   rÒ   N)ÚgetattrÚto_tuple)r­   Úkr„   s     €r%   ú	<genexpr>z/GroupViTModelOutput.to_tuple.<locals>.<genexpr>3  sc   øè è € ð 
ð 
àð Ð LÐLÐLˆD�ŒGˆGÕRYÐZ^Ð`aÑRbÔRb×RkÒRkÑRmÔRmð
ð 
ð 
ð 
ð 
ð 
r'   )ÚtupleÚkeys©r„   s   `r%   rÖ   zGroupViTModelOutput.to_tuple2  sC   ø€ Ýð 
ð 
ð 
ð 
à—Y’Y‘[”[ð
ñ 
ô 
ñ 
ô 
ð 	
r'   )rŒ   r�   rŽ   Ú__doc__rË   r"   ÚFloatTensorÚ__annotations__rÌ   rÍ   rÎ   rÏ   rÐ   rÑ   r   rÒ   rÙ   r   rÖ   r¬   r'   r%   rÊ   rÊ     s÷   € € € € € € ðð ð> &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø15Ð�eÔ'¨$Ñ.Ð5Ð5Ñ5Ø04€O�UÔ&¨Ñ-Ð4Ð4Ñ4Ø48Ð˜Ô*¨TÑ1Ð8Ð8Ñ8Ø,0€K�Ô" TÑ)Ð0Ð0Ñ0Ø-1€L�%Ô# dÑ*Ð1Ð1Ñ1Ø48ÐÐ1Ð8Ð8Ñ8Ø6:ÐÐ3Ð:Ð:Ñ:ð
˜% œ*ð 
ð 
ð 
ð 
ð 
ð 
r'   rÊ   c            	       ó¦   ‡ — e Zd ZdZ	 	 	 	 ddeee         z  eeef         z  deeeef         z  ded	efˆ fd
„Zddej	        de
dej	        fd„Zˆ xZS )ÚGroupViTPatchEmbeddingsz#
    Image to Patch Embedding.
    éà   é   r   é   Ú
image_sizeÚ
patch_sizeÚnum_channelsÚ	embed_dimc                 ó†  •— t          ¦   «                              ¦   «          t          |t          j        j        ¦  «        r|n||f}t          |t          j        j        ¦  «        r|n||f}|d         |d         z  |d         |d         z  z  }|| _        || _        || _        t          j
        ||||¬¦  «        | _        d S )Nr   r   )Úkernel_sizeÚstride)rx   ry   r±   r³   r´   rµ   rä   rå   Únum_patchesr   ÚConv2dÚ
projection)r„   rä   rå   ræ   rç   rë   r…   s         €r%   ry   z GroupViTPatchEmbeddings.__init__>  s½   ø€ õ 	‰Œ×ÒÑÔÐÝ#-¨j½+¼/Ô:RÑ#SÔ#SÐq�Z�ZÐZdÐfpÐYqˆ
Ý#-¨j½+¼/Ô:RÑ#SÔ#SÐq�Z�ZÐZdÐfpÐYqˆ
Ø! !”}¨
°1¬Ñ5¸*ÀQ¼-È:ÐVWÌ=Ñ:XÑYˆØ$ˆŒØ$ˆŒØ&ˆÔåœ) L°)ÈÐ\fÐgÑgÔgˆŒˆˆr'   FÚpixel_valuesÚinterpolate_pos_encodingr   c                 óB  — |j         \  }}}}|sT|| j        d         k    s|| j        d         k    r2t          d|› d|› d| j        d         › d| j        d         › d�	¦  «        ‚|                      |¦  «                             d¦  «                             dd¦  «        }|S )Nr   r   zInput image size (Ú*z) doesn't match model (ú).rP   )rK   rä   Ú
ValueErrorrí   Úflattenr¢   )r„   rî   rï   rb   ræ   r]   r^   rŠ   s           r%   r‹   zGroupViTPatchEmbeddings.forwardO  sÚ   € Ø2>Ô2DÑ/ˆ
�L &¨%Ø'ð 	Ø˜œ¨Ô+Ò+Ð+¨u¸¼ÈÔ8JÒ/JÐ/JÝ ðE¨ð Eð E°%ð Eð EØœ¨Ô+ðEð EØ.2¬o¸aÔ.@ðEð Eð Eñô ð ð �OŠO˜LÑ)Ô)×1Ò1°!Ñ4Ô4×>Ò>¸qÀ!ÑDÔDˆØˆr'   )rá   râ   r   rã   ©F)rŒ   r�   rŽ   rÜ   rW   ÚlistrÙ   ry   r"   ÚTensorÚboolr‹   r�   r�   s   @r%   rà   rà   9  sá   ø€ € € € € ðð ð 9<Ø,.ØØðhð hà˜$˜sœ)‘O e¨C°¨H¤oÑ5ðhð ˜%  S œ/Ñ)ðhð ð	hð
 ðhð hð hð hð hð hð"	ð 	 E¤Lð 	ÈDð 	Ð]bÔ]ið 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	r'   rà   c                   óz   ‡ — e Zd Zdefˆ fd„Zdej        dededej        fd„Zdd	ej        d
e	dej        fd„Z
ˆ xZS )ÚGroupViTVisionEmbeddingsrs   c                 óÈ  •— t          ¦   «                              ¦   «          t          |j        |j        |j        |j        ¬¦  «        | _        | j        j        }t          j
        t          j        d||j        ¦  «        ¦  «        | _        t          j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        |j        | _        || _        d S )N)rä   rå   ræ   rç   r   rv   )rx   ry   rà   rä   rå   ræ   r}   Úpatch_embeddingsrë   r   Ú	Parameterr"   ÚzerosÚposition_embeddingsÚDropoutÚdropoutr|   r~   Ú	layernormrs   )r„   rs   rë   r…   s      €r%   ry   z!GroupViTVisionEmbeddings.__init__\  s¼   ø€ Ý‰Œ×ÒÑÔÐå 7ØÔ(ØÔ(ØÔ,ØÔ(ð	!
ñ !
ô !
ˆÔð Ô+Ô7ˆÝ#%¤<µ´¸A¸{ÈFÔL^Ñ0_Ô0_Ñ#`Ô#`ˆÔ Ý”z &¤.Ñ1Ô1ˆŒÝœ fÔ&8¸fÔ>SÐTÑTÔTˆŒØ Ô+ˆŒØˆŒˆˆr'   Ú
embeddingsr]   r^   r   c                 ó  — |j         d         }| j        j         d         }t          j                             ¦   «         s||k    r||k    r| j        S | j        }|j         d         }|| j        z  }|| j        z  }	t          |dz  ¦  «        }
|                     d|
|
|¦  «        }|                     dddd¦  «        }t          j
                             |||	fdd¬	¦  «        }|                     dddd¦  «                             dd|¦  «        }|S )
a  
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing and no class embeddings.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   r@   rQ   r   r   rP   ÚbicubicFrS   )rK   rÿ   r"   ÚjitÚ
is_tracingrå   r   rZ   rg   r   r    r[   Úview)r„   r  r]   r^   rë   Únum_positionsÚpatch_pos_embedr.   Ú
new_heightÚ	new_widthÚsqrt_num_positionss              r%   rï   z1GroupViTVisionEmbeddings.interpolate_pos_encodingl  s*  € ð !Ô& qÔ)ˆØÔ0Ô6°qÔ9ˆõ Œy×#Ò#Ñ%Ô%ð 	,¨+¸Ò*FÐ*FÈ6ÐUZÊ?È?ØÔ+Ð+àÔ2ˆàÔ˜rÔ"ˆà˜tœÑ.ˆ
Ø˜Tœ_Ñ,ˆ	å& }°cÑ'9Ñ:Ô:ÐØ)×1Ò1°!Ð5GÐI[Ð]`ÑaÔaˆØ)×1Ò1°!°Q¸¸1Ñ=Ô=ˆåœ-×3Ò3ØØ˜iÐ(ØØð	 4ñ 
ô 
ˆð *×1Ò1°!°Q¸¸1Ñ=Ô=×BÒBÀ1ÀbÈ#ÑNÔNˆØÐr'   Frî   rï   c                 ó  — |j         \  }}}}|                      ||¬¦  «        }|                      |¦  «        }|                     ¦   «         \  }}}	|r||                      |||¦  «        z   }n
|| j        z   }|                      |¦  «        }|S )N)rï   )rK   rü   r  rT   rï   rÿ   r  )
r„   rî   rï   rb   ræ   r]   r^   r  Úseq_lenÚ_s
             r%   r‹   z GroupViTVisionEmbeddings.forward’  s¡   € Ø2>Ô2DÑ/ˆ
�L &¨%Ø×*Ò*¨<ÐRjÐ*ÑkÔkˆ
à—^’^ JÑ/Ô/ˆ
à!+§¢Ñ!2Ô!2Ñˆ
�G˜Qð $ð 	?Ø# d×&CÒ&CÀJÐPVÐX]Ñ&^Ô&^Ñ^ˆJˆJà# dÔ&>Ñ>ˆJà—\’\ *Ñ-Ô-ˆ
àÐr'   rõ   )rŒ   r�   rŽ   r   ry   r"   r÷   rW   rï   rø   r‹   r�   r�   s   @r%   rú   rú   [  s±   ø€ € € € € ðÐ3ð ð ð ð ð ð ð $°5´<ð $Èð $ÐUXð $Ð]bÔ]ið $ð $ð $ð $ðLð  E¤Lð ÈDð Ð]bÔ]ið ð ð ð ð ð ð ð r'   rú   c            	       ó~   ‡ — e Zd Zdefˆ fd„Z	 	 	 d	dej        dz  dej        dz  dej        dz  dej        fd„Z	ˆ xZ
S )
ÚGroupViTTextEmbeddingsrs   c                 óV  •— t          ¦   «                              ¦   «          |j        }t          j        |j        |¦  «        | _        t          j        |j        |¦  «        | _        |  	                    dt          j        |j        ¦  «                             d¦  «        d¬¦  «         d S )NÚposition_ids©r   r@   F)Ú
persistent)rx   ry   r}   r   Ú	EmbeddingÚ
vocab_sizeÚtoken_embeddingÚmax_position_embeddingsÚposition_embeddingÚregister_bufferr"   r#   Úexpand©r„   rs   rç   r…   s      €r%   ry   zGroupViTTextEmbeddings.__init__§  sœ   ø€ Ý‰Œ×ÒÑÔÐØÔ&ˆ	å!œ|¨FÔ,=¸yÑIÔIˆÔÝ"$¤,¨vÔ/MÈyÑ"YÔ"YˆÔð 	×ÒØ�EœL¨Ô)GÑHÔH×OÒOÐPWÑXÔXÐejð 	ñ 	
ô 	
ð 	
ð 	
ð 	
r'   NÚ	input_idsr  Úinputs_embedsr   c                 ó.  — |�|j         d         n|j         d         }| j        j        j         d         }||k    rt          d|› d|› �¦  «        ‚|€| j        d d …d |…f         }|€|                      |¦  «        }|                      |¦  «        }||z   }|S )Nr@   rœ   r   zRSequence length must be less than max_position_embeddings (got `sequence length`: z and max_position_embeddings: )rK   r  Úweightró   r  r  )r„   r  r  r   Ú
seq_lengthÚmax_position_embeddingrÿ   r  s           r%   r‹   zGroupViTTextEmbeddings.forward³  sØ   € ð -6Ð,A�Y”_ RÔ(Ð(À}ÔGZÐ[]ÔG^ˆ
Ø!%Ô!8Ô!?Ô!EÀaÔ!HÐàÐ.Ò.Ð.ÝðVØðVð VØ=SðVð Vñô ð ð
 ÐØÔ,¨Q¨Q¨Q°°°¨^Ô<ˆLàÐ Ø ×0Ò0°Ñ;Ô;ˆMà"×5Ò5°lÑCÔCÐØ"Ð%8Ñ8ˆ
àÐr'   ©NNN)rŒ   r�   rŽ   r   ry   r"   Ú
LongTensorrÝ   r÷   r‹   r�   r�   s   @r%   r  r  ¦  s©   ø€ € € € € ð

Ð1ð 

ð 

ð 

ð 

ð 

ð 

ð .2Ø04Ø26ð	ð àÔ# dÑ*ðð Ô&¨Ñ-ðð Ô(¨4Ñ/ð	ð
 
Œðð ð ð ð ð ð ð r'   r  c            
       óâ   ‡ — e Zd ZdZdededededef
ˆ fd„Zed„ ¦   «         Zd	„ Z	dde
j        de
j        d
z  de
j        fd„Z	 	 dde
j        de
j        d
z  ded
z  dee
j                 fd„Zˆ xZS )ÚGroupViTStagezMThis corresponds to the `GroupingLayer` class in the GroupViT implementation.rs   ÚdepthÚnum_prev_group_tokenr¾   r¯   c           	      ó^  •‡— t          ¦   «                              ¦   «          || _        || _        |dk    r3t	          j        t          j        d|‰j        ¦  «        ¦  «        | _	        nd | _	        t	          j
        ˆfd„t          |¦  «        D ¦   «         ¦  «        | _        |dk    rt          ‰||¬¦  «        | _        nd | _        |dk    rX|dk    rRt	          j        t	          j        ‰j        ‰j        ¬¦  «        t%          ‰|‰j        dz  |¦  «        ¦  «        | _        d S d | _        d S )Nr   r   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r¬   ©ÚGroupViTEncoderLayer©r­   r  rs   s     €r%   r®   z*GroupViTStage.__init__.<locals>.<listcomp>à  s"   ø€ Ð$XÐ$XÐ$XÀaÕ%9¸&Ñ%AÔ%AÐ$XÐ$XÐ$Xr'   )rs   r¾   r¯   rv   rP   )rx   ry   r)  r¾   r   rý   r"   rþ   r}   Úgroup_tokenÚ
ModuleListÚrangeÚlayersr©   Ú
downsampleÚ
Sequentialr|   r~   r¶   Úgroup_projector)r„   rs   r)  r*  r¾   r¯   r…   s    `    €r%   ry   zGroupViTStage.__init__Ñ  s9  øø€ õ 	‰Œ×ÒÑÔÐØˆŒ
Ø.ˆÔØ˜QÒÐÝ!œ|­E¬K¸¸?ÈFÔL^Ñ,_Ô,_Ñ`Ô`ˆDÔÐà#ˆDÔÝ”mÐ$XÐ$XÐ$XÐ$XÍ5ÐQVÉ<Ì<Ð$XÑ$XÔ$XÑYÔYˆŒà˜QÒÐÝ1ØØ /Ø!1ðñ ô ˆDŒOˆOð #ˆDŒOà !Ò#Ð#¨¸!Ò(;Ð(;Ý#%¤=Ý”˜VÔ/°VÔ5JÐKÑKÔKÝ  Ð)=¸vÔ?QÐUVÑ?VÐXgÑhÔhñ$ô $ˆDÔ Ð Ð ð
 $(ˆDÔ Ð Ð r'   c                 ó   — | j         d uS ©N)r0  rÛ   s    r%   Úwith_group_tokenzGroupViTStage.with_group_tokenó  s   € àÔ tÐ+Ð+r'   c                 óh   — | j         r(|d d …d | j         …f         |d d …| j         d …f         fS |d fS r8  )r9  r¾   )r„   rŠ   s     r%   Úsplit_xzGroupViTStage.split_x÷  sU   € ØÔ ð 	Ø�Q�Q�QÐ/˜4Ô/Ð/Ð/Ð/Ô0°!°A°A°A¸Ô8LÐ7LÐ7NÐ7NÐ4NÔ2OÐOÐOà�d�7ˆNr'   NrŠ   r0  r   c                 ó:   — |€|S t          j        ||gd¬¦  «        S )Nr   r�   )r"   Úcat)r„   rŠ   r0  s      r%   Úconcat_xzGroupViTStage.concat_xý  s'   € ØÐØˆHÝŒy˜!˜[Ð)¨qÐ1Ñ1Ô1Ð1r'   FÚhidden_statesÚprev_group_tokenÚoutput_attentionsc                 ó   — | j         rO| j                             |                     d¦  «        dd¦  «        }| j        �||                      |¦  «        z   }nd}|}|                      ||¦  «        }| j        D ]} ||d¬¦  «        }Œ|                      |¦  «        \  }}d}| j        �|                      ||¦  «        \  }}||f}	|r|	|fz   }	|	S )aø  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
                `(config.encoder_attention_heads,)`.
            output_attentions (`bool`, *optional*):
                Whether or not to return the grouping tensors of Grouping block.
        r   r@   N)Úattention_mask)	r9  r0  r  rT   r6  r>  r3  r;  r4  )
r„   r?  r@  rA  r0  rŠ   Úcat_xÚlayerrÈ   Úoutputss
             r%   r‹   zGroupViTStage.forward  sú   € ð Ô ð 	ØÔ*×1Ò1°-×2DÒ2DÀQÑ2GÔ2GÈÈRÑPÔPˆKØÔ#Ð/Ø)¨D×,@Ò,@ÐAQÑ,RÔ,RÑR�øàˆKàˆà—’˜a Ñ-Ô-ˆØ”[ð 	6ð 	6ˆEØ�E˜%°Ð5Ñ5Ô5ˆEˆEàŸš eÑ,Ô,‰ˆˆ;àˆ	ØŒ?Ð&ØŸ?š?¨1¨kÑ:Ô:‰LˆAˆyà�kÐ"ˆØð 	-Ø  Ñ,ˆGàˆr'   r8  )NF)rŒ   r�   rŽ   rÜ   r   rW   ry   Úpropertyr9  r;  r"   r÷   r>  rø   rÙ   rÝ   r‹   r�   r�   s   @r%   r(  r(  Î  s5  ø€ € € € € ØWÐWð (à$ð (ð ð (ð "ð	 (ð
 ð (ð ð (ð  (ð  (ð  (ð  (ð  (ðD ð,ð ,ñ „Xð,ðð ð ð2ð 2˜%œ,ð 2°U´\ÀDÑ5Hð 2ÐTYÔT`ð 2ð 2ð 2ð 2ð 15Ø).ð	&ð &à”|ð&ð  œ,¨Ñ-ð&ð   $™;ð	&ð
 
ˆuÔ Ô	!ð&ð &ð &ð &ð &ð &ð &ð &r'   r(  c            
       ón   ‡ — e Zd Z	 	 	 d
dededz  dedz  dedz  fˆ fd„Zdej        dej        fd	„Zˆ xZ	S )r€   Nrs   r}   Úintermediate_sizeÚoutput_sizec                 ó$  •— t          ¦   «                              ¦   «          || _        t          |j                 | _        |�|n|j        }|�|n|j        }|�|n|}t          j	        ||¦  «        | _
        t          j	        ||¦  «        | _        d S r8  )rx   ry   rs   r   Ú
hidden_actÚactivation_fnr}   rI  r   r•   Úfc1Úfc2)r„   rs   r}   rI  rJ  r…   s        €r%   ry   zGroupViTMLP.__init__,  s’   ø€ õ 	‰Œ×ÒÑÔÐØˆŒÝ# FÔ$5Ô6ˆÔØ%0Ð%<�k�kÀ&ÔBTˆØ1BÐ1NÐ-Ð-ÐTZÔTlÐØ%0Ð%<�k�kÀ+ˆÝ”9˜[Ð*;Ñ<Ô<ˆŒÝ”9Ð.°Ñ<Ô<ˆŒˆˆr'   r?  r   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r8  )rN  rM  rO  )r„   r?  s     r%   r‹   zGroupViTMLP.forward<  s=   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆØÐr'   r%  )
rŒ   r�   rŽ   r   rW   ry   r"   r÷   r‹   r�   r�   s   @r%   r€   r€   +  s¥   ø€ € € € € ð #'Ø(,Ø"&ð=ð =à$ð=ð ˜4‘Zð=ð  ™:ð	=ð
 ˜4‘Zð=ð =ð =ð =ð =ð =ð  U¤\ð °e´lð ð ð ð ð ð ð ð r'   r€   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )r¶   c                 óš   •— t          ¦   «                              |                     dd¦  «        ¦  «        }|                     dd¦  «        S ©Nr   rP   )rx   r‹   r¢   )r„   rŠ   r…   s     €r%   r‹   zGroupViTMixerMLP.forwardD  s:   ø€ Ý‰GŒG�OŠO˜AŸKšK¨¨1Ñ-Ô-Ñ.Ô.ˆØ�{Š{˜1˜aÑ Ô Ð r'   )rŒ   r�   rŽ   r‹   r�   r�   s   @r%   r¶   r¶   C  s8   ø€ € € € € ð!ð !ð !ð !ð !ð !ð !ð !ð !r'   r¶   c                   ó²   ‡ — e Zd ZdZˆ fd„Zdej        dedefd„Z	 	 ddej        d	ej        dz  d
ej	        dz  de
ej        ej        dz  f         fd„Zˆ xZS )rz   z=Multi-headed attention from 'Attention Is All You Need' paperc                 ót  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        | j        | j        z  | _        | j        | j        z  | j        k    r t          d| j        › d| j        › d�¦  «        ‚| j        dz  | _	        |j
        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        d S )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: rò   r”   )rx   ry   rs   r}   rç   Únum_attention_headsÚ	num_headsÚhead_dimró   r_   Úattention_dropoutr  r   r•   r—   r˜   r–   Úout_projrƒ   s     €r%   ry   zGroupViTAttention.__init__L  s  ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØÔ3ˆŒØœ¨$¬.Ñ8ˆŒØŒ=˜4œ>Ñ)¨T¬^Ò;Ð;Ýð'ÈdÌnð 'ð 'Ø”Nð'ð 'ð 'ñô ð ð ”] DÑ(ˆŒ
ØÔ/ˆŒå”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝœ	 $¤.°$´.ÑAÔAˆŒˆˆr'   rI   r  Úbszc                 ó’   — |                      ||| j        | j        ¦  «                             dd¦  «                             ¦   «         S rS  )r  rW  rX  r¢   rh   )r„   rI   r  r[  s       r%   Ú_shapezGroupViTAttention._shape_  s<   € Ø�{Š{˜3 ¨¬¸¼ÑGÔG×QÒQÐRSÐUVÑWÔW×bÒbÑdÔdÐdr'   Nr?  rC  r‡   r   c                 óP  — |                      ¦   «         \  }}}|du}|                      |¦  «        | j        z  }	|rU|                      |                      |¦  «        d|¦  «        }
|                      |                      |¦  «        d|¦  «        }nT|                      |                      |¦  «        d|¦  «        }
|                      |                      |¦  «        d|¦  «        }|| j        z  d| j        f} |                      |	||¦  «        j        |Ž }	 |
j        |Ž }
 |j        |Ž }|
                      d¦  «        }t          j
        |	|
                     dd¦  «        ¦  «        }|                      ¦   «         || j        z  ||fk    r2t          d|| j        z  ||f› d|                      ¦   «         › �¦  «        ‚|�†|                      ¦   «         |d||fk    r+t          d|d||f› d|                      ¦   «         › �¦  «        ‚|                     || j        ||¦  «        |z   }|                     || j        z  ||¦  «        }t          j                             |d¬¦  «        }|                     || j        ||¦  «        }|                     || j        z  ||¦  «        }t          j                             || j        | j        ¬	¦  «        }t          j
        ||¦  «        }|                      ¦   «         || j        z  || j        fk    r5t          d
|| j        || j        f› d|                      ¦   «         › �¦  «        ‚|                     || j        || j        ¦  «        }|                     dd¦  «        }|                     |||¦  «        }|                      |¦  «        }||fS )z#Input shape: Batch x Time x ChannelNr@   r   rP   z$Attention weights should be of size z	, but is z!Attention mask should be of size r�   )Úprž   z `attn_output` should be of size )rT   r–   r_   r]  r—   r˜   rW  rX  r  r"   Úbmmr¢   ró   r   r    r5   r  rž   rZ   rZ  )r„   r?  rC  r‡   Úkwargsr[  Útgt_lenrç   Úis_cross_attentionÚquery_statesÚ
key_statesÚvalue_statesÚ
proj_shapeÚsrc_lenÚattn_weightsÚattn_weights_reshapedÚ
attn_probsÚattn_outputs                     r%   r‹   zGroupViTAttention.forwardb  sÎ  € ð #0×"4Ò"4Ñ"6Ô"6ÑˆˆW�iØ2¸$Ð>Ðð —{’{ =Ñ1Ô1°D´JÑ>ˆØð 	LØŸš T§[¢[Ð1FÑ%GÔ%GÈÈSÑQÔQˆJØŸ;š; t§{¢{Ð3HÑ'IÔ'IÈ2ÈsÑSÔSˆLˆLàŸš T§[¢[°Ñ%?Ô%?ÀÀSÑIÔIˆJØŸ;š; t§{¢{°=Ñ'AÔ'AÀ2ÀsÑKÔKˆLà˜DœNÑ*¨B°´Ð>ˆ
ØC�t—{’{ <°¸#Ñ>Ô>ÔCÀZÐPˆØ$�Z”_ jÐ1ˆ
Ø(�|Ô(¨*Ð5ˆà—/’/ !Ñ$Ô$ˆÝ”y ¨z×/CÒ/CÀAÀqÑ/IÔ/IÑJÔJˆà×ÒÑÔ 3¨¬Ñ#7¸À'Ð"JÒJÐJÝð*¸¸d¼nÑ8LÈgÐW^Ð7_ð *ð *Ø ×%Ò%Ñ'Ô'ð*ð *ñô ð ð
 Ð%Ø×"Ò"Ñ$Ô$¨¨a°¸'Ð(BÒBÐBÝ Øt¸¸aÀÈ'Ð8RÐtÐtÐ]k×]pÒ]pÑ]rÔ]rÐtÐtñô ð ð (×,Ò,¨S°$´.À'È7ÑSÔSÐVdÑdˆLØ'×,Ò,¨S°4´>Ñ-AÀ7ÈGÑTÔTˆLå”}×,Ò,¨\¸rÐ,ÑBÔBˆð !-× 1Ò 1°#°t´~ÀwÐPWÑ XÔ XÐØ,×1Ò1°#¸¼Ñ2FÈÐQXÑYÔYˆå”]×*Ò*¨<¸4¼<ÐRVÔR_Ð*Ñ`Ô`ˆ
å”i 
¨LÑ9Ô9ˆà×ÒÑÔ #¨¬Ñ"6¸ÀÄÐ!OÒOÐOÝð)°C¸¼ÈÐRVÔR_Ð3`ð )ð )Ø×$Ò$Ñ&Ô&ð)ð )ñô ð ð
 "×&Ò& s¨D¬N¸GÀTÄ]ÑSÔSˆØ!×+Ò+¨A¨qÑ1Ô1ˆØ!×)Ò)¨#¨w¸	ÑBÔBˆà—m’m KÑ0Ô0ˆàÐ1Ð1Ð1r'   ©NN)rŒ   r�   rŽ   rÜ   ry   r"   r÷   rW   r]  rÝ   rÙ   r‹   r�   r�   s   @r%   rz   rz   I  sí   ø€ € € € € ØGÐGðBð Bð Bð Bð Bð&e˜Uœ\ð e°Cð e¸cð eð eð eð eð /3Ø:>ð	D2ð D2à”|ðD2ð œ tÑ+ðD2ð  %Ô0°4Ñ7ð	D2ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ðD2ð D2ð D2ð D2ð D2ð D2ð D2ð D2r'   rz   c                   óf   ‡ — e Zd Zdefˆ fd„Zdej        dej        dee         dej	        fd„Z
ˆ xZS )r.  rs   c                 óD  •— t          ¦   «                              ¦   «          |j        | _        t	          |¦  «        | _        t          j        | j        |j        ¬¦  «        | _	        t          |¦  «        | _        t          j        | j        |j        ¬¦  «        | _        d S ru   )rx   ry   r}   rç   rz   Ú	self_attnr   r|   r~   Úlayer_norm1r€   r�   Úlayer_norm2rƒ   s     €r%   ry   zGroupViTEncoderLayer.__init__«  s   ø€ Ý‰Œ×ÒÑÔÐØÔ+ˆŒÝ*¨6Ñ2Ô2ˆŒÝœ<¨¬¸FÔ<QÐRÑRÔRˆÔÝ˜vÑ&Ô&ˆŒÝœ<¨¬¸FÔ<QÐRÑRÔRˆÔÐÐr'   r?  rC  ra  r   c                 óÆ   — |}|                       |¦  «        } | j        d||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N)r?  rC  r¬   )rq  rp  rr  r�   )r„   r?  rC  ra  Úresidualr  s         r%   r‹   zGroupViTEncoderLayer.forward³  s“   € ð !ˆà×(Ò(¨Ñ7Ô7ˆØ)˜4œ>ð 
Ø'Ø)ð
ð 
ð ð
ð 
Ñˆ�qð
 ! =Ñ0ˆà ˆØ×(Ò(¨Ñ7Ô7ˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆàÐr'   )rŒ   r�   rŽ   r   ry   r"   r÷   r   r   rÝ   r‹   r�   r�   s   @r%   r.  r.  ª  s“   ø€ € € € € ðSÐ3ð Sð Sð Sð Sð Sð Sðà”|ðð œðð Ð+Ô,ð	ð
 
Ô	ðð ð ð ð ð ð ð r'   r.  c                   ój   ‡ — e Zd ZU eed<   dZdZdZee	dœZ
 ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚGroupViTPreTrainedModelrs   Úgroupvit)ÚimageÚtextT)r?  r\   c                 ó˜  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          ¦  «        r™t          j        |j        j	        d|dz  ¬¦  «         t          j        |j
        j	        d|dz  ¬¦  «         t          j        |j        t          j        |j        j        d         ¦  «                             d¦  «        ¦  «         d	S t	          |t"          ¦  «        r»| j        j        }|j        dz  d|j        j        z  dz  z  |z  }|j        dz  |z  }t          j        |j        j	        |¬¦  «         t          j        |j        j	        |¬¦  «         t          j        |j        j	        |¬¦  «         t          j        |j        j	        |¬¦  «         d	S t	          |t0          ¦  «        rˆ| j        j        }|j        j        dz  d|j        j        z  dz  z  |z  }d|j        j        z  dz  |z  }t          j        |j        j	        |¬¦  «         t          j        |j        j	        |¬¦  «         d	S d	S )
zInitialize the weightsrD   g{®Gáz”?)ÚmeanÚstdr@   r  r”   rP   )r|  N)rx   Ú_init_weightsrs   Úinitializer_factorr±   r  ÚinitÚnormal_r  r"  r  Úcopy_r  r"   r#   rK   r  rz   rç   Únum_hidden_layersr–   r—   r˜   rZ  r€   r}   rN  rO  )r„   ÚmoduleÚfactorÚin_proj_stdÚout_proj_stdÚfc_stdr…   s         €r%   r}  z%GroupViTPreTrainedModel._init_weightsÖ  s%  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ø”Ô/ˆÝ�fÕ4Ñ5Ô5ð 	=ÝŒL˜Ô/Ô6¸SÀfÈtÁmÐTÑTÔTÐTÝŒL˜Ô2Ô9ÀÈÐRVÉÐWÑWÔWÐWÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÑhÔhÐhÐhÐhÝ˜Õ 1Ñ2Ô2ð 	=Ø”[Ô3ˆFØ!Ô+¨TÑ1°q¸6¼=Ô;ZÑ7ZÐ_cÑ6cÑdÐgmÑmˆKØ"Ô,¨dÑ2°fÑ<ˆLÝŒL˜œÔ-°;Ð?Ñ?Ô?Ð?ÝŒL˜œÔ-°;Ð?Ñ?Ô?Ð?ÝŒL˜œÔ-°;Ð?Ñ?Ô?Ð?ÝŒL˜œÔ/°\ÐBÑBÔBÐBÐBÐBÝ˜¥Ñ,Ô,ð 	=Ø”[Ô3ˆFØ!œ=Ô4°dÑ:ÀÀFÄMÔDcÑ@cÐhlÑ?lÑmÐpvÑvˆKØ˜&œ-Ô3Ñ3¸Ñ<¸vÑEˆFÝŒL˜œÔ*°Ð7Ñ7Ô7Ð7ÝŒL˜œÔ*°Ð<Ñ<Ô<Ð<Ð<Ð<ð	=ð 	=r'   )rŒ   r�   rŽ   r   rÞ   Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingr.  rz   Ú_can_record_outputsr"   rf   r}  r�   r�   s   @r%   rv  rv  Ë  s   ø€ € € € € € àÐÐÑØ"ÐØ(ÐØ&*Ð#à-Ø'ðð Ðð
 €U„]�_„_ð=ð =ð =ð =ñ „_ð=ð =ð =ð =ð =r'   rv  c                   ón   ‡ — e Zd Zdeddfˆ fd„Z	 	 	 d
dej        dedz  dedz  dedz  dee	z  f
d	„Z
ˆ xZS )ÚGroupViTVisionEncoderrs   r   Nc                 óî   •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          t          ‰j        ¦  «        ¦  «        D ¦   «         ¦  «        | _        d| _	        d S )Nc                 ó¦   •— g | ]M}t          ‰‰j        |         ‰j        |         ‰j        |         |d k    r‰j        |dz
           nd ¬¦  «        ‘ŒNS )r   r   )rs   r)  r¾   r¯   r*  )r(  ÚdepthsÚnum_group_tokensÚnum_output_groups)r­   Úirs   s     €r%   r®   z2GroupViTVisionEncoder.__init__.<locals>.<listcomp>õ  s{   ø€ ð 	ð 	ð 	ð õ Ø!Ø œ-¨Ô*Ø$*Ô$;¸AÔ$>Ø%+Ô%=¸aÔ%@ØLMÐPQÊEÈE¨Ô)AÀ!ÀaÁ%Ô)HÐ)HÐWXðñ ô ð	ð 	ð 	r'   F)
rx   ry   rs   r   r1  r2  r$   r�  ÚstagesÚgradient_checkpointingrƒ   s    `€r%   ry   zGroupViTVisionEncoder.__init__ñ  s~   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”mð	ð 	ð 	ð 	õ �s 6¤=Ñ1Ô1Ñ2Ô2ð	ñ 	ô 	ñ
ô 
ˆŒð ',ˆÔ#Ð#Ð#r'   r?  Úoutput_hidden_statesrA  Úreturn_dictc                 ó˜  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|rdnd }|rdnd }d }t	          | j        ¦  «        D ]@\  }}	|r||fz   } |	|||¦  «        }
|
d         }|
d         }|r|
d         �||
d         fz   }ŒA|r||fz   }|st          d„ |||fD ¦   «         ¦  «        S t          |||¬¦  «        S )Nr¬   r   r   rP   c              3   ó   K  — | ]}|®|V — Œ	d S r8  r¬   )r­   Úvs     r%   rØ   z0GroupViTVisionEncoder.forward.<locals>.<genexpr>$  s(   è è € ÐgÐg˜qÐYZÐYf˜ÐYfÐYfÐYfÐYfÐgÐgr'   )Úlast_hidden_stater?  r\   )rs   rA  r–  r—  Ú	enumerater”  rÙ   r   )r„   r?  r–  rA  r—  Úall_hidden_statesÚall_groupingsrÂ   r“  ÚstageÚlayer_outputss              r%   r‹   zGroupViTVisionEncoder.forward  sU  € ð 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆà"6Ð@˜B˜B¸DÐØ/Ð9˜˜°Tˆàˆå! $¤+Ñ.Ô.ð 
	Dð 
	D‰HˆAˆuØ#ð IØ$5¸Ð8HÑ$HÐ!à!˜E -°Ð?PÑQÔQˆMà)¨!Ô,ˆMØ(¨Ô+ˆLà ð D ]°1Ô%5Ð%AØ -°¸qÔ1AÐ0CÑ C�øàð 	EØ 1°]Ð4DÑ DÐàð 	hÝÐgÐg ]Ð4EÀ}Ð$UÐgÑgÔgÑgÔgÐgÝØ+Ð;LÐYfð
ñ 
ô 
ð 	
r'   r%  )rŒ   r�   rŽ   r   ry   r"   r÷   rø   rÙ   r   r‹   r�   r�   s   @r%   r�  r�  ð  s³   ø€ € € € € ð,Ð3ð ,¸ð ,ð ,ð ,ð ,ð ,ð ,ð( -1Ø)-Ø#'ð%
ð %
à”|ð%
ð # T™kð%
ð   $™;ð	%
ð
 ˜D‘[ð%
ð 
�Ñ	 ð%
ð %
ð %
ð %
ð %
ð %
ð %
ð %
r'   r�  c                   ób   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej        dz  dee	         de
ez  fd„Zˆ xZS )
ÚGroupViTTextEncoderz¹
    Transformer encoder consisting of `config.num_hidden_layers` self-attention layers. Each layer is a
    [`GroupViTEncoderLayer`].

    Args:
        config: GroupViTTextConfig
    rs   c                 óÔ   •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r¬   r-  r/  s     €r%   r®   z0GroupViTTextEncoder.__init__.<locals>.<listcomp>6  s"   ø€ Ð$kÐ$kÐ$kÀaÕ%9¸&Ñ%AÔ%AÐ$kÐ$kÐ$kr'   F)	rx   ry   rs   r   r1  r2  r‚  r3  r•  rƒ   s    `€r%   ry   zGroupViTTextEncoder.__init__3  sa   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”mÐ$kÐ$kÐ$kÐ$kÍ5ÐQWÔQiÑKjÔKjÐ$kÑ$kÔ$kÑlÔlˆŒØ&+ˆÔ#Ð#Ð#r'   NrC  ra  r   c                 óN   — |}| j         D ]} |||fi |¤Ž}Œt          |¬¦  «        S )a7  
        Args:
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
        )r›  )r3  r   )r„   r   rC  ra  r?  Úencoder_layers         r%   r‹   zGroupViTTextEncoder.forward9  s^   € ð( &ˆØ!œ[ð 	ð 	ˆMØ)˜MØØðð ð ðð ˆMˆMõ Ø+ð
ñ 
ô 
ð 	
r'   r8  )rŒ   r�   rŽ   rÜ   r   ry   r"   r÷   r   r   rÙ   r   r‹   r�   r�   s   @r%   r¢  r¢  *  s�   ø€ € € € € ðð ð,Ð1ð ,ð ,ð ,ð ,ð ,ð ,ð /3ð
ð 
ð œ tÑ+ð
ð Ð+Ô,ð	
ð
 
�Ñ	 ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r'   r¢  c                   óÈ   ‡ — e Zd Zdefˆ fd„Ze ed¬¦  «        e	 	 	 ddej	        dz  dej	        dz  dej	        dz  d	e
e         d
ef
d„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚGroupViTTextTransformerrs   c                 ó(  •— t          ¦   «                              |¦  «         |j        }t          |¦  «        | _        t          |¦  «        | _        t          j        ||j	        ¬¦  «        | _
        |j        | _        |                      ¦   «          d S ru   )rx   ry   r}   r  r  r¢  Úencoderr   r|   r~   Úfinal_layer_normÚeos_token_idÚ	post_initr  s      €r%   ry   z GroupViTTextTransformer.__init__[  s~   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØÔ&ˆ	Ý0°Ñ8Ô8ˆŒÝ*¨6Ñ2Ô2ˆŒÝ "¤¨Y¸FÔ<QÐ RÑ RÔ RˆÔð #Ô/ˆÔà�ŠÑÔÐÐÐr'   F)Útie_last_hidden_statesNr  rC  r  ra  r   c                 ór  — |€t          d¦  «        ‚|                     ¦   «         }|                     d|d         ¦  «        }|                      ||¬¦  «        }t	          | j        ||d ¬¦  «        }|                     dd ¦  «          | j        d||ddœ|¤Ž}|d         }|                      |¦  «        }| j	        d	k    rg|t          j        |j        d         |j        ¬
¦  «        |                     t          j        |j        ¬¦  «                             d¬¦  «        f         }	n�|t          j        |j        d         |j        ¬
¦  «        |                     t          j        |j        ¬¦  «        | j	        k                         ¦   «                              d¬¦  «        f         }	t#          ||	¬¦  «        S )NzYou have to specify input_idsr@   )r  r  )rs   r   rC  Úpast_key_valuesÚ	is_causalT)r   rC  r±  r   rP   r   )rE   r   r�   )r›  Úpooler_outputr¬   )ró   rT   r  r  r	   rs   Úpoprª  r«  r¬  r"   r#   rK   r   ÚtorW   Úargmaxr   )
r„   r  rC  r  ra  Úinput_shaper?  Úencoder_outputsr›  Úpooled_outputs
             r%   r‹   zGroupViTTextTransformer.forwardg  sÓ  € ð ÐÝÐ<Ñ=Ô=Ð=à—n’nÑ&Ô&ˆØ—N’N 2 {°2¤Ñ7Ô7ˆ	àŸš°)È,˜ÑWÔWˆå+Ø”;Ø'Ø)Ø ð	
ñ 
ô 
ˆð 	�
Š
�; Ñ%Ô%Ð%Ø+7¨4¬<ð ,
Ø'Ø)Øð,
ð ,
ð ð	,
ð ,
ˆð ,¨AÔ.ÐØ ×1Ò1Ð2CÑDÔDÐàÔ Ò!Ð!ð .Ý”Ð.Ô4°QÔ7Ð@QÔ@XÐYÑYÔYØ—’¥5¤9Ð5FÔ5M�ÑNÔN×UÒUÐZ\ÐUÑ]Ô]ð_ôˆMˆMð .Ý”Ð.Ô4°QÔ7Ð@QÔ@XÐYÑYÔYð —’¥E¤IÐ6GÔ6N�ÑOÔOÐSWÔSdÒdß’‘”ß’˜B�‘”ð!ôˆMõ *Ø/Ø'ð
ñ 
ô 
ð 	
r'   r%  )rŒ   r�   rŽ   r   ry   r   r   r   r"   r÷   r   r   r   r‹   r�   r�   s   @r%   r¨  r¨  Z  sã   ø€ € € € € ð
Ð1ð 
ð 
ð 
ð 
ð 
ð 
ð  Ø€_¨EÐ2Ñ2Ô2Øð *.Ø.2Ø,0ð	:
ð :
à”< $Ñ&ð:
ð œ tÑ+ð:
ð ”l TÑ)ð	:
ð
 Ð+Ô,ð:
ð 
$ð:
ð :
ð :
ñ „^ñ 3Ô2ñ  Ôð:
ð :
ð :
ð :
ð :
r'   r¨  c                   óÆ   ‡ — e Zd ZU eed<   dZdefˆ fd„Zdej        fd„Z	d„ Z
e	 	 	 ddej        dz  d	ej        dz  d
ej        dz  dee         deez  f
d„¦   «         Zˆ xZS )ÚGroupViTTextModelrs   )ry  c                 óš   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |                      ¦   «          d S r8  )rx   ry   r¨  Ú
text_modelr­  rƒ   s     €r%   ry   zGroupViTTextModel.__init__«  s@   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý1°&Ñ9Ô9ˆŒà�ŠÑÔÐÐÐr'   r   c                 ó$   — | j         j        j        S r8  ©r¼  r  r  rÛ   s    r%   Úget_input_embeddingsz&GroupViTTextModel.get_input_embeddings±  s   € ØŒÔ)Ô9Ð9r'   c                 ó(   — || j         j        _        d S r8  r¾  )r„   r¤   s     r%   Úset_input_embeddingsz&GroupViTTextModel.set_input_embeddings´  s   € Ø5:ˆŒÔ"Ô2Ð2Ð2r'   Nr  rC  r  ra  c                 ó$   —  | j         d|||dœ|¤ŽS )a9  
        Examples:

        ```python
        >>> from transformers import CLIPTokenizer, GroupViTTextModel

        >>> tokenizer = CLIPTokenizer.from_pretrained("nvidia/groupvit-gcc-yfcc")
        >>> model = GroupViTTextModel.from_pretrained("nvidia/groupvit-gcc-yfcc")

        >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> last_hidden_state = outputs.last_hidden_state
        >>> pooled_output = outputs.pooler_output  # pooled (EOS token) states
        ```©r  rC  r  r¬   )r¼  )r„   r  rC  r  ra  s        r%   r‹   zGroupViTTextModel.forward·  s7   € ð. ˆtŒð 
ØØ)Ø%ð
ð 
ð ð	
ð 
ð 	
r'   r%  )rŒ   r�   rŽ   r   rÞ   r‰  ry   r   ÚModuler¿  rÁ  r   r"   r÷   r   r   rÙ   r   r‹   r�   r�   s   @r%   rº  rº  §  s  ø€ € € € € € ØÐÐÑØ ÐðÐ1ð ð ð ð ð ð ð: b¤ið :ð :ð :ð :ð;ð ;ð ;ð ð *.Ø.2Ø,0ð	
ð 
à”< $Ñ&ð
ð œ tÑ+ð
ð ”l TÑ)ð	
ð
 Ð+Ô,ð
ð 
Ð+Ñ	+ð
ð 
ð 
ñ „^ð
ð 
ð 
ð 
ð 
r'   rº  c                   ó‚   ‡ — e Zd Zdefˆ fd„Ze	 	 	 	 d
dej        dz  dedz  dedz  dedz  de	e
z  f
d	„¦   «         Zˆ xZS )ÚGroupViTVisionTransformerrs   c                 óô   •— t          ¦   «                              ¦   «          || _        |j        }t	          |¦  «        | _        t          |¦  «        | _        t          j	        ||j
        ¬¦  «        | _        d S ru   )rx   ry   rs   r}   rú   r  r�  rª  r   r|   r~   r  r  s      €r%   ry   z"GroupViTVisionTransformer.__init__×  sc   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ&ˆ	å2°6Ñ:Ô:ˆŒÝ,¨VÑ4Ô4ˆŒÝœ i°VÔ5JÐKÑKÔKˆŒˆˆr'   Nrî   r–  rA  r—  r   c                 ó¢  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|€t	          d¦  «        ‚|                      |¦  «        }|                      ||||¬¦  «        }|d         }|                      |¦  «        }|                     d¬¦  «        }|s||f|dd …         z   S t          |||j
        |j        ¬¦  «        S )Nz You have to specify pixel_values)r?  r–  rA  r—  r   r   r�   )r›  r²  r?  r\   )rs   rA  r–  r—  ró   r  rª  r  r{  r   r?  r\   )	r„   rî   r–  rA  r—  r?  r·  r›  r¸  s	            r%   r‹   z!GroupViTVisionTransformer.forwardà  s  € ð 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàÐÝÐ?Ñ@Ô@Ð@àŸš¨Ñ5Ô5ˆàŸ,š,Ø'Ø!5Ø/Ø#ð	 'ñ 
ô 
ˆð ,¨AÔ.Ðð !ŸNšNÐ+<Ñ=Ô=ÐØ)×.Ò.°1Ð.Ñ5Ô5ˆàð 	LØ% }Ð5¸ÈÈÈÔ8KÑKÐKå)Ø/Ø'Ø)Ô7Ø&Ô1ð	
ñ 
ô 
ð 	
r'   ©NNNN)rŒ   r�   rŽ   r   ry   r   r"   rÝ   rø   rÙ   r   r‹   r�   r�   s   @r%   rÆ  rÆ  Ö  sÉ   ø€ € € € € ðLÐ3ð Lð Lð Lð Lð Lð Lð ð 26Ø,0Ø)-Ø#'ð'
ð '
àÔ'¨$Ñ.ð'
ð # T™kð'
ð   $™;ð	'
ð
 ˜D‘[ð'
ð 
Ð+Ñ	+ð'
ð '
ð '
ñ „^ð'
ð '
ð '
ð '
ð '
r'   rÆ  c                   ó¦   ‡ — e Zd ZU eed<   dZdZi Zdefˆ fd„Zde	fd„Z
e	 	 	 	 ddej        dz  dedz  d	edz  d
edz  deez  f
d„¦   «         Zˆ xZS )ÚGroupViTVisionModelrs   rî   )rx  c                 óš   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |                      ¦   «          d S r8  )rx   ry   rÆ  Úvision_modelr­  rƒ   s     €r%   ry   zGroupViTVisionModel.__init__  sA   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý5°fÑ=Ô=ˆÔà�ŠÑÔÐÐÐr'   r   c                 ó$   — | j         j        j        S r8  )rÍ  r  rü   rÛ   s    r%   r¿  z(GroupViTVisionModel.get_input_embeddings  s   € ØÔ Ô+Ô<Ð<r'   NrA  r–  r—  c                 ó4   — |                       ||||¬¦  «        S )a)  
        Examples:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, GroupViTVisionModel

        >>> processor = AutoProcessor.from_pretrained("nvidia/groupvit-gcc-yfcc")
        >>> model = GroupViTVisionModel.from_pretrained("nvidia/groupvit-gcc-yfcc")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=image, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> last_hidden_state = outputs.last_hidden_state
        >>> pooled_output = outputs.pooler_output  # pooled CLS states
        ```©rî   rA  r–  r—  )rÍ  )r„   rî   rA  r–  r—  ra  s         r%   r‹   zGroupViTVisionModel.forward  s-   € ð> × Ò Ø%Ø/Ø!5Ø#ð	 !ñ 
ô 
ð 	
r'   rÉ  )rŒ   r�   rŽ   r   rÞ   Úmain_input_namer‰  r‹  ry   rà   r¿  r   r"   rÝ   rø   rÙ   r   r‹   r�   r�   s   @r%   rË  rË    sý   ø€ € € € € € Ø Ð Ð Ñ Ø$€OØ!ÐØÐðÐ3ð ð ð ð ð ð ð=Ð&=ð =ð =ð =ð =ð ð 26Ø)-Ø,0Ø#'ð#
ð #
àÔ'¨$Ñ.ð#
ð   $™;ð#
ð # T™kð	#
ð
 ˜D‘[ð#
ð 
Ð+Ñ	+ð#
ð #
ð #
ñ „^ð#
ð #
ð #
ð #
ð #
r'   rË  c                   óÊ  ‡ — e Zd ZU eed<   defˆ fd„Zee	 	 ddej	        dej	        dz  dej	        dz  de
e         deez  f
d	„¦   «         ¦   «         Zeed
ej	        de
e         deez  fd„¦   «         ¦   «         Zee	 	 	 	 	 	 	 	 ddej        dz  d
ej        dz  dej	        dz  dej        dz  dedz  dedz  dedz  dedz  de
e         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚGroupViTModelrs   c           
      ó´  •— t          ¦   «                              |¦  «         t          |j        t          ¦  «        s%t          dt          |j        ¦  «        › d�¦  «        ‚t          |j        t          ¦  «        s%t          dt          |j        ¦  «        › d�¦  «        ‚|j        }|j        }|j	        | _	        |j
        | _
        |j        | _        |j        | _        t          |¦  «        | _        t!          |¦  «        | _        t%          j        t%          j        | j        | j
        d¬¦  «        t%          j        | j
        ¦  «        t%          j        d¬¦  «        t%          j        | j
        | j	        d¬¦  «        ¦  «        | _        t%          j        t%          j        | j        | j
        d¬¦  «        t%          j        | j
        ¦  «        t%          j        d¬¦  «        t%          j        | j
        | j	        d¬¦  «        ¦  «        | _        t%          j        t5          j        | j        j        ¦  «        ¦  «        | _        |                      ¦   «          d S )NzOconfig.text_config is expected to be of type GroupViTTextConfig but is of type ú.zSconfig.vision_config is expected to be of type GroupViTVisionConfig but is of type T)Úbias)Úinplace) rx   ry   r±   Útext_configr   Ú	TypeErrorÚtypeÚvision_configr   Úprojection_dimÚprojection_intermediate_dimr}   Útext_embed_dimÚvision_embed_dimr¨  r¼  rÆ  rÍ  r   r5  r•   ÚBatchNorm1dÚReLUÚvisual_projectionÚtext_projectionrý   r"   rI   rs   Úlogit_scale_init_valueÚlogit_scaler­  )r„   rs   rØ  rÛ  r…   s       €r%   ry   zGroupViTModel.__init__E  s  ø€ Ý‰Œ×Ò˜Ñ Ô Ð å˜&Ô,Õ.@ÑAÔAð 	Ýð0Ý˜Ô+Ñ,Ô,ð0ð 0ð 0ñô ð õ
 ˜&Ô.Õ0DÑEÔEð 	Ýð2Ý˜Ô-Ñ.Ô.ð2ð 2ð 2ñô ð ð
 Ô(ˆØÔ,ˆà$Ô3ˆÔØ+1Ô+MˆÔ(Ø)Ô5ˆÔØ -Ô 9ˆÔå1°+Ñ>Ô>ˆŒÝ5°mÑDÔDˆÔå!#¤ÝŒI�dÔ+¨TÔ-MÐTXÐYÑYÔYÝŒN˜4Ô;Ñ<Ô<ÝŒG˜DÐ!Ñ!Ô!ÝŒI�dÔ6¸Ô8KÐRVÐWÑWÔWñ	"
ô "
ˆÔõ  "œ}ÝŒI�dÔ)¨4Ô+KÐRVÐWÑWÔWÝŒN˜4Ô;Ñ<Ô<ÝŒG˜DÐ!Ñ!Ô!ÝŒI�dÔ6¸Ô8KÐRVÐWÑWÔWñ	 
ô  
ˆÔõ œ<­¬°T´[Ô5WÑ(XÔ(XÑYÔYˆÔð 	�ŠÑÔÐÐÐr'   Nr  rC  r  ra  r   c                 ól   —  | j         d|||ddœ|¤Ž}|j        }|                      |¦  «        |_        |S )a  
        Examples:

        ```python
        >>> import torch
        >>> from transformers import CLIPTokenizer, GroupViTModel

        >>> model = GroupViTModel.from_pretrained("nvidia/groupvit-gcc-yfcc")
        >>> tokenizer = CLIPTokenizer.from_pretrained("nvidia/groupvit-gcc-yfcc")

        >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
        >>> with torch.inference_mode():
        ...     text_features = model.get_text_features(**inputs)
        ```T)r  rC  r  r—  r¬   )r¼  r²  rã  )r„   r  rC  r  ra  Útext_outputsr¸  s          r%   Úget_text_featureszGroupViTModel.get_text_featuresp  s^   € ð. 4C°4´?ð 4
ØØ)Ø%Øð	4
ð 4
ð
 ð4
ð 4
ˆð %Ô2ˆØ%)×%9Ò%9¸-Ñ%HÔ%HˆÔ"àÐr'   rî   c                 ód   —  | j         |fddi|¤Ž}|                      |j        ¦  «        |_        |S )aŠ  
        Examples:

        ```python
        >>> import torch
        >>> from transformers import AutoProcessor, GroupViTModel
        >>> from transformers.image_utils import load_image

        >>> model = GroupViTModel.from_pretrained("nvidia/groupvit-gcc-yfcc")
        >>> processor = AutoProcessor.from_pretrained("nvidia/groupvit-gcc-yfcc")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = load_image(url)

        >>> inputs = processor(images=image, return_tensors="pt")

        >>> with torch.inference_mode():
        ...     image_features = model.get_image_features(**inputs)
        ```r—  T)rÍ  râ  r²  )r„   rî   ra  Úvision_outputss       r%   Úget_image_featuresz GroupViTModel.get_image_features“  sE   € ð4 6G°TÔ5FÀ|Ð5pÐ5pÐaeÐ5pÐioÐ5pÐ5pˆØ'+×'=Ò'=¸nÔ>ZÑ'[Ô'[ˆÔ$àÐr'   Úreturn_lossrA  r–  Úoutput_segmentationc	           
      ó~  — |�|n| j         j        }|�|n| j         j        }|rd}|�|n| j         j        }|                      |||d¬¦  «        }
 | j        d|||dœ|	¤Ž}|
j        }|                      |¦  «        }|j        }|                      |¦  «        }|| 	                    dd¬¦  «        z  }|| 	                    dd¬¦  «        z  }| j
                             ¦   «         }t          j        ||                     ¦   «         ¦  «        |z  }|                     ¦   «         }d}|�rn|
j        }|                      |                     d|j        d         ¦  «        ¦  «        }|
j        }t'          ||j        dd…         ¦  «        }|| 	                    dd¬¦  «        z  }t          j        ||                     ¦   «         ¦  «        |z  }|                     |j        d         d|j        d         ¦  «                             ddd	¦  «        }|                     |j        d         |j        d	         d¦  «        }t          j        ||¦  «        |z  }|                     |j        d         |j        d	         |j        d         |j        d
         ¦  «        }d}|rt+          |¦  «        }t-          ||||||||
¬¦  «        S )až  
        return_loss (`bool`, *optional*):
            Whether or not to return the contrastive loss.
        output_segmentation (`bool`, *optional*):
            Whether or not to return the segmentation logits.

        Examples:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, GroupViTModel

        >>> model = GroupViTModel.from_pretrained("nvidia/groupvit-gcc-yfcc")
        >>> processor = AutoProcessor.from_pretrained("nvidia/groupvit-gcc-yfcc")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(
        ...     text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True
        ... )

        >>> outputs = model(**inputs)
        >>> logits_per_image = outputs.logits_per_image  # this is the image-text similarity score
        >>> probs = logits_per_image.softmax(dim=1)  # we can take the softmax to get the label probabilities
        ```NTrÐ  rÃ  r@   r¡   rP   r   r   r   )rË   rÌ   rÍ   rÎ   rÏ   rÐ   rÑ   rÒ   r¬   )rs   rA  rí  r–  rÍ  r¼  r²  râ  rã  Únormrå  Úexpr"   ÚmatmulÚtr›  rZ   rK   r\   rp   rg   r-   rÊ   )r„   r  rî   rC  r  rì  rA  r–  rí  ra  rê  rç  rÐ   rÏ   rå  rÍ   rÌ   Ú
seg_logitsÚimage_group_embedsr\   ÚgroupingÚlogits_per_image_groupÚflatten_groupingrË   s                           r%   r‹   zGroupViTModel.forward²  s5  € ðX 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà#6Ð#BÐÐÈÌÔHgð 	ð ð 	%Ø $Ðà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð
 ×*Ò*Ø%Ø/Ø!5Øð	 +ñ 
ô 
ˆð 4C°4´?ð 4
ØØ)Ø%ð4
ð 4
ð ð	4
ð 4
ˆð &Ô3ˆØ×-Ò-¨lÑ;Ô;ˆà"Ô0ˆØ×*Ò*¨;Ñ7Ô7ˆð $ l×&7Ò&7¸BÈÐ&7Ñ&MÔ&MÑMˆØ! K×$4Ò$4¸ÀTÐ$4Ñ$JÔ$JÑJˆð Ô&×*Ò*Ñ,Ô,ˆÝœ, {°L·N²NÑ4DÔ4DÑEÔEÈÑSˆØ*×,Ò,Ñ.Ô.Ðàˆ
Øñ 	ð "0Ô!AÐà!%×!7Ò!7Ð8J×8RÒ8RÐSUÐWiÔWoÐprÔWsÑ8tÔ8tÑ!uÔ!uÐØ'Ô2ˆJå3°JÀÔ@RÐSTÐSUÐSUÔ@VÑWÔWˆHð "4Ð6H×6MÒ6MÐRTÐ^bÐ6MÑ6cÔ6cÑ!cÐå%*¤\Ð2DÀkÇmÂmÁoÄoÑ%VÔ%VÐYdÑ%dÐ"à%;×%CÒ%CØÔ" 1Ô% r¨;Ô+<¸QÔ+?ñ&ô &çŠg�a˜˜AÑÔð #ð
  (×/Ò/°´¸qÔ0AÀ8Ä>ÐRSÔCTÐVXÑYÔYÐõ œÐ&<Ð>NÑOÔOÐR]Ñ]ˆJØ#×+Ò+ØÔ  Ô# ZÔ%5°aÔ%8¸(¼.ÈÔ:KÈXÌ^Ð\]ÔM^ñô ˆJð ˆØð 	@Ý.¨Ñ?Ô?ˆDå"ØØ-Ø+Ø *Ø#Ø%Ø*Ø .ð	
ñ 	
ô 	
ð 		
r'   rm  )NNNNNNNN)rŒ   r�   rŽ   r   rÞ   ry   r   r   r"   r÷   r   r   rÙ   r   rè  rë  r&  rÝ   rø   rÊ   r‹   r�   r�   s   @r%   rÓ  rÓ  A  s'  ø€ € € € € € àÐÐÑð)˜~ð )ð )ð )ð )ð )ð )ðV Øð /3Ø,0ð	ð à”<ðð œ tÑ+ðð ”l TÑ)ð	ð
 Ð+Ô,ðð 
Ð+Ñ	+ðð ð ñ „^ñ ÔððB Øðà”lðð Ð+Ô,ðð 
Ð+Ñ	+ð	ð ð ñ „^ñ Ôðð: Øð .2Ø15Ø.2Ø04Ø#'Ø)-Ø,0Ø+/ð|
ð |
àÔ# dÑ*ð|
ð Ô'¨$Ñ.ð|
ð œ tÑ+ð	|
ð
 Ô&¨Ñ-ð|
ð ˜D‘[ð|
ð   $™;ð|
ð # T™kð|
ð " D™[ð|
ð Ð+Ô,ð|
ð 
Ð$Ñ	$ð|
ð |
ð |
ñ „^ñ Ôð|
ð |
ð |
ð |
ð |
r'   rÓ  )rÓ  rv  rº  rË  )r   Fr@   rõ   )MrÜ   Úcollections.abcr³   Údataclassesr   Útypingr   ÚnumpyrX   r"   r   Ú r   r  Úactivationsr   Úmasking_utilsr	   Úmodeling_layersr
   Úmodeling_outputsr   r   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úconfiguration_groupvitr   r   r   Ú
get_loggerrŒ   Úloggerr÷   r&   r-   rW   r?   Úfloatrø   rN   rd   rp   rÄ  rr   r’   r©   rÊ   rà   rú   r  r(  r€   r¶   rz   r.  rv  r�  r¢  r¨  rº  rÆ  rË  rÓ  Ú__all__r¬   r'   r%   ú<module>r     s(  ðð Ð à Ð Ð Ð Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð à Ð Ð Ð Ø €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø /Ð /Ð /Ð /Ð /Ð /Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ -Ð -Ð -Ð -Ð -Ð -Ø &Ð &Ð &Ð &Ð &Ð &Ø jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jØ 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \ð 
ˆÔ	˜HÑ	%Ô	%€ð
`˜Uœ\ð `¨e¬lð `ð `ð `ð `ð
-¨E¬Lð -¸U¼\ð -ð -ð -ð -ð˜œð ¨Cð ð ð ð ðð ˜5œ<ð ¨eð ¸tð ÐRUð Ð_dÔ_kð ð ð ð ð,ð ð ð ð<ð ð ð:ð ð ð ð  "¤)ñ ô ð ð -ð -ð -ð -ð -˜bœiñ -ô -ð -ð`4+ð 4+ð 4+ð 4+ð 4+˜"œ)ñ 4+ô 4+ð 4+ðn Ø
ð-
ð -
ð -
ð -
ð -
˜+ñ -
ô -
ñ „ñ „ð-
ð`ð ð ð ð ˜bœiñ ô ð ðDGð Gð Gð Gð G˜rœyñ Gô Gð GðV%ð %ð %ð %ð %˜RœYñ %ô %ð %ðPZð Zð Zð Zð Z�B”Iñ Zô Zð Zðzð ð ð ð �"”)ñ ô ð ð0!ð !ð !ð !ð !�{ñ !ô !ð !ð]2ð ]2ð ]2ð ]2ð ]2˜œ	ñ ]2ô ]2ð ]2ðBð ð ð ð Ð5ñ ô ð ðB ð!=ð !=ð !=ð !=ð !=˜oñ !=ô !=ñ „ð!=ðH7
ð 7
ð 7
ð 7
ð 7
˜BœIñ 7
ô 7
ð 7
ðt-
ð -
ð -
ð -
ð -
˜"œ)ñ -
ô -
ð -
ð`J
ð J
ð J
ð J
ð J
Ð5ñ J
ô J
ð J
ðZ,
ð ,
ð ,
ð ,
ð ,
Ð/ñ ,
ô ,
ð ,
ð^2
ð 2
ð 2
ð 2
ð 2
 ¤	ñ 2
ô 2
ð 2
ðj3
ð 3
ð 3
ð 3
ð 3
Ð1ñ 3
ô 3
ð 3
ðl ðn
ð n
ð n
ð n
ð n
Ð+ñ n
ô n
ñ „ðn
ðb cÐ
bÐ
b€€€r'   