§
    ‚Štj÷ ã                   ó¼  — d Z ddlZddlZddlmZ ddlmZ ddlmZ ddl	Z	ddl	m
Z
 ddlmZ dd	lmZ dd
lmZmZmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z* ddl+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1m2Z2m3Z3  e)j4        e5¦  «        Z6e' G d„ de!¦  «        ¦   «         Z7dXde	j8        de	j9        de:dz  fd„Z;	 dYde	j<        de	j9        de	j=        de:fd„Z>e'e G d „ d!e¦  «        ¦   «         ¦   «         Z? e'd"¬#¦  «        e G d$„ d%e%¦  «        ¦   «         ¦   «         Z@ e'd&¬#¦  «        e G d'„ d(e%¦  «        ¦   «         ¦   «         ZA G d)„ d*e
jB        ¦  «        ZC	 dZd,e
jB        d-e	j8        d.e	j8        d/e	j8        d0e	j8        dz  d1eDd2eDfd3„ZE G d4„ d5e
jB        ¦  «        ZF G d6„ d7e
jB        ¦  «        ZG G d8„ d9e¦  «        ZH G d:„ d;e
jB        ¦  «        ZI G d<„ d=e7¦  «        ZJ G d>„ d?e
jB        ¦  «        ZK G d@„ dAe
jB        ¦  «        ZL G dB„ dCe
jB        ¦  «        ZM G dD„ dEe¦  «        ZN G dF„ dGe7¦  «        ZO G dH„ dIe7¦  «        ZP G dJ„ dKe7¦  «        ZQ e'dL¬#¦  «         G dM„ dNe7e¦  «        ¦   «         ZR G dO„ dPe
jB        ¦  «        ZS e'dQ¬#¦  «         G dR„ dSe7¦  «        ¦   «         ZT e'dT¬#¦  «         G dU„ dVe7e¦  «        ¦   «         ZUg dW¢ZVdS )[zPyTorch KOSMOS-2 model.é    N)ÚCallable)Ú	dataclass)ÚAny)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚBaseModelOutputWithPoolingÚ!CausalLMOutputWithCrossAttentions)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚ	torch_int)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚKosmos2ConfigÚKosmos2TextConfigÚKosmos2VisionConfigc                   ó€   ‡ — e Zd ZU eed<   dZdZddgZdZdZ	dZ
 ej        ¦   «         dej        fˆ fd„¦   «         Zˆ xZS )	ÚKosmos2PreTrainedModelÚconfig)ÚimageÚtextTÚKosmos2VisionEncoderLayerÚKosmos2TextBlockFÚmodulec                 ór
  •— t          ¦   «                              |¦  «         t          | j        d¦  «        r| j        j        }n&t          | j        d¦  «        r| j        j        j        }t          | j        d¦  «        r| j        j        }n&t          | j        d¦  «        r| j        j        j        }t          |t          ¦  «        rÒt          j        |j        d|j        dz  |z  ¬¦  «         t          j        |j        j        |j        j        |z  ¬¦  «         t          j        |j        j        |j        j        |z  ¬¦  «         t          j        |j        t)          j        |j        j        d	         ¦  «                             d
¦  «        ¦  «         dS t          |t0          ¦  «        r¯|j        dz  d|j        j        z  dz  z  |z  }|j        dz  |z  }t          j        |j        j        |¬¦  «         t          j        |j        j        |¬¦  «         t          j        |j        j        |¬¦  «         t          j        |j        j        |¬¦  «         dS t          |t<          ¦  «        r||j        j        dz  d|j        j        z  dz  z  |z  }d|j        j        z  dz  |z  }t          j        |j         j        |¬¦  «         t          j        |j!        j        |¬¦  «         dS t          |tD          ¦  «        r‚t          j        |j        j        |¬¦  «         t          j        |j        j        |¬¦  «         t          j        |j        j        |¬¦  «         t          j        |j        j        |¬¦  «         dS t          |tF          ¦  «        rBt          j        |j         j        |¬¦  «         t          j        |j!        j        |¬¦  «         dS t          |tH          ¦  «        r"t          j        |j%        j        |¬¦  «         dS t          |tL          ¦  «        r;t          j        |j'        j        |¬¦  «         t          j        |j(        ¦  «         dS t          |tR          ¦  «        r_t          j        |j*        j        d|¬¦  «         |j*        j+        �0t          j,        |j*        j        |j*        j+                 ¦  «         dS dS t          |tZ          ¦  «        rJ| .                    |j/        |j0        z   |j1        |j+        ¦  «        }t          j        |j2        |¦  «         dS dS )zInitialize the weightsÚinitializer_factorÚvision_configÚinit_stdÚtext_configç        ç      à¿)ÚmeanÚstd)r4   éÿÿÿÿ©r    r5   é   N)3ÚsuperÚ_init_weightsÚhasattrr&   r-   r.   r/   r0   Ú
isinstanceÚKosmos2VisionEmbeddingsÚinitÚnormal_Úclass_embeddingÚ	embed_dimÚpatch_embeddingÚweightÚinitializer_rangeÚposition_embeddingÚcopy_Úposition_idsÚtorchÚarangeÚshapeÚexpandÚKosmos2VisionAttentionÚnum_hidden_layersÚq_projÚk_projÚv_projÚout_projÚKosmos2VisionMLPÚhidden_sizeÚfc1Úfc2ÚKosmosTextAttentionÚKosmos2TextFFNÚKosmos2TextForCausalLMÚlm_headÚKosmos2ImageToTextProjectionÚdenseÚlatent_queryÚKosmos2TextTransformerÚembed_tokensÚpadding_idxÚzeros_Ú(Kosmos2TextSinusoidalPositionalEmbeddingÚget_embeddingÚnum_positionsÚoffsetÚembedding_dimÚweights)	Úselfr+   Úfactorr4   Úin_proj_stdÚout_proj_stdÚfc_stdÚemb_weightsÚ	__class__s	           €új/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/kosmos2/modeling_kosmos2.pyr9   z$Kosmos2PreTrainedModel._init_weights:   sË  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�4”;Ð 4Ñ5Ô5ð 	BØ”[Ô3ˆFˆFÝ�T”[ /Ñ2Ô2ð 	BØ”[Ô.ÔAˆFå�4”; 
Ñ+Ô+ð 	3Ø”+Ô&ˆCˆCÝ�T”[ -Ñ0Ô0ð 	3Ø”+Ô)Ô2ˆCå�fÕ5Ñ6Ô6ð &	4ÝŒL˜Ô/°c¸vÔ?OÐQUÑ?UÐX^Ñ?^Ð_Ñ_Ô_Ð_ÝŒL˜Ô/Ô6¸F¼MÔ<[Ð^dÑ<dÐeÑeÔeÐeÝŒL˜Ô2Ô9¸v¼}Ô?^ÐagÑ?gÐhÑhÔhÐhÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÑhÔhÐhÐhÐhÝ˜Õ 6Ñ7Ô7ð !	4Ø!Ô+¨TÑ1°q¸6¼=Ô;ZÑ7ZÐ_cÑ6cÑdÐgmÑmˆKØ"Ô,¨dÑ2°fÑ<ˆLÝŒL˜œÔ-°;Ð?Ñ?Ô?Ð?ÝŒL˜œÔ-°;Ð?Ñ?Ô?Ð?ÝŒL˜œÔ-°;Ð?Ñ?Ô?Ð?ÝŒL˜œÔ/°\ÐBÑBÔBÐBÐBÐBÝ˜Õ 0Ñ1Ô1ð 	4Ø!œ=Ô4°dÑ:ÀÀFÄMÔDcÑ@cÐhlÑ?lÑmÐpvÑvˆKØ˜&œ-Ô3Ñ3¸Ñ<¸vÑEˆFÝŒL˜œÔ*°Ð7Ñ7Ô7Ð7ÝŒL˜œÔ*°Ð<Ñ<Ô<Ð<Ð<Ð<Ý˜Õ 3Ñ4Ô4ð 	4ÝŒL˜œÔ-°3Ð7Ñ7Ô7Ð7ÝŒL˜œÔ-°3Ð7Ñ7Ô7Ð7ÝŒL˜œÔ-°3Ð7Ñ7Ô7Ð7ÝŒL˜œÔ/°SÐ9Ñ9Ô9Ð9Ð9Ð9Ý˜¥Ñ/Ô/ð 	4ÝŒL˜œÔ*°Ð4Ñ4Ô4Ð4ÝŒL˜œÔ*°Ð4Ñ4Ô4Ð4Ð4Ð4Ý˜Õ 6Ñ7Ô7ð 	4ÝŒL˜œÔ.°CÐ8Ñ8Ô8Ð8Ð8Ð8Ý˜Õ <Ñ=Ô=ð 	4ÝŒL˜œÔ,°#Ð6Ñ6Ô6Ð6ÝŒL˜Ô,Ñ-Ô-Ð-Ð-Ð-Ý˜Õ 6Ñ7Ô7ð 	4ÝŒL˜Ô,Ô3¸#À3ÐGÑGÔGÐGØÔ"Ô.Ð:Ý”˜FÔ/Ô6°vÔ7JÔ7VÔWÑXÔXÐXÐXÐXð ;Ð:å˜Õ HÑIÔIð 	4Ø ×.Ò.ØÔ$ v¤}Ñ4°fÔ6JÈFÔL^ñô ˆKõ ŒJ�v”~ {Ñ3Ô3Ð3Ð3Ð3ð		4ð 	4ó    )Ú__name__Ú
__module__Ú__qualname__r!   Ú__annotations__Úinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_attention_backendÚ_supports_flash_attnÚ_supports_sdparG   Úno_gradr   ÚModuler9   Ú__classcell__©rl   s   @rm   r%   r%   0   s�   ø€ € € € € € àÐÐÑØ(ÐØ&*Ð#Ø4Ð6HÐIÐØ"&ÐØ ÐØ€Nà€U„]�_„_ð34 B¤Ið 34ð 34ð 34ð 34ð 34ñ „_ð34ð 34ð 34ð 34ð 34rn   r%   ÚmaskÚdtypeÚtgt_lenc                 óL  — |                       ¦   «         \  }}|�|n|}| dd…dddd…f                              |d||¦  «                             |¦  «        }d|z
  }|                     |                     t          j        ¦  «        t	          j        |¦  «        j        ¦  «        S )z_
    Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
    Nr    ç      ð?)ÚsizerJ   ÚtoÚmasked_fillrG   ÚboolÚfinfoÚmin)r}   r~   r   ÚbszÚsrc_lenÚexpanded_maskÚinverted_masks          rm   Ú_expand_maskrŒ   q   s�   € ð —9’9‘;”;�L€CˆØ Ð,ˆgˆg°'€Gà˜˜˜˜D $¨¨¨Ð)Ô*×1Ò1°#°q¸'À7ÑKÔK×NÒNÈuÑUÔU€Mà˜-Ñ'€Mà×$Ò$ ]×%5Ò%5µe´jÑ%AÔ%AÅ5Ä;ÈuÑCUÔCUÔCYÑZÔZÐZrn   Úinput_ids_shapeÚdeviceÚpast_key_values_lengthc                 ó*  — | \  }}t          j        ||ft          j        |¦  «        j        |¬¦  «        }t          j        |                     d¦  «        |¬¦  «        }|                     ||dz                        |                     d¦  «        d¦  «        k     d¦  «         |                     |¦  «        }|dk    r.t          j	        t          j
        ||||¬¦  «        |gd¬¦  «        }|dddd…dd…f                              |d|||z   ¦  «        S )zB
    Make causal mask used for bi-directional self-attention.
    )rŽ   r5   r    r   ©r~   rŽ   ©ÚdimN)rG   Úfullr†   r‡   rH   r‚   Úmasked_fill_Úviewrƒ   ÚcatÚzerosrJ   )r�   r~   rŽ   r�   rˆ   r   r}   Ú	mask_conds           rm   Ú_make_causal_maskrš      s  € ð #�L€CˆÝŒ:�w Ð(­%¬+°eÑ*<Ô*<Ô*@ÈÐPÑPÔP€DÝ”˜TŸYšY r™]œ]°6Ð:Ñ:Ô:€IØ×Ò�i 9¨q¡=×"6Ò"6°t·y²yÀ±}´}ÀaÑ"HÔ"HÒHÈ!ÑLÔLÐLØ�7Š7�5‰>Œ>€Dà Ò!Ð!ÝŒy�%œ+ gÐ/EÈUÐ[aÐbÑbÔbÐdhÐiÐoqÐrÑrÔrˆØ��d˜A˜A˜A˜q˜q˜qÐ Ô!×(Ò(¨¨a°¸'ÐDZÑ:ZÑ[Ô[Ð[rn   c                   ó>   — e Zd ZU dZdZeej                 dz  ed<   dS )Ú'BaseModelOutputWithProjectionAttentionsaq  
    projection_attentions (`tuple(torch.FloatTensor)`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.

        Attentions weights given by `Kosmos2ImageToTextProjection`, after the attention softmax, used to compute
        the weighted average in the self-attention heads.
    NÚprojection_attentions)	ro   rp   rq   Ú__doc__r�   ÚtuplerG   ÚFloatTensorrr   © rn   rm   rœ   rœ   �   s=   € € € € € € ðð ð >BÐ˜5 Ô!2Ô3°dÑ:ÐAÐAÑAÐAÐArn   rœ   ze
    Base class for text model's outputs that also contains a pooling of the last hidden states.
    )Úcustom_introc                   ó  — e Zd ZU dZdZej        dz  ed<   dZe	dz  ed<   dZ
eej                 dz  ed<   dZeej                 dz  ed<   dZej        dz  ed<   dZeej                 dz  ed<   dZeed	<   d
ee         fd„ZdS )ÚKosmos2ModelOutputa±  
    image_embeds (`torch.FloatTensor` of shape `(batch_size, latent_query_num, hidden_size)`, *optional*):
        Sequence of hidden-states at the output of `Kosmos2ImageToTextProjection`.
    projection_attentions (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.

        Attentions weights given by `Kosmos2ImageToTextProjection`, after the attention softmax, used to compute
        the weighted average in the self-attention heads.
    vision_model_output (`BaseModelOutputWithPooling`, *optional*):
        The output of the [`Kosmos2VisionModel`].
    NÚlast_hidden_stateÚpast_key_valuesÚhidden_statesÚ
attentionsÚimage_embedsr�   Úvision_model_outputÚreturnc                 ó^   ‡ — t          ˆ fd„‰                      ¦   «         D ¦   «         ¦  «        S )Nc              3   ót   •K  — | ]2}|d vr‰|         n!t          ‰|¦  «                             ¦   «         V — Œ3dS ©)Útext_model_outputrª   N©ÚgetattrÚto_tuple©Ú.0Úkrf   s     €rm   ú	<genexpr>z.Kosmos2ModelOutput.to_tuple.<locals>.<genexpr>¼   óc   øè è € ð 
ð 
àð Ð LÐLÐLˆD�ŒGˆGÕRYÐZ^Ð`aÑRbÔRb×RkÒRkÑRmÔRmð
ð 
ð 
ð 
ð 
ð 
rn   ©rŸ   Úkeys©rf   s   `rm   r²   zKosmos2ModelOutput.to_tuple»   óC   ø€ Ýð 
ð 
ð 
ð 
à—Y’Y‘[”[ð
ñ 
ô 
ñ 
ô 
ð 	
rn   )ro   rp   rq   rž   r¥   rG   r    rr   r¦   r
   r§   rŸ   r¨   r©   r�   rª   r   r   r²   r¡   rn   rm   r¤   r¤   Ÿ   sð   € € € € € € ðð ð 37Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø-1€L�%Ô# dÑ*Ð1Ð1Ñ1Ø=AÐ˜5 Ô!2Ô3°dÑ:ÐAÐAÑAØ6:ÐÐ3Ð:Ð:Ñ:ð
˜% œ*ð 
ð 
ð 
ð 
ð 
ð 
rn   r¤   zC
    Model output class for `Kosmos2ForConditionalGeneration`.
    c                   ó&  — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZej        dz  ed<   dZeej                 dz  ed	<   dZeed
<   dee         fd„ZdS )Ú*Kosmos2ForConditionalGenerationModelOutputa  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    image_embeds (`torch.FloatTensor` of shape `(batch_size, latent_query_num, hidden_size)`, *optional*):
        Sequence of hidden-states at the output of `Kosmos2ImageToTextProjection`.
    projection_attentions (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.

        Attentions weights given by `Kosmos2ImageToTextProjection`, after the attention softmax, used to compute
        the weighted average in the self-attention heads.
    vision_model_output (`BaseModelOutputWithPooling`, *optional*):
        The output of the [`Kosmos2VisionModel`].
    NÚlossÚlogitsr¦   r§   r¨   r©   r�   rª   r«   c                 ó^   ‡ — t          ˆ fd„‰                      ¦   «         D ¦   «         ¦  «        S )Nc              3   ót   •K  — | ]2}|d vr‰|         n!t          ‰|¦  «                             ¦   «         V — Œ3dS r®   r°   r³   s     €rm   r¶   zFKosmos2ForConditionalGenerationModelOutput.to_tuple.<locals>.<genexpr>ä   r·   rn   r¸   rº   s   `rm   r²   z3Kosmos2ForConditionalGenerationModelOutput.to_tupleã   r»   rn   )ro   rp   rq   rž   r¾   rG   r    rr   r¿   r¦   r
   r§   rŸ   r¨   r©   r�   rª   r   r   r²   r¡   rn   rm   r½   r½   Â   s  € € € € € € ðð ð" &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø-1€L�%Ô# dÑ*Ð1Ð1Ñ1Ø=AÐ˜5 Ô!2Ô3°dÑ:ÐAÐAÑAØ6:ÐÐ3Ð:Ð:Ñ:ð
˜% œ*ð 
ð 
ð 
ð 
ð 
ð 
rn   r½   c                   óv   ‡ — e Zd Zdefˆ fd„Zdej        dededej        fd„Zdd	ej	        dej        fd
„Z
ˆ xZS )r<   r&   c                 óz  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        t          j        t          j
        | j        ¦  «        ¦  «        | _        t          j        |j        | j        | j        | j        d¬¦  «        | _        | j        | j        z  dz  | _        | j        dz   | _        t          j        | j        | j        ¦  «        | _        |                      dt          j        | j        ¦  «                             d¦  «        d¬¦  «         d S )NF)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚbiasr7   r    rF   r6   ©Ú
persistent)r8   Ú__init__r&   rR   r@   Ú
image_sizeÚ
patch_sizer   Ú	ParameterrG   Úrandnr?   ÚConv2dÚnum_channelsrA   Únum_patchesrb   Ú	EmbeddingrD   Úregister_bufferrH   rJ   ©rf   r&   rl   s     €rm   rË   z Kosmos2VisionEmbeddings.__init__ì   s  ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØ Ô+ˆŒØ Ô+ˆŒå!œ|­E¬K¸¼Ñ,GÔ,GÑHÔHˆÔå!œyØÔ+ØœØœØ”?Øð 
ñ  
ô  
ˆÔð !œO¨t¬Ñ>À1ÑDˆÔØ!Ô-°Ñ1ˆÔÝ"$¤,¨tÔ/AÀ4Ä>Ñ"RÔ"RˆÔØ×Ò˜^­U¬\¸$Ô:LÑ-MÔ-M×-TÒ-TÐU\Ñ-]Ô-]ÐjoÐÑpÔpÐpÐpÐprn   Ú
embeddingsÚheightÚwidthr«   c                 óÚ  — |j         d         dz
  }| j        j                             d¦  «        }|j         d         dz
  }t          j                             ¦   «         s&||k    r ||k    r|                      | j        ¦  «        S |dd…dd…f         }|dd…dd…f         }|j         d         }	|| j        z  }
|| j        z  }t          |dz  ¦  «        }| 
                    d|||	¦  «        }|                     dddd¦  «        }t          j                             ||
|fdd	¬
¦  «        }|                     dddd¦  «                             dd|	¦  «        }t	          j        ||fd¬¦  «        S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r    r   Nr5   g      à?r   r7   ÚbicubicF)r‚   ÚmodeÚalign_cornersr’   )rI   rD   rB   Ú	unsqueezerG   ÚjitÚ
is_tracingrF   rÍ   r   ÚreshapeÚpermuter   Ú
functionalÚinterpolater–   r—   )rf   rÖ   r×   rØ   rÒ   rD   rb   Úclass_pos_embedÚpatch_pos_embedr“   Ú
new_heightÚ	new_widthÚsqrt_num_positionss                rm   Úinterpolate_pos_encodingz0Kosmos2VisionEmbeddings.interpolate_pos_encoding  s‘  € ð !Ô& qÔ)¨AÑ-ˆØ!Ô4Ô;×EÒEÀaÑHÔHÐØ*Ô0°Ô3°aÑ7ˆõ Œy×#Ò#Ñ%Ô%ð 	>¨+¸Ò*FÐ*FÈ6ÐUZÊ?È?Ø×*Ò*¨4Ô+<Ñ=Ô=Ð=à,¨Q¨Q¨Q°°°¨UÔ3ˆØ,¨Q¨Q¨Q°°°¨UÔ3ˆàÔ˜rÔ"ˆà˜tœÑ.ˆ
Ø˜Tœ_Ñ,ˆ	å& }°cÑ'9Ñ:Ô:ÐØ)×1Ò1°!Ð5GÐI[Ð]`ÑaÔaˆØ)×1Ò1°!°Q¸¸1Ñ=Ô=ˆåœ-×3Ò3ØØ˜iÐ(ØØð	 4ñ 
ô 
ˆð *×1Ò1°!°Q¸¸1Ñ=Ô=×BÒBÀ1ÀbÈ#ÑNÔNˆåŒy˜/¨?Ð;ÀÐCÑCÔCÐCrn   FÚpixel_valuesc                 ó<  — |j         \  }}}}|s<|| j        k    s|| j        k    r&t          d|› d|› d| j        › d| j        › d�	¦  «        ‚| j        j        j        }|                      |                     |¬¦  «        ¦  «        }|                     d¦  «                             dd¦  «        }| j	         
                    |dd¦  «        }	t          j        |	|gd¬	¦  «        }
|r|
|                      |
||¦  «        z   }
n|
|                      | j        ¦  «        z   }
|
S )
NzInput image size (Ú*z) doesn't match model (ú).©r~   r7   r    r5   r’   )rI   rÌ   Ú
ValueErrorrA   rB   r~   rƒ   ÚflattenÚ	transposer?   rJ   rG   r—   ré   rD   rF   )rf   rê   ré   Ú
batch_sizeÚ_r×   rØ   Útarget_dtypeÚpatch_embedsÚclass_embedsrÖ   s              rm   ÚforwardzKosmos2VisionEmbeddings.forward+  sD  € Ø'3Ô'9Ñ$ˆ
�A�v˜uØ'ð 	¨V°t´Ò-FÐ-FÈ%ÐSWÔSbÒJbÐJbÝØq VÐqÐq¨eÐqÐqÈDÌOÐqÐqÐ^bÔ^mÐqÐqÐqñô ð ð Ô+Ô2Ô8ˆØ×+Ò+¨L¯OªOÀ,¨OÑ,OÔ,OÑPÔPˆØ#×+Ò+¨AÑ.Ô.×8Ò8¸¸AÑ>Ô>ˆàÔ+×2Ò2°:¸qÀ"ÑEÔEˆÝ”Y ¨lÐ;ÀÐCÑCÔCˆ
Ø#ð 	QØ# d×&CÒ&CÀJÐPVÐX]Ñ&^Ô&^Ñ^ˆJˆJà# d×&=Ò&=¸dÔ>OÑ&PÔ&PÑPˆJØÐrn   ©F)ro   rp   rq   r#   rË   rG   ÚTensorÚintré   r    r÷   r{   r|   s   @rm   r<   r<   ë   sº   ø€ € € € € ðqÐ2ð qð qð qð qð qð qð,'D°5´<ð 'DÈð 'DÐUXð 'DÐ]bÔ]ið 'Dð 'Dð 'Dð 'DðRð  EÔ$5ð ÐZ_ÔZfð ð ð ð ð ð ð ð rn   r<   r1   r+   ÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutc                 óz  — t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |d¬¦  «        }t          j                             ||| j        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «                             ¦   «         }	|	|fS )Nr5   éþÿÿÿr’   ©ÚpÚtrainingr    r7   )	rG   Úmatmulrñ   r   râ   Úsoftmaxr   r  Ú
contiguous)
r+   rû   rü   rý   rþ   rÿ   r   ÚkwargsÚattn_weightsÚattn_outputs
             rm   Úeager_attention_forwardr  ?  s­   € õ ”<  s§}¢}°R¸Ñ'<Ô'<Ñ=Ô=ÀÑG€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€KØ˜Ð$Ð$rn   c                   óŽ   ‡ — e Zd ZdZˆ fd„Z	 d	dej        dej        dz  dee         de	ej        ej        dz  f         fd„Z
ˆ xZS )
rK   ú=Multi-headed attention from 'Attention Is All You Need' paperc                 ó‚  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        | j        | j        z  | _        | j        | j        z  | j        k    r t          d| j        › d| j        › d�¦  «        ‚| j        dz  | _	        |j
        | _        d| _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        d S )Nú;embed_dim must be divisible by num_heads (got `embed_dim`: ú and `num_heads`: rí   r2   F)r8   rË   r&   rR   r@   Únum_attention_headsÚ	num_headsÚhead_dimrï   ÚscaleÚattention_dropoutr   Ú	is_causalr   ÚLinearrN   rO   rM   rP   rÕ   s     €rm   rË   zKosmos2VisionAttention.__init__X  s  ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØÔ3ˆŒØœ¨$¬.Ñ8ˆŒØŒ=˜4œ>Ñ)¨T¬^Ò;Ð;Ýð'ÈdÌnð 'ð 'Ø”Nð'ð 'ð 'ñô ð ð ”] DÑ(ˆŒ
ØÔ/ˆŒØˆŒå”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝœ	 $¤.°$´.ÑAÔAˆŒˆˆrn   Nr§   rþ   r	  r«   c                 óÄ  — |j         dd…         }g |¢d‘| j        ‘R }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                     |¦  «                             dd¦  «        }|                     |¦  «                             dd¦  «        }|                     |¦  «                             dd¦  «        }t          j        | j	        j
        t          ¦  «        }	 |	| ||||f| j        | j        | j        sdn| j        dœ|¤Ž\  }
} |
j        g |¢d‘R Ž                      ¦   «         }
|                      |
¦  «        }
|
|fS )ú#Input shape: Batch x Time x ChannelNr5   r    r7   r1   )r  rÿ   r   )rI   r  rM   rN   rO   r–   rñ   r   Úget_interfacer&   Ú_attn_implementationr  r  r  r  r   rà   r  rP   )rf   r§   rþ   r	  Úinput_shapeÚhidden_shapeÚqueriesr¹   ÚvaluesÚattention_interfacer  r
  s               rm   r÷   zKosmos2VisionAttention.forwardl  s�  € ð $Ô)¨#¨2¨#Ô.ˆà8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆØ—+’+˜mÑ,Ô,ˆØ�{Š{˜=Ñ)Ô)ˆØ—’˜]Ñ+Ô+ˆà—,’,˜|Ñ,Ô,×6Ò6°q¸!Ñ<Ô<ˆØ�yŠy˜Ñ&Ô&×0Ò0°°AÑ6Ô6ˆØ—’˜\Ñ*Ô*×4Ò4°Q¸Ñ:Ô:ˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð ”nØ”JØ#œ}Ð>�C�C°$´,ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—m’m KÑ0Ô0ˆØ˜LÐ(Ð(rn   ©N)ro   rp   rq   rž   rË   rG   rù   r   r   rŸ   r÷   r{   r|   s   @rm   rK   rK   U  s©   ø€ € € € € ØGÐGðBð Bð Bð Bð Bð. /3ð%)ð %)à”|ð%)ð œ tÑ+ð%)ð Ð+Ô,ð	%)ð
 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð%)ð %)ð %)ð %)ð %)ð %)ð %)ð %)rn   rK   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )rQ   c                 ó  •— t          ¦   «                              ¦   «          || _        t          |j                 | _        t          j        |j        |j	        ¦  «        | _
        t          j        |j	        |j        ¦  «        | _        d S r"  )r8   rË   r&   r	   Ú
hidden_actÚactivation_fnr   r  rR   Úintermediate_sizerS   rT   rÕ   s     €rm   rË   zKosmos2VisionMLP.__init__–  sf   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ# FÔ$5Ô6ˆÔÝ”9˜VÔ/°Ô1IÑJÔJˆŒÝ”9˜VÔ5°vÔ7IÑJÔJˆŒˆˆrn   r§   r«   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r"  )rS   r&  rT   ©rf   r§   s     rm   r÷   zKosmos2VisionMLP.forward�  s=   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆØÐrn   )ro   rp   rq   rË   rG   rù   r÷   r{   r|   s   @rm   rQ   rQ   •  sc   ø€ € € € € ðKð Kð Kð Kð Kð U¤\ð °e´lð ð ð ð ð ð ð ð rn   rQ   c                   óf   ‡ — e Zd Zdefˆ fd„Zdej        dej        dee         dej	        fd„Z
ˆ xZS )r)   r&   c                 óD  •— t          ¦   «                              ¦   «          |j        | _        t	          |¦  «        | _        t          j        | j        |j        ¬¦  «        | _	        t          |¦  «        | _        t          j        | j        |j        ¬¦  «        | _        d S ©N©Úeps)r8   rË   rR   r@   rK   Ú	self_attnr   Ú	LayerNormÚlayer_norm_epsÚlayer_norm1rQ   ÚmlpÚlayer_norm2rÕ   s     €rm   rË   z"Kosmos2VisionEncoderLayer.__init__¦  s   ø€ Ý‰Œ×ÒÑÔÐØÔ+ˆŒÝ/°Ñ7Ô7ˆŒÝœ<¨¬¸FÔ<QÐRÑRÔRˆÔÝ# FÑ+Ô+ˆŒÝœ<¨¬¸FÔ<QÐRÑRÔRˆÔÐÐrn   r§   rþ   r	  r«   c                 óÆ   — |}|                       |¦  «        } | j        d||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N)r§   rþ   r¡   )r2  r/  r4  r3  )rf   r§   rþ   r	  Úresidualró   s         rm   r÷   z!Kosmos2VisionEncoderLayer.forward®  s“   € ð !ˆà×(Ò(¨Ñ7Ô7ˆØ)˜4œ>ð 
Ø'Ø)ð
ð 
ð ð
ð 
Ñˆ�qð
 ! =Ñ0ˆà ˆØ×(Ò(¨Ñ7Ô7ˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆàÐrn   )ro   rp   rq   r#   rË   rG   rù   r   r   r    r÷   r{   r|   s   @rm   r)   r)   ¥  s“   ø€ € € € € ðSÐ2ð Sð Sð Sð Sð Sð Sðà”|ðð œðð Ð+Ô,ð	ð
 
Ô	ðð ð ð ð ð ð ð rn   r)   c                   ób   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej        dz  dee	         de
ez  fd„Zˆ xZS )
ÚKosmos2VisionEncoderz¿
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`Kosmos2VisionEncoderLayer`].

    Args:
        config: Kosmos2VisionConfig
    r&   c                 óÔ   •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r¡   )r)   )r´   ró   r&   s     €rm   ú
<listcomp>z1Kosmos2VisionEncoder.__init__.<locals>.<listcomp>Ò  s"   ø€ Ð$pÐ$pÐ$pÈ1Õ%>¸vÑ%FÔ%FÐ$pÐ$pÐ$prn   F)	r8   rË   r&   r   Ú
ModuleListÚrangerL   ÚlayersÚgradient_checkpointingrÕ   s    `€rm   rË   zKosmos2VisionEncoder.__init__Ï  sb   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”mÐ$pÐ$pÐ$pÐ$pÕPUÐV\ÔVnÑPoÔPoÐ$pÑ$pÔ$pÑqÔqˆŒØ&+ˆÔ#Ð#Ð#rn   Nrþ   r	  r«   c                 óN   — |}| j         D ]} |||fi |¤Ž}Œt          |¬¦  «        S )a7  
        Args:
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
        )r¥   )r>  rœ   )rf   Úinputs_embedsrþ   r	  r§   Úencoder_layers         rm   r÷   zKosmos2VisionEncoder.forwardÕ  s^   € ð( &ˆØ!œ[ð 	ð 	ˆMØ)˜MØØðð ð ðð ˆMˆMõ 7Ø+ð
ñ 
ô 
ð 	
rn   r"  )ro   rp   rq   rž   r#   rË   rG   rù   r   r   rŸ   r   r÷   r{   r|   s   @rm   r8  r8  Æ  s�   ø€ € € € € ðð ð,Ð2ð ,ð ,ð ,ð ,ð ,ð ,ð /3ð
ð 
ð œ tÑ+ð
ð Ð+Ô,ð	
ð
 
�Ñ	 ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
rn   r8  c                   ó¬   ‡ — e Zd ZeedœZdefˆ fd„Ze e	d¬¦  «        e
	 	 ddej        dz  ded	ee         d
efd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚKosmos2VisionTransformer)r§   r¨   r&   c                 óP  •— t          ¦   «                              |¦  «         |j        }t          |¦  «        | _        t          j        ||j        ¬¦  «        | _        t          |¦  «        | _
        t          j        ||j        ¬¦  «        | _        |                      ¦   «          d S r,  )r8   rË   rR   r<   rÖ   r   r0  r1  Úpre_layrnormr8  ÚencoderÚpost_layernormÚ	post_init)rf   r&   r@   rl   s      €rm   rË   z!Kosmos2VisionTransformer.__init__ý  s‹   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØÔ&ˆ	å1°&Ñ9Ô9ˆŒÝœL¨¸Ô8MÐNÑNÔNˆÔÝ+¨FÑ3Ô3ˆŒÝ œl¨9¸&Ô:OÐPÑPÔPˆÔà�ŠÑÔÐÐÐrn   F)Útie_last_hidden_statesNrê   ré   r	  r«   c                 ó  — |€t          d¦  «        ‚|                      ||¬¦  «        }|                      |¦  «        } | j        dd|i|¤Ž}|d         }|d d …dd d …f         }|                      |¦  «        }t          ||¬¦  «        S )Nz You have to specify pixel_values)ré   rA  r   )r¥   Úpooler_outputr¡   )rï   rÖ   rF  rG  rH  r   )rf   rê   ré   r	  r§   Úencoder_outputsr¥   Úpooled_outputs           rm   r÷   z Kosmos2VisionTransformer.forward  s¿   € ð ÐÝÐ?Ñ@Ô@Ð@àŸš¨ÐOg˜ÑhÔhˆØ×)Ò)¨-Ñ8Ô8ˆà&˜$œ,ð 
ð 
Ø'ð
àð
ð 
ˆð
 ,¨AÔ.ÐØ)¨!¨!¨!¨Q°°°¨'Ô2ˆØ×+Ò+¨MÑ:Ô:ˆå)Ø/Ø'ð
ñ 
ô 
ð 	
rn   ©NF)ro   rp   rq   r)   rK   Ú_can_record_outputsr#   rË   r   r   r   rG   r    r…   r   r   r   r÷   r{   r|   s   @rm   rD  rD  ÷  sÜ   ø€ € € € € à2Ø,ðð Ðð
	Ð2ð 	ð 	ð 	ð 	ð 	ð 	ð  Ø€_¨EÐ2Ñ2Ô2Øð 26Ø).ð
ð 
àÔ'¨$Ñ.ð
ð #'ð
ð Ð+Ô,ð	
ð
 
$ð
ð 
ð 
ñ „^ñ 3Ô2ñ  Ôð
ð 
ð 
ð 
ð 
rn   rD  c                   ó*  ‡ — e Zd ZdZddedededz  fˆ fd„Zddedededz  fd„Zeddedededz  fd	„¦   «         Z e	j
        ¦   «         	 	 	 	 dde	j        dz  de	j        dz  dede	j        dz  fd„¦   «         Zed„ ¦   «         Zedd„¦   «         Zˆ xZS )r`   zDThis module produces sinusoidal positional embeddings of any length.Nrb   rd   r^   c                 ó¾   •— t          ¦   «                              ¦   «          d| _        || _        || _        || _        |                      || j        z   ||¦  «         d S )Nr7   )r8   rË   rc   rb   rd   r^   Úmake_weights)rf   rb   rd   r^   rl   s       €rm   rË   z1Kosmos2TextSinusoidalPositionalEmbedding.__init__+  s]   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ*ˆÔØ*ˆÔØ&ˆÔØ×Ò˜-¨$¬+Ñ5°}ÀkÑRÔRÐRÐRÐRrn   Únum_embeddingsc                 óÚ   — |                       |||¦  «        }t          | d¦  «        r+|                     | j        j        | j        j        ¬¦  «        }|                      d|d¬¦  «         d S )Nre   r‘   FrÉ   )ra   r:   rƒ   re   r~   rŽ   rÔ   )rf   rT  rd   r^   rk   s        rm   rS  z5Kosmos2TextSinusoidalPositionalEmbedding.make_weights4  sl   € Ø×(Ò(¨¸ÈÑTÔTˆÝ�4˜Ñ#Ô#ð 	_à%Ÿ.š.¨t¬|Ô/AÈ$Ì,ÔJ]˜.Ñ^Ô^ˆKà×Ò˜Y¨ÀÐÑFÔFÐFÐFÐFrn   c                 óð  — |dz  }t          j        d¦  «        |dz
  z  }t          j        t          j        |t          j        ¬¦  «                             ¦   «         | z  ¦  «        }t          j        | t          j        ¬¦  «                             ¦   «                              d¦  «        |                     d¦  «        z  }t          j        t          j	        |¦  «        t          j
        |¦  «        gd¬¦  «                             | d¦  «        }|dz  dk    r+t          j        |t          j        | d¦  «        gd¬¦  «        }|�	d||dd…f<   |                     t          j        ¦   «         ¦  «        S )	zÊ
        Build sinusoidal embeddings.

        This matches the implementation in tensor2tensor, but differs slightly from the description in Section 3.5 of
        "Attention Is All You Need".
        r7   i'  r    rî   r   r’   r5   N)ÚmathÚlogrG   ÚexprH   Úint64ÚfloatrÝ   r—   ÚsinÚcosr–   r˜   rƒ   Úget_default_dtype)rT  rd   r^   Úhalf_dimÚembs        rm   ra   z6Kosmos2TextSinusoidalPositionalEmbedding.get_embedding<  s?  € ð ! AÑ%ˆÝŒh�u‰oŒo ¨A¡Ñ.ˆÝŒi�œ XµU´[ÐAÑAÔA×GÒGÑIÔIÈSÈDÑPÑQÔQˆÝŒl˜>µ´Ð=Ñ=Ô=×CÒCÑEÔE×OÒOÐPQÑRÔRÐUX×UbÒUbÐcdÑUeÔUeÑeˆÝŒi�œ 3™œ­¬°3©¬Ð8¸aÐ@Ñ@Ô@×EÒEÀnÐVXÑYÔYˆØ˜1Ñ Ò!Ð!å”)˜S¥%¤+¨n¸aÑ"@Ô"@ÐAÀqÐIÑIÔIˆCØÐ"Ø"#ˆC�˜Q˜Q˜Q�Ñà�vŠv•eÔ-Ñ/Ô/Ñ0Ô0Ð0rn   r   Ú	input_idsrA  r�   rF   c                 ó”  — |�N|                      ¦   «         \  }}|€4|                      || j        |¦  «                             |j        ¦  «        }n=|                      ¦   «         d d…         \  }}|€|                      ||| j        ¦  «        }| j        dz   |z   |z   }|| j                              d¦  «        k    r)|                      || j        z   | j	        | j        ¦  «         | j         
                    d|                     d¦  «        ¦  «                             ||| j        j        d         ¦  «                             ¦   «         S )Nr5   r    r   )r‚   Ú"create_position_ids_from_input_idsr^   rƒ   rŽ   Ú&create_position_ids_from_inputs_embedsre   rS  rc   rd   Úindex_selectr–   rI   Údetach)rf   ra  rA  r�   rF   rˆ   Úseq_lenÚmax_poss           rm   r÷   z0Kosmos2TextSinusoidalPositionalEmbedding.forwardR  sH  € ð Ð Ø$Ÿ>š>Ñ+Ô+‰LˆC�ØÐ#à#×FÒFØ˜tÔ/Ð1Gñ ô  ç’"�YÔ%Ñ&Ô&ð øð )×-Ò-Ñ/Ô/°°°Ô4‰LˆC�ØÐ#Ø#×JÒJØ!Ð#9¸4Ô;Kñ ô  �ð
 Ô" QÑ&¨Ñ0Ð3IÑIˆØ�T”\×&Ò& qÑ)Ô)Ò)Ð)Ø×Ò˜g¨¬Ñ3°TÔ5GÈÔIYÑZÔZÐZàŒ|×(Ò(¨¨L×,=Ò,=¸bÑ,AÔ,AÑBÔB×GÒGÈÈWÐVZÔVbÔVhÐikÔVlÑmÔm×tÒtÑvÔvÐvrn   c                 ó$  — |                       ¦   «         dd…         }|d         }t          j        |dz   ||z   dz   t          j        | j        ¬¦  «        }|                     d¦  «                             |¦  «                             ¦   «         |z   S )z×
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nr5   r    r‘   r   )r‚   rG   rH   ÚlongrŽ   rÝ   rJ   r  )rA  r�   r^   r  Úsequence_lengthrF   s         rm   rd  zOKosmos2TextSinusoidalPositionalEmbedding.create_position_ids_from_inputs_embedso  s�   € ð $×(Ò(Ñ*Ô*¨3¨B¨3Ô/ˆØ% aœ.ˆå”|Ø˜!‰O˜_¨{Ñ:¸QÑ>ÅeÄjÐYfÔYmð
ñ 
ô 
ˆð ×%Ò% aÑ(Ô(×/Ò/°Ñ<Ô<×GÒGÑIÔIÐLbÑbÐbrn   c                 óÜ   — |                       |¦  «                             ¦   «         }t          j        |d¬¦  «                             |¦  «        |z   |z  }|                     ¦   «         |z   S )a  
        Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
        are ignored. This is modified from fairseq's `utils.make_positions`.

        Args:
            x: torch.Tensor x:

        Returns: torch.Tensor
        r    r’   )Únerú   rG   ÚcumsumÚtype_asrj  )ra  r^   r�   r}   Úincremental_indicess        rm   rc  zKKosmos2TextSinusoidalPositionalEmbedding.create_position_ids_from_input_ids‚  sg   € ð �|Š|˜KÑ(Ô(×,Ò,Ñ.Ô.ˆÝ$œ|¨D°aÐ8Ñ8Ô8×@Ò@ÀÑFÔFÐI_Ñ_ÐcgÑgÐØ"×'Ò'Ñ)Ô)¨KÑ7Ð7rn   r"  )NNr   N©r   )ro   rp   rq   rž   rú   rË   rS  Ústaticmethodra   rG   ry   rù   r÷   rd  rc  r{   r|   s   @rm   r`   r`   '  s¥  ø€ € € € € ØNÐNðSð S cð S¸#ð SÈCÐRVÉJð Sð Sð Sð Sð Sð SðGð G¨3ð G¸sð GÐQTÐW[ÑQ[ð Gð Gð Gð Gð ð1ð 1 cð 1¸#ð 1ÈCÐRVÉJð 1ð 1ð 1ñ „\ð1ð( €U„]�_„_ð *.Ø-1Ø&'Ø,0ðwð wà”< $Ñ&ðwð ”| dÑ*ðwð !$ð	wð
 ”l TÑ)ðwð wð wñ „_ðwð8 ðcð cñ „\ðcð" ð8ð 8ð 8ñ „\ð8ð 8ð 8ð 8ð 8rn   r`   c                   óê   ‡ — e Zd ZdZ	 	 	 	 	 ddededed	edz  d
edz  dedz  dedz  fˆ fd„Z	 	 	 ddej	        dej	        dz  de
dz  dej	        dz  deej	        ej	        dz  e
dz  f         f
d„Zˆ xZS )rU   r  r1   FTNr@   r  r   Ú
is_decoderÚadd_inner_attn_layernormrÈ   Ú	layer_idxc	                 ón  •— t          ¦   «                              ¦   «          || _        || _        || _        || _        ||z  | _        d| _        | j        |z  | j        k    rt          d| j        › d|› d�¦  «        ‚| j        dz  | _	        || _
        || _        t          j        |||¬¦  «        | _        t          j        |||¬¦  «        | _        t          j        |||¬¦  «        | _        t          j        |||¬¦  «        | _        d | _        |r"t          j        ||j        ¬¦  «        | _        d S d S )NTr  r  rí   r2   )rÈ   r-  )r8   rË   r&   r@   r  r   r  r  rï   rÿ   rt  rv  r   r  rN   rO   rM   rP   Úinner_attn_lnr0  r1  )
rf   r&   r@   r  r   rt  ru  rÈ   rv  rl   s
            €rm   rË   zKosmosTextAttention.__init__˜  sR  ø€ õ 	‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ"ˆŒØˆŒØ! YÑ.ˆŒØˆŒàŒM˜IÑ%¨$¬.Ò8Ð8Ýð3ÈdÌnð 3ð 3Ø%.ð3ð 3ð 3ñô ð ð ”} dÑ*ˆŒØ$ˆŒØ"ˆŒå”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝ”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝ”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝœ	 )¨Y¸TÐBÑBÔBˆŒð "ˆÔØ#ð 	TÝ!#¤¨i¸VÔ=RÐ!SÑ!SÔ!SˆDÔÐÐð	Tð 	Trn   r§   Úencoder_hidden_statesr¦   rþ   r«   c                 óÂ  — |du}|j         dd…         }g |¢d‘| j        ‘R }|                      |¦  «        }	|	                     |¦  «                             dd¦  «        }	d}
|�Ht          |t          ¦  «        r1|j                             | j	        ¦  «        }
|r|j
        }n
|j        }n|}|r|n|}|r3|�1|
r/|j        | j	                 j        }|j        | j	                 j        }nÚg |j         dd…         ¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|�E|                     ||| j	        ¦  «        \  }}|r$t          |t          ¦  «        rd|j        | j	        <   t%          j        | j        j        t,          ¦  «        } || |	|||f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }| j        �|                      |¦  «        }|                      |¦  «        }||fS )	r  Nr5   r    r7   FTr1   )r   rÿ   )rI   r  rM   r–   rñ   r;   r   Ú
is_updatedÚgetrv  Úcross_attention_cacheÚself_attention_cacher>  r¹   r   rN   rO   Úupdater   r  r&   r  r  r  r   rÿ   rà   r  rx  rP   )rf   r§   ry  r¦   rþ   r	  Úis_cross_attentionr  r  Úquery_statesr{  Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚkv_shaper!  r  r
  s                      rm   r÷   zKosmosTextAttention.forward¾  s¿  € ð 3¸$Ð>ÐØ#Ô)¨#¨2¨#Ô.ˆà8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆØ—{’{ =Ñ1Ô1ˆØ#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆàˆ
ØÐ&Ý˜/Õ+>Ñ?Ô?ð 7Ø,Ô7×;Ò;¸D¼NÑKÔK�
Ø%ð Pà+:Ô+PÐ(Ð(à+:Ô+OÐ(Ð(à'6Ð$à2DÐWÐ.Ð.È-ˆØð 	F /Ð"=À*Ð"=à-Ô4°T´^ÔDÔIˆJØ/Ô6°t´~ÔFÔMˆLˆLàF˜Ô-¨c¨r¨cÔ2ÐF°BÐF¸¼ÐFÐFˆHØŸš ^Ñ4Ô4×9Ò9¸(ÑCÔC×MÒMÈaÐQRÑSÔSˆJØŸ;š; ~Ñ6Ô6×;Ò;¸HÑEÔE×OÒOÐPQÐSTÑUÔUˆLàÐ*à+?×+FÒ+FÀzÐS_ÐaeÔaoÑ+pÔ+pÑ(�
˜Là%ð F­*°_ÕFYÑ*ZÔ*Zð FØAE�OÔ.¨t¬~Ñ>å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}Ð>�C�C°$´,Ø”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØÔÐ)Ø×,Ò,¨[Ñ9Ô9ˆKà—m’m KÑ0Ô0ˆà˜LÐ(Ð(rn   )r1   FFTN)NNN)ro   rp   rq   rž   rú   r[  r…   rË   rG   rù   r
   rŸ   r÷   r{   r|   s   @rm   rU   rU   ”  sM  ø€ € € € € ØGÐGð Ø"'Ø05Ø Ø!%ð$Tð $Tð ð$Tð ð	$Tð
 ð$Tð ˜4‘Kð$Tð #'¨¡+ð$Tð �T‰kð$Tð ˜$‘;ð$Tð $Tð $Tð $Tð $Tð $TðR 6:Ø(,Ø.2ðE)ð E)à”|ðE)ð  %œ|¨dÑ2ðE)ð  ™ð	E)ð
 œ tÑ+ðE)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸$±,Ð>Ô	?ðE)ð E)ð E)ð E)ð E)ð E)ð E)ð E)rn   rU   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )rV   r&   c                 ó€  •— t          ¦   «                              ¦   «          |j        | _        t          |j                 | _        |j        | _        t          j        |j	        |j
        ¦  «        | _        t          j        |j
        |j	        ¦  «        | _        t          j        |j
        |j        ¬¦  «        | _        d S r,  )r8   rË   r   r	   Úactivation_functionr&  Úactivation_dropoutr   r  r@   Úffn_dimrS   rT   r0  r1  Úffn_layernormrÕ   s     €rm   rË   zKosmos2TextFFN.__init__  s�   ø€ Ý‰Œ×ÒÑÔÐà”~ˆŒÝ# FÔ$>Ô?ˆÔØ"(Ô";ˆÔå”9˜VÔ-¨v¬~Ñ>Ô>ˆŒÝ”9˜Vœ^¨VÔ-=Ñ>Ô>ˆŒåœ\¨&¬.¸fÔ>SÐTÑTÔTˆÔÐÐrn   c                 óZ  — |                       |                      |¦  «        ¦  «        }t          j                             || j        | j        ¬¦  «        }|                      |¦  «        }|                      |¦  «        }t          j                             || j        | j        ¬¦  «        }|S )Nr  )	r&  rS   r   râ   r   rŠ  r  rŒ  rT   r)  s     rm   r÷   zKosmos2TextFFN.forward  s�   € Ø×*Ò*¨4¯8ª8°MÑ+BÔ+BÑCÔCˆÝœ×-Ò-¨m¸tÔ?VÐaeÔanÐ-ÑoÔoˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆÝœ×-Ò-¨m¸t¼|ÐVZÔVcÐ-ÑdÔdˆàÐrn   )ro   rp   rq   r"   rË   r÷   r{   r|   s   @rm   rV   rV     s[   ø€ € € € € ð
UÐ0ð 
Uð 
Uð 
Uð 
Uð 
Uð 
Uðð ð ð ð ð ð rn   rV   c                   óà   ‡ — e Zd Zddefˆ fd„Z	 	 	 	 	 ddej        dej        dz  dej        dz  dej        dz  d	edz  d
edz  de	ej
        e	ej
        ej
        f         dz  f         fd„Zˆ xZS )r*   Nr&   c           	      ó4  •— t          ¦   «                              ¦   «          |j        | _        t          || j        |j        |j        dd|¬¦  «        | _        |j        | _        t          j	        | j        |j
        ¬¦  «        | _        |j        rOt          || j        |j        |j        dd|¬¦  «        | _        t          j	        | j        |j
        ¬¦  «        | _        t          |¦  «        | _        t          j	        | j        |j
        ¬¦  «        | _        d S )NT)r@   r  r   rt  ru  rv  r-  F)r8   rË   r@   rU   Úattention_headsr  r/  r   r   r0  r1  Úself_attn_layer_normÚadd_cross_attentionÚencoder_attnÚencoder_attn_layer_normrV   ÚffnÚfinal_layer_norm)rf   r&   rv  rl   s      €rm   rË   zKosmos2TextBlock.__init__  s  ø€ Ý‰Œ×ÒÑÔÐØÔ)ˆŒå,ØØ”nØÔ,ØÔ,ØØ%)Øð
ñ 
ô 
ˆŒð ”~ˆŒÝ$&¤L°´ÀVÔEZÐ$[Ñ$[Ô$[ˆÔ!àÔ%ð 
	cÝ 3ØØœ.Ø Ô0ØÔ0ØØ).Ø#ð!ñ !ô !ˆDÔõ ,.¬<¸¼ÈFÔLaÐ+bÑ+bÔ+bˆDÔ(å! &Ñ)Ô)ˆŒÝ "¤¨T¬^ÀÔAVÐ WÑ WÔ WˆÔÐÐrn   Fr§   rþ   ry  Úencoder_attention_maskr¦   Úoutput_attentionsr«   c           	      ó(  — |}|                       |¦  «        } | j        d||||dœ|¤Ž\  }}	t          j                             || j        | j        ¬¦  «        }||z   }|��t          | d¦  «        st          d| › d�¦  «        ‚|}|                      |¦  «        } | j	        d|||||dœ|¤Ž\  }}	t          j                             || j        | j        ¬¦  «        }||z   }|}|  
                    |¦  «        }|                      |¦  «        }||z   }|S )N)r§   r¦   rþ   r˜  r  r“  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`)r§   ry  rþ   r¦   r˜  r¡   )r‘  r/  r   râ   r   r  r:   rï   r”  r“  r–  r•  )
rf   r§   rþ   ry  r—  r¦   r˜  r	  r6  ró   s
             rm   r÷   zKosmos2TextBlock.forward=  s‰  € ð !ˆØ×1Ò1°-Ñ@Ô@ˆà)˜4œ>ð 
Ø'Ø+Ø)Ø/ð	
ð 
ð
 ð
ð 
Ñˆ�qõ œ×-Ò-¨m¸t¼|ÐVZÔVcÐ-ÑdÔdˆØ  =Ñ0ˆð !Ð,Ý˜4 Ñ0Ô0ð Ý ðD¸dð Dð Dð Dñô ð ð
 %ˆHØ ×8Ò8¸ÑGÔGˆMà0˜tÔ0ð  Ø+Ø&;Ø5Ø /Ø"3ð ð  ð ð ð  ÑˆM˜1õ œM×1Ò1°-À4Ä<ÐZ^ÔZgÐ1ÑhÔhˆMØ$ }Ñ4ˆMð !ˆà×-Ò-¨mÑ<Ô<ˆð Ÿš Ñ/Ô/ˆØ  =Ñ0ˆàÐrn   r"  )NNNNF)ro   rp   rq   r"   rË   rG   rù   r
   r…   rŸ   r    r÷   r{   r|   s   @rm   r*   r*     s  ø€ € € € € ðXð XÐ0ð Xð Xð Xð Xð Xð XðD /3Ø59Ø6:Ø(,Ø).ð6ð 6à”|ð6ð œ tÑ+ð6ð  %œ|¨dÑ2ð	6ð
 !&¤¨tÑ 3ð6ð  ™ð6ð   $™;ð6ð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uð6ð 6ð 6ð 6ð 6ð 6ð 6ð 6rn   r*   c            #       ó  ‡ — e Zd ZU eed<   dZe eedd¬¦  «         eedd¬¦  «        dœZ	defˆ fd„Z
d	„ Z	 	 	 	 	 ddej        d
z  dej        d
z  dej        d
z  dedej        d
z  f
d„Zeee	 	 	 	 	 	 	 	 	 	 	 	 	 d dej        d
z  dej        d
z  dej        d
z  dej        d
z  dej        d
z  dej        d
z  ded
z  dej        d
z  dej        d
z  ded
z  ded
z  ded
z  ded
z  dee         deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )!r\   r&   ©r(   r    r/  )ÚindexÚ
layer_namer“  )r§   r¨   Úcross_attentionsc                 óX  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        ‰j        rt          j        ‰j        ¦  «        nd| _        t          j
        ‰j        ‰j        ‰j        ¬¦  «        | _        t          ‰j        ‰j        ‰j        ¬¦  «        | _        t          j        ˆfd„t%          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j        ‰j        ¦  «        | _        d| _        |                      ¦   «          d S )Nr�   )r^   )rb   rd   r^   c                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS ))rv  )r*   )r´   Úir&   s     €rm   r;  z3Kosmos2TextTransformer.__init__.<locals>.<listcomp>�  s'   ø€ Ð$iÐ$iÐ$iÈqÕ%5°fÈÐ%JÑ%JÔ%JÐ$iÐ$iÐ$irn   F)r8   rË   r   Ú	layerdropÚscale_embeddingrW  Úsqrtr@   Úembed_scaler   rÓ   Ú
vocab_sizeÚpad_token_idr]   r`   Úmax_position_embeddingsÚembed_positionsr<  r=  r>  r0  r1  Ú
layer_normr?  rI  rÕ   s    `€rm   rË   zKosmos2TextTransformer.__init__  s
  øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø”~ˆŒØÔ)ˆŒà:@Ô:PÐY�4œ9 VÔ%5Ñ6Ô6Ð6ÐVYˆÔÝœL¨Ô):¸FÔ<LÐZ`ÔZmÐnÑnÔnˆÔåGØ Ô8Ø Ô*ØÔ+ð 
ñ  
ô  
ˆÔõ ”mÐ$iÐ$iÐ$iÐ$iÕTYÐZ`ÔZgÑThÔThÐ$iÑ$iÔ$iÑjÔjˆŒÝœ, vÔ'7¸Ô9NÑOÔOˆŒà&+ˆÔ#à�ŠÑÔÐÐÐrn   c                 óÜ   — d }|d         dk    rt          ||j        |j        |¬¦  «        }|�>t          ||j        |d         ¬¦  «                             |j        ¦  «        }|€|n||z   }|S )Nr5   r    )rŽ   r�   ©r   )rš   r~   rŽ   rŒ   rƒ   )rf   rþ   r  rA  r�   Úcombined_attention_maskÚexpanded_attn_masks          rm   Ú_prepare_decoder_attention_maskz6Kosmos2TextTransformer._prepare_decoder_attention_mask”  s¡   € ð #'ÐØ�rŒ?˜QÒÐÝ&7ØØÔ#Ø$Ô+Ø'=ð	'ñ 'ô 'Ð#ð Ð%å!-¨n¸mÔ>QÐ[fÐgiÔ[jÐ!kÑ!kÔ!k×!nÒ!nØÔ$ñ"ô "Ðð '>Ð&EÐ"Ð"ÐK]Ð`wÑKwð $ð 'Ð&rn   Nr   rA  r©   Úimg_input_maskr�   rF   c                 óØ  — |€|                       |¦  «        }|�b|                     |j        ¦  «                             d|                     d¦  «        ¦  «        ||                     t
          j        ¬¦  «        <   || j        z  }|                      ||||¬¦  «        }|                     |j        ¦  «        }||z   }t          j
                             || j        | j        ¬¦  «        }|S )Nr5   rî   )ra  rA  r�   rF   r  )r]   rƒ   rŽ   r–   r‚   rG   r…   r¥  r©  r   râ   r   r  )	rf   ra  rA  r©   r°  r�   rF   Ú	positionsr§   s	            rm   Úforward_embeddingz(Kosmos2TextTransformer.forward_embedding«  sô   € ð Ð Ø ×-Ò-¨iÑ8Ô8ˆMàÐ#ØAMÇÂÐQ^ÔQeÑAfÔAf×AkÒAkØ�L×%Ò% bÑ)Ô)ñBô BˆM˜.×+Ò+µ%´*Ð+Ñ=Ô=Ñ>ð &¨Ô(8Ñ8ˆð ×(Ò(ØØ'Ø#9Ø%ð	 )ñ 
ô 
ˆ	ð —L’L Ô!5Ñ6Ô6ˆ	à%¨	Ñ1ˆåœ×-Ò-¨m¸t¼|ÐVZÔVcÐ-ÑdÔdˆàÐrn   ra  rþ   Úimage_embeds_position_maskry  r—  r¦   Ú	use_cacher˜  Úoutput_hidden_statesÚreturn_dictr	  r«   c           	      óœ  — |�|�t          d¦  «        ‚|�$|j        }|                     d|d         ¦  «        }n.|�|                     ¦   «         dd…         }nt          d¦  «        ‚|
r[|€Y|€| j        j        r6t          t          | j        ¬¦  «        t          | j        ¬¦  «        ¦  «        nt          | j        ¬¦  «        }|�|                     ¦   «         nd}|dk    rd}d}|  	                    ||||||	¬¦  «        }|  
                    ||||¦  «        }|�|�t          ||j        |d         ¬¦  «        }t          j                             || j        | j        ¬	¦  «        }| j        D ]9}| j        r t%          j        g ¦  «        }|| j        k     rŒ) ||||f||||
d
œ|¤Ž}Œ:|                      |¦  «        }t-          ||¬¦  «        S )áN  
        image_embeds (`torch.FloatTensor` of shape `(batch_size, latent_query_num, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of `Kosmos2ImageToTextProjection`.
        image_embeds_position_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to indicate the location in a sequence to insert the image features . Mask values selected in `[0,
            1]`:

            - 1 for places where to put the image features,
            - 0 for places that are not for image features (i.e. for text tokens).
        NzDYou cannot specify both input_ids and inputs_embeds at the same timer5   z5You have to specify either input_ids or inputs_embeds)r&   r   )ra  rA  r©   r°  r�   rF   r¬  r  )r—  r¦   r˜  rµ  )r¥   r¦   )rï   rI   r–   r‚   r&   Úis_encoder_decoderr   r   Úget_seq_lengthr³  r¯  rŒ   r~   r   râ   r   r  r>  rG   Úrandr¢  rª  r   )rf   ra  rþ   r©   r´  ry  r—  r¦   rA  rF   rµ  r˜  r¶  r·  r	  r  r�   r§   Údecoder_layerÚdropout_probabilitys                       rm   r÷   zKosmos2TextTransformer.forwardÎ  sT  € ð< Ð  ]Ð%>ÝÐcÑdÔdÐdØÐ"Ø#œ/ˆKØ!Ÿš r¨;°r¬?Ñ;Ô;ˆIˆIØÐ&Ø'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKˆKåÐTÑUÔUÐUàð 	˜Ð0ð )Ð4¸¼Ô8VÐ4õ $¥L¸¼Ð$DÑ$DÔ$DÅlÐZ^ÔZeÐFfÑFfÔFfÑgÔgÐgå!¨¬Ð5Ñ5Ô5ð ð FUÐE` ×!?Ò!?Ñ!AÔ!AÐ!AÐfgÐð " AÒ%Ð%ØˆLØ)-Ð&à×.Ò.ØØ'Ø%Ø5Ø#9Ø%ð /ñ 
ô 
ˆð ×=Ò=Ø˜K¨Ð8Nñ
ô 
ˆð
 !Ð,Ð1GÐ1Så%1Ð2HÈ-ÔJ]ÐgrÐsuÔgvÐ%wÑ%wÔ%wÐ"åœ×-Ò-¨m¸t¼|ÐVZÔVcÐ-ÑdÔdˆà!œ[ð 	ð 	ˆMàŒ}ð Ý&+¤j°¡n¤nÐ#Ø&¨¬Ò7Ð7Øà)˜MØØØ%ð	ð (>Ø /Ø"3Ø#ð	ð 	ð ð	ð 	ˆMˆMð Ÿš¨Ñ6Ô6ˆå8Ø+Ø+ð
ñ 
ô 
ð 	
rn   )NNNr   N)NNNNNNNNNNNNN)ro   rp   rq   r"   rr   rs   r*   r   rU   rP  rË   r¯  rG   rù   rú   r³  r   r   r   r
   r…   r   r   rŸ   r   r÷   r{   r|   s   @rm   r\   r\   v  sƒ  ø€ € € € € € ØÐÐÑØ Ðà)Ø$�nÐ%8ÀÈkÐZÑZÔZØ*˜NÐ+>ÀaÐTbÐcÑcÔcðð ÐðÐ0ð ð ð ð ð ð ð*'ð 'ð 'ð4 .2Ø,0Ø.2Ø&'Ø,0ð!ð !ð ”| dÑ*ð!ð ”l TÑ)ð	!ð
 œ tÑ+ð!ð !$ð!ð ”l TÑ)ð!ð !ð !ð !ðF  ØØð *.Ø.2Ø,0Ø:>Ø59Ø6:Ø(,Ø-1Ø,0Ø!%Ø)-Ø,0Ø#'ð_
ð _
à”< $Ñ&ð_
ð œ tÑ+ð_
ð ”l TÑ)ð	_
ð
 %*¤L°4Ñ$7ð_
ð  %œ|¨dÑ2ð_
ð !&¤¨tÑ 3ð_
ð  ™ð_
ð ”| dÑ*ð_
ð ”l TÑ)ð_
ð ˜$‘;ð_
ð   $™;ð_
ð # T™kð_
ð ˜D‘[ð_
ð Ð-Ô.ð_
ð  
Ð:Ñ	:ð!_
ð _
ð _
ñ „^ñ „_ñ  Ôð_
ð _
ð _
ð _
ð _
rn   r\   c                   ó®   ‡ — e Zd ZU eed<   dZdZdefˆ fd„Zdej	        fd„Z
ee	 	 ddej        dz  d	ed
ee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚKosmos2VisionModelr&   rê   )r'   c                 óš   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |                      ¦   «          d S r"  )r8   rË   rD  ÚmodelrI  rÕ   s     €rm   rË   zKosmos2VisionModel.__init__8  s@   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý-¨fÑ5Ô5ˆŒ
à�ŠÑÔÐÐÐrn   r«   c                 ó$   — | j         j        j        S r"  )rÂ  rÖ   rA   rº   s    rm   Úget_input_embeddingsz'Kosmos2VisionModel.get_input_embeddings>  s   € ØŒzÔ$Ô4Ð4rn   NFré   r	  c                 ó"   —  | j         d||dœ|¤ŽS )N)rê   ré   r¡   ©rÂ  )rf   rê   ré   r	  s       rm   r÷   zKosmos2VisionModel.forwardA  s4   € ð ˆtŒzð 
Ø%Ø%=ð
ð 
ð ð
ð 
ð 	
rn   rO  )ro   rp   rq   r#   rr   Úmain_input_namers   rË   r   rz   rÄ  r   r   rG   r    r…   r   r   rŸ   rœ   r÷   r{   r|   s   @rm   rÀ  rÀ  3  sí   ø€ € € € € € ØÐÐÑØ$€OØ!ÐðÐ2ð ð ð ð ð ð ð5 b¤ið 5ð 5ð 5ð 5ð Øð 26Ø).ð

ð 

àÔ'¨$Ñ.ð

ð #'ð

ð Ð+Ô,ð	

ð
 
Ð8Ñ	8ð

ð 

ð 

ñ „^ñ Ôð

ð 

ð 

ð 

ð 

rn   rÀ  c                   óV  ‡ — e Zd ZU eed<   dZdefˆ fd„Zdej        fd„Z	e
e	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  d	ej        dz  d
ej        dz  dej        dz  dej        dz  dedz  dej        dz  dej        dz  dedz  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚKosmos2TextModelr&   r›  c                 óš   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |                      ¦   «          d S r"  )r8   rË   r\   rÂ  rI  rÕ   s     €rm   rË   zKosmos2TextModel.__init__T  s@   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý+¨FÑ3Ô3ˆŒ
à�ŠÑÔÐÐÐrn   r«   c                 ó   — | j         j        S r"  ©rÂ  r]   rº   s    rm   rÄ  z%Kosmos2TextModel.get_input_embeddingsZ  ó   € ØŒzÔ&Ð&rn   Nra  rþ   r©   r´  ry  r—  r¦   rA  rF   rµ  r	  c                 ó2   —  | j         d|||||||||	|
dœ
|¤ŽS )r¹  ©
ra  rþ   r©   r´  ry  r—  r¦   rA  rF   rµ  r¡   rÆ  )rf   ra  rþ   r©   r´  ry  r—  r¦   rA  rF   rµ  r	  s               rm   r÷   zKosmos2TextModel.forward]  sL   € ð4 ˆtŒzð 
ØØ)Ø%Ø'AØ"7Ø#9Ø+Ø'Ø%Øð
ð 
ð ð
ð 
ð 	
rn   )
NNNNNNNNNN)ro   rp   rq   r"   rr   rs   rË   r   rz   rÄ  r   r   rG   rù   r
   r…   r   r   rŸ   r   r÷   r{   r|   s   @rm   rÉ  rÉ  P  sˆ  ø€ € € € € € ØÐÐÑØ ÐðÐ0ð ð ð ð ð ð ð' b¤ið 'ð 'ð 'ð 'ð Øð *.Ø.2Ø,0Ø:>Ø59Ø6:Ø(,Ø-1Ø,0Ø!%ð$
ð $
à”< $Ñ&ð$
ð œ tÑ+ð$
ð ”l TÑ)ð	$
ð
 %*¤L°4Ñ$7ð$
ð  %œ|¨dÑ2ð$
ð !&¤¨tÑ 3ð$
ð  ™ð$
ð ”| dÑ*ð$
ð ”l TÑ)ð$
ð ˜$‘;ð$
ð Ð+Ô,ð$
ð 
Ð:Ñ	:ð$
ð $
ð $
ñ „^ñ Ôð$
ð $
ð $
ð $
ð $
rn   rÉ  z‰
    The text model from KOSMOS-2 with a language modeling head on top (linear layer with weights tied to the input
    embeddings).
    c                    ó¶  ‡ — e Zd ZU eed<   ddiZdefˆ fd„Zdej        fd„Z	dej        fd„Z
ee	 	 	 	 	 	 	 	 	 	 	 	 dd
ej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dej        dz  dej        dz  dej        dz  dedz  deej        z  dee         deez  fd„¦   «         ¦   «         Z	 	 	 	 	 	 	 dˆ fd„	Zˆ xZS )rW   r&   zlm_head.weightzmodel.embed_tokens.weightc                 óæ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S )NF)Úin_featuresÚout_featuresrÈ   )
r8   rË   r\   rÂ  r   r  r@   r¦  rX   rI  rÕ   s     €rm   rË   zKosmos2TextForCausalLM.__init__�  sa   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å+¨FÑ3Ô3ˆŒ
Ý”y¨VÔ-=ÈFÔL]ÐdiÐjÑjÔjˆŒð 	�ŠÑÔÐÐÐrn   r«   c                 ó   — | j         j        S r"  rÌ  rº   s    rm   rÄ  z+Kosmos2TextForCausalLM.get_input_embeddings™  rÍ  rn   c                 ó   — | j         S r"  )rX   rº   s    rm   Úget_output_embeddingsz,Kosmos2TextForCausalLM.get_output_embeddingsœ  s
   € ØŒ|Ðrn   Nr   ra  rþ   r©   r´  ry  r—  r¦   rA  rF   Úlabelsrµ  Úlogits_to_keepr	  c                 ó¤  — |
�|rt                                d¦  «         d} | j        d|||||||||	|dœ
|¤Ž}|j        }t	          |t
          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|
� | j        d||
| j	        j
        dœ|¤Ž}t          |||j        |j        |j        |j        ¬¦  «        S )a  
        image_embeds (`torch.FloatTensor` of shape `(batch_size, latent_query_num, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of `Kosmos2ImageToTextProjection`.
        image_embeds_position_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to indicate the location in a sequence to insert the image features . Mask values selected in `[0,
            1]`:

            - 1 for places where to put the image features,
            - 0 for places that are not for image features (i.e. for text tokens).
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        NzJThe `use_cache` argument is changed to `False` since `labels` is provided.FrÏ  )r¿   r×  r¦  )r¾   r¿   r¦   r§   r¨   rž  r¡   )ÚloggerÚwarningrÂ  r¥   r;   rú   ÚslicerX   Úloss_functionr&   r¦  r   r¦   r§   r¨   rž  )rf   ra  rþ   r©   r´  ry  r—  r¦   rA  rF   r×  rµ  rØ  r	  Úoutputsr§   Úslice_indicesr¿   r¾   s                      rm   r÷   zKosmos2TextForCausalLM.forwardŸ  s.  € ð@ ÐØð mÝ—’ÐkÑlÔlÐlØˆIà=G¸T¼Zð >
ØØ)Ø%Ø'AØ"7Ø#9Ø+Ø'Ø%Øð>
ð >
ð ð>
ð >
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDå0ØØØ#Ô3Ø!Ô/ØÔ)Ø$Ô5ð
ñ 
ô 
ð 	
rn   Fc	                 ó®  •— |s|rd }d }n’|��|�|                      ¦   «         d d…         n|                      ¦   «         \  }
}|                      ¦   «         d         }t          j        |t          j        |
||z
  ft          j        |j        ¬¦  «        fd¬¦  «        } t          ¦   «         j        |f|||||||dœ|	¤Ž}|                     dd ¦  «         |S )Nr5   )r‚   r~   rŽ   r    r’   )r¦   rþ   r©   r´  rA  rµ  Úis_first_iterationrF   )	r‚   rG   r—   r˜   r…   rŽ   r8   Úprepare_inputs_for_generationÚpop)rf   ra  r©   r´  r¦   rþ   rA  rµ  rá  Úmodel_kwargsrò   rg  Úmask_lenÚmodel_inputsrl   s                 €rm   râ  z4Kosmos2TextForCausalLM.prepare_inputs_for_generationä  s"  ø€ ð$ "ð 	 ið 	ØˆLØ)-Ð&Ð&ð (Ð3Ø?LÐ?X -×"4Ò"4Ñ"6Ô"6°s¸°sÔ";Ð";Ð^l×^qÒ^qÑ^sÔ^sÑˆJ˜Ø1×6Ò6Ñ8Ô8¸Ô<ˆHÝ).¬à.Ý”K j°'¸HÑ2DÐ%EÍUÌZÐ`iÔ`pÐqÑqÔqðð ð*ñ *ô *Ð&ð =•u‘w”wÔ<Øð

à+Ø)Ø%Ø'AØ'ØØ1ð

ð 

ð ð

ð 

ˆð 	×Ò˜¨Ñ.Ô.Ð.àÐrn   )NNNNNNNNNNNr   )NNNNNNF)ro   rp   rq   r"   rr   Ú_tied_weights_keysrË   r   rz   rÄ  rÖ  r   r   rG   rù   r
   Ú
LongTensorr…   rú   r   r   rŸ   r   r÷   râ  r{   r|   s   @rm   rW   rW   †  s  ø€ € € € € € ð ÐÐÑØ*Ð,GÐHÐðÐ0ð ð ð ð ð ð ð' b¤ið 'ð 'ð 'ð 'ð r¤yð ð ð ð ð Øð *.Ø.2Ø,0Ø:>Ø59Ø6:Ø(,Ø-1Ø,0Ø*.Ø!%Ø-.ðA
ð A
à”< $Ñ&ðA
ð œ tÑ+ðA
ð ”l TÑ)ð	A
ð
 %*¤L°4Ñ$7ðA
ð  %œ|¨dÑ2ðA
ð !&¤¨tÑ 3ðA
ð  ™ðA
ð ”| dÑ*ðA
ð ”l TÑ)ðA
ð Ô  4Ñ'ðA
ð ˜$‘;ðA
ð ˜eœlÑ*ðA
ð Ð+Ô,ðA
ð 
Ð2Ñ	2ðA
ð A
ð A
ñ „^ñ ÔðA
ðL Ø#'ØØØØØ ð0ð 0ð 0ð 0ð 0ð 0ð 0ð 0ð 0ð 0rn   rW   c                   ó.   ‡ — e Zd ZdZdefˆ fd„Zd„ Zˆ xZS )rY   zmThe layer that transforms the image model's output to part of the text model's input (namely, image features)r&   c                 ó”  •— t          ¦   «                              ¦   «          t          j        |j        j        |j        j        ¦  «        | _        t          j	        t          j        |j        |j        j        ¦  «        ¦  «        | _        t          |j        |j        j        |j        j        |j        j        dd¬¦  «        | _        d S )NF)r   rt  ru  )r8   rË   r   r  r.   rR   r0   r@   rZ   rÎ   rG   rÏ   Úlatent_query_numr[   rU   r�  r  Úx_attnrÕ   s     €rm   rË   z%Kosmos2ImageToTextProjection.__init__  s    ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ3Ô?ÀÔASÔA]Ñ^Ô^ˆŒ
ÝœL­¬°VÔ5LÈfÔN`ÔNjÑ)kÔ)kÑlÔlˆÔå)ØÔØÔÔ(ØÔÔ.ØÔ&Ô8ØØ%*ð
ñ 
ô 
ˆŒˆˆrn   c                 ó"  — |                       |¦  «        }| j                             d¦  «                             |                     d¦  «        dd¦  «        }t          j        ||gd¬¦  «        }|                      ||d d d ¬¦  «        \  }}||fS )Nr   r5   r    r’   )r§   ry  r¦   rþ   r˜  )rZ   r[   rÝ   rJ   r‚   rG   r—   rì  )rf   Úfeaturesr§   r[   Úkey_value_statesr
  s         rm   r÷   z$Kosmos2ImageToTextProjection.forward(  s¡   € ØŸ
š
 8Ñ,Ô,ˆð Ô(×2Ò2°1Ñ5Ô5×<Ò<¸]×=OÒ=OÐPQÑ=RÔ=RÐTVÐXZÑ[Ô[ˆÝ œ9 m°\Ð%BÈÐJÑJÔJÐà&*§k¢kØ&Ø"2Ø ØØ"ð '2ñ '
ô '
Ñ#ˆ�|ð ˜lÐ*Ð*rn   )ro   rp   rq   rž   r!   rË   r÷   r{   r|   s   @rm   rY   rY     sY   ø€ € € € € ØwÐwð
˜}ð 
ð 
ð 
ð 
ð 
ð 
ð+ð +ð +ð +ð +ð +ð +rn   rY   z}
    KOSMOS-2 Model for generating text and image features. The model consists of a vision encoder and a language model.
    c                   óª  ‡ — e Zd ZU eed<   dZdefˆ fd„Zdej        fd„Z	d„ Z
ee	 ddej        ded	z  d
ee         deez  fd„¦   «         ¦   «         Zee	 	 	 	 	 	 	 	 	 	 ddej        d	z  dej        d	z  dej        d	z  dej        d	z  ded	z  dej        d	z  dej        d	z  dej        d	z  ded	z  ded
ee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚKosmos2Modelr&   rê   c                 óþ   •— t          ¦   «                              |¦  «         t          |j        ¦  «        | _        t          |j        ¦  «        | _        t          |¦  «        | _	        |  
                    ¦   «          d S r"  )r8   rË   rÉ  r0   Ú
text_modelrÀ  r.   Úvision_modelrY   Úimage_to_text_projectionrI  rÕ   s     €rm   rË   zKosmos2Model.__init__C  sh   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å*¨6Ô+=Ñ>Ô>ˆŒÝ.¨vÔ/CÑDÔDˆÔÝ(DÀVÑ(LÔ(LˆÔ%ð 	�ŠÑÔÐÐÐrn   r«   c                 ó$   — | j         j        j        S r"  ©ró  rÂ  r]   rº   s    rm   rÄ  z!Kosmos2Model.get_input_embeddingsM  ó   € ØŒÔ$Ô1Ð1rn   c                 ó(   — || j         j        _        d S r"  r÷  ©rf   rý   s     rm   Úset_input_embeddingsz!Kosmos2Model.set_input_embeddingsP  ó   € Ø-2ˆŒÔÔ*Ð*Ð*rn   Fré   Nr	  c                 óh  — d|v r0t          j        dt          ¦  «         |                     dd ¦  «          | j        d||ddœ|¤Ž}| j        j                             |d         ¦  «        }t          j         	                    |d¬¦  «        }|  
                    |¦  «        \  }}||_        ||_        |S )	NÚreturn_attentionsz¯`return_attentions` is deprecated and will be removed in a future version. Please use `return_dict` and access `projection_attentions` from the returned `ModelOutput` instead.T)rê   ré   r·  r   r5   r’   r¡   )ÚwarningsÚwarnÚFutureWarningrã  rô  rÂ  rH  r   râ   Ú	normalizerõ  rL  r�   )rf   rê   ré   r	  Úvision_outputr©   r�   s          rm   Úget_image_featureszKosmos2Model.get_image_featuresS  sâ   € ð  &Ð(Ð(ÝŒMð_åñô ð ð
 �JŠJÐ*¨DÑ1Ô1Ð1àARÀÔARð B
Ø%Ø%=ØðB
ð B
ð ð	B
ð B
ˆð Ô(Ô.×=Ò=¸mÈAÔ>NÑOÔOˆå”}×.Ò.¨|ÀÐ.ÑDÔDˆØ.2×.KÒ.KÈLÑ.YÔ.YÑ+ˆÐ+Ø&2ˆÔ#Ø.CˆÔ+àÐrn   ra  r´  rþ   r¦   r©   rA  rF   rµ  c                 óð   — d}d}|€0|€t          d¦  «        ‚ | j        |f|
ddœ|¤Ž}|j        }|j        } | j        d||||||||	ddœ	|¤Ž}t          |j        |j        |j        |j	        |||¬¦  «        S )a–  
        image_embeds_position_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to indicate the location in a sequence to insert the image features . Mask values selected in `[0,
            1]`:

            - 1 for places where to put the image features,
            - 0 for places that are not for image features (i.e. for text tokens).
        image_embeds (`torch.FloatTensor` of shape `(batch_size, latent_query_num, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of `Kosmos2ImageToTextProjection`.

        Examples:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Kosmos2Model

        >>> model = Kosmos2Model.from_pretrained("microsoft/kosmos-2-patch14-224")
        >>> processor = AutoProcessor.from_pretrained("microsoft/kosmos-2-patch14-224")

        >>> url = "https://huggingface.co/microsoft/kosmos-2-patch14-224/resolve/main/snowman.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> text = (
        ...     "<grounding> An image of<phrase> a snowman</phrase><object><patch_index_0044><patch_index_0863>"
        ...     "</object> warming himself by<phrase> a fire</phrase><object><patch_index_0005><patch_index_0911>"
        ...     "</object>"
        ... )

        >>> inputs = processor(text=text, images=image, return_tensors="pt", add_eos_token=True)

        >>> last_hidden_state = model(
        ...     pixel_values=inputs["pixel_values"],
        ...     input_ids=inputs["input_ids"],
        ...     attention_mask=inputs["attention_mask"],
        ...     image_embeds_position_mask=inputs["image_embeds_position_mask"],
        ... ).last_hidden_state
        >>> list(last_hidden_state.shape)
        [1, 91, 2048]
        ```Nú<You have to specify either `pixel_values` or `image_embeds`.T)ré   r·  )	ra  rþ   r©   r´  r¦   rA  rF   rµ  r·  )r¥   r¦   r§   r¨   r©   r�   rª   r¡   )
rï   r  rL  r�   ró  r¤   r¥   r¦   r§   r¨   )rf   rê   ra  r´  rþ   r¦   r©   rA  rF   rµ  ré   r	  rª   r�   Úimage_featuresrÞ  s                   rm   r÷   zKosmos2Model.forwards  sê   € ðt #ÐØ $ÐØÐØÐ#Ý Ð!_Ñ`Ô`Ð`Ø4˜TÔ4ØðØ7OÐ]aðð Øekðð ˆNð *Ô7ˆLØ$2Ô$HÐ!à!�$”/ð 
ØØ)Ø%Ø'AØ+Ø'Ø%ØØð
ð 
ð ð
ð 
ˆõ "Ø%Ô7Ø#Ô3Ø!Ô/ØÔ)Ø%Ø"7Ø 3ð
ñ 
ô 
ð 	
rn   rø   )
NNNNNNNNNF)ro   rp   rq   r!   rr   rÇ  rË   r   rz   rÄ  rû  r   r   rG   r    r…   r   r   rŸ   rœ   r  rù   r
   r¤   r÷   r{   r|   s   @rm   rñ  rñ  :  s  ø€ € € € € € ð ÐÐÑØ$€Oð˜}ð ð ð ð ð ð ð2 b¤ið 2ð 2ð 2ð 2ð3ð 3ð 3ð Øð 16ðð àÔ'ðð #'¨¡+ðð Ð+Ô,ð	ð
 
Ð8Ñ	8ðð ð ñ „^ñ Ôðð< Øð -1Ø)-Ø:>Ø.2Ø(,Ø,0Ø-1Ø,0Ø!%Ø).ðX
ð X
à”l TÑ)ðX
ð ”< $Ñ&ðX
ð %*¤L°4Ñ$7ð	X
ð
 œ tÑ+ðX
ð  ™ðX
ð ”l TÑ)ðX
ð ”| dÑ*ðX
ð ”l TÑ)ðX
ð ˜$‘;ðX
ð #'ðX
ð Ð+Ô,ðX
ð 
Ð#Ñ	#ðX
ð X
ð X
ñ „^ñ ÔðX
ð X
ð X
ð X
ð X
rn   rñ  z�
    KOSMOS-2 Model for generating text and bounding boxes given an image. The model consists of a vision encoder and a
    language model.
    c                   óN  ‡ — e Zd ZU eed<   dZddiZdefˆ fd„Zdej	        fd„Z
d„ Zdej	        fd	„Zd
„ Zee	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dedz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  deej        z  dee         deez  fd„¦   «         ¦   «         Z ej        ¦   «         	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  fd„¦   «         Zˆ xZS )ÚKosmos2ForConditionalGenerationr&   rê   ztext_model.lm_head.weightz$text_model.model.embed_tokens.weightc                 óþ   •— t          ¦   «                              |¦  «         t          |j        ¦  «        | _        t          |j        ¦  «        | _        t          |¦  «        | _	        |  
                    ¦   «          d S r"  )r8   rË   rW   r0   ró  rÀ  r.   rô  rY   rõ  rI  rÕ   s     €rm   rË   z(Kosmos2ForConditionalGeneration.__init__Û  sh   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å0°Ô1CÑDÔDˆŒÝ.¨vÔ/CÑDÔDˆÔå(DÀVÑ(LÔ(LˆÔ%ð 	�ŠÑÔÐÐÐrn   r«   c                 ó$   — | j         j        j        S r"  r÷  rº   s    rm   rÄ  z4Kosmos2ForConditionalGeneration.get_input_embeddingsæ  rø  rn   c                 ó(   — || j         j        _        d S r"  r÷  rú  s     rm   rû  z4Kosmos2ForConditionalGeneration.set_input_embeddingsé  rü  rn   c                 ó4   — | j                              ¦   «         S r"  )ró  rÖ  rº   s    rm   rÖ  z5Kosmos2ForConditionalGeneration.get_output_embeddingsì  s   € ØŒ×4Ò4Ñ6Ô6Ð6rn   c                 ó:   — | j                              |¦  «         d S r"  )ró  Úset_output_embeddings)rf   Únew_embeddingss     rm   r  z5Kosmos2ForConditionalGeneration.set_output_embeddingsï  s   € ØŒ×-Ò-¨nÑ=Ô=Ð=Ð=Ð=rn   Nr   ra  r´  rþ   r¦   r©   rA  rF   r×  rµ  rØ  r	  c                 ó¨  — d}d}|€…|€t          d¦  «        ‚|                      |¬¦  «        }| j        j                             |d         ¦  «        }t          j                             |d¬¦  «        }|                      |¦  «        \  }} | j        d	||||||||	|
|dœ
|¤Ž}t          |j
        |j        |j        |j        |j        |||¬¦  «        S )
a†  
        image_embeds_position_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to indicate the location in a sequence to insert the image features . Mask values selected in `[0,
            1]`:

            - 1 for places where to put the image features,
            - 0 for places that are not for image features (i.e. for text tokens).
        image_embeds (`torch.FloatTensor` of shape `(batch_size, latent_query_num, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of `Kosmos2ImageToTextProjection`.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`

        Examples:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Kosmos2ForConditionalGeneration

        >>> model = Kosmos2ForConditionalGeneration.from_pretrained("microsoft/kosmos-2-patch14-224")
        >>> processor = AutoProcessor.from_pretrained("microsoft/kosmos-2-patch14-224")

        >>> url = "https://huggingface.co/microsoft/kosmos-2-patch14-224/resolve/main/snowman.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> prompt = "<grounding> An image of"

        >>> inputs = processor(text=prompt, images=image, return_tensors="pt")

        >>> generated_ids = model.generate(
        ...     pixel_values=inputs["pixel_values"],
        ...     input_ids=inputs["input_ids"],
        ...     attention_mask=inputs["attention_mask"],
        ...     image_embeds=None,
        ...     image_embeds_position_mask=inputs["image_embeds_position_mask"],
        ...     use_cache=True,
        ...     max_new_tokens=64,
        ... )
        >>> generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        >>> processed_text = processor.post_process_generation(generated_text, cleanup_and_extract=False)
        >>> processed_text
        '<grounding> An image of<phrase> a snowman</phrase><object><patch_index_0044><patch_index_0863></object> warming himself by<phrase> a fire</phrase><object><patch_index_0005><patch_index_0911></object>.'

        >>> caption, entities = processor.post_process_generation(generated_text)
        >>> caption
        'An image of a snowman warming himself by a fire.'

        >>> entities
        [('a snowman', (12, 21), [(0.390625, 0.046875, 0.984375, 0.828125)]), ('a fire', (41, 47), [(0.171875, 0.015625, 0.484375, 0.890625)])]
        ```Nr  )rê   r   r5   r’   )
ra  rþ   r©   r´  r¦   rA  rF   r×  rµ  rØ  )r¾   r¿   r¦   r§   r¨   r©   r�   rª   r¡   )rï   rô  rÂ  rH  r   râ   r  rõ  ró  r½   r¾   r¿   r¦   r§   r¨   )rf   rê   ra  r´  rþ   r¦   r©   rA  rF   r×  rµ  rØ  r	  rª   r�   Ú
lm_outputss                   rm   r÷   z'Kosmos2ForConditionalGeneration.forwardò  s$  € ðN #ÐØ $ÐØÐØÐ#Ý Ð!_Ñ`Ô`Ð`à"&×"3Ò"3Ø)ð #4ñ #ô #Ðð  Ô,Ô2×AÒAÐBUÐVWÔBXÑYÔYˆLåœ=×2Ò2°<ÀRÐ2ÑHÔHˆLØ26×2OÒ2OÐP\Ñ2]Ô2]Ñ/ˆLÐ/à8G¸¼ð 9
ØØ)Ø%Ø'AØ+Ø'Ø%ØØØ)ð9
ð 9
ð ð9
ð 9
ˆ
õ :Ø”ØÔ$Ø&Ô6Ø$Ô2Ø!Ô,Ø%Ø"7Ø 3ð	
ñ 	
ô 	
ð 		
rn   c           	      ó†  — |                      dd ¦  «        }|�|�t          d|› d�¦  «        ‚|€|�|}|€s|                      |¦  «        }	| j        j                             |	d         ¦  «        }t
          j                             |d¬¦  «        }|                      |¦  «        \  }}
 | j	        j
        d|||||dœ|¤Ž}|S )	NÚinputsz
`inputs`: zp were passed alongside `pixel_values` which is not allowed.Make sure to either pass `inputs` or pixel_values=...r   r5   r’   )ra  rþ   r©   r´  rA  r¡   )rã  rï   rô  rÂ  rH  r   râ   r  rõ  ró  Úgenerate)rf   rê   r´  ra  rþ   r©   rA  r	  r  rª   r�   Úoutputs               rm   r  z(Kosmos2ForConditionalGeneration.generatea  s  € ð —’˜H dÑ+Ô+ˆØÐ#¨Ð(:ÝðI˜Vð Ið Ið Iñô ð ð Ð FÐ$6Ø!ˆLàÐØ"&×"3Ò"3°LÑ"AÔ"AÐàÔ,Ô2×AÒAÐBUÐVWÔBXÑYÔYˆLåœ=×2Ò2°<ÀRÐ2ÑHÔHˆLØ26×2OÒ2OÐP\Ñ2]Ô2]Ñ/ˆLÐ/à)�”Ô)ð 
ØØ)Ø%Ø'AØ'ð
ð 
ð ð
ð 
ˆð ˆrn   )NNNNNNNNNNr   )NNNNNN)ro   rp   rq   r!   rr   rÇ  rç  rË   r   rz   rÄ  rû  rÖ  r  r   r   rG   rù   r
   rè  r…   rú   r   r   rŸ   r½   r÷   ry   r  r{   r|   s   @rm   r	  r	  Ð  sœ  ø€ € € € € € ð ÐÐÑØ$€OØ5Ð7]Ð^Ðð	˜}ð 	ð 	ð 	ð 	ð 	ð 	ð2 b¤ið 2ð 2ð 2ð 2ð3ð 3ð 3ð7 r¤yð 7ð 7ð 7ð 7ð>ð >ð >ð Øð -1Ø)-Ø:>Ø.2Ø(,Ø,0Ø-1Ø,0Ø*.Ø!%Ø-.ðk
ð k
à”l TÑ)ðk
ð ”< $Ñ&ðk
ð %*¤L°4Ñ$7ð	k
ð
 œ tÑ+ðk
ð  ™ðk
ð ”l TÑ)ðk
ð ”| dÑ*ðk
ð ”l TÑ)ðk
ð Ô  4Ñ'ðk
ð ˜$‘;ðk
ð ˜eœlÑ*ðk
ð Ð+Ô,ðk
ð 
Ð;Ñ	;ðk
ð k
ð k
ñ „^ñ Ôðk
ðZ €U„]�_„_ð -1Ø:>Ø)-Ø.2Ø,0Ø-1ð%ð %à”l TÑ)ð%ð %*¤L°4Ñ$7ð%ð ”< $Ñ&ð	%ð
 œ tÑ+ð%ð ”l TÑ)ð%ð ”| dÑ*ð%ð %ð %ñ „_ð%ð %ð %ð %ð %rn   r	  )r	  rñ  r%   r"  rq  )r1   )Wrž   rW  rÿ  Úcollections.abcr   Údataclassesr   Útypingr   rG   r   Ú r   r=   Úactivationsr	   Úcache_utilsr
   r   r   Ú
generationr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úconfiguration_kosmos2r!   r"   r#   Ú
get_loggerro   rÚ  r%   rù   r~   rú   rŒ   ÚSizerŽ   rš   rœ   r¤   r½   rz   r<   r[  r  rK   rQ   r)   r8  rD  r`   rU   rV   r*   r\   rÀ  rÉ  rW   rY   rñ  r	  Ú__all__r¡   rn   rm   ú<module>r*     sÆ  ðð Ð à €€€Ø €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ )Ð )Ð )Ð )Ð )Ð )Ø BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð GÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jØ 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ Xð 
ˆÔ	˜HÑ	%Ô	%€ð ð=4ð =4ð =4ð =4ð =4˜_ñ =4ô =4ñ „ð=4ð@[ð [�u”|ð [¨E¬Kð [À#ÈÁ*ð [ð [ð [ð [ð jkð\ð \Ø”Zð\Ø(-¬ð\Ø=B¼\ð\Øcfð\ð \ð \ð \ð" Ø
ð
Bð 
Bð 
Bð 
Bð 
BÐ.Hñ 
Bô 
Bñ „ñ „ð
Bð €ððñ ô ð
 ð
ð 
ð 
ð 
ð 
˜ñ 
ô 
ñ „ñô ð
ð: €ððñ ô ð
 ð
ð 
ð 
ð 
ð 
°ñ 
ô 
ñ „ñô ð
ðFPð Pð Pð Pð P˜bœiñ Pô Pð Pðv ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð %ð %ð %ð,<)ð <)ð <)ð <)ð <)˜RœYñ <)ô <)ð <)ð@ð ð ð ð �r”yñ ô ð ð ð ð ð ð Ð :ñ ô ð ðB-
ð -
ð -
ð -
ð -
˜2œ9ñ -
ô -
ð -
ðb,
ð ,
ð ,
ð ,
ð ,
Ð5ñ ,
ô ,
ð ,
ð`j8ð j8ð j8ð j8ð j8¨r¬yñ j8ô j8ð j8ðZo)ð o)ð o)ð o)ð o)˜"œ)ñ o)ô o)ð o)ðdð ð ð ð �R”Yñ ô ð ð.Vð Vð Vð Vð VÐ1ñ Vô Vð Vðrz
ð z
ð z
ð z
ð z
Ð3ñ z
ô z
ð z
ðz
ð 
ð 
ð 
ð 
Ð/ñ 
ô 
ð 
ð:3
ð 3
ð 3
ð 3
ð 3
Ð-ñ 3
ô 3
ð 3
ðl €ððñ ô ðHð Hð Hð Hð HÐ3°_ñ Hô Hñô ðHðV +ð  +ð  +ð  +ð  + 2¤9ñ  +ô  +ð  +ðF €ððñ ô ð
N
ð N
ð N
ð N
ð N
Ð)ñ N
ô N
ñô ð
N
ðb €ððñ ô ðqð qð qð qð qÐ&<¸oñ qô qñô ðqðh XÐ
WÐ
W€€€rn   