§
    ‚ŠtjâÙ  ã                   ój  — d dl Z d dlmZ d dlmZmZ d dlZd dlmZ d dl	m
Z
 ddlmZ ddlmZmZmZ ddlmZ dd	lmZmZmZ dd
lmZ ddlmZ ddlmZmZmZm Z m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z,m-Z-m.Z.m/Z/m0Z0 ddl1m2Z2 ddl3m4Z4m5Z5 ddl6m7Z7 ddl8m9Z9m:Z: ddl;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZF ddlGmHZH ddlImJZJmKZKmLZL  e/jM        eN¦  «        ZO e-d¬¦  «        e
 G d„ de:e¦  «        ¦   «         ¦   «         ZP e-d¬¦  «        e
 G d„ de9¦  «        ¦   «         ¦   «         ZQ e-d¬¦  «        e
 G d„ d e:e¦  «        ¦   «         ¦   «         ZR e-d¬¦  «        e
 G d!„ d"e¦  «        ¦   «         ¦   «         ZS G d#„ d$e@¦  «        ZT G d%„ d&e=¦  «        ZU G d'„ d(eA¦  «        ZV G d)„ d*e<¦  «        ZW G d+„ d,e<¦  «        ZXdNd.eYd/efd0„ZZ G d1„ d2eK¦  «        Z[ G d3„ d4eK¦  «        Z\ G d5„ d6eL¦  «        Z] G d7„ d8eJ¦  «        Z^ G d9„ d:e>¦  «        Z_ G d;„ d<eB¦  «        Z`e- G d=„ d>e?¦  «        ¦   «         Za G d?„ d@ea¦  «        Zb G dA„ dBea¦  «        Zc G dC„ dDea¦  «        Zde- G dE„ dFea¦  «        ¦   «         Ze G dG„ dHeae¦  «        Zfe- G dI„ dJea¦  «        ¦   «         Zge- G dK„ dLea¦  «        ¦   «         Zhg dM¢ZidS )Oé    N)ÚCallable)ÚAnyÚOptional)Ústricté   )Úinitialization)ÚDynamicCacheÚEncoderDecoderCacheÚStaticCache)ÚPreTrainedConfig)ÚGenerationConfigÚGenerationMixinÚGenerationMode)Úcreate_bidirectional_mask)ÚFlashAttentionKwargs)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚBaseModelOutputWithPoolingÚSeq2SeqLMOutputÚSeq2SeqModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚROPE_INIT_FUNCTIONS)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚtorch_compilable_check)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Ú	AutoModel)ÚGemma3ConfigÚGemma3TextConfig)ÚGemma3AttentionÚ	Gemma3MLPÚGemma3MultiModalProjectorÚGemma3PreTrainedModelÚGemma3RMSNormÚGemma3RotaryEmbeddingÚGemma3TextScaledWordEmbeddingÚapply_rotary_pos_embÚcreate_causal_maskÚ!create_sliding_window_causal_maskÚeager_attention_forward)ÚSiglipVisionConfig)ÚT5GemmaClassificationHeadÚT5GemmaEncoderLayerÚT5GemmaLMHeadzgoogle/t5gemma-2-270m-270m)Ú
checkpointc                   ó0   — e Zd ZdZdZ e¦   «         Zd„ ZdS )ÚT5Gemma2TextConfigát  
    query_pre_attn_scalar (`float`, *optional*, defaults to 256):
        Scaling factor used on the attention scores
    final_logit_softcapping (`float`, *optional*):
        Scaling factor when applying tanh softcapping on the logits.
    attn_logit_softcapping (`float`, *optional*):
        Scaling factor when applying tanh softcapping on the attention scores.
    Út5gemma2_textc                 ó®   ‡— |                      dd¦  «        Š| j        €%ˆfd„t          | j        ¦  «        D ¦   «         | _        t	          j        di |¤Ž d S )NÚsliding_window_patterné   c                 óB   •— g | ]}t          |d z   ‰z  ¦  «        rdnd‘ŒS ©é   Úsliding_attentionÚfull_attention©Úbool©Ú.0ÚiÚ_sliding_window_patterns     €úk/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/t5gemma2/modular_t5gemma2.pyú
<listcomp>z4T5Gemma2TextConfig.__post_init__.<locals>.<listcomp>_   óG   ø€ ð  ð  ð  àõ (,¨Q°©UÐ6MÑ,MÑ'NÔ'NÐdÐ#Ð#ÐTdð ð  ð  ó    © ©ÚpopÚlayer_typesÚrangeÚnum_hidden_layersr   Ú__post_init__©ÚselfÚkwargsrJ   s     @rK   rU   z T5Gemma2TextConfig.__post_init__[   óv   ø€ à"(§*¢*Ð-EÀqÑ"IÔ"IÐØÔÐ#ð ð  ð  ð  å˜tÔ5Ñ6Ô6ð ñ  ô  ˆDÔõ
 	Ô&Ð0Ð0¨Ð0Ð0Ð0Ð0Ð0rN   N©Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚAttributeErrorÚuse_bidirectional_attentionrU   rO   rN   rK   r:   r:   L   sB   € € € € € ðð ð !€JØ"0 .Ñ"2Ô"2Ðð	1ð 	1ð 	1ð 	1ð 	1rN   r:   c                   ó   — e Zd ZdZeedœZdS )ÚT5Gemma2EncoderConfigÚt5gemma2_encoder)Útext_configÚvision_configN)r[   r\   r]   r_   r:   r4   Úsub_configsrO   rN   rK   rc   rc   g   s)   € € € € € ð $€Jð *Ø+ðð €K€K€KrN   rc   c                   ó0   — e Zd ZdZdZ e¦   «         Zd„ ZdS )ÚT5Gemma2DecoderConfigr;   Út5gemma2_decoderc                 ó®   ‡— |                      dd¦  «        Š| j        €%ˆfd„t          | j        ¦  «        D ¦   «         | _        t	          j        di |¤Ž d S )Nr>   r?   c                 óB   •— g | ]}t          |d z   ‰z  ¦  «        rdnd‘ŒS rA   rE   rG   s     €rK   rL   z7T5Gemma2DecoderConfig.__post_init__.<locals>.<listcomp>…   rM   rN   rO   rP   rV   s     @rK   rU   z#T5Gemma2DecoderConfig.__post_init__�   rY   rN   NrZ   rO   rN   rK   ri   ri   r   sB   € € € € € ðð ð $€JØ"0 .Ñ"2Ô"2Ðð	1ð 	1ð 	1ð 	1ð 	1rN   ri   c                   ó$  ‡ — e Zd ZU dZdZdgZeedœZdddœZ	dZ
eeeef         z  dz  ed	<   dZeeeef         z  dz  ed
<   dZeed<   dZeez  ed<   dZeez  ed<   dZeez  ed<   dZeed<   dZeed<   dZedz  ed<   dZeed<   ˆ fd„Zd„ Zˆ xZS )ÚT5Gemma2Configa`  
    encoder (`Union[T5Gemma2EncoderConfig, dict]`, optional, *optional*):
        Configuration for the encoder.
    decoder (`Union[T5Gemma2DecoderConfig, dict]`, optional, *optional*):
        Configuration for the decoder.
    eoi_token_index (`int`, *optional*):
        The end-of-image token index to wrap the image prompt. Will be same as
        `self.encoder.eoi_token_index`

    ```python
    >>> from transformers import T5Gemma2Config, T5Gemma2Model
    >>> t5gemma2_config = T5Gemma2Config.from_pretrained("google/t5gemma-270m-270m")
    >>> model = T5Gemma2Model(t5gemma2_config)
    ```
    Út5gemma2Úpast_key_values)ÚencoderÚdecoderÚimage_token_indexÚeoi_token_index)Úimage_token_idÚeoi_token_idNrq   rr   TÚis_encoder_decoderç        Údropout_rateÚattention_dropoutÚclassifier_dropout_rateg{®Gáz”?Úinitializer_rangeiè Útie_word_embeddingsc                 ó  •— t          | j        t          ¦  «        rt          di | j        ¤Ž| _        n4| j        €-t          ¦   «         | _        t                               d¦  «         t          | j        t          ¦  «        rt          di | j        ¤Ž| _        n4| j        €-t          ¦   «         | _        t                               d¦  «         | j        | j        j	        _        | j
        | j        j	        _
        | j
        | j        j        _
        | j        | j        _        | j        | j        _        | j
        | j        _
        | j        j        | _        dD ]}||vrt          | j        |¦  «        ||<   Œ t          ¦   «         j        di |¤Ž d S )NzDencoder is None, using default T5Gemma2EncoderConfig encoder config.zDdecoder is None, using default T5Gemma2DecoderConfig decoder config.)Úbos_token_idÚpad_token_idÚeos_token_idÚ
vocab_sizerO   )Ú
isinstancerq   Údictrc   ÚloggerÚinforr   ri   ry   re   rz   rf   rs   rt   ÚgetattrÚsuperrU   )rW   rX   Úspecial_token_keyÚ	__class__s      €rK   rU   zT5Gemma2Config.__post_init__¸   sl  ø€ Ý�d”l¥DÑ)Ô)ð 	`Ý0Ð@Ð@°4´<Ð@Ð@ˆDŒLˆLØŒ\Ð!Ý0Ñ2Ô2ˆDŒLÝ�KŠKÐ^Ñ_Ô_Ð_å�d”l¥DÑ)Ô)ð 	`Ý0Ð@Ð@°4´<Ð@Ð@ˆDŒLˆLØŒ\Ð!Ý0Ñ2Ô2ˆDŒLÝ�KŠKÐ^Ñ_Ô_Ð_à04Ô0AˆŒÔ Ô-Ø59Ô5KˆŒÔ Ô2Ø7;Ô7MˆŒÔ"Ô4Ø)-Ô)?ˆŒÔ&à$(Ô$5ˆŒÔ!Ø)-Ô)?ˆŒÔ&Ø#œ|Ô;ˆÔà!_ð 	Uð 	UÐØ ¨Ð.Ð.Ý,3°D´LÐBSÑ,TÔ,T�Ð(Ñ)øà�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'rN   c                 ój  — | j         j        j        | j        j        k    r/t	          d| j         j        j        › d| j        j        › d�¦  «        ‚| j        st	          d¦  «        ‚| j         j        j        | j        j        k    r/t	          d| j         j        j        › d| j        j        › d�¦  «        ‚dS )zOPart of `@strict`-powered validation. Validates the architecture of the config.zBImbalanced encoder-decoder is not supported in T5Gemma2: encoder (z) vs decoder (z).z4T5Gemma2Model only support encoder-decoder modeling.zRImbalanced encoder-decoder vocabulary size is not supported in T5Gemma2: encoder (N)rq   re   Úhidden_sizerr   Ú
ValueErrorrw   r‚   ©rW   s    rK   Úvalidate_architecturez$T5Gemma2Config.validate_architectureÔ   së   € àŒ<Ô#Ô/°4´<Ô3KÒKÐKÝðmØ œLÔ4Ô@ðmð mØPTÔP\ÔPhðmð mð mñô ð ð
 Ô&ð 	UÝÐSÑTÔTÐTàŒ<Ô#Ô.°$´,Ô2IÒIÐIÝðkØ œLÔ4Ô?ðkð kØOSÌ|ÔOfðkð kð kñô ð ð JÐIrN   )r[   r\   r]   r^   r_   Úkeys_to_ignore_at_inferencerc   ri   rg   Úattribute_maprq   r„   Ústrr   Ú__annotations__rr   rw   rF   ry   ÚfloatÚintrz   r{   r|   rs   rt   r}   rU   r�   Ú__classcell__©rŠ   s   @rK   rn   rn   �   sj  ø€ € € € € € ðð ð  €JØ#4Ð"5Ðð )Ø(ðð €Kð .Ø)ðð €Mð
 >B€GÐ" T¨#¨s¨(¤^Ñ3°dÑ:ÐAÐAÑAØ=A€GÐ" T¨#¨s¨(¤^Ñ3°dÑ:ÐAÐAÑAØ#Ð˜Ð#Ð#Ñ#Ø #€L�%˜#‘+Ð#Ð#Ñ#Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø+.Ð˜U S™[Ð.Ð.Ñ.Ø#Ð�uÐ#Ð#Ñ#Ø$Ð�sÐ$Ð$Ñ$Ø"&€O�S˜4‘ZÐ&Ð&Ñ&Ø $Ð˜Ð$Ð$Ñ$ð(ð (ð (ð (ð (ð8ð ð ð ð ð ð rN   rn   c                   ó   — e Zd ZdS )ÚT5Gemma2RMSNormN©r[   r\   r]   rO   rN   rK   r™   r™   æ   ó   € € € € € Ø€DrN   r™   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚT5Gemma2MLPÚconfigc                 ó†   •— t          ¦   «                              |¦  «         t          j        |j        ¦  «        | _        d S ©N)rˆ   Ú__init__ÚnnÚDropoutry   Údropout©rW   rž   rŠ   s     €rK   r¡   zT5Gemma2MLP.__init__ë   s3   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”z &Ô"5Ñ6Ô6ˆŒˆˆrN   c                 óÖ   — |                       |                      |¦  «        ¦  «        |                      |¦  «        z  }|                      |¦  «        }|                      |¦  «        }|S r    )Úact_fnÚ	gate_projÚup_projr¤   Ú	down_proj)rW   ÚxÚhidden_statesrª   s       rK   ÚforwardzT5Gemma2MLP.forwardï   sV   € ØŸš D§N¢N°1Ñ$5Ô$5Ñ6Ô6¸¿ºÀa¹¼ÑHˆØŸš ]Ñ3Ô3ˆØ—N’N =Ñ1Ô1ˆ	ØÐrN   )r[   r\   r]   r:   r¡   r­   r–   r—   s   @rK   r�   r�   ê   sT   ø€ € € € € ð7Ð1ð 7ð 7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð rN   r�   c                   óŽ   ‡ — e Zd Zddefˆ fd„Ze	 	 	 	 ddedz  ded         dedz  dedz  de	d	e
f         f
ˆ fd
„¦   «         Zˆ xZS )ÚT5Gemma2RotaryEmbeddingNrž   c                 óL   •— t          ¦   «                              ||¦  «         d S r    ©rˆ   r¡   )rW   rž   ÚdevicerŠ   s      €rK   r¡   z T5Gemma2RotaryEmbedding.__init__÷   s#   ø€ Ý‰Œ×Ò˜ Ñ(Ô(Ð(Ð(Ð(rN   r²   ztorch.deviceÚseq_lenÚ
layer_typeÚreturnztorch.Tensorc                 óL   •— t          ¦   «                              | |||¦  «        S r    )rˆ   Úcompute_default_rope_parameters)rž   r²   r³   r´   rŠ   s       €rK   r·   z7T5Gemma2RotaryEmbedding.compute_default_rope_parametersú   s$   ø€ õ ‰wŒw×6Ò6°v¸vÀwÐPZÑ[Ô[Ð[rN   r    )NNNN)r[   r\   r]   r:   r¡   Ústaticmethodr   r•   r’   Útupler”   r·   r–   r—   s   @rK   r¯   r¯   ö   sß   ø€ € € € € ð)ð )Ð1ð )ð )ð )ð )ð )ð )ð à,0Ø+/Ø"Ø!%ð	\ð \Ø" TÑ)ð\à˜Ô(ð\ð �t‘ð\ð ˜$‘Jð	\ð
 
ˆ~˜uÐ$Ô	%ð\ð \ð \ð \ð \ñ „\ð\ð \ð \ð \ð \rN   r¯   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚT5Gemma2SelfAttentionrž   Ú	layer_idxc                 óZ   •— t          ¦   «                              ||¦  «         d| _        d S ©NF©rˆ   r¡   Ú	is_causal©rW   rž   r¼   rŠ   s      €rK   r¡   zT5Gemma2SelfAttention.__init__  ó(   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+ØˆŒˆˆrN   )r[   r\   r]   r:   r•   r¡   r–   r—   s   @rK   r»   r»     sL   ø€ € € € € ðÐ1ð ¸cð ð ð ð ð ð ð ð ð ð rN   r»   c                   óö   ‡ — e Zd ZdZdedefˆ fd„Z	 ddej        de	ej        ej        f         dej        dz  d	ej        d
e
dz  dee         de	ej        ej        dz  e	ej                 dz  f         fd„Zˆ xZS )ÚT5Gemma2MergedAttentionz6Merged self-attention and cross-attention for decoder.rž   r¼   c                 óZ   •— t          ¦   «                              ||¦  «         d| _        d S r¾   r¿   rÁ   s      €rK   r¡   z T5Gemma2MergedAttention.__init__  rÂ   rN   Nr¬   Úposition_embeddingsÚmerged_attention_maskÚencoder_hidden_statesrp   rX   rµ   c                 ó¤  — |j         d d…         }g |¢d‘| j        ‘R }|j         d d…         }	g |	¢d‘| j        ‘R }
|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «        }|                      |¦  «        }|\  }}t          ||||¦  «        \  }}|�L|j
        }|                     ||| j        ¦  «        \  }}|j                             | j        ¦  «        }|j        }|�|s¾|                      |¦  «                             |
¦  «                             dd¦  «        }|                      |¦  «                             |
¦  «                             dd¦  «        }|                      |¦  «        }|�.|                     ||| j        ¦  «        \  }}d|j        | j        <   n.|j        | j                 j        }|j        | j                 j        }|}|	d         }t'          j        ||gd¬¦  «        }t'          j        ||gd¬¦  «        }t+          j        | j        j        t2          ¦  «        } || ||||f| j        r| j        nd| j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }|�|dd | …f         }|d| d …f         }nd	\  }}|||fS )
NéÿÿÿÿrB   r%   T©Údimrx   )r¤   Úscaling.)NN) ÚshapeÚhead_dimÚq_projÚviewÚ	transposeÚk_projÚv_projÚq_normÚk_normr0   Úself_attention_cacheÚupdater¼   Ú
is_updatedÚgetÚcross_attention_cacheÚlayersÚkeysÚvaluesÚtorchÚcatr   Úget_interfacerž   Ú_attn_implementationr3   Útrainingrz   rÍ   ÚreshapeÚ
contiguousÚo_proj)rW   r¬   rÆ   rÇ   rÈ   rp   rX   Úinput_shapeÚhidden_shapeÚcross_input_shapeÚcross_hidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinr×   rÙ   rÛ   Úcross_key_statesÚcross_value_statesÚcross_key_sizeÚattention_interfaceÚattn_outputÚattn_weightsÚself_attn_weightsÚcross_attn_weightss                              rK   r­   zT5Gemma2MergedAttention.forward  s¹  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆØ1Ô7¸¸¸Ô<ÐØDÐ0ÐD°"ÐD°d´mÐDÐDÐð —{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà—{’{ <Ñ0Ô0ˆØ—[’[ Ñ,Ô,ˆ
à&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&à#2Ô#GÐ Ø';×'BÒ'BÀ:È|Ð]aÔ]kÑ'lÔ'lÑ$ˆJ˜ð )Ô3×7Ò7¸¼ÑGÔGˆJØ$3Ô$IÐ!àÐ"¨*Ð"Ø#Ÿ{š{Ð+@ÑAÔA×FÒFÐGYÑZÔZ×dÒdÐefÐhiÑjÔjÐØ!%§¢Ð-BÑ!CÔ!C×!HÒ!HÐI[Ñ!\Ô!\×!fÒ!fÐghÐjkÑ!lÔ!lÐà#Ÿ{š{Ð+;Ñ<Ô<ÐàÐ*Ø7L×7SÒ7SØ$Ð&8¸$¼.ñ8ô 8Ñ4Ð Ð"4ð >B�Ô*¨4¬>Ñ:øà4Ô;¸D¼NÔKÔPÐØ!6Ô!=¸d¼nÔ!MÔ!TÐð $ˆØ*¨1Ô-ˆÝ”Y 
Ð,<Ð=À1ÐEÑEÔEˆ
Ý”y ,Ð0BÐ!CÈÐKÑKÔKˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØ!ð	%
ð /3¬mÐD�DÔ*Ð*ÀØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆð Ð#Ø ,¨SÐ2B°N°?Ð2BÐ-BÔ CÐØ!-¨c°N°?Ð3CÐ3CÐ.CÔ!DÐÐà4>Ñ1ÐÐ1ØÐ-Ð/AÐAÐArN   r    )r[   r\   r]   r^   r:   r•   r¡   rß   ÚTensorr¹   r
   r   r   r­   r–   r—   s   @rK   rÄ   rÄ   
  s  ø€ € € € € Ø@Ð@ðÐ1ð ¸cð ð ð ð ð ð ð 7;ðTBð TBð ”|ðTBð # 5¤<°´Ð#=Ô>ð	TBð
  %œ|¨dÑ2ðTBð  %œ|ðTBð -¨tÑ3ðTBð Ð-Ô.ðTBð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	MðTBð TBð TBð TBð TBð TBð TBð TBrN   rÄ   TÚsliding_windowrµ   c           
      ó^   ‡ ‡— dt           dt           dt           dt           dt          f
ˆˆ fd„}|S )zL
    This creates uni/bidirectional attention mask with sliding window.
    Ú	batch_idxÚhead_idxÚq_idxÚkv_idxrµ   c                 ó|   •— ‰	r‰
d}}n‰
dz   dz  ‰
dz  dz   }}||z
  }|dk    ||k     z  }|dk     | |k     z  }||z  S )Nr   rB   r%   rO   )rû   rü   rý   rþ   Úleft_window_sizeÚright_window_sizeÚdistÚ	left_maskÚ
right_maskrÀ   rù   s            €€rK   Ú
inner_maskz0sliding_window_mask_function.<locals>.inner_maskm  sy   ø€ Øð 	iØ2@À!Ð/ÐÐà4BÀQÑ4FÈ1Ñ3LÈ~ÐbcÑNcÐfgÑNgÐ/Ðà�v‰~ˆØ˜Q’Y 4Ð*:Ò#:Ñ;ˆ	Ø˜Q’h D 5Ð+<Ò#<Ñ=ˆ
Ø˜:Ñ%Ð%rN   )r•   rF   )rù   rÀ   r  s   `` rK   Úsliding_window_mask_functionr  h  sR   øø€ ð
	&�cð 	&­Sð 	&½ð 	&Åcð 	&Ídð 	&ð 	&ð 	&ð 	&ð 	&ð 	&ð 	&ð ÐrN   c                   ó   — e Zd ZdS )ÚT5Gemma2EncoderLayerNrš   rO   rN   rK   r  r  {  r›   rN   r  c                   óÐ   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 	 	 ddej        deej        ej        f         dej        dz  d	ej	        dz  d
e
dz  dedz  dej        dz  dej        fd„Zˆ xZS )ÚT5Gemma2DecoderLayerzFDecoder sub-layer: merged attention instead of vanilla self-attention.r¼   c                 óx   •— t          ¦   «                              ||¦  «         t          ||¬¦  «        | _        d S )N)rž   r¼   )rˆ   r¡   rÄ   Ú	self_attnrÁ   s      €rK   r¡   zT5Gemma2DecoderLayer.__init__‚  s?   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+õ 1ØØð
ñ 
ô 
ˆŒˆˆrN   NFr¬   rÆ   rÇ   Úposition_idsrp   Ú	use_cacherÈ   rµ   c                 ór  — |}	|                       |¦  «        } | j        d|||||||dœ|¤Ž\  }}
}
|                      |¦  «        }|	|                      |¦  «        z   }|}	|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|	|                      |¦  «        z   }|S )N)r¬   rÆ   rÇ   r  rp   r  rÈ   rO   )Úpre_self_attn_layernormr  Úpost_self_attn_layernormr¤   Úpre_feedforward_layernormÚmlpÚpost_feedforward_layernorm)rW   r¬   rÆ   rÇ   r  rp   r  rÈ   rX   ÚresidualÚ_s              rK   r­   zT5Gemma2DecoderLayer.forward‹  sâ   € ð !ˆØ×4Ò4°]ÑCÔCˆà,˜dœnð 	
Ø'Ø 3Ø"7Ø%Ø+ØØ"7ð	
ð 	
ð ð	
ð 	
Ñˆ�q˜!ð ×5Ò5°mÑDÔDˆØ  4§<¢<°Ñ#>Ô#>Ñ>ˆà ˆØ×6Ò6°}ÑEÔEˆØŸš Ñ/Ô/ˆØ×7Ò7¸ÑFÔFˆØ  4§<¢<°Ñ#>Ô#>Ñ>ˆØÐrN   )NNNFN)r[   r\   r]   r^   r•   r¡   rß   rø   r¹   Ú
LongTensorr
   rF   ÚFloatTensorr­   r–   r—   s   @rK   r
  r
    sô   ø€ € € € € ØPÐPð
¨#ð 
ð 
ð 
ð 
ð 
ð 
ð 6:Ø04Ø6:Ø!&Ø59ð ð  à”|ð ð # 5¤<°´Ð#=Ô>ð ð  %œ|¨dÑ2ð	 ð
 Ô&¨Ñ-ð ð -¨tÑ3ð ð ˜$‘;ð ð  %œ|¨dÑ2ð ð 
Ô	ð ð  ð  ð  ð  ð  ð  ð  rN   r
  c                   ó   — e Zd ZdS )ÚT5Gemma2LMHeadNrš   rO   rN   rK   r  r  ®  r›   rN   r  c                   ó   — e Zd ZdS )ÚT5Gemma2ClassificationHeadNrš   rO   rN   rK   r  r  ²  r›   rN   r  c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚT5Gemma2MultiModalProjectorrž   c                 óJ   •— t          ¦   «                              |¦  «         d S r    r±   r¥   s     €rK   r¡   z$T5Gemma2MultiModalProjector.__init__·  s!   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ð Ð rN   )r[   r\   r]   rc   r¡   r–   r—   s   @rK   r  r  ¶  sE   ø€ € € € € ð!Ð4ð !ð !ð !ð !ð !ð !ð !ð !ð !ð !rN   r  c                   óX   ‡ — e Zd ZdZ	 	 ddededededef
ˆ fd	„Zd
ej        fˆ fd„Z	ˆ xZ
S )ÚT5Gemma2TextScaledWordEmbeddingzCT5Gemma2 Embedding: override to add eoi token embedding separately.ç      ð?é è Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scalert   c                 ó¾   •— t          ¦   «                              ||||¦  «         || _        t          j        t          j        | j        ¦  «        ¦  «        | _        d S r    )	rˆ   r¡   rt   r¢   Ú	Parameterrß   Úzerosr%  Úeoi_embedding)rW   r$  r%  r&  r'  rt   rŠ   s         €rK   r¡   z(T5Gemma2TextScaledWordEmbedding.__init__¾  sP   ø€ õ 	‰Œ×Ò˜¨¸À[ÑQÔQÐQØ.ˆÔÝœ\­%¬+°dÔ6HÑ*IÔ*IÑJÔJˆÔÐÐrN   Ú	input_idsc                 óê   •— t          ¦   «                              |¦  «        | j                             | j        j        ¦  «        z  }| j                             |j        ¦  «        ||| j        k    <   |S r    )rˆ   r­   r'  ÚtoÚweightÚdtyper+  rt   )rW   r,  Úinput_embeddingsrŠ   s      €rK   r­   z'T5Gemma2TextScaledWordEmbedding.forwardÊ  sb   ø€ Ý ™7œ7Ÿ?š?¨9Ñ5Ô5¸Ô8H×8KÒ8KÈDÌKÔL]Ñ8^Ô8^Ñ^ÐØ>BÔ>P×>SÒ>SÐTdÔTjÑ>kÔ>kÐ˜ dÔ&:Ò:Ñ;ØÐrN   )r"  r#  )r[   r\   r]   r^   r•   r”   r¡   rß   rø   r­   r–   r—   s   @rK   r!  r!  »  s³   ø€ € € € € ØMÐMð !Ø&ð
Kð 
Kàð
Kð ð
Kð ð	
Kð
 ð
Kð ð
Kð 
Kð 
Kð 
Kð 
Kð 
Kð  ¤ð  ð  ð  ð  ð  ð  ð  ð  ð  ð  rN   r!  c                   óR   — e Zd ZU eed<   dZdZdZdZg d¢Z	dZ
d„ Zdej        fd	„ZdS )
ÚT5Gemma2PreTrainedModelrž   ÚmodelTF)r  r
  ÚSiglipVisionEmbeddingsÚSiglipEncoderLayerÚ#SiglipMultiheadAttentionPoolingHeadNc                 ó(  — t          j        | |¦  «         t          |t          ¦  «        rt	          j        |j        ¦  «         d S t          |t          ¦  «        r:t	          j        |j        ¦  «         t	          j	        |j
        |j        ¦  «         d S t          |t          ¦  «        r�|j        j        j        d         dz  }t	          j        |j        j        d| j        j        |z  ¬¦  «         t'          |j        d¦  «        r,|j        j        �"t	          j        |j        j        ¦  «         d S d S d S d|j        j        v rt	          j        |j        ¦  «         d S t          |t.          ¦  «        r›|j        D ]•}|j        }|j        |         dk    rt6          |j        |                  } ||j        |¬¦  «        \  }}t	          j        t;          ||› d	�¦  «        |¦  «         t	          j        t;          ||› d
�¦  «        |¦  «         Œ”d S d S )Nr   g      à¿rx   )ÚmeanÚstdÚbiasÚRMSNormÚdefault)r´   Ú	_inv_freqÚ_original_inv_freq)r   Ú_init_weightsrƒ   r  ÚinitÚzeros_Úmm_input_projection_weightr!  r+  Ú	constant_r'  Úscalar_embed_scaler  Úout_projr/  rÎ   Únormal_rž   r|   Úhasattrr;  rŠ   r[   r¯   rR   r·   Ú	rope_typer   Úcopy_r‡   )rW   ÚmoduleÚscaler´   Úrope_init_fnÚcurr_inv_freqr  s          rK   r@  z%T5Gemma2PreTrainedModel._init_weightsæ  s"  € ÝÔ% d¨FÑ3Ô3Ð3Ý�fÕ9Ñ:Ô:ð 	^ÝŒK˜Ô9Ñ:Ô:Ð:Ð:Ð:Ý˜Õ ?Ñ@Ô@ð 	^ÝŒK˜Ô,Ñ-Ô-Ð-ÝŒN˜6Ô-¨vÔ/HÑIÔIÐIÐIÐIÝ˜Õ :Ñ;Ô;ð 	^Ø”OÔ*Ô0°Ô3°tÑ;ˆEÝŒL˜œÔ/°c¸t¼{Ô?\Ð_dÑ?dÐeÑeÔeÐeÝ�v”¨Ñ/Ô/ð 2°F´OÔ4HÐ4TÝ”˜FœOÔ0Ñ1Ô1Ð1Ð1Ð1ð2ð 2Ð4TÐ4Tð ˜&Ô*Ô3Ð3Ð3ÝŒK˜œÑ&Ô&Ð&Ð&Ð&Ý˜Õ 7Ñ8Ô8ð 	^Ø$Ô0ð ^ð ^�
Ø%ÔE�ØÔ# JÔ/°9Ò<Ð<Ý#6°vÔ7GÈ
Ô7SÔ#T�LØ#/ <°´È*Ð#UÑ#UÔ#UÑ �˜qÝ”
�7 6¨jÐ+CÐ+CÐ+CÑDÔDÀmÑTÔTÐTÝ”
�7 6¨jÐ+LÐ+LÐ+LÑMÔMÈ}Ñ]Ô]Ð]Ð]ð	^ð 	^ð^ð ^rN   Úlabelsc                 ó:  — | j         j        }|j        }|j        }|€t	          d¦  «        ‚|                     |j        ¦  «        }|ddd…f                              ¦   «         |ddd…f<   ||d<   |€t	          d¦  «        ‚|                     |dk    |¦  «         |S )	zú
        Shifts input_ids to the right, prepends the decoder_start_token_id, and handles
        pad_token_id replacement for labels that were -100.
        This is a common preparation step for decoder inputs in sequence-to-sequence models.
        Nz:self.model.config.decoder.bos_token_id has to be defined. .rÊ   rB   ).r   z9self.model.config.decoder.pad_token_id has to be defined.iœÿÿÿ)	rž   rr   r   r€   r�   Ú	new_zerosrÎ   ÚcloneÚmasked_fill_)rW   rO  Údecoder_configÚdecoder_start_token_idr€   Úshifted_input_idss         rK   Ú%prepare_decoder_input_ids_from_labelsz=T5Gemma2PreTrainedModel.prepare_decoder_input_ids_from_labelsþ  s¼   € ð œÔ,ˆØ!/Ô!<ÐØ%Ô2ˆà!Ð)ÝÐYÑZÔZÐZð #×,Ò,¨V¬\Ñ:Ô:ÐØ%+¨C°°"°¨HÔ%5×%;Ò%;Ñ%=Ô%=Ð˜#˜q˜r˜r˜'Ñ"Ø$:Ð˜&Ñ!àÐÝÐXÑYÔYÐYð 	×&Ò&Ð'8¸DÒ'@À,ÑOÔOÐOà Ð rN   )r[   r\   r]   rn   r“   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_flex_attnÚ_no_split_modulesÚ_can_record_outputsr@  rß   rø   rW  rO   rN   rK   r3  r3  Ð  s…   € € € € € € àÐÐÑØÐØ&*Ð#ð !ÐàÐðð ð Ðð Ðð^ð ^ð ^ð0!¸E¼Lð !ð !ð !ð !ð !ð !rN   r3  c                   óþ   ‡ — e Zd ZU eed<   eedœZ	 ddedefˆ fd„Z	e
ee	 	 	 	 	 ddej        dz  dej        dz  d	ej        dz  d
ej        dz  dej        dz  dee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚT5Gemma2TextEncoderrž   )Ú
attentionsr¬   r#  rt   c                 ó$  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          ‰j        ‰j        | j        ‰j        dz  |¬¦  «        | _        t          ‰j        ‰j	        ¬¦  «        | _
        d| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j        ¦  «        | _        t)          ‰¦  «        | _        |                      ¦   «          d S )Nç      à?©r'  rt   ©ÚepsFc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS rO   )r  ©rH   r¼   rž   s     €rK   rL   z0T5Gemma2TextEncoder.__init__.<locals>.<listcomp>5  ó$   ø€ ÐfÐfÐf¸Õ! &¨)Ñ4Ô4ÐfÐfÐfrN   ©rˆ   r¡   r€   r&  r‚   r!  rŒ   Úembed_tokensr™   Úrms_norm_epsÚnormÚgradient_checkpointingr¢   Ú
ModuleListrS   rT   rÜ   r£   ry   r¤   r¯   Ú
rotary_embÚ	post_init©rW   rž   rt   rŠ   s    ` €rK   r¡   zT5Gemma2TextEncoder.__init__!  s   øø€ õ
 	‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒå;ØÔØÔØÔØÔ*¨CÑ/Ø+ð
ñ 
ô 
ˆÔõ $ FÔ$6¸FÔ<OÐPÑPÔPˆŒ	Ø&+ˆÔ#å”mØfÐfÐfÐfÅeÈFÔLdÑFeÔFeÐfÑfÔfñ
ô 
ˆŒõ ”z &Ô"5Ñ6Ô6ˆŒÝ1°&Ñ9Ô9ˆŒð 	�ŠÑÔÐÐÐrN   Nr,  Úattention_maskr  Úinputs_embedsÚtoken_type_idsrX   rµ   c           
      óŽ  — |d u |d uz  rt          d¦  «        ‚|                     dd ¦  «         |€|                      |¦  «        }|€:t          j        d|j        d         |j        ¬¦  «                             d¦  «        }t          |x}t          ¦  «        sA| j
        ||dœ}t          di |¤Žt          di |¤dt          | j
        j        d¬	¦  «        i¤Žd
œ}|}	i }
t          | j
        j        ¦  «        D ]}|                      |	||¦  «        |
|<   Œ|                      |	¦  «        }	t%          | j        d | j
        j        …         ¦  «        D ]=\  }} ||	|
| j
        j        |                  || j
        j        |                  |fi |¤Ž}	Œ>|                      |	¦  «        }	|                      |	¦  «        }	t-          |	¬¦  «        S )Nú:You must specify exactly one of input_ids or inputs_embedsrp   r   rB   ©r²   )rž   rs  rr  Úand_mask_functionF)rÀ   ©rD   rC   )Úlast_hidden_staterO   )r�   rQ   rj  rß   ÚarangerÎ   r²   Ú	unsqueezerƒ   r„   rž   r   r  rù   ÚsetrR   ro  r¤   Ú	enumeraterÜ   rT   rl  r   )rW   r,  rr  r  rs  rt  rX   Úself_attn_mask_mappingÚmask_kwargsr¬   rÆ   r´   rI   Úlayer_modules                 rK   r­   zT5Gemma2TextEncoder.forward=  s5  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZð 	�
Š
Ð$ dÑ+Ô+Ð+àÐ Ø ×-Ò-¨iÑ8Ô8ˆMàÐÝ œ<¨¨=Ô+>¸qÔ+AÈ-ÔJ^Ð_Ñ_Ô_×iÒiÐjkÑlÔlˆLå°NÐBÐ0ÅDÑIÔIð 	àœ+Ø!.Ø"0ðð ˆKõ #<Ð"JÐ"J¸kÐ"JÐ"JÝ%>ð &ð &Ø!ð&ð &å&BÀ4Ä;ÔC]ÐinÐ&oÑ&oÔ&oð&ð &ð &ð&ð &Ð"ð &ˆð !ÐÝ˜dœkÔ5Ñ6Ô6ð 	gð 	gˆJØ.2¯oªo¸mÈ\Ð[eÑ.fÔ.fÐ 
Ñ+Ð+ð Ÿš ]Ñ3Ô3ˆå(¨¬Ð5T°t´{Ô7TÐ5TÔ)UÑVÔVð 	ð 	‰OˆAˆ|Ø(˜LØØ# D¤KÔ$;¸AÔ$>Ô?Ø& t¤{Ô'>¸qÔ'AÔBØð	ð ð
 ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆØŸš ]Ñ3Ô3ˆÝØ+ð
ñ 
ô 
ð 	
rN   ©r#  )NNNNN)r[   r\   r]   r:   r“   r»   r  r]  r•   r¡   r"   r$   r   rß   r  rø   r  r   r   r   r­   r–   r—   s   @rK   r_  r_    s5  ø€ € € € € € ØÐÐÑà+Ø-ðð Ðð  'ðð à"ðð ðð ð ð ð ð ð8  ØØð .2Ø.2Ø04Ø26à.2ð<
ð <
àÔ# dÑ*ð<
ð œ tÑ+ð<
ð Ô&¨Ñ-ð	<
ð
 Ô(¨4Ñ/ð<
ð œ tÑ+ð<
ð Ð+Ô,ð<
ð 
ð<
ð <
ð <
ñ „^ñ „_ñ  Ôð<
ð <
ð <
ð <
ð <
rN   r_  c                   ó„  ‡ — e Zd ZU eed<   	 ddedefˆ fd„Zd„ Zd„ Ze	e
dej        dee         d	eez  fd
„¦   «         ¦   «         Zdej        dz  dej        dz  dej        fd„Ze
	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dee         d	efd„¦   «         Zˆ xZS )ÚT5Gemma2Encoderrž   r#  rt   c                 ó$  •— t          ¦   «                              |¦  «         t                               |j        |¬¦  «        | _        t          j        |j        ¬¦  «        | _	        t          |¦  «        | _        |                      ¦   «          d S )N)rt   ©rž   )rˆ   r¡   r_  Ú_from_configre   Ú
text_modelr&   Úfrom_configrf   Úvision_towerr  Úmulti_modal_projectorrp  rq  s      €rK   r¡   zT5Gemma2Encoder.__init__‚  s|   ø€ õ
 	‰Œ×Ò˜Ñ Ô Ð å-×:Ò:¸6Ô;MÐ_nÐ:ÑoÔoˆŒÝ%Ô1¸Ô9MÐNÑNÔNˆÔÝ%@ÀÑ%HÔ%HˆÔ"ð 	�ŠÑÔÐÐÐrN   c                 ó4   — | j                              ¦   «         S r    )rˆ  Úget_input_embeddingsrŽ   s    rK   r�  z$T5Gemma2Encoder.get_input_embeddings�  s   € ØŒ×3Ò3Ñ5Ô5Ð5rN   c                 ó6   — | j                              |¦  «        S r    )rˆ  Úset_input_embeddings©rW   Únew_embeddingss     rK   r�  z$T5Gemma2Encoder.set_input_embeddings“  s   € ØŒ×3Ò3°NÑCÔCÐCrN   Úpixel_valuesrX   rµ   c                 ól   —  | j         d|ddœ|¤Ž}|j        }|                      |¦  «        }||_        |S )NT)r’  Úreturn_dictrO   )rŠ  rz  r‹  Úpooler_output)rW   r’  rX   Úvision_outputsrz  Úimage_featuress         rK   Úget_image_featuresz"T5Gemma2Encoder.get_image_features–  sP   € ð +˜Ô*Ða¸ÐRVÐaÐaÐZ`ÐaÐaˆØ*Ô<ÐØ×3Ò3Ð4EÑFÔFˆØ'5ˆÔ$àÐrN   r,  Nrs  r—  c                 ó2  — | j         j        }|€l|€t          d¦  «        ‚| |                      ¦   «         t	          j        |t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }n||k    }| 	                    ¦   «         }| 
                    d¦  «                             |j        ¦  «        }|j        d         |j        d         z  }t          ||j        d         z  |                     ¦   «         k    d|› d|› �¦  «         |S )	zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        Nz9Either `input_ids` or `inputs_embeds` has to be provided.)r0  r²   rÊ   r   rB   z6Image features and image tokens do not match: tokens: z, features )rž   ru   r�   r�  rß   ÚtensorÚlongr²   ÚallÚsumr|  r.  rÎ   r!   Únumel)rW   r,  rs  r—  ru   Úspecial_image_maskÚn_image_tokensÚn_image_featuress           rK   Úget_image_placeholder_maskz*T5Gemma2Encoder.get_image_placeholder_mask¤  s,  € ð œÔ3ˆØÐØÐ$Ý Ð!\Ñ]Ô]Ð]Ø!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”˜^µ5´:ÀmÔFZÐ[Ñ[Ô[ñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;ÐÐà!*¨nÒ!<Ðà+×/Ò/Ñ1Ô1ˆØ/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐØ)Ô/°Ô2°^Ô5IÈ!Ô5LÑLÐÝØ˜]Ô0°Ô4Ñ4¸×8LÒ8LÑ8NÔ8NÒNØrÀ^ÐrÐrÐ`pÐrÐrñ	
ô 	
ð 	
ð "Ð!rN   rr  r  rt  c                 óh  — |d u |d uz  rt          d¦  «        ‚|€| j                             |¦  «        }|�j|                      |d¬¦  «        j        }|                     |j        |j        ¦  «        }|                      |||¬¦  «        }	| 	                    |	|¦  «        } | j        d|||dœ|¤Ž}
|
S )Nrv  T)r”  )rs  r—  )rs  rr  r  rO   )
r�   rˆ  rj  r˜  r•  r.  r²   r0  r¢  Úmasked_scatter)rW   r,  rr  r  rs  r’  rt  rX   r—  Ú
image_maskÚoutputss              rK   r­   zT5Gemma2Encoder.forwardÂ  sñ   € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø œO×8Ò8¸ÑCÔCˆMàÐ#Ø!×4Ò4°\ÈtÐ4ÑTÔTÔbˆNØ+×.Ò.¨}Ô/CÀ]ÔEXÑYÔYˆNà×8Ò8Ø¨À~ð 9ñ ô ˆJð *×8Ò8¸À^ÑTÔTˆMà!�$”/ð 
Ø'Ø)Ø%ð
ð 
ð ð	
ð 
ˆð ˆrN   r‚  )NNNNNN)r[   r\   r]   rc   r“   r•   r¡   r�  r�  r   r   rß   rø   r   r   r¹   r   r˜  r  r  r¢  r   r­   r–   r—   s   @rK   r„  r„    s×  ø€ € € € € € Ø!Ð!Ð!Ñ!ð
  'ðð à%ðð ðð ð ð ð ð ð6ð 6ð 6ðDð Dð Dð Øð
Ø!œLð
Ø4:Ð;MÔ4Nð
à	Ð+Ñ	+ð
ð 
ð 
ñ „^ñ Ôð
ð"àÔ# dÑ*ð"ð Ô(¨4Ñ/ð"ð Ô)ð	"ð "ð "ð "ð< ð .2Ø.2Ø04Ø26Ø15à.2ð!ð !àÔ# dÑ*ð!ð œ tÑ+ð!ð Ô&¨Ñ-ð	!ð
 Ô(¨4Ñ/ð!ð Ô'¨$Ñ.ð!ð œ tÑ+ð!ð Ð+Ô,ð!ð 
ð!ð !ð !ñ „^ð!ð !ð !ð !ð !rN   r„  c                   óX  ‡ — e Zd ZU eed<    eed¬¦  «         eed¬¦  «        edœZddede	fˆ fd„Z
eee	 	 	 	 	 	 	 	 dd
ej        d	z  dej        d	z  dej        d	z  ded	z  dej        d	z  ded	z  dej        d	z  dej        d	z  dee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚT5Gemma2Decoderrž   rB   )Úindexr%   )r`  Úcross_attentionsr¬   r#  rt   c                 ó$  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          ‰j        ‰j        ‰j        ‰j        dz  |¬¦  «        | _        t          ‰j        ‰j	        ¬¦  «        | _
        d| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j        ¦  «        | _        t)          ‰¦  «        | _        |                      ¦   «          d S )Nrb  rc  rd  Fc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS rO   )r
  rg  s     €rK   rL   z,T5Gemma2Decoder.__init__.<locals>.<listcomp>ÿ  rh  rN   ri  rq  s    ` €rK   r¡   zT5Gemma2Decoder.__init__ï  sü   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒå;ØÔØÔØÔØÔ*¨CÑ/Ø+ð
ñ 
ô 
ˆÔõ $ FÔ$6¸FÔ<OÐPÑPÔPˆŒ	Ø&+ˆÔ#å”mØfÐfÐfÐfÅeÈFÔLdÑFeÔFeÐfÑfÔfñ
ô 
ˆŒõ ”z &Ô"5Ñ6Ô6ˆŒÝ1°&Ñ9Ô9ˆŒØ�ŠÑÔÐÐÐrN   Nr,  rr  r  rp   rs  r  rÈ   Úencoder_attention_maskrX   rµ   c	           
      ó  — |d u |d uz  rt          d¦  «        ‚|€t          d¦  «        ‚|€|                      |¦  «        }| j        s3|r1|€/t          t	          | j        ¬¦  «        t	          ¦   «         ¦  «        }|€V|�|                     ¦   «         nd}
t          j        |j	        d         |j
        ¬¦  «        |
z   }|                     d¦  «        }t          |x}t          ¦  «        s3d„ }| j        |||�|j        nd ||dœ}t          di |¤Žt!          di |¤Žd	œ}t          |x}t          ¦  «        sd
t#          | j        ||||¬¦  «        i}t          j        |d
         |d
         gd¬¦  «        t          j        |d         |d
         gd¬¦  «        d	œ}|}i }t'          | j        j        ¦  «        D ]}|                      |||¦  «        ||<   Œ|                      |¦  «        }t/          | j        d | j        j        …         ¦  «        D ]@\  }} |||| j        j        |                  || j        j        |                  ||||fi |	¤Ž}ŒA|                      |¦  «        }|                      |¦  «        }t7          ||¬¦  «        S )Nrv  z0`encoder_hidden_states` must be given in decoderr†  r   rB   rw  c                  óB   — t          j        dt           j        ¬¦  «        S )NT)r0  )rß   rš  rF   )Úargss    rK   ú<lambda>z)T5Gemma2Decoder.forward.<locals>.<lambda>'  s   € µE´LÀÍUÌZÐ4XÑ4XÔ4X€ rN   )rž   rs  rr  rp   r  rx  ry  rD   )rž   rs  rr  rÈ   rx  rÊ   rË   rC   )rz  rp   rO   )r�   rj  rã   r
   r	   rž   Úget_seq_lengthrß   r{  rÎ   r²   r|  rƒ   r„   r×   r1   r2   r   rà   r}  rR   ro  r¤   r~  rÜ   rT   rl  r   )rW   r,  rr  r  rp   rs  r  rÈ   r­  rX   Úpast_seen_tokensr  Údummy_and_mask_functionr€  Úcross_attn_mask_mappingÚmerged_attn_mask_mappingr¬   rÆ   r´   rI   r�  s                        rK   r­   zT5Gemma2Decoder.forward  sF  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZØ Ð(ÝÐOÑPÔPÐPàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàŒ}ð 	d ð 	d¨Ð/FÝ1µ,ÀdÄkÐ2RÑ2RÔ2RÕT`ÑTbÔTbÑcÔcˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå°NÐBÐ0ÅDÑIÔIð 	ð 'YÐ&XÐ#àœ+Ø!.Ø"0ØKZÐKf ?Ô#GÐ#GÐlpØ ,Ø%<ðð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ%FÐ%UÐ%UÈÐ%UÐ%Uð&ð &Ð"õ
 Ð5KÐKÐ1ÍTÑRÔRð 		à Õ";Øœ;Ø"/Ø#9Ø*?Ø&=ð#ñ #ô #ð'Ð#õ $œiØ'Ð(8Ô9Ð;RÐScÔ;dÐeÐkmðñ ô õ "'¤Ø'Ð(;Ô<Ð>UÐVfÔ>gÐhÐnpð"ñ "ô "ð	$
ð $
Ð ð &ˆð !ÐÝ˜dœkÔ5Ñ6Ô6ð 	gð 	gˆJØ.2¯oªo¸mÈ\Ð[eÑ.fÔ.fÐ 
Ñ+Ð+ð Ÿš ]Ñ3Ô3ˆå(¨¬Ð5T°t´{Ô7TÐ5TÔ)UÑVÔVð 
	ð 
	‰OˆAˆ|Ø(˜LØØ# D¤KÔ$;¸AÔ$>Ô?Ø(¨¬Ô)@ÀÔ)CÔDØØØØ%ð	ð 	ð ð	ð 	ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆØŸš ]Ñ3Ô3ˆÝ8Ø+Ø+ð
ñ 
ô 
ð 	
rN   r‚  )NNNNNNNN)r[   r\   r]   ri   r“   r#   rÄ   r
  r]  r•   r¡   r"   r$   r   rß   r  rø   r
   r  rF   r   r   r   r­   r–   r—   s   @rK   r¨  r¨  ç  sš  ø€ € € € € € Ø!Ð!Ð!Ñ!à$�nÐ%<ÀAÐFÑFÔFØ*˜NÐ+BÈ!ÐLÑLÔLØ-ðð Ððð Ð4ð Àsð ð ð ð ð ð ð,  ØØð .2Ø.2Ø04Ø6:Ø26Ø!%Ø59Ø6:ð]
ð ]
àÔ# dÑ*ð]
ð œ tÑ+ð]
ð Ô&¨Ñ-ð	]
ð
 -¨tÑ3ð]
ð Ô(¨4Ñ/ð]
ð ˜$‘;ð]
ð  %œ|¨dÑ2ð]
ð !&¤¨tÑ 3ð]
ð Ð+Ô,ð]
ð 
3ð]
ð ]
ð ]
ñ „^ñ „_ñ  Ôð]
ð ]
ð ]
ð ]
ð ]
rN   r¨  c                   ó|  ‡ — e Zd ZdddœZdefˆ fd„Zddedz  fd„Zd	„ Zd
„ Z	d„ Z
ee	 	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dedz  dej        dz  dej        dz  dedz  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚT5Gemma2Modelz&encoder.text_model.embed_tokens.weightz-encoder.text_model.embed_tokens.eoi_embedding)zdecoder.embed_tokens.weightz"decoder.embed_tokens.eoi_embeddingrž   c                 óî   •— t          ¦   «                              |¦  «         t          |j        |j        ¦  «        | _        t          |j        |j        ¦  «        | _        |                      ¦   «          d S r    )rˆ   r¡   r„  rq   rt   r¨  rr   rp  r¥   s     €rK   r¡   zT5Gemma2Model.__init__o  s_   ø€ Ý‰Œ×Ò˜Ñ Ô Ð õ ' v¤~°vÔ7MÑNÔNˆŒÝ& v¤~°vÔ7MÑNÔNˆŒà�ŠÑÔÐÐÐrN   NÚmodalityc                 ó   — | j         S r    )rq   ©rW   rº  s     rK   Úget_encoderzT5Gemma2Model.get_encoderx  ó
   € ØŒ|ÐrN   c                 ó   — | j         S r    ©rr   rŽ   s    rK   Úget_decoderzT5Gemma2Model.get_decoder{  r¾  rN   c                 ó4   — | j                              ¦   «         S r    )rq   r�  rŽ   s    rK   r�  z"T5Gemma2Model.get_input_embeddings~  s   € ØŒ|×0Ò0Ñ2Ô2Ð2rN   c                 ó6   — | j                              |¦  «        S r    )rq   r�  r�  s     rK   r�  z"T5Gemma2Model.set_input_embeddings�  s   € ØŒ|×0Ò0°Ñ@Ô@Ð@rN   r,  r’  rr  r  Údecoder_input_idsÚdecoder_attention_maskÚdecoder_position_idsÚencoder_outputsrp   rs  Údecoder_inputs_embedsr  rX   rµ   c                 óè   — |€ | j         d||||
|ddœ|¤Ž}|j        } | j        d|||||	|||ddœ	|¤Ž}t          |j        |j        |j        |j        |j        |j        |j        |j        ¬¦  «        S )aX  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        NT)r,  rr  r  rs  r’  r”  )	r,  rr  r  rs  rp   rÈ   r­  r  r”  )rz  rp   Údecoder_hidden_statesÚdecoder_attentionsrª  Úencoder_last_hidden_staterÈ   Úencoder_attentionsrO   )rq   rz  rr   r   rp   r¬   r`  rª  )rW   r,  r’  rr  r  rÄ  rÅ  rÆ  rÇ  rp   rs  rÈ  r  rX   rÈ   Údecoder_outputss                   rK   r­   zT5Gemma2Model.forward„  sÛ   € ð6 Ð"Ø*˜dœlð Ø#Ø-Ø)Ø+Ø)Ø ðð ð ðð ˆOð !0Ô AÐð '˜$œ,ð 
Ø'Ø1Ø-Ø/Ø+Ø"7Ø#1ØØð
ð 
ð ð
ð 
ˆõ "Ø-Ô?Ø+Ô;Ø"1Ô"?Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð	
ñ 	
ô 	
ð 		
rN   r    )NNNNNNNNNNNN)r[   r\   r]   Ú_tied_weights_keysrn   r¡   r’   r½  rÁ  r�  r�  r   r   rß   r  r  Ú
BoolTensorr   r
   rø   rF   r   r   r   r­   r–   r—   s   @rK   r¸  r¸  h  sê  ø€ € € € € ð (PØ.]ðð Ðð
˜~ð ð ð ð ð ð ðð  C¨$¡Jð ð ð ð ðð ð ð3ð 3ð 3ðAð Að Að Øð .2Ø15Ø37Ø04à59Ø:>Ø8<à26Ø6:Ø-1Ø59Ø!%ð!=
ð =
ð Ô# dÑ*ð=
ð Ô'¨$Ñ.ð	=
ð
 Ô)¨DÑ0ð=
ð Ô&¨Ñ-ð=
ð !Ô+¨dÑ2ð=
ð !&Ô 0°4Ñ 7ð=
ð $Ô.°Ñ5ð=
ð )¨4Ñ/ð=
ð -¨tÑ3ð=
ð ”| dÑ*ð=
ð  %œ|¨dÑ2ð=
ð  ˜$‘;ð!=
ð" Ð+Ô,ð#=
ð$ 
ð%=
ð =
ð =
ñ „^ñ Ôð=
ð =
ð =
ð =
ð =
rN   r¸  c            $       óp  ‡ — e Zd ZddiZddiZddgdgfiZdefˆ fd„Zd	„ Zd
„ Z	d„ Z
d„ Zd+dedz  fd„Zd„ Zeedej        dee         deez  fd„¦   «         ¦   «         Zed„ ¦   «         Zee	 	 	 	 	 	 	 	 	 	 	 	 	 	 d,dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dedz  dej        dz  d ej        dz  d!ej        dz  d"edz  d#e ej        z  dee         deej                 e!z  f d$„¦   «         ¦   «         Z"d%e#d&e$d'e%d(e d)e defˆ fd*„Z&ˆ xZ'S )-Ú T5Gemma2ForConditionalGenerationzlm_head.out_proj.weightz,model.encoder.text_model.embed_tokens.weightzlm_head.out_projÚcolwise_gather_outputr¬   Úlogitsrž   c                 ó  •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        j        | _        t          |j        j        | j        ¦  «        | _        d| _	        |  
                    ¦   «          d S )NÚForMaskedLM)rˆ   r¡   r¸  r4  rr   r‚   r  rŒ   Úlm_headÚ	loss_typerp  r¥   s     €rK   r¡   z)T5Gemma2ForConditionalGeneration.__init__Í  sk   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å" 6Ñ*Ô*ˆŒ
Ø œ.Ô3ˆŒÝ% f¤nÔ&@À$Ä/ÑRÔRˆŒØ&ˆŒà�ŠÑÔÐÐÐrN   c                 ó   — || j         _        d S r    ©r×  rF  r�  s     rK   Úset_output_embeddingsz6T5Gemma2ForConditionalGeneration.set_output_embeddings×  s   € Ø .ˆŒÔÐÐrN   c                 ó   — | j         j        S r    rÚ  rŽ   s    rK   Úget_output_embeddingsz6T5Gemma2ForConditionalGeneration.get_output_embeddingsÚ  s   € ØŒ|Ô$Ð$rN   c                 ó4   — | j                              ¦   «         S r    ©r4  r�  rŽ   s    rK   r�  z5T5Gemma2ForConditionalGeneration.get_input_embeddingsÝ  ó   € ØŒz×.Ò.Ñ0Ô0Ð0rN   c                 ó:   — | j                              |¦  «         d S r    ©r4  r�  ©rW   Úvalues     rK   r�  z5T5Gemma2ForConditionalGeneration.set_input_embeddingsà  ó   € ØŒ
×'Ò'¨Ñ.Ô.Ð.Ð.Ð.rN   Nrº  c                 ó8   — | j                              |¬¦  «        S )N)rº  )r4  r½  r¼  s     rK   r½  z,T5Gemma2ForConditionalGeneration.get_encoderã  s   € ØŒz×%Ò%¨xÐ%Ñ8Ô8Ð8rN   c                 ó4   — | j                              ¦   «         S r    )r4  rÁ  rŽ   s    rK   rÁ  z,T5Gemma2ForConditionalGeneration.get_decoderæ  s   € ØŒz×%Ò%Ñ'Ô'Ð'rN   r’  rX   rµ   c                 óB   —  |                       ¦   «         j        |fi |¤ŽS r    )r½  r˜  )rW   r’  rX   s      rK   r˜  z3T5Gemma2ForConditionalGeneration.get_image_featuresé  s+   € ð
 5ˆt×ÒÑ!Ô!Ô4°\ÐLÐLÀVÐLÐLÐLrN   c                 ó4   — |                       ¦   «         j        S r    )r½  rŠ  rŽ   s    rK   rŠ  z-T5Gemma2ForConditionalGeneration.vision_towerð  s   € à×ÒÑ!Ô!Ô.Ð.rN   r   r,  rr  r  rÄ  rÅ  rÆ  rÇ  rp   rs  rÈ  rO  r  Úlogits_to_keepc                 ó,  — |�|€|€|                       |¦  «        } | j        d|||||||||	|
||dœ|¤Ž}|j        }t          |t          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }| j        j        }|j	        �(||j	        z  }t          j        |¦  «        }||j	        z  }d}|� | j        ||| j        fi |¤Ž}t          |||j        |j        |j        |j        |j        |j        |j        ¬¦	  «	        S )aü  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        N)r,  r’  rr  r  rÄ  rÅ  rÆ  rÇ  rp   rs  rÈ  r  )	ÚlossrÔ  rp   rÊ  rË  rª  rÌ  rÈ   rÍ  rO   )rW  r4  rz  rƒ   r•   Úslicer×  rž   rr   Úfinal_logit_softcappingrß   ÚtanhÚloss_functionr‚   r   rp   rÊ  rË  rª  rÌ  rÈ   rÍ  )rW   r,  r’  rr  r  rÄ  rÅ  rÆ  rÇ  rp   rs  rÈ  rO  r  rê  rX   rÎ  r¬   Úslice_indicesrÔ  rT  rì  s                         rK   r­   z(T5Gemma2ForConditionalGeneration.forwardô  sz  € ðB ÐÐ"3Ð";Ð@UÐ@]à $× JÒ JÈ6Ñ RÔ RÐà.8¨d¬jð /
ØØ%Ø)Ø%Ø/Ø#9Ø!5Ø+Ø+Ø'Ø"7Øð/
ð /
ð ð/
ð /
ˆð  (Ô9ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàœÔ,ˆØÔ1Ð=Ø˜nÔDÑDˆFÝ”Z Ñ'Ô'ˆFØ˜nÔDÑDˆFàˆØÐà%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDåØØØ+Ô;Ø"1Ô"GØ.ÔAØ,Ô=Ø&5Ô&OØ"1Ô"GØ.ÔAð

ñ 

ô 

ð 
	
rN   Úgeneration_configÚmodel_kwargsÚgeneration_modeÚ
batch_sizeÚmax_cache_lengthc           	      óÀ  •— t          ¦   «                              |||||¦  «         |j        du rdS |j        }|€d}n	d|j        v }t	          j        | j                             d¬¦  «        ¦  «        }d|_        dg|j	        z  |_
        ||dœ}	|                     d¦  «        }
|
�¡t          |
t          ¦  «        st          d	¦  «        ‚t          |
j        ¦  «        d
k    r|
j                             d
¦  «        rdS t#          |
j        ¦  «        }|t&          k    r|d         d
         j        d         |	d<    |di |	¤Ž|
_        nEt          t+          di | j                             d¬¦  «        |dœ¤Žt+          ¦   «         ¦  «        |d<   t-          | d¦  «        r?| j        �:t          | j        t          ¦  «        st          d¦  «        ‚|d         | _        dS dS dS )zMOverride cache preparation to support T5Gemma2-specific EncoderDecoder Cache.FNÚ	offloadedTrÀ  rD   )rž   Ú
offloadingrp   zaThe `past_key_values` in `model_kwargs` must be of type `EncoderDecoderCache` for T5Gemma2 model.r   rÇ  rB   Úmax_cache_lenÚ_cachezLThe internal cache must be of type `EncoderDecoderCache` for T5Gemma2 model.rO   )rˆ   Ú_prepare_cache_for_generationr  Úcache_implementationÚcopyÚdeepcopyrž   Úget_text_configrù   rT   rR   rÚ   rƒ   r
   r�   ÚlenrÙ   ÚtyperÛ   r   rÎ   r	   rH  rû  )rW   rò  ró  rô  rõ  rö  rý  Úoffload_cacheÚcross_attn_configÚcross_attn_cache_kwargsrp   Úcross_attn_clsrŠ   s               €rK   rü  z>T5Gemma2ForConditionalGeneration._prepare_cache_for_generationE  sN  ø€ õ 	‰Œ×-Ò-ØØØØØñ	
ô 	
ð 	
ð Ô&¨%Ð/Ð/ØˆFà0ÔEÐØÐ'Ø!ˆMˆMà'Ð+<Ô+QÐQˆMõ !œM¨$¬+×*EÒ*EÈdÐ*EÑ*SÔ*SÑTÔTÐð ,0ÐÔ(Ø)9Ð(:Ð=NÔ=`Ñ(`ÐÔ%ð (Ø'ð#
ð #
Ðð
 '×*Ò*Ð+<Ñ=Ô=ˆØÐ&Ý˜oÕ/BÑCÔCð Ý Øwñô ð õ
 �?Ô-Ñ.Ô.°Ò2Ð2°Ô7Q×7UÒ7UÐVWÑ7XÔ7XÐ2Ø�å! /Ô"GÑHÔHˆNØ¥Ò,Ð,Ø;GÐHYÔ;ZÐ[\Ô;]Ô;cÐdeÔ;fÐ'¨Ñ8à4B°NÐ4]Ð4]ÐE\Ð4]Ð4]ˆOÔ1Ð1õ /BÝð ð à"&¤+×"=Ò"=ÀdÐ"=Ñ"KÔ"KØ&3ðð ðð õ ‘”ñ/ô /ˆLÐ*Ñ+õ �4˜Ñ"Ô"ð 	: t¤{Ð'>Ý˜dœkÕ+>Ñ?Ô?ð qÝ Ð!oÑpÔpÐpà&Ð'8Ô9ˆDŒKˆKˆKð		:ð 	:Ð'>Ð'>rN   r    )NNNNNNNNNNNNNr   )(r[   r\   r]   rÏ  Ú_tp_planÚ_pp_planrn   r¡   rÛ  rÝ  r�  r�  r’   r½  rÁ  r   r   rß   rø   r   r   r¹   r   r˜  ÚpropertyrŠ  r  r  rÐ  r   r
   rF   r•   r   r­   r   r„   r   rü  r–   r—   s   @rK   rÒ  rÒ  Æ  sG  ø€ € € € € à!Ð#QðÐð #Ð$;Ð<€HØ" oÐ%6¸¸
Ð$CÐD€Hð˜~ð ð ð ð ð ð ð/ð /ð /ð%ð %ð %ð1ð 1ð 1ð/ð /ð /ð9ð 9 C¨$¡Jð 9ð 9ð 9ð 9ð(ð (ð (ð ØðMØ!œLðMØ4:Ð;MÔ4NðMà	Ð+Ñ	+ðMð Mð Mñ „^ñ ÔðMð
 ð/ð /ñ „Xð/ð Øð .2Ø15Ø37Ø04à59Ø:>Ø8<à26Ø6:Ø26Ø:>Ø*.Ø!%Ø-.ð%M
ð M
ð Ô# dÑ*ðM
ð Ô'¨$Ñ.ð	M
ð
 Ô)¨DÑ0ðM
ð Ô&¨Ñ-ðM
ð !Ô+¨dÑ2ðM
ð !&Ô 0°4Ñ 7ðM
ð $Ô.°Ñ5ðM
ð )¨4Ñ/ðM
ð -¨tÑ3ðM
ð Ô(¨4Ñ/ðM
ð  %Ô0°4Ñ7ðM
ð  Ô  4Ñ'ð!M
ð" ˜$‘;ð#M
ð$ ˜eœlÑ*ð%M
ð& Ð+Ô,ð'M
ð( 
ˆuÔ Ô	! OÑ	3ð)M
ð M
ð M
ñ „^ñ ÔðM
ð^I:à+ðI:ð ðI:ð (ð	I:ð
 ðI:ð ðI:ð 
ðI:ð I:ð I:ð I:ð I:ð I:ð I:ð I:ð I:ð I:rN   rÒ  c                   óV  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 	 	 dde	j
        dz  de	j        dz  de	j        dz  d	e	j
        dz  d
e	j
        dz  de	j        dz  de	j
        dz  dedz  de	j        dz  de	j        dz  de	j
        dz  dee         defd„¦   «         ¦   «         Zˆ xZS )Ú!T5Gemma2ForSequenceClassificationrž   c                 ó6  •— t          ¦   «                              |¦  «         |j        | _        |j        j        | _        t          |¦  «        | _        t          |dd¦  «        }t          | j        | j        |¦  «        | _	        |  
                    ¦   «          d S ©Nr{   gš™™™™™¹?©rˆ   r¡   Ú
num_labelsrr   rŒ   r¸  r4  r‡   r  Úscorerp  ©rW   rž   Úclassifier_dropoutrŠ   s      €rK   r¡   z*T5Gemma2ForSequenceClassification.__init__“  sƒ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒØ!œ>Ô5ˆÔå" 6Ñ*Ô*ˆŒ
å$ VÐ-FÈÑLÔLÐÝ/°Ô0@À$Ä/ÐSeÑfÔfˆŒ
Ø�ŠÑÔÐÐÐrN   c                 ó4   — | j                              ¦   «         S r    rß  rŽ   s    rK   r�  z6T5Gemma2ForSequenceClassification.get_input_embeddingsž  rà  rN   c                 ó:   — | j                              |¦  «         d S r    râ  rã  s     rK   r�  z6T5Gemma2ForSequenceClassification.set_input_embeddings¡  rå  rN   Nr,  r’  rr  r  rÄ  rÅ  rÆ  rÇ  rs  rÈ  rO  rX   rµ   c                 ó$  — |	€|
�t          d| j        j        › d�¦  «        ‚|€t          d¦  «        ‚|€|                      |¦  «        } | j        |f||||||||	|
ddœ
|¤Ž}|j        }|j        }|j        }|  	                    |¦  «        }|j
        d         }|| j        j        k                         |j        t          j        ¦  «        }t          j        |j
        d         |j        t          j        ¬	¦  «        }||z                       d¦  «        }t          j        ||j
        d         d
z
  ¬¦  «        }|t          j        ||j        ¬¦  «        |f         }d}|�|                      |||| j        ¬¦  «        }t+          ||||¬¦  «        S )áÜ  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nú8Passing input embeddings is currently not supported for ú.úYou have to specify input_idsF©
r’  rr  r  rÄ  rÅ  rÆ  rÇ  rs  rÈ  r  r   rÊ   )r²   r0  rB   )Úmaxrw  )rÔ  rO  Úpooled_logitsrž   ©rì  rÔ  r¬   r`  )ÚNotImplementedErrorrŠ   r[   r�   rW  r4  rz  rÊ  rË  r  rÎ   rž   r€   r.  r²   rß   Úint32r{  ÚargmaxÚclamprð  r   )rW   r,  r’  rr  r  rÄ  rÅ  rÆ  rÇ  rs  rÈ  rO  rX   r¦  rz  r¬   r`  rÔ  rõ  Únon_pad_maskÚtoken_indicesÚlast_non_pad_tokenr  rì  s                           rK   r­   z)T5Gemma2ForSequenceClassification.forward¤  sà  € ð4 Ð$Ð(=Ð(IÝ%ØeÈ4Ì>ÔKbÐeÐeÐeñô ð ð ÐÝÐ<Ñ=Ô=Ð=àÐ$Ø $× JÒ JÈ9Ñ UÔ UÐà&0 d¤jØð'
à%Ø)Ø%Ø/Ø#9Ø!5Ø+Ø'Ø"7Øð'
ð '
ð ð'
ð '
ˆð $Ô5ÐØÔ5ˆØÔ/ˆ
à—’Ð-Ñ.Ô.ˆà”_ QÔ'ˆ
à)¨T¬[Ô-EÒE×IÒIÈ&Ì-ÕY^ÔYdÑeÔeˆÝœÐ%6Ô%<¸RÔ%@ÈÌÕ^cÔ^iÐjÑjÔjˆØ+¨lÑ:×BÒBÀ2ÑFÔFÐÝ"œ[Ð);ÐARÔAXÐY[ÔA\Ð_`ÑA`ÐaÑaÔaÐà�uœ|¨J¸v¼}ÐMÑMÔMÐOaÐaÔbˆàˆØÐØ×%Ò%¨V¸FÐR_ÐhlÔhsÐ%ÑtÔtˆDå'ØØ Ø'Ø!ð	
ñ 
ô 
ð 	
rN   ©NNNNNNNNNNN)r[   r\   r]   rn   r¡   r�  r�  r   r   rß   r  r  rø   r   r   r   r   r­   r–   r—   s   @rK   r  r  ‘  s¤  ø€ € € € € ð	˜~ð 	ð 	ð 	ð 	ð 	ð 	ð1ð 1ð 1ð/ð /ð /ð Øð .2Ø15Ø.2Ø04Ø59Ø6:Ø8<Ø26Ø26Ø:>Ø*.ðJ
ð J
àÔ# dÑ*ðJ
ð Ô'¨$Ñ.ðJ
ð œ tÑ+ð	J
ð
 Ô&¨Ñ-ðJ
ð !Ô+¨dÑ2ðJ
ð !&¤¨tÑ 3ðJ
ð $Ô.°Ñ5ðJ
ð )¨4Ñ/ðJ
ð Ô(¨4Ñ/ðJ
ð  %Ô0°4Ñ7ðJ
ð Ô  4Ñ'ðJ
ð Ð+Ô,ðJ
ð 
"ðJ
ð J
ð J
ñ „^ñ ÔðJ
ð J
ð J
ð J
ð J
rN   r  c                   óV  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 	 	 dde	j
        dz  de	j        dz  de	j        dz  d	e	j
        dz  d
e	j
        dz  de	j        dz  de	j
        dz  dedz  de	j        dz  de	j        dz  de	j
        dz  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚT5Gemma2ForTokenClassificationrž   c                 ó6  •— t          ¦   «                              |¦  «         |j        | _        |j        j        | _        t          |¦  «        | _        t          |dd¦  «        }t          | j        | j        |¦  «        | _	        |  
                    ¦   «          d S r  r  r  s      €rK   r¡   z'T5Gemma2ForTokenClassification.__init__õ  sƒ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒØ!œ>Ô5ˆÔå" 6Ñ*Ô*ˆŒ
å$ VÐ-FÈÑLÔLÐÝ/°Ô0@À$Ä/ÐSeÑfÔfˆŒ
à�ŠÑÔÐÐÐrN   c                 ó4   — | j                              ¦   «         S r    rß  rŽ   s    rK   r�  z3T5Gemma2ForTokenClassification.get_input_embeddings  rà  rN   c                 ó:   — | j                              |¦  «         d S r    râ  rã  s     rK   r�  z3T5Gemma2ForTokenClassification.set_input_embeddings  rå  rN   Nr,  r’  rr  r  rÄ  rÅ  rÆ  rÇ  rs  rÈ  rO  rX   rµ   c                 ó€  — |	€|
�t          d| j        j        › d�¦  «        ‚|€t          d¦  «        ‚|€|                      |¦  «        } | j        |f||||||||	|
ddœ
|¤Ž}|j        }|j        }|j        }|  	                    |¦  «        }d}|�|  
                    ||| j        ¦  «        }t          ||||¬¦  «        S )r  Nr  r  r  Fr  r  )r  rŠ   r[   r�   rW  r4  rz  rÊ  rË  r  rð  rž   r   )rW   r,  r’  rr  r  rÄ  rÅ  rÆ  rÇ  rs  rÈ  rO  rX   r¦  rz  r¬   r`  rÔ  rì  s                      rK   r­   z&T5Gemma2ForTokenClassification.forward  s"  € ð4 Ð$Ð(=Ð(IÝ%ØeÈ4Ì>ÔKbÐeÐeÐeñô ð ð ÐÝÐ<Ñ=Ô=Ð=àÐ$Ø $× JÒ JÈ9Ñ UÔ UÐà&0 d¤jØð'
à%Ø)Ø%Ø/Ø#9Ø!5Ø+Ø'Ø"7Øð'
ð '
ð ð'
ð '
ˆð $Ô5ÐØÔ5ˆØÔ/ˆ
à—’Ð-Ñ.Ô.ˆàˆØÐØ×%Ò% f¨f°d´kÑBÔBˆDå$ØØØ'Ø!ð	
ñ 
ô 
ð 	
rN   r%  )r[   r\   r]   rn   r¡   r�  r�  r   r   rß   r  r  rø   r   r   r   r   r­   r–   r—   s   @rK   r'  r'  ó  s¤  ø€ € € € € ð
˜~ð 
ð 
ð 
ð 
ð 
ð 
ð1ð 1ð 1ð/ð /ð /ð Øð .2Ø15Ø.2Ø04Ø59Ø6:Ø8<Ø26Ø26Ø:>Ø*.ð@
ð @
àÔ# dÑ*ð@
ð Ô'¨$Ñ.ð@
ð œ tÑ+ð	@
ð
 Ô&¨Ñ-ð@
ð !Ô+¨dÑ2ð@
ð !&¤¨tÑ 3ð@
ð $Ô.°Ñ5ð@
ð )¨4Ñ/ð@
ð Ô(¨4Ñ/ð@
ð  %Ô0°4Ñ7ð@
ð Ô  4Ñ'ð@
ð Ð+Ô,ð@
ð 
ð@
ð @
ð @
ñ „^ñ Ôð@
ð @
ð @
ð @
ð @
rN   r'  )
rn   r:   rc   ri   rÒ  r¸  r„  r3  r  r'  )T)jrþ  Úcollections.abcr   Útypingr   r   rß   Útorch.nnr¢   Úhuggingface_hub.dataclassesr   Ú r   rA  Úcache_utilsr	   r
   r   Úconfiguration_utilsr   Ú
generationr   r   r   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_outputsr   r   r   r   r   r   r   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r    r!   Úutils.genericr"   Úutils.output_capturingr#   r$   Úautor&   Úgemma3.configuration_gemma3r'   r(   Úgemma3.modeling_gemma3r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   Úsiglipr4   Út5gemma.modeling_t5gemmar5   r6   r7   Ú
get_loggerr[   r…   r:   rc   ri   rn   r™   r�   r¯   r»   rÄ   r•   r  r  r
  r  r  r  r!  r3  r_  r„  r¨  r¸  rÒ  r  r'  Ú__all__rO   rN   rK   ú<module>rD     sE  ðð €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø  Ð  Ð  Ð  Ð  Ð  Ð  Ð  à €€€Ø Ð Ð Ð Ð Ð Ø .Ð .Ð .Ð .Ð .Ð .à &Ð &Ð &Ð &Ð &Ð &Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø BÐ BÐ BÐ BÐ BÐ Bðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 7Ð 6Ð 6Ð 6Ð 6Ð 6Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &ðð ð ð ð ð ð ð ð ð ð ð ð ð ð 8Ð 7Ð 7Ð 7Ð 7Ð 7Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ Ð Ð Ð Ð Ð Ø HÐ HÐ HÐ HÐ HÐ HÐ HÐ Hðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð (Ð 'Ð 'Ð 'Ð 'Ð 'ðð ð ð ð ð ð ð ð ð ð 
ˆÔ	˜HÑ	%Ô	%€ð €Ð7Ð8Ñ8Ô8Øð1ð 1ð 1ð 1ð 1Ð)Ð+;ñ 1ô 1ñ „ñ 9Ô8ð1ð2 €Ð7Ð8Ñ8Ô8Øðð ð ð ð ˜Lñ ô ñ „ñ 9Ô8ðð €Ð7Ð8Ñ8Ô8Øð1ð 1ð 1ð 1ð 1Ð,Ð.>ñ 1ô 1ñ „ñ 9Ô8ð1ð2 €Ð7Ð8Ñ8Ô8ØðTð Tð Tð Tð TÐ%ñ Tô Tñ „ñ 9Ô8ðTðn	ð 	ð 	ð 	ð 	�mñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	�)ñ 	ô 	ð 	ð\ð \ð \ð \ð \Ð3ñ \ô \ð \ðð ð ð ð ˜Oñ ô ð ð[Bð [Bð [Bð [Bð [B˜oñ [Bô [Bð [Bð|ð °ð Èð ð ð ð ð&	ð 	ð 	ð 	ð 	Ð.ñ 	ô 	ð 	ð,ð ,ð ,ð ,ð ,Ð.ñ ,ô ,ð ,ð^	ð 	ð 	ð 	ð 	�]ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð!:ñ 	ô 	ð 	ð!ð !ð !ð !ð !Ð";ñ !ô !ð !ð
 ð  ð  ð  ð  Ð&Cñ  ô  ð  ð* ðF!ð F!ð F!ð F!ð F!Ð3ñ F!ô F!ñ „ðF!ðRb
ð b
ð b
ð b
ð b
Ð1ñ b
ô b
ð b
ðJeð eð eð eð eÐ-ñ eô eð eðP~
ð ~
ð ~
ð ~
ð ~
Ð-ñ ~
ô ~
ð ~
ðB ðZ
ð Z
ð Z
ð Z
ð Z
Ð+ñ Z
ô Z
ñ „ðZ
ðzH:ð H:ð H:ð H:ð H:Ð'>Àñ H:ô H:ð H:ðV ð^
ð ^
ð ^
ð ^
ð ^
Ð(?ñ ^
ô ^
ñ „ð^
ðB ðU
ð U
ð U
ð U
ð U
Ð%<ñ U
ô U
ñ „ðU
ðpð ð €€€rN   