§
    ‚ŠtjmÊ  ã                   ó¢  — d dl Z d dlmZ d dlmZ d dlZd dlmZ d dlm	Z	 ddl
mZ ddlmZmZmZmZ ddlmZ dd	lmZmZmZ dd
lmZmZmZmZ ddlmZ ddlm Z  ddl!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z' ddl(m)Z)m*Z* ddl+m,Z, ddl-m.Z.m/Z/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5m6Z6 ddl7m8Z8 ddl9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?  e1j@        eA¦  «        ZB e/d¬¦  «        e	 G d„ de8¦  «        ¦   «         ¦   «         ZC e/d¬¦  «        e	 G d„ de¦  «        ¦   «         ¦   «         ZD G d„ de=¦  «        ZE G d„ de;¦  «        ZF G d „ d!e>¦  «        ZG G d"„ d#e:¦  «        ZH G d$„ d%e:¦  «        ZI G d&„ d'e ¦  «        ZJ G d(„ d)e ¦  «        ZK G d*„ d+ejL        ¦  «        ZM G d,„ d-ejL        ¦  «        ZNe/ G d.„ d/e<¦  «        ¦   «         ZOd0ejP        dz  d1ejQ        d2eRdz  d3ejQ        fd4„ZS G d5„ d6eO¦  «        ZT G d7„ d8eO¦  «        ZUe/ G d9„ d:eO¦  «        ¦   «         ZVe/ G d;„ d<eO¦  «        ¦   «         ZW G d=„ d>eOe¦  «        ZXe/ G d?„ d@eO¦  «        ¦   «         ZYe/ G dA„ dBeO¦  «        ¦   «         ZZg dC¢Z[dS )Dé    N)ÚCallable)ÚAny)Ústricté   )Úinitialization)ÚCacheÚDynamicCacheÚEncoderDecoderCacheÚStaticCache)ÚPreTrainedConfig)ÚGenerationConfigÚGenerationMixinÚGenerationMode)Úcreate_bidirectional_maskÚ(create_bidirectional_sliding_window_maskÚcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚGemma2Config)ÚGemma2AttentionÚ	Gemma2MLPÚGemma2PreTrainedModelÚGemma2RMSNormÚGemma2RotaryEmbeddingÚeager_attention_forwardzgoogle/t5_gemma_module-7b)Ú
checkpointc                   ó6   — e Zd ZU dZdZeed<    e¦   «         ZdS )ÚT5GemmaModuleConfigaA  
    query_pre_attn_scalar (`float`, *optional*, defaults to 256):
        scaling factor used on the attention scores
    final_logit_softcapping (`float`, *optional*, defaults to 30.0):
        scaling factor when applying tanh softcapping on the logits.
    attn_logit_softcapping (`float`, *optional*, defaults to 50.0):
        scaling factor when applying tanh softcapping on the attention scores.

    ```python
    >>> from transformers import T5GemmaModuleModel, T5GemmaModuleConfig
    >>> # Initializing a T5GemmaModule t5_gemma_module-7b style configuration
    >>> configuration = T5GemmaModuleConfig()
    >>> # Initializing a model from the t5_gemma_module-7b style configuration
    >>> model = T5GemmaModuleModel(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```FÚ
is_decoderN)	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r1   ÚboolÚ__annotations__ÚAttributeErrorÚuse_bidirectional_attention© ó    úi/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/t5gemma/modular_t5gemma.pyr0   r0   C   s<   € € € € € € ðð ð$ €J�ÐÐÑØ"0 .Ñ"2Ô"2ÐÐÐr;   r0   c                   óò   ‡ — e Zd ZU dZdZdgZeedœZdZee	e
e
f         z  dz  ed<   dZee	e
e
f         z  dz  ed<   dZeed	<   d
Zeez  ed<   d
Zeez  ed<   d
Zeez  ed<   dZeed<   dZeed<   ˆ fd„Zˆ xZS )ÚT5GemmaConfigaÈ  
    encoder (`Union[T5GemmaModuleConfig, dict]`, optional, *optional*):
        Configuration for the encoder.
    decoder (`Union[T5GemmaModuleConfig, dict]`, optional, *optional*):
        Configuration for the decoder.

    Example:

    ```python
    >>> from transformers import T5GemmaConfig, T5GemmaModel
    >>> t5gemma_config = T5GemmaConfig.from_pretrained("google/t5gemma-2b-2b-prefixlm-it")
    >>> model = T5GemmaModel(t5gemma_config)
    ```Út5gemmaÚpast_key_values)ÚencoderÚdecoderNrA   rB   TÚis_encoder_decoderç        Údropout_rateÚclassifier_dropout_rateÚattention_dropoutÚtie_word_embeddingsi è Ú
vocab_sizec                 óî  •— t          | j        t          ¦  «        rt          di | j        ¤Ž| _        n| j        €t          ¦   «         | _        t          | j        t          ¦  «        rt          di | j        ¤Ž| _        n| j        €t          ¦   «         | _        d| j        _        | j        | j        _        | j        | j        _        d| j        _        d| j        _        | j        | j        _        | j        | j        _        | j        j	        | j        _
        |                     d| j        j        ¦  «        | _        dD ]}||vrt          | j        |¦  «        ||<   Œ t          ¦   «         j        di |¤Ž d S )NFTÚinitializer_range)Úbos_token_idÚpad_token_idÚeos_token_idr:   )Ú
isinstancerA   Údictr0   rB   r1   rE   rG   Ú	use_cacheÚhidden_sizeÚcross_attention_hidden_sizeÚpoprK   ÚgetattrÚsuperÚ__post_init__)ÚselfÚkwargsÚspecial_token_keyÚ	__class__s      €r<   rW   zT5GemmaConfig.__post_init__z   s]  ø€ Ý�d”l¥DÑ)Ô)ð 	1Ý.Ð>Ð>°´Ð>Ð>ˆDŒLˆLØŒ\Ð!Ý.Ñ0Ô0ˆDŒLå�d”l¥DÑ)Ô)ð 	1Ý.Ð>Ð>°´Ð>Ð>ˆDŒLˆLØŒ\Ð!Ý.Ñ0Ô0ˆDŒLà"'ˆŒÔØ$(Ô$5ˆŒÔ!Ø)-Ô)?ˆŒÔ&à"&ˆŒÔØ!%ˆŒÔØ$(Ô$5ˆŒÔ!Ø)-Ô)?ˆŒÔ&Ø37´<Ô3KˆŒÔ0à!'§¢Ð,?ÀÄÔA_Ñ!`Ô!`ˆÔà!Qð 	Uð 	UÐØ ¨Ð.Ð.Ý,3°D´LÐBSÑ,TÔ,T�Ð(Ñ)øà�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'r;   )r2   r3   r4   r5   Ú
model_typeÚkeys_to_ignore_at_inferencer0   Úsub_configsrA   rP   r   r7   rB   rC   r6   rE   ÚintÚfloatrF   rG   rH   rI   rW   Ú__classcell__©r[   s   @r<   r>   r>   \   s  ø€ € € € € € ðð ð €JØ#4Ð"5ÐØ1Ð>QÐRÐR€Kà;?€GÐ  4¨¨S¨¤>Ñ1°DÑ8Ð?Ð?Ñ?Ø;?€GÐ  4¨¨S¨¤>Ñ1°DÑ8Ð?Ð?Ñ?Ø#Ð˜Ð#Ð#Ñ#Ø #€L�#˜‘+Ð#Ð#Ñ#Ø+.Ð˜S 5™[Ð.Ð.Ñ.Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø $Ð˜Ð$Ð$Ñ$Ø€J�ÐÐÑð(ð (ð (ð (ð (ð (ð (ð (ð (r;   r>   c                   ó   — e Zd ZdS )ÚT5GemmaRMSNormN©r2   r3   r4   r:   r;   r<   rd   rd   ˜   ó   € € € € € Ø€Dr;   rd   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú
T5GemmaMLPc                 ó†   •— t          ¦   «                              |¦  «         t          j        |j        ¦  «        | _        d S ©N)rV   Ú__init__ÚnnÚDropoutrE   Údropout©rX   Úconfigr[   s     €r<   rk   zT5GemmaMLP.__init__�   s3   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”z &Ô"5Ñ6Ô6ˆŒˆˆr;   c                 óÖ   — |                       |                      |¦  «        ¦  «        |                      |¦  «        z  }|                      |¦  «        }|                      |¦  «        }|S rj   )Úact_fnÚ	gate_projÚup_projrn   Ú	down_proj)rX   ÚxÚhidden_statesru   s       r<   ÚforwardzT5GemmaMLP.forward¡   sV   € ØŸš D§N¢N°1Ñ$5Ô$5Ñ6Ô6¸¿ºÀa¹¼ÑHˆØŸš ]Ñ3Ô3ˆØ—N’N =Ñ1Ô1ˆ	ØÐr;   )r2   r3   r4   rk   rx   ra   rb   s   @r<   rh   rh   œ   sG   ø€ € € € € ð7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð r;   rh   c                   ó   — e Zd ZdS )ÚT5GemmaRotaryEmbeddingNre   r:   r;   r<   rz   rz   ¨   rf   r;   rz   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚT5GemmaSelfAttentionrp   Ú	layer_idxc                 ód   •— t          ¦   «                              ||¦  «         |j        | _        d S rj   )rV   rk   r1   Ú	is_causal©rX   rp   r}   r[   s      €r<   rk   zT5GemmaSelfAttention.__init__­   s+   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+àÔ*ˆŒˆˆr;   )r2   r3   r4   r0   r_   rk   ra   rb   s   @r<   r|   r|   ¬   sL   ø€ € € € € ð+Ð2ð +¸sð +ð +ð +ð +ð +ð +ð +ð +ð +ð +r;   r|   c                   óÐ   ‡ — e Zd Zdedefˆ fd„Z	 ddej        dej        dz  dej        dz  dedz  d	e	e
         d
eej        ej        dz  eej                 dz  f         fd„Zˆ xZS )ÚT5GemmaCrossAttentionrp   r}   c                 óZ  •— t          ¦   «                              ||¦  «         | `| `d| _        |j        €t          d¦  «        ‚t          j        |j        |j	        | j
        z  |j        ¬¦  «        | _        t          j        |j        |j	        | j
        z  |j        ¬¦  «        | _        d S )NFzBCross-attention needs cross_attention_hidden_size to be specified.©Úbias)rV   rk   Úsliding_windowÚ
layer_typer   rS   Ú
ValueErrorrl   ÚLinearÚnum_key_value_headsÚhead_dimÚattention_biasÚk_projÚv_projr€   s      €r<   rk   zT5GemmaCrossAttention.__init__´   s­   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+ØÐØˆOØˆŒàÔ-Ð5ÝÐaÑbÔbÐbå”iØÔ.°Ô0JÈTÌ]Ñ0ZÐagÔavð
ñ 
ô 
ˆŒõ ”iØÔ.°Ô0JÈTÌ]Ñ0ZÐagÔavð
ñ 
ô 
ˆŒˆˆr;   Nrw   Úattention_maskÚencoder_hidden_statesr@   rY   Úreturnc                 ó,  — |€t          d¦  «        ‚|j        d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|�&|j                             | j        ¦  «        }	|j	        }
|�|	sÆ|j        d d…         }g |¢d‘| j        ‘R }|  
                    |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|�.|
                     ||| j        ¦  «        \  }}d|j        | j        <   n.|
j        | j                 j        }|
j        | j                 j        }t!          j        | j        j        t(          ¦  «        } || ||||f| j        r| j        nd| j        d | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nz5Encoder hidden state is required for cross attention.éÿÿÿÿé   r&   TrD   )rn   Úscalingr†   Úsoftcap)rˆ   Úshaper‹   Úq_projÚviewÚ	transposeÚ
is_updatedÚgetr}   Úcross_attention_cacher�   rŽ   ÚupdateÚlayersÚkeysÚvaluesr   Úget_interfacerp   Ú_attn_implementationr-   ÚtrainingrG   r•   Úattn_logit_softcappingÚreshapeÚ
contiguousÚo_proj)rX   rw   r�   r�   r@   rY   Úinput_shapeÚhidden_shapeÚquery_statesr›   Úcurr_past_key_valuesÚencoder_input_shapeÚencoder_hidden_shapeÚ
key_statesÚvalue_statesÚattention_interfaceÚattn_outputÚattn_weightss                     r<   rx   zT5GemmaCrossAttention.forwardÄ   sV  € ð !Ð(ÝÐTÑUÔUÐUà#Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆØ—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆàÐ&Ø(Ô3×7Ò7¸¼ÑGÔGˆJØ#2Ô#HÐ àÐ"¨*Ð"Ø"7Ô"=¸c¸r¸cÔ"BÐØ#LÐ%8Ð#L¸"Ð#L¸d¼mÐ#LÐ#LÐ ØŸšÐ%:Ñ;Ô;×@Ò@ÐAUÑVÔV×`Ò`ÐabÐdeÑfÔfˆJØŸ;š;Ð'<Ñ=Ô=×BÒBÐCWÑXÔX×bÒbÐcdÐfgÑhÔhˆLàÐ*Ø+?×+FÒ+FÀzÐS_ÐaeÔaoÑ+pÔ+pÑ(�
˜LØ=A�Ô*¨4¬>Ñ:øà-Ô4°T´^ÔDÔIˆJØ/Ô6°t´~ÔFÔMˆLå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð%
ð /3¬mÐD�DÔ*Ð*ÀØ”LØØÔ/ð%
ð %
ð ð%
ð %
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r;   rj   )r2   r3   r4   r0   r_   rk   ÚtorchÚTensorr   r   r   Útuplerx   ra   rb   s   @r<   r‚   r‚   ³   sß   ø€ € € € € ð
Ð2ð 
¸sð 
ð 
ð 
ð 
ð 
ð 
ð* )-ð3)ð 3)à”|ð3)ð œ tÑ+ð3)ð  %œ|¨dÑ2ð	3)ð
  ™ð3)ð Ð-Ô.ð3)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð3)ð 3)ð 3)ð 3)ð 3)ð 3)ð 3)ð 3)r;   r‚   c                   ó¸   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 ddej        deej        ej        f         dz  dej        dz  dej	        dz  d	eej
        f         f
d
„Zˆ xZS )ÚT5GemmaEncoderLayerzEncoder sub-layer.r}   c                 ó0  •— t          ¦   «                              ¦   «          |j        | _        || _        || _        |j        |         | _        t          ||¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        t          |¦  «        | _        t          |j        |j
        ¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        t#          j        |j        ¦  «        | _        d S ©N)rp   r}   ©Úeps)rV   rk   rR   rp   r}   Úlayer_typesÚattention_typer|   Ú	self_attnrd   Úrms_norm_epsÚpre_self_attn_layernormÚpost_self_attn_layernormrh   ÚmlpÚpre_feedforward_layernormÚpost_feedforward_layernormrl   rm   rE   rn   r€   s      €r<   rk   zT5GemmaEncoderLayer.__init__ý   sõ   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØˆŒØ"ˆŒØ$Ô0°Ô;ˆÔå-ØØð
ñ 
ô 
ˆŒõ (6°fÔ6HÈfÔNaÐ'bÑ'bÔ'bˆÔ$Ý(6°vÔ7IÈvÔObÐ(cÑ(cÔ(cˆÔ%å˜fÑ%Ô%ˆŒÝ)7¸Ô8JÐPVÔPcÐ)dÑ)dÔ)dˆÔ&Ý*8¸Ô9KÐQWÔQdÐ*eÑ*eÔ*eˆÔ'å”z &Ô"5Ñ6Ô6ˆŒˆˆr;   Nrw   Úposition_embeddingsr�   Úposition_idsr‘   c           	      ól  — |}|                       |¦  «        } | j        d||||d dœ|¤Ž\  }}|                      |¦  «        }||                      |¦  «        z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||                      |¦  «        z   }|S )N)rw   rÆ   r�   rÇ   r@   r:   )rÁ   r¿   rÂ   rn   rÄ   rÃ   rÅ   )rX   rw   rÆ   r�   rÇ   rY   ÚresidualÚ_s           r<   rx   zT5GemmaEncoderLayer.forward  sÚ   € ð !ˆØ×4Ò4°]ÑCÔCˆØ)˜4œ>ð 
Ø'Ø 3Ø)Ø%Ø ð
ð 
ð ð
ð 
Ñˆ�qð ×5Ò5°mÑDÔDˆØ  4§<¢<°Ñ#>Ô#>Ñ>ˆà ˆØ×6Ò6°}ÑEÔEˆØŸš Ñ/Ô/ˆØ×7Ò7¸ÑFÔFˆØ  4§<¢<°Ñ#>Ô#>Ñ>ˆØÐr;   )NNN)r2   r3   r4   r5   r_   rk   r´   rµ   r¶   Ú
LongTensorÚFloatTensorrx   ra   rb   s   @r<   r¸   r¸   ú   sÏ   ø€ € € € € ØÐð7¨#ð 7ð 7ð 7ð 7ð 7ð 7ð. IMØ.2Ø04ðð à”|ðð # 5¤<°´Ð#=Ô>ÀÑEðð œ tÑ+ð	ð
 Ô&¨Ñ-ðð 
ˆuÔ Ð!Ô	"ðð ð ð ð ð ð ð r;   r¸   c                   óî   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 	 	 	 	 ddej        deej        ej        f         dz  dej        dz  d	ej	        dz  d
e
dz  dedz  dej        dz  dej        dz  dej        fd„Zˆ xZS )ÚT5GemmaDecoderLayerz2Decoder sub-layer: an extra cross-attention layer.r}   c                 óÜ  •— t          ¦   «                              ¦   «          |j        | _        || _        || _        |j        |         | _        t          ||¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        t          |¦  «        | _        t          |j        |j
        ¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        t#          j        |j        ¦  «        | _        t+          ||¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        d S rº   )rV   rk   rR   rp   r}   r½   r¾   r|   r¿   rd   rÀ   rÁ   rÂ   rh   rÃ   rÄ   rÅ   rl   rm   rE   rn   r‚   Ú
cross_attnÚpre_cross_attn_layernormÚpost_cross_attn_layernormr€   s      €r<   rk   zT5GemmaDecoderLayer.__init__1  sB  ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØˆŒØ"ˆŒØ$Ô0°Ô;ˆÔå-ØØð
ñ 
ô 
ˆŒõ (6°fÔ6HÈfÔNaÐ'bÑ'bÔ'bˆÔ$Ý(6°vÔ7IÈvÔObÐ(cÑ(cÔ(cˆÔ%å˜fÑ%Ô%ˆŒÝ)7¸Ô8JÐPVÔPcÐ)dÑ)dÔ)dˆÔ&Ý*8¸Ô9KÐQWÔQdÐ*eÑ*eÔ*eˆÔ'å”z &Ô"5Ñ6Ô6ˆŒÝ/°vÈÐSÑSÔSˆŒÝ(6°vÔ7IÈvÔObÐ(cÑ(cÔ(cˆÔ%Ý)7¸Ô8JÐPVÔPcÐ)dÑ)dÔ)dˆÔ&Ð&Ð&r;   NFrw   rÆ   r�   rÇ   r@   rQ   r�   Úencoder_attention_maskr‘   c	           
      ó4  — |}
|                       |¦  «        } | j        d|||||�|j        nd |dœ|	¤Ž\  }}|                      |¦  «        }|
|                      |¦  «        z   }|}
|                      |¦  «        } | j        d|||||dœ|	¤Ž\  }}|                      |¦  «        }|
|                      |¦  «        z   }|}
|                      |¦  «        }|  	                    |¦  «        }|  
                    |¦  «        }|
|                      |¦  «        z   }|S )N)rw   rÆ   r�   rÇ   r@   rQ   )rw   r�   r�   r@   rQ   r:   )rÁ   r¿   Úself_attention_cacherÂ   rn   rÑ   rÐ   rÒ   rÄ   rÃ   rÅ   )rX   rw   rÆ   r�   rÇ   r@   rQ   r�   rÓ   rY   rÉ   rÊ   s               r<   rx   zT5GemmaDecoderLayer.forwardH  si  € ð !ˆØ×4Ò4°]ÑCÔCˆØ)˜4œ>ð 
Ø'Ø 3Ø)Ø%ØDSÐD_˜OÔ@Ð@ÐeiØð
ð 
ð ð
ð 
Ñˆ�qð ×5Ò5°mÑDÔDˆØ  4§<¢<°Ñ#>Ô#>Ñ>ˆà ˆØ×5Ò5°mÑDÔDˆØ*˜4œ?ð 
Ø'Ø"7Ø1Ø+Øð
ð 
ð ð
ð 
Ñˆ�qð ×6Ò6°}ÑEÔEˆØ  4§<¢<°Ñ#>Ô#>Ñ>ˆà ˆØ×6Ò6°}ÑEÔEˆØŸš Ñ/Ô/ˆØ×7Ò7¸ÑFÔFˆØ  4§<¢<°Ñ#>Ô#>Ñ>ˆØÐr;   )NNNNFNN)r2   r3   r4   r5   r_   rk   r´   rµ   r¶   rË   r
   r6   rÌ   rx   ra   rb   s   @r<   rÎ   rÎ   .  s  ø€ € € € € Ø<Ð<ðe¨#ð eð eð eð eð eð eð4 IMØ.2Ø04Ø6:Ø!&Ø59Ø6:ð,ð ,à”|ð,ð # 5¤<°´Ð#=Ô>ÀÑEð,ð œ tÑ+ð	,ð
 Ô&¨Ñ-ð,ð -¨tÑ3ð,ð ˜$‘;ð,ð  %œ|¨dÑ2ð,ð !&¤¨tÑ 3ð,ð 
Ô	ð,ð ,ð ,ð ,ð ,ð ,ð ,ð ,r;   rÎ   c                   óV   ‡ — e Zd ZdZd
dededefˆ fd„Zdej        dej        fd	„Z	ˆ xZ
S )ÚT5GemmaClassificationHeadz-Head for sentence-level classification tasks.rD   rR   Ú
num_labelsrF   c                 ó°   •— t          ¦   «                              ¦   «          t          j        |¬¦  «        | _        t          j        ||¦  «        | _        d S )N)Úp)rV   rk   rl   rm   rn   r‰   Úout_proj)rX   rR   rØ   rF   r[   s       €r<   rk   z"T5GemmaClassificationHead.__init__z  sE   ø€ Ý‰Œ×ÒÑÔÐÝ”zÐ$;Ð<Ñ<Ô<ˆŒÝœ	 +¨zÑ:Ô:ˆŒˆˆr;   rw   r‘   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rj   )rn   rÛ   )rX   rw   s     r<   rx   z!T5GemmaClassificationHead.forward  s*   € ØŸš ]Ñ3Ô3ˆØŸš mÑ4Ô4ˆØÐr;   )rD   )r2   r3   r4   r5   r_   r`   rk   r´   rµ   rx   ra   rb   s   @r<   r×   r×   w  s„   ø€ € € € € Ø7Ð7ð;ð ; Cð ;°Sð ;ÐSXð ;ð ;ð ;ð ;ð ;ð ;ð
 U¤\ð °e´lð ð ð ð ð ð ð ð r;   r×   c                   óV   ‡ — e Zd ZdZd
dededefˆ fd„Zdej        dej        fd	„Z	ˆ xZ
S )ÚT5GemmaLMHeadz.Head for language modeling (generation) tasks.FrR   rI   r…   c                 ó€   •— t          ¦   «                              ¦   «          t          j        |||¬¦  «        | _        d S )Nr„   )rV   rk   rl   r‰   rÛ   )rX   rR   rI   r…   r[   s       €r<   rk   zT5GemmaLMHead.__init__ˆ  s5   ø€ Ý‰Œ×ÒÑÔÐÝœ	 +¨zÀÐEÑEÔEˆŒˆˆr;   rw   r‘   c                 ó0   — |                       |¦  «        }|S rj   )rÛ   )rX   rw   Úlogitss      r<   rx   zT5GemmaLMHead.forwardŒ  s   € Ø—’˜}Ñ-Ô-ˆØˆr;   )F)r2   r3   r4   r5   r_   r6   rk   r´   rµ   rx   ra   rb   s   @r<   rÞ   rÞ   …  s�   ø€ € € € € Ø8Ð8ðFð F Cð F°Sð FÀð Fð Fð Fð Fð Fð Fð U¤\ð °e´lð ð ð ð ð ð ð ð r;   rÞ   c                   ód   — e Zd ZU eed<   dZdZddgZdZ e	j
        ¦   «         d„ ¦   «         Zd„ ZdS )	ÚT5GemmaPreTrainedModelrp   ÚmodelTr¸   rÎ   Nc                 ó�  — t          j        | |¦  «         | j        j        }t	          |t
          ¦  «        rƒ|j        j        j        d         dz  }t          j
        |j        j        d||z  ¬¦  «         t          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         d S d S d S t	          |t          ¦  «        rN| j        j        s@|j        j        j        d         dz  }t          j
        |j        j        d||z  ¬¦  «         d S d S d|j        j        v rt          j        |j        ¦  «         d S d S )Nr   g      à¿rD   )ÚmeanÚstdr…   ÚRMSNorm)r   Ú_init_weightsrp   rK   rO   r×   rÛ   Úweightr—   ÚinitÚnormal_Úhasattrr…   Úzeros_rÞ   rH   r[   r2   )rX   Úmodulerç   Úscales       r<   ré   z$T5GemmaPreTrainedModel._init_weightsš  sP  € õ 	Ô% d¨FÑ3Ô3Ð3ØŒkÔ+ˆÝ�fÕ7Ñ8Ô8ð 	'Ø”OÔ*Ô0°Ô3°tÑ;ˆEÝŒL˜œÔ/°c¸sÀU¹{ÐKÑKÔKÐKÝ�v”¨Ñ/Ô/ð 2°F´OÔ4HÐ4TÝ”˜FœOÔ0Ñ1Ô1Ð1Ð1Ð1ð2ð 2Ð4TÐ4Tå˜¥Ñ.Ô.ð 	'Ø”;Ô2ð PØœÔ.Ô4°QÔ7¸4Ñ?�Ý”˜Vœ_Ô3¸#À3ÈÁ;ÐOÑOÔOÐOÐOÐOðPð Pð ˜&Ô*Ô3Ð3Ð3ÝŒK˜œÑ&Ô&Ð&Ð&Ð&ð 4Ð3r;   c                 óJ  — | j         j        j        }| j         j        j        }|€t	          d¦  «        ‚|                     |j        ¦  «        }|ddd…f                              ¦   «         |ddd…f<   ||d<   |€t	          d¦  «        ‚|                     |dk    |¦  «         |S )	zú
        Shifts input_ids to the right, prepends the decoder_start_token_id, and handles
        pad_token_id replacement for labels that were -100.
        This is a common preparation step for decoder inputs in sequence-to-sequence models.
        Nz:self.model.config.decoder.bos_token_id has to be defined. .r“   r”   ).r   z9self.model.config.decoder.pad_token_id has to be defined.iœÿÿÿ)	rp   rB   rL   rM   rˆ   Ú	new_zerosr—   ÚcloneÚmasked_fill_)rX   Ú	input_idsÚdecoder_start_token_idrM   Úshifted_input_idss        r<   Ú_shift_rightz#T5GemmaPreTrainedModel._shift_right¬  s¼   € ð "&¤Ô!4Ô!AÐØ”{Ô*Ô7ˆà!Ð)ÝÐYÑZÔZÐZð &×/Ò/°	´Ñ@Ô@ÐØ%.¨s°C°R°C¨xÔ%8×%>Ò%>Ñ%@Ô%@Ð˜#˜q˜r˜r˜'Ñ"Ø$:Ð˜&Ñ!àÐÝÐXÑYÔYÐYð 	×&Ò&Ð'8¸DÒ'@À,ÑOÔOÐOà Ð r;   )r2   r3   r4   r>   r7   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_can_record_outputsr´   Úno_gradré   rø   r:   r;   r<   rã   rã   ‘  sp   € € € € € € àÐÐÑØÐØ&*Ð#Ø.Ð0EÐFÐàÐà€U„]�_„_ð'ð 'ñ „_ð'ð"!ð !ð !ð !ð !r;   rã   Ú	token_idsrw   rM   r‘   c                 óü   — | �;|€t          d¦  «        ‚| |k                         |j        t          j        ¦  «        }n>t          j        |j        d         |j        d         f|j        t          j        ¬¦  «        }|S )z%Construct the default attention mask.Nz3`pad_token_id` is required for padding information.r   r”   ©ÚdeviceÚdtype)rˆ   Útor  r´   ÚlongÚonesr—   )rþ   rw   rM   r�   s       r<   Úmake_default_2d_attention_maskr  Ç  s�   € ð ÐØÐÝÐRÑSÔSÐSØ# |Ò3×7Ò7¸Ô8LÍeÌjÑYÔYˆˆåœØÔ  Ô# ]Ô%8¸Ô%;Ð<À]ÔEYÕafÔakð
ñ 
ô 
ˆð Ðr;   c                   óÄ   ‡ — e Zd ZeedœZˆ fd„Zee	 	 	 	 dde	j
        dz  de	j        dz  de	j
        dz  de	j        dz  dee         d	eez  fd
„¦   «         ¦   «         Zˆ xZS )ÚT5GemmaEncoder)Ú
attentionsrw   c                 ó  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          ‰j        ‰j
        ¬¦  «        | _        d| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j        ¦  «        | _        t)          ‰¬¦  «        | _        |                      ¦   «          d S )Nr»   Fc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r:   )r¸   ©Ú.0r}   rp   s     €r<   ú
<listcomp>z+T5GemmaEncoder.__init__.<locals>.<listcomp>è  ó$   ø€ ÐeÐeÐe¸	Õ  ¨Ñ3Ô3ÐeÐeÐer;   ©rp   ©rV   rk   rM   Úpadding_idxrI   rl   Ú	EmbeddingrR   Úembed_tokensrd   rÀ   ÚnormÚgradient_checkpointingÚ
ModuleListÚrangeÚnum_hidden_layersrŸ   rm   rE   rn   rz   Ú
rotary_embÚ	post_initro   s    `€r<   rk   zT5GemmaEncoder.__init__Þ  óè   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔÝ" 6Ô#5¸6Ô;NÐOÑOÔOˆŒ	Ø&+ˆÔ#å”mØeÐeÐeÐeÅUÈ6ÔKcÑEdÔEdÐeÑeÔeñ
ô 
ˆŒõ ”z &Ô"5Ñ6Ô6ˆŒÝ0¸Ð?Ñ?Ô?ˆŒð 	�ŠÑÔÐÐÐr;   Nrõ   r�   rÇ   Úinputs_embedsrY   r‘   c                 óz  — |d u |d uz  rt          d¦  «        ‚|                     dd ¦  «         |€|                      |¦  «        }|€;t          j        |j        d         |j        ¬¦  «        }|                     d¦  «        }|€t          ||| j	        j
        ¦  «        }t          |x}t          ¦  «        s$| j	        ||dœ}t          di |¤Žt          di |¤Ždœ}|}t          j        | j	        j        dz  |j        ¬	¦  «        }	||	z  }|                      |¦  «        }|                      ||¦  «        }
t)          | j        d | j	        j        …         ¦  «        D ]'\  }} |||
|| j	        j        |                  |fi |¤Ž}Œ(|                      |¦  «        }|                      |¦  «        }t3          |¬
¦  «        S )Nú:You must specify exactly one of input_ids or inputs_embedsr@   r”   ©r  r   )rp   r  r�   ©Úfull_attentionÚsliding_attentionç      à?©r  )Úlast_hidden_stater:   )rˆ   rT   r  r´   Úaranger—   r  Ú	unsqueezer  rp   rM   rO   rP   r   r   ÚtensorrR   r  rn   r  Ú	enumeraterŸ   r  r½   r  r   )rX   rõ   r�   rÇ   r  rY   Úself_attn_mask_mappingÚmask_kwargsrw   Ú
normalizerrÆ   ÚiÚlayer_modules                r<   rx   zT5GemmaEncoder.forwardð  s  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZð 	�
Š
Ð$ dÑ+Ô+Ð+àÐ Ø ×-Ò-¨iÑ8Ô8ˆMàÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\ˆLØ'×1Ò1°!Ñ4Ô4ˆLàÐ!Ý;¸IÀ}ÐVZÔVaÔVnÑoÔoˆNå°NÐBÐ0ÅDÑIÔIð 		àœ+Ø!.Ø"0ðð ˆKõ #<Ð"JÐ"J¸kÐ"JÐ"JÝ%MÐ%\Ð%\ÐP[Ð%\Ð%\ð&ð &Ð"ð
 &ˆÝ”\ $¤+Ô"9¸3Ñ">ÀmÔFYÐZÑZÔZˆ
Ø%¨
Ñ2ˆØŸš ]Ñ3Ô3ˆà"Ÿošo¨m¸\ÑJÔJÐå(¨¬Ð5T°t´{Ô7TÐ5TÔ)UÑVÔVð 	ð 	‰OˆAˆ|Ø(˜LØØ#Ø& t¤{Ô'>¸qÔ'AÔBØð	ð ð
 ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆØŸš ]Ñ3Ô3ˆÝØ+ð
ñ 
ô 
ð 	
r;   ©NNNN)r2   r3   r4   r|   r¸   rü   rk   r#   r%   r´   rË   rµ   rÌ   r   r   r¶   r   rx   ra   rb   s   @r<   r  r  Ø  sí   ø€ € € € € à*Ø,ðð Ðð
ð ð ð ð ð$  Øð .2Ø.2Ø04Ø26ð6
ð 6
àÔ# dÑ*ð6
ð œ tÑ+ð6
ð Ô&¨Ñ-ð	6
ð
 Ô(¨4Ñ/ð6
ð Ð+Ô,ð6
ð 
�Ñ	 ð6
ð 6
ð 6
ñ „_ñ  Ôð6
ð 6
ð 6
ð 6
ð 6
r;   r  c                   ó6  ‡ — e Zd Z eed¬¦  «         eed¬¦  «        edœZˆ fd„Ze	e
	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  d	edz  d
ej        dz  dedz  dej        dz  dej        dz  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚT5GemmaDecoderr”   )Úindex)r	  Úcross_attentionsrw   c                 ó  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          ‰j        ‰j
        ¬¦  «        | _        d| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j        ¦  «        | _        t)          ‰¬¦  «        | _        |                      ¦   «          d S )Nr»   Fc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r:   )rÎ   r  s     €r<   r  z+T5GemmaDecoder.__init__.<locals>.<listcomp><  r  r;   r  r  ro   s    `€r<   rk   zT5GemmaDecoder.__init__2  r  r;   Nrõ   r�   rÇ   r@   r  rQ   r�   rÓ   rY   r‘   c	                 ó¬  — |d u |d uz  rt          d¦  «        ‚|€t          d¦  «        ‚|€|                      |¦  «        }| j        s3|r1|€/t          t	          | j        ¬¦  «        t	          ¦   «         ¦  «        }|€V|�|                     ¦   «         nd}
t          j        |j	        d         |j
        ¬¦  «        |
z   }|                     d¦  «        }|€|€t          ||| j        j        ¦  «        }t          |x}t          ¦  «        s/| j        |||�|j        nd |dœ}t#          di |¤Žt%          di |¤Ždœ}t          |x}t          ¦  «        sd	t'          | j        |||¬
¦  «        i}|}t          j        | j        j        dz  |j        ¬¦  «        }||z  }|                      |¦  «        }|                      ||¦  «        }t3          | j        d | j        j        …         ¦  «        D ]1\  }} ||||| j        j        |                  |||||d	         fi |	¤Ž}Œ2|                      |¦  «        }|                      |¦  «        }t=          ||¬¦  «        S )Nr  z0`encoder_hidden_states` must be given in decoderr  r   r”   r   )rp   r  r�   r@   rÇ   r!  r"  )rp   r  r�   r�   r$  r%  )r&  r@   r:   )rˆ   r  r¤   r
   r	   rp   Úget_seq_lengthr´   r'  r—   r  r(  r  rM   rO   rP   rÕ   r   r   r   r)  rR   r  rn   r  r*  rŸ   r  r½   r  r   )rX   rõ   r�   rÇ   r@   r  rQ   r�   rÓ   rY   Úpast_seen_tokensr+  r,  Úcross_attn_mask_mappingrw   r-  rÆ   r.  r/  s                      r<   rx   zT5GemmaDecoder.forwardD  sâ  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZØ Ð(ÝÐOÑPÔPÐPàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàŒ}ð 	d ð 	d¨Ð/Fõ 2µ,ÀdÄkÐ2RÑ2RÔ2RÕT`ÑTbÔTbÑcÔcˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLàÐ! oÐ&=Ý;¸IÀ}ÐVZÔVaÔVnÑoÔoˆNå°NÐBÐ0ÅDÑIÔIð 	àœ+Ø!.Ø"0ØKZÐKf ?Ô#GÐ#GÐlpØ ,ðð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ%FÐ%UÐ%UÈÐ%UÐ%Uð&ð &Ð"õ
 Ð5KÐKÐ1ÍTÑRÔRð 	à Õ";Øœ;Ø"/Ø#9Ø*?ð	#ñ #ô #ð'Ð#ð &ˆÝ”\ $¤+Ô"9¸3Ñ">ÀmÔFYÐZÑZÔZˆ
Ø%¨
Ñ2ˆØŸš ]Ñ3Ô3ˆà"Ÿošo¨m¸\ÑJÔJÐå(¨¬Ð5T°t´{Ô7TÐ5TÔ)UÑVÔVð 	ð 	‰OˆAˆ|Ø(˜LØØ#Ø& t¤{Ô'>¸qÔ'AÔBØØØØ%Ø'Ð(8Ô9ð
ð 
ð ð
ð 
ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆØŸš ]Ñ3Ô3ˆÝ8Ø+Ø+ð
ñ 
ô 
ð 	
r;   )NNNNNNNN)r2   r3   r4   r$   r|   r‚   rÎ   rü   rk   r#   r%   r´   rË   rµ   r
   rÌ   r6   r   r   r¶   r   rx   ra   rb   s   @r<   r2  r2  +  so  ø€ € € € € à$�nÐ%9ÀÐCÑCÔCØ*˜NÐ+@ÈÐJÑJÔJØ,ðð Ððð ð ð ð ð$  Øð .2Ø.2Ø04Ø6:Ø26Ø!%Ø59Ø6:ðP
ð P
àÔ# dÑ*ðP
ð œ tÑ+ðP
ð Ô&¨Ñ-ð	P
ð
 -¨tÑ3ðP
ð Ô(¨4Ñ/ðP
ð ˜$‘;ðP
ð  %œ|¨dÑ2ðP
ð !&¤¨tÑ 3ðP
ð Ð+Ô,ðP
ð 
Ð:Ñ	:ðP
ð P
ð P
ñ „_ñ  ÔðP
ð P
ð P
ð P
ð P
r;   r2  c                   óB  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 	 	 dde	j
        dz  de	j        dz  de	j
        dz  d	e	j
        dz  d
e	j        dz  de	j
        dz  dedz  dedz  de	j        dz  de	j        dz  dedz  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚT5GemmaModelrp   c                 ó  •— t          ¦   «                              |¦  «         |j        st          d¦  «        ‚t	          |j        ¦  «        | _        t          |j        ¦  «        | _        |                      ¦   «          d S )NzVT5GemmaModel only support encoder-decoder modeling. Use `T5GemmaEncoderModel` instead.)	rV   rk   rC   rˆ   r  rA   r2  rB   r  ro   s     €r<   rk   zT5GemmaModel.__init__›  sn   ø€ Ý‰Œ×Ò˜Ñ Ô Ð àÔ(ð 	wÝÐuÑvÔvÐvå% f¤nÑ5Ô5ˆŒÝ% f¤nÑ5Ô5ˆŒà�ŠÑÔÐÐÐr;   c                 ó4   — | j                              ¦   «         S rj   ©rA   Úget_input_embeddings©rX   s    r<   r@  z!T5GemmaModel.get_input_embeddings¦  ó   € ØŒ|×0Ò0Ñ2Ô2Ð2r;   c                 ó6   — | j                              |¦  «        S rj   ©rA   Úset_input_embeddings©rX   Únew_embeddingss     r<   rE  z!T5GemmaModel.set_input_embeddings©  ó   € ØŒ|×0Ò0°Ñ@Ô@Ð@r;   Nrõ   r�   rÇ   Údecoder_input_idsÚdecoder_attention_maskÚdecoder_position_idsÚencoder_outputsr@   r  Údecoder_inputs_embedsrQ   rY   r‘   c                 ó  — |€ | j         d||||	dœ|¤Ž}|j        } | j        d||||
||||dœ|¤Ž}t          |j        |j        |                     dd¦  «        r|j        n|j        f|j        |j        |j        |j        |j        ¬¦  «        S )aX  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        N©rõ   r�   rÇ   r  )rõ   r�   rÇ   r  r@   r�   rÓ   rQ   Úoutput_hidden_statesF)r&  r@   Údecoder_hidden_statesÚdecoder_attentionsr4  Úencoder_last_hidden_stater�   Úencoder_attentionsr:   )	rA   r&  rB   r   r@   rœ   rw   r	  r4  )rX   rõ   r�   rÇ   rI  rJ  rK  rL  r@   r  rM  rQ   rY   r�   Údecoder_outputss                  r<   rx   zT5GemmaModel.forward¬  sò   € ð, Ð"Ø*˜dœlð Ø#Ø-Ø)Ø+ð	ð ð
 ðð ˆOð !0Ô AÐà&˜$œ,ð 

Ø'Ø1Ø-Ø/Ø+Ø"7Ø#1Øð

ð 

ð ð

ð 

ˆõ "Ø-Ô?Ø+Ô;à�zŠzÐ0°%Ñ8Ô8ð#6 /Ô"?Ð"?à!Ô3Ð5Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð
ñ 
ô 
ð 	
r;   )NNNNNNNNNNN)r2   r3   r4   r>   rk   r@  rE  r!   r    r´   rË   rÌ   Ú
BoolTensorr   r
   rµ   r6   r   r   r   rx   ra   rb   s   @r<   r<  r<  ™  sŠ  ø€ € € € € ð	˜}ð 	ð 	ð 	ð 	ð 	ð 	ð3ð 3ð 3ðAð Að Að Øð .2Ø37Ø04Ø59Ø:>Ø8<Ø26Ø6:Ø-1Ø59Ø!%ð6
ð 6
àÔ# dÑ*ð6
ð Ô)¨DÑ0ð6
ð Ô&¨Ñ-ð	6
ð
 !Ô+¨dÑ2ð6
ð !&Ô 0°4Ñ 7ð6
ð $Ô.°Ñ5ð6
ð )¨4Ñ/ð6
ð -¨tÑ3ð6
ð ”| dÑ*ð6
ð  %œ|¨dÑ2ð6
ð ˜$‘;ð6
ð Ð+Ô,ð6
ð 
ð6
ð 6
ð 6
ñ „^ñ Ôð6
ð 6
ð 6
ð 6
ð 6
r;   r<  c                   óÆ   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 	 	 	 dde	j
        dz  de	j        dz  de	j
        dz  d	e	j        dz  d
ee         defd„¦   «         ¦   «         Zˆ xZS )ÚT5GemmaEncoderModelrp   c                 óÐ   •— t          ¦   «                              |¦  «         |j        rt          d¦  «        ‚t	          |j        ¦  «        | _        |                      ¦   «          d S )NzQT5GemmaEncoderModel only supports encoder-only model. Use `T5GemmaModel` instead.)rV   rk   rC   rˆ   r  rA   r  ro   s     €r<   rk   zT5GemmaEncoderModel.__init__é  s]   ø€ Ý‰Œ×Ò˜Ñ Ô Ð àÔ$ð 	rÝÐpÑqÔqÐqå% f¤nÑ5Ô5ˆŒØ�ŠÑÔÐÐÐr;   c                 ó4   — | j                              ¦   «         S rj   r?  rA  s    r<   r@  z(T5GemmaEncoderModel.get_input_embeddingsò  rB  r;   c                 ó6   — | j                              |¦  «        S rj   rD  rF  s     r<   rE  z(T5GemmaEncoderModel.set_input_embeddingsõ  rH  r;   Nrõ   r�   rÇ   r  rY   r‘   c                 ó*   —  | j         d||||dœ|¤Ž}|S )NrO  r:   )rA   )rX   rõ   r�   rÇ   r  rY   rL  s          r<   rx   zT5GemmaEncoderModel.forwardø  s?   € ð '˜$œ,ð 
ØØ)Ø%Ø'ð	
ð 
ð
 ð
ð 
ˆð Ðr;   r0  )r2   r3   r4   r>   rk   r@  rE  r!   r    r´   rË   rÌ   rµ   r   r   r   rx   ra   rb   s   @r<   rX  rX  ç  s  ø€ € € € € ð˜}ð ð ð ð ð ð ð3ð 3ð 3ðAð Að Að Øð .2Ø37Ø04Ø-1ðð àÔ# dÑ*ðð Ô)¨DÑ0ðð Ô&¨Ñ-ð	ð
 ”| dÑ*ðð Ð+Ô,ðð 
ðð ð ñ „^ñ Ôðð ð ð ð r;   rX  c            "       óä  ‡ — e Zd ZddiZddiZddgdgfiZdefˆ fd„Zd	„ Zd
„ Z	e
e	 	 	 	 	 	 	 	 	 	 	 	 	 d$dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dedz  dej        dz  dej        dz  dej        dz  dedz  deej        z  dee         deej                 ez  fd„¦   «         ¦   «         Zdej        fd„Zdeded ed!ed"edefˆ fd#„Zˆ xZS )%ÚT5GemmaForConditionalGenerationzlm_head.out_proj.weightz!model.decoder.embed_tokens.weightzlm_head.out_projÚcolwise_gather_outputrw   rá   rp   c                 ó   •— d|_         t          ¦   «                              |¦  «         t          |¦  «        | _        |j        j        | _        t          |j        j        | j        ¦  «        | _	        d| _
        |                      ¦   «          d S )NTÚForMaskedLM)rC   rV   rk   r<  rä   rB   rI   rÞ   rR   Úlm_headÚ	loss_typer  ro   s     €r<   rk   z(T5GemmaForConditionalGeneration.__init__  ss   ø€ Ø$(ˆÔ!Ý‰Œ×Ò˜Ñ Ô Ð å! &Ñ)Ô)ˆŒ
Ø œ.Ô3ˆŒÝ$ V¤^Ô%?ÀÄÑQÔQˆŒØ&ˆŒà�ŠÑÔÐÐÐr;   c                 ó¼   — || j         _        | j        j        rC|j        | j        j        j        _        |j        j        d         | j        j        j        _	        d S d S )Nr   )
rb  rÛ   rp   rH   rê   rä   rB   r  r—   Únum_embeddingsrF  s     r<   Úset_output_embeddingsz5T5GemmaForConditionalGeneration.set_output_embeddings  s]   € Ø .ˆŒÔð Œ;Ô*ð 	\Ø5CÔ5JˆDŒJÔÔ+Ô2Ø=KÔ=RÔ=XÐYZÔ=[ˆDŒJÔÔ+Ô:Ð:Ð:ð	\ð 	\r;   c                 ó   — | j         j        S rj   )rb  rÛ   rA  s    r<   Úget_output_embeddingsz5T5GemmaForConditionalGeneration.get_output_embeddings%  s   € ØŒ|Ô$Ð$r;   Nr   rõ   r�   rÇ   rI  rJ  rK  rL  r@   r  rM  ÚlabelsrQ   Úlogits_to_keeprY   r‘   c                 óD  — |�|€|
€|                       |¦  «        } | j        d|||||||||	|
|dœ|¤Ž}|j        }t          |t          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }|                      ¦   «         j        }|j	        �(||j	        z  }t          j        |¦  «        }||j	        z  }d}|� | j        ||| j        fi |¤Ž}t          |||j        |j        |j        |j        |j        |j        |j        ¬¦	  «	        S )aü  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        N)rõ   r�   rÇ   rI  rJ  rK  rL  r@   r  rM  rQ   )	Úlossrá   r@   rQ  rR  r4  rS  r�   rT  r:   )rø   rä   r&  rO   r_   Úslicerb  Úget_decoderrp   Úfinal_logit_softcappingr´   ÚtanhÚloss_functionrI   r   r@   rQ  rR  r4  rS  r�   rT  )rX   rõ   r�   rÇ   rI  rJ  rK  rL  r@   r  rM  ri  rQ   rj  rY   rU  rw   Úslice_indicesrá   Údecoder_configrl  s                        r<   rx   z'T5GemmaForConditionalGeneration.forward(  s€  € ð: ÐÐ"3Ð";Ð@UÐ@]à $× 1Ò 1°&Ñ 9Ô 9Ðà.8¨d¬jð /
ØØ)Ø%Ø/Ø#9Ø!5Ø+Ø+Ø'Ø"7Øð/
ð /
ð ð/
ð /
ˆð (Ô9ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆØ×)Ò)Ñ+Ô+Ô2ˆØÔ1Ð=Ø˜nÔDÑDˆFÝ”Z Ñ'Ô'ˆFØ˜nÔDÑDˆFàˆØÐà%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDåØØØ+Ô;Ø"1Ô"GØ.ÔAØ,Ô=Ø&5Ô&OØ"1Ô"GØ.ÔAð

ñ 

ô 

ð 
	
r;   c                 ó,   — |                       |¦  «        S rj   )rø   )rX   ri  s     r<   Ú%prepare_decoder_input_ids_from_labelszET5GemmaForConditionalGeneration.prepare_decoder_input_ids_from_labelss  s   € Ø× Ò  Ñ(Ô(Ð(r;   Úgeneration_configÚmodel_kwargsÚgeneration_modeÚ
batch_sizeÚmax_cache_lengthc           	      óÀ  •— t          ¦   «                              |||||¦  «         |j        du rdS |j        }|€d}n	d|j        v }t	          j        | j                             d¬¦  «        ¦  «        }d|_        dg|j	        z  |_
        ||dœ}	|                     d¦  «        }
|
�¡t          |
t          ¦  «        st          d	¦  «        ‚t          |
j        ¦  «        d
k    r|
j                             d
¦  «        rdS t#          |
j        ¦  «        }|t&          k    r|d         d
         j        d         |	d<    |di |	¤Ž|
_        nEt          t+          di | j                             d¬¦  «        |dœ¤Žt+          ¦   «         ¦  «        |d<   t-          | d¦  «        r?| j        �:t          | j        t          ¦  «        st          d¦  «        ‚|d         | _        dS dS dS )a7  Override cache preparation to force full attention on the cross-attention cache.

        The decoder config may declare sliding-window layers, but cross-attention must always use full attention.
        The default `_prepare_cache_for_generation` would otherwise build a sliding cross-attention cache.
        FNÚ	offloadedT)rB   r"  )rp   Ú
offloadingr@   z`The `past_key_values` in `model_kwargs` must be of type `EncoderDecoderCache` for T5Gemma model.r   rL  r”   Úmax_cache_lenÚ_cachezKThe internal cache must be of type `EncoderDecoderCache` for T5Gemma model.r:   )rV   Ú_prepare_cache_for_generationrQ   Úcache_implementationÚcopyÚdeepcopyrp   Úget_text_configr†   r  r½   rœ   rO   r
   rˆ   Úlenr›   Útyper�   r   r—   r	   rí   r  )rX   rv  rw  rx  ry  rz  r�  Úoffload_cacheÚcross_attn_configÚcross_attn_cache_kwargsr@   Úcross_attn_clsr[   s               €r<   r€  z=T5GemmaForConditionalGeneration._prepare_cache_for_generationv  sL  ø€ õ 	‰Œ×-Ò-ØØØØØñ	
ô 	
ð 	
ð Ô&¨%Ð/Ð/ØˆFà0ÔEÐØÐ'Ø!ˆMˆMà'Ð+<Ô+QÐQˆMõ !œM¨$¬+×*EÒ*EÈdÐ*EÑ*SÔ*SÑTÔTÐØ+/ÐÔ(Ø)9Ð(:Ð=NÔ=`Ñ(`ÐÔ%ð (Ø'ð#
ð #
Ðð
 '×*Ò*Ð+<Ñ=Ô=ˆØÐ&Ý˜oÕ/BÑCÔCð Ý Øvñô ð õ
 �?Ô-Ñ.Ô.°Ò2Ð2°Ô7Q×7UÒ7UÐVWÑ7XÔ7XÐ2Ø�å! /Ô"GÑHÔHˆNØ¥Ò,Ð,Ø;GÐHYÔ;ZÐ[\Ô;]Ô;cÐdeÔ;fÐ'¨Ñ8à4B°NÐ4]Ð4]ÐE\Ð4]Ð4]ˆOÔ1Ð1õ /BÝð ð à"&¤+×"=Ò"=ÀdÐ"=Ñ"KÔ"KØ&3ðð ðð õ ‘”ñ/ô /ˆLÐ*Ñ+õ �4˜Ñ"Ô"ð 	: t¤{Ð'>Ý˜dœkÕ+>Ñ?Ô?ð pÝ Ð!nÑoÔoÐoà&Ð'8Ô9ˆDŒKˆKˆKð		:ð 	:Ð'>Ð'>r;   )NNNNNNNNNNNNr   ) r2   r3   r4   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr>   rk   rf  rh  r!   r    r´   rË   rÌ   rV  r   r
   r6   r_   rµ   r   r   r¶   r   rx   ru  r   rP   r   r€  ra   rb   s   @r<   r^  r^    s‡  ø€ € € € € Ø3Ð5XÐYÐØ"Ð$;Ð<€HØ" oÐ%6¸¸
Ð$CÐD€Hð	˜}ð 	ð 	ð 	ð 	ð 	ð 	ð\ð \ð \ð%ð %ð %ð Øð .2Ø37Ø04Ø59Ø:>Ø8<Ø26Ø6:Ø26Ø:>Ø*.Ø!%Ø-.ðG
ð G
àÔ# dÑ*ðG
ð Ô)¨DÑ0ðG
ð Ô&¨Ñ-ð	G
ð
 !Ô+¨dÑ2ðG
ð !&Ô 0°4Ñ 7ðG
ð $Ô.°Ñ5ðG
ð )¨4Ñ/ðG
ð -¨tÑ3ðG
ð Ô(¨4Ñ/ðG
ð  %Ô0°4Ñ7ðG
ð Ô  4Ñ'ðG
ð ˜$‘;ðG
ð ˜eœlÑ*ðG
ð Ð+Ô,ðG
ð  
ˆuÔ Ô	! OÑ	3ð!G
ð G
ð G
ñ „^ñ ÔðG
ðR)¸E¼Lð )ð )ð )ð )ðI:à+ðI:ð ðI:ð (ð	I:ð
 ðI:ð ðI:ð 
ðI:ð I:ð I:ð I:ð I:ð I:ð I:ð I:ð I:ð I:r;   r^  c                   óL  ‡ — e Zd Zddededz  fˆ fd„Zd„ Zd„ Zee		 	 	 	 	 	 	 	 	 	 dde
j        dz  de
j        dz  d	e
j        dz  d
e
j        dz  de
j        dz  de
j        dz  dedz  de
j        dz  de
j        dz  de
j        dz  dee         defd„¦   «         ¦   «         Zˆ xZS )Ú T5GemmaForSequenceClassificationNrp   rC   c                 ó’  •— |�||_         t          ¦   «                              |¦  «         |j        | _        |j         rt	          |¦  «        | _        nt          |¦  «        | _        |j        j        }|j         r|j	        j        }t          |dd¦  «        }t          || j        |¦  «        | _        |                      ¦   «          dS )z¬
        is_encoder_decoder (`Optional`, *optional*):
            Whether use encoder_decoder for sequence classification. When set to False, only encoder is used.
        NrF   çš™™™™™¹?©rC   rV   rk   rØ   r<  rä   rX  rA   rR   rB   rU   r×   Úscorer  ©rX   rp   rC   rR   Úclassifier_dropoutr[   s        €r<   rk   z)T5GemmaForSequenceClassification.__init__Ä  s¾   ø€ ð
 Ð)Ø(:ˆFÔ%Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒàÔ$ð 	5Ý% fÑ-Ô-ˆDŒJˆJå,¨VÑ4Ô4ˆDŒJà”nÔ0ˆØÔ$ð 	5Ø œ.Ô4ˆKå$ VÐ-FÈÑLÔLÐÝ.¨{¸D¼OÐM_Ñ`Ô`ˆŒ
Ø�ŠÑÔÐÐÐr;   c                 ó4   — | j                              ¦   «         S rj   ©rä   r@  rA  s    r<   r@  z5T5GemmaForSequenceClassification.get_input_embeddingsÛ  ó   € ØŒz×.Ò.Ñ0Ô0Ð0r;   c                 ó:   — | j                              |¦  «         d S rj   ©rä   rE  ©rX   Úvalues     r<   rE  z5T5GemmaForSequenceClassification.set_input_embeddingsÞ  ó   € ØŒ
×'Ò'¨Ñ.Ô.Ð.Ð.Ð.r;   rõ   r�   rÇ   rI  rJ  rK  rL  r  rM  ri  rY   r‘   c                 ó¸  — | j         j        r!|€|�t          d| j        j        › d�¦  «        ‚| j         j        r*|€(|	€&|€t          d¦  «        ‚|                      |¦  «        }| j         j        r. | j        |f||||||||	ddœ	|¤Ž}|j        }|j	        }|j
        }n' | j        |f|||dœ|¤Ž}|j        }|j        }|j        }|                      |¦  «        }|�|j        d         }n|j        d         }| j         j        €|d	k    rt          d
¦  «        ‚| j         j        €d}nÝ|�²|| j         j        k                         |j        t$          j        ¦  «        }t%          j        |j        d         |j        t$          j        ¬¦  «        }||z                       d¦  «        }| j         j        r)|d	z  }t%          j        ||j        d         d	z
  ¬¦  «        }n)d}t.                               | j        j        › d�¦  «         |t%          j        ||j        ¬¦  «        |f         }d}|
�|                      ||
|| j         ¬¦  «        }t5          ||||¬¦  «        S )áÜ  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nú8Passing input embeddings is currently not supported for ú in encoder-decoder mode.ú°If no `decoder_input_ids` or `decoder_inputs_embeds` are passed, `input_ids` cannot be `None`. Please pass either `input_ids` or `decoder_input_ids` or `decoder_inputs_embeds`.F©	r�   rÇ   rI  rJ  rK  rL  r  rM  rQ   ©r�   rÇ   r  r   r”   z=Cannot handle batch sizes > 1 if no padding token is defined.r“   r   )ÚmaxzŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r   )rá   ri  Úpooled_logitsrp   ©rl  rá   rw   r	  )rp   rC   ÚNotImplementedErrorr[   r2   rˆ   rø   rä   r&  rQ  rR  rw   r	  r“  r—   rM   r  r  r´   Úint32r'  ÚargmaxÚclampÚloggerÚwarning_oncerq  r   )rX   rõ   r�   rÇ   rI  rJ  rK  rL  r  rM  ri  rY   Úoutputsr&  rw   r	  rá   ry  Úlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesr¦  rl  s                          r<   rx   z(T5GemmaForSequenceClassification.forwardá  s  € ð2 Œ;Ô)ð 	¨yÐ/@À]ÐE^Ý%Ø}È4Ì>ÔKbÐ}Ð}Ð}ñô ð ð
 Œ;Ô)ð 	=Ð/@Ð/HÐMbÐMjØÐ Ý ðUñô ð ð
 !%× 1Ò 1°)Ñ <Ô <ÐàŒ;Ô)ð 	,Ø*4¨$¬*Øð+à-Ø)Ø"3Ø'=Ø%9Ø /Ø+Ø&;Øð+ð +ð ð+ð +ˆGð !(Ô 9ÐØ#Ô9ˆMØ Ô3ˆJˆJà'1 t¤zØð(à-Ø)Ø+ð	(ð (ð
 ð(ð (ˆGð !(Ô 9ÐØ#Ô1ˆMØ Ô+ˆJà—’Ð-Ñ.Ô.ˆàÐ Ø"œ¨Ô+ˆJˆJà&Ô,¨QÔ/ˆJàŒ;Ô#Ð+°
¸a²°ÝÐ\Ñ]Ô]Ð]ØŒ;Ô#Ð+Ø!#ÐÐØÐ"à%¨¬Ô)AÒA×EÒEÀfÄmÕUZÔU`ÑaÔaˆLÝ!œL¨¬¸Ô)<ÀVÄ]ÕZ_ÔZeÐfÑfÔfˆMØ"/°,Ñ">×!FÒ!FÀrÑ!JÔ!JÐàŒ{Ô-ð jØ" aÑ'Ð"Ý%*¤[Ð1CÐIZÔI`ÐacÔIdÐghÑIhÐ%iÑ%iÔ%iÐ"øà!#ÐÝ×ÒØ”>Ô*ð Zð Zð Zñô ð ð
 �uœ|¨J¸v¼}ÐMÑMÔMÐOaÐaÔbˆàˆØÐØ×%Ò%¨V¸FÐR_ÐhlÔhsÐ%ÑtÔtˆDå'ØØ Ø'Ø!ð	
ñ 
ô 
ð 	
r;   rj   ©
NNNNNNNNNN)r2   r3   r4   r>   r6   rk   r@  rE  r!   r    r´   rË   rµ   r   rÌ   r   r   r   rx   ra   rb   s   @r<   r�  r�  Â  sž  ø€ € € € € ðð ˜}ð À$ÈÁ+ð ð ð ð ð ð ð.1ð 1ð 1ð/ð /ð /ð Øð .2Ø.2Ø04Ø59Ø6:Ø8<Ø26Ø26Ø:>Ø*.ði
ð i
àÔ# dÑ*ði
ð œ tÑ+ði
ð Ô&¨Ñ-ð	i
ð
 !Ô+¨dÑ2ði
ð !&¤¨tÑ 3ði
ð $Ô.°Ñ5ði
ð )¨4Ñ/ði
ð Ô(¨4Ñ/ði
ð  %Ô0°4Ñ7ði
ð Ô  4Ñ'ði
ð Ð+Ô,ði
ð 
"ði
ð i
ð i
ñ „^ñ Ôði
ð i
ð i
ð i
ð i
r;   r�  c                   óL  ‡ — e Zd Zddededz  fˆ fd„Zd„ Zd„ Zee		 	 	 	 	 	 	 	 	 	 dde
j        dz  de
j        dz  d	e
j        dz  d
e
j        dz  de
j        dz  de
j        dz  dedz  de
j        dz  de
j        dz  de
j        dz  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚT5GemmaForTokenClassificationNrp   rC   c                 ó’  •— |�||_         t          ¦   «                              |¦  «         |j        | _        |j         rt	          |¦  «        | _        nt          |¦  «        | _        |j        j        }|j         r|j	        j        }t          |dd¦  «        }t          || j        |¦  «        | _        |                      ¦   «          dS )z©
        is_encoder_decoder (`Optional`, *optional*):
            Whether use encoder_decoder for token classification. When set to False, only encoder is used.
        NrF   r‘  r’  r”  s        €r<   rk   z&T5GemmaForTokenClassification.__init__Q  s¾   ø€ ð
 Ð)Ø(:ˆFÔ%Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒàÔ$ð 	5Ý% fÑ-Ô-ˆDŒJˆJå,¨VÑ4Ô4ˆDŒJà”nÔ0ˆØÔ$ð 	5Ø œ.Ô4ˆKå$ VÐ-FÈÑLÔLÐÝ.¨{¸D¼OÐM_Ñ`Ô`ˆŒ
à�ŠÑÔÐÐÐr;   c                 ó4   — | j                              ¦   «         S rj   r—  rA  s    r<   r@  z2T5GemmaForTokenClassification.get_input_embeddingsi  r˜  r;   c                 ó:   — | j                              |¦  «         d S rj   rš  r›  s     r<   rE  z2T5GemmaForTokenClassification.set_input_embeddingsl  r�  r;   rõ   r�   rÇ   rI  rJ  rK  rL  r  rM  ri  rY   r‘   c                 ó  — | j         j        r!|€|�t          d| j        j        › d�¦  «        ‚| j         j        r*|€(|	€&|€t          d¦  «        ‚|                      |¦  «        }| j         j        r. | j        |f||||||||	ddœ	|¤Ž}|j        }|j	        }|j
        }n' | j        |f|||dœ|¤Ž}|j        }|j        }|j        }|                      |¦  «        }d}|
�|                      ||
| j         ¦  «        }t          ||||¬¦  «        S )	rŸ  Nr   r¡  r¢  Fr£  r¤  r§  )rp   rC   r¨  r[   r2   rˆ   rø   rä   r&  rQ  rR  rw   r	  r“  rq  r   )rX   rõ   r�   rÇ   rI  rJ  rK  rL  r  rM  ri  rY   r®  r&  rw   r	  rá   rl  s                     r<   rx   z%T5GemmaForTokenClassification.forwardo  s¥  € ð4 Œ;Ô)ð 	¨yÐ/@À]ÐE^Ý%Ø}È4Ì>ÔKbÐ}Ð}Ð}ñô ð ð Œ;Ô)ð 	=Ð/@Ð/HÐMbÐMjØÐ Ý ðUñô ð ð
 !%× 1Ò 1°)Ñ <Ô <ÐàŒ;Ô)ð 	,Ø*4¨$¬*Øð+à-Ø)Ø"3Ø'=Ø%9Ø /Ø+Ø&;Øð+ð +ð ð+ð +ˆGð !(Ô 9ÐØ#Ô9ˆMØ Ô3ˆJˆJà'1 t¤zØð(à-Ø)Ø+ð	(ð (ð
 ð(ð (ˆGð !(Ô 9ÐØ#Ô1ˆMØ Ô+ˆJà—’Ð-Ñ.Ô.ˆàˆØÐØ×%Ò% f¨f°d´kÑBÔBˆDå$ØØØ'Ø!ð	
ñ 
ô 
ð 	
r;   rj   r²  )r2   r3   r4   r>   r6   rk   r@  rE  r!   r    r´   rË   rµ   r   rÌ   r   r   r   rx   ra   rb   s   @r<   r´  r´  O  sž  ø€ € € € € ðð ˜}ð À$ÈÁ+ð ð ð ð ð ð ð01ð 1ð 1ð/ð /ð /ð Øð .2Ø.2Ø04Ø59Ø6:Ø8<Ø26Ø26Ø:>Ø*.ðN
ð N
àÔ# dÑ*ðN
ð œ tÑ+ðN
ð Ô&¨Ñ-ð	N
ð
 !Ô+¨dÑ2ðN
ð !&¤¨tÑ 3ðN
ð $Ô.°Ñ5ðN
ð )¨4Ñ/ðN
ð Ô(¨4Ñ/ðN
ð  %Ô0°4Ñ7ðN
ð Ô  4Ñ'ðN
ð Ð+Ô,ðN
ð 
ðN
ð N
ð N
ñ „^ñ ÔðN
ð N
ð N
ð N
ð N
r;   r´  )r>   r0   r^  r<  rX  rã   r�  r´  )\r‚  Úcollections.abcr   Útypingr   r´   Útorch.nnrl   Úhuggingface_hub.dataclassesr   Ú r   rë   Úcache_utilsr   r	   r
   r   Úconfiguration_utilsr   Ú
generationr   r   r   Úmasking_utilsr   r   r   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r    r!   r"   Úutils.genericr#   Úutils.output_capturingr$   r%   Úgemma2.configuration_gemma2r'   Úgemma2.modeling_gemma2r(   r)   r*   r+   r,   r-   Ú
get_loggerr2   r¬  r0   r>   rd   rh   rz   r|   r‚   r¸   rÎ   ÚModuler×   rÞ   rã   rË   rµ   r_   r  r  r2  r<  rX  r^  r�  r´  Ú__all__r:   r;   r<   ú<module>rÏ     sä  ðð €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ø .Ð .Ð .Ð .Ð .Ð .à &Ð &Ð &Ð &Ð &Ð &Ø PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PØ 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ Kðð ð ð ð ð ð ð ð ð ð ð ð CÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð GÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &ðð ð ð ð ð ð ð ð ð ð ð ð 8Ð 7Ð 7Ð 7Ð 7Ð 7Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ 6Ð 6Ð 6Ð 6Ð 6Ð 6ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 
ˆÔ	˜HÑ	%Ô	%€ð €Ð6Ð7Ñ7Ô7Øð3ð 3ð 3ð 3ð 3˜,ñ 3ô 3ñ „ñ 8Ô7ð3ð. €Ð6Ð7Ñ7Ô7Øð7(ð 7(ð 7(ð 7(ð 7(Ð$ñ 7(ô 7(ñ „ñ 8Ô7ð7(ðt	ð 	ð 	ð 	ð 	�]ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	�ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð2ñ 	ô 	ð 	ð+ð +ð +ð +ð +˜?ñ +ô +ð +ðD)ð D)ð D)ð D)ð D)˜Oñ D)ô D)ð D)ðN1ð 1ð 1ð 1ð 1Ð4ñ 1ô 1ð 1ðhFð Fð Fð Fð FÐ4ñ Fô Fð FðRð ð ð ð  ¤	ñ ô ð ð	ð 	ð 	ð 	ð 	�B”Iñ 	ô 	ð 	ð ð2!ð 2!ð 2!ð 2!ð 2!Ð2ñ 2!ô 2!ñ „ð2!ðjØÔ $Ñ&ðà”<ðð ˜‘*ðð „\ð	ð ð ð ð"P
ð P
ð P
ð P
ð P
Ð+ñ P
ô P
ð P
ðfk
ð k
ð k
ð k
ð k
Ð+ñ k
ô k
ð k
ð\ ðJ
ð J
ð J
ð J
ð J
Ð)ñ J
ô J
ñ „ðJ
ðZ ð!ð !ð !ð !ð !Ð0ñ !ô !ñ „ð!ðHs:ð s:ð s:ð s:ð s:Ð&<¸oñ s:ô s:ð s:ðl ðI
ð I
ð I
ð I
ð I
Ð'=ñ I
ô I
ñ „ðI
ðX ðo
ð o
ð o
ð o
ð o
Ð$:ñ o
ô o
ñ „ðo
ðd	ð 	ð 	€€€r;   