§
    ‚Štjsí  ã                   ón  — d dl Z d dlmZ d dlmZ d dlZd dlmZ ddlm	Z
 ddlmZ ddlmZmZmZmZ ddlmZmZmZ dd	lmZmZ dd
lmZmZmZmZ ddlmZ ddl m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( ddl)m*Z*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0 ddl1m2Z2m3Z3m4Z4m5Z5 ddl6m7Z7m8Z8 ddl9m:Z:m;Z; ddl<m=Z=m>Z>  e5j?        e@¦  «        ZA G d„ dejB        ¦  «        ZC G d„ dejB        ¦  «        ZD G d„ dejB        ¦  «        ZEd„ ZF ed¦  «        dMd„¦   «         ZGdejH        d eId!ejH        fd"„ZJ	 	 	 dNd$ejB        d%ejH        d&ejH        d'ejH        d(ejH        dz  d)eKeIz  d*eKdz  d+eKdz  d!eLejH        ejH        f         fd,„ZM eeG¦  «         G d-„ d.ejB        ¦  «        ¦   «         ZN eeG¦  «         G d/„ d0ejB        ¦  «        ¦   «         ZO G d1„ d2e!¦  «        ZP G d3„ d4e!¦  «        ZQ G d5„ d6ejB        ¦  «        ZR G d7„ d8ejB        ¦  «        ZSe3 G d9„ d:e.¦  «        ¦   «         ZTd;ejU        dz  dejH        d<eIdz  d!ejH        fd=„ZV G d>„ d?eT¦  «        ZW G d@„ dAeT¦  «        ZXe3 G dB„ dCeT¦  «        ¦   «         ZYe3 G dD„ dEeT¦  «        ¦   «         ZZ G dF„ dGeTe¦  «        Z[e3 G dH„ dIeT¦  «        ¦   «         Z\e3 G dJ„ dKeT¦  «        ¦   «         Z]g dL¢Z^dS )Oé    N)ÚCallable)ÚOptionalé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCacheÚStaticCache)ÚGenerationConfigÚGenerationMixinÚGenerationMode)Úuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚ(create_bidirectional_sliding_window_maskÚcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚT5GemmaConfigÚT5GemmaModuleConfigc                   ó<   ‡ — e Zd Zddedefˆ fd„Zd„ Zd„ Zd„ Zˆ xZ	S )	ÚT5GemmaRMSNormç�íµ ÷Æ°>ÚdimÚepsc                 ó¬   •— t          ¦   «                              ¦   «          || _        t          j        t          j        |¦  «        ¦  «        | _        d S ©N)ÚsuperÚ__init__r1   ÚnnÚ	ParameterÚtorchÚzerosÚweight)Úselfr0   r1   Ú	__class__s      €új/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/t5gemma/modeling_t5gemma.pyr5   zT5GemmaRMSNorm.__init__>   s?   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”l¥5¤;¨sÑ#3Ô#3Ñ4Ô4ˆŒˆˆó    c                 ó�   — |t          j        |                     d¦  «                             dd¬¦  «        | j        z   ¦  «        z  S )Né   éÿÿÿÿT)Úkeepdim)r8   ÚrsqrtÚpowÚmeanr1   )r;   Úxs     r=   Ú_normzT5GemmaRMSNorm._normC   s8   € Ø•5”;˜qŸušu Q™xœxŸ}š}¨R¸˜}Ñ>Ô>ÀÄÑIÑJÔJÑJÐJr>   c                 ó¸   — |                       |                     ¦   «         ¦  «        }|d| j                             ¦   «         z   z  }|                     |¦  «        S )Nç      ð?)rG   Úfloatr:   Útype_as)r;   rF   Úoutputs      r=   ÚforwardzT5GemmaRMSNorm.forwardF   sL   € Ø—’˜AŸGšG™IœIÑ&Ô&ˆð ˜3 ¤×!2Ò!2Ñ!4Ô!4Ñ4Ñ5ˆØ�~Š~˜aÑ Ô Ð r>   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)Útupler:   Úshaper1   ©r;   s    r=   Ú
extra_reprzT5GemmaRMSNorm.extra_reprM   s%   € Ý˜œÔ)Ñ*Ô*Ð<Ð<°$´(Ð<Ð<Ð<r>   )r/   )
Ú__name__Ú
__module__Ú__qualname__ÚintrJ   r5   rG   rM   rR   Ú__classcell__©r<   s   @r=   r.   r.   =   s€   ø€ € € € € ð5ð 5˜Cð 5 eð 5ð 5ð 5ð 5ð 5ð 5ð
Kð Kð Kð!ð !ð !ð=ð =ð =ð =ð =ð =ð =r>   r.   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú
T5GemmaMLPc                 óÔ  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _	        t          |j                 | _        t          j        |j        ¦  «        | _        d S )NF©Úbias)r4   r5   ÚconfigÚhidden_sizeÚintermediate_sizer6   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Úhidden_activationÚact_fnÚDropoutÚdropout_rateÚdropout©r;   r^   r<   s     €r=   r5   zT5GemmaMLP.__init__R   sº   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔÝœ 4Ô#3°TÔ5KÐRWÐXÑXÔXˆŒÝ”y Ô!1°4Ô3IÐPUÐVÑVÔVˆŒÝœ 4Ô#9¸4Ô;KÐRWÐXÑXÔXˆŒÝ˜VÔ5Ô6ˆŒÝ”z &Ô"5Ñ6Ô6ˆŒˆˆr>   c                 óÖ   — |                       |                      |¦  «        ¦  «        |                      |¦  «        z  }|                      |¦  «        }|                      |¦  «        }|S r3   )rf   rb   rc   ri   rd   )r;   rF   Úhidden_statesrd   s       r=   rM   zT5GemmaMLP.forward]   sV   € ØŸš D§N¢N°1Ñ$5Ô$5Ñ6Ô6¸¿ºÀa¹¼ÑHˆØŸš ]Ñ3Ô3ˆØ—N’N =Ñ1Ô1ˆ	ØÐr>   )rS   rT   rU   r5   rM   rW   rX   s   @r=   rZ   rZ   Q   sG   ø€ € € € € ð	7ð 	7ð 	7ð 	7ð 	7ðð ð ð ð ð ð r>   rZ   c                   óÔ   ‡ — e Zd ZU ej        ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d         de
dz  ded	ef         fd
„¦   «         Z ej        ¦   «         ed„ ¦   «         ¦   «         Zˆ xZS )ÚT5GemmaRotaryEmbeddingÚinv_freqNr^   c                 ó²  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        |                      d|d¬¦  «         |                      d|                     ¦   «         d¬¦  «         d S )NÚ	rope_typeÚdefaultro   F)Ú
persistentÚoriginal_inv_freq)r4   r5   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr^   Úrope_parametersrq   Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)r;   r^   ÚdeviceÚrope_init_fnro   r<   s        €r=   r5   zT5GemmaRotaryEmbedding.__init__g   sÊ   ø€ Ý‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(à×Ò˜Z¨¸eÐÑDÔDÐDØ×ÒÐ0°(·.².Ñ2BÔ2BÈuÐÑUÔUÐUÐUÐUr>   r}   ztorch.deviceÚseq_lenÚreturnztorch.Tensorc                 óü   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «                             |t          j        ¬¦  «        |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNrI   r   r@   ©Údtype©r}   r…   )	rx   Úgetattrr_   Únum_attention_headsr8   ÚarangeÚint64ÚtorJ   )r^   r}   r   Úbaser0   Úattention_factorro   s          r=   ry   z6T5GemmaRotaryEmbedding.compute_default_rope_parametersw   sŒ   € ð& Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐð Ø•U”\ ! S¨!µ5´;Ð?Ñ?Ô?×BÒBÈ&ÕX]ÔXcÐBÑdÔdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r>   c                 óN  — | j         d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  |                     ¦   «         |                     ¦   «         z   
                    dd¦  «        }t          j        ||fd¬	¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬
¦  «        |	                     |j        ¬
¦  «        fS )Nr   rA   r*   ÚmpsÚcpuF)Údevice_typeÚenabledr@   ©r0   r„   )ro   rJ   ÚexpandrP   r‹   r}   Ú
isinstanceÚtypeÚstrr&   Ú	transposer8   ÚcatÚcosrz   Úsinr…   )
r;   rF   Úposition_idsÚinv_freq_expandedÚposition_ids_expandedr‘   ÚfreqsÚembrš   r›   s
             r=   rM   zT5GemmaRotaryEmbedding.forward•   s·  € ð !œM¨$°°°°4¨-Ô8×>Ò>Ñ@Ô@×GÒGÈÔHZÐ[\ÔH]Ð_aÐcdÑeÔe×hÒhÐijÔiqÑrÔrÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	5ð 	5Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃBE&Å&E*Å-E*r3   ©NNN)rS   rT   rU   r8   ÚTensorÚ__annotations__r+   r5   Ústaticmethodr   rV   rO   rJ   ry   Úno_gradr   rM   rW   rX   s   @r=   rn   rn   d   sù   ø€ € € € € € ØŒlÐÐÑðVð V˜}ð Vð Vð Vð Vð Vð Vð  à'+Ø+/Ø"ð*ð *Ø Ñ$ð*à˜Ô(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ô	%ð	*ð *ð *ñ „\ð*ð: €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r>   rn   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..NrA   r@   r“   )rP   r8   r™   )rF   Úx1Úx2s      r=   Úrotate_halfr©   ¥   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r>   Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer©   )ÚqÚkrš   r›   Úunsqueeze_dimÚq_embedÚk_embeds          r=   Úapply_rotary_pos_embr²   ¬   sc   € ð& �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr>   rl   Ún_repr€   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r*   N)rP   r”   Úreshape)rl   r³   ÚbatchÚnum_key_value_headsÚslenrƒ   s         r=   Ú	repeat_kvr¹   Æ   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr>   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskri   ÚscalingÚsoftcapc                 ól  — |€
| j         dz  }t          || j        ¦  «        }	t          || j        ¦  «        }
t          j        ||	                     dd¦  «        ¦  «        |z  }|�||z  }t          j        |¦  «        }||z  }|�||z   }t          j         	                    |dt          j
        ¬¦  «                             |j        ¦  «        }t          j                             ||| j        ¬¦  «        }t          j        ||
¦  «        }|                     dd¦  «                             ¦   «         }||fS )Nç      à¿r@   r   rA   )r0   r…   )ÚpÚtrainingr*   )rƒ   r¹   Únum_key_value_groupsr8   Úmatmulr˜   Útanhr6   Ú
functionalÚsoftmaxÚfloat32r‹   r…   ri   rÅ   Ú
contiguous)r»   r¼   r½   r¾   r¿   ri   rÀ   rÁ   ÚkwargsÚ
key_statesÚvalue_statesÚattn_weightsÚattn_outputs                r=   Úeager_attention_forwardrÒ   Ò   s%  € ð €Ø”/ 4Ñ'ˆå˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LàÐØ# gÑ-ˆÝ”z ,Ñ/Ô/ˆØ# gÑ-ˆØÐ!Ø# nÑ4ˆõ ”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€KØ˜Ð$Ð$r>   c                   óò   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej        de	ej        ej        f         dz  dej        dz  d	e
dz  d
ee         de	ej        ej        dz  e	ej                 dz  f         fd„Zˆ xZS )ÚT5GemmaSelfAttentionú=Multi-headed attention from 'Attention Is All You Need' paperr^   Ú	layer_idxc                 ó^  •— t          ¦   «                              ¦   «          t          |d¦  «        r|j        |         nd | _        || _        || _        t          |d|j        |j	        z  ¦  «        | _
        |j	        |j        z  | _        |j        dz  | _        | j        j        | _        |j        | _        t%          j        |j        |j	        | j
        z  |j        ¬¦  «        | _        t%          j        |j        |j        | j
        z  |j        ¬¦  «        | _        t%          j        |j        |j        | j
        z  |j        ¬¦  «        | _        t%          j        |j	        | j
        z  |j        |j        ¬¦  «        | _        | j        j        | _        | j        dk    r|j        nd | _        d S )NÚlayer_typesrƒ   rÃ   r\   Úsliding_attention)r4   r5   ÚhasattrrØ   Ú
layer_typer^   rÖ   r‡   r_   rˆ   rƒ   r·   rÆ   Úquery_pre_attn_scalarrÀ   Úattention_dropoutÚ
is_decoderÚ	is_causalr6   ra   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚattn_logit_softcappingÚsliding_window©r;   r^   rÖ   r<   s      €r=   r5   zT5GemmaSelfAttention.__init__ø   s›  ø€ Ý‰Œ×ÒÑÔÐÝ;BÀ6È=Ñ;YÔ;YÐc˜&Ô,¨YÔ7Ð7Ð_cˆŒØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!ØÔ3°TÑ9ˆŒØ!%¤Ô!>ˆÔàÔ*ˆŒå”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒð '+¤kÔ&HˆÔ#Ø7;´ÐJ]Ò7]Ð7]˜fÔ3Ð3ÐcgˆÔÐÐr>   Nrl   Úposition_embeddingsr¿   Úpast_key_valuesrÍ   r€   c                 ó:  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|\  }}t          ||	||¦  «        \  }}	|�|                     |	|
| j	        ¦  «        \  }	}
t          j        | j        j        t          ¦  «        } || ||	|
|f| j        r| j        nd| j        | j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )NrA   r*   r@   rº   ©ri   rÀ   ræ   rÁ   )rP   rƒ   rá   Úviewr˜   râ   rã   r²   ÚupdaterÖ   r   Úget_interfacer^   Ú_attn_implementationrÒ   rÅ   rÝ   rÀ   ræ   rå   rµ   rÌ   rä   )r;   rl   rè   r¿   ré   rÍ   Úinput_shapeÚhidden_shapeÚquery_statesrÎ   rÏ   rš   r›   Úattention_interfacerÑ   rÐ   s                   r=   rM   zT5GemmaSelfAttention.forward  sÍ  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð%
ð /3¬mÐD�DÔ*Ð*ÀØ”LØÔ.ØÔ/ð%
ð %
ð ð%
ð %
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r>   r¡   )rS   rT   rU   Ú__doc__r,   rV   r5   r8   r¢   rO   r   r!   r   rM   rW   rX   s   @r=   rÔ   rÔ   ô   s  ø€ € € € € àGÐGðhÐ2ð h¸sð hð hð hð hð hð hð< IMØ.2Ø(,ð()ð ()à”|ð()ð # 5¤<°´Ð#=Ô>ÀÑEð()ð œ tÑ+ð	()ð
  ™ð()ð Ð-Ô.ð()ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð()ð ()ð ()ð ()ð ()ð ()ð ()ð ()r>   rÔ   c                   óÔ   ‡ — e Zd ZdZdedefˆ fd„Z	 ddej        dej        dz  dej        dz  d	e	dz  d
e
e         deej        ej        dz  eej                 dz  f         fd„Zˆ xZS )ÚT5GemmaCrossAttentionrÕ   r^   rÖ   c                 ó  •— t          ¦   «                              ¦   «          || _        || _        t	          |d|j        |j        z  ¦  «        | _        |j        |j        z  | _	        |j
        dz  | _        | j        j        | _        d| _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        | j        z  |j        |j        ¬¦  «        | _        | j        j        | _        |j        €t/          d¦  «        ‚d S )Nrƒ   rÃ   Fr\   zBCross-attention needs cross_attention_hidden_size to be specified.)r4   r5   r^   rÖ   r‡   r_   rˆ   rƒ   r·   rÆ   rÜ   rÀ   rÝ   rß   r6   ra   rà   rá   Úcross_attention_hidden_sizerâ   rã   rä   rå   Ú
ValueErrorrç   s      €r=   r5   zT5GemmaCrossAttention.__init__B  sp  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!ØÔ3°TÑ9ˆŒØ!%¤Ô!>ˆÔØˆŒå”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ.°Ô0JÈTÌ]Ñ0ZÐagÔavð
ñ 
ô 
ˆŒõ ”iØÔ.°Ô0JÈTÌ]Ñ0ZÐagÔavð
ñ 
ô 
ˆŒõ ”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒð '+¤kÔ&HˆÔ#àÔ-Ð5ÝÐaÑbÔbÐbð 6Ð5r>   Nrl   r¿   Úencoder_hidden_statesré   rÍ   r€   c                 ó,  — |€t          d¦  «        ‚|j        d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|�&|j                             | j        ¦  «        }	|j	        }
|�|	sÆ|j        d d…         }g |¢d‘| j        ‘R }|  
                    |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|�.|
                     ||| j        ¦  «        \  }}d|j        | j        <   n.|
j        | j                 j        }|
j        | j                 j        }t!          j        | j        j        t(          ¦  «        } || ||||f| j        r| j        nd| j        d | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nz5Encoder hidden state is required for cross attention.rA   r*   r@   Trº   rë   )rù   rP   rƒ   rá   rì   r˜   Ú
is_updatedÚgetrÖ   Úcross_attention_cacherâ   rã   rí   ÚlayersÚkeysÚvaluesr   rî   r^   rï   rÒ   rÅ   rÝ   rÀ   rå   rµ   rÌ   rä   )r;   rl   r¿   rú   ré   rÍ   rð   rñ   rò   rü   Úcurr_past_key_valuesÚencoder_input_shapeÚencoder_hidden_shaperÎ   rÏ   ró   rÑ   rÐ   s                     r=   rM   zT5GemmaCrossAttention.forward^  sV  € ð !Ð(ÝÐTÑUÔUÐUà#Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆØ—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆàÐ&Ø(Ô3×7Ò7¸¼ÑGÔGˆJØ#2Ô#HÐ àÐ"¨*Ð"Ø"7Ô"=¸c¸r¸cÔ"BÐØ#LÐ%8Ð#L¸"Ð#L¸d¼mÐ#LÐ#LÐ ØŸšÐ%:Ñ;Ô;×@Ò@ÐAUÑVÔV×`Ò`ÐabÐdeÑfÔfˆJØŸ;š;Ð'<Ñ=Ô=×BÒBÐCWÑXÔX×bÒbÐcdÐfgÑhÔhˆLàÐ*Ø+?×+FÒ+FÀzÐS_ÐaeÔaoÑ+pÔ+pÑ(�
˜LØ=A�Ô*¨4¬>Ñ:øà-Ô4°T´^ÔDÔIˆJØ/Ô6°t´~ÔFÔMˆLå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð%
ð /3¬mÐD�DÔ*Ð*ÀØ”LØØÔ/ð%
ð %
ð ð%
ð %
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r>   r3   )rS   rT   rU   rô   r,   rV   r5   r8   r¢   r   r!   r   rO   rM   rW   rX   s   @r=   rö   rö   >  sî   ø€ € € € € àGÐGðcÐ2ð c¸sð cð cð cð cð cð cðB )-ð3)ð 3)à”|ð3)ð œ tÑ+ð3)ð  %œ|¨dÑ2ð	3)ð
  ™ð3)ð Ð-Ô.ð3)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð3)ð 3)ð 3)ð 3)ð 3)ð 3)ð 3)ð 3)r>   rö   c                   ó¸   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 ddej        deej        ej        f         dz  dej        dz  dej	        dz  d	eej
        f         f
d
„Zˆ xZS )ÚT5GemmaEncoderLayerzEncoder sub-layer.rÖ   c                 ó0  •— t          ¦   «                              ¦   «          |j        | _        || _        || _        |j        |         | _        t          ||¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        t          |¦  «        | _        t          |j        |j
        ¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        t#          j        |j        ¦  «        | _        d S ©N)r^   rÖ   ©r1   )r4   r5   r_   r^   rÖ   rØ   Úattention_typerÔ   Ú	self_attnr.   Úrms_norm_epsÚpre_self_attn_layernormÚpost_self_attn_layernormrZ   ÚmlpÚpre_feedforward_layernormÚpost_feedforward_layernormr6   rg   rh   ri   rç   s      €r=   r5   zT5GemmaEncoderLayer.__init__—  sõ   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØˆŒØ"ˆŒØ$Ô0°Ô;ˆÔå-ØØð
ñ 
ô 
ˆŒõ (6°fÔ6HÈfÔNaÐ'bÑ'bÔ'bˆÔ$Ý(6°vÔ7IÈvÔObÐ(cÑ(cÔ(cˆÔ%å˜fÑ%Ô%ˆŒÝ)7¸Ô8JÐPVÔPcÐ)dÑ)dÔ)dˆÔ&Ý*8¸Ô9KÐQWÔQdÐ*eÑ*eÔ*eˆÔ'å”z &Ô"5Ñ6Ô6ˆŒˆˆr>   Nrl   rè   r¿   rœ   r€   c           	      ól  — |}|                       |¦  «        } | j        d||||d dœ|¤Ž\  }}|                      |¦  «        }||                      |¦  «        z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||                      |¦  «        z   }|S )N)rl   rè   r¿   rœ   ré   © )r  r  r  ri   r  r  r  )r;   rl   rè   r¿   rœ   rÍ   ÚresidualÚ_s           r=   rM   zT5GemmaEncoderLayer.forward«  sÚ   € ð !ˆØ×4Ò4°]ÑCÔCˆØ)˜4œ>ð 
Ø'Ø 3Ø)Ø%Ø ð
ð 
ð ð
ð 
Ñˆ�qð ×5Ò5°mÑDÔDˆØ  4§<¢<°Ñ#>Ô#>Ñ>ˆà ˆØ×6Ò6°}ÑEÔEˆØŸš Ñ/Ô/ˆØ×7Ò7¸ÑFÔFˆØ  4§<¢<°Ñ#>Ô#>Ñ>ˆØÐr>   r¡   )rS   rT   rU   rô   rV   r5   r8   r¢   rO   Ú
LongTensorÚFloatTensorrM   rW   rX   s   @r=   r  r  ”  sÏ   ø€ € € € € ØÐð7¨#ð 7ð 7ð 7ð 7ð 7ð 7ð. IMØ.2Ø04ðð à”|ðð # 5¤<°´Ð#=Ô>ÀÑEðð œ tÑ+ð	ð
 Ô&¨Ñ-ðð 
ˆuÔ Ð!Ô	"ðð ð ð ð ð ð ð r>   r  c                   óî   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 	 	 	 	 ddej        deej        ej        f         dz  dej        dz  d	ej	        dz  d
e
dz  dedz  dej        dz  dej        dz  dej        fd„Zˆ xZS )ÚT5GemmaDecoderLayerz2Decoder sub-layer: an extra cross-attention layer.rÖ   c                 óÜ  •— t          ¦   «                              ¦   «          |j        | _        || _        || _        |j        |         | _        t          ||¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        t          |¦  «        | _        t          |j        |j
        ¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        t#          j        |j        ¦  «        | _        t+          ||¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        t          |j        |j
        ¬¦  «        | _        d S r  )r4   r5   r_   r^   rÖ   rØ   r
  rÔ   r  r.   r  r  r  rZ   r  r  r  r6   rg   rh   ri   rö   Ú
cross_attnÚpre_cross_attn_layernormÚpost_cross_attn_layernormrç   s      €r=   r5   zT5GemmaDecoderLayer.__init__Ë  sB  ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØˆŒØ"ˆŒØ$Ô0°Ô;ˆÔå-ØØð
ñ 
ô 
ˆŒõ (6°fÔ6HÈfÔNaÐ'bÑ'bÔ'bˆÔ$Ý(6°vÔ7IÈvÔObÐ(cÑ(cÔ(cˆÔ%å˜fÑ%Ô%ˆŒÝ)7¸Ô8JÐPVÔPcÐ)dÑ)dÔ)dˆÔ&Ý*8¸Ô9KÐQWÔQdÐ*eÑ*eÔ*eˆÔ'å”z &Ô"5Ñ6Ô6ˆŒÝ/°vÈÐSÑSÔSˆŒÝ(6°vÔ7IÈvÔObÐ(cÑ(cÔ(cˆÔ%Ý)7¸Ô8JÐPVÔPcÐ)dÑ)dÔ)dˆÔ&Ð&Ð&r>   NFrl   rè   r¿   rœ   ré   Ú	use_cacherú   Úencoder_attention_maskr€   c	           
      ó4  — |}
|                       |¦  «        } | j        d|||||�|j        nd |dœ|	¤Ž\  }}|                      |¦  «        }|
|                      |¦  «        z   }|}
|                      |¦  «        } | j        d|||||dœ|	¤Ž\  }}|                      |¦  «        }|
|                      |¦  «        z   }|}
|                      |¦  «        }|  	                    |¦  «        }|  
                    |¦  «        }|
|                      |¦  «        z   }|S )N)rl   rè   r¿   rœ   ré   r  )rl   rú   r¿   ré   r  r  )r  r  Úself_attention_cacher  ri   r  r  r  r  r  r  )r;   rl   rè   r¿   rœ   ré   r  rú   r  rÍ   r  r  s               r=   rM   zT5GemmaDecoderLayer.forwardâ  si  € ð !ˆØ×4Ò4°]ÑCÔCˆØ)˜4œ>ð 
Ø'Ø 3Ø)Ø%ØDSÐD_˜OÔ@Ð@ÐeiØð
ð 
ð ð
ð 
Ñˆ�qð ×5Ò5°mÑDÔDˆØ  4§<¢<°Ñ#>Ô#>Ñ>ˆà ˆØ×5Ò5°mÑDÔDˆØ*˜4œ?ð 
Ø'Ø"7Ø1Ø+Øð
ð 
ð ð
ð 
Ñˆ�qð ×6Ò6°}ÑEÔEˆØ  4§<¢<°Ñ#>Ô#>Ñ>ˆà ˆØ×6Ò6°}ÑEÔEˆØŸš Ñ/Ô/ˆØ×7Ò7¸ÑFÔFˆØ  4§<¢<°Ñ#>Ô#>Ñ>ˆØÐr>   )NNNNFNN)rS   rT   rU   rô   rV   r5   r8   r¢   rO   r  r
   Úboolr  rM   rW   rX   s   @r=   r  r  È  s  ø€ € € € € Ø<Ð<ðe¨#ð eð eð eð eð eð eð4 IMØ.2Ø04Ø6:Ø!&Ø59Ø6:ð,ð ,à”|ð,ð # 5¤<°´Ð#=Ô>ÀÑEð,ð œ tÑ+ð	,ð
 Ô&¨Ñ-ð,ð -¨tÑ3ð,ð ˜$‘;ð,ð  %œ|¨dÑ2ð,ð !&¤¨tÑ 3ð,ð 
Ô	ð,ð ,ð ,ð ,ð ,ð ,ð ,ð ,r>   r  c                   óV   ‡ — e Zd ZdZd
dededefˆ fd„Zdej        dej        fd	„Z	ˆ xZ
S )ÚT5GemmaClassificationHeadz-Head for sentence-level classification tasks.rº   r_   Ú
num_labelsÚclassifier_dropout_ratec                 ó°   •— t          ¦   «                              ¦   «          t          j        |¬¦  «        | _        t          j        ||¦  «        | _        d S )N)rÄ   )r4   r5   r6   rg   ri   ra   Úout_proj)r;   r_   r%  r&  r<   s       €r=   r5   z"T5GemmaClassificationHead.__init__  sE   ø€ Ý‰Œ×ÒÑÔÐÝ”zÐ$;Ð<Ñ<Ô<ˆŒÝœ	 +¨zÑ:Ô:ˆŒˆˆr>   rl   r€   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r3   )ri   r(  )r;   rl   s     r=   rM   z!T5GemmaClassificationHead.forward  s*   € ØŸš ]Ñ3Ô3ˆØŸš mÑ4Ô4ˆØÐr>   )rº   )rS   rT   rU   rô   rV   rJ   r5   r8   r¢   rM   rW   rX   s   @r=   r$  r$    s„   ø€ € € € € Ø7Ð7ð;ð ; Cð ;°Sð ;ÐSXð ;ð ;ð ;ð ;ð ;ð ;ð
 U¤\ð °e´lð ð ð ð ð ð ð ð r>   r$  c                   óV   ‡ — e Zd ZdZd
dededefˆ fd„Zdej        dej        fd	„Z	ˆ xZ
S )ÚT5GemmaLMHeadz.Head for language modeling (generation) tasks.Fr_   Ú
vocab_sizer]   c                 ó€   •— t          ¦   «                              ¦   «          t          j        |||¬¦  «        | _        d S )Nr\   )r4   r5   r6   ra   r(  )r;   r_   r,  r]   r<   s       €r=   r5   zT5GemmaLMHead.__init__"  s5   ø€ Ý‰Œ×ÒÑÔÐÝœ	 +¨zÀÐEÑEÔEˆŒˆˆr>   rl   r€   c                 ó0   — |                       |¦  «        }|S r3   )r(  )r;   rl   Úlogitss      r=   rM   zT5GemmaLMHead.forward&  s   € Ø—’˜}Ñ-Ô-ˆØˆr>   )F)rS   rT   rU   rô   rV   r"  r5   r8   r¢   rM   rW   rX   s   @r=   r+  r+    s�   ø€ € € € € Ø8Ð8ðFð F Cð F°Sð FÀð Fð Fð Fð Fð Fð Fð U¤\ð °e´lð ð ð ð ð ð ð ð r>   r+  c                   óˆ   ‡ — e Zd ZU eed<   dZdZddgZdgZdZ	dZ
dZdZdZdZ ej        ¦   «         ˆ fd„¦   «         Zd	„ Zˆ xZS )
ÚT5GemmaPreTrainedModelr^   ÚmodelTr  r  ré   Nc                 óª  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          ¦  «        rƒ|j        j        j        d         dz  }t          j
        |j        j        d||z  ¬¦  «         t          |j        d¦  «        r,|j        j        �"t          j        |j        j        ¦  «         d S d S d S t	          |t          ¦  «        rN| j        j        s@|j        j        j        d         dz  }t          j
        |j        j        d||z  ¬¦  «         d S d S d|j        j        v rt          j        |j        ¦  «         d S d S )Nr   rÃ   rº   )rE   Ústdr]   ÚRMSNorm)r4   Ú_init_weightsr^   Úinitializer_ranger•   r$  r(  r:   rP   ÚinitÚnormal_rÚ   r]   Úzeros_r+  Útie_word_embeddingsr<   rS   )r;   r»   r4  Úscaler<   s       €r=   r6  z$T5GemmaPreTrainedModel._init_weights;  sV  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%ØŒkÔ+ˆÝ�fÕ7Ñ8Ô8ð 	'Ø”OÔ*Ô0°Ô3°tÑ;ˆEÝŒL˜œÔ/°c¸sÀU¹{ÐKÑKÔKÐKÝ�v”¨Ñ/Ô/ð 2°F´OÔ4HÐ4TÝ”˜FœOÔ0Ñ1Ô1Ð1Ð1Ð1ð2ð 2Ð4TÐ4Tå˜¥Ñ.Ô.ð 	'Ø”;Ô2ð PØœÔ.Ô4°QÔ7¸4Ñ?�Ý”˜Vœ_Ô3¸#À3ÈÁ;ÐOÑOÔOÐOÐOÐOðPð Pð ˜&Ô*Ô3Ð3Ð3ÝŒK˜œÑ&Ô&Ð&Ð&Ð&ð 4Ð3r>   c                 óJ  — | j         j        j        }| j         j        j        }|€t	          d¦  «        ‚|                     |j        ¦  «        }|ddd…f                              ¦   «         |ddd…f<   ||d<   |€t	          d¦  «        ‚|                     |dk    |¦  «         |S )	zú
        Shifts input_ids to the right, prepends the decoder_start_token_id, and handles
        pad_token_id replacement for labels that were -100.
        This is a common preparation step for decoder inputs in sequence-to-sequence models.
        Nz:self.model.config.decoder.bos_token_id has to be defined. .rA   r*   ).r   z9self.model.config.decoder.pad_token_id has to be defined.iœÿÿÿ)	r^   ÚdecoderÚbos_token_idÚpad_token_idrù   Ú	new_zerosrP   r|   Úmasked_fill_)r;   Ú	input_idsÚdecoder_start_token_idr@  Úshifted_input_idss        r=   Ú_shift_rightz#T5GemmaPreTrainedModel._shift_rightM  s¼   € ð "&¤Ô!4Ô!AÐØ”{Ô*Ô7ˆà!Ð)ÝÐYÑZÔZÐZð &×/Ò/°	´Ñ@Ô@ÐØ%.¨s°C°R°C¨xÔ%8×%>Ò%>Ñ%@Ô%@Ð˜#˜q˜r˜r˜'Ñ"Ø$:Ð˜&Ñ!àÐÝÐXÑYÔYÐYð 	×&Ò&Ð'8¸DÒ'@À,ÑOÔOÐOà Ð r>   )rS   rT   rU   r+   r£   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendÚ_can_record_outputsr8   r¥   r6  rF  rW   rX   s   @r=   r1  r1  +  s«   ø€ € € € € € àÐÐÑØÐØ&*Ð#Ø.Ð0EÐFÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&ÐàÐà€U„]�_„_ð'ð 'ð 'ð 'ñ „_ð'ð"!ð !ð !ð !ð !ð !ð !r>   r1  Ú	token_idsr@  c                 óü   — | �;|€t          d¦  «        ‚| |k                         |j        t          j        ¦  «        }n>t          j        |j        d         |j        d         f|j        t          j        ¬¦  «        }|S )z%Construct the default attention mask.Nz3`pad_token_id` is required for padding information.r   r*   r†   )rù   r‹   r}   r8   ÚlongÚonesrP   )rQ  rl   r@  r¿   s       r=   Úmake_default_2d_attention_maskrU  h  s�   € ð ÐØÐÝÐRÑSÔSÐSØ# |Ò3×7Ò7¸Ô8LÍeÌjÑYÔYˆˆåœØÔ  Ô# ]Ô%8¸Ô%;Ð<À]ÔEYÕafÔakð
ñ 
ô 
ˆð Ðr>   c                   óÄ   ‡ — e Zd ZeedœZˆ fd„Zee	 	 	 	 dde	j
        dz  de	j        dz  de	j
        dz  de	j        dz  dee         d	eez  fd
„¦   «         ¦   «         Zˆ xZS )ÚT5GemmaEncoder)Ú
attentionsrl   c                 ó  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          ‰j        ‰j
        ¬¦  «        | _        d| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j        ¦  «        | _        t)          ‰¬¦  «        | _        |                      ¦   «          d S )Nr	  Fc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r  )r  ©Ú.0rÖ   r^   s     €r=   ú
<listcomp>z+T5GemmaEncoder.__init__.<locals>.<listcomp>‰  ó$   ø€ ÐeÐeÐe¸	Õ  ¨Ñ3Ô3ÐeÐeÐer>   ©r^   ©r4   r5   r@  Úpadding_idxr,  r6   Ú	Embeddingr_   Úembed_tokensr.   r  ÚnormÚgradient_checkpointingÚ
ModuleListÚrangeÚnum_hidden_layersrÿ   rg   rh   ri   rn   Ú
rotary_embÚ	post_initrj   s    `€r=   r5   zT5GemmaEncoder.__init__  óè   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔÝ" 6Ô#5¸6Ô;NÐOÑOÔOˆŒ	Ø&+ˆÔ#å”mØeÐeÐeÐeÅUÈ6ÔKcÑEdÔEdÐeÑeÔeñ
ô 
ˆŒõ ”z &Ô"5Ñ6Ô6ˆŒÝ0¸Ð?Ñ?Ô?ˆŒð 	�ŠÑÔÐÐÐr>   NrC  r¿   rœ   Úinputs_embedsrÍ   r€   c                 óz  — |d u |d uz  rt          d¦  «        ‚|                     dd ¦  «         |€|                      |¦  «        }|€;t          j        |j        d         |j        ¬¦  «        }|                     d¦  «        }|€t          ||| j	        j
        ¦  «        }t          |x}t          ¦  «        s$| j	        ||dœ}t          di |¤Žt          di |¤Ždœ}|}t          j        | j	        j        dz  |j        ¬	¦  «        }	||	z  }|                      |¦  «        }|                      ||¦  «        }
t)          | j        d | j	        j        …         ¦  «        D ]'\  }} |||
|| j	        j        |                  |fi |¤Ž}Œ(|                      |¦  «        }|                      |¦  «        }t3          |¬
¦  «        S )Nú:You must specify exactly one of input_ids or inputs_embedsré   r*   ©r}   r   )r^   rl  r¿   ©Úfull_attentionrÙ   ç      à?r„   )Úlast_hidden_stater  )rù   Úpoprc  r8   r‰   rP   r}   r¬   rU  r^   r@  r•   Údictr   r   Útensorr_   r…   ri   ri  Ú	enumeraterÿ   rh  rØ   rd  r   )r;   rC  r¿   rœ   rl  rÍ   Úself_attn_mask_mappingÚmask_kwargsrl   Ú
normalizerrè   ÚiÚlayer_modules                r=   rM   zT5GemmaEncoder.forward‘  s  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZð 	�
Š
Ð$ dÑ+Ô+Ð+àÐ Ø ×-Ò-¨iÑ8Ô8ˆMàÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\ˆLØ'×1Ò1°!Ñ4Ô4ˆLàÐ!Ý;¸IÀ}ÐVZÔVaÔVnÑoÔoˆNå°NÐBÐ0ÅDÑIÔIð 		àœ+Ø!.Ø"0ðð ˆKõ #<Ð"JÐ"J¸kÐ"JÐ"JÝ%MÐ%\Ð%\ÐP[Ð%\Ð%\ð&ð &Ð"ð
 &ˆÝ”\ $¤+Ô"9¸3Ñ">ÀmÔFYÐZÑZÔZˆ
Ø%¨
Ñ2ˆØŸš ]Ñ3Ô3ˆà"Ÿošo¨m¸\ÑJÔJÐå(¨¬Ð5T°t´{Ô7TÐ5TÔ)UÑVÔVð 	ð 	‰OˆAˆ|Ø(˜LØØ#Ø& t¤{Ô'>¸qÔ'AÔBØð	ð ð
 ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆØŸš ]Ñ3Ô3ˆÝØ+ð
ñ 
ô 
ð 	
r>   ©NNNN)rS   rT   rU   rÔ   r  rP  r5   r'   r)   r8   r  r¢   r  r!   r"   rO   r   rM   rW   rX   s   @r=   rW  rW  y  sí   ø€ € € € € à*Ø,ðð Ðð
ð ð ð ð ð$  Øð .2Ø.2Ø04Ø26ð6
ð 6
àÔ# dÑ*ð6
ð œ tÑ+ð6
ð Ô&¨Ñ-ð	6
ð
 Ô(¨4Ñ/ð6
ð Ð+Ô,ð6
ð 
�Ñ	 ð6
ð 6
ð 6
ñ „_ñ  Ôð6
ð 6
ð 6
ð 6
ð 6
r>   rW  c                   ó6  ‡ — e Zd Z eed¬¦  «         eed¬¦  «        edœZˆ fd„Ze	e
	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  d	edz  d
ej        dz  dedz  dej        dz  dej        dz  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚT5GemmaDecoderr*   )Úindex)rX  Úcross_attentionsrl   c                 ó  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          ‰j        ‰j
        ¬¦  «        | _        d| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j        ¦  «        | _        t)          ‰¬¦  «        | _        |                      ¦   «          d S )Nr	  Fc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r  )r  r[  s     €r=   r]  z+T5GemmaDecoder.__init__.<locals>.<listcomp>Ý  r^  r>   r_  r`  rj   s    `€r=   r5   zT5GemmaDecoder.__init__Ó  rk  r>   NrC  r¿   rœ   ré   rl  r  rú   r  rÍ   r€   c	                 ó¬  — |d u |d uz  rt          d¦  «        ‚|€t          d¦  «        ‚|€|                      |¦  «        }| j        s3|r1|€/t          t	          | j        ¬¦  «        t	          ¦   «         ¦  «        }|€V|�|                     ¦   «         nd}
t          j        |j	        d         |j
        ¬¦  «        |
z   }|                     d¦  «        }|€|€t          ||| j        j        ¦  «        }t          |x}t          ¦  «        s/| j        |||�|j        nd |dœ}t#          di |¤Žt%          di |¤Ždœ}t          |x}t          ¦  «        sd	t'          | j        |||¬
¦  «        i}|}t          j        | j        j        dz  |j        ¬¦  «        }||z  }|                      |¦  «        }|                      ||¦  «        }t3          | j        d | j        j        …         ¦  «        D ]1\  }} ||||| j        j        |                  |||||d	         fi |	¤Ž}Œ2|                      |¦  «        }|                      |¦  «        }t=          ||¬¦  «        S )Nrn  z0`encoder_hidden_states` must be given in decoderr_  r   r*   ro  )r^   rl  r¿   ré   rœ   rp  rq  )r^   rl  r¿   rú   rr  r„   )rs  ré   r  )rù   rc  rÅ   r
   r	   r^   Úget_seq_lengthr8   r‰   rP   r}   r¬   rU  r@  r•   ru  r!  r   r   r   rv  r_   r…   ri   ri  rw  rÿ   rh  rØ   rd  r   )r;   rC  r¿   rœ   ré   rl  r  rú   r  rÍ   Úpast_seen_tokensrx  ry  Úcross_attn_mask_mappingrl   rz  rè   r{  r|  s                      r=   rM   zT5GemmaDecoder.forwardå  sâ  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZØ Ð(ÝÐOÑPÔPÐPàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàŒ}ð 	d ð 	d¨Ð/Fõ 2µ,ÀdÄkÐ2RÑ2RÔ2RÕT`ÑTbÔTbÑcÔcˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLàÐ! oÐ&=Ý;¸IÀ}ÐVZÔVaÔVnÑoÔoˆNå°NÐBÐ0ÅDÑIÔIð 	àœ+Ø!.Ø"0ØKZÐKf ?Ô#GÐ#GÐlpØ ,ðð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ%FÐ%UÐ%UÈÐ%UÐ%Uð&ð &Ð"õ
 Ð5KÐKÐ1ÍTÑRÔRð 	à Õ";Øœ;Ø"/Ø#9Ø*?ð	#ñ #ô #ð'Ð#ð &ˆÝ”\ $¤+Ô"9¸3Ñ">ÀmÔFYÐZÑZÔZˆ
Ø%¨
Ñ2ˆØŸš ]Ñ3Ô3ˆà"Ÿošo¨m¸\ÑJÔJÐå(¨¬Ð5T°t´{Ô7TÐ5TÔ)UÑVÔVð 	ð 	‰OˆAˆ|Ø(˜LØØ#Ø& t¤{Ô'>¸qÔ'AÔBØØØØ%Ø'Ð(8Ô9ð
ð 
ð ð
ð 
ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆØŸš ]Ñ3Ô3ˆÝ8Ø+Ø+ð
ñ 
ô 
ð 	
r>   )NNNNNNNN)rS   rT   rU   r(   rÔ   rö   r  rP  r5   r'   r)   r8   r  r¢   r
   r  r"  r!   r"   rO   r   rM   rW   rX   s   @r=   r  r  Ì  so  ø€ € € € € à$�nÐ%9ÀÐCÑCÔCØ*˜NÐ+@ÈÐJÑJÔJØ,ðð Ððð ð ð ð ð$  Øð .2Ø.2Ø04Ø6:Ø26Ø!%Ø59Ø6:ðP
ð P
àÔ# dÑ*ðP
ð œ tÑ+ðP
ð Ô&¨Ñ-ð	P
ð
 -¨tÑ3ðP
ð Ô(¨4Ñ/ðP
ð ˜$‘;ðP
ð  %œ|¨dÑ2ðP
ð !&¤¨tÑ 3ðP
ð Ð+Ô,ðP
ð 
Ð:Ñ	:ðP
ð P
ð P
ñ „_ñ  ÔðP
ð P
ð P
ð P
ð P
r>   r  c                   óB  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 	 	 dde	j
        dz  de	j        dz  de	j
        dz  d	e	j
        dz  d
e	j        dz  de	j
        dz  dedz  dedz  de	j        dz  de	j        dz  dedz  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚT5GemmaModelr^   c                 ó  •— t          ¦   «                              |¦  «         |j        st          d¦  «        ‚t	          |j        ¦  «        | _        t          |j        ¦  «        | _        |                      ¦   «          d S )NzVT5GemmaModel only support encoder-decoder modeling. Use `T5GemmaEncoderModel` instead.)	r4   r5   Úis_encoder_decoderrù   rW  Úencoderr  r>  rj  rj   s     €r=   r5   zT5GemmaModel.__init__<  sn   ø€ Ý‰Œ×Ò˜Ñ Ô Ð àÔ(ð 	wÝÐuÑvÔvÐvå% f¤nÑ5Ô5ˆŒÝ% f¤nÑ5Ô5ˆŒà�ŠÑÔÐÐÐr>   c                 ó4   — | j                              ¦   «         S r3   ©rŒ  Úget_input_embeddingsrQ   s    r=   r�  z!T5GemmaModel.get_input_embeddingsG  ó   € ØŒ|×0Ò0Ñ2Ô2Ð2r>   c                 ó6   — | j                              |¦  «        S r3   ©rŒ  Úset_input_embeddings©r;   Únew_embeddingss     r=   r“  z!T5GemmaModel.set_input_embeddingsJ  ó   € ØŒ|×0Ò0°Ñ@Ô@Ð@r>   NrC  r¿   rœ   Údecoder_input_idsÚdecoder_attention_maskÚdecoder_position_idsÚencoder_outputsré   rl  Údecoder_inputs_embedsr  rÍ   r€   c                 ó  — |€ | j         d||||	dœ|¤Ž}|j        } | j        d||||
||||dœ|¤Ž}t          |j        |j        |                     dd¦  «        r|j        n|j        f|j        |j        |j        |j        |j        ¬¦  «        S )aX  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        N©rC  r¿   rœ   rl  )rC  r¿   rœ   rl  ré   rú   r  r  Úoutput_hidden_statesF)rs  ré   Údecoder_hidden_statesÚdecoder_attentionsr�  Úencoder_last_hidden_staterú   Úencoder_attentionsr  )	rŒ  rs  r>  r   ré   rý   rl   rX  r�  )r;   rC  r¿   rœ   r—  r˜  r™  rš  ré   rl  r›  r  rÍ   rú   Údecoder_outputss                  r=   rM   zT5GemmaModel.forwardM  sò   € ð, Ð"Ø*˜dœlð Ø#Ø-Ø)Ø+ð	ð ð
 ðð ˆOð !0Ô AÐà&˜$œ,ð 

Ø'Ø1Ø-Ø/Ø+Ø"7Ø#1Øð

ð 

ð ð

ð 

ˆõ "Ø-Ô?Ø+Ô;à�zŠzÐ0°%Ñ8Ô8ð#6 /Ô"?Ð"?à!Ô3Ð5Ø.Ô9Ø,Ô=Ø&5Ô&GØ"1Ô"?Ø.Ô9ð
ñ 
ô 
ð 	
r>   )NNNNNNNNNNN)rS   rT   rU   r+   r5   r�  r“  r$   r#   r8   r  r  Ú
BoolTensorr   r
   r¢   r"  r!   r"   r   rM   rW   rX   s   @r=   r‰  r‰  :  sŠ  ø€ € € € € ð	˜}ð 	ð 	ð 	ð 	ð 	ð 	ð3ð 3ð 3ðAð Að Að Øð .2Ø37Ø04Ø59Ø:>Ø8<Ø26Ø6:Ø-1Ø59Ø!%ð6
ð 6
àÔ# dÑ*ð6
ð Ô)¨DÑ0ð6
ð Ô&¨Ñ-ð	6
ð
 !Ô+¨dÑ2ð6
ð !&Ô 0°4Ñ 7ð6
ð $Ô.°Ñ5ð6
ð )¨4Ñ/ð6
ð -¨tÑ3ð6
ð ”| dÑ*ð6
ð  %œ|¨dÑ2ð6
ð ˜$‘;ð6
ð Ð+Ô,ð6
ð 
ð6
ð 6
ð 6
ñ „^ñ Ôð6
ð 6
ð 6
ð 6
ð 6
r>   r‰  c                   óÆ   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 	 	 	 dde	j
        dz  de	j        dz  de	j
        dz  d	e	j        dz  d
ee         defd„¦   «         ¦   «         Zˆ xZS )ÚT5GemmaEncoderModelr^   c                 óÐ   •— t          ¦   «                              |¦  «         |j        rt          d¦  «        ‚t	          |j        ¦  «        | _        |                      ¦   «          d S )NzQT5GemmaEncoderModel only supports encoder-only model. Use `T5GemmaModel` instead.)r4   r5   r‹  rù   rW  rŒ  rj  rj   s     €r=   r5   zT5GemmaEncoderModel.__init__Š  s]   ø€ Ý‰Œ×Ò˜Ñ Ô Ð àÔ$ð 	rÝÐpÑqÔqÐqå% f¤nÑ5Ô5ˆŒØ�ŠÑÔÐÐÐr>   c                 ó4   — | j                              ¦   «         S r3   rŽ  rQ   s    r=   r�  z(T5GemmaEncoderModel.get_input_embeddings“  r�  r>   c                 ó6   — | j                              |¦  «        S r3   r’  r”  s     r=   r“  z(T5GemmaEncoderModel.set_input_embeddings–  r–  r>   NrC  r¿   rœ   rl  rÍ   r€   c                 ó*   —  | j         d||||dœ|¤Ž}|S )Nr�  r  )rŒ  )r;   rC  r¿   rœ   rl  rÍ   rš  s          r=   rM   zT5GemmaEncoderModel.forward™  s?   € ð '˜$œ,ð 
ØØ)Ø%Ø'ð	
ð 
ð
 ð
ð 
ˆð Ðr>   r}  )rS   rT   rU   r+   r5   r�  r“  r$   r#   r8   r  r  r¢   r!   r"   r   rM   rW   rX   s   @r=   r¦  r¦  ˆ  s  ø€ € € € € ð˜}ð ð ð ð ð ð ð3ð 3ð 3ðAð Að Að Øð .2Ø37Ø04Ø-1ðð àÔ# dÑ*ðð Ô)¨DÑ0ðð Ô&¨Ñ-ð	ð
 ”| dÑ*ðð Ð+Ô,ðð 
ðð ð ñ „^ñ Ôðð ð ð ð r>   r¦  c            "       óä  ‡ — e Zd ZddiZddiZddgdgfiZdefˆ fd„Zd	„ Zd
„ Z	e
e	 	 	 	 	 	 	 	 	 	 	 	 	 d$dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dedz  dej        dz  dej        dz  dej        dz  dedz  deej        z  dee         deej                 ez  fd„¦   «         ¦   «         Zdej        fd„Zdeded ed!ed"edefˆ fd#„Zˆ xZS )%ÚT5GemmaForConditionalGenerationzlm_head.out_proj.weightz!model.decoder.embed_tokens.weightzlm_head.out_projÚcolwise_gather_outputrl   r/  r^   c                 ó   •— d|_         t          ¦   «                              |¦  «         t          |¦  «        | _        |j        j        | _        t          |j        j        | j        ¦  «        | _	        d| _
        |                      ¦   «          d S )NTÚForMaskedLM)r‹  r4   r5   r‰  r2  r>  r,  r+  r_   Úlm_headÚ	loss_typerj  rj   s     €r=   r5   z(T5GemmaForConditionalGeneration.__init__²  ss   ø€ Ø$(ˆÔ!Ý‰Œ×Ò˜Ñ Ô Ð å! &Ñ)Ô)ˆŒ
Ø œ.Ô3ˆŒÝ$ V¤^Ô%?ÀÄÑQÔQˆŒØ&ˆŒà�ŠÑÔÐÐÐr>   c                 ó¼   — || j         _        | j        j        rC|j        | j        j        j        _        |j        j        d         | j        j        j        _	        d S d S )Nr   )
r°  r(  r^   r;  r:   r2  r>  rc  rP   Únum_embeddingsr”  s     r=   Úset_output_embeddingsz5T5GemmaForConditionalGeneration.set_output_embeddings½  s]   € Ø .ˆŒÔð Œ;Ô*ð 	\Ø5CÔ5JˆDŒJÔÔ+Ô2Ø=KÔ=RÔ=XÐYZÔ=[ˆDŒJÔÔ+Ô:Ð:Ð:ð	\ð 	\r>   c                 ó   — | j         j        S r3   )r°  r(  rQ   s    r=   Úget_output_embeddingsz5T5GemmaForConditionalGeneration.get_output_embeddingsÆ  s   € ØŒ|Ô$Ð$r>   Nr   rC  r¿   rœ   r—  r˜  r™  rš  ré   rl  r›  Úlabelsr  Úlogits_to_keeprÍ   r€   c                 óD  — |�|€|
€|                       |¦  «        } | j        d|||||||||	|
|dœ|¤Ž}|j        }t          |t          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }|                      ¦   «         j        }|j	        �(||j	        z  }t          j        |¦  «        }||j	        z  }d}|� | j        ||| j        fi |¤Ž}t          |||j        |j        |j        |j        |j        |j        |j        ¬¦	  «	        S )aü  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        N)rC  r¿   rœ   r—  r˜  r™  rš  ré   rl  r›  r  )	Úlossr/  ré   rŸ  r   r�  r¡  rú   r¢  r  )rF  r2  rs  r•   rV   Úslicer°  Úget_decoderr^   Úfinal_logit_softcappingr8   rÈ   Úloss_functionr,  r   ré   rŸ  r   r�  r¡  rú   r¢  )r;   rC  r¿   rœ   r—  r˜  r™  rš  ré   rl  r›  r·  r  r¸  rÍ   r£  rl   Úslice_indicesr/  Údecoder_configrº  s                        r=   rM   z'T5GemmaForConditionalGeneration.forwardÉ  s€  € ð: ÐÐ"3Ð";Ð@UÐ@]à $× 1Ò 1°&Ñ 9Ô 9Ðà.8¨d¬jð /
ØØ)Ø%Ø/Ø#9Ø!5Ø+Ø+Ø'Ø"7Øð/
ð /
ð ð/
ð /
ˆð (Ô9ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆØ×)Ò)Ñ+Ô+Ô2ˆØÔ1Ð=Ø˜nÔDÑDˆFÝ”Z Ñ'Ô'ˆFØ˜nÔDÑDˆFàˆØÐà%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDåØØØ+Ô;Ø"1Ô"GØ.ÔAØ,Ô=Ø&5Ô&OØ"1Ô"GØ.ÔAð

ñ 

ô 

ð 
	
r>   c                 ó,   — |                       |¦  «        S r3   )rF  )r;   r·  s     r=   Ú%prepare_decoder_input_ids_from_labelszET5GemmaForConditionalGeneration.prepare_decoder_input_ids_from_labels  s   € Ø× Ò  Ñ(Ô(Ð(r>   Úgeneration_configÚmodel_kwargsÚgeneration_modeÚ
batch_sizeÚmax_cache_lengthc           	      óÀ  •— t          ¦   «                              |||||¦  «         |j        du rdS |j        }|€d}n	d|j        v }t	          j        | j                             d¬¦  «        ¦  «        }d|_        dg|j	        z  |_
        ||dœ}	|                     d¦  «        }
|
�¡t          |
t          ¦  «        st          d	¦  «        ‚t          |
j        ¦  «        d
k    r|
j                             d
¦  «        rdS t#          |
j        ¦  «        }|t&          k    r|d         d
         j        d         |	d<    |di |	¤Ž|
_        nEt          t+          di | j                             d¬¦  «        |dœ¤Žt+          ¦   «         ¦  «        |d<   t-          | d¦  «        r?| j        �:t          | j        t          ¦  «        st          d¦  «        ‚|d         | _        dS dS dS )a7  Override cache preparation to force full attention on the cross-attention cache.

        The decoder config may declare sliding-window layers, but cross-attention must always use full attention.
        The default `_prepare_cache_for_generation` would otherwise build a sliding cross-attention cache.
        FNÚ	offloadedT)r>  rq  )r^   Ú
offloadingré   z`The `past_key_values` in `model_kwargs` must be of type `EncoderDecoderCache` for T5Gemma model.r   rš  r*   Úmax_cache_lenÚ_cachezKThe internal cache must be of type `EncoderDecoderCache` for T5Gemma model.r  )r4   Ú_prepare_cache_for_generationr  Úcache_implementationÚcopyÚdeepcopyr^   Úget_text_configræ   rh  rØ   rý   r•   r
   rù   Úlenrü   r–   rþ   r   rP   r	   rÚ   rÌ  )r;   rÃ  rÄ  rÅ  rÆ  rÇ  rÎ  Úoffload_cacheÚcross_attn_configÚcross_attn_cache_kwargsré   Úcross_attn_clsr<   s               €r=   rÍ  z=T5GemmaForConditionalGeneration._prepare_cache_for_generation  sL  ø€ õ 	‰Œ×-Ò-ØØØØØñ	
ô 	
ð 	
ð Ô&¨%Ð/Ð/ØˆFà0ÔEÐØÐ'Ø!ˆMˆMà'Ð+<Ô+QÐQˆMõ !œM¨$¬+×*EÒ*EÈdÐ*EÑ*SÔ*SÑTÔTÐØ+/ÐÔ(Ø)9Ð(:Ð=NÔ=`Ñ(`ÐÔ%ð (Ø'ð#
ð #
Ðð
 '×*Ò*Ð+<Ñ=Ô=ˆØÐ&Ý˜oÕ/BÑCÔCð Ý Øvñô ð õ
 �?Ô-Ñ.Ô.°Ò2Ð2°Ô7Q×7UÒ7UÐVWÑ7XÔ7XÐ2Ø�å! /Ô"GÑHÔHˆNØ¥Ò,Ð,Ø;GÐHYÔ;ZÐ[\Ô;]Ô;cÐdeÔ;fÐ'¨Ñ8à4B°NÐ4]Ð4]ÐE\Ð4]Ð4]ˆOÔ1Ð1õ /BÝð ð à"&¤+×"=Ò"=ÀdÐ"=Ñ"KÔ"KØ&3ðð ðð õ ‘”ñ/ô /ˆLÐ*Ñ+õ �4˜Ñ"Ô"ð 	: t¤{Ð'>Ý˜dœkÕ+>Ñ?Ô?ð pÝ Ð!nÑoÔoÐoà&Ð'8Ô9ˆDŒKˆKˆKð		:ð 	:Ð'>Ð'>r>   )NNNNNNNNNNNNr   ) rS   rT   rU   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr+   r5   r´  r¶  r$   r#   r8   r  r  r¤  r   r
   r"  rV   r¢   r!   r"   rO   r   rM   rÂ  r   ru  r   rÍ  rW   rX   s   @r=   r¬  r¬  ­  s‡  ø€ € € € € Ø3Ð5XÐYÐØ"Ð$;Ð<€HØ" oÐ%6¸¸
Ð$CÐD€Hð	˜}ð 	ð 	ð 	ð 	ð 	ð 	ð\ð \ð \ð%ð %ð %ð Øð .2Ø37Ø04Ø59Ø:>Ø8<Ø26Ø6:Ø26Ø:>Ø*.Ø!%Ø-.ðG
ð G
àÔ# dÑ*ðG
ð Ô)¨DÑ0ðG
ð Ô&¨Ñ-ð	G
ð
 !Ô+¨dÑ2ðG
ð !&Ô 0°4Ñ 7ðG
ð $Ô.°Ñ5ðG
ð )¨4Ñ/ðG
ð -¨tÑ3ðG
ð Ô(¨4Ñ/ðG
ð  %Ô0°4Ñ7ðG
ð Ô  4Ñ'ðG
ð ˜$‘;ðG
ð ˜eœlÑ*ðG
ð Ð+Ô,ðG
ð  
ˆuÔ Ô	! OÑ	3ð!G
ð G
ð G
ñ „^ñ ÔðG
ðR)¸E¼Lð )ð )ð )ð )ðI:à+ðI:ð ðI:ð (ð	I:ð
 ðI:ð ðI:ð 
ðI:ð I:ð I:ð I:ð I:ð I:ð I:ð I:ð I:ð I:r>   r¬  c                   óL  ‡ — e Zd Zddededz  fˆ fd„Zd„ Zd„ Zee		 	 	 	 	 	 	 	 	 	 dde
j        dz  de
j        dz  d	e
j        dz  d
e
j        dz  de
j        dz  de
j        dz  dedz  de
j        dz  de
j        dz  de
j        dz  dee         defd„¦   «         ¦   «         Zˆ xZS )Ú T5GemmaForSequenceClassificationNr^   r‹  c                 ó’  •— |�||_         t          ¦   «                              |¦  «         |j        | _        |j         rt	          |¦  «        | _        nt          |¦  «        | _        |j        j        }|j         r|j	        j        }t          |dd¦  «        }t          || j        |¦  «        | _        |                      ¦   «          dS )z¬
        is_encoder_decoder (`Optional`, *optional*):
            Whether use encoder_decoder for sequence classification. When set to False, only encoder is used.
        Nr&  çš™™™™™¹?©r‹  r4   r5   r%  r‰  r2  r¦  rŒ  r_   r>  r‡   r$  Úscorerj  ©r;   r^   r‹  r_   Úclassifier_dropoutr<   s        €r=   r5   z)T5GemmaForSequenceClassification.__init__e  s¾   ø€ ð
 Ð)Ø(:ˆFÔ%Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒàÔ$ð 	5Ý% fÑ-Ô-ˆDŒJˆJå,¨VÑ4Ô4ˆDŒJà”nÔ0ˆØÔ$ð 	5Ø œ.Ô4ˆKå$ VÐ-FÈÑLÔLÐÝ.¨{¸D¼OÐM_Ñ`Ô`ˆŒ
Ø�ŠÑÔÐÐÐr>   c                 ó4   — | j                              ¦   «         S r3   ©r2  r�  rQ   s    r=   r�  z5T5GemmaForSequenceClassification.get_input_embeddings|  ó   € ØŒz×.Ò.Ñ0Ô0Ð0r>   c                 ó:   — | j                              |¦  «         d S r3   ©r2  r“  ©r;   r¾   s     r=   r“  z5T5GemmaForSequenceClassification.set_input_embeddings  ó   € ØŒ
×'Ò'¨Ñ.Ô.Ð.Ð.Ð.r>   rC  r¿   rœ   r—  r˜  r™  rš  rl  r›  r·  rÍ   r€   c                 ó¸  — | j         j        r!|€|�t          d| j        j        › d�¦  «        ‚| j         j        r*|€(|	€&|€t          d¦  «        ‚|                      |¦  «        }| j         j        r. | j        |f||||||||	ddœ	|¤Ž}|j        }|j	        }|j
        }n' | j        |f|||dœ|¤Ž}|j        }|j        }|j        }|                      |¦  «        }|�|j        d         }n|j        d         }| j         j        €|d	k    rt          d
¦  «        ‚| j         j        €d}nÝ|�²|| j         j        k                         |j        t$          j        ¦  «        }t%          j        |j        d         |j        t$          j        ¬¦  «        }||z                       d¦  «        }| j         j        r)|d	z  }t%          j        ||j        d         d	z
  ¬¦  «        }n)d}t.                               | j        j        › d�¦  «         |t%          j        ||j        ¬¦  «        |f         }d}|
�|                      ||
|| j         ¬¦  «        }t5          ||||¬¦  «        S )áÜ  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nú8Passing input embeddings is currently not supported for ú in encoder-decoder mode.ú°If no `decoder_input_ids` or `decoder_inputs_embeds` are passed, `input_ids` cannot be `None`. Please pass either `input_ids` or `decoder_input_ids` or `decoder_inputs_embeds`.F©	r¿   rœ   r—  r˜  r™  rš  rl  r›  r  ©r¿   rœ   rl  r   r*   z=Cannot handle batch sizes > 1 if no padding token is defined.rA   r†   )ÚmaxzŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`ro  )r/  r·  Úpooled_logitsr^   ©rº  r/  rl   rX  )r^   r‹  ÚNotImplementedErrorr<   rS   rù   rF  r2  rs  rŸ  r   rl   rX  rß  rP   r@  r‹   r}   r8   Úint32r‰   ÚargmaxÚclampÚloggerÚwarning_oncer¾  r   )r;   rC  r¿   rœ   r—  r˜  r™  rš  rl  r›  r·  rÍ   Úoutputsrs  rl   rX  r/  rÆ  Úlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesrñ  rº  s                          r=   rM   z(T5GemmaForSequenceClassification.forward‚  s  € ð2 Œ;Ô)ð 	¨yÐ/@À]ÐE^Ý%Ø}È4Ì>ÔKbÐ}Ð}Ð}ñô ð ð
 Œ;Ô)ð 	=Ð/@Ð/HÐMbÐMjØÐ Ý ðUñô ð ð
 !%× 1Ò 1°)Ñ <Ô <ÐàŒ;Ô)ð 	,Ø*4¨$¬*Øð+à-Ø)Ø"3Ø'=Ø%9Ø /Ø+Ø&;Øð+ð +ð ð+ð +ˆGð !(Ô 9ÐØ#Ô9ˆMØ Ô3ˆJˆJà'1 t¤zØð(à-Ø)Ø+ð	(ð (ð
 ð(ð (ˆGð !(Ô 9ÐØ#Ô1ˆMØ Ô+ˆJà—’Ð-Ñ.Ô.ˆàÐ Ø"œ¨Ô+ˆJˆJà&Ô,¨QÔ/ˆJàŒ;Ô#Ð+°
¸a²°ÝÐ\Ñ]Ô]Ð]ØŒ;Ô#Ð+Ø!#ÐÐØÐ"à%¨¬Ô)AÒA×EÒEÀfÄmÕUZÔU`ÑaÔaˆLÝ!œL¨¬¸Ô)<ÀVÄ]ÕZ_ÔZeÐfÑfÔfˆMØ"/°,Ñ">×!FÒ!FÀrÑ!JÔ!JÐàŒ{Ô-ð jØ" aÑ'Ð"Ý%*¤[Ð1CÐIZÔI`ÐacÔIdÐghÑIhÐ%iÑ%iÔ%iÐ"øà!#ÐÝ×ÒØ”>Ô*ð Zð Zð Zñô ð ð
 �uœ|¨J¸v¼}ÐMÑMÔMÐOaÐaÔbˆàˆØÐØ×%Ò%¨V¸FÐR_ÐhlÔhsÐ%ÑtÔtˆDå'ØØ Ø'Ø!ð	
ñ 
ô 
ð 	
r>   r3   ©
NNNNNNNNNN)rS   rT   rU   r+   r"  r5   r�  r“  r$   r#   r8   r  r¢   r   r  r!   r"   r   rM   rW   rX   s   @r=   rÛ  rÛ  c  sž  ø€ € € € € ðð ˜}ð À$ÈÁ+ð ð ð ð ð ð ð.1ð 1ð 1ð/ð /ð /ð Øð .2Ø.2Ø04Ø59Ø6:Ø8<Ø26Ø26Ø:>Ø*.ði
ð i
àÔ# dÑ*ði
ð œ tÑ+ði
ð Ô&¨Ñ-ð	i
ð
 !Ô+¨dÑ2ði
ð !&¤¨tÑ 3ði
ð $Ô.°Ñ5ði
ð )¨4Ñ/ði
ð Ô(¨4Ñ/ði
ð  %Ô0°4Ñ7ði
ð Ô  4Ñ'ði
ð Ð+Ô,ði
ð 
"ði
ð i
ð i
ñ „^ñ Ôði
ð i
ð i
ð i
ð i
r>   rÛ  c                   óL  ‡ — e Zd Zddededz  fˆ fd„Zd„ Zd„ Zee		 	 	 	 	 	 	 	 	 	 dde
j        dz  de
j        dz  d	e
j        dz  d
e
j        dz  de
j        dz  de
j        dz  dedz  de
j        dz  de
j        dz  de
j        dz  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚT5GemmaForTokenClassificationNr^   r‹  c                 ó’  •— |�||_         t          ¦   «                              |¦  «         |j        | _        |j         rt	          |¦  «        | _        nt          |¦  «        | _        |j        j        }|j         r|j	        j        }t          |dd¦  «        }t          || j        |¦  «        | _        |                      ¦   «          dS )z©
        is_encoder_decoder (`Optional`, *optional*):
            Whether use encoder_decoder for token classification. When set to False, only encoder is used.
        Nr&  rÝ  rÞ  rà  s        €r=   r5   z&T5GemmaForTokenClassification.__init__ò  s¾   ø€ ð
 Ð)Ø(:ˆFÔ%Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒàÔ$ð 	5Ý% fÑ-Ô-ˆDŒJˆJå,¨VÑ4Ô4ˆDŒJà”nÔ0ˆØÔ$ð 	5Ø œ.Ô4ˆKå$ VÐ-FÈÑLÔLÐÝ.¨{¸D¼OÐM_Ñ`Ô`ˆŒ
à�ŠÑÔÐÐÐr>   c                 ó4   — | j                              ¦   «         S r3   rã  rQ   s    r=   r�  z2T5GemmaForTokenClassification.get_input_embeddings
  rä  r>   c                 ó:   — | j                              |¦  «         d S r3   ræ  rç  s     r=   r“  z2T5GemmaForTokenClassification.set_input_embeddings  rè  r>   rC  r¿   rœ   r—  r˜  r™  rš  rl  r›  r·  rÍ   r€   c                 ó  — | j         j        r!|€|�t          d| j        j        › d�¦  «        ‚| j         j        r*|€(|	€&|€t          d¦  «        ‚|                      |¦  «        }| j         j        r. | j        |f||||||||	ddœ	|¤Ž}|j        }|j	        }|j
        }n' | j        |f|||dœ|¤Ž}|j        }|j        }|j        }|                      |¦  «        }d}|
�|                      ||
| j         ¦  «        }t          ||||¬¦  «        S )	rê  Nrë  rì  rí  Frî  rï  rò  )r^   r‹  ró  r<   rS   rù   rF  r2  rs  rŸ  r   rl   rX  rß  r¾  r   )r;   rC  r¿   rœ   r—  r˜  r™  rš  rl  r›  r·  rÍ   rù  rs  rl   rX  r/  rº  s                     r=   rM   z%T5GemmaForTokenClassification.forward  s¥  € ð4 Œ;Ô)ð 	¨yÐ/@À]ÐE^Ý%Ø}È4Ì>ÔKbÐ}Ð}Ð}ñô ð ð Œ;Ô)ð 	=Ð/@Ð/HÐMbÐMjØÐ Ý ðUñô ð ð
 !%× 1Ò 1°)Ñ <Ô <ÐàŒ;Ô)ð 	,Ø*4¨$¬*Øð+à-Ø)Ø"3Ø'=Ø%9Ø /Ø+Ø&;Øð+ð +ð ð+ð +ˆGð !(Ô 9ÐØ#Ô9ˆMØ Ô3ˆJˆJà'1 t¤zØð(à-Ø)Ø+ð	(ð (ð
 ð(ð (ˆGð !(Ô 9ÐØ#Ô1ˆMØ Ô+ˆJà—’Ð-Ñ.Ô.ˆàˆØÐØ×%Ò% f¨f°d´kÑBÔBˆDå$ØØØ'Ø!ð	
ñ 
ô 
ð 	
r>   r3   rý  )rS   rT   rU   r+   r"  r5   r�  r“  r$   r#   r8   r  r¢   r   r  r!   r"   r   rM   rW   rX   s   @r=   rÿ  rÿ  ð  sž  ø€ € € € € ðð ˜}ð À$ÈÁ+ð ð ð ð ð ð ð01ð 1ð 1ð/ð /ð /ð Øð .2Ø.2Ø04Ø59Ø6:Ø8<Ø26Ø26Ø:>Ø*.ðN
ð N
àÔ# dÑ*ðN
ð œ tÑ+ðN
ð Ô&¨Ñ-ð	N
ð
 !Ô+¨dÑ2ðN
ð !&¤¨tÑ 3ðN
ð $Ô.°Ñ5ðN
ð )¨4Ñ/ðN
ð Ô(¨4Ñ/ðN
ð  %Ô0°4Ñ7ðN
ð Ô  4Ñ'ðN
ð Ð+Ô,ðN
ð 
ðN
ð N
ð N
ñ „^ñ ÔðN
ð N
ð N
ð N
ð N
r>   rÿ  )r¬  r‰  r¦  r1  rÛ  rÿ  )r*   )rº   NN)_rÏ  Úcollections.abcr   Útypingr   r8   Útorch.nnr6   Ú r   r8  Úactivationsr   Úcache_utilsr   r	   r
   r   Ú
generationr   r   r   Úintegrationsr   r   Úmasking_utilsr   r   r   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r    Úprocessing_utilsr!   Úutilsr"   r#   r$   r%   Úutils.genericr&   r'   Úutils.output_capturingr(   r)   Úconfiguration_t5gemmar+   r,   Ú
get_loggerrS   r÷  ÚModuler.   rZ   rn   r©   r²   r¢   rV   r¹   rJ   rO   rÒ   rÔ   rö   r  r  r$  r+  r1  r  rU  rW  r  r‰  r¦  r¬  rÛ  rÿ  Ú__all__r  r>   r=   ú<module>r     s\  ðð* €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ IÐ IÐ IÐ IÐ IÐ IÐ IÐ Iðð ð ð ð ð ð ð ð ð ð ð ð CÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð LÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ Eð 
ˆÔ	˜HÑ	%Ô	%€ð=ð =ð =ð =ð =�R”Yñ =ô =ð =ð(ð ð ð ð �”ñ ô ð ð&><ð ><ð ><ð ><ð ><˜RœYñ ><ô ><ð ><ðB(ð (ð (ð ÐÐ*Ñ+Ô+ðð ð ñ ,Ô+ðð2	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð$ Ø Ø ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �S‰[ð%ð �T‰\ð%ð �T‰\ð%ð ˆ5Œ<˜œÐ%Ô&ð%ð %ð %ð %ðD ÐÐ)Ñ*Ô*ðF)ð F)ð F)ð F)ð F)˜2œ9ñ F)ô F)ñ +Ô*ðF)ðR ÐÐ)Ñ*Ô*ðR)ð R)ð R)ð R)ð R)˜BœIñ R)ô R)ñ +Ô*ðR)ðj1ð 1ð 1ð 1ð 1Ð4ñ 1ô 1ð 1ðhFð Fð Fð Fð FÐ4ñ Fô Fð FðRð ð ð ð  ¤	ñ ô ð ð	ð 	ð 	ð 	ð 	�B”Iñ 	ô 	ð 	ð ð9!ð 9!ð 9!ð 9!ð 9!˜_ñ 9!ô 9!ñ „ð9!ðxØÔ $Ñ&ðà”<ðð ˜‘*ðð „\ð	ð ð ð ð"P
ð P
ð P
ð P
ð P
Ð+ñ P
ô P
ð P
ðfk
ð k
ð k
ð k
ð k
Ð+ñ k
ô k
ð k
ð\ ðJ
ð J
ð J
ð J
ð J
Ð)ñ J
ô J
ñ „ðJ
ðZ ð!ð !ð !ð !ð !Ð0ñ !ô !ñ „ð!ðHs:ð s:ð s:ð s:ð s:Ð&<¸oñ s:ô s:ð s:ðl ðI
ð I
ð I
ð I
ð I
Ð'=ñ I
ô I
ñ „ðI
ðX ðo
ð o
ð o
ð o
ð o
Ð$:ñ o
ô o
ñ „ðo
ðdð ð €€€r>   