§
    ‚ŠtjØÈ  ã                   óF  — d dl mZ d dlmZ d dlmZ d dlZd dlmZ ddl	m
Z ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZmZmZmZmZmZmZ ddl m!Z!m"Z" ddl#m$Z$m%Z%m&Z&m'Z' ddl(m)Z)m*Z* ddl+m,Z,m-Z- ddl.m/Z/ ddl0m1Z1m2Z2m3Z3m4Z4m5Z5 ddl6m7Z7m8Z8 ddl9m:Z: ddl;m<Z< ddl=m>Z>m?Z?  e3d¬¦  «        e G d„ de$¦  «        ¦   «         ¦   «         Z@ e3d¬¦  «        e G d„ de1¦  «        ¦   «         ¦   «         ZA G d „ d!ejB        ¦  «        ZC G d"„ d#ejD        ¦  «        ZE G d$„ d%ejD        ¦  «        ZF G d&„ d'ejD        ¦  «        ZGd(„ ZH ed)¦  «        d[d*„¦   «         ZId+ejJ        d,eKd-ejJ        fd.„ZL	 	 	 d\d0ejD        d1ejJ        d2ejJ        d3ejJ        d4ejJ        dz  d5eMeKz  d6eMdz  d7eMdz  d-eNejJ        ejJ        f         fd8„ZO eeI¦  «         G d9„ d:ejD        ¦  «        ¦   «         ZP G d;„ d<e"¦  «        ZQe3 G d=„ d>e-¦  «        ¦   «         ZRd?eKd-eeKeKeKeKgeSf         fd@„ZTe3 G dA„ dBeR¦  «        ¦   «         ZUe3 G dC„ dDeRe¦  «        ¦   «         ZV G dE„ dFejD        ¦  «        ZWd]dGejJ        dHejX        dz  d-ejJ        fdI„ZYdJedKejJ        d4ejJ        dz  dLedz  dMejJ        dz  dNejJ        d-eZfdO„Z[ e3dP¬¦  «         G dQ„ dReR¦  «        ¦   «         Z\ e3dP¬¦  «         G dS„ dTeRe¦  «        ¦   «         Z] e3dU¬¦  «         G dV„ dWe!eR¦  «        ¦   «         Z^ G dX„ dYe!eR¦  «        Z_g dZ¢Z`dS )^é    )ÚCallable)Ú	dataclass)ÚOptionalNé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚPreTrainedConfig)ÚGenerationMixin)Úuse_kernel_func_from_hubÚuse_kernelized_func)Ú_preprocess_mask_argumentsÚblockwise_overlayÚcreate_causal_maskÚcreate_masks_for_generateÚ!create_sliding_window_causal_maskÚmaybe_pad_block_sequence_idsÚsliding_window_overlay)Ú GenericForSequenceClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚCausalLMOutputWithPastÚ SequenceClassifierOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚtorch_compilable_check)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )Ú	AutoModelé   )ÚGemma3ConfigÚGemma3TextConfigzK
    Base class for Gemma3 outputs, with hidden states and attentions.
    ©Úcustom_introc                   ó2   — e Zd ZU dZdZej        dz  ed<   dS )ÚGemma3ModelOutputWithPasta  
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    NÚimage_hidden_states)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r2   ÚtorchÚFloatTensorÚ__annotations__© ó    úh/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/gemma3/modeling_gemma3.pyr1   r1   <   s7   € € € € € € ðð ð 59Ð˜Ô*¨TÑ1Ð8Ð8Ñ8Ð8Ð8r;   r1   zR
    Base class for Gemma3 causal language model (or autoregressive) outputs.
    c                   óÖ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZej        dz  ed<   dS )	ÚGemma3CausalLMOutputWithPasta8  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.text_config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder after projecting last hidden state.
    NÚlossÚlogitsÚpast_key_valuesÚhidden_statesÚ
attentionsr2   )r3   r4   r5   r6   r?   r7   r8   r9   r@   rA   r	   rB   ÚtuplerC   r2   r:   r;   r<   r>   r>   L   sµ   € € € € € € ðð ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø48Ð˜Ô*¨TÑ1Ð8Ð8Ñ8Ð8Ð8r;   r>   c            	       óP   ‡ — e Zd ZdZd
dedededefˆ fd„Zdej        fˆ fd	„Z	ˆ xZ
S )ÚGemma3TextScaledWordEmbeddingz\
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    ç      ð?Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scalec                 ó°   •— t          ¦   «                              |||¦  «         || _        |                      dt	          j        |¦  «        d¬¦  «         d S )NrK   F©Ú
persistent)ÚsuperÚ__init__Úscalar_embed_scaleÚregister_bufferr7   Útensor)ÚselfrH   rI   rJ   rK   Ú	__class__s        €r<   rP   z&Gemma3TextScaledWordEmbedding.__init__o   sS   ø€ Ý‰Œ×Ò˜¨¸ÑDÔDÐDØ"-ˆÔØ×Ò˜]­E¬L¸Ñ,EÔ,EÐRWÐÑXÔXÐXÐXÐXr;   Ú	input_idsc                 ó�   •— t          ¦   «                              |¦  «        | j                             | j        j        ¦  «        z  S ©N)rO   ÚforwardrK   ÚtoÚweightÚdtype)rT   rV   rU   s     €r<   rY   z%Gemma3TextScaledWordEmbedding.forwardt   s4   ø€ Ý‰wŒw�Š˜yÑ)Ô)¨DÔ,<×,?Ò,?ÀÄÔ@QÑ,RÔ,RÑRÐRr;   )rG   )r3   r4   r5   r6   ÚintÚfloatrP   r7   ÚTensorrY   Ú__classcell__©rU   s   @r<   rF   rF   j   s¦   ø€ € € € € ðð ðYð Y sð Y¸3ð YÈSð YÐ_dð Yð Yð Yð Yð Yð Yð
S ¤ð Sð Sð Sð Sð Sð Sð Sð Sð Sð Sr;   rF   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )Ú	Gemma3MLPÚconfigc                 ó˜  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _	        t          |j                 | _        d S ©NF©Úbias)rO   rP   rd   Úhidden_sizeÚintermediate_sizeÚnnÚLinearÚ	gate_projÚup_projÚ	down_projr   Úhidden_activationÚact_fn©rT   rd   rU   s     €r<   rP   zGemma3MLP.__init__y   s¦   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔÝœ 4Ô#3°TÔ5KÐRWÐXÑXÔXˆŒÝ”y Ô!1°4Ô3IÐPUÐVÑVÔVˆŒÝœ 4Ô#9¸4Ô;KÐRWÐXÑXÔXˆŒÝ˜VÔ5Ô6ˆŒˆˆr;   c                 ó¨   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        }|S rX   )ro   rq   rm   rn   )rT   Úxro   s      r<   rY   zGemma3MLP.forwardƒ   sA   € Ø—N’N 4§;¢;¨t¯~ª~¸aÑ/@Ô/@Ñ#AÔ#AÀDÇLÂLÐQRÁOÄOÑ#SÑTÔTˆ	ØÐr;   )r3   r4   r5   r-   rP   rY   r`   ra   s   @r<   rc   rc   x   sT   ø€ € € € € ð7Ð/ð 7ð 7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð r;   rc   c                   ó<   ‡ — e Zd Zddedefˆ fd„Zd„ Zd„ Zd„ Zˆ xZ	S )	ÚGemma3RMSNormç�íµ ÷Æ°>ÚdimÚepsc                 ó¬   •— t          ¦   «                              ¦   «          || _        t          j        t          j        |¦  «        ¦  «        | _        d S rX   )rO   rP   ry   rk   Ú	Parameterr7   Úzerosr[   )rT   rx   ry   rU   s      €r<   rP   zGemma3RMSNorm.__init__‰   s?   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”l¥5¤;¨sÑ#3Ô#3Ñ4Ô4ˆŒˆˆr;   c                 ó�   — |t          j        |                     d¦  «                             dd¬¦  «        | j        z   ¦  «        z  S )Nr)   éÿÿÿÿT)Úkeepdim)r7   ÚrsqrtÚpowÚmeanry   )rT   rt   s     r<   Ú_normzGemma3RMSNorm._normŽ   s8   € Ø•5”;˜qŸušu Q™xœxŸ}š}¨R¸˜}Ñ>Ô>ÀÄÑIÑJÔJÑJÐJr;   c                 ó¸   — |                       |                     ¦   «         ¦  «        }|d| j                             ¦   «         z   z  }|                     |¦  «        S )NrG   )rƒ   r^   r[   Útype_as)rT   rt   Úoutputs      r<   rY   zGemma3RMSNorm.forward‘   sL   € Ø—’˜AŸGšG™IœIÑ&Ô&ˆð ˜3 ¤×!2Ò!2Ñ!4Ô!4Ñ4Ñ5ˆØ�~Š~˜aÑ Ô Ð r;   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)rD   r[   Úshapery   )rT   s    r<   Ú
extra_reprzGemma3RMSNorm.extra_repr˜   s%   € Ý˜œÔ)Ñ*Ô*Ð<Ð<°$´(Ð<Ð<Ð<r;   )rw   )
r3   r4   r5   r]   r^   rP   rƒ   rY   r‰   r`   ra   s   @r<   rv   rv   ˆ   s€   ø€ € € € € ð5ð 5˜Cð 5 eð 5ð 5ð 5ð 5ð 5ð 5ð
Kð Kð Kð!ð !ð !ð=ð =ð =ð =ð =ð =ð =r;   rv   c                   óà   ‡ — e Zd ZU ej        ed<   defˆ fd„Ze	 	 	 	 ddedz  de	d         de
dz  dedz  d	ed
ef         f
d„¦   «         Z ej        ¦   «         edd„¦   «         ¦   «         Zˆ xZS )ÚGemma3RotaryEmbeddingÚinv_freqrd   c                 ó€  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        t          t          |j        ¦  «        ¦  «        | _        i | _	        | j        D ]È}| j        j
        |         }|€Œ|d         | j	        |<   | j        }| j	        |         dk    rt          | j	        |                  } || j        |¬¦  «        \  }}|                      |› d�|d¬¦  «         |                      |› d�|                     ¦   «         d¬¦  «         t          | |› d�|¦  «         ŒÉd S )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqFrM   Ú_original_inv_freqÚ_attention_scaling)rO   rP   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrd   ÚlistÚsetÚlayer_typesrŽ   Úrope_parametersÚcompute_default_rope_parametersr   rR   ÚcloneÚsetattr)rT   rd   r‘   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingrU   s          €r<   rP   zGemma3RotaryEmbedding.__init__Ÿ   s^  ø€ Ý‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!ØˆŒÝ¥ FÔ$6Ñ 7Ô 7Ñ8Ô8ˆÔØˆŒØÔ*ð 	Uð 	UˆJØœ+Ô5°jÔAˆKØÐ"Øà)4°[Ô)AˆDŒN˜:Ñ&Ø%)Ô%IˆLØŒ~˜jÔ)¨YÒ6Ð6Ý2°4´>À*Ô3MÔN�Ø4@°LÀÄÐYcÐ4dÑ4dÔ4dÑ1ˆMÐ1Ø× Ò  JÐ!9Ð!9Ð!9¸=ÐUZÐ Ñ[Ô[Ð[Ø× Ò  JÐ!BÐ!BÐ!BÀM×DWÒDWÑDYÔDYÐfkÐ ÑlÔlÐlÝ�D˜ZÐ;Ð;Ð;Ð=SÑTÔTÐTÐTð	Uð 	Ur;   NÚdeviceztorch.deviceÚseq_lenr‘   Úreturnztorch.Tensorc                 ó  — | j         |         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «                             |t          j        ¬¦  «        |z  z  z  }||fS )	a|  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNrG   r   r)   ©r\   )r£   r\   )	r›   Úgetattrri   Únum_attention_headsr7   ÚarangeÚint64rZ   r^   )rd   r£   r¤   r‘   Úbaserx   Úattention_factorrŒ   s           r<   rœ   z5Gemma3RotaryEmbedding.compute_default_rope_parameters´   s‘   € ð2 Ô% jÔ1°,Ô?ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐð Ø•U”\ ! S¨!µ5´;Ð?Ñ?Ô?×BÒBÈ&ÕX]ÔXcÐBÑdÔdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r;   c                 ó|  — t          | |› d�¦  «        }t          | |› d�¦  «        }|d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬	¦  «        5  |                     ¦   «         |                     ¦   «         z   
                    dd
¦  «        }	t          j        |	|	fd¬¦  «        }
|
                     ¦   «         |z  }|
                     ¦   «         |z  }d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |                     |j        ¬¦  «        fS )Nr’   r”   r   r~   r+   ÚmpsÚcpuF)Údevice_typeÚenabledr)   ©rx   r©   )rª   r^   Úexpandrˆ   rZ   r£   Ú
isinstanceÚtypeÚstrr&   Ú	transposer7   ÚcatÚcosÚsinr\   )rT   rt   Úposition_idsr‘   rŒ   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedr³   ÚfreqsÚembr¼   r½   s                r<   rY   zGemma3RotaryEmbedding.forwardØ   sâ  € õ ˜4 JÐ!9Ð!9Ð!9Ñ:Ô:ˆÝ# D¨ZÐ*KÐ*KÐ*KÑLÔLÐà$ T¨1¨1¨1¨d ]Ô3×9Ò9Ñ;Ô;×BÒBÀ<ÔCUÐVWÔCXÐZ\Ð^_Ñ`Ô`×cÒcÐdeÔdlÑmÔmÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	0ð 	0Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)Ð/Ñ/ˆCØ—'’'‘)”)Ð/Ñ/ˆCð		0ð 	0ð 	0ñ 	0ô 	0ð 	0ð 	0ð 	0ð 	0ð 	0ð 	0øøøð 	0ð 	0ð 	0ð 	0ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   Ã-BE=Å=FÆF©NNNNrX   )r3   r4   r5   r7   r_   r9   r-   rP   Ústaticmethodr   r]   r¹   rD   r^   rœ   Úno_gradr   rY   r`   ra   s   @r<   r‹   r‹   œ   s  ø€ € € € € € ØŒlÐÐÑðUÐ/ð Uð Uð Uð Uð Uð Uð* à*.Ø+/Ø"Ø!%ð	!*ð !*Ø  4Ñ'ð!*à˜Ô(ð!*ð �t‘ð!*ð ˜$‘Jð	!*ð
 
ˆ~˜uÐ$Ô	%ð!*ð !*ð !*ñ „\ð!*ðF €U„]�_„_Øð<ð <ð <ñ Ôñ „_ð<ð <ð <ð <ð <r;   r‹   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr~   r)   rµ   )rˆ   r7   r»   )rt   Úx1Úx2s      r<   Úrotate_halfrÊ   ë   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r;   Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezerÊ   )ÚqÚkr¼   r½   Úunsqueeze_dimÚq_embedÚk_embeds          r<   Úapply_rotary_pos_embrÓ   ò   sc   € ð& �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr;   rB   Ún_repr¥   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r+   N)rˆ   r¶   Úreshape)rB   rÔ   ÚbatchÚnum_key_value_headsÚslenr¨   s         r<   Ú	repeat_kvrÚ     s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr;   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚdropoutÚscalingÚsoftcapc                 ól  — |€
| j         dz  }t          || j        ¦  «        }	t          || j        ¦  «        }
t          j        ||	                     dd¦  «        ¦  «        |z  }|�||z  }t          j        |¦  «        }||z  }|�||z   }t          j         	                    |dt          j
        ¬¦  «                             |j        ¦  «        }t          j                             ||| j        ¬¦  «        }t          j        ||
¦  «        }|                     dd¦  «                             ¦   «         }||fS )Nç      à¿r)   r   r~   )rx   r\   )ÚpÚtrainingr+   )r¨   rÚ   Únum_key_value_groupsr7   Úmatmulrº   Útanhrk   Ú
functionalÚsoftmaxÚfloat32rZ   r\   rá   rç   Ú
contiguous)rÜ   rÝ   rÞ   rß   rà   rá   râ   rã   ÚkwargsÚ
key_statesÚvalue_statesÚattn_weightsÚattn_outputs                r<   Úeager_attention_forwardrô     s%  € ð €Ø”/ 4Ñ'ˆå˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LàÐØ# gÑ-ˆÝ”z ,Ñ/Ô/ˆØ# gÑ-ˆØÐ!Ø# nÑ4ˆõ ”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€KØ˜Ð$Ð$r;   c                   óÒ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej        dej        dej        dz  d	e	dz  d
e
e         deej        ej        dz  eej                 dz  f         fd„Zˆ xZS )ÚGemma3Attentionz=Multi-headed attention from 'Attention Is All You Need' paperrd   Ú	layer_idxc                 ó
  •— t          ¦   «                              ¦   «          t          |d¦  «        r|j        |         nd | _        || _        || _        t          |d|j        |j	        z  ¦  «        | _
        |j	        |j        z  | _        |j        dz  | _        | j        j        | _        | j        j         | _        t%          j        |j        |j	        | j
        z  |j        ¬¦  «        | _        t%          j        |j        |j        | j
        z  |j        ¬¦  «        | _        t%          j        |j        |j        | j
        z  |j        ¬¦  «        | _        t%          j        |j	        | j
        z  |j        |j        ¬¦  «        | _        | j        j        | _        | j        dk    r|j        nd | _        | j        dk    | _        t9          |j
        |j        ¬¦  «        | _        t9          |j
        |j        ¬¦  «        | _        d S )Nrš   r¨   rå   rg   Úsliding_attention)rx   ry   ) rO   rP   Úhasattrrš   r‘   rd   r÷   rª   ri   r«   r¨   rØ   rè   Úquery_pre_attn_scalarrâ   Úattention_dropoutÚuse_bidirectional_attentionÚ	is_causalrk   rl   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚattn_logit_softcappingÚsliding_windowÚ
is_slidingrv   Úrms_norm_epsÚq_normÚk_norm©rT   rd   r÷   rU   s      €r<   rP   zGemma3Attention.__init__>  sß  ø€ Ý‰Œ×ÒÑÔÐÝ;BÀ6È=Ñ;YÔ;YÐc˜&Ô,¨YÔ7Ð7Ð_cˆŒØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!ØÔ3°TÑ9ˆŒØ!%¤Ô!>ˆÔØ!œ[ÔDÐDˆŒå”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒð '+¤kÔ&HˆÔ#Ø7;´ÐJ]Ò7]Ð7]˜fÔ3Ð3ÐcgˆÔØœ/Ð-@Ò@ˆŒå#¨¬¸VÔ=PÐQÑQÔQˆŒÝ#¨¬¸VÔ=PÐQÑQÔQˆŒˆˆr;   NrB   Úposition_embeddingsrà   rA   rï   r¥   c                 ó‚  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|                      |¦  «        }|                      |	¦  «        }	|\  }}t          ||	||¦  «        \  }}	|�| 
                    |	|
| j        ¦  «        \  }	}
t          j        | j        j        t           ¦  «        } || ||	|
|f| j        r| j        nd| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr~   r+   r)   rÛ   )rá   râ   r  )rˆ   r¨   r   Úviewrº   r  r  r  r	  rÓ   Úupdater÷   r   Úget_interfacerd   Ú_attn_implementationrô   rç   rü   râ   r  rÖ   rî   r  )rT   rB   r  rà   rA   rï   Úinput_shapeÚhidden_shapeÚquery_statesrð   rñ   r¼   r½   Úattention_interfaceró   rò   s                   r<   rY   zGemma3Attention.forward\  sé  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà—{’{ <Ñ0Ô0ˆØ—[’[ Ñ,Ô,ˆ
à&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð /3¬mÐD�DÔ*Ð*ÀØ”LØÔ.ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r;   )NNN)r3   r4   r5   r6   r-   r]   rP   r7   r_   r	   r    r"   rD   rY   r`   ra   s   @r<   rö   rö   :  sï   ø€ € € € € àGÐGðRÐ/ð R¸Cð Rð Rð Rð Rð Rð RðB -1Ø.2Ø(,ð*)ð *)à”|ð*)ð #œ\ð*)ð œ tÑ+ð	*)ð
  ™ð*)ð Ð+Ô,ð*)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð*)ð *)ð *)ð *)ð *)ð *)ð *)ð *)r;   rö   c                   óà   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 ddej        dej        dej        dz  dej        dz  d	e	dz  d
e
e         deej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚGemma3DecoderLayerrd   r÷   c                 óÐ  •— t          ¦   «                              ¦   «          || _        |j        | _        || _        t          ||¬¦  «        | _        t          |¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        d S )N)rd   r÷   ©ry   )rO   rP   rd   ri   r÷   rö   Ú	self_attnrc   Úmlprv   r  Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormr
  s      €r<   rP   zGemma3DecoderLayer.__init__Š  sÇ   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ"ˆŒÝ(°À)ÐLÑLÔLˆŒÝ˜VÑ$Ô$ˆŒÝ,¨TÔ-=À6ÔCVÐWÑWÔWˆÔÝ(5°dÔ6FÈFÔL_Ð(`Ñ(`Ô(`ˆÔ%Ý)6°tÔ7GÈVÔM`Ð)aÑ)aÔ)aˆÔ&Ý*7¸Ô8HÈfÔNaÐ*bÑ*bÔ*bˆÔ'Ð'Ð'r;   NrB   r  rà   r¾   rA   rï   r¥   c           	      ó   — |}|                       |¦  «        } | j        d|||||dœ|¤Ž\  }}|                      |¦  «        }||z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z   }|S )N)rB   r  rà   r¾   rA   r:   )r  r  r  r  r  r  )	rT   rB   r  rà   r¾   rA   rï   ÚresidualÚ_s	            r<   rY   zGemma3DecoderLayer.forward–  sÂ   € ð !ˆà×,Ò,¨]Ñ;Ô;ˆà)˜4œ>ð 
Ø'Ø 3Ø)Ø%Ø+ð
ð 
ð ð
ð 
Ñˆ�qð ×5Ò5°mÑDÔDˆØ  =Ñ0ˆà ˆØ×6Ò6°}ÑEÔEˆØŸš Ñ/Ô/ˆØ×7Ò7¸ÑFÔFˆØ  =Ñ0ˆàÐr;   rÄ   )r3   r4   r5   r-   r]   rP   r7   r_   Ú
LongTensorr	   r    r"   rD   r8   rY   r`   ra   s   @r<   r  r  ‰  sÿ   ø€ € € € € ð
cÐ/ð 
c¸Cð 
cð 
cð 
cð 
cð 
cð 
cð -1Ø.2Ø04Ø(,ðð à”|ðð #œ\ðð œ tÑ+ð	ð
 Ô&¨Ñ-ðð  ™ðð Ð+Ô,ðð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uðð ð ð ð ð ð ð r;   r  c                   óŒ   ‡ — e Zd ZU eed<   dZdZg d¢ZdgZdZ	dZ
dZdZdZeedœZdZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )	ÚGemma3PreTrainedModelrd   ÚmodelT)r  ÚSiglipVisionEmbeddingsÚSiglipEncoderLayerÚ#SiglipMultiheadAttentionPoolingHeadrA   )rB   rC   )ÚimageÚtextc                 óÌ  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rt	          j        |j        ¦  «         d S d|j        j        v rt	          j        |j	        ¦  «         d S t          |t          ¦  «        r!t	          j        |j        |j        ¦  «         d S t          |t          ¦  «        r›|j        D ]•}|j        }|j        |         dk    rt$          |j        |                  } ||j        |¬¦  «        \  }}t	          j        t+          ||› d�¦  «        |¦  «         t	          j        t+          ||› d�¦  «        |¦  «         Œ”d S d S )NÚRMSNormr�   r�   r’   r“   )rO   Ú_init_weightsr·   ÚGemma3MultiModalProjectorÚinitÚzeros_Úmm_input_projection_weightrU   r3   r[   rF   Ú	constant_rK   rQ   r‹   rš   rœ   rŽ   r   rd   Úcopy_rª   )rT   rÜ   r‘   r    r¡   r!  rU   s         €r<   r-  z#Gemma3PreTrainedModel._init_weightsÏ  sx  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ7Ñ8Ô8ð 	^ÝŒK˜Ô9Ñ:Ô:Ð:Ð:Ð:à˜&Ô*Ô3Ð3Ð3ÝŒK˜œÑ&Ô&Ð&Ð&Ð&Ý˜Õ =Ñ>Ô>ð 		^ÝŒN˜6Ô-¨vÔ/HÑIÔIÐIÐIÐIÝ˜Õ 5Ñ6Ô6ð 	^Ø$Ô0ð ^ð ^�
Ø%ÔE�ØÔ# JÔ/°9Ò<Ð<Ý#6°vÔ7GÈ
Ô7SÔ#T�LØ#/ <°´È*Ð#UÑ#UÔ#UÑ �˜qÝ”
�7 6¨jÐ+CÐ+CÐ+CÑDÔDÀmÑTÔTÐTÝ”
�7 6¨jÐ+LÐ+LÐ+LÑMÔMÈ}Ñ]Ô]Ð]Ð]ð	^ð 	^ð^ð ^r;   )r3   r4   r5   r,   r9   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr  rö   Ú_can_record_outputsÚinput_modalitiesr7   rÆ   r-  r`   ra   s   @r<   r$  r$  ·  sÂ   ø€ € € € € € àÐÐÑØÐØ&*Ð#ðð ð Ðð $5Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðà+Ø%ðð Ðð )Ðà€U„]�_„_ð^ð ^ð ^ð ^ñ „_ð^ð ^ð ^ð ^ð ^r;   r$  r  c           
      óZ   ‡ — dt           dt           dt           dt           dt          f
ˆ fd„}|S )zA
    Enables a bidirectional mask within the sliding window.
    Ú	batch_idxÚhead_idxÚq_idxÚkv_idxr¥   c                 ó0   •— t          ||z
  ¦  «        ‰k     S )z“A token can attend to any other token if their absolute distance is within
        the (exclusive) sliding window size (distance < sliding_window).)Úabs)r@  rA  rB  rC  r  s       €r<   Ú
inner_maskz1_bidirectional_window_overlay.<locals>.inner_maskè  s   ø€ õ �5˜6‘>Ñ"Ô" ^Ò3Ð3r;   )r]   Úbool)r  rF  s   ` r<   Ú_bidirectional_window_overlayrH  ã  sL   ø€ ð
4�cð 4­Sð 4½ð 4Åcð 4Ídð 4ð 4ð 4ð 4ð 4ð 4ð
 Ðr;   c                   óò   ‡ — e Zd ZU eed<   dZdefˆ fd„Zeee		 	 	 	 	 	 dde
j        dz  de
j        dz  de
j        dz  dedz  d	e
j        dz  d
edz  dee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚGemma3TextModelrd   ©r*  c                 óð  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          ‰j        ‰j        | j        | j        j        dz  ¬¦  «        | _        t          j
        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t#          ‰¦  «        | _        d| _        |                      ¦   «          d S )Nç      à?)rK   c                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r:   )r  )Ú.0r÷   rd   s     €r<   ú
<listcomp>z,Gemma3TextModel.__init__.<locals>.<listcomp>ÿ  s$   ø€ ÐdÐdÐd°yÕ ¨	Ñ2Ô2ÐdÐdÐdr;   r  F)rO   rP   Úpad_token_idrJ   Ú
vocab_sizerF   ri   rd   Úembed_tokensrk   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayersrv   r  Únormr‹   Ú
rotary_embÚgradient_checkpointingÚ	post_initrr   s    `€r<   rP   zGemma3TextModel.__init__õ  së   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒõ :ØÔ˜vÔ1°4Ô3CÐQUÔQ\ÔQhÐjmÑQmð
ñ 
ô 
ˆÔõ ”mØdÐdÐdÐdÅEÈ&ÔJbÑDcÔDcÐdÑdÔdñ
ô 
ˆŒõ " &Ô"4¸&Ô:MÐNÑNÔNˆŒ	Ý/°Ñ7Ô7ˆŒØ&+ˆÔ#ð 	�ŠÑÔÐÐÐr;   NrV   rà   r¾   rA   Úinputs_embedsÚ	use_cacherï   r¥   c           	      óÌ  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          |x}	t          ¦  «        sh| j        ||||dœ}
|
                     ¦   «         }| j        j        r"d„ |
d<   t          | j        j        ¦  «        |d<   t!          di |
¤Žt#          di |¤Žd	œ}	|}i }t%          | j        j        ¦  «        D ]}|                      |||¦  «        ||<   Œt+          | j        d | j        j        …         ¦  «        D ]?\  }} ||f|	| j        j        |                  || j        j        |                  ||d
œ|¤Ž}Œ@|                      |¦  «        }t3          ||¬¦  «        S )Nú:You must specify exactly one of input_ids or inputs_embeds©rd   r   r+   ©r£   ©rd   r\  rà   rA   r¾   c                  óB   — t          j        dt           j        ¬¦  «        S )NTr©   )r7   rS   rG  )Úargss    r<   ú<lambda>z)Gemma3TextModel.forward.<locals>.<lambda>0  s   € ÅÄÈTÕY^ÔYcÐ@dÑ@dÔ@d€ r;   Úor_mask_function©Úfull_attentionrù   )rà   r  r¾   rA   )Úlast_hidden_staterA   r:   )Ú
ValueErrorrS  r
   rd   Úget_seq_lengthr7   r¬   rˆ   r£   rÍ   r·   ÚdictÚcopyrý   rH  r  r   r   r™   rš   rY  Ú	enumeraterW  rV  rX  r   )rT   rV   rà   r¾   rA   r\  r]  rï   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsÚsliding_mask_kwargsrB   r  r‘   ÚiÚdecoder_layers                    r<   rY   zGemma3TextModel.forward  sq  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLõ °Ð?Ð-ÅÑFÔFð 	ð œ+Ø!.Ø"0Ø#2Ø ,ðð ˆKð #.×"2Ò"2Ñ"4Ô"4ÐàŒ{Ô6ð tØ2dÐ2d�Ð.Ñ/Ý:WÐX\ÔXcÔXrÑ:sÔ:sÐ#Ð$6Ñ7õ #5Ð"CÐ"C°{Ð"CÐ"CÝ%FÐ%]Ð%]ÐI\Ð%]Ð%]ð#ð #Ðð &ˆØ ÐÝ˜dœkÔ5Ñ6Ô6ð 	gð 	gˆJØ.2¯oªo¸mÈ\Ð[eÑ.fÔ.fÐ 
Ñ+Ð+å )¨$¬+Ð6U¸¼Ô8UÐ6UÔ*VÑ WÔ Wð 	ð 	ÑˆAˆ}Ø)˜MØðà2°4´;Ô3JÈ1Ô3MÔNØ$7¸¼Ô8OÐPQÔ8RÔ$SØ)Ø /ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå&Ø+Ø+ð
ñ 
ô 
ð 	
r;   )NNNNNN)r3   r4   r5   r-   r9   r>  rP   r'   r(   r#   r7   r"  r_   r	   r8   rG  r    r"   r   rY   r`   ra   s   @r<   rJ  rJ  ð  s3  ø€ € € € € € àÐÐÑØ ÐðÐ/ð ð ð ð ð ð ð&  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ðC
ð C
àÔ# dÑ*ðC
ð œ tÑ+ðC
ð Ô&¨Ñ-ð	C
ð
  ™ðC
ð Ô(¨4Ñ/ðC
ð ˜$‘;ðC
ð Ð+Ô,ðC
ð 
!ðC
ð C
ð C
ñ „^ñ „_ñ  ÔðC
ð C
ð C
ð C
ð C
r;   rJ  c                   ó*  ‡ — e Zd ZU ddiZddiZddgdgfiZeed<   defˆ fd„Ze	e
	 	 	 	 	 	 	 	 ddej        d	z  dej        d	z  dej        d	z  ded	z  dej        d	z  dej        d	z  ded	z  deej        z  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚGemma3ForCausalLMúlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrB   r@   rd   c                 óþ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S rf   )
rO   rP   rJ  r%  rR  rk   rl   ri   rx  r[  rr   s     €r<   rP   zGemma3ForCausalLM.__init__X  sj   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý$ VÑ,Ô,ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒð 	�ŠÑÔÐÐÐr;   Nr   rV   rà   r¾   rA   r\  Úlabelsr]  Úlogits_to_keeprï   r¥   c	           
      óÀ  —  | j         d||||||dœ|	¤Ž}
|
j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }| j        j        �2|| j        j        z  }t          j	        |¦  «        }|| j        j        z  }d}|� | j
        ||| j        fi |	¤Ž}t          |||
j        |
j        |
j        ¬¦  «        S )a‚  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, Gemma3ForCausalLM

        >>> model = Gemma3ForCausalLM.from_pretrained("google/gemma-2-9b")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-9b")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```)rV   rà   r¾   rA   r\  r]  N)r?   r@   rA   rB   rC   r:   )r%  ri  r·   r]   Úslicerx  rd   Úfinal_logit_softcappingr7   rê   Úloss_functionrR  r   rA   rB   rC   )rT   rV   rà   r¾   rA   r\  r{  r]  r|  rï   ÚoutputsrB   Úslice_indicesr@   r?   s                  r<   rY   zGemma3ForCausalLM.forwarda  s&  € ð@ ,6¨4¬:ð ,
ØØ)Ø%Ø+Ø'Øð,
ð ,
ð ð,
ð ,
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆØŒ;Ô.Ð:Ø˜dœkÔAÑAˆFÝ”Z Ñ'Ô'ˆFØ˜dœkÔAÑAˆFàˆØÐØ%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDå%ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r;   )NNNNNNNr   )r3   r4   r5   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr-   r9   rP   r$   r#   r7   r"  r_   r	   r8   rG  r]   r    r"   r   rY   r`   ra   s   @r<   rv  rv  Q  sf  ø€ € € € € € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐÐÑðÐ/ð ð ð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð;
ð ;
àÔ# dÑ*ð;
ð œ tÑ+ð;
ð Ô&¨Ñ-ð	;
ð
  ™ð;
ð Ô(¨4Ñ/ð;
ð Ô  4Ñ'ð;
ð ˜$‘;ð;
ð ˜eœlÑ*ð;
ð Ð+Ô,ð;
ð 
 ð;
ð ;
ð ;
ñ „^ñ Ôð;
ð ;
ð ;
ð ;
ð ;
r;   rv  c                   ó:   ‡ — e Zd Zdefˆ fd„Zdej        fd„Zˆ xZS )r.  rd   c                 ó  •— t          ¦   «                              ¦   «          t          j        t	          j        |j        j        |j        j        ¦  «        ¦  «        | _	        t          |j        j        |j        j        ¬¦  «        | _        t          |j        j        |j        j        z  ¦  «        | _        t          |j        dz  ¦  «        | _        | j        | j        z  | _        t          j        | j        | j        ¬¦  «        | _        d S )Nr  rM  )Úkernel_sizeÚstride)rO   rP   rk   r{   r7   r|   Úvision_configri   Útext_configr1  rv   Úlayer_norm_epsÚmm_soft_emb_normr]   Ú
image_sizeÚ
patch_sizeÚpatches_per_imageÚmm_tokens_per_imageÚtokens_per_siderˆ  Ú	AvgPool2dÚavg_poolrr   s     €r<   rP   z"Gemma3MultiModalProjector.__init__¢  sà   ø€ Ý‰Œ×ÒÑÔÐå*,¬,ÝŒK˜Ô,Ô8¸&Ô:LÔ:XÑYÔYñ+
ô +
ˆÔ'õ !.ØÔ Ô,°&Ô2FÔ2Uð!
ñ !
ô !
ˆÔõ "% VÔ%9Ô%DÈÔH\ÔHgÑ%gÑ!hÔ!hˆÔÝ" 6Ô#=¸sÑ#BÑCÔCˆÔØÔ1°TÔ5IÑIˆÔÝœ°Ô1AÈ$ÔJZÐ[Ñ[Ô[ˆŒˆˆr;   Úvision_outputsc                 ó¸  — |j         \  }}}|                     dd¦  «        }|                     ||| j        | j        ¦  «        }|                     ¦   «         }|                      |¦  «        }|                     d¦  «        }|                     dd¦  «        }|                      |¦  «        }t          j	        || j
        ¦  «        }|                     |¦  «        S )Nr+   r)   )rˆ   rº   rÖ   r�  rî   r”  Úflattenr�  r7   ré   r1  r…   )	rT   r•  Ú
batch_sizer!  ri   Úreshaped_vision_outputsÚpooled_vision_outputsÚnormed_vision_outputsÚprojected_vision_outputss	            r<   rY   z!Gemma3MultiModalProjector.forward²  sÜ   € Ø%3Ô%9Ñ"ˆ
�A�{à"0×":Ò":¸1¸aÑ"@Ô"@ÐØ"9×"AÒ"AØ˜ TÔ%;¸TÔ=Sñ#
ô #
Ðð #:×"DÒ"DÑ"FÔ"FÐà $§¢Ð.EÑ FÔ FÐØ 5× =Ò =¸aÑ @Ô @ÐØ 5× ?Ò ?ÀÀ1Ñ EÔ EÐà $× 5Ò 5Ð6KÑ LÔ LÐå#(¤<Ð0EÀtÔGfÑ#gÔ#gÐ Ø'×/Ò/°Ñ?Ô?Ð?r;   )	r3   r4   r5   r,   rP   r7   r_   rY   r`   ra   s   @r<   r.  r.  ¡  sq   ø€ € € € € ð\˜|ð \ð \ð \ð \ð \ð \ð @ e¤lð @ð @ð @ð @ð @ð @ð @ð @r;   r.  Útoken_type_idsr£   c                 ó$  — | dk                          |¬¦  «        }t          j                             |dd¬¦  «        d d …d d…f         }|| z  }t	          j        |                     ¦   «         d¬¦  «        dz
  }t	          j        ||d¦  «        }|S )Nr+   ra  )r+   r   r   )rß   r~   rµ   )rZ   rk   rë   Úpadr7   Úcumsumr]   Úwhere)r�  r£   Úis_imageÚis_previous_imageÚnew_image_startÚ	group_idsÚblock_sequence_idss          r<   Úget_block_sequence_ids_for_maskr§  Å  sš   € ð  !Ò#×'Ò'¨vÐ'Ñ6Ô6€HÝœ×)Ò)¨(°FÀ!Ð)ÑDÔDÀQÀQÀQÈÈÈÀVÔLÐØÐ"3Ð!3Ñ3€OÝ”˜_×0Ò0Ñ2Ô2¸Ð:Ñ:Ô:¸QÑ>€IÝœ X¨y¸"Ñ=Ô=ÐØÐr;   rd   r\  rA   r¾   r¦  c                 óú   — | ||||dœ}t          di |¤d|i¤Ž}t          di |¤ddi¤Ž\  }}	}	}	}
}	}|r|}nt          |||
|¦  «        }t          di |¤t          |¦  «        t	          | j        ¦  «        dœ¤Ž}||dœS )zãCreate full_attention and sliding_attention masks with correct composition.

    For global (full attention) layers:  OR(causal, blockwise)
    For local (sliding window) layers:  AND(sliding_window, OR(causal, blockwise))
    rb  r¦  r÷   r   )rf  Úand_mask_functionrg  r:   )r   r   r   r   r   r  )rd   r\  rà   rA   r¾   r¦  rq  Ú	full_maskÚ
early_exitr!  Ú	kv_lengthÚ	kv_offsetÚpadded_block_sequence_idsÚsliding_masks                 r<   Úcreate_masks_for_vision_modelr°  Ð  s
  € ð Ø&Ø(Ø*Ø$ðð €Kõ #ÐXÐX [ÐXÐXÐEWÐXÐXÐX€Iõ 4Nð 4ð 4Ø
ð4ð 4àð4ð 4ð 4Ñ0€J��1�a˜ A yð ð 
Ø$6Ð!Ð!å$@Ø °	¸9ñ%
ô %
Ð!õ &ð ð Ø
ðå*Ð+DÑEÔEÝ0°Ô1FÑGÔGðð ð ð €Lð $Ø)ðð ð r;   zy
    The Base Gemma3 model which consists of a vision backbone and a language model without language modeling head.,
    c                   ó´  ‡ — e Zd ZdZdefˆ fd„Ze ed¬¦  «        dej	        de
e         deez  fd	„¦   «         ¦   «         Zd
ej        dej	        dej	        fd„Zee	 	 	 	 	 	 	 	 	 dd
ej        dz  dej	        dz  dej        dz  dej        dz  dedz  dej        dz  dej	        dz  dej        dz  dedz  de
e         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚGemma3ModelFrd   c                 ó<  •— t          ¦   «                              |¦  «         t          j        |j        ¬¦  «        | _        t          |¦  «        | _        |j        j	        | _	        t          j        |j        ¬¦  «        }|| _
        |                      ¦   «          d S )Nr`  )rO   rP   r*   Úfrom_configrŠ  Úvision_towerr.  Úmulti_modal_projectorr‹  rR  Úlanguage_modelr[  )rT   rd   r·  rU   s      €r<   rP   zGemma3Model.__init__  s…   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý%Ô1¸Ô9MÐNÑNÔNˆÔÝ%>¸vÑ%FÔ%FˆÔ"Ø Ô,Ô7ˆŒå"Ô.°fÔ6HÐIÑIÔIˆØ,ˆÔØ�ŠÑÔÐÐÐr;   zOProjects the last hidden state from the vision model into language model space.r.   Úpixel_valuesrï   r¥   c                 óh   —  | j         d|ddœ|¤Ž}|j        }|                      |¦  «        |_        |S )NT)r¸  Úreturn_dictr:   )rµ  ri  r¶  Úpooler_output)rT   r¸  rï   r•  ri  s        r<   Úget_image_featureszGemma3Model.get_image_features  sK   € ð
 +˜Ô*Ða¸ÐRVÐaÐaÐZ`ÐaÐaˆØ*Ô<ÐØ'+×'AÒ'AÐBSÑ'TÔ'TˆÔ$àÐr;   rV   r\  Úimage_featuresc                 ó   — |€e| |                       ¦   «         t          j        | j        j        t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }n|| j        j        k    }|                     ¦   «         }|j	        d         |j	        d         z  }| 
                    d¦  «                             |j        ¦  «        }t          ||j	        d         z  |                     ¦   «         k    d|› d|› �¦  «         |S )zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        N)r\   r£   r~   r   r+   z6Image features and image tokens do not match, tokens: z, features: )Úget_input_embeddingsr7   rS   rd   Úimage_token_idÚlongr£   ÚallÚsumrˆ   rÍ   rZ   r%   Únumel)rT   rV   r\  r½  Úspecial_image_maskÚn_image_tokensÚn_image_featuress          r<   Úget_placeholder_maskz Gemma3Model.get_placeholder_mask"  s  € ð ÐØ!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”˜Tœ[Ô7½u¼zÐR_ÔRfÐgÑgÔgñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;ÐÐà!*¨d¬kÔ.HÒ!HÐà+×/Ò/Ñ1Ô1ˆØ)Ô/°Ô2°^Ô5IÈ!Ô5LÑLÐØ/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐÝØ˜]Ô0°Ô4Ñ4¸×8LÒ8LÑ8NÔ8NÒNØsÀ^ÐsÐsÐaqÐsÐsñ	
ô 	
ð 	
ð "Ð!r;   Nrà   r¾   rA   r�  r{  r]  Ú	lm_kwargsc
           
      ó   — |du |duz  rt          d¦  «        ‚|�?| j        j        | j        k    r*|| j        j        k    }|                     ¦   «         }d||<   n|}|€ |                      ¦   «         |¦  «        }|�j|                      |d¬¦  «        j        }|                     |j	        |j
        ¦  «        }|                      |||¬¦  «        }|                     ||¦  «        }t          |x}t          ¦  «        sR| j                             ¦   «         ||||dœ}|�%t!          ||j	        ¬¦  «        }t#          dd	|i|¤Ž}nt%          di |¤Ž} | j        d|||||	dd
œ|
¤Ž}t)          |j        |j        |j        |j        |�|nd¬¦  «        S )a®  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration

        >>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma32-3b-mix-224")
        >>> processor = AutoProcessor.from_pretrained("google/gemma32-3b-mix-224")

        >>> prompt = "Where is the cat standing?"
        >>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=image, text=prompt,  return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(**inputs,)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Where is the cat standing?\nsnow"
        ```Nr_  r   T)rº  )r\  r½  rb  ra  r¦  )rà   r¾   rA   r\  r]  rº  )ri  rA   rB   rC   r2   r:   )rj  rd   rÀ  rR  r�   r¿  r¼  r»  rZ   r£   r\   rÈ  Úmasked_scatterr·   rl  Úget_text_configr§  r°  r   r·  r1   ri  rA   rB   rC   )rT   rV   r¸  rà   r¾   rA   r�  r\  r{  r]  rÉ  rÅ  Úllm_input_idsr½  rp  rq  r¦  r�  s                     r<   rY   zGemma3Model.forward:  s*  € ðX ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZð Ð  T¤[Ô%?À4Ä?Ò%RÐ%RØ!*¨d¬kÔ.HÒ!HÐØ%ŸOšOÑ-Ô-ˆMØ01ˆMÐ,Ñ-Ð-à%ˆMàÐ Ø7˜D×5Ò5Ñ7Ô7¸ÑFÔFˆMð Ð#Ø!×4Ò4°\ÈtÐ4ÑTÔTÔbˆNØ+×.Ò.¨}Ô/CÀ]ÔEXÑYÔYˆNØ!%×!:Ò!:Ø¨À~ð ";ñ "ô "Ðð *×8Ò8Ð9KÈ^Ñ\Ô\ˆMõ °Ð?Ð-ÅÑFÔFð 	Oàœ+×5Ò5Ñ7Ô7Ø!.Ø"0Ø#2Ø ,ðð ˆKð Ð)Ý%DÀ^Ð\iÔ\pÐ%qÑ%qÔ%qÐ"Ý&Cð 'ð 'Ø'9ð'à!ð'ð 'Ð#Ð#õ
 '@Ð&NÐ&NÀ+Ð&NÐ&NÐ#à%�$Ô%ð 
Ø.Ø%Ø+Ø'ØØð
ð 
ð ð
ð 
ˆõ )Ø%Ô7Ø#Ô3Ø!Ô/ØÔ)Ø2>Ð2J  ÐPTð
ñ 
ô 
ð 	
r;   )	NNNNNNNNN)r3   r4   r5   Úaccepts_loss_kwargsr,   rP   r$   r#   r7   r8   r    r"   rD   r   r¼  r"  rÈ  r_   r	   rG  r1   rY   r`   ra   s   @r<   r²  r²    sû  ø€ € € € € ð  Ðð˜|ð ð ð ð ð ð ð Ø€^Ð!rÐsÑsÔsðØ!Ô-ðØ9?Ð@RÔ9Sðà	Ð+Ñ	+ðð ð ñ tÔsñ Ôðð"ØÔ)ð"Ø:?Ô:Kð"Ø]bÔ]nð"ð "ð "ð "ð0 Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.Ø!%ðd
ð d
àÔ# dÑ*ðd
ð Ô'¨$Ñ.ðd
ð œ tÑ+ð	d
ð
 Ô&¨Ñ-ðd
ð  ™ðd
ð Ô(¨4Ñ/ðd
ð Ô(¨4Ñ/ðd
ð Ô  4Ñ'ðd
ð ˜$‘;ðd
ð Ð.Ô/ðd
ð 
Ð*Ñ	*ðd
ð d
ð d
ñ „^ñ Ôðd
ð d
ð d
ð d
ð d
r;   r²  c                   ó  ‡ — e Zd ZddiZdZdefˆ fd„Zedej	        de
e         fd„¦   «         Zee	 	 	 	 	 	 	 	 	 	 ddej        d	z  dej	        d	z  dej        d	z  dej        d	z  ded	z  dej        d	z  dej	        d	z  dej        d	z  ded	z  deej        z  de
e         deez  fd„¦   «         ¦   «         Z	 	 	 	 	 	 	 	 	 	 dˆ fd„	Ze	 	 ddedej        dej        d	z  ded	z  dej        d	z  dej        d	z  ded	z  defd„¦   «         Zˆ xZS )ÚGemma3ForConditionalGenerationrw  z(model.language_model.embed_tokens.weightFrd   c                 óú   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        j        |j        j        d¬¦  «        | _	        |  
                    ¦   «          d S rf   )rO   rP   r²  r%  rk   rl   r‹  ri   rR  rx  r[  rr   s     €r<   rP   z'Gemma3ForConditionalGeneration.__init__®  se   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý  Ñ(Ô(ˆŒ
Ý”y Ô!3Ô!?ÀÔASÔA^ÐejÐkÑkÔkˆŒØ�ŠÑÔÐÐÐr;   r¸  rï   c                 ó(   —  | j         j        |fi |¤ŽS rX   )r%  r¼  )rT   r¸  rï   s      r<   r¼  z1Gemma3ForConditionalGeneration.get_image_features´  s   € à,ˆtŒzÔ,¨\ÐDÐD¸VÐDÐDÐDr;   Nr   rV   rà   r¾   rA   r�  r\  r{  r]  r|  rÉ  r¥   c                 ó  —  | j         d	||||||||	|ddœ
|¤Ž}|d         }t          |
t          ¦  «        rt          |
 d¦  «        n|
}|                      |dd…|dd…f         ¦  «        }d}|��i|                     ¦   «         }|ddd…dd…f         }|ddd…f         }|�Ÿ|dd…|j        d          d…f                              |j        ¦  «        }||                     |j        ¦  «        dk              	                    ¦   «         }||                     |j        ¦  «        dk              	                    ¦   «         }n(| 	                    ¦   «         }| 	                    ¦   «         }t          j        ¦   «         }|                     d| j        j        j        ¦  «        }|                     d¦  «                             |j        ¦  «        } |||¦  «        }t!          |||j        |j        |j        |j        ¬¦  «        S )
a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration

        >>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma-3-4b-it")
        >>> processor = AutoProcessor.from_pretrained("google/gemma-3-4b-it")

        >>> messages = [
        ...     {
        ...         "role": "system",
        ...         "content": [
        ...             {"type": "text", "text": "You are a helpful assistant."}
        ...         ]
        ...     },
        ...     {
        ...         "role": "user", "content": [
        ...             {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
        ...             {"type": "text", "text": "Where is the cat standing?"},
        ...         ]
        ...     },
        ... ]

        >>> inputs = processor.apply_chat_template(
        ...     messages,
        ...     tokenize=True,
        ...     return_dict=True,
        ...     return_tensors="pt",
        ...     add_generation_prompt=True
        ... )
        >>> # Generate
        >>> generate_ids = model.generate(**inputs)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "user\nYou are a helpful assistant.\n\n\n\n\n\nWhere is the cat standing?\nmodel\nBased on the image, the cat is standing in a snowy area, likely outdoors. It appears to"
        ```
        T)
rV   r¸  r�  rà   r¾   rA   r\  r]  r{  rº  r   N.r~   r+   )r?   r@   rA   rB   rC   r2   r:   )r%  r·   r]   r~  rx  r^   rˆ   rZ   r£   rî   rk   ÚCrossEntropyLossr  rd   r‹  rR  r>   rA   rB   rC   r2   )rT   rV   r¸  rà   r¾   rA   r�  r\  r{  r]  r|  rÉ  r�  rB   r‚  r@   r?   Úshift_logitsÚshift_labelsÚshift_attention_maskÚloss_fctÚflat_logitsÚflat_labelss                          r<   rY   z&Gemma3ForConditionalGeneration.forward¸  s5  € ðz �$”*ð 
ØØ%Ø)Ø)Ø%Ø+Ø'ØØØð
ð 
ð ð
ð 
ˆð   œ
ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÑà—\’\‘^”^ˆFØ! # s¨ s¨A¨A¨A +Ô.ˆLØ! # q r r 'œ?ˆLØÐ)ð (6°a°a°a¸,Ô:LÈQÔ:OÐ9OÐ9QÐ9QÐ6QÔ'R×'UÒ'UÐV\ÔVcÑ'dÔ'dÐ$Ø+Ð,@×,CÒ,CÀFÄMÑ,RÔ,RÐVWÒ,WÔX×cÒcÑeÔe�Ø+Ð,@×,CÒ,CÀLÔDWÑ,XÔ,XÐ\]Ò,]Ô^×iÒiÑkÔk��à+×6Ò6Ñ8Ô8�Ø+×6Ò6Ñ8Ô8�åÔ*Ñ,Ô,ˆHà&×+Ò+¨B°´Ô0GÔ0RÑSÔSˆKØ&×+Ò+¨BÑ/Ô/×2Ò2°<Ô3FÑGÔGˆKØ�8˜K¨Ñ5Ô5ˆDå+ØØØ#Ô3Ø!Ô/ØÔ)Ø 'Ô ;ð
ñ 
ô 
ð 	
r;   Tc                 ól   •—  t          ¦   «         j        |f||||||	||dœ|¤Ž}|s|s||d<   nd |d<   |S )N)rA   r\  rà   r¾   r]  r|  r�  Úis_first_iterationr¸  r�  )rO   Úprepare_inputs_for_generation)rT   rV   rA   r\  r¾   r¸  rà   r�  r]  r|  r{  rÜ  rï   Úmodel_inputsrU   s                 €r<   rÝ  z<Gemma3ForConditionalGeneration.prepare_inputs_for_generation'  s}   ø€ ð  =•u‘w”wÔ<Øð
à+Ø'Ø)Ø%ØØ)Ø)Ø1ð
ð 
ð ð
ð 
ˆð" ð 	2 Yð 	2Ø+7ˆL˜Ñ(Ð(ð .2ˆLÐ)Ñ*àÐr;   rÜ  c                 óš   — |                       ¦   «         ||||dœ}|�$t          ||j        ¬¦  «        }	t          dd|	i|¤ŽS t	          di |¤ŽS )Nrb  ra  r¦  r:   )rÌ  r§  r£   r°  r   )
rd   r\  rà   rA   r¾   r�  rÜ  rï   rq  r¦  s
             r<   r   z8Gemma3ForConditionalGeneration.create_masks_for_generateP  s‡   € ð ×,Ò,Ñ.Ô.Ø*Ø,Ø.Ø(ð
ð 
ˆð Ð%Ý!@ÀÐXeÔXlÐ!mÑ!mÔ!mÐÝ0ð ð Ø#5ðàðð ð õ
 )Ð7Ð7¨;Ð7Ð7Ð7r;   )
NNNNNNNNNr   )
NNNNNNTNNF)NF)r3   r4   r5   rƒ  rÎ  r,   rP   r#   r7   r8   r    r"   r¼  r$   r"  r_   r	   rG  r]   rD   r>   rY   rÝ  rÅ   r   rl  r   r`   ra   s   @r<   rÐ  rÐ  £  s˜  ø€ € € € € ð +Ð,VÐWÐð  Ðð˜|ð ð ð ð ð ð ð ðE¨uÔ/@ð EÈFÐSeÔLfð Eð Eð Eñ „^ðEð Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.Ø!%Ø-.ðk
ð k
àÔ# dÑ*ðk
ð Ô'¨$Ñ.ðk
ð œ tÑ+ð	k
ð
 Ô&¨Ñ-ðk
ð  ™ðk
ð Ô(¨4Ñ/ðk
ð Ô(¨4Ñ/ðk
ð Ô  4Ñ'ðk
ð ˜$‘;ðk
ð ˜eœlÑ*ðk
ð Ð.Ô/ðk
ð 
Ð-Ñ	-ðk
ð k
ð k
ñ „^ñ Ôðk
ð` ØØØØØØØØØ ð'ð 'ð 'ð 'ð 'ð 'ðR ð /3Ø*/ð8ð 8Ø ð8à”|ð8ð œ tÑ+ð8ð  ™ð	8ð
 ”l TÑ)ð8ð œ tÑ+ð8ð ! 4™Kð8ð 
ð8ð 8ð 8ñ „\ð8ð 8ð 8ð 8ð 8r;   rÐ  zÎ
Gemma3TextForSequenceClassification is a text-only sequence classification model that works with Gemma3TextConfig.
It uses the generic sequence classification implementation for efficiency and consistency.c                   ó   — e Zd ZU eed<   dZdS )Ú#Gemma3TextForSequenceClassificationrd   rK  N)r3   r4   r5   r-   r9   r>  r:   r;   r<   rá  rá  m  s&   € € € € € € ð ÐÐÑØ ÐÐÐr;   rá  c                   óÜ   ‡ — e Zd Z	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dedz  dej        dz  dej        dz  d	ej        dz  d
ee	         de
fˆ fd„Zˆ xZS )ÚGemma3ForSequenceClassificationNrV   r¸  rà   r¾   rA   r�  r\  r{  rï   r¥   c	                 óH   •—  t          ¦   «         j        d||||||||dœ|	¤ŽS )N)rV   rà   r¾   rA   r\  r¸  r�  r{  r:   )rO   rY   )rT   rV   r¸  rà   r¾   rA   r�  r\  r{  rï   rU   s             €r<   rY   z'Gemma3ForSequenceClassification.forwardx  sK   ø€ ð �u‰wŒwŒð 

ØØ)Ø%Ø+Ø'Ø%Ø)Øð

ð 

ð ð

ð 

ð 
	
r;   )NNNNNNNN)r3   r4   r5   r7   r"  r8   r_   r	   r    r"   r   rY   r`   ra   s   @r<   rã  rã  w  sÿ   ø€ € € € € ð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.ð
ð 
àÔ# dÑ*ð
ð Ô'¨$Ñ.ð
ð œ tÑ+ð	
ð
 Ô&¨Ñ-ð
ð  ™ð
ð Ô(¨4Ñ/ð
ð Ô(¨4Ñ/ð
ð Ô  4Ñ'ð
ð Ð+Ô,ð
ð 
*ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r;   rã  )r$  rJ  rv  rÐ  r²  rã  rá  )r+   )rÛ   NNrX   )aÚcollections.abcr   Údataclassesr   Útypingr   r7   Útorch.nnrk   Ú r   r/  Úactivationsr   Úcache_utilsr	   r
   Úconfiguration_utilsr   Ú
generationr   Úintegrationsr   r   Úmasking_utilsr   r   r   r   r   r   r   Úmodeling_layersr   r   Úmodeling_outputsr   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr    Úutilsr!   r"   r#   r$   r%   Úutils.genericr&   r'   Úutils.output_capturingr(   Úautor*   Úconfiguration_gemma3r,   r-   r1   r>   Ú	EmbeddingrF   ÚModulerc   rv   r‹   rÊ   rÓ   r_   r]   rÚ   r^   rD   rô   rö   r  r$  rG  rH  rJ  rv  r.  r£   r§  rl  r°  r²  rÐ  rá  rã  Ú__all__r:   r;   r<   ú<module>rý     sk  ðð* %Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø )Ð )Ð )Ð )Ð )Ð )Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ Iðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð \Ð [Ð [Ð [Ð [Ð [Ð [Ð [ðð ð ð ð ð ð ð ð ð ð ð ð LÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø nÐ nÐ nÐ nÐ nÐ nÐ nÐ nÐ nÐ nÐ nÐ nÐ nÐ nØ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø Ð Ð Ð Ð Ð Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @ð €ððñ ô ð
 ð9ð 9ð 9ð 9ð 9Ð 7ñ 9ô 9ñ „ñô ð9ð €ððñ ô ð
 ð9ð 9ð 9ð 9ð 9 ;ñ 9ô 9ñ „ñô ð9ð0Sð Sð Sð Sð S B¤Lñ Sô Sð Sðð ð ð ð �”	ñ ô ð ð =ð =ð =ð =ð =�B”Iñ =ô =ð =ð(L<ð L<ð L<ð L<ð L<˜BœIñ L<ô L<ð L<ð^(ð (ð (ð ÐÐ*Ñ+Ô+ðð ð ñ ,Ô+ðð2	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð$ Ø Ø ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �S‰[ð%ð �T‰\ð%ð �T‰\ð%ð ˆ5Œ<˜œÐ%Ô&ð%ð %ð %ð %ðD ÐÐ)Ñ*Ô*ðK)ð K)ð K)ð K)ð K)�b”iñ K)ô K)ñ +Ô*ðK)ð\+ð +ð +ð +ð +Ð3ñ +ô +ð +ð\ ð(^ð (^ð (^ð (^ð (^˜Oñ (^ô (^ñ „ð(^ðV
°#ð 
¸(ÀCÈÈcÐSVÐCWÐY]ÐC]Ô:^ð 
ð 
ð 
ð 
ð ð]
ð ]
ð ]
ð ]
ð ]
Ð+ñ ]
ô ]
ñ „ð]
ð@ ðL
ð L
ð L
ð L
ð L
Ð-¨ñ L
ô L
ñ „ðL
ð^!@ð !@ð !@ð !@ð !@ ¤	ñ !@ô !@ð !@ðHð °E´Lð È%Ì,ÐY]ÑJ]ð ÐinÔiuð ð ð ð ð1Øð1à”<ð1ð ”L 4Ñ'ð1ð ˜T‘\ð	1ð
 ”, Ñ%ð1ð œð1ð 
ð1ð 1ð 1ð 1ðh €ððñ ô ð
W
ð W
ð W
ð W
ð W
Ð'ñ W
ô W
ñô ð
W
ðt €ððñ ô ð
B8ð B8ð B8ð B8ð B8Ð%:¸Oñ B8ô B8ñô ð
B8ðJ €ð^ðñ ô ð
!ð !ð !ð !ð !Ð*JÐLañ !ô !ñô ð
!ð

ð 
ð 
ð 
ð 
Ð&FÐH]ñ 
ô 
ð 
ð4ð ð €€€r;   