§
    ‚Štj¾£  ã                   óô  — d dl mZ d dlmZmZ d dlZd dlmZ d dlm	Z	 ddl
mZ ddlmZmZ ddlmZ dd	lmZmZmZmZmZmZmZ dd
lmZmZ ddlmZmZm Z  ddl!m"Z"m#Z# ddl$m%Z%m&Z& ddl'm(Z( ddl)m*Z*m+Z+m,Z,m-Z- ddl.m/Z/ ddl0m1Z1 ddl2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z; ddl<m=Z=m>Z>m?Z?m@Z@ ddlAmBZB  e-jC        eD¦  «        ZE e+d¬¦  «        e	 G d„ de1e¦  «        ¦   «         ¦   «         ZF e+d¬¦  «        e	 G d„ de¦  «        ¦   «         ¦   «         ZG G d„ de@¦  «        ZH G d„ de=¦  «        ZI G d „ d!ejJ        ¦  «        ZK G d"„ d#e5¦  «        ZL G d$„ d%e8¦  «        ZM G d&„ d'e9ejN        ¦  «        ZO G d(„ d)e3¦  «        ZP G d*„ d+e¦  «        ZQdZR G d,„ d-e7¦  «        ZSd.eTd/eeTeTeTeTgeUf         fd0„ZV G d1„ d2e6¦  «        ZW G d3„ d4e4¦  «        ZX G d5„ d6ejN        ¦  «        ZYdLd7ejZ        d8ej[        dz  d/ejZ        fd9„Z\d:ed;ejZ        d<ejZ        dz  d=edz  d>ejZ        dz  d?ejZ        d/e]fd@„Z^ G dA„ dBe?¦  «        Z_ G dC„ dDe>¦  «        Z` e+dE¬F¦  «         G dG„ dHeeS¦  «        ¦   «         Za G dI„ dJeeS¦  «        Zbg dK¢ZcdS )Mé    )ÚCallable)ÚAnyÚOptionalN)Ústricté   )Úinitialization)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Ú_preprocess_mask_argumentsÚblockwise_overlayÚcreate_causal_maskÚcreate_masks_for_generateÚ!create_sliding_window_causal_maskÚmaybe_pad_block_sequence_idsÚsliding_window_overlay)Ú GenericForSequenceClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚ SequenceClassifierOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmaybe_autocasté   )ÚGemma2Config)	ÚGemma2AttentionÚGemma2ForCausalLMÚ	Gemma2MLPÚGemma2ModelÚGemma2PreTrainedModelÚGemma2RMSNormÚGemma2RotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)ÚPaliGemmaCausalLMOutputWithPastÚ!PaliGemmaForConditionalGenerationÚPaliGemmaModelÚPaligemmaModelOutputWithPast)ÚSiglipVisionConfigzgoogle/gemma-3-4b-it)Ú
checkpointc            
       óÒ   — e Zd ZU dZdZddddddddddœ	Zddd	œZd
Zee	d<   dZ
ee	d<   dZee         dz  e	d<   dZedz  e	d<   dZedz  e	d<   dZedz  e	d<   dZedz  e	d<   d„ Zd„ ZdS )ÚGemma3TextConfigaé  
    query_pre_attn_scalar (`float`, *optional*, defaults to 256):
        scaling factor used on the attention scores
    final_logit_softcapping (`float`, *optional*):
        Scaling factor when applying tanh softcapping on the logits.
    attn_logit_softcapping (`float`, *optional*):
        Scaling factor when applying tanh softcapping on the attention scores.
    use_bidirectional_attention (`bool`, *optional*, defaults to `False`):
        If True, the model will attend to all text tokens instead of using a causal mask. This does not change
        behavior for vision tokens.

    ```python
    >>> from transformers import Gemma3TextModel, Gemma3TextConfig
    >>> # Initializing a Gemma3Text gemma3_text-7b style configuration
    >>> configuration = Gemma3TextConfig()
    >>> # Initializing a model from the gemma3_text-7b style configuration
    >>> model = Gemma3TextModel(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    Úgemma3_textÚcolwiseÚreplicated_with_grad_allreduceÚrowwise)	zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.q_normzlayers.*.self_attn.k_normzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projg    €„.Ag     ˆÃ@)ÚglobalÚlocali@  Ú
vocab_sizei   Úmax_position_embeddingsNÚlayer_typesÚfinal_logit_softcappingÚattn_logit_softcappingÚrope_parametersFÚuse_bidirectional_attentionc                 óê   ‡ — ‰ j         r‰ j        dz  dz   ‰ _        |                     dd¦  «        ‰ _        ‰ j        €%ˆ fd„t          ‰ j        ¦  «        D ¦   «         ‰ _        t          j        di |¤Ž d S )Nr"   é   Úsliding_window_patterné   c                 óL   •— g | ] }t          |d z   ‰j        z  ¦  «        rdnd‘Œ!S )rC   Úsliding_attentionÚfull_attention)ÚboolÚ_sliding_window_pattern)Ú.0ÚiÚselfs     €úg/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/gemma3/modular_gemma3.pyú
<listcomp>z2Gemma3TextConfig.__post_init__.<locals>.<listcomp>{   sI   ø€ ð  ð  ð  àõ (,¨Q°©U°dÔ6RÑ,RÑ'SÔ'SÐiÐ#Ð#ÐYið ð  ð  ó    © )	rA   Úsliding_windowÚgetrJ   r=   ÚrangeÚnum_hidden_layersr   Ú__post_init__)rM   Úkwargss   ` rN   rV   zGemma3TextConfig.__post_init__s   s›   ø€ ØÔ+ð 	AØ#'Ô#6¸!Ñ#;¸qÑ"@ˆDÔð (.§z¢zÐ2JÈAÑ'NÔ'NˆÔ$àÔÐ#ð ð  ð  ð  å˜tÔ5Ñ6Ô6ð ñ  ô  ˆDÔõ
 	Ô&Ð0Ð0¨Ð0Ð0Ð0Ð0Ð0rP   c                 ór  — |                      dd ¦  «        }ddiddidœ}| j        �| j        n|| _        |� | j        d                              |¦  «         | j                             d¦  «        €ddi| j        d<   | j        d                              d|                      d| j        d         ¦  «        ¦  «         | j                             d¦  «        €ddi| j        d<   | j        d                              d|                      d	| j        d
         ¦  «        ¦  «         |                      ¦   «          |S )NÚrope_scalingÚ	rope_typeÚdefault)rG   rH   rH   Ú
rope_thetar9   rG   Úrope_local_base_freqr:   )Úpopr@   ÚupdaterS   Ú
setdefaultÚdefault_thetaÚstandardize_rope_params)rM   rW   rY   Údefault_rope_paramss       rN   Úconvert_rope_params_to_dictz,Gemma3TextConfig.convert_rope_params_to_dict‚   s_  € Ø—z’z .°$Ñ7Ô7ˆð
 #.¨yÐ!9Ø*¨IÐ6ð
ð 
Ðð 8<Ô7KÐ7W˜tÔ3Ð3Ð]pˆÔØÐ#ØÔ Ð!1Ô2×9Ò9¸,ÑGÔGÐGð Ô×#Ò#Ð$4Ñ5Ô5Ð=Ø6AÀ9Ð5MˆDÔ Ð!1Ñ2ØÔÐ-Ô.×9Ò9Ø˜&Ÿ*š* \°4Ô3EÀhÔ3OÑPÔPñ	
ô 	
ð 	
ð Ô×#Ò#Ð$7Ñ8Ô8Ð@Ø9DÀiÐ8PˆDÔ Ð!4Ñ5ØÔÐ0Ô1×<Ò<Ø˜&Ÿ*š*Ð%;¸TÔ=OÐPWÔ=XÑYÔYñ	
ô 	
ð 	
ð
 	×$Ò$Ñ&Ô&Ð&ØˆrP   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚbase_model_tp_planra   r;   ÚintÚ__annotations__r<   r=   ÚlistÚstrr>   Úfloatr?   r@   ÚdictrA   rI   rV   rd   rQ   rP   rN   r4   r4   D   s	  € € € € € € ðð ð, €Jà%.Ø%.Ø%.Ø%EØ%EØ%.Ø"+Ø )Ø"+ð
ð 
Ðð  +°XÐ>Ð>€Mà€J�ÐÐÑØ#*Ð˜SÐ*Ð*Ñ*Ø$(€K��c”˜TÑ!Ð(Ð(Ñ(Ø,0Ð˜U T™\Ð0Ð0Ñ0Ø+/Ð˜E D™LÐ/Ð/Ñ/Ø#'€O�T˜D‘[Ð'Ð'Ñ'Ø/4Ð ¨¡Ð4Ð4Ñ4ð1ð 1ð 1ðð ð ð ð rP   r4   c                   ó
  ‡ — e Zd ZU dZdZddddœZeedœZdZ	ee
eef         z  dz  ed	<   dZee
eef         z  dz  ed
<   dZedz  ed<   dZedz  ed<   dZedz  ed<   dZedz  ed<   dZedz  ed<   dZedz  ed<   ˆ fd„Zˆ xZS )ÚGemma3Configa  
    mm_tokens_per_image (`int`, *optional*, defaults to 256):
        The number of tokens per image embedding.
    boi_token_index (`int`, *optional*, defaults to 255999):
        The begin-of-image token index to wrap the image prompt.
    eoi_token_index (`int`, *optional*, defaults to 256000):
        The end-of-image token index to wrap the image prompt.

    Example:

    ```python
    >>> from transformers import Gemma3ForConditionalGeneration, Gemma3Config, SiglipVisionConfig, Gemma3TextConfig

    >>> # Initializing a Siglip-like vision config
    >>> vision_config = SiglipVisionConfig()

    >>> # Initializing a Gemma3 Text config
    >>> text_config = Gemma3TextConfig()

    >>> # Initializing a Gemma3 gemma-3-4b style configuration
    >>> configuration = Gemma3Config(vision_config, text_config)

    >>> # Initializing a model from the gemma-3-4b style configuration
    >>> model = Gemma3TextConfig(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Úgemma3Úimage_token_indexÚboi_token_indexÚeoi_token_index)Úimage_token_idÚboi_token_idÚeoi_token_id)Útext_configÚvision_configNrz   r{   é   Úmm_tokens_per_imageiÿç i è i   g{®Gáz”?Úinitializer_rangeTÚtie_word_embeddingsc                 óÎ  •— | j         €.t          ¦   «         | _         t                               d¦  «         n0t	          | j         t
          ¦  «        rt          di | j         ¤Ž| _         t	          | j        t
          ¦  «        rt          di | j        ¤Ž| _        n4| j        €-t          ¦   «         | _        t                               d¦  «          t          ¦   «         j	        di |¤Ž d S )Nz@text_config is None, using default Gemma3TextConfig text config.zFvision_config is None, using default SiglipVisionConfig vision config.rQ   )
rz   r4   ÚloggerÚinfoÚ
isinstancerp   r{   r1   ÚsuperrV   )rM   rW   Ú	__class__s     €rN   rV   zGemma3Config.__post_init__Ô   sÞ   ø€ ØÔÐ#Ý/Ñ1Ô1ˆDÔÝ�KŠKÐZÑ[Ô[Ð[Ð[Ý˜Ô(­$Ñ/Ô/ð 	DÝ/ÐCÐC°$Ô2BÐCÐCˆDÔå�dÔ(­$Ñ/Ô/ð 	bÝ!3Ð!IÐ!I°dÔ6HÐ!IÐ!IˆDÔÐØÔÐ'Ý!3Ñ!5Ô!5ˆDÔÝ�KŠKÐ`ÑaÔaÐaà�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'rP   )re   rf   rg   rh   ri   Úattribute_mapr4   r1   Úsub_configsrz   rp   rn   r   rl   r{   r}   rk   ru   rv   rt   r~   ro   r   rI   rV   Ú__classcell__©r…   s   @rN   rr   rr       s9  ø€ € € € € € ðð ð: €Jà-Ø)Ø)ðð €Mð (Ø+ðð €Kð
 =A€KÐ! D¨¨c¨¤NÑ2°TÑ9Ð@Ð@Ñ@Ø@D€MÐ%¨¨S°#¨X¬Ñ6¸Ñ=ÐDÐDÑDØ&)Ð˜˜t™Ð)Ð)Ñ)Ø")€O�S˜4‘ZÐ)Ð)Ñ)Ø")€O�S˜4‘ZÐ)Ð)Ñ)Ø$+Ð�s˜T‘zÐ+Ð+Ñ+Ø&*Ð�u˜t‘|Ð*Ð*Ñ*Ø'+Ð˜ ™Ð+Ð+Ñ+ð(ð (ð (ð (ð (ð (ð (ð (ð (rP   rr   c                   ó   — e Zd ZdS )ÚGemma3ModelOutputWithPastN©re   rf   rg   rQ   rP   rN   r‹   r‹   ä   ó   € € € € € Ø€DrP   r‹   c                   ó   — e Zd ZdS )ÚGemma3CausalLMOutputWithPastNrŒ   rQ   rP   rN   r�   r�   è   r�   rP   r�   c            	       óP   ‡ — e Zd ZdZd
dedededefˆ fd„Zdej        fˆ fd	„Z	ˆ xZ
S )ÚGemma3TextScaledWordEmbeddingz\
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    ç      ð?Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scalec                 ó°   •— t          ¦   «                              |||¦  «         || _        |                      dt	          j        |¦  «        d¬¦  «         d S )Nr–   F©Ú
persistent)r„   Ú__init__Úscalar_embed_scaleÚregister_bufferÚtorchÚtensor)rM   r“   r”   r•   r–   r…   s        €rN   rš   z&Gemma3TextScaledWordEmbedding.__init__ñ   sS   ø€ Ý‰Œ×Ò˜¨¸ÑDÔDÐDØ"-ˆÔØ×Ò˜]­E¬L¸Ñ,EÔ,EÐRWÐÑXÔXÐXÐXÐXrP   Ú	input_idsc                 ó�   •— t          ¦   «                              |¦  «        | j                             | j        j        ¦  «        z  S ©N)r„   Úforwardr–   ÚtoÚweightÚdtype)rM   rŸ   r…   s     €rN   r¢   z%Gemma3TextScaledWordEmbedding.forwardö   s4   ø€ Ý‰wŒw�Š˜yÑ)Ô)¨DÔ,<×,?Ò,?ÀÄÔ@QÑ,RÔ,RÑRÐRrP   )r’   )re   rf   rg   rh   rk   ro   rš   r�   ÚTensorr¢   rˆ   r‰   s   @rN   r‘   r‘   ì   s¦   ø€ € € € € ðð ðYð Y sð Y¸3ð YÈSð YÐ_dð Yð Yð Yð Yð Yð Yð
S ¤ð Sð Sð Sð Sð Sð Sð Sð Sð Sð SrP   r‘   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )Ú	Gemma3MLPÚconfigc                 óJ   •— t          ¦   «                              |¦  «         d S r¡   ©r„   rš   ©rM   r©   r…   s     €rN   rš   zGemma3MLP.__init__û   s!   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ð Ð rP   )re   rf   rg   r4   rš   rˆ   r‰   s   @rN   r¨   r¨   ú   sE   ø€ € € € € ð!Ð/ð !ð !ð !ð !ð !ð !ð !ð !ð !ð !rP   r¨   c                   ó*   ‡ — e Zd Zddedefˆ fd„Zˆ xZS )ÚGemma3RMSNormç�íµ ÷Æ°>ÚdimÚepsc                 óN   •— t          ¦   «                              ||¬¦  «         d S )N©r°   r±   r«   )rM   r°   r±   r…   s      €rN   rš   zGemma3RMSNorm.__init__   s&   ø€ Ý‰Œ×Ò˜S cÐÑ*Ô*Ð*Ð*Ð*rP   )r¯   )re   rf   rg   rk   ro   rš   rˆ   r‰   s   @rN   r®   r®   ÿ   sP   ø€ € € € € ð+ð +˜Cð + eð +ð +ð +ð +ð +ð +ð +ð +ð +ð +rP   r®   c                   óÀ   — e Zd Zdefd„Ze	 	 	 	 ddedz  ded         dedz  dedz  de	d	e
f         f
d
„¦   «         Z ej        ¦   «         edd„¦   «         ¦   «         ZdS )ÚGemma3RotaryEmbeddingr©   c                 ó|  — t           j                             | ¦  «         |j        | _        |j        | _        || _        t          t          |j	        ¦  «        ¦  «        | _	        i | _
        | j	        D ]È}| j        j        |         }|€Œ|d         | j
        |<   | j        }| j
        |         dk    rt          | j
        |                  } || j        |¬¦  «        \  }}|                      |› d�|d¬¦  «         |                      |› d�|                     ¦   «         d¬¦  «         t!          | |› d�|¦  «         ŒÉd S )	NrZ   r[   ©Ú
layer_typeÚ	_inv_freqFr˜   Ú_original_inv_freqÚ_attention_scaling)ÚnnÚModulerš   r<   Úmax_seq_len_cachedÚoriginal_max_seq_lenr©   rm   Úsetr=   rZ   r@   Úcompute_default_rope_parametersr   rœ   ÚcloneÚsetattr)rM   r©   r¸   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalings          rN   rš   zGemma3RotaryEmbedding.__init__  s]  € Ý
Œ	×Ò˜4Ñ Ô Ð Ø"(Ô"@ˆÔØ$*Ô$BˆÔ!ØˆŒÝ¥ FÔ$6Ñ 7Ô 7Ñ8Ô8ˆÔØˆŒØÔ*ð 	Uð 	UˆJØœ+Ô5°jÔAˆKØÐ"Øà)4°[Ô)AˆDŒN˜:Ñ&Ø%)Ô%IˆLØŒ~˜jÔ)¨YÒ6Ð6Ý2°4´>À*Ô3MÔN�Ø4@°LÀÄÐYcÐ4dÑ4dÔ4dÑ1ˆMÐ1Ø× Ò  JÐ!9Ð!9Ð!9¸=ÐUZÐ Ñ[Ô[Ð[Ø× Ò  JÐ!BÐ!BÐ!BÀM×DWÒDWÑDYÔDYÐfkÐ ÑlÔlÐlÝ�D˜ZÐ;Ð;Ð;Ð=SÑTÔTÐTÐTð	Uð 	UrP   NÚdeviceztorch.deviceÚseq_lenr¸   Úreturnztorch.Tensorc                 ó  — | j         |         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «                             |t          j        ¬¦  «        |z  z  z  }||fS )	a|  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        r\   Úhead_dimNr’   r   r"   ©r¥   )rÈ   r¥   )	r@   ÚgetattrÚhidden_sizeÚnum_attention_headsr�   ÚarangeÚint64r£   ro   )r©   rÈ   rÉ   r¸   Úbaser°   Úattention_factorÚinv_freqs           rN   rÁ   z5Gemma3RotaryEmbedding.compute_default_rope_parameters  s‘   € ð2 Ô% jÔ1°,Ô?ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐð Ø•U”\ ! S¨!µ5´;Ð?Ñ?Ô?×BÒBÈ&ÕX]ÔXcÐBÑdÔdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)rP   c                 ó|  — t          | |› d�¦  «        }t          | |› d�¦  «        }|d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬	¦  «        5  |                     ¦   «         |                     ¦   «         z   
                    dd
¦  «        }	t          j        |	|	fd¬¦  «        }
|
                     ¦   «         |z  }|
                     ¦   «         |z  }d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |                     |j        ¬¦  «        fS )Nr¹   r»   r   éÿÿÿÿrC   ÚmpsÚcpuF)Údevice_typeÚenabledr"   ©r°   rÍ   )rÎ   ro   ÚexpandÚshaper£   rÈ   rƒ   Útypern   r!   Ú	transposer�   ÚcatÚcosÚsinr¥   )rM   ÚxÚposition_idsr¸   rÕ   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedrÚ   ÚfreqsÚembrâ   rã   s                rN   r¢   zGemma3RotaryEmbedding.forward>  sâ  € õ ˜4 JÐ!9Ð!9Ð!9Ñ:Ô:ˆÝ# D¨ZÐ*KÐ*KÐ*KÑLÔLÐà$ T¨1¨1¨1¨d ]Ô3×9Ò9Ñ;Ô;×BÒBÀ<ÔCUÐVWÔCXÐZ\Ð^_Ñ`Ô`×cÒcÐdeÔdlÑmÔmÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	0ð 	0Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)Ð/Ñ/ˆCØ—'’'‘)”)Ð/Ñ/ˆCð		0ð 	0ð 	0ñ 	0ô 	0ð 	0ð 	0ð 	0ð 	0ð 	0ð 	0øøøð 	0ð 	0ð 	0ð 	0ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   Ã-BE=Å=FÆF©NNNNr¡   )re   rf   rg   r4   rš   Ústaticmethodr   rk   rn   Útuplero   rÁ   r�   Úno_gradr   r¢   rQ   rP   rN   rµ   rµ     sä   € € € € € ðUÐ/ð Uð Uð Uð Uð* à*.Ø+/Ø"Ø!%ð	!*ð !*Ø  4Ñ'ð!*à˜Ô(ð!*ð �t‘ð!*ð ˜$‘Jð	!*ð
 
ˆ~˜uÐ$Ô	%ð!*ð !*ð !*ñ „\ð!*ðF €U„]�_„_Øð<ð <ð <ñ Ôñ „_ð<ð <ð <rP   rµ   c                   óÎ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej        dej        dej        dz  dedz  d	e	e
         d
eej        ej        dz  eej                 dz  f         fd„Zˆ xZS )ÚGemma3Attentionr©   Ú	layer_idxc                 óB  •— t          ¦   «                              ||¦  «         | j        dk    r|j        nd | _        | j        dk    | _        | j        j         | _        t          |j	        |j
        ¬¦  «        | _        t          |j	        |j
        ¬¦  «        | _        d S )NrG   r³   )r„   rš   r¸   rR   Ú
is_slidingr©   rA   Ú	is_causalr®   rÌ   Úrms_norm_epsÚq_normÚk_norm©rM   r©   rñ   r…   s      €rN   rš   zGemma3Attention.__init__S  s�   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+Ø7;´ÐJ]Ò7]Ð7]˜fÔ3Ð3ÐcgˆÔØœ/Ð-@Ò@ˆŒØ!œ[ÔDÐDˆŒå#¨¬¸VÔ=PÐQÑQÔQˆŒÝ#¨¬¸VÔ=PÐQÑQÔQˆŒˆˆrP   NÚhidden_statesÚposition_embeddingsÚattention_maskÚpast_key_valuesrW   rÊ   c                 ó‚  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|                      |¦  «        }|                      |	¦  «        }	|\  }}t          ||	||¦  «        \  }}	|�| 
                    |	|
| j        ¦  «        \  }	}
t          j        | j        j        t           ¦  «        } || ||	|
|f| j        r| j        nd| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr×   rC   r"   g        )ÚdropoutÚscalingrR   )rÞ   rÌ   Úq_projÚviewrà   Úk_projÚv_projrö   r÷   r+   r_   rñ   r   Úget_interfacer©   Ú_attn_implementationr,   ÚtrainingÚattention_dropoutrÿ   rR   ÚreshapeÚ
contiguousÚo_proj)rM   rù   rú   rû   rü   rW   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesrâ   rã   Úattention_interfaceÚattn_outputÚattn_weightss                   rN   r¢   zGemma3Attention.forward\  sé  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà—{’{ <Ñ0Ô0ˆØ—[’[ Ñ,Ô,ˆ
à&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð /3¬mÐD�DÔ*Ð*ÀØ”LØÔ.ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(rP   )NNN)re   rf   rg   r4   rk   rš   r�   r¦   r	   r   r   rí   r¢   rˆ   r‰   s   @rN   rð   rð   R  sè   ø€ € € € € ðRÐ/ð R¸Cð Rð Rð Rð Rð Rð Rð -1Ø.2Ø(,ð*)ð *)à”|ð*)ð #œ\ð*)ð œ tÑ+ð	*)ð
  ™ð*)ð Ð+Ô,ð*)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð*)ð *)ð *)ð *)ð *)ð *)ð *)ð *)rP   rð   c                   óà   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 ddej        dej        dej        dz  dej        dz  d	e	dz  d
e
e         deej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚGemma3DecoderLayerr©   rñ   c                 óÐ  •— t          ¦   «                              ¦   «          || _        |j        | _        || _        t          ||¬¦  «        | _        t          |¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        d S )N)r©   rñ   ©r±   )r„   rš   r©   rÏ   rñ   rð   Ú	self_attnr¨   Úmlpr®   rõ   Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormrø   s      €rN   rš   zGemma3DecoderLayer.__init__Š  sÇ   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ"ˆŒÝ(°À)ÐLÑLÔLˆŒÝ˜VÑ$Ô$ˆŒÝ,¨TÔ-=À6ÔCVÐWÑWÔWˆÔÝ(5°dÔ6FÈFÔL_Ð(`Ñ(`Ô(`ˆÔ%Ý)6°tÔ7GÈVÔM`Ð)aÑ)aÔ)aˆÔ&Ý*7¸Ô8HÈfÔNaÐ*bÑ*bÔ*bˆÔ'Ð'Ð'rP   Nrù   rú   rû   rå   rü   rW   rÊ   c           	      ó   — |}|                       |¦  «        } | j        d|||||dœ|¤Ž\  }}|                      |¦  «        }||z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z   }|S )N)rù   rú   rû   rå   rü   rQ   )r  r  r  r  r  r  )	rM   rù   rú   rû   rå   rü   rW   ÚresidualÚ_s	            rN   r¢   zGemma3DecoderLayer.forward–  sÂ   € ð !ˆà×,Ò,¨]Ñ;Ô;ˆà)˜4œ>ð 
Ø'Ø 3Ø)Ø%Ø+ð
ð 
ð ð
ð 
Ñˆ�qð ×5Ò5°mÑDÔDˆØ  =Ñ0ˆà ˆØ×6Ò6°}ÑEÔEˆØŸš Ñ/Ô/ˆØ×7Ò7¸ÑFÔFˆØ  =Ñ0ˆàÐrP   rë   )re   rf   rg   r4   rk   rš   r�   r¦   Ú
LongTensorr	   r   r   rí   ÚFloatTensorr¢   rˆ   r‰   s   @rN   r  r  ‰  sÿ   ø€ € € € € ð
cÐ/ð 
c¸Cð 
cð 
cð 
cð 
cð 
cð 
cð -1Ø.2Ø04Ø(,ðð à”|ðð #œ\ðð œ tÑ+ð	ð
 Ô&¨Ñ-ðð  ™ðð Ð+Ô,ðð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uðð ð ð ð ð ð ð rP   r  c                   óN   — e Zd ZdZdZg d¢Z ej        ¦   «         d„ ¦   «         ZdS )ÚGemma3PreTrainedModelÚmodel)ÚimageÚtext)r  ÚSiglipVisionEmbeddingsÚSiglipEncoderLayerÚ#SiglipMultiheadAttentionPoolingHeadc                 ó²  — t          j        | |¦  «         t          |t          ¦  «        rt	          j        |j        ¦  «         d S d|j        j        v rt	          j        |j	        ¦  «         d S t          |t          ¦  «        r!t	          j        |j        |j        ¦  «         d S t          |t          ¦  «        r›|j        D ]•}|j        }|j        |         dk    rt$          |j        |                  } ||j        |¬¦  «        \  }}t	          j        t+          ||› d�¦  «        |¦  «         t	          j        t+          ||› d�¦  «        |¦  «         Œ”d S d S )NÚRMSNormr[   r·   r¹   rº   )r   Ú_init_weightsrƒ   ÚGemma3MultiModalProjectorÚinitÚzeros_Úmm_input_projection_weightr…   re   r¤   r‘   Ú	constant_r–   r›   rµ   r=   rÁ   rZ   r   r©   Úcopy_rÎ   )rM   Úmoduler¸   rÅ   rÆ   r  s         rN   r,  z#Gemma3PreTrainedModel._init_weightsÄ  sr  € åÔ% d¨FÑ3Ô3Ð3Ý�fÕ7Ñ8Ô8ð 	^ÝŒK˜Ô9Ñ:Ô:Ð:Ð:Ð:à˜&Ô*Ô3Ð3Ð3ÝŒK˜œÑ&Ô&Ð&Ð&Ð&Ý˜Õ =Ñ>Ô>ð 		^ÝŒN˜6Ô-¨vÔ/HÑIÔIÐIÐIÐIÝ˜Õ 5Ñ6Ô6ð 	^Ø$Ô0ð ^ð ^�
Ø%ÔE�ØÔ# JÔ/°9Ò<Ð<Ý#6°vÔ7GÈ
Ô7SÔ#T�LØ#/ <°´È*Ð#UÑ#UÔ#UÑ �˜qÝ”
�7 6¨jÐ+CÐ+CÐ+CÑDÔDÀmÑTÔTÐTÝ”
�7 6¨jÐ+LÐ+LÐ+LÑMÔMÈ}Ñ]Ô]Ð]Ð]ð	^ð 	^ð^ð ^rP   N)	re   rf   rg   Úbase_model_prefixÚinput_modalitiesÚ_no_split_modulesr�   rî   r,  rQ   rP   rN   r#  r#  º  sZ   € € € € € ØÐØ(Ððð ð Ðð €U„]�_„_ð^ð ^ñ „_ð^ð ^ð ^rP   r#  rR   rÊ   c           
      óZ   ‡ — dt           dt           dt           dt           dt          f
ˆ fd„}|S )zA
    Enables a bidirectional mask within the sliding window.
    Ú	batch_idxÚhead_idxÚq_idxÚkv_idxrÊ   c                 ó0   •— t          ||z
  ¦  «        ‰k     S )z“A token can attend to any other token if their absolute distance is within
        the (exclusive) sliding window size (distance < sliding_window).)Úabs)r8  r9  r:  r;  rR   s       €rN   Ú
inner_maskz1_bidirectional_window_overlay.<locals>.inner_maskÝ  s   ø€ õ �5˜6‘>Ñ"Ô" ^Ò3Ð3rP   )rk   rI   )rR   r>  s   ` rN   Ú_bidirectional_window_overlayr?  Ø  sL   ø€ ð
4�cð 4­Sð 4½ð 4Åcð 4Ídð 4ð 4ð 4ð 4ð 4ð 4ð
 ÐrP   c                   óÂ   ‡ — e Zd ZU eed<   dZdefˆ fd„Z	 	 	 	 	 	 ddej        dz  dej	        dz  dej        dz  de
dz  d	ej        dz  d
edz  dee         defd„Zˆ xZS )ÚGemma3TextModelr©   ©r&  c                 ó²   •— t          ¦   «                              |¦  «         t          |j        |j        | j        | j        j        dz  ¬¦  «        | _        d S )Nç      à?)r–   )r„   rš   r‘   r;   rÏ   r•   r©   Úembed_tokensr¬   s     €rN   rš   zGemma3TextModel.__init__é  sY   ø€ Ý‰Œ×Ò˜Ñ Ô Ð õ :ØÔ˜vÔ1°4Ô3CÐQUÔQ\ÔQhÐjmÑQmð
ñ 
ô 
ˆÔÐÐrP   NrŸ   rû   rå   rü   Úinputs_embedsÚ	use_cacherW   rÊ   c           	      óÌ  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          |x}	t          ¦  «        sh| j        ||||dœ}
|
                     ¦   «         }| j        j        r"d„ |
d<   t          | j        j        ¦  «        |d<   t!          di |
¤Žt#          di |¤Žd	œ}	|}i }t%          | j        j        ¦  «        D ]}|                      |||¦  «        ||<   Œt+          | j        d | j        j        …         ¦  «        D ]?\  }} ||f|	| j        j        |                  || j        j        |                  ||d
œ|¤Ž}Œ@|                      |¦  «        }t3          ||¬¦  «        S )Nú:You must specify exactly one of input_ids or inputs_embeds)r©   r   rC   ©rÈ   ©r©   rF  rû   rü   rå   c                  óB   — t          j        dt           j        ¬¦  «        S )NTrÍ   )r�   rž   rI   )Úargss    rN   ú<lambda>z)Gemma3TextModel.forward.<locals>.<lambda>  s   € ÅÄÈTÕY^ÔYcÐ@dÑ@dÔ@d€ rP   Úor_mask_function©rH   rG   )rû   rú   rå   rü   )Úlast_hidden_staterü   rQ   )Ú
ValueErrorrE  r
   r©   Úget_seq_lengthr�   rÑ   rÞ   rÈ   Ú	unsqueezerƒ   rp   ÚcopyrA   r?  rR   r   r   rÀ   r=   Ú
rotary_embÚ	enumerateÚlayersrU   Únormr   )rM   rŸ   rû   rå   rü   rF  rG  rW   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsÚsliding_mask_kwargsrù   rú   r¸   rL   Údecoder_layers                    rN   r¢   zGemma3TextModel.forwardñ  sq  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLõ °Ð?Ð-ÅÑFÔFð 	ð œ+Ø!.Ø"0Ø#2Ø ,ðð ˆKð #.×"2Ò"2Ñ"4Ô"4ÐàŒ{Ô6ð tØ2dÐ2d�Ð.Ñ/Ý:WÐX\ÔXcÔXrÑ:sÔ:sÐ#Ð$6Ñ7õ #5Ð"CÐ"C°{Ð"CÐ"CÝ%FÐ%]Ð%]ÐI\Ð%]Ð%]ð#ð #Ðð &ˆØ ÐÝ˜dœkÔ5Ñ6Ô6ð 	gð 	gˆJØ.2¯oªo¸mÈ\Ð[eÑ.fÔ.fÐ 
Ñ+Ð+å )¨$¬+Ð6U¸¼Ô8UÐ6UÔ*VÑ WÔ Wð 	ð 	ÑˆAˆ}Ø)˜MØðà2°4´;Ô3JÈ1Ô3MÔNØ$7¸¼Ô8OÐPQÔ8RÔ$SØ)Ø /ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå&Ø+Ø+ð
ñ 
ô 
ð 	
rP   )NNNNNN)re   rf   rg   r4   rl   r5  rš   r�   r   r¦   r	   r!  rI   r   r   r   r¢   rˆ   r‰   s   @rN   rA  rA  å  s  ø€ € € € € € ØÐÐÑØ Ðð
Ð/ð 
ð 
ð 
ð 
ð 
ð 
ð .2Ø.2Ø04Ø(,Ø26Ø!%ðC
ð C
àÔ# dÑ*ðC
ð œ tÑ+ðC
ð Ô&¨Ñ-ð	C
ð
  ™ðC
ð Ô(¨4Ñ/ðC
ð ˜$‘;ðC
ð Ð+Ô,ðC
ð 
!ðC
ð C
ð C
ð C
ð C
ð C
ð C
ð C
rP   rA  c                   ó0   ‡ — e Zd ZU eed<   defˆ fd„Zˆ xZS )ÚGemma3ForCausalLMr©   c                 ór   •— t          ¦   «                              |¦  «         t          |¦  «        | _        d S r¡   )r„   rš   rA  r$  r¬   s     €rN   rš   zGemma3ForCausalLM.__init__:  s.   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý$ VÑ,Ô,ˆŒ
ˆ
ˆ
rP   )re   rf   rg   r4   rl   rš   rˆ   r‰   s   @rN   r`  r`  7  sS   ø€ € € € € € ØÐÐÑð-Ð/ð -ð -ð -ð -ð -ð -ð -ð -ð -ð -rP   r`  c                   ó:   ‡ — e Zd Zdefˆ fd„Zdej        fd„Zˆ xZS )r-  r©   c                 ó  •— t          ¦   «                              ¦   «          t          j        t	          j        |j        j        |j        j        ¦  «        ¦  «        | _	        t          |j        j        |j        j        ¬¦  «        | _        t          |j        j        |j        j        z  ¦  «        | _        t          |j        dz  ¦  «        | _        | j        | j        z  | _        t          j        | j        | j        ¬¦  «        | _        d S )Nr  rD  )Úkernel_sizeÚstride)r„   rš   r¼   Ú	Parameterr�   Úzerosr{   rÏ   rz   r0  r®   Úlayer_norm_epsÚmm_soft_emb_normrk   Ú
image_sizeÚ
patch_sizeÚpatches_per_imager}   Útokens_per_siderd  Ú	AvgPool2dÚavg_poolr¬   s     €rN   rš   z"Gemma3MultiModalProjector.__init__@  sà   ø€ Ý‰Œ×ÒÑÔÐå*,¬,ÝŒK˜Ô,Ô8¸&Ô:LÔ:XÑYÔYñ+
ô +
ˆÔ'õ !.ØÔ Ô,°&Ô2FÔ2Uð!
ñ !
ô !
ˆÔõ "% VÔ%9Ô%DÈÔH\ÔHgÑ%gÑ!hÔ!hˆÔÝ" 6Ô#=¸sÑ#BÑCÔCˆÔØÔ1°TÔ5IÑIˆÔÝœ°Ô1AÈ$ÔJZÐ[Ñ[Ô[ˆŒˆˆrP   Úvision_outputsc                 ó¸  — |j         \  }}}|                     dd¦  «        }|                     ||| j        | j        ¦  «        }|                     ¦   «         }|                      |¦  «        }|                     d¦  «        }|                     dd¦  «        }|                      |¦  «        }t          j	        || j
        ¦  «        }|                     |¦  «        S )NrC   r"   )rÞ   rà   r  rl  r	  ro  Úflattenri  r�   Úmatmulr0  Útype_as)	rM   rp  Ú
batch_sizer  rÏ   Úreshaped_vision_outputsÚpooled_vision_outputsÚnormed_vision_outputsÚprojected_vision_outputss	            rN   r¢   z!Gemma3MultiModalProjector.forwardP  sÜ   € Ø%3Ô%9Ñ"ˆ
�A�{à"0×":Ò":¸1¸aÑ"@Ô"@ÐØ"9×"AÒ"AØ˜ TÔ%;¸TÔ=Sñ#
ô #
Ðð #:×"DÒ"DÑ"FÔ"FÐà $§¢Ð.EÑ FÔ FÐØ 5× =Ò =¸aÑ @Ô @ÐØ 5× ?Ò ?ÀÀ1Ñ EÔ EÐà $× 5Ò 5Ð6KÑ LÔ LÐå#(¤<Ð0EÀtÔGfÑ#gÔ#gÐ Ø'×/Ò/°Ñ?Ô?Ð?rP   )	re   rf   rg   rr   rš   r�   r¦   r¢   rˆ   r‰   s   @rN   r-  r-  ?  sq   ø€ € € € € ð\˜|ð \ð \ð \ð \ð \ð \ð @ e¤lð @ð @ð @ð @ð @ð @ð @ð @rP   r-  Útoken_type_idsrÈ   c                 ó$  — | dk                          |¬¦  «        }t          j                             |dd¬¦  «        d d …d d…f         }|| z  }t	          j        |                     ¦   «         d¬¦  «        dz
  }t	          j        ||d¦  «        }|S )NrC   rJ  )rC   r   r   )Úvaluer×   rÜ   )r£   r¼   Ú
functionalÚpadr�   Úcumsumrk   Úwhere)rz  rÈ   Úis_imageÚis_previous_imageÚnew_image_startÚ	group_idsÚblock_sequence_idss          rN   Úget_block_sequence_ids_for_maskr†  c  sš   € ð  !Ò#×'Ò'¨vÐ'Ñ6Ô6€HÝœ×)Ò)¨(°FÀ!Ð)ÑDÔDÀQÀQÀQÈÈÈÀVÔLÐØÐ"3Ð!3Ñ3€OÝ”˜_×0Ò0Ñ2Ô2¸Ð:Ñ:Ô:¸QÑ>€IÝœ X¨y¸"Ñ=Ô=ÐØÐrP   r©   rF  rû   rü   rå   r…  c                 óú   — | ||||dœ}t          di |¤d|i¤Ž}t          di |¤ddi¤Ž\  }}	}	}	}
}	}|r|}nt          |||
|¦  «        }t          di |¤t          |¦  «        t	          | j        ¦  «        dœ¤Ž}||dœS )zãCreate full_attention and sliding_attention masks with correct composition.

    For global (full attention) layers:  OR(causal, blockwise)
    For local (sliding window) layers:  AND(sliding_window, OR(causal, blockwise))
    rK  r…  rñ   r   )rO  Úand_mask_functionrP  rQ   )r   r   r   r   r   rR   )r©   rF  rû   rü   rå   r…  r\  Ú	full_maskÚ
early_exitr  Ú	kv_lengthÚ	kv_offsetÚpadded_block_sequence_idsÚsliding_masks                 rN   Úcreate_masks_for_vision_modelr�  n  s
  € ð Ø&Ø(Ø*Ø$ðð €Kõ #ÐXÐX [ÐXÐXÐEWÐXÐXÐX€Iõ 4Nð 4ð 4Ø
ð4ð 4àð4ð 4ð 4Ñ0€J��1�a˜ A yð ð 
Ø$6Ð!Ð!å$@Ø °	¸9ñ%
ô %
Ð!õ &ð ð Ø
ðå*Ð+DÑEÔEÝ0°Ô1FÑGÔGðð ð ð €Lð $Ø)ðð ð rP   c                   ó‚  ‡ — e Zd ZdZdefˆ fd„Ze ed¬¦  «        dej	        de
e         deez  fd	„¦   «         ¦   «         Zee	 	 	 	 	 	 	 	 	 ddej        d
z  dej	        d
z  dej        d
z  dej        d
z  ded
z  dej        d
z  dej	        d
z  dej        d
z  ded
z  de
e         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚGemma3ModelFr©   c                 óN   •— t          ¦   «                              |¦  «         | `d S r¡   )r„   rš   Útext_config_dtyper¬   s     €rN   rš   zGemma3Model.__init__¦  s'   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØÐ"Ð"Ð"rP   zOProjects the last hidden state from the vision model into language model space.©Úcustom_introÚpixel_valuesrW   rÊ   c                 óh   —  | j         d|ddœ|¤Ž}|j        }|                      |¦  «        |_        |S )NT)r–  Úreturn_dictrQ   )Úvision_towerrQ  Úmulti_modal_projectorÚpooler_output)rM   r–  rW   rp  rQ  s        rN   Úget_image_featureszGemma3Model.get_image_featuresª  sK   € ð
 +˜Ô*Ða¸ÐRVÐaÐaÐZ`ÐaÐaˆØ*Ô<ÐØ'+×'AÒ'AÐBSÑ'TÔ'TˆÔ$àÐrP   NrŸ   rû   rå   rü   rz  rF  ÚlabelsrG  Ú	lm_kwargsc
           
      ó   — |d u |d uz  rt          d¦  «        ‚|�?| j        j        | j        k    r*|| j        j        k    }|                     ¦   «         }d||<   n|}|€ |                      ¦   «         |¦  «        }|�j|                      |d¬¦  «        j        }|                     |j	        |j
        ¦  «        }|                      |||¬¦  «        }|                     ||¦  «        }t          |x}t          ¦  «        sR| j                             ¦   «         ||||dœ}|�%t!          ||j	        ¬¦  «        }t#          dd|i|¤Ž}nt%          di |¤Ž} | j        d|||||	dd	œ|
¤Ž}t)          |j        |j        |j        |j        |�|nd ¬
¦  «        S )NrI  r   T)r˜  )rF  Úimage_featuresrK  rJ  r…  )rû   rå   rü   rF  rG  r˜  )rQ  rü   rù   Ú
attentionsÚimage_hidden_statesrQ   )rR  r©   rw   r;   rÂ   Úget_input_embeddingsrœ  r›  r£   rÈ   r¥   Úget_placeholder_maskÚmasked_scatterrƒ   rp   Úget_text_configr†  r�  r   Úlanguage_modelr‹   rQ  rü   rù   r¡  )rM   rŸ   r–  rû   rå   rü   rz  rF  r�  rG  rž  Úspecial_image_maskÚllm_input_idsr   r[  r\  r…  Úoutputss                     rN   r¢   zGemma3Model.forwardµ  s)  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZð Ð  T¤[Ô%?À4Ä?Ò%RÐ%RØ!*¨d¬kÔ.HÒ!HÐØ%ŸOšOÑ-Ô-ˆMØ01ˆMÐ,Ñ-Ð-à%ˆMàÐ Ø7˜D×5Ò5Ñ7Ô7¸ÑFÔFˆMð Ð#Ø!×4Ò4°\ÈtÐ4ÑTÔTÔbˆNØ+×.Ò.¨}Ô/CÀ]ÔEXÑYÔYˆNØ!%×!:Ò!:Ø¨À~ð ";ñ "ô "Ðð *×8Ò8Ð9KÈ^Ñ\Ô\ˆMõ °Ð?Ð-ÅÑFÔFð 	Oàœ+×5Ò5Ñ7Ô7Ø!.Ø"0Ø#2Ø ,ðð ˆKð Ð)Ý%DÀ^Ð\iÔ\pÐ%qÑ%qÔ%qÐ"Ý&Cð 'ð 'Ø'9ð'à!ð'ð 'Ð#Ð#õ
 '@Ð&NÐ&NÀ+Ð&NÐ&NÐ#à%�$Ô%ð 
Ø.Ø%Ø+Ø'ØØð
ð 
ð ð
ð 
ˆõ )Ø%Ô7Ø#Ô3Ø!Ô/ØÔ)Ø2>Ð2J  ÐPTð
ñ 
ô 
ð 	
rP   )	NNNNNNNNN)re   rf   rg   Úaccepts_loss_kwargsrr   rš   r   r   r�   r!  r   r   rí   r   rœ  r   r¦   r	   rI   r‹   r¢   rˆ   r‰   s   @rN   r‘  r‘  ¢  sÄ  ø€ € € € € àÐð#˜|ð #ð #ð #ð #ð #ð #ð Ø€^Ð!rÐsÑsÔsðØ!Ô-ðØ9?Ð@RÔ9Sðà	Ð+Ñ	+ðð ð ñ tÔsñ Ôðð Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.Ø!%ðG
ð G
àÔ# dÑ*ðG
ð Ô'¨$Ñ.ðG
ð œ tÑ+ð	G
ð
 Ô&¨Ñ-ðG
ð  ™ðG
ð Ô(¨4Ñ/ðG
ð Ô(¨4Ñ/ðG
ð Ô  4Ñ'ðG
ð ˜$‘;ðG
ð Ð.Ô/ðG
ð 
Ð*Ñ	*ðG
ð G
ð G
ñ „^ñ ÔðG
ð G
ð G
ð G
ð G
rP   r‘  c                   ó¸  ‡ — e Zd ZdZee	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej	        dz  dej        dz  de
dz  d	ej        dz  d
ej        dz  dej        dz  dedz  deej	        z  dee         deez  fd„¦   «         ¦   «         Z	 	 	 	 	 	 	 	 	 	 dˆ fd„	Z	 	 dded
ej	        dej	        dz  de
dz  dej	        dz  d	ej	        dz  dedz  defd„Zˆ xZS )ÚGemma3ForConditionalGenerationFNr   rŸ   r–  rû   rå   rü   rz  rF  r�  rG  Úlogits_to_keeprž  rÊ   c                 ó  —  | j         d	||||||||	|ddœ
|¤Ž}|d         }t          |
t          ¦  «        rt          |
 d¦  «        n|
}|                      |dd…|dd…f         ¦  «        }d}|��i|                     ¦   «         }|ddd…dd…f         }|ddd…f         }|�Ÿ|dd…|j        d          d…f                              |j        ¦  «        }||                     |j        ¦  «        dk              	                    ¦   «         }||                     |j        ¦  «        dk              	                    ¦   «         }n(| 	                    ¦   «         }| 	                    ¦   «         }t          j        ¦   «         }|                     d| j        j        j        ¦  «        }|                     d¦  «                             |j        ¦  «        } |||¦  «        }t!          |||j        |j        |j        |j        ¬¦  «        S )
a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration

        >>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma-3-4b-it")
        >>> processor = AutoProcessor.from_pretrained("google/gemma-3-4b-it")

        >>> messages = [
        ...     {
        ...         "role": "system",
        ...         "content": [
        ...             {"type": "text", "text": "You are a helpful assistant."}
        ...         ]
        ...     },
        ...     {
        ...         "role": "user", "content": [
        ...             {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
        ...             {"type": "text", "text": "Where is the cat standing?"},
        ...         ]
        ...     },
        ... ]

        >>> inputs = processor.apply_chat_template(
        ...     messages,
        ...     tokenize=True,
        ...     return_dict=True,
        ...     return_tensors="pt",
        ...     add_generation_prompt=True
        ... )
        >>> # Generate
        >>> generate_ids = model.generate(**inputs)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "user\nYou are a helpful assistant.\n\n\n\n\n\nWhere is the cat standing?\nmodel\nBased on the image, the cat is standing in a snowy area, likely outdoors. It appears to"
        ```
        T)
rŸ   r–  rz  rû   rå   rü   rF  rG  r�  r˜  r   N.r×   rC   )ÚlossÚlogitsrü   rù   r¡  r¢  rQ   )r$  rƒ   rk   ÚsliceÚlm_headro   rÞ   r£   rÈ   r	  r¼   ÚCrossEntropyLossr  r©   rz   r;   r�   rü   rù   r¡  r¢  )rM   rŸ   r–  rû   rå   rü   rz  rF  r�  rG  r®  rž  rª  rù   Úslice_indicesr±  r°  Úshift_logitsÚshift_labelsÚshift_attention_maskÚloss_fctÚflat_logitsÚflat_labelss                          rN   r¢   z&Gemma3ForConditionalGeneration.forward  s5  € ðz �$”*ð 
ØØ%Ø)Ø)Ø%Ø+Ø'ØØØð
ð 
ð ð
ð 
ˆð   œ
ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÑà—\’\‘^”^ˆFØ! # s¨ s¨A¨A¨A +Ô.ˆLØ! # q r r 'œ?ˆLØÐ)ð (6°a°a°a¸,Ô:LÈQÔ:OÐ9OÐ9QÐ9QÐ6QÔ'R×'UÒ'UÐV\ÔVcÑ'dÔ'dÐ$Ø+Ð,@×,CÒ,CÀFÄMÑ,RÔ,RÐVWÒ,WÔX×cÒcÑeÔe�Ø+Ð,@×,CÒ,CÀLÔDWÑ,XÔ,XÐ\]Ò,]Ô^×iÒiÑkÔk��à+×6Ò6Ñ8Ô8�Ø+×6Ò6Ñ8Ô8�åÔ*Ñ,Ô,ˆHà&×+Ò+¨B°´Ô0GÔ0RÑSÔSˆKØ&×+Ò+¨BÑ/Ô/×2Ò2°<Ô3FÑGÔGˆKØ�8˜K¨Ñ5Ô5ˆDå+ØØØ#Ô3Ø!Ô/ØÔ)Ø 'Ô ;ð
ñ 
ô 
ð 	
rP   Tc                 ól   •—  t          ¦   «         j        |f||||||	||dœ|¤Ž}|s|s||d<   nd |d<   |S )N)rü   rF  rû   rå   rG  r®  rz  Úis_first_iterationr–  rz  )r„   Úprepare_inputs_for_generation)rM   rŸ   rü   rF  rå   r–  rû   rz  rG  r®  r�  r½  rW   Úmodel_inputsr…   s                 €rN   r¾  z<Gemma3ForConditionalGeneration.prepare_inputs_for_generationu  s}   ø€ ð  =•u‘w”wÔ<Øð
à+Ø'Ø)Ø%ØØ)Ø)Ø1ð
ð 
ð ð
ð 
ˆð" ð 	2 Yð 	2Ø+7ˆL˜Ñ(Ð(ð .2ˆLÐ)Ñ*àÐrP   r©   r½  c                 óš   — |                       ¦   «         ||||dœ}|�$t          ||j        ¬¦  «        }	t          dd|	i|¤ŽS t	          di |¤ŽS )NrK  rJ  r…  rQ   )r¦  r†  rÈ   r�  r   )
r©   rF  rû   rü   rå   rz  r½  rW   r\  r…  s
             rN   r   z8Gemma3ForConditionalGeneration.create_masks_for_generatež  s‡   € ð ×,Ò,Ñ.Ô.Ø*Ø,Ø.Ø(ð
ð 
ˆð Ð%Ý!@ÀÐXeÔXlÐ!mÑ!mÔ!mÐÝ0ð ð Ø#5ðàðð ð õ
 )Ð7Ð7¨;Ð7Ð7Ð7rP   )
NNNNNNNNNr   )
NNNNNNTNNF)NF)re   rf   rg   r«  r   r   r�   r   r!  r¦   r	   rI   rk   r   r   rí   r�   r¢   r¾  r   rp   r   rˆ   r‰   s   @rN   r­  r­    s  ø€ € € € € ð  ÐàØð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.Ø!%Ø-.ðk
ð k
àÔ# dÑ*ðk
ð Ô'¨$Ñ.ðk
ð œ tÑ+ð	k
ð
 Ô&¨Ñ-ðk
ð  ™ðk
ð Ô(¨4Ñ/ðk
ð Ô(¨4Ñ/ðk
ð Ô  4Ñ'ðk
ð ˜$‘;ðk
ð ˜eœlÑ*ðk
ð Ð.Ô/ðk
ð 
Ð-Ñ	-ðk
ð k
ð k
ñ „^ñ Ôðk
ð` ØØØØØØØØØ ð'ð 'ð 'ð 'ð 'ð 'ð^ /3Ø*/ð8ð 8Ø ð8à”|ð8ð œ tÑ+ð8ð  ™ð	8ð
 ”l TÑ)ð8ð œ tÑ+ð8ð ! 4™Kð8ð 
ð8ð 8ð 8ð 8ð 8ð 8ð 8ð 8rP   r­  zÎ
Gemma3TextForSequenceClassification is a text-only sequence classification model that works with Gemma3TextConfig.
It uses the generic sequence classification implementation for efficiency and consistency.r”  c                   ó   — e Zd ZU eed<   dZdS )Ú#Gemma3TextForSequenceClassificationr©   rB  N)re   rf   rg   r4   rl   r5  rQ   rP   rN   rÂ  rÂ  º  s&   € € € € € € ð ÐÐÑØ ÐÐÐrP   rÂ  c                   óÜ   ‡ — e Zd Z	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dedz  dej        dz  dej        dz  d	ej        dz  d
ee	         de
fˆ fd„Zˆ xZS )ÚGemma3ForSequenceClassificationNrŸ   r–  rû   rå   rü   rz  rF  r�  rW   rÊ   c	                 óH   •—  t          ¦   «         j        d||||||||dœ|	¤ŽS )N)rŸ   rû   rå   rü   rF  r–  rz  r�  rQ   )r„   r¢   )rM   rŸ   r–  rû   rå   rü   rz  rF  r�  rW   r…   s             €rN   r¢   z'Gemma3ForSequenceClassification.forwardÅ  sK   ø€ ð �u‰wŒwŒð 

ØØ)Ø%Ø+Ø'Ø%Ø)Øð

ð 

ð ð

ð 

ð 
	
rP   )NNNNNNNN)re   rf   rg   r�   r   r!  r¦   r	   r   r   r   r¢   rˆ   r‰   s   @rN   rÄ  rÄ  Ä  sÿ   ø€ € € € € ð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.ð
ð 
àÔ# dÑ*ð
ð Ô'¨$Ñ.ð
ð œ tÑ+ð	
ð
 Ô&¨Ñ-ð
ð  ™ð
ð Ô(¨4Ñ/ð
ð Ô(¨4Ñ/ð
ð Ô  4Ñ'ð
ð Ð+Ô,ð
ð 
*ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
rP   rÄ  )	rr   r4   r#  rA  r`  r­  r‘  rÄ  rÂ  r¡   )dÚcollections.abcr   Útypingr   r   r�   Útorch.nnr¼   Úhuggingface_hub.dataclassesr   Ú r   r.  Úcache_utilsr	   r
   Úconfiguration_utilsr   Úmasking_utilsr   r   r   r   r   r   r   Úmodeling_layersr   r   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r    Úutils.genericr!   Úgemma2.configuration_gemma2r#   Úgemma2.modeling_gemma2r$   r%   r&   r'   r(   r)   r*   r+   r,   Úpaligemma.modeling_paligemmar-   r.   r/   r0   Úsiglipr1   Ú
get_loggerre   r�   r4   rr   r‹   r�   Ú	Embeddingr‘   r¨   r®   r½   rµ   rð   r  ÚGEMMA3_START_DOCSTRINGr#  rk   rI   r?  rA  r`  r-  r¦   rÈ   r†  rp   r�  r‘  r­  rÂ  rÄ  Ú__all__rQ   rP   rN   ú<module>rÝ     sa  ðð %Ð $Ð $Ð $Ð $Ð $Ø  Ð  Ð  Ð  Ð  Ð  Ð  Ð  à €€€Ø Ð Ð Ð Ð Ð Ø .Ð .Ð .Ð .Ð .Ð .à &Ð &Ð &Ð &Ð &Ð &Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð \Ð [Ð [Ð [Ð [Ð [Ð [Ð [Ø uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uðð ð ð ð ð ð ð ð GÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ +Ð +Ð +Ð +Ð +Ð +Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ðð ð ð ð ð ð ð ð ð ð ð ð (Ð 'Ð 'Ð 'Ð 'Ð 'ð 
ˆÔ	˜HÑ	%Ô	%€ð €Ð1Ð2Ñ2Ô2ØðWð Wð Wð Wð W�|Ð%5ñ Wô Wñ „ñ 3Ô2ðWðt €Ð1Ð2Ñ2Ô2Øð?(ð ?(ð ?(ð ?(ð ?(Ð#ñ ?(ô ?(ñ „ñ 3Ô2ð?(ðD	ð 	ð 	ð 	ð 	Ð <ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð#Bñ 	ô 	ð 	ðSð Sð Sð Sð S B¤Lñ Sô Sð Sð!ð !ð !ð !ð !�	ñ !ô !ð !ð
+ð +ð +ð +ð +�Mñ +ô +ð +ð
J<ð J<ð J<ð J<ð J<Ð1°2´9ñ J<ô J<ð J<ð\4)ð 4)ð 4)ð 4)ð 4)�oñ 4)ô 4)ð 4)ðn+ð +ð +ð +ð +Ð3ñ +ô +ð +ð\ Ð ð^ð ^ð ^ð ^ð ^Ð1ñ ^ô ^ð ^ð<
°#ð 
¸(ÀCÈÈcÐSVÐCWÐY]ÐC]Ô:^ð 
ð 
ð 
ð 
ðO
ð O
ð O
ð O
ð O
�kñ O
ô O
ð O
ðd-ð -ð -ð -ð -Ð)ñ -ô -ð -ð!@ð !@ð !@ð !@ð !@ ¤	ñ !@ô !@ð !@ðHð °E´Lð È%Ì,ÐY]ÑJ]ð ÐinÔiuð ð ð ð ð1Øð1à”<ð1ð ”L 4Ñ'ð1ð ˜T‘\ð	1ð
 ”, Ñ%ð1ð œð1ð 
ð1ð 1ð 1ð 1ðh\
ð \
ð \
ð \
ð \
�.ñ \
ô \
ð \
ð~v8ð v8ð v8ð v8ð v8Ð%Fñ v8ô v8ð v8ðr €ð^ðñ ô ð
!ð !ð !ð !ð !Ð*JÐLañ !ô !ñô ð
!ð

ð 
ð 
ð 
ð 
Ð&FÐH]ñ 
ô 
ð 
ð4
ð 
ð 
€€€rP   