§
    ‚Štj« ã                   óˆ  — d dl Z d dlmZ d dlmZ d dlmZ d dlmZ d dl	Z	d dl	m
Z
 d dlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZmZmZmZmZmZm Z  ddl!m"Z" ddl#m$Z$m%Z% ddl&m'Z'm(Z( ddl)m*Z*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4 ddl5m6Z6m7Z7m8Z8 ddl9m:Z:m;Z; ddl<m=Z= ddl>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZE ddlFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZO ddlPmQZQ ddlRmSZS ddlTmUZU ddlVmWZWmXZXmYZYmZZZ  e2¦   «         r	  e3j[        e\¦  «        Z]ded e	j^        d!e	j^        dz  d"edz  d#e	j^        dz  d$e	j^        d%e_fd&„Z` G d'„ d(eJ¦  «        Za G d)„ d*eG¦  «        Zbe G d+„ d,e$¦  «        ¦   «         Zce0e G d-„ d.e%¦  «        ¦   «         ¦   «         Zd G d/„ d0e
je        ¦  «        Zf G d1„ d2eM¦  «        Zg G d3„ d4e
je        ¦  «        Zh G d5„ d6e
je        ¦  «        Zi G d7„ d8e
je        ¦  «        Zj G d9„ d:e
je        ¦  «        Zk G d;„ d<e
je        ¦  «        Zl G d=„ d>e
jm        ¦  «        Zn G d?„ d@e
je        ¦  «        Zo G dA„ dBe
je        ¦  «        Zp G dC„ dDe
je        ¦  «        Zq G dE„ dFe
je        ¦  «        Zr G dG„ dHeB¦  «        Zs	 d€dIe	j^        dJe	j^        dKe	j^        d#e	j^        dLetd%e	j^        fdM„Zu G dN„ dOeQ¦  «        Zve8 G dP„ dQe?¦  «        ¦   «         Zw G dR„ dSe@¦  «        Zx G dT„ dUe
je        ¦  «        Zy G dV„ dWeB¦  «        Zz G dX„ dYeC¦  «        Z{ G dZ„ d[e
je        ¦  «        Z| G d\„ d]eS¦  «        Z} G d^„ d_e
je        ¦  «        Z~ G d`„ dae@¦  «        Z G db„ dceE¦  «        Z€ G dd„ deeL¦  «        Z� e0df¬g¦  «         G dh„ dieD¦  «        ¦   «         Z‚ e0dj¬g¦  «         G dk„ dleA¦  «        ¦   «         Zƒ G dm„ dne�¦  «        Z„ G do„ dpe�¦  «        Z… G dq„ dreK¦  «        Z†dse	j^        dz  dte	j^        dz  d%edz  fdu„Z‡dve	j^        dwe	jˆ        d%e	j^        fdx„Z‰ e0dy¬g¦  «         G dz„ d{eI¦  «        ¦   «         ZŠ e0d|¬g¦  «         G d}„ d~eH¦  «        ¦   «         Z‹g d¢ZŒdS )�é    N)ÚUserDict)ÚCallable)Ú	dataclass)Úcached_property)Únn)Ú
functionalé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Ú_preprocess_mask_argumentsÚblockwise_overlayÚcreate_bidirectional_maskÚcreate_causal_maskÚcreate_masks_for_generateÚ!create_sliding_window_causal_maskÚmaybe_pad_block_sequence_idsÚsliding_window_overlay)ÚFlashAttentionKwargs)ÚBaseModelOutputWithPastÚBaseModelOutputWithPooling)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_accelerate_availableÚloggingÚtorch_compilable_check)Úmaybe_autocastÚmerge_with_config_defaultsÚno_inherit_decorator)ÚOutputRecorderÚcapture_outputsé   )Ú	AutoModel)ÚGemma3AttentionÚGemma3DecoderLayerÚGemma3ForCausalLMÚ	Gemma3MLPÚGemma3RotaryEmbeddingÚGemma3TextModelÚGemma3TextScaledWordEmbedding)	ÚGemma3nCausalLMOutputWithPastÚGemma3nForConditionalGenerationÚGemma3nModelÚGemma3nModelOutputWithPastÚGemma3nMultimodalEmbedderÚGemma3nPreTrainedModelÚGemma3nRMSNormÚapply_rotary_pos_embÚeager_attention_forward)ÚLlamaRotaryEmbedding)ÚMixtralExperts)Úsliding_window_mask_functioné   )ÚGemma4AudioConfigÚGemma4ConfigÚGemma4TextConfigÚGemma4VisionConfigÚconfigÚinputs_embedsÚattention_maskÚpast_key_valuesÚposition_idsÚblock_sequence_idsÚreturnc                 óò   — | ||||dœ}t          di |¤Ž}t          di |¤ddi¤Ž\  }}	}	}	}
}	}|r|}nt          |||
|¦  «        }t          di |¤t          |¦  «        t	          | j        ¦  «        dœ¤Ž}||dœS )a‹  Create full_attention and sliding_attention masks with correct composition.

    For global (full attention) layers:  causal only (no bidirectional)
    For local (sliding window) layers:  AND(sliding_window, OR(causal, blockwise))

    Unlike Gemma 3 (which applies bidirectional attention on all layers), Gemma 4
    explicitly disables bidirectional attention on global attention layers.
    ©rD   rE   rF   rG   rH   Ú	layer_idxr   )Úor_mask_functionÚand_mask_function©Úfull_attentionÚsliding_attention© )r   r   r   r   r   Úsliding_window)rD   rE   rF   rG   rH   rI   Úmask_kwargsÚ	full_maskÚ
early_exitÚ_Ú	kv_lengthÚ	kv_offsetÚpadded_block_sequence_idsÚsliding_masks                 úg/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/gemma4/modular_gemma4.pyÚcreate_masks_for_vision_modelr^   X   sþ   € ð" Ø&Ø(Ø*Ø$ðð €Kõ #Ð1Ð1 [Ð1Ð1€Iõ 4Nð 4ð 4Ø
ð4ð 4àð4ð 4ð 4Ñ0€J��1�a˜ A yð ð 
Ø$6Ð!Ð!å$@Ø °	¸9ñ%
ô %
Ð!õ &ð ð Ø
ðå*Ð+DÑEÔEÝ0°Ô1FÑGÔGðð ð ð €Lð $Ø)ðð ð ó    c                   ó\   — e Zd ZU dZdZeeeej	        ej	        f         f         dz  e
d<   dS )ÚGemma4ModelOutputWithPasta¯  
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    audio_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        audio_hidden_states of the model produced by the audio encoder and after projecting the last hidden state.
    shared_kv_states (`dict`, *optional*):
        Dictionary mapping layer type strings to tuples of (key_states, value_states) tensors.
        Used to pass shared KV states between layers during KV sharing.
    NÚshared_kv_states©Ú__name__Ú
__module__Ú__qualname__Ú__doc__rb   ÚdictÚstrÚtupleÚtorchÚTensorÚ__annotations__rS   r_   r]   ra   ra   �   sN   € € € € € € ðð ð" MQÐ�d˜3  e¤l°E´LÐ&@Ô AÐAÔBÀTÑIÐPÐPÑPÐPÐPr_   ra   c                   ó\   — e Zd ZU dZdZeeeej	        ej	        f         f         dz  e
d<   dS )ÚGemma4CausalLMOutputWithPasta  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.text_config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder after projecting last hidden state.
    audio_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        audio_hidden_states of the model produced by the audio encoder and after projecting the last hidden state.
    shared_kv_states (`dict`, *optional*):
        Dictionary mapping layer type strings to tuples of (key_states, value_states) tensors.
        Used to pass shared KV states between layers during KV sharing.
    Nrb   rc   rS   r_   r]   ro   ro   ¤   sN   € € € € € € ðð ð* MQÐ�d˜3  e¤l°E´LÐ&@Ô AÐAÔBÀTÑIÐPÐPÑPÐPÐPr_   ro   c                   ó\   — e Zd ZU dZdZeeeej	        ej	        f         f         dz  e
d<   dS )ÚGemma4TextModelOutputWithPasta9  
    BaseModelOutputWithPast extended with shared_kv_states for KV sharing.

    Args:
        shared_kv_states (`dict`, *optional*):
            Dictionary mapping layer type strings to tuples of (key_states, value_states) tensors.
            Used to pass shared KV states between layers during KV sharing.
    Nrb   rc   rS   r_   r]   rq   rq   ½   sN   € € € € € € ðð ð MQÐ�d˜3  e¤l°E´LÐ&@Ô AÐAÔBÀTÑIÐPÐPÑPÐPÐPr_   rq   c                   ó2   — e Zd ZU dZdZej        dz  ed<   dS )ÚGemma4AudioModelOutputz§
    attention_mask (`torch.BoolTensor`, *optional*):
        A torch.BoolTensor of shape `(batch_size, num_frames)`. True for valid positions, False for padding.
    NrF   )rd   re   rf   rg   rF   rk   Ú
BoolTensorrm   rS   r_   r]   rs   rs   Ë   s6   € € € € € € ðð ð
 /3€N�EÔ$ tÑ+Ð2Ð2Ñ2Ð2Ð2r_   rs   c                   óZ   ‡ — e Zd Zdeez  dededdfˆ fd„Zdej        dej        fd„Z	ˆ xZ
S )	ÚGemma4ClippableLinearrD   Úin_featuresÚout_featuresrJ   Nc                 óV  •— t          ¦   «                              ¦   «          |j        | _        t          j        ||d¬¦  «        | _        | j        rØ|                      dt          j        t          d¦  «         ¦  «        ¦  «         |                      dt          j        t          d¦  «        ¦  «        ¦  «         |                      dt          j        t          d¦  «         ¦  «        ¦  «         |                      dt          j        t          d¦  «        ¦  «        ¦  «         d S d S )NF©ÚbiasÚ	input_minÚinfÚ	input_maxÚ
output_minÚ
output_max)
ÚsuperÚ__init__Úuse_clipped_linearsr   ÚLinearÚlinearÚregister_bufferrk   ÚtensorÚfloat)ÚselfrD   rw   rx   Ú	__class__s       €r]   r‚   zGemma4ClippableLinear.__init__×   sø   ø€ õ 	‰Œ×ÒÑÔÐØ#)Ô#=ˆÔ Ý”i ¨\ÀÐFÑFÔFˆŒàÔ#ð 	KØ× Ò  ­e¬l½EÀ%¹L¼L¸=Ñ.IÔ.IÑJÔJÐJØ× Ò  ­e¬l½5À¹<¼<Ñ.HÔ.HÑIÔIÐIØ× Ò  ­u¬|½UÀ5¹\¼\¸MÑ/JÔ/JÑKÔKÐKØ× Ò  ­u¬|½EÀ%¹L¼LÑ/IÔ/IÑJÔJÐJÐJÐJð		Kð 	Kr_   Úhidden_statesc                 óÌ   — | j         r t          j        || j        | j        ¦  «        }|                      |¦  «        }| j         r t          j        || j        | j        ¦  «        }|S ©N)rƒ   rk   Úclampr|   r~   r…   r   r€   )r‰   r‹   s     r]   ÚforwardzGemma4ClippableLinear.forwardç   s_   € ØÔ#ð 	WÝ!œK¨°t´~ÀtÄ~ÑVÔVˆMàŸš MÑ2Ô2ˆàÔ#ð 	YÝ!œK¨°t´ÈÌÑXÔXˆMàÐr_   )rd   re   rf   rC   r@   Úintr‚   rk   rl   r�   Ú__classcell__©rŠ   s   @r]   rv   rv   Ö   s™   ø€ € € € € ðKà"Ð%6Ñ6ðKð ðKð ð	Kð
 
ðKð Kð Kð Kð Kð Kð 	 U¤\ð 	°e´lð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	r_   rv   c                   ó   — e Zd ZdS )ÚGemma4RMSNormN©rd   re   rf   rS   r_   r]   r”   r”   ó   ó   € € € € € Ø€Dr_   r”   c                   óŒ   ‡ — e Zd ZU dZej        ed<   defˆ fd„Z ej	        ¦   «         dej        dej        fd„¦   «         Z
ˆ xZS )Ú Gemma4AudioRelPositionalEncodingzíSinusoidal relative positional encoding for the audio encoder.

    Produces position embeddings of shape [1, context_size // 2 + 1, hidden_size] with
    concatenated [sin..., cos...] layout matching the original Gemma4 convention.
    Úinv_timescalesrD   c                 óæ  •— t          ¦   «                              ¦   «          |j        | _        |j        |j        z   dz
  |j        z   | _        d}d}| j        dz  }t          j        ||z  ¦  «        t          |dz
  d¦  «        z  }|t          j        t          j        |¦  «        | z  ¦  «        z  }|                      d|                     d¦  «                             d¦  «        d¬¦  «         d S )	Nr?   ç      ð?ç     ˆÃ@r*   r™   r   F©Ú
persistent)r�   r‚   Úhidden_sizeÚattention_chunk_sizeÚattention_context_leftÚattention_context_rightÚcontext_sizeÚmathÚlogÚmaxrk   ÚexpÚaranger†   Ú	unsqueeze)r‰   rD   Úmin_timescaleÚmax_timescaleÚnum_timescalesÚlog_timescale_incrementr™   rŠ   s          €r]   r‚   z)Gemma4AudioRelPositionalEncoding.__init__   só   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔàÔ'¨&Ô*GÑGÈ!ÑKÈfÔNlÑlð 	Ôð ˆØˆØÔ)¨QÑ.ˆÝ"&¤(¨=¸=Ñ+HÑ"IÔ"IÍCÐP^ÐabÑPbÐdeÑLfÔLfÑ"fÐØ&­¬µ5´<ÀÑ3OÔ3OÐSjÐRjÑ3jÑ)kÔ)kÑkˆØ×ÒÐ-¨~×/GÒ/GÈÑ/JÔ/J×/TÒ/TÐUVÑ/WÔ/WÐdiÐÑjÔjÐjÐjÐjr_   r‹   rJ   c                 óP  — t          j        | j        dz  dd|j        ¬¦  «        }|d         }|| j                             |j        ¬¦  «        z  }t          j        t          j        |¦  «        t          j        |¦  «        gd¬¦  «        }|                     |j	        ¬¦  «        S )Nr*   éÿÿÿÿ©Údevice©.N©Údim©Údtype)
rk   r¨   r£   r±   r™   ÚtoÚcatÚsinÚcosr¶   )r‰   r‹   rH   Úscaled_timeÚ	pos_embeds        r]   r�   z(Gemma4AudioRelPositionalEncoding.forward  s•   € å”| DÔ$5¸Ñ$:¸BÀÈ=ÔK_Ð`Ñ`Ô`ˆØ# IÔ.ˆØ" TÔ%8×%;Ò%;À=ÔCWÐ%;Ñ%XÔ%XÑXˆÝ”I�uœy¨Ñ5Ô5µu´yÀÑ7MÔ7MÐNÐTVÐWÑWÔWˆ	Ø�|Š| -Ô"5ˆ|Ñ6Ô6Ð6r_   )rd   re   rf   rg   rk   rl   rm   r@   r‚   Úno_gradr�   r‘   r’   s   @r]   r˜   r˜   ÷   s¢   ø€ € € € € € ðð ð ”LÐ Ð Ñ ðkÐ0ð kð kð kð kð kð kð €U„]�_„_ð7 U¤\ð 7°e´lð 7ð 7ð 7ñ „_ð7ð 7ð 7ð 7ð 7r_   r˜   c                   óò   ‡ — e Zd ZdZdedefˆ fd„Zdej        dej        fd„Z	dej        dej        fd„Z
d	ej        dej        fd
„Z	 ddej        dej        dej        dz  deej        df         fd„Zˆ xZS )ÚGemma4AudioAttentionz3Chunked local attention with relative position biasrD   rM   c                 óJ  •— t          ¦   «                              ¦   «          || _        || _        |j        | _        |j        |j        z  | _        |j        | _	        | j        dz  t          j        d¦  «        z  | _        t          j        dt          j        z   ¦  «        t          j        d¦  «        z  | _        |j        | _        |j        dz
  | _        |j        | _        | j        | j        z   | j        z   | _        t-          ||j        | j	        | j        z  ¦  «        | _        t-          ||j        | j	        | j        z  ¦  «        | _        t-          ||j        | j	        | j        z  ¦  «        | _        t-          ||j        |j        ¦  «        | _        t7          j        |j        | j	        | j        z  d¬¦  «        | _        t7          j        t?          j         | j        ¦  «        ¦  «        | _!        |  "                    dt?          j#        | j        ¦  «        d¬¦  «         d S )Nç      à¿r*   r?   Frz   Úsoftcapr�   )$r�   r‚   rD   rM   Úattention_logit_capÚattention_logits_soft_caprŸ   Únum_attention_headsÚhead_dimÚ	num_headsr¤   r¥   Úq_scaleÚeÚk_scaler    Ú
chunk_sizer¡   Úmax_past_horizonr¢   Úmax_future_horizonr£   rv   Úq_projÚk_projÚv_projÚpostr   r„   Úrelative_k_projÚ	Parameterrk   ÚzerosÚper_dim_scaler†   r‡   ©r‰   rD   rM   rŠ   s      €r]   r‚   zGemma4AudioAttention.__init__  s»  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ)/Ô)CˆÔ&ØÔ*¨fÔ.HÑHˆŒØÔ3ˆŒàœ tÑ+­t¬x¸©{¬{Ñ:ˆŒÝ”x ¥D¤F¡
Ñ+Ô+­d¬h°q©k¬kÑ9ˆŒà Ô5ˆŒØ &Ô =ÀÑ AˆÔØ"(Ô"@ˆÔØ œO¨dÔ.CÑCÀdÔF]Ñ]ˆÔå+¨F°FÔ4FÈÌÐY]ÔYfÑHfÑgÔgˆŒÝ+¨F°FÔ4FÈÌÐY]ÔYfÑHfÑgÔgˆŒÝ+¨F°FÔ4FÈÌÐY]ÔYfÑHfÑgÔgˆŒÝ)¨&°&Ô2DÀfÔFXÑYÔYˆŒ	å!œy¨Ô);¸T¼^ÈdÌmÑ=[ÐbgÐhÑhÔhˆÔÝœ\­%¬+°d´mÑ*DÔ*DÑEÔEˆÔà×Ò˜Y­¬°TÔ5SÑ(TÔ(TÐafÐÑgÔgÐgÐgÐgr_   r‹   rJ   c           	      óô   — |j         \  }}}}|| j        z   dz
  | j        z  }|| j        z  |z
  }t          j        |ddddd|f¦  «        }|                     ||| j        ||¦  «                             ¦   «         S )z€Splits a `(batch_size, seq_len, num_heads, head_dim)` tensor into non-overlapping blocks of `chunk_size` along the sequence dim.r?   r   )ÚshaperË   ÚFÚpadÚreshapeÚ
contiguous)r‰   r‹   Ú
batch_sizeÚseq_lenrÇ   rÆ   Ú
num_blocksrÚ   s           r]   Ú_convert_to_blockz&Gemma4AudioAttention._convert_to_block3  s…   € à3@Ô3FÑ0ˆ
�G˜Y¨Ø ¤Ñ/°!Ñ3¸¼ÑGˆ
Ø˜4œ?Ñ*¨WÑ4ˆÝœ˜m¨a°°A°q¸!¸SÐ-AÑBÔBˆØ×$Ò$ Z°¸T¼_ÈiÐYaÑbÔb×mÒmÑoÔoÐor_   c           
      ó  — |j         \  }}}}t          j        |dddd| j        | j        | j        z   dz
  f¦  «        }|                     d| j        | j        ¦  «        }t          j	        |dd¦  «        }| 
                    ¦   «         S )z`Extracts overlapping context windows of `context_size` for every block, strided by `chunk_size`.r   r?   r¯   r*   )rØ   rÙ   rÚ   rÌ   rÍ   rË   Úunfoldr£   rk   ÚmovedimrÜ   )r‰   r‹   rÝ   rÞ   rÇ   rÆ   s         r]   Ú_extract_block_contextz+Gemma4AudioAttention._extract_block_context;  sŽ   € à3@Ô3FÑ0ˆ
�G˜Y¨ÝœØ˜A˜q ! Q¨Ô(=¸tÔ?VÐY]ÔYhÑ?hÐklÑ?lÐmñ
ô 
ˆð &×,Ò,¨Q°Ô0AÀ4Ä?ÑSÔSˆÝœ m°R¸Ñ;Ô;ˆØ×'Ò'Ñ)Ô)Ð)r_   Úxc                 óð   — |j         \  }}}}}| j        }t          j        |d|dz   |z
  f¦  «        }|                     |||||dz   z  ¦  «        }|dd||z  …f         }|                     |||||¦  «        S )zjRelative position shift for blocked attention. See appendix B of https://huggingface.co/papers/1901.02860.r   r?   .N)rØ   r£   rÙ   rÚ   Úview)r‰   rå   rÝ   rÇ   rß   Ú
block_sizeÚposition_lengthr£   s           r]   Ú
_rel_shiftzGemma4AudioAttention._rel_shiftE  s’   € àIJÌÑFˆ
�I˜z¨:°ØÔ(ˆÝŒE�!�a˜¨Ñ)¨OÑ;Ð<Ñ=Ô=ˆØ�FŠF�:˜y¨*°jÀLÐSTÑDTÑ6UÑVÔVˆØˆcÐ.�Z ,Ñ.Ð.Ð.Ô/ˆØ�vŠv�j )¨Z¸À\ÑRÔRÐRr_   NÚposition_embeddingsrF   c                 óö  — |j         \  }}}||| j        | j        f}|                      |¦  «                             ¦   «                              |¦  «        }|                      |¦  «                             ¦   «                              |¦  «        }	|                      |¦  «                             ¦   «                              |¦  «        }
|| j        z  t          j
        | j        ¦  «        z  }|	| j        z  }	|                      |¦  «        }|                      |	¦  «        }	|                      |
¦  «        }
|j         d         }|                      |¦  «        }|                     d| j        | j        ¦  «        }|                     |j        ¬¦  «        }|                     ddddd¦  «        }||	                     ddddd¦  «        z  }|                     || j        d| j        ¦  «        }||                     ddd¦  «        z  }|                     || j        || j        d¦  «        }|                      |¦  «        }||z   }|| j        z  }t/          j        |¦  «        }|| j        z  }|�2|                     |                     ¦   «         | j        j        ¦  «        }t          j        |dt.          j        ¬¦  «                             |
j        ¦  «        }||
                     ddddd¦  «        z  }|                     ddddd¦  «                             ||| j        z  d¦  «        }|d d …d |…f                              ¦   «         }|                       |                     |j        ¦  «        ¦  «        }||fS )	Nr?   r¯   rµ   r   r	   r*   é   ©r´   r¶   )!rØ   rÇ   rÆ   rÎ   rˆ   rç   rÏ   rÐ   rÈ   rÙ   ÚsoftplusrÕ   rÊ   rà   rä   rÒ   r·   r¶   ÚpermuterÛ   rË   rê   rÂ   rk   ÚtanhÚmasked_fillÚlogical_notrD   Úattention_invalid_logits_valueÚsoftmaxÚfloat32rÜ   rÑ   )r‰   r‹   rë   rF   rÝ   Ú
seq_lengthrX   Úhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesrß   Úrelative_key_statesÚqueriesÚ	matrix_acÚqueries_flatÚ	matrix_bdÚattn_weightsÚattn_outputs                      r]   r�   zGemma4AudioAttention.forwardN  s6  € ð %2Ô$7Ñ!ˆ
�J Ø" J°´ÀÄÐNˆà—{’{ =Ñ1Ô1×7Ò7Ñ9Ô9×>Ò>¸|ÑLÔLˆØ—[’[ Ñ/Ô/×5Ò5Ñ7Ô7×<Ò<¸\ÑJÔJˆ
Ø—{’{ =Ñ1Ô1×7Ò7Ñ9Ô9×>Ò>¸|ÑLÔLˆà# d¤lÑ2µQ´ZÀÔ@RÑ5SÔ5SÑSˆØ $¤,Ñ.ˆ
à×-Ò-¨lÑ;Ô;ˆØ×0Ò0°Ñ<Ô<ˆ
Ø×2Ò2°<Ñ@Ô@ˆØ!Ô'¨Ô*ˆ
à"×2Ò2Ð3FÑGÔGÐØ1×6Ò6°r¸4¼>È4Ì=ÑYÔYÐØ1×4Ò4¸<Ô;MÐ4ÑNÔNÐà×&Ò& q¨!¨Q°°1Ñ5Ô5ˆØ˜j×0Ò0°°A°q¸!¸QÑ?Ô?Ñ?ˆ	à—’ z°4´>À2ÀtÄ}ÑUÔUˆØ Ð#6×#>Ò#>¸qÀ!ÀQÑ#GÔ#GÑGˆ	Ø×%Ò% j°$´.À*ÈdÌoÐ_aÑbÔbˆ	Ø—O’O IÑ.Ô.ˆ	à  9Ñ,ˆØ# d¤lÑ2ˆÝ”z ,Ñ/Ô/ˆØ# d¤lÑ2ˆàÐ%Ø'×3Ò3Ø×*Ò*Ñ,Ô,¨d¬kÔ.Xñô ˆLõ ”y °2½U¼]ÐKÑKÔK×NÒNÈ|ÔOaÑbÔbˆØ" \×%9Ò%9¸!¸QÀÀ1ÀaÑ%HÔ%HÑHˆØ!×)Ò)¨!¨Q°°1°aÑ8Ô8×@Ò@ÀÈZÐZ^ÔZiÑMiÐkmÑnÔnˆØ! ! ! ! [ j [ .Ô1×<Ò<Ñ>Ô>ˆØ—i’i §¢¨}Ô/BÑ CÔ CÑDÔDˆà˜LÐ(Ð(r_   r�   )rd   re   rf   rg   r@   r�   r‚   rk   rl   rà   rä   rê   rt   rj   r�   r‘   r’   s   @r]   r¿   r¿     s4  ø€ € € € € Ø=Ð=ðhÐ0ð h¸Sð hð hð hð hð hð hð4p¨u¬|ð pÀÄð pð pð pð pð*°E´Lð *ÀUÄ\ð *ð *ð *ð *ðS˜EœLð S¨U¬\ð Sð Sð Sð Sð 37ð	1)ð 1)à”|ð1)ð #œ\ð1)ð Ô(¨4Ñ/ð	1)ð
 
ˆuŒ|˜TÐ!Ô	"ð1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)r_   r¿   c                   óJ   ‡ — e Zd Zˆ fd„Zddej        dej        dz  fd„Zˆ xZS )Ú'Gemma4AudioSubSampleConvProjectionLayerc                 óð   •— t          ¦   «                              ¦   «          t          j        ||dddd¬¦  «        | _        t          j        ||dd¬¦  «        | _        t          j        ¦   «         | _        d S )N)r	   r	   )r*   r*   r?   F)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingr{   T)ÚepsÚelementwise_affiner{   )	r�   r‚   r   ÚConv2dÚconvÚ	LayerNormÚnormÚReLUÚact)r‰   r  r  Únorm_epsrŠ   s       €r]   r‚   z0Gemma4AudioSubSampleConvProjectionLayer.__init__ƒ  sr   ø€ Ý‰Œ×ÒÑÔÐÝ”IØ#Ø%ØØØØð
ñ 
ô 
ˆŒ	õ ”L °8ÐPTÐ[`ÐaÑaÔaˆŒ	Ý”7‘9”9ˆŒˆˆr_   Nr‹   Úmaskc           
      óÆ  — |�.|                      |j        ¬¦  «        }||d d …d d d …d f         z  }|                      |                      | j        j        j        ¦  «        ¦  «        }|                      |                      |                     dddd¦  «        ¦  «                             dddd¦  «                             ¦   «         ¦  «        }|�|d d …d d d…f         }||fS )Nr°   r   r*   r	   r?   )	r·   r±   r  Úweightr¶   r  r  rð   rÜ   )r‰   r‹   r  s      r]   r�   z/Gemma4AudioSubSampleConvProjectionLayer.forward�  sâ   € ØÐØ—7’7 -Ô"6�7Ñ7Ô7ˆDØ)¨D°°°°D¸!¸!¸!¸TÐ1AÔ,BÑBˆMàŸ	š	 -×"2Ò"2°4´9Ô3CÔ3IÑ"JÔ"JÑKÔKˆØŸš §¢¨=×+@Ò+@ÀÀAÀqÈ!Ñ+LÔ+LÑ!MÔ!M×!UÒ!UÐVWÐYZÐ\]Ð_`Ñ!aÔ!a×!lÒ!lÑ!nÔ!nÑoÔoˆàÐØ˜˜˜˜3˜3˜Q˜3˜”<ˆDà˜dÐ"Ð"r_   r�   )rd   re   rf   r‚   rk   rl   r�   r‘   r’   s   @r]   r  r  ‚  sh   ø€ € € € € ðð ð ð ð ð#ð # U¤\ð #¸¼ÈÑ9Lð #ð #ð #ð #ð #ð #ð #ð #r_   r  c            	       óz   ‡ — e Zd Zdefˆ fd„Z	 ddej        dej        dz  deej        ej        f         fd„Zˆ xZ	S )	Ú"Gemma4AudioSubSampleConvProjectionrD   c                 óx  •— t          ¦   «                              ¦   «          t          d|j        d         |j        ¬¦  «        | _        t          |j        d         |j        d         |j        ¬¦  «        | _        |j        d         dz  |j        d         z  }t          j        ||j	        d¬¦  «        | _
        d S )Nr?   r   )r  r  r  rí   Frz   )r�   r‚   r  Úsubsampling_conv_channelsÚrms_norm_epsÚlayer0Úlayer1r   r„   rŸ   Úinput_proj_linear)r‰   rD   Úproj_input_dimrŠ   s      €r]   r‚   z+Gemma4AudioSubSampleConvProjection.__init__Ÿ  s¼   ø€ Ý‰Œ×ÒÑÔÐÝ=ØØÔ9¸!Ô<ØÔ(ð
ñ 
ô 
ˆŒõ
 >ØÔ8¸Ô;ØÔ9¸!Ô<ØÔ(ð
ñ 
ô 
ˆŒð
 !Ô:¸1Ô=ÀÑBÀfÔFfÐghÔFiÑiˆÝ!#¤¨>¸6Ô;MÐTYÐ!ZÑ!ZÔ!ZˆÔÐÐr_   NÚinput_featuresÚinput_features_maskrJ   c                 óT  — |                      d¦  «        }|                      ||¦  «        \  }}|                      ||¦  «        \  }}|j        \  }}}}|                     dddd¦  «                             ¦   «                              ||d¦  «        }|                      |¦  «        |fS )Nr?   r   r*   r	   r¯   )r©   r  r  rØ   rð   rÜ   rÛ   r  )r‰   r   r!  r‹   r  rÝ   rX   rÞ   s           r]   r�   z*Gemma4AudioSubSampleConvProjection.forward®  s«   € ð
 '×0Ò0°Ñ3Ô3ˆØ"Ÿkšk¨-Ð9LÑMÔMÑˆ�tØ"Ÿkšk¨-¸Ñ>Ô>Ñˆ�tà$1Ô$7Ñ!ˆ
�A�w Ø%×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔF×NÒNÈzÐ[bÐdfÑgÔgˆØ×%Ò% mÑ4Ô4°dÐ:Ð:r_   r�   )
rd   re   rf   r@   r‚   rk   rl   rj   r�   r‘   r’   s   @r]   r  r  ž  s�   ø€ € € € € ð[Ð0ð [ð [ð [ð [ð [ð [ð$ 48ð;ð ;àœð;ð #œ\¨DÑ0ð;ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð	;ð ;ð ;ð ;ð ;ð ;ð ;ð ;r_   r  c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚGemma4AudioFeedForwardrD   c                 ó¤  •— t          ¦   «                              ¦   «          || _        t          ||j        |j        dz  ¦  «        | _        t          ||j        dz  |j        ¦  «        | _        t          |j        ¦  «        | _        t          |j        ¦  «        | _	        t          |j                 | _        |j        | _        |j        | _        d S )Nrí   )r�   r‚   rD   rv   rŸ   Úffw_layer_1Úffw_layer_2r”   Úpre_layer_normÚpost_layer_normr   Ú
hidden_actÚact_fnÚgradient_clippingÚresidual_weightÚpost_layer_scale©r‰   rD   rŠ   s     €r]   r‚   zGemma4AudioFeedForward.__init__½  s±   ø€ Ý‰Œ×ÒÑÔÐØˆŒå0°¸Ô9KÈVÔM_ÐbcÑMcÑdÔdˆÔÝ0°¸Ô9KÈaÑ9OÐQWÔQcÑdÔdˆÔå+¨FÔ,>Ñ?Ô?ˆÔÝ,¨VÔ-?Ñ@Ô@ˆÔÝ˜VÔ.Ô/ˆŒà!'Ô!9ˆÔØ &Ô 6ˆÔÐÐr_   r‹   rJ   c                 ó¸  — t          | j        t          j        |j        ¦  «        j        ¦  «        }|}t          j        || |¦  «        }|                      |¦  «        }|                      |¦  «        }|  	                    |¦  «        }|  
                    |¦  «        }t          j        || |¦  «        }|                      |¦  «        }|| j        z  }||z  }|S r�   )Úminr,  rk   Úfinfor¶   r¦   rŽ   r(  r&  r+  r'  r)  r.  )r‰   r‹   r,  Úresiduals       r]   r�   zGemma4AudioFeedForward.forwardË  sÔ   € å Ô 6½¼ÀMÔDWÑ8XÔ8XÔ8\Ñ]Ô]Ðà ˆÝœ MÐ4EÐ3EÐGXÑYÔYˆØ×+Ò+¨MÑ:Ô:ˆà×(Ò(¨Ñ7Ô7ˆØŸš MÑ2Ô2ˆØ×(Ò(¨Ñ7Ô7ˆåœ MÐ4EÐ3EÐGXÑYÔYˆØ×,Ò,¨]Ñ;Ô;ˆØ˜Ô.Ñ.ˆØ˜Ñ!ˆàÐr_   ©	rd   re   rf   r@   r‚   rk   rl   r�   r‘   r’   s   @r]   r$  r$  ¼  sk   ø€ € € € € ð7Ð0ð 7ð 7ð 7ð 7ð 7ð 7ð U¤\ð °e´lð ð ð ð ð ð ð ð r_   r$  c                   óR   ‡ — e Zd Zed„ ¦   «         Zdej        dej        fˆ fd„Zˆ xZS )ÚGemma4AudioCausalConv1dc                 ód   — | j         d         dz
  | j        d         z  dz   }|| j        d         z
  S )Nr   r?   )r  Údilationr	  )r‰   Úeffective_kernel_sizes     r]   Úleft_padz Gemma4AudioCausalConv1d.left_padî  s7   € à!%Ô!1°!Ô!4°qÑ!8¸D¼MÈ!Ô<LÑ LÈqÑ PÐØ$ t¤{°1¤~Ñ5Ð5r_   rå   rJ   c                 ó”   •— t           j                             || j        df¦  «        }t	          ¦   «                              |¦  «        S )Nr   )r   r   rÚ   r:  r�   r�   )r‰   rå   rŠ   s     €r]   r�   zGemma4AudioCausalConv1d.forwardó  s9   ø€ õ ŒM×Ò˜a $¤-°Ð!3Ñ4Ô4ˆå‰wŒw�Š˜qÑ!Ô!Ð!r_   )	rd   re   rf   r   r:  rk   rl   r�   r‘   r’   s   @r]   r6  r6  à  sn   ø€ € € € € ð ð6ð 6ñ „_ð6ð"àŒ<ð"ð 
Œð	"ð "ð "ð "ð "ð "ð "ð "ð "ð "r_   r6  c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚGemma4AudioLightConv1drD   c                 ó   •— t          ¦   «                              ¦   «          || _        t          ||j        |j        dz  ¦  «        | _        t          ||j        |j        ¦  «        | _        t          |j        |j        |j        |j        d¬¦  «        | _	        t          |j        |j        d¬¦  «        | _        t          |j        |j        d¬¦  «        | _        t          |j                 | _        |j        | _        d S )Nr*   F)r  r  r  Úgroupsr{   T©r  Ú
with_scale)r�   r‚   rD   rv   rŸ   Úlinear_startÚ
linear_endr6  Úconv_kernel_sizeÚdepthwise_conv1dr”   r  r(  Ú	conv_normr   r*  r+  r,  r/  s     €r]   r‚   zGemma4AudioLightConv1d.__init__  sè   ø€ Ý‰Œ×ÒÑÔÐØˆŒå1°&¸&Ô:LÈfÔN`ÐcdÑNdÑeÔeˆÔÝ/°¸Ô8JÈFÔL^Ñ_Ô_ˆŒÝ 7ØÔ*ØÔ+ØÔ/ØÔ%Øð!
ñ !
ô !
ˆÔõ ,¨FÔ,>ÀFÔDWÐdhÐiÑiÔiˆÔÝ& vÔ'9¸vÔ?RÐ_cÐdÑdÔdˆŒÝ˜VÔ.Ô/ˆŒà!'Ô!9ˆÔÐÐr_   r‹   rJ   c                 ó2  — |}|                       |¦  «        }|                      |¦  «        }t          j                             |d¬¦  «        }|                      |                     dd¦  «        ¦  «                             dd¦  «        }t          | j        t          j
        |j        ¦  «        j        ¦  «        }t          j        || |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z  }|S )Nr¯   r³   r?   r*   )r(  rB  r   r   ÚglurE  Ú	transposer1  r,  rk   r2  r¶   r¦   rŽ   rF  r+  rC  )r‰   r‹   r3  r,  s       r]   r�   zGemma4AudioLightConv1d.forward  sü   € Ø ˆà×+Ò+¨MÑ:Ô:ˆØ×)Ò)¨-Ñ8Ô8ˆÝœ×)Ò)¨-¸RÐ)Ñ@Ô@ˆà×-Ò-¨m×.EÒ.EÀaÈÑ.KÔ.KÑLÔL×VÒVÐWXÐZ[Ñ\Ô\ˆõ   Ô 6½¼ÀMÔDWÑ8XÔ8XÔ8\Ñ]Ô]ÐÝœ MÐ4EÐ3EÐGXÑYÔYˆØŸš }Ñ5Ô5ˆàŸš MÑ2Ô2ˆØŸš¨Ñ6Ô6ˆØ˜Ñ!ˆØÐr_   r4  r’   s   @r]   r=  r=    sk   ø€ € € € € ð:Ð0ð :ð :ð :ð :ð :ð :ð( U¤\ð °e´lð ð ð ð ð ð ð ð r_   r=  c            
       ó~   ‡ — e Zd Zdedefˆ fd„Zdej        dej        dz  dej        de	e
         d	ej        f
d
„Zˆ xZS )ÚGemma4AudioLayerrD   rM   c                 ó¦  •— t          ¦   «                              ¦   «          || _        t          |¦  «        | _        t          |¦  «        | _        t          ||¦  «        | _        t          |¦  «        | _	        t          |j        ¦  «        | _        t          |j        ¦  «        | _        t          |j        ¦  «        | _        |j        | _        d S r�   )r�   r‚   rD   r$  Úfeed_forward1Úfeed_forward2r¿   Ú	self_attnr=  Úlconv1dr”   rŸ   Únorm_pre_attnÚnorm_post_attnÚnorm_outr,  rÖ   s      €r]   r‚   zGemma4AudioLayer.__init__+  s©   ø€ Ý‰Œ×ÒÑÔÐØˆŒå3°FÑ;Ô;ˆÔÝ3°FÑ;Ô;ˆÔÝ-¨f°iÑ@Ô@ˆŒÝ-¨fÑ5Ô5ˆŒå*¨6Ô+=Ñ>Ô>ˆÔÝ+¨FÔ,>Ñ?Ô?ˆÔÝ% fÔ&8Ñ9Ô9ˆŒà!'Ô!9ˆÔÐÐr_   r‹   rF   Nrë   ÚkwargsrJ   c                 óF  — t          | j        t          j        | j        j        j        ¦  «        j        ¦  «        }|                      |¦  «        }|}t          j	        || |¦  «        }|                      |¦  «        }|  
                    |||¬¦  «        \  }}t          j	        || |¦  «        }|                      |¦  «        }||z  }|                      |¦  «        }|                      |¦  «        }t          j	        || |¦  «        }|                      |¦  «        }|S )N)r‹   rë   rF   )r1  r,  rk   r2  rQ  r  r¶   r¦   rM  rŽ   rO  rR  rP  rN  rS  )r‰   r‹   rF   rë   rT  r,  r3  rX   s           r]   r�   zGemma4AudioLayer.forward:  s   € õ   Ô 6½¼ÀDÔDVÔD]ÔDcÑ8dÔ8dÔ8hÑiÔiÐà×*Ò*¨=Ñ9Ô9ˆØ ˆåœ MÐ4EÐ3EÐGXÑYÔYˆØ×*Ò*¨=Ñ9Ô9ˆàŸ>š>Ø'Ø 3Ø)ð *ñ 
ô 
Ñˆ�qõ œ MÐ4EÐ3EÐGXÑYÔYˆØ×+Ò+¨MÑ:Ô:ˆØ˜Ñ!ˆàŸš ]Ñ3Ô3ˆØ×*Ò*¨=Ñ9Ô9ˆåœ MÐ4EÐ3EÐGXÑYÔYˆØŸš mÑ4Ô4ˆàÐr_   )rd   re   rf   r@   r�   r‚   rk   rl   rt   r   r   r�   r‘   r’   s   @r]   rK  rK  *  s¤   ø€ € € € € ð:Ð0ð :¸Sð :ð :ð :ð :ð :ð :ð à”|ð ð Ô(¨4Ñ/ð ð #œ\ð	 ð
 Ð+Ô,ð ð 
Œð ð  ð  ð  ð  ð  ð  ð  r_   rK  c                   ó–   ‡ — e Zd Zdefˆ fd„Zdej        dej        dej        fd„Zdej        dej        dej        dej        fd„Zˆ xZ	S )	ÚGemma4VisionPatchEmbedderrD   c                 ód  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        t          j        d| j        dz  z  | j        d¬¦  «        | _        t          j	        t          j        d| j        | j        ¦  «        ¦  «        | _        d S )Nr	   r*   Frz   )r�   r‚   rD   rŸ   Ú
patch_sizeÚposition_embedding_sizer   r„   Ú
input_projrÓ   rk   ÚonesÚposition_embedding_tabler/  s     €r]   r‚   z"Gemma4VisionPatchEmbedder.__init__a  s•   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ Ô+ˆŒØ'-Ô'EˆÔ$åœ) A¨¬¸Ñ(:Ñ$:¸DÔ<LÐSXÐYÑYÔYˆŒÝ(*¬µU´ZÀÀ4ÔC_ÐaeÔaqÑ5rÔ5rÑ(sÔ(sˆÔ%Ð%Ð%r_   Úpixel_position_idsÚpadding_positionsrJ   c                 ó&  — |                      d¬¦  «        }t          j        |d         | j        d         ¦  «        }t          j        |d         | j        d         ¦  «        }||z   }t	          j        |                     d¦  «        d|¦  «        }|S )ak  Compute 2-D patch position embeddings via embedding lookup.

        ``pixel_position_ids`` has shape ``(batch, num_patches, 2)`` where the
        last dimension holds (x, y) indices into ``position_embedding_table``
        (shape ``(2, position_embedding_size, hidden_size)``).  The result is the
        sum of the x- and y-embeddings for each patch.
        r   ©r1  ©.r   ©.r?   r?   r¯   ç        )rŽ   rÙ   Ú	embeddingr]  rk   Úwherer©   )r‰   r^  r_  Úclamped_positionsÚx_embÚy_embrë   s          r]   Ú_position_embeddingsz.Gemma4VisionPatchEmbedder._position_embeddingsk  s‘   € ð /×4Ò4¸Ð4Ñ;Ô;Ðõ ”Ð-¨fÔ5°tÔ7TÐUVÔ7WÑXÔXˆÝ”Ð-¨fÔ5°tÔ7TÐUVÔ7WÑXÔXˆØ# e™mÐÝ#œkÐ*;×*EÒ*EÀbÑ*IÔ*IÈ3ÐPcÑdÔdÐØ"Ð"r_   Úpixel_valuesc                 óÌ   — d|dz
  z  }| j         j        j        x}j        r|                     |¦  «        }|                       |¦  «        }|                      ||¦  «        }||z   S )Nr*   ç      à?)r[  r  r¶   Úis_floating_pointr·   rj  )r‰   rk  r^  r_  Útarget_dtyper‹   rë   s          r]   r�   z!Gemma4VisionPatchEmbedder.forward�  sq   € ð ˜L¨3Ñ.Ñ/ˆØ œOÔ2Ô8Ð8ˆLÔKð 	9Ø'Ÿ?š?¨<Ñ8Ô8ˆLØŸš¨Ñ5Ô5ˆØ"×7Ò7Ð8JÐL]Ñ^Ô^ÐØÐ2Ñ2Ð2r_   )
rd   re   rf   rC   r‚   rk   rl   rj  r�   r‘   r’   s   @r]   rW  rW  `  s¼   ø€ € € € € ðtÐ1ð tð tð tð tð tð tð#°u´|ð #ÐX]ÔXdð #ÐinÔiuð #ð #ð #ð #ð,	3Ø!œLð	3Ø>C¼lð	3Ø_dÔ_kð	3à	Œð	3ð 	3ð 	3ð 	3ð 	3ð 	3ð 	3ð 	3r_   rW  c                   óà   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        dede	ej        ej        f         fd„Z
	 ddej        dej        d
ej        ded	z  de	ej        ej        f         f
d„Zˆ xZS )ÚGemma4VisionPoolera[  Spatial pooling and ``sqrt(hidden_size)`` scaling for vision encodings.

    The scaling expands the activation magnitude, which can exceed the float16 range, so it is
    computed in float32 and the pooled features are returned in float32. The caller
    (``Gemma4VisionModel.forward``) standardizes them and casts back to the working dtype.
    rD   c                 ó~   •— t          ¦   «                              ¦   «          |j        | _        | j        dz  | _        d S )Nrm  )r�   r‚   rŸ   Úroot_hidden_sizer/  s     €r]   r‚   zGemma4VisionPooler.__init__•  s:   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ $Ô 0°#Ñ 5ˆÔÐÐr_   r‹   r^  ÚlengthrJ   c                 óÜ  — |j         d         }t          ||z  dz  ¦  «        }|dz  }||z  |k    r$t          d|j         › d|› d|›d|›d|› d	�¦  «        ‚|                     d
¬¦  «        }|d                              dd¬¦  «        d
         dz   }t          j        ||d¬¦  «        }	|	d         ||z  |	d         z  z   }	t          j        |	 	                    ¦   «         |¦  «         
                    ¦   «         |z  }
|
                     dd¦  «        | 
                    ¦   «         z  }t          j        |
d
k                         d¬¦  «        ¦  «        }|                     |j        ¦  «        |fS )zÙ
        2D spatial pooling according to patch positions.
        Pools the input tokens by averaging patches within a `k^2` grid, where `k` is determined by the ratio between
        input and output lengths
        r?   rm  r*   zCannot pool z to z: k=z^2 times length=z	 must be ú.r   ra  rb  r¯   T©r´   ÚkeepdimÚfloor)Úrounding_moderc  r³   )rØ   r�   Ú
ValueErrorrŽ   r¦   rk   ÚdivrÙ   Úone_hotÚlongrˆ   rI  ró   Úallr·   r¶   )r‰   r‹   r^  rt  Úinput_seq_lenÚkÚ	k_squaredrg  Úmax_xÚkernel_idxsÚweightsÚoutputr  s                r]   Ú_avg_pool_by_positionsz)Gemma4VisionPooler._avg_pool_by_positionsš  sŒ  € ð &Ô+¨AÔ.ˆÝ� &Ñ(¨SÑ0Ñ1Ô1ˆØ�q‘Dˆ	Ø�vÑ Ò.Ð.ÝØq˜}Ô2ÐqÐq¸ÐqÐqÀ!ÐqÐqÈvÐqÐqÐanÐqÐqÐqñô ð ð /×4Ò4¸Ð4Ñ;Ô;ÐØ! &Ô)×-Ò-°"¸dÐ-ÑCÔCÀAÔFÈÑJˆÝ”iÐ 1°1ÀGÐLÑLÔLˆØ! &Ô)¨U°a©Z¸;ÀvÔ;NÑ,NÑNˆÝ”)˜K×,Ò,Ñ.Ô.°Ñ7Ô7×=Ò=Ñ?Ô?À)ÑKˆØ×"Ò" 1 aÑ(Ô(¨=×+>Ò+>Ñ+@Ô+@Ñ@ˆÝÔ  '¨Q¢,×!3Ò!3¸Ð!3Ñ!:Ô!:Ñ;Ô;ˆØ�yŠy˜Ô,Ñ-Ô-¨tÐ3Ð3r_   Nr_  Úoutput_lengthc                 óN  — ||j         d         k    r!t          d|› d|j         d         › d�¦  «        ‚|                     |                     d¦  «        d¦  «        }|j         d         |k    r|                      |||¦  «        \  }}|                     ¦   «         | j        z  }||fS )Nr?   z*Cannot output more soft tokens (requested z) than there are patches (z9). Change the value of `num_soft_tokens` when processing.r¯   rd  )rØ   r{  rò   r©   r‡  rˆ   rs  )r‰   r‹   r^  r_  rˆ  s        r]   r�   zGemma4VisionPooler.forwardµ  sÞ   € ð ˜=Ô.¨qÔ1Ò1Ð1Ýðg¸]ð gð gØ"Ô(¨Ô+ðgð gð gñô ð ð
 &×1Ò1Ð2C×2MÒ2MÈbÑ2QÔ2QÐSVÑWÔWˆàÔ˜qÔ! ]Ò2Ð2Ø/3×/JÒ/JØÐ1°=ñ0ô 0Ñ,ˆMÐ,ð &×+Ò+Ñ-Ô-°Ô0EÑEˆØÐ/Ð/Ð/r_   r�   )rd   re   rf   rg   rC   r‚   rk   rl   r�   rj   r‡  r�   r‘   r’   s   @r]   rq  rq  �  sü   ø€ € € € € ðð ð6Ð1ð 6ð 6ð 6ð 6ð 6ð 6ð
4Ø"œ\ð4Ø?D¼|ð4ØUXð4à	ˆuŒ|˜Uœ\Ð)Ô	*ð4ð 4ð 4ð 4ð@ %)ð0ð 0à”|ð0ð "œLð0ð !œ<ð	0ð
 ˜T‘zð0ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð0ð 0ð 0ð 0ð 0ð 0ð 0ð 0r_   rq  c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚGemma4VisionMLPrD   c                 ó  •— t          ¦   «                              | |¦  «         t          || j        | j        ¦  «        | _        t          || j        | j        ¦  «        | _        t          || j        | j        ¦  «        | _        d S r�   )r�   r‚   rv   rŸ   Úintermediate_sizeÚ	gate_projÚup_projÚ	down_projr/  s     €r]   r‚   zGemma4VisionMLP.__init__Ñ  sl   ø€ Ý‰Œ×Ò˜˜vÑ&Ô&Ð&Ý.¨v°tÔ7GÈÔI_Ñ`Ô`ˆŒÝ,¨V°TÔ5EÀtÔG]Ñ^Ô^ˆŒÝ.¨v°tÔ7MÈtÔO_Ñ`Ô`ˆŒˆˆr_   )rd   re   rf   rC   r‚   r‘   r’   s   @r]   r‹  r‹  Ð  sP   ø€ € € € € ðaÐ1ð að að að að að að að að að ar_   r‹  rå   rº   r¹   Úunsqueeze_dimc           	      ó�  ‡‡
‡‡— |j         d         }| j         d         }d|d|z  z  z  }|dk    rt          d|› d|› d|› d�¦  «        ‚|g|z  }t          j        | |d¬¦  «        Št          j        ||d¬¦  «        Š
t          j        ||d¬¦  «        Šˆ
ˆˆˆfd	„t	          |¦  «        D ¦   «         }	t          j        |	d¬¦  «        S )
ak  Applies multidimensional RoPE to inputs.

    Args:
        x (`torch.Tensor`): The tensor to embed.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            If position_ids.ndim + 2 == x.ndim, then this function passes through to `apply_rotary_pos_emb()`.
            Otherwise, position_ids is used to split the inputs, x, into multiple pieces, where each piece is fed to
            `apply_rotary_pos_emb()`, and then concatenated back together.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.

    Returns:
      Tensor of shape [B, L, N, H] with RoPE applied.
    r¯   r*   r   zEInvalid configuration: num_rotated_channels_per_dim must be > 0, got z (num_input_channels=z, ndim=ú)r³   c                 óZ   •— g | ]'}t          ‰|         ‰|         ‰|         ‰¬ ¦  «        ‘Œ(S ))rå   rº   r¹   r‘  )r:   )Ú.0r�  Ú	cos_partsÚ	sin_partsr‘  Úx_partss     €€€€r]   ú
<listcomp>z/apply_multidimensional_rope.<locals>.<listcomp>  sR   ø€ ð ð ð ð õ 	Ø�aŒjØ˜!”Ø˜!”Ø'ð		
ñ 	
ô 	
ðð ð r_   )rØ   r{  rk   ÚsplitÚranger¸   )rå   rº   r¹   rH   r‘  ÚndimÚnum_input_channelsÚnum_rotated_channels_per_dimÚsplit_sizesÚy_partsr–  r—  r˜  s       `     @@@r]   Úapply_multidimensional_roper¡  Ø  s+  øøøø€ ð8 Ô˜bÔ!€DØœ œÐØ#$Ð(:¸qÀ4¹xÑ(HÑ#IÐ à# qÒ(Ð(ÝðØ,ðð ØCUðð àðð ð ñ
ô 
ð 	
ð 0Ð0°4Ñ7€KÝŒk˜!˜[¨bÐ1Ñ1Ô1€GÝ”˜C °"Ð5Ñ5Ô5€IÝ”˜C °"Ð5Ñ5Ô5€Iðð ð ð ð ð ð õ �t‘”ðñ ô €Gõ Œ9�W "Ð%Ñ%Ô%Ð%r_   c                   óª   — e Zd Ze	 	 	 d	dedz  dej        dz  dedz  dede	f         fd„¦   «         Z
 ej        ¦   «         ed„ ¦   «         ¦   «         ZdS )
ÚGemma4VisionRotaryEmbeddingNrD   r±   rÞ   rJ   ztorch.Tensorc                 ó  — | j         d         }t          | dd¦  «        p| j        | j        z  }|dz  }d}d|t	          j        d|dt          j        ¬¦  «                             |t          j        ¬¦  «        |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetarÆ   Nr*   r›   r   rµ   )r±   r¶   )	Úrope_parametersÚgetattrrŸ   rÅ   rk   r¨   Úint64r·   rˆ   )rD   r±   rÞ   Úbaser´   Úspatial_dimÚattention_factorÚinv_freqs           r]   Úcompute_default_rope_parametersz;Gemma4VisionRotaryEmbedding.compute_default_rope_parameters  s™   € ð& Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆð ˜Q‘hˆàÐØØÝ”˜Q ¨Qµe´kÐBÑBÔB×EÒEÈVÕ[`Ô[fÐEÑgÔgÐjuÑuñwñ
ˆð Ð)Ð)Ð)r_   c                 óH  — | j         d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}g g }}t          d¦  «        D �]}|d d …d d …|f         }|d d …d d d …f                              ¦   «         }	t          |d¬¦  «        5  |                     ¦   «         |	                     ¦   «         z                       dd¦  «        }
t          j        |
|
fd¬	¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }d d d ¦  «         n# 1 swxY w Y   |                     |¦  «         |                     |¦  «         �Œt          j        |d¬	¦  «                             |j        ¬
¦  «        }t          j        |d¬	¦  «                             |j        ¬
¦  «        }||fS )Nr   r¯   r?   ÚmpsÚcpur*   F)Údevice_typeÚenabledr³   rµ   )r¬  rˆ   ÚexpandrØ   r·   r±   Ú
isinstanceÚtyperi   r›  r%   rI  rk   r¸   rº   Úattention_scalingr¹   Úappendr¶   )r‰   rå   rH   Úinv_freq_expandedr±  Úall_cosÚall_sinÚiÚdim_position_idsÚdim_position_ids_expandedÚfreqsÚembrº   r¹   s                 r]   r�   z#Gemma4VisionRotaryEmbedding.forward4  sC  € ð !œM¨$°°°°4¨-Ô8×>Ò>Ñ@Ô@×GÒGÈÔHZÐ[\ÔH]Ð_aÐcdÑeÔe×hÒhÐijÔiqÑrÔrÐÝ'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆð ˜r�ˆÝ�q‘”ð 
	 ñ 
	 ˆAØ+¨A¨A¨A¨q¨q¨q°!¨GÔ4ÐØ(8¸¸¸¸DÀ!À!À!¸Ô(D×(JÒ(JÑ(LÔ(LÐ%å¨KÀÐGÑGÔGð 9ð 9Ø*×0Ò0Ñ2Ô2Ð5N×5TÒ5TÑ5VÔ5VÑV×aÒaÐbcÐefÑgÔg�Ý”i ¨ °BÐ7Ñ7Ô7�Ø—g’g‘i”i $Ô"8Ñ8�Ø—g’g‘i”i $Ô"8Ñ8�ð	9ð 9ð 9ñ 9ô 9ð 9ð 9ð 9ð 9ð 9ð 9øøøð 9ð 9ð 9ð 9ð
 �NŠN˜3ÑÔÐØ�NŠN˜3ÑÔÐÑåŒi˜ RÐ(Ñ(Ô(×+Ò+°!´'Ð+Ñ:Ô:ˆÝŒi˜ RÐ(Ñ(Ô(×+Ò+°!´'Ð+Ñ:Ô:ˆØ�Cˆxˆs   Ã1BFÆF	ÆF	©NNN)rd   re   rf   ÚstaticmethodrC   rk   r±   r�   rj   rˆ   r­  r½   r   r�   rS   r_   r]   r£  r£    s­   € € € € € Øà,0Ø&*Ø"ð *ð  *Ø" TÑ)ð *à”˜tÑ#ð *ð �t‘ð *ð 
ˆ~˜uÐ$Ô	%ð	 *ð  *ð  *ñ „\ð *ðD €U„]�_„_Øðð ñ Ôñ „_ðð ð r_   r£  c                   óØ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej        dej        dej        dz  dej        dz  d	e	e
         d
eej        ej        dz  eej                 dz  f         fd„Zˆ xZS )ÚGemma4VisionAttentionrD   rM   c                 óø  •— t          ¦   «                              | ||¦  «         | `| `| `d| _        d| _        t          ||j        |j	        | j
        z  ¦  «        | _        t          ||j        |j        | j
        z  ¦  «        | _        t          ||j        |j	        | j
        z  ¦  «        | _        t          ||j        | j
        z  |j        ¦  «        | _        t!          | j
        |j        d¬¦  «        | _        d S )Nr›   Fr@  )r�   r‚   Úattn_logit_softcappingrT   Ú
is_slidingÚscalingÚ	is_causalrv   rŸ   Únum_key_value_headsrÆ   rÏ   rÅ   rÎ   rÐ   Úo_projr”   r  Úv_normrÖ   s      €r]   r‚   zGemma4VisionAttention.__init__O  sç   ø€ Ý‰Œ×Ò˜˜v yÑ1Ô1Ð1ØÐ'ØÐØˆOØˆŒØˆŒÝ+¨F°FÔ4FÈÔHbÐeiÔerÑHrÑsÔsˆŒÝ+¨F°FÔ4FÈÔHbÐeiÔerÑHrÑsÔsˆŒÝ+¨F°FÔ4FÈÔHbÐeiÔerÑHrÑsÔsˆŒÝ+¨F°FÔ4NÐQUÔQ^Ñ4^Ð`fÔ`rÑsÔsˆŒÝ# D¤M°vÔ7JÐW\Ð]Ñ]Ô]ˆŒˆˆr_   Nr‹   rë   rF   rH   rT  rJ   c                 óˆ  — |j         d d…         }g |¢d‘| j        ‘R }|\  }}	|                      |¦  «                             |¦  «        }
|                      |
¦  «        }
t          |
||	|¦  «        }
|
                     dd¦  «        }
|                      |¦  «                             |¦  «        }|                      |¦  «        }t          |||	|¦  «        }|                     dd¦  «        }|  	                    |¦  «                             |¦  «        }|  
                    |¦  «        }|                     dd¦  «        }t          j        | j        j        t          ¦  «        } || |
|||f| j        r| j        nd| j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr¯   r?   r*   rd  )ÚdropoutrÇ  )rØ   rÆ   rÎ   rç   Úq_normr¡  rI  rÏ   Úk_normrÐ   rË  r   Úget_interfacerD   Ú_attn_implementationr;   ÚtrainingÚattention_dropoutrÇ  rÛ   rÜ   rÊ  )r‰   r‹   rë   rF   rH   rT  Úinput_shaperø   rº   r¹   rù   rú   rû   Úattention_interfacer  r  s                   r]   r�   zGemma4VisionAttention.forward\  sæ  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà&‰ˆˆSà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆØ—{’{ <Ñ0Ô0ˆÝ2°<ÀÀcÈ<ÑXÔXˆØ#×-Ò-¨a°Ñ3Ô3ˆà—[’[ Ñ/Ô/×4Ò4°\ÑBÔBˆ
Ø—[’[ Ñ,Ô,ˆ
Ý0°¸SÀ#À|ÑTÔTˆ
Ø×)Ò)¨!¨QÑ/Ô/ˆ
à—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆØ—{’{ <Ñ0Ô0ˆØ#×-Ò-¨a°Ñ3Ô3ˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð /3¬mÐD�DÔ*Ð*ÀØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r_   rÀ  )rd   re   rf   rC   r�   r‚   rk   rl   Ú
LongTensorr   r   rj   r�   r‘   r’   s   @r]   rÃ  rÃ  M  sì   ø€ € € € € ð^Ð1ð ^¸cð ^ð ^ð ^ð ^ð ^ð ^ð  -1Ø.2Ø04ð,)ð ,)à”|ð,)ð #œ\ð,)ð œ tÑ+ð	,)ð
 Ô&¨Ñ-ð,)ð Ð+Ô,ð,)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð,)ð ,)ð ,)ð ,)ð ,)ð ,)ð ,)ð ,)r_   rÃ  c                   óÔ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej        dej        dej        dz  dej        dz  d	e	e
         d
eej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚGemma4VisionEncoderLayerrD   rM   c                 ó¢   •— t          ¦   «                              | ||¦  «         t          ||¬¦  «        | _        t	          |¦  «        | _        d S )N©rD   rM   )r�   r‚   rÃ  rO  r‹  ÚmlprÖ   s      €r]   r‚   z!Gemma4VisionEncoderLayer.__init__�  sF   ø€ Ý‰Œ×Ò˜˜v yÑ1Ô1Ð1Ý.°fÈ	ÐRÑRÔRˆŒÝ" 6Ñ*Ô*ˆŒˆˆr_   Nr‹   rë   rF   rH   rT  rJ   c                 ó  — |}|                       |¦  «        } | j        d||||dœ|¤Ž\  }}|                      |¦  «        }||z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z   }|S )N)r‹   rë   rF   rH   rS   )Úinput_layernormrO  Úpost_attention_layernormÚpre_feedforward_layernormrÛ  Úpost_feedforward_layernorm)r‰   r‹   rë   rF   rH   rT  r3  rX   s           r]   r�   z Gemma4VisionEncoderLayer.forward’  s¿   € ð !ˆà×,Ò,¨]Ñ;Ô;ˆà)˜4œ>ð 
Ø'Ø 3Ø)Ø%ð	
ð 
ð
 ð
ð 
Ñˆ�qð ×5Ò5°mÑDÔDˆØ  =Ñ0ˆà ˆØ×6Ò6°}ÑEÔEˆØŸš Ñ/Ô/ˆØ×7Ò7¸ÑFÔFˆØ  =Ñ0ˆàÐr_   rÀ  )rd   re   rf   rC   r�   r‚   rk   rl   rÖ  r   r   rj   ÚFloatTensorr�   r‘   r’   s   @r]   rØ  rØ  Œ  sæ   ø€ € € € € ð+Ð1ð +¸cð +ð +ð +ð +ð +ð +ð -1Ø.2Ø04ðð à”|ðð #œ\ðð œ tÑ+ð	ð
 Ô&¨Ñ-ðð Ð+Ô,ðð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uðð ð ð ð ð ð ð r_   rØ  c                   ót   ‡ — e Zd Zdefˆ fd„Z	 d
dej        dej        dej        dz  dee	         de
f
d	„Zˆ xZS )ÚGemma4VisionEncoderrD   c                 ó  •‡— t          ¦   «                              ¦   «          ‰| _        ‰j        | _        t          ‰¦  «        | _        t          j        ˆfd„t          | j        ¦  «        D ¦   «         ¦  «        | _
        d S )Nc                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS )rÚ  )rØ  )r•  r»  rD   s     €r]   r™  z0Gemma4VisionEncoder.__init__.<locals>.<listcomp>¸  s'   ø€ ÐbÐbÐbÀaÕ%¨V¸qÐAÑAÔAÐbÐbÐbr_   )r�   r‚   rD   Únum_hidden_layersÚ
num_layersr£  Ú
rotary_embr   Ú
ModuleListr›  Úlayersr/  s    `€r]   r‚   zGemma4VisionEncoder.__init__²  st   øø€ Ý‰Œ×ÒÑÔÐØˆŒØ Ô2ˆŒÝ5°fÑ=Ô=ˆŒÝ”mØbÐbÐbÐbÍ5ÐQUÔQ`ÑKaÔKaÐbÑbÔbñ
ô 
ˆŒˆˆr_   NrE   rF   r^  rT  rJ   c                 óÒ   — t          | j        ||¬¦  «        }|}|                      ||¦  «        }| j        d| j        j        …         D ]} ||f|||dœ|¤Ž}Œt          |¬¦  «        S )z�
        pixel_position_ids (torch.Tensor):
            Patch positions as (x, y) coordinates in the image as [batch, num_patches, 2].
        )rD   rE   rF   N)rF   rë   rH   ©Úlast_hidden_state)r   rD   rè  rê  ræ  r   )r‰   rE   rF   r^  rT  r‹   rë   Údecoder_layers           r]   r�   zGemma4VisionEncoder.forward»  s¨   € õ 3Ø”;Ø'Ø)ð
ñ 
ô 
ˆð &ˆØ"Ÿošo¨mÐ=OÑPÔPÐð "œ[Ð)H¨4¬;Ô+HÐ)HÔIð 	ð 	ˆMØ)˜MØðà-Ø$7Ø/ð	ð ð
 ðð ˆMˆMõ '¸ÐGÑGÔGÐGr_   r�   )rd   re   rf   rC   r‚   rk   rl   rÖ  r   r   r   r�   r‘   r’   s   @r]   rã  rã  ±  s³   ø€ € € € € ð
Ð1ð 
ð 
ð 
ð 
ð 
ð 
ð 7;ð	Hð Hà”|ðHð œðHð "Ô,¨tÑ3ð	Hð
 Ð+Ô,ðHð 
!ðHð Hð Hð Hð Hð Hð Hð Hr_   rã  c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚGemma4TextMLPrD   rM   c                 ó¾   •— |j         |j        z
  }||cxk    odk    nc }|j        o|}t          ¦   «                              ¦   «          |j        |rdndz  | _        d S )Nr   r*   r?   )ræ  Únum_kv_shared_layersÚuse_double_wide_mlpr�   r‚   r�  )r‰   rD   rM   Úfirst_kv_shared_layer_idxÚis_kv_shared_layerró  rŠ   s         €r]   r‚   zGemma4TextMLP.__init__á  s}   ø€ Ø$*Ô$<¸vÔ?ZÑ$ZÐ!Ø&Ð*CÐGÐGÒGÐGÀaÒGÐGÐGÐGÐØ$Ô8ÐOÐ=OÐÝ‰Œ×ÒÑÔÐØ!'Ô!9ÐBUÐ=\¸Q¸QÐ[\Ñ!]ˆÔÐÐr_   )rd   re   rf   rB   r�   r‚   r‘   r’   s   @r]   rð  rð  à  sX   ø€ € € € € ð^Ð/ð ^¸Cð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^ð ^r_   rð  c                   ó   — e Zd Zddefd„ZdS )ÚGemma4TextRotaryEmbeddingNrD   c                 ó‚  — t           j                             | ¦  «         |j        | _        |j        | _        || _        t          |j        ¦  «        | _        i | _	        i | _
        | j        D ]Ñ}| j        j        |         }|€Œ|d         x}dk    rt          |         }n| j        }|| j	        |<   || j
        |<   ||dœ}|dk    r|dk    rd|d<    || j        fi |¤Ž\  }}	|                      |› d�|d	¬
¦  «         |                      |› d�|                     ¦   «         d	¬
¦  «         t!          | |› d�|	¦  «         ŒÒd S )NÚ	rope_typeÚdefault)r±   Ú
layer_typerQ   ÚproportionalÚglobal_head_dimÚhead_dim_keyÚ	_inv_freqFr�   Ú_original_inv_freqÚ_attention_scaling)r   ÚModuler‚   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrD   ÚsetÚlayer_typesÚrope_init_fnsrù  r¦  r   r­  r†   ÚcloneÚsetattr)
r‰   rD   r±   rû  Úrope_paramsrù  Úrope_init_fnÚrope_init_fn_kwargsÚcurr_inv_freqÚcurr_attention_scalings
             r]   r‚   z"Gemma4TextRotaryEmbedding.__init__ê  s•  € Ý
Œ	×Ò˜4Ñ Ô Ð Ø"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒÝ˜vÔ1Ñ2Ô2ˆÔØSUˆÔØ)+ˆŒàÔ*ð 	Uð 	UˆJØœ+Ô5°jÔAˆKØÐ"Øà(¨Ô5Ð5�	¸)ÒCÐCÝ2°9Ô=��à#ÔC�à-9ˆDÔ˜zÑ*Ø)2ˆDŒN˜:Ñ&à-3À:Ð"NÐ"NÐØÐ-Ò-Ð-°)¸~Ò2MÐ2MØ6GÐ# NÑ3à4@°LÀÄÐ4dÐ4dÐPcÐ4dÐ4dÑ1ˆMÐ1Ø× Ò  JÐ!9Ð!9Ð!9¸=ÐUZÐ Ñ[Ô[Ð[Ø× Ò  JÐ!BÐ!BÐ!BÀM×DWÒDWÑDYÔDYÐfkÐ ÑlÔlÐlÝ�D˜ZÐ;Ð;Ð;Ð=SÑTÔTÐTÐTð)	Uð 	Ur_   ©NN)rd   re   rf   rB   r‚   rS   r_   r]   r÷  r÷  é  s=   € € € € € ðUð UÐ/ð Uð Uð Uð Uð Uð Ur_   r÷  c                   óè   ‡ — e Zd ZdZdedefˆ fd„Z	 ddej        dej        dej        dz  d	e	e
eej        ej        f         f         d
edz  dee         deej        ej        dz  f         fd„Zˆ xZS )ÚGemma4TextAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrD   rM   c                 ó†  •— t          ¦   «                              ¦   «          t          |d¦  «        r|j        |         nd | _        || _        || _        | j        dk    | _        | j        r|j        nd | _        | j        s|j	        r|j	        n|j
        | _
        |j        o| j         | _        | j        r|j        n|j        }|j        |z  | _        d| _        | j        j        | _        |j        dk    | _        | j        j        t-          | j        dd¦  «        z
  }||cxk    odk    nc | _        |j        d |…         }| j         o?|t1          |¦  «        dz
  |d d d…                              |j        |         ¦  «        z
  k    | _        t7          j        |j        |j        | j
        z  |j        ¬	¦  «        | _        tA          | j
        |j!        ¬
¦  «        | _"        | j        s¦tA          | j
        |j!        ¬
¦  «        | _#        tA          | j
        |j!        d¬¦  «        | _$        t7          j        |j        || j
        z  |j        ¬	¦  «        | _%        | j        s)t7          j        |j        || j
        z  |j        ¬	¦  «        nd | _&        t7          j        |j        | j
        z  |j        |j        ¬	¦  «        | _'        d S )Nr  rR   r›   r  rò  r   r?   r¯   rz   )r´   r  Fr@  )(r�   r‚   Úhasattrr  rû  rD   rM   rÆ  rT   rý  rÆ   Úattention_k_eq_vÚuse_alternative_attentionÚnum_global_key_value_headsrÉ  rÅ   Únum_key_value_groupsrÇ  rÓ  Úuse_bidirectional_attentionrÈ  ræ  r§  rõ  ÚlenÚindexÚstore_full_length_kvr   r„   rŸ   Úattention_biasrÎ   r”   r  rÎ  rÏ  rË  rÏ   rÐ   rÊ  )r‰   rD   rM   rÉ  rô  Úprev_layersrŠ   s         €r]   r‚   zGemma4TextAttention.__init__  sÝ  ø€ Ý‰Œ×ÒÑÔÐÝ;BÀ6È=Ñ;YÔ;YÐc˜&Ô,¨YÔ7Ð7Ð_cˆŒØˆŒØ"ˆŒØœ/Ð-@Ò@ˆŒØ7;´ÐP˜fÔ3Ð3ÈDˆÔà6:´oÐuÈ&ÔJ`Ðu˜Ô.Ð.ÐflÔfuˆŒØ)/Ô)@Ð)XÈÌÐEXˆÔ&à15Ô1OÐoˆFÔ-Ð-ÐU[ÔUoð 	ð %+Ô$>ÐBUÑ$UˆÔ!ØˆŒØ!%¤Ô!>ˆÔØÔ;¸uÒDˆŒð %)¤KÔ$AÅGÈDÌKÐYoÐqrÑDsÔDsÑ$sÐ!Ø"+Ð/HÐ"MÐ"MÒ"MÐ"MÈAÒ"MÐ"MÐ"MÐ"MˆÔØÔ(Ð)CÐ*CÐ)CÔDˆØ(,Ô(?Ð$?ð %/ÀIÕQTÐU`ÑQaÔQaÐdeÑQeÐhsØˆDˆbˆDôi
ç
Š%�Ô" 9Ô-Ñ
.Ô
.ñR/ò E/ˆÔ!õ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ $¨¬¸6Ô;NÐOÑOÔOˆŒð Ô&ð 	Ý'¨D¬M¸vÔ?RÐSÑSÔSˆDŒKÝ'¨¬¸6Ô;NÐ[`ÐaÑaÔaˆDŒKåœ)ØÔ"Ð$7¸$¼-Ñ$GÈfÔNcðñ ô ˆDŒKð
 Ô5ð•”	˜&Ô,Ð.AÀDÄMÑ.QÐX^ÔXmÐnÑnÔnÐnàð ŒKõ ”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒˆˆr_   Nr‹   rë   rF   rb   rG   rT  rJ   c                 ó¸  — |j         d d…         }g |¢d‘| j        ‘R }|\  }	}
|                      |¦  «                             |¦  «        }|                      |¦  «        }t          ||	|
d¬¦  «        }|                     dd¦  «        }| j        rE|| j                 \  }}| 	                    |j
        ¦  «        }| 	                    |j
        ¦  «        }nÂ|                      |¦  «                             |¦  «        }| j        �(|                      |¦  «                             |¦  «        n|}|                      |¦  «        }t          ||	|
d¬¦  «        }|                     dd¦  «        }|                      |¦  «        }|                     dd¦  «        }|�&| j        s|                     ||| j        ¦  «        \  }}| j        r||f|| j        <   t%          j        | j        j        t,          ¦  «        } || ||||f| j        r| j        nd| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr¯   r*   )r‘  r?   rd  )rÍ  rÇ  rT   )rØ   rÆ   rÎ   rç   rÎ  r:   rI  rõ  rû  r·   r±   rÏ   rÐ   rÏ  rË  ÚupdaterM   r  r   rÐ  rD   rÑ  r;   rÒ  rÓ  rÇ  rT   rÛ   rÜ   rÊ  )r‰   r‹   rë   rF   rb   rG   rT  rÔ  rø   rº   r¹   rù   rú   rû   rÕ  r  r  s                    r]   r�   zGemma4TextAttention.forward?  s•  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà&‰ˆˆSà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆØ—{’{ <Ñ0Ô0ˆÝ+¨L¸#¸sÐRSÐTÑTÔTˆØ#×-Ò-¨a°Ñ3Ô3ˆð
 Ô"ð 	8Ø'7¸¼Ô'HÑ$ˆJ˜à#Ÿš |Ô':Ñ;Ô;ˆJØ'Ÿ?š?¨<Ô+>Ñ?Ô?ˆLˆLàŸš ]Ñ3Ô3×8Ò8¸ÑFÔFˆJØLPÌKÐLc˜4Ÿ;š; }Ñ5Ô5×:Ò:¸<ÑHÔHÐHÐisˆLàŸš ZÑ0Ô0ˆJÝ-¨j¸#¸sÐRSÐTÑTÔTˆJØ#×-Ò-¨a°Ñ3Ô3ˆJàŸ;š; |Ñ4Ô4ˆLØ'×1Ò1°!°QÑ7Ô7ˆLàÐ&¨tÔ/FÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜ØÔ$ð 	IØ0:¸LÐ0HÐ˜Tœ_Ñ-å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð /3¬mÐD�DÔ*Ð*ÀØ”LØÔ.ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r_   r�   )rd   re   rf   rg   rB   r�   r‚   rk   rl   rh   ri   rj   r   r   r   r�   r‘   r’   s   @r]   r  r    sõ   ø€ € € € € ØGÐGð/
Ð/ð /
¸Cð /
ð /
ð /
ð /
ð /
ð /
ðn )-ð=)ð =)à”|ð=)ð #œ\ð=)ð œ tÑ+ð	=)ð
 ˜s E¨%¬,¸¼Ð*DÔ$EÐEÔFð=)ð  ™ð=)ð Ð-Ô.ð=)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð=)ð =)ð =)ð =)ð =)ð =)ð =)ð =)r_   r  c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚGemma4TextExpertsrD   c                 ó¦   •— t          ¦   «                              ¦   «          |j        | _        |j        | _        t
          |j                 | _        d S r�   )r�   r‚   Únum_expertsÚmoe_intermediate_sizeÚintermediate_dimr   Úhidden_activationr+  r/  s     €r]   r‚   zGemma4TextExperts.__init__€  sB   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ &Ô <ˆÔÝ˜VÔ5Ô6ˆŒˆˆr_   )rd   re   rf   rB   r‚   r‘   r’   s   @r]   r"  r"    sE   ø€ € € € € ð7Ð/ð 7ð 7ð 7ð 7ð 7ð 7ð 7ð 7ð 7ð 7r_   r"  c                   ób   ‡ — e Zd Zdefˆ fd„Zdej        deej        ej        f         fd„Zˆ xZ	S )ÚGemma4TextRouterrD   c                 óò  •— t          ¦   «                              ¦   «          || _        |j        | _        | j        dz  | _        |j        | _        t          | j        | j        d¬¦  «        | _        t          j
        |j        |j        d¬¦  «        | _        t          j        t          j        | j        ¦  «        ¦  «        | _        t          j        t          j        |j        ¦  «        ¦  «        | _        d S )NrÁ   Fr@  rz   )r�   r‚   rD   rŸ   Úscalar_root_sizer  r  r”   r  r   r„   r$  ÚprojrÓ   rk   r\  ÚscaleÚper_expert_scaler/  s     €r]   r‚   zGemma4TextRouter.__init__ˆ  sÁ   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ $Ô 0°$Ñ 6ˆÔØÔ&ˆŒå! $Ô"2¸¼ÈUÐSÑSÔSˆŒ	Ý”I˜fÔ0°&Ô2DÈ5ÐQÑQÔQˆŒ	Ý”\¥%¤*¨TÔ-=Ñ">Ô">Ñ?Ô?ˆŒ
Ý "¤­U¬Z¸Ô8JÑ-KÔ-KÑ LÔ LˆÔÐÐr_   r‹   rJ   c                 óx  — |                       |¦  «        }|| j        z  | j        z  }|                      |¦  «        }t          j                             |dt          j        ¬¦  «        }t          j	        || j
        j        d¬¦  «        \  }}||                     dd¬¦  «        z  }|| j        |         z  }|||fS )Nr¯   rî   )r�  r´   Trw  )r  r-  r+  r,  r   r   rõ   rk   rö   ÚtopkrD   Útop_k_expertsÚsumr.  )r‰   r‹   Úexpert_scoresÚrouter_probabilitiesÚtop_k_weightsÚtop_k_indexs         r]   r�   zGemma4TextRouter.forward”  sÄ   € ØŸ	š	 -Ñ0Ô0ˆØ%¨¬
Ñ2°TÔ5JÑJˆàŸ	š	 -Ñ0Ô0ˆå!œ}×4Ò4°]ÈÕRWÔR_Ð4Ñ`Ô`Ðõ &+¤ZØ ØŒkÔ'Øð&
ñ &
ô &
Ñ"ˆ�{ð 	˜×*Ò*¨r¸4Ð*Ñ@Ô@Ñ@ˆð &¨Ô(=¸kÔ(JÑJˆà# ]°KÐ?Ð?r_   )
rd   re   rf   rB   r‚   rk   rl   rj   r�   r‘   r’   s   @r]   r)  r)  ‡  sˆ   ø€ € € € € ð
MÐ/ð 
Mð 
Mð 
Mð 
Mð 
Mð 
Mð@ U¤\ð @°e¸E¼LÈ%Ì,Ð<VÔ6Wð @ð @ð @ð @ð @ð @ð @ð @r_   r)  c                   óì   ‡ — e Zd Zdeez  defˆ fd„Z	 	 	 	 	 	 ddej        dej        de	e
eej        ej        f         f         dz  dej        d	ej        dz  d
ej        dz  dedz  dej        fd„Zˆ xZS )ÚGemma4TextDecoderLayerrD   rM   c                 óX  •— t          ¦   «                              ||¦  «         t          ||¬¦  «        | _        t	          ||¦  «        | _        |                      dt          j        d¦  «        ¦  «         |j	        | _	        | j	        rƒt          |j                 | _        t          j        | j        | j	        d¬¦  «        | _        t          j        | j	        | j        d¬¦  «        | _        t%          | j        |j        ¬¦  «        | _        |j        | _        | j        rŠt-          |¦  «        | _        t1          |¦  «        | _        t%          | j        |j        ¬¦  «        | _        t%          | j        |j        ¬¦  «        | _        t%          | j        |j        ¬¦  «        | _        d S d S )NrÚ  Úlayer_scalarr?   Frz   ©r  )r�   r‚   r  rO  rð  rÛ  r†   rk   r\  Úhidden_size_per_layer_inputr   r'  r+  r   r„   rŸ   Úper_layer_input_gateÚper_layer_projectionr”   r  Úpost_per_layer_input_normÚenable_moe_blockr)  Úrouterr"  ÚexpertsÚpost_feedforward_layernorm_1Úpost_feedforward_layernorm_2Úpre_feedforward_layernorm_2rÖ   s      €r]   r‚   zGemma4TextDecoderLayer.__init__­  s~  ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+Ý,°FÀiÐPÑPÔPˆŒÝ  ¨Ñ3Ô3ˆŒØ×Ò˜^­U¬Z¸©]¬]Ñ;Ô;Ð;à+1Ô+MˆÔ(ØÔ+ð 	fÝ  Ô!9Ô:ˆDŒKÝ(*¬	°$Ô2BÀDÔDdÐkpÐ(qÑ(qÔ(qˆDÔ%Ý(*¬	°$Ô2RÐTXÔTdÐkpÐ(qÑ(qÔ(qˆDÔ%Ý-:¸4Ô;KÐQWÔQdÐ-eÑ-eÔ-eˆDÔ*à &Ô 7ˆÔØÔ ð 	hÝ*¨6Ñ2Ô2ˆDŒKÝ,¨VÑ4Ô4ˆDŒLÝ0=¸dÔ>NÐTZÔTgÐ0hÑ0hÔ0hˆDÔ-Ý0=¸dÔ>NÐTZÔTgÐ0hÑ0hÔ0hˆDÔ-Ý/<¸TÔ=MÐSYÔSfÐ/gÑ/gÔ/gˆDÔ,Ð,Ð,ð	hð 	hr_   Nr‹   Úper_layer_inputrb   rë   rF   rH   rG   rJ   c           
      óp  — |}	|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}
|                      |¦  «        }|	|z   }|}	|                      |¦  «        }|                      |¦  «        }| j        r¯|                      |¦  «        }|	                     d|	j        d         ¦  «        }|  	                    |¦  «        \  }
}}|  
                    |¦  «        }|                      |||¦  «        }|                     |	j        ¦  «        }|                      |¦  «        }||z   }|                      |¦  «        }|	|z   }| j        r`|}	|                      |¦  «        }|                      |¦  «        }||z  }|                      |¦  «        }|                      |¦  «        }|	|z   }|| j        z  }|S )N)r‹   rë   rF   rb   rH   rG   r¯   rS   )rÝ  rO  rÞ  rß  rÛ  r@  rC  rÛ   rØ   rA  rE  rB  rD  rà  r<  r=  r+  r>  r?  r:  )r‰   r‹   rF  rb   rë   rF   rH   rG   rT  r3  rX   Úhidden_states_1Úhidden_states_flatr5  r6  Úhidden_states_2s                   r]   r�   zGemma4TextDecoderLayer.forwardÂ  só  € ð !ˆà×,Ò,¨]Ñ;Ô;ˆØ)˜4œ>ð 
Ø'Ø 3Ø)Ø-Ø%Ø+ð
ð 
ð ð
ð 
Ñˆ�qð ×5Ò5°mÑDÔDˆØ  =Ñ0ˆà ˆØ×6Ò6°}ÑEÔEˆØŸš Ñ/Ô/ˆàÔ ð 	>Ø"×?Ò?ÀÑNÔNˆOð "*×!1Ò!1°"°h´nÀRÔ6HÑ!IÔ!IÐØ,0¯KªKÐ8JÑ,KÔ,KÑ)ˆAˆ}˜kØ"×>Ò>Ð?QÑRÔRˆOØ"Ÿlšl¨?¸KÈÑWÔWˆOØ-×5Ò5°h´nÑEÔEˆOØ"×?Ò?ÀÑPÔPˆOð ,¨oÑ=ˆMà×7Ò7¸ÑFÔFˆØ  =Ñ0ˆàÔ+ð 	5Ø$ˆHØ ×5Ò5°mÑDÔDˆMØ ŸKšK¨Ñ6Ô6ˆMØ)¨OÑ;ˆMØ ×5Ò5°mÑDÔDˆMØ ×:Ò:¸=ÑIÔIˆMØ$ }Ñ4ˆMà˜Ô*Ñ*ˆØÐr_   )NNNNNN)rd   re   rf   rB   rC   r�   r‚   rk   rl   rh   ri   rj   rÖ  r   r�   r‘   r’   s   @r]   r8  r8  ¬  s  ø€ € € € € ðhÐ/Ð2DÑDð hÐQTð hð hð hð hð hð hð0 )-ØPTØ,0Ø.2Ø04Ø(,ð9ð 9à”|ð9ð œð9ð ˜s E¨%¬,¸¼Ð*DÔ$EÐEÔFÈÑMð	9ð
 #œ\ð9ð œ tÑ+ð9ð Ô&¨Ñ-ð9ð  ™ð9ð 
Œð9ð 9ð 9ð 9ð 9ð 9ð 9ð 9r_   r8  c                   ó   — e Zd ZdS )ÚGemma4TextScaledWordEmbeddingNr•   rS   r_   r]   rL  rL  þ  r–   r_   rL  c                   óN   — e Zd Zg d¢ZdZdZ ej        ¦   «         d„ ¦   «         ZdS )ÚGemma4PreTrainedModel)r8  rØ  rW  rK  )ÚimageÚtextÚvideoÚaudioNc                 óJ	  — t          j        |¦  «         t          |t          ¦  «        rt	          j        |j        ¦  «         d S t          |t          ¦  «        r¨d}d}|j        dz  }t          j
        ||z  ¦  «        t          |dz
  d¦  «        z  }|t          j        t          j        |¦  «        | z  ¦  «        z  }t	          j        |j        |                     d¦  «                             d¦  «        ¦  «         d S t          |t$          ¦  «        r:t	          j        |j        |j        ¦  «         t	          j        |j        ¦  «         d S t          |t0          ¦  «        rž|j                             ¦   «         D ]‚\  }}d|i}	|dk    r|j        |         dk    rd	|	d
<    ||j        fi |	¤Ž\  }
}t	          j        t;          ||› d�¦  «        |
¦  «         t	          j        t;          ||› d�¦  «        |
¦  «         Œƒd S t          |t<          ¦  «        rm|j        dk    rt>          |j                 n|j         } ||j        ¦  «        \  }}t	          j        |j!        |¦  «         t	          j        |j"        |¦  «         d S t          |tF          ¦  «        r!t	          j        |j$        |j%        ¦  «         d S t          |tL          ¦  «        r4t	          j        |j'        ¦  «         t	          j        |j(        ¦  «         d S t          |tR          ¦  «        rF| j        j*        }t	          j+        |j,        d|¬¦  «         t	          j+        |j-        d|¬¦  «         d S t          |t\          ¦  «        rt	          j        |j/        ¦  «         d S t          |t`          ¦  «        r§|j1        r t	          j        |j2        tg          d¦  «         ¦  «         t	          j        |j4        tg          d¦  «        ¦  «         t	          j        |j5        tg          d¦  «         ¦  «         t	          j        |j6        tg          d¦  «        ¦  «         d S t          |tn          ¦  «        r@|j        j8        r6t	          j        |j9        ¦  «         t	          j        |j:        ¦  «         d S d S d S )Nr›   rœ   r*   r?   r   rû  rQ   rü  rý  rþ  rÿ  r   rú  rd  )ÚmeanÚstdr}   );r   Ú_init_weightsr´  rW  ÚinitÚones_r]  r˜   rŸ   r¤   r¥   r¦   rk   r§   r¨   Úcopy_r™   r©   r¿   Ú	constant_rÂ   rÄ   Úzeros_rÕ   r÷  r  Úitemsrù  rD   r§  r£  r   r­  r¬  Úoriginal_inv_freqrL  Úembed_scaleÚscalar_embed_scaler)  r-  r.  r"  Úinitializer_rangeÚnormal_Úgate_up_projr�  r8  r:  rv   rƒ   r|   rˆ   r~   r   r€   ÚGemma4VisionModelÚstandardizeÚstd_biasÚ	std_scale)r‰   Úmodulerª   r«   r¬   r­   r™   rû  r  r  r  rX   Úrope_fnÚbuffer_valuerU  s                  r]   rV  z#Gemma4PreTrainedModel._init_weights  s†  € åÔ% fÑ-Ô-Ð-Ý�fÕ7Ñ8Ô8ð 0	)ÝŒJ�vÔ6Ñ7Ô7Ð7Ð7Ð7Ý˜Õ @ÑAÔAð .	)ØˆMØ#ˆMØ#Ô/°1Ñ4ˆNÝ&*¤h¨}¸}Ñ/LÑ&MÔ&MÕPSÐTbÐefÑTfÐhiÑPjÔPjÑ&jÐ#Ø*­U¬Yµu´|ÀNÑ7SÔ7SÐWnÐVnÑ7nÑ-oÔ-oÑoˆNÝŒJ�vÔ,¨n×.FÒ.FÀqÑ.IÔ.I×.SÒ.SÐTUÑ.VÔ.VÑWÔWÐWÐWÐWÝ˜Õ 4Ñ5Ô5ð '	)ÝŒN˜6œ>¨6Ô+KÑLÔLÐLÝŒK˜Ô,Ñ-Ô-Ð-Ð-Ð-Ý˜Õ 9Ñ:Ô:ð $	)Ø,2Ô,@×,FÒ,FÑ,HÔ,Hð ^ð ^Ñ(�
˜LØ'3°ZÐ&@Ð#ØÐ!1Ò1Ð1°fÔ6FÀzÔ6RÐVdÒ6dÐ6dØ:KÐ'¨Ñ7à#/ <°´Ð#UÐ#UÐATÐ#UÐ#UÑ �˜qÝ”
�7 6¨jÐ+CÐ+CÐ+CÑDÔDÀmÑTÔTÐTÝ”
�7 6¨jÐ+LÐ+LÐ+LÑMÔMÈ}Ñ]Ô]Ð]Ð]ð^ð ^õ ˜Õ ;Ñ<Ô<ð 	)ð Ô# yÒ0Ð0õ $ FÔ$4Ô5Ð5àÔ;ð ð
 &˜g f¤mÑ4Ô4‰OˆL˜!ÝŒJ�v”¨Ñ5Ô5Ð5ÝŒJ�vÔ/°Ñ>Ô>Ð>Ð>Ð>Ý˜Õ =Ñ>Ô>ð 	)ÝŒN˜6Ô-¨vÔ/HÑIÔIÐIÐIÐIÝ˜Õ 0Ñ1Ô1ð 	)ÝŒJ�v”|Ñ$Ô$Ð$ÝŒJ�vÔ.Ñ/Ô/Ð/Ð/Ð/Ý˜Õ 1Ñ2Ô2ð 	)Ø”+Ô/ˆCÝŒL˜Ô,°3¸CÐ@Ñ@Ô@Ð@ÝŒL˜Ô)°¸Ð=Ñ=Ô=Ð=Ð=Ð=Ý˜Õ 6Ñ7Ô7ð 		)ÝŒJ�vÔ*Ñ+Ô+Ð+Ð+Ð+Ý˜Õ 5Ñ6Ô6ð 	)¸6Ô;Uð 	)ÝŒN˜6Ô+­e°E©l¬l¨]Ñ;Ô;Ð;ÝŒN˜6Ô+­U°5©\¬\Ñ:Ô:Ð:ÝŒN˜6Ô,­u°U©|¬|¨mÑ<Ô<Ð<ÝŒN˜6Ô,­e°E©l¬lÑ;Ô;Ð;Ð;Ð;Ý˜Õ 1Ñ2Ô2ð 	)°v´}Ô7Pð 	)ÝŒK˜œÑ(Ô(Ð(ÝŒJ�vÔ'Ñ(Ô(Ð(Ð(Ð(ð	)ð 	)ð 	)ð 	)r_   )	rd   re   rf   Ú_no_split_modulesÚinput_modalitiesÚ_can_record_outputsrk   r½   rV  rS   r_   r]   rN  rN    sU   € € € € € ðð ð Ðð ;ÐØÐà€U„]�_„_ð2)ð 2)ñ „_ð2)ð 2)ð 2)r_   rN  zAThe base Gemma 4 language model without a language modeling head.©Úcustom_introc                   ó   ‡ — e Zd ZU eed<    eed¬¦  «        eedœZ	defˆ fd„Z
dej        dz  dej        dz  d	ej        fd
„Z	 ddej        dej        dz  d	ej        fd„Zeee	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dedz  dej        dz  dej        dz  dedz  dee         d	efd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚGemma4TextModelrD   r   )r  )Úrouter_logitsr‹   Ú
attentionsc                 ó"  •‡‡— t          ¦   «                              ‰¦  «         t          j        ˆfd„t	          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰¦  «        | _        t          | j
        j        ¦  «        | _        ‰j        | _        | j        r›t          ‰j        ‰j        ‰j        z  | j        ‰j        dz  ¬¦  «        | _        d| _        t          j        ‰j        ‰j        ‰j        z  d¬¦  «        | _        ‰j        dz  | _        t/          ‰j        ‰j        ¬¦  «        | _        g | _        t7          | j        ¦  «        D ]7\  Š}|j        j        r&| j                             ˆfd	„d
D ¦   «         ¦  «         Œ8d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS rS   )r8  ©r•  rM   rD   s     €r]   r™  z,Gemma4TextModel.__init__.<locals>.<listcomp>Q  s$   ø€ ÐhÐhÐh¸9Õ# F¨IÑ6Ô6ÐhÐhÐhr_   rm  )r^  gÍ;fž æ?Frz   rÁ   r;  c                 ó    •— g | ]
}d ‰› d|› �‘ŒS )zlayers.z.self_attn.rS   )r•  Únamer»  s     €r]   r™  z,Gemma4TextModel.__init__.<locals>.<listcomp>o  s*   ø€ ÐiÐiÐi¸Ð3˜qÐ3Ð3¨TÐ3Ð3ÐiÐiÐir_   )rÏ   rÐ   rÏ  rË  )r�   r‚   r   ré  r›  ræ  rê  r÷  rè  r  rD   r  Úunique_layer_typesr<  rL  Úvocab_size_per_layer_inputÚpadding_idxÚembed_tokens_per_layerÚper_layer_input_scaler„   rŸ   Úper_layer_model_projectionÚ per_layer_model_projection_scaler”   r  Úper_layer_projection_normÚ"_keys_to_ignore_on_load_unexpectedÚ	enumeraterO  rõ  Úextend)r‰   rD   Úlayerr»  rŠ   s    ` @€r]   r‚   zGemma4TextModel.__init__N  s§  øøø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”mØhÐhÐhÐhÍÈfÔNfÑHgÔHgÐhÑhÔhñ
ô 
ˆŒõ 4°FÑ;Ô;ˆŒÝ"% d¤kÔ&=Ñ">Ô">ˆÔð
 ,2Ô+MˆÔ(ØÔ+ð 	xÝ*GØÔ1ØÔ(¨6Ô+MÑMØÔ Ø"Ô>ÀÑCð	+ñ +ô +ˆDÔ'ð *3ˆDÔ&Ý.0¬iØÔ"ØÔ(¨6Ô+MÑMØð/ñ /ô /ˆDÔ+ð
 5;Ô4FÈÑ4LˆDÔ1Ý-:¸6Ô;]ÐciÔcvÐ-wÑ-wÔ-wˆDÔ*ð 35ˆÔ/Ý! $¤+Ñ.Ô.ð 	ð 	‰HˆAˆuØŒÔ1ð ØÔ7×>Ò>ØiÐiÐiÐiÐ@hÐiÑiÔiñô ð øð	ð 	r_   Ú	input_idsNrE   rJ   c                 óh  — | j         st          d| j        › �¦  «        ‚|€Ût          j        ¦   «         5  |dd…dd…ddd…f         | j        j        dddd…dd…f         | j        j        dz  z  k                         d¬¦  «         	                    ¦   «         dd…df         }	 | 
                    |j        dd…         ¦  «        }n# t          $ r t          d¦  «        ‚w xY w	 ddd¦  «         n# 1 swxY w Y    |                      |¦  «        j        g |j        ¢| j        j        ‘| j         ‘R Ž S )a  Compute the token-identity component of Per-Layer Embeddings (PLE).

        Looks up `input_ids` in `embed_tokens_per_layer` (a scaled embedding that multiplies
        by `sqrt(hidden_size_per_layer_input)`) and reshapes the packed output from
        `[batch, seq, num_hidden_layers * hidden_size_per_layer_input]` to
        `[batch, seq, num_hidden_layers, hidden_size_per_layer_input]`.

        If only `inputs_embeds` is provided (no `input_ids`), reverses the main embedding
        to recover `input_ids` for the PLE lookup.
        z}Attempting to call get_per_layer_inputs() from a model initialized with a config that does not support per-layer embeddings. Nrm  r	   r³   r*   a)  It seems like you tried to call `forward` from `inputs_embeds` without providing `input_ids`, and that the `inputs_embeds` you provided do not exactly match the embedding weights. Since Gemma4 needs to reverse the embedding to compute another embedding, make sure you provide exact `inputs_embeds`)r<  ÚRuntimeErrorrD   rk   r½   Úembed_tokensr  rŸ   r  Únonzerorç   rØ   r{  rÛ   ræ  )r‰   r„  rE   s      r]   Úget_per_layer_inputsz$Gemma4TextModel.get_per_layer_inputsr  sÉ  € ð Ô/ð 	Ýð8Ø*.¬+ð8ð 8ñô ð ð ÐÝ”‘”ð ð ð & a a a¨¨¨¨D°!°!°! mÔ4ØÔ,Ô3°D¸$ÀÀÀÀ1À1À1Ð4DÔEÈÌÔH_ÐadÑHdÑdòe÷ ’S˜Q�S‘Z”Zß’W‘Y”Y˜q˜q˜q !˜tô%ð ðØ )§¢¨}Ô/BÀ2ÀAÀ2Ô/FÑ GÔ G�I�IøÝ#ð ð ð Ý&ðrñô ð ðøøøð ðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð$ >ˆt×*Ò*¨9Ñ5Ô5Ô=ð 
ØŒ_ð
àŒKÔ)ð
ð Ô,ð
ð 
ð 
ð 	
s*   ´A/C1Â$"CÃC1ÃC!Ã!C1Ã1C5Ã8C5Úper_layer_inputsc                 ó  — | j         st          d| j        › �¦  «        ‚|                      |¦  «        | j        z  } |j        g |j        dd…         ¢| j        j        ‘| j         ‘R Ž }|                      |¦  «        }|€|S ||z   | j	        z  S )aŒ  Compute the context-aware component of PLE and combine with token-identity.

        Projects `inputs_embeds` through `per_layer_model_projection` (Linear), scales by
        `1/sqrt(hidden_size)`, reshapes to `[batch, seq, num_layers, ple_dim]`, and normalizes
        with `per_layer_projection_norm` (RMSNorm).

        If `per_layer_inputs` (the token-identity component from `get_per_layer_inputs()`)
        is provided, combines both: `(context_projection + token_identity) * (1/sqrt(2))`.
        If `per_layer_inputs` is None (e.g. for multimodal inputs where input_ids are not
        available), returns just the context projection.
        z�Attempting to call project_per_layer_inputs() from a model initialized with a config that does not support per-layer embeddings. Nr¯   )
r<  r†  rD   r}  r~  rÛ   rØ   ræ  r  r|  )r‰   rE   rŠ  r>  s       r]   Úproject_per_layer_inputsz(Gemma4TextModel.project_per_layer_inputsž  s×   € ð  Ô/ð 	Ýð@Ø26´+ð@ð @ñô ð ð
  $×>Ò>¸}ÑMÔMÐPTÔPuÑuÐØ;Ð3Ô;ð  
ØÔ   " Ô%ð 
àŒKÔ)ð 
ð Ô,ð 
ð  
ð  
Ðð
  $×=Ò=Ð>RÑSÔSÐàÐ#Ø'Ð'à$Ð'7Ñ7¸4Ô;UÑUÐUr_   rF   rH   rG   Ú	use_cacherT  c           
      óZ  — |du |duz  rt          d¦  «        ‚|�|�t          d¦  «        ‚|�|                      |¦  «        }| j        r.|€|                      ||¦  «        }|                      ||¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}	t          j	        |j
        d         |j        ¬¦  «        |	z   }|                     d¦  «        }t          |x}
t          ¦  «        s&| j        ||||dœ}t          di |¤Žt!          di |¤Žd	œ}
|}i }| j        D ]}|                      |||¦  «        ||<   Œ|                     d
t)          ¦   «         ¦  «        }t+          | j        d| j        j        …         ¦  «        D ]W\  }}|�|dd…dd…|dd…f         nd} |||f||| j        j        |                  |
| j        j        |                  ||dœ|¤Ž}ŒX|                      |¦  «        }t5          |||                     dd¦  «        r|nd¬¦  «        S )ak  
        per_layer_inputs (`torch.Tensor`, *optional*):
            Pre-computed per-layer input text embeddings of shape `(batch_size, sequence_length, num_hidden_layers,
            hidden_size_per_layer_input)`. When provided, these are used directly instead of being computed from `input_ids`
            via `get_per_layer_inputs()` in the text model. If calling the `forward` with `inputs_embeds` instead of `input_ids`,
            you should probably precompute them and forward them along `inputs_embeds`, otherwise recomputing them needs
            to reverse the main embedding, which is expensive.
        Nú:You must specify exactly one of input_ids or inputs_embedsú<You cannot specify per_layer_inputs if input_ids is provided)rD   r   r?   r°   rL   rP   rb   )rb   rë   rF   rH   rG   Úreturn_shared_kv_statesF)rí  rG   rb   rS   )r{  r‡  r<  r‰  rŒ  r   rD   Úget_seq_lengthrk   r¨   rØ   r±   r©   r´  rh   r   r   rx  rè  Úpopr   r�  rê  ræ  r  r  rq   Úget)r‰   r„  rF   rH   rG   rE   rŠ  r�  rT  Úpast_seen_tokensÚcausal_mask_mappingrU   r‹   rë   rû  rb   r»  rî  rF  s                      r]   r�   zGemma4TextModel.forwardÁ  sã  € ð, ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ð%5Ð%AÝÐ[Ñ\Ô\Ð\àÐ Ø ×-Ò-¨iÑ8Ô8ˆMàÔ+ð 	^ØÐ'Ø#'×#<Ò#<¸YÈÑ#VÔ#VÐ Ø#×<Ò<¸]ÐL\Ñ]Ô]Ðàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLõ °Ð?Ð-ÅÑFÔFð 	ð œ+Ø!.Ø"0Ø#2Ø ,ðð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ%FÐ%UÐ%UÈÐ%UÐ%Uð#ð #Ðð &ˆØ ÐØÔ1ð 	gð 	gˆJØ.2¯oªo¸mÈ\Ð[eÑ.fÔ.fÐ 
Ñ+Ð+ð
 "Ÿ:š:Ð&8½(¹*¼*ÑEÔEÐõ !*¨$¬+Ð6U¸¼Ô8UÐ6UÔ*VÑ WÔ Wð 	ð 	ÑˆAˆ}Ø>NÐ>ZÐ.¨q¨q¨q°!°!°!°Q¸¸¸¨zÔ:Ð:Ð`dˆOà)˜MØØð	ð "2Ø$7¸¼Ô8OÐPQÔ8RÔ$SØ2°4´;Ô3JÈ1Ô3MÔNØ)Ø /ð	ð 	ð ð	ð 	ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå,Ø+Ø+Ø17·²Ð<UÐW\Ñ1]Ô1]ÐgÐ-Ð-Ðcgð
ñ 
ô 
ð 	
r_   r�   )NNNNNNN)rd   re   rf   rB   rm   r(   r)  r8  r  rl  r‚   rk   rl   r‰  rŒ  r&   r)   r    rÖ  r   rá  Úboolr   r   rq   r�   r‘   r’   s   @r]   rp  rp  E  sö  ø€ € € € € € àÐÐÑà'˜Ð(8ÀÐBÑBÔBØ/Ø)ðð Ðð"Ð/ð "ð "ð "ð "ð "ð "ðH*
¨e¬l¸TÑ.Að *
ÐRWÔR^ÐaeÑReð *
ÐjoÔjvð *
ð *
ð *
ð *
ð^ 15ð!Vð !Và”|ð!Vð  œ,¨Ñ-ð!Vð 
Œð	!Vð !Vð !Vð !VðF  ØØð .2Ø.2Ø04Ø(,Ø26Ø04Ø!%ðY
ð Y
àÔ# dÑ*ðY
ð œ tÑ+ðY
ð Ô&¨Ñ-ð	Y
ð
  ™ðY
ð Ô(¨4Ñ/ðY
ð  œ,¨Ñ-ðY
ð ˜$‘;ðY
ð Ð+Ô,ðY
ð 
'ðY
ð Y
ð Y
ñ „^ñ „_ñ  ÔðY
ð Y
ð Y
ð Y
ð Y
r_   rp  z>The base Gemma 4 language model with a language modeling head.c                   ó  — e Zd ZdZee	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  de	dz  dej
        dz  d	ej        dz  d
edz  deej        z  dej        dz  dee         defd„¦   «         ¦   «         ZdS )ÚGemma4ForCausalLMÚmodelNr   r„  rF   rH   rG   rE   Úlabelsr�  Úlogits_to_keeprŠ  rT  rJ   c
                 óÎ  —  | j         d||||||	|dœ|
¤Ž}|j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }| j        j        �2|| j        j        z  }t          j	        |¦  «        }|| j        j        z  }d}|� | j
        ||| j        fi |
¤Ž}t          |||j        |j        |j        |j        ¬¦  «        S )aí  
        per_layer_inputs (`torch.Tensor`, *optional*):
            Pre-computed per-layer input text embeddings of shape `(batch_size, sequence_length, num_hidden_layers,
            hidden_size_per_layer_input)`. When provided, these are used directly instead of being computed from `input_ids`
            via `get_per_layer_inputs()` in the text model. If calling the `forward` with `inputs_embeds` instead of `input_ids`,
            you should probably precompute them and forward them along `inputs_embeds`, otherwise recomputing them needs
            to reverse the main embedding, which is expensive.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, Gemma4ForCausalLM

        >>> model = Gemma4ForCausalLM.from_pretrained("google/gemma-4-E2B-it")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-4-E2B-it")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```)r„  rF   rH   rG   rE   rŠ  r�  N)ÚlossÚlogitsrG   r‹   rr  rb   rS   )rš  rí  r´  r�   ÚsliceÚlm_headrD   Úfinal_logit_softcappingrk   rñ   Úloss_functionÚ
vocab_sizero   rG   r‹   rr  rb   )r‰   r„  rF   rH   rG   rE   r›  r�  rœ  rŠ  rT  Úoutputsr‹   Úslice_indicesrŸ  rž  s                   r]   r�   zGemma4ForCausalLM.forward$  s/  € ðP 2<°´ð 	2
ØØ)Ø%Ø+Ø'Ø-Øð	2
ð 	2
ð ð	2
ð 	2
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆØŒ;Ô.Ð:Ø˜dœkÔAÑAˆFÝ”Z Ñ'Ô'ˆFØ˜dœkÔAÑAˆFàˆØÐØ%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDå+ØØØ#Ô3Ø!Ô/ØÔ)Ø$Ô5ð
ñ 
ô 
ð 	
r_   )	NNNNNNNr   N)rd   re   rf   Úbase_model_prefixr!   r    rk   rÖ  rl   r   rá  r—  r�   r   r   ro   r�   rS   r_   r]   r™  r™     s&  € € € € € àÐàØð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.Ø04ðE
ð E
àÔ# dÑ*ðE
ð œ tÑ+ðE
ð Ô&¨Ñ-ð	E
ð
  ™ðE
ð Ô(¨4Ñ/ðE
ð Ô  4Ñ'ðE
ð ˜$‘;ðE
ð ˜eœlÑ*ðE
ð  œ,¨Ñ-ðE
ð Ð+Ô,ðE
ð 
&ðE
ð E
ð E
ñ „^ñ ÔðE
ð E
ð E
r_   r™  c                   ó  ‡ — e Zd ZU dZeed<   dZdZee	dœZ
defˆ fd„Zdej        dej        fd	„Zee ed
¬¦  «        	 ddej        dej        dz  dee         deej        ej        f         fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚGemma4AudioModelznAn audio encoder based on the [Universal Speech Model](https://huggingface.co/papers/2303.01037) architecture.rD   r   zmodel.audio_tower©r‹   rr  c                 óŒ  •‡— t          ¦   «                              ‰¦  «         ‰| _        t          ‰¦  «        | _        t          ‰¦  «        | _        t          j        ˆfd„t          ‰j
        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j        ‰j        d¬¦  «        | _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS rS   )rK  ru  s     €r]   r™  z-Gemma4AudioModel.__init__.<locals>.<listcomp>€  s$   ø€ ÐbÐbÐb°YÕ˜f iÑ0Ô0ÐbÐbÐbr_   Trz   )r�   r‚   rD   r  Úsubsample_conv_projectionr˜   Úrel_pos_encr   ré  r›  ræ  rê  r„   rŸ   Úoutput_proj_dimsÚoutput_projÚ	post_initr/  s    `€r]   r‚   zGemma4AudioModel.__init__y  s°   øø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒå)KÈFÑ)SÔ)SˆÔ&Ý;¸FÑCÔCˆÔÝ”mØbÐbÐbÐbÅ%ÈÔH`ÑBaÔBaÐbÑbÔbñ
ô 
ˆŒõ œ9 VÔ%7¸Ô9PÐW[Ð\Ñ\Ô\ˆÔà�ŠÑÔÐÐÐr_   Úmask_4drJ   c                 óR  — |j         \  }}}}|j        }| j        j        }| j        j        dz
  }| j        j        }||z   dz
  |z  }	|	|z  }
|
|z
  }t          j        |d|d|fd¬¦  «        }|                     |d|	||
¦  «        }t          j        |||fd¬¦  «        }t          j
        |	|¬¦  «        |z  }t          j
        ||z   |z   |¬¦  «        }|dd…df         |ddd…f         z   }|dddd…ddd…f                              |dd|d¦  «        }|                     d|¦  «        S )zÜ
        Convert a standard 4D attention mask `[batch_size, 1, seq_len, seq_len]` to the 5D blocked format
        `[batch_size, 1, num_blocks, chunk_size, context_size]` expected by the chunked local attention,
        r?   r   F)Úvaluer°   Nr¯   )rØ   r±   rD   r    r¡   r¢   rÙ   rÚ   rÛ   rk   r¨   r³  Úgather)r‰   r²  rÝ   rX   rÞ   r±   rË   rÌ   rÍ   rß   Úpadded_seq_lenÚ
pad_amountÚmask_5dÚblock_startsÚoffsetsÚ
kv_indicess                   r]   Ú_convert_4d_mask_to_blocked_5dz/Gemma4AudioModel._convert_4d_mask_to_blocked_5d†  sd  € ð
 %,¤MÑ!ˆ
�A�w Ø”ˆà”[Ô5ˆ
Øœ;Ô=ÀÑAÐØ!œ[Ô@Ðà 
Ñ*¨QÑ.°:Ñ=ˆ
Ø# jÑ0ˆØ# gÑ-ˆ
å”%˜ ! Z°°JÐ!?ÀuÐMÑMÔMˆØ—/’/ *¨a°¸ZÈÑXÔXˆÝ”%˜Ð"2Ð4FÐ!GÈuÐUÑUÔUˆå”| J°vÐ>Ñ>Ô>ÀÑKˆÝ”,˜zÐ,<Ñ<Ð?QÑQÐZ`ÐaÑaÔaˆØ! ! ! ! T 'Ô*¨W°T¸1¸1¸1°WÔ-=Ñ=ˆ
Ø  d¨A¨A¨A¨t°Q°Q°QÐ 6Ô7×>Ò>¸zÈ1ÈbÐR\Ð^`ÑaÔaˆ
à�~Š~˜b *Ñ-Ô-Ð-r_   z&Encodes audio features to soft tokens.rm  NrF   rT  c           	      ó¤  — |                       ||¦  «        \  }}|                      |¦  «        }t          | j        ||t	          | j        j        dz
  | j        j        f¦  «        ¬¦  «        }|�|                      |¦  «        }| j        d | j        j	        …         D ]} ||f||dœ|¤Ž}Œ|  
                    |¦  «        }t          ||¬¦  «        S )Nr?   )rD   rE   rF   rO   )rF   rë   )rí  rF   )r­  r®  r   rD   r>   r¡   r¢   r¼  rê  ræ  r°  rs   )r‰   r   rF   rT  r‹   Úoutput_maskrë   Úencoder_layers           r]   r�   zGemma4AudioModel.forward¡  s  € ð &*×%CÒ%CÀNÐTbÑ%cÔ%cÑ"ˆ�{Ø"×.Ò.¨}Ñ=Ô=Ðå2Ø”;Ø'Ø&Ý:Ø”Ô3°aÑ7¸¼Ô9\Ð]ñô ð	
ñ 
ô 
ˆð Ð%Ø!×@Ò@ÀÑPÔPˆNà!œ[Ð)H¨4¬;Ô+HÐ)HÔIð 	ð 	ˆMØ)˜MØðà-Ø$7ðð ð ð	ð ˆMˆMð ×(Ò(¨Ñ7Ô7ˆÝ%¸ÐVaÐbÑbÔbÐbr_   r�   )rd   re   rf   rg   r@   rm   Úmain_input_namer§  rK  r¿   rl  r‚   rk   rl   r¼  r&   r)   r    r   r   rj   rt   r�   r‘   r’   s   @r]   r©  r©  n  s=  ø€ € € € € € ØxÐxàÐÐÑØ&€OØ+Ðà)Ø*ðð Ðð
Ð0ð ð ð ð ð ð ð.°e´lð .ÀuÄ|ð .ð .ð .ð .ð6  ØØ€^Ð!IÐJÑJÔJð /3ðcð càœðcð œ tÑ+ðcð Ð+Ô,ð	cð
 
ˆuŒ|˜UÔ-Ð-Ô	.ðcð cð cñ KÔJñ „_ñ  Ôðcð cð cð cð cr_   r©  c                   ó²   ‡ — e Zd ZdZeZeedœZdefˆ fd„Z	e
e ed¬¦  «        dej        dej        d	ee         d
efd„¦   «         ¦   «         ¦   «         Zˆ xZS )rc  zThe Gemma 4 Vision Encoder.rª  rD   c                 óÊ  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¦  «        | _        t          |¦  «        | _        | j        j	        rd|  
                    dt          j        | j        j        ¦  «        ¦  «         |  
                    dt          j        | j        j        ¦  «        ¦  «         |                      ¦   «          d S )Nre  rf  )r�   r‚   rW  Úpatch_embedderrã  Úencoderrq  ÚpoolerrD   rd  r†   rk   ÚemptyrŸ   r±  r/  s     €r]   r‚   zGemma4VisionModel.__init__Í  s·   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý7¸Ñ?Ô?ˆÔÝ*¨6Ñ2Ô2ˆŒÝ(¨Ñ0Ô0ˆŒàŒ;Ô"ð 	TØ× Ò  ­U¬[¸¼Ô9PÑ-QÔ-QÑRÔRÐRØ× Ò  ­e¬k¸$¼+Ô:QÑ.RÔ.RÑSÔSÐSà�ŠÑÔÐÐÐr_   z1Encodes image pixels to soft tokens from patches.rm  rk  r^  rT  rJ   c                 óð  — | j         j        }|j        d         ||z  z  }|dk                         d¬¦  «        }|                      |||¦  «        } | j        d|| |dœ|¤Ž}|                      |j        |||¬¦  «        \  }	}
|	|
         }	| j         j        r6|	| j	         
                    ¦   «         z
  | j         
                    ¦   «         z  }	|	                     |j        ¦  «        }	t          |	¬¦  «        S )aà  
        pixel_values (`torch.FloatTensor` or `list[torch.FloatTensor]`):
            The images to encode. Either a single `[batch, channels, height, width]` tensor
            (all images same size) or a list of `[1, channels, height, width]` tensors (different sizes).
        pixel_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`):
            The patch positions as (x, y) coordinates in the image. Padding patches are indicated by (-1, -1).
        éþÿÿÿr¯   r³   )rE   rF   r^  )r‹   r^  r_  rˆ  rì  rS   )rD   Úpooling_kernel_sizerØ   r  rÃ  rÄ  rÅ  rí  rd  re  rˆ   rf  r·   r¶   r   )r‰   rk  r^  rT  rÉ  rˆ  r_  rE   r†  r‹   Úpooler_masks              r]   r�   zGemma4VisionModel.forwardÙ  s,  € ð  #œkÔ=ÐØ$Ô*¨2Ô.Ð3FÐI\Ñ3\Ñ]ˆà/°2Ò5×:Ò:¸rÐ:ÑBÔBÐØ×+Ò+¨LÐ:LÐN_Ñ`Ô`ˆØ�”ð 
Ø'Ø-Ð-Ø1ð
ð 
ð ð	
ð 
ˆð &*§[¢[Ø Ô2Ø1Ø/Ø'ð	 &1ñ &
ô &
Ñ"ˆ�{ð & kÔ2ˆð Œ;Ô"ð 	]Ø*¨T¬]×-@Ò-@Ñ-BÔ-BÑBÀdÄn×FZÒFZÑF\ÔF\Ñ\ˆMØ%×(Ò(¨Ô)<Ñ=Ô=ˆå&¸ÐGÑGÔGÐGr_   )rd   re   rf   rg   rC   rD   rØ  rÃ  rl  r‚   r&   r)   r    rk   rá  rÖ  r   r   r   r�   r‘   r’   s   @r]   rc  rc  Ä  sæ   ø€ € € € € Ø%Ð%à€Fà1Ø+ðð Ðð

Ð1ð 
ð 
ð 
ð 
ð 
ð 
ð  ØØ€^Ð!TÐUÑUÔUð)HàÔ'ð)Hð "Ô,ð)Hð Ð+Ô,ð	)Hð
 
!ð)Hð )Hð )Hñ VÔUñ „_ñ  Ôð)Hð )Hð )Hð )Hð )Hr_   rc  c                   óR   ‡ — e Zd Zdeez  defˆ fd„Zdej        dej        fd„Z	ˆ xZ
S )ÚGemma4MultimodalEmbedderÚmultimodal_configÚtext_configc                 óÜ   •— t          ¦   «                              ||¦  «         | `| `| `| `| `| `t          |d|j	        ¦  «        | _
        t          | j
        | j        d¬¦  «        | _        d S )Nr¯  Fr@  )r�   r‚   re  Úhard_embedding_normÚsoft_embedding_normÚvocab_offsetr¤  Úembedding_post_projection_normr§  rŸ   Úmultimodal_hidden_sizer”   r  Úembedding_pre_projection_norm)r‰   rÍ  rÎ  rŠ   s      €r]   r‚   z!Gemma4MultimodalEmbedder.__init__	  s�   ø€ õ 	‰Œ×ÒÐ*¨KÑ8Ô8Ð8ØˆNØÐ$ØÐ$ØÐØˆOØÐ/å&-Ð.?ÐASÐUfÔUrÑ&sÔ&sˆÔ#Ý-:¸4Ô;VÐ\`Ô\dÐqvÐ-wÑ-wÔ-wˆÔ*Ð*Ð*r_   rE   rJ   c                 óV   — |                       |¦  «        }|                      |¦  «        S )a:  Embeds token ids or soft tokens for multimodal content into language model space.
        Args:
            inputs_embeds: A torch.Tensor containing the soft tokens to embed.
        Returns:
            A torch.Tensor of embeddings with shape `[batch_size, seq_len, self.config.text_config.hidden_size]`.
        )rÕ  Úembedding_projection)r‰   rE   Úembs_normeds      r]   r�   z Gemma4MultimodalEmbedder.forward  s+   € ð ×8Ò8¸ÑGÔGˆØ×(Ò(¨Ñ5Ô5Ð5r_   )rd   re   rf   r@   rC   rB   r‚   rk   rl   r�   r‘   r’   s   @r]   rÌ  rÌ    sƒ   ø€ € € € € ðxà,Ð/AÑAðxð &ðxð xð xð xð xð xð$6 U¤\ð 6°e´lð 6ð 6ð 6ð 6ð 6ð 6ð 6ð 6r_   rÌ  Útoken_type_idsÚimage_group_idsc           
      ób   ‡— | €dS dt           dt           dt           dt           dt          f
ˆfd„}|S )z•
    This function adds the correct offsets to the `q_idx` and `kv_idx` as the torch API can only accept lengths,
    not start and end indices.
    NÚ	batch_idxÚhead_idxÚq_idxÚkv_idxrJ   c                 ó,  •— ‰	j         d         }|                     |dz
  ¬¦  «        }|                     |dz
  ¬¦  «        }‰	| |f         }‰	| |f         }t          j        ||k     |d¦  «        }t          j        ||k     |d¦  «        }||k    |dk    z  S )Nr¯   r?   )r¦   r   )rØ   rŽ   rk   rf  )
rÜ  rÝ  rÞ  rß  r÷   Úq_idx_clampedÚkv_idx_clampedÚq_groupÚkv_grouprÚ  s
            €r]   Ú
inner_maskz0token_type_ids_mask_function.<locals>.inner_mask3  s¤   ø€ Ø$Ô*¨2Ô.ˆ
ð Ÿš¨
°Q©˜Ñ7Ô7ˆØŸš¨*°q©.˜Ñ9Ô9ˆð " )¨]Ð":Ô;ˆØ" 9¨nÐ#<Ô=ˆÝ”+˜e jÒ0°'¸2Ñ>Ô>ˆÝ”;˜v¨
Ò2°H¸bÑAÔAˆØ˜8Ò#¨°1ªÑ5Ð5r_   )r�   r—  )rÙ  rÚ  rå  s    ` r]   Útoken_type_ids_mask_functionræ  '  sY   ø€ ð ÐØˆtð6�cð 6­Sð 6½ð 6Åcð 6Ídð 6ð 6ð 6ð 6ð 6ð 6ð Ðr_   Úmm_token_type_idsr±   c                 ó  — |                       |¦  «        } | dk    | dk    z  }t          j        |dd¬¦  «        }d|d<   || z  }t          j        |                     ¦   «         d¬¦  «        dz
  }t          j        ||d¦  «        }|S )Nr?   r*   r¯   )ÚshiftsÚdimsFrb  r³   )r·   rk   ÚrollÚcumsumr�   rf  )rç  r±   Ú	is_visionÚis_prev_visionÚnew_vision_startsÚvision_group_idsrI   s          r]   Úget_block_sequence_ids_for_maskrñ  D  sš   € Ø)×,Ò,¨VÑ4Ô4Ðà" aÒ'Ð,=ÀÒ,BÑC€IÝ”Z 	°!¸"Ð=Ñ=Ô=€NØ"€N�6ÑØ! ^ OÑ3ÐÝ”|Ð$5×$9Ò$9Ñ$;Ô$;ÀÐCÑCÔCÀaÑGÐÝœ YÐ0@À"ÑEÔEÐØÐr_   zŒ
    The base Gemma 4 model comprising a vision backbone, an audio backbone, and a language model without a
    language modeling head.
    c            $       óf  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Ze ed¬¦  «        	 dde	j
        d	e	j        dz  d
ee         defd„¦   «         ¦   «         Ze ed¬¦  «        	 dde	j
        de	j        dz  d
ee         defd„¦   «         ¦   «         Z	 	 d de	j        dz  de	j
        dz  dee	j        e	j        e	j        f         fd„Zeee	 	 	 	 	 	 	 	 	 	 	 	 	 	 d!de	j        dz  de	j
        dz  de	j
        dz  de	j
        dz  de	j        dz  de	j        dz  de	j        dz  dedz  de	j        dz  de	j
        dz  dedz  d	e	j        dz  de	j        dz  de	j        dz  d
ee         def d„¦   «         ¦   «         ¦   «         Ze ed¬¦  «        de	j        de	j        d
ee         deez  fd„¦   «         ¦   «         Zˆ xZS )"ÚGemma4ModelrD   c                 ó†  •— t          ¦   «                              |¦  «         |j        �t          j        |j        ¦  «        nd | _        |j        �t          |j        |j        ¦  «        nd | _        |j	        �t          j        |j	        ¦  «        nd | _
        |j	        �t          |j	        |j        ¦  «        nd | _        d S r�   )r�   r‚   Úvision_configr+   Úfrom_configÚvision_towerrÌ  rÎ  Úembed_visionÚaudio_configÚaudio_towerÚembed_audior/  s     €r]   r‚   zGemma4Model.__init__W  sÉ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØKQÔK_ÐKk�IÔ1°&Ô2FÑGÔGÐGÐquˆÔð Ô#Ð/õ % VÔ%9¸6Ô;MÑNÔNÐNàð 	Ôð
 JPÔI\ÐIh�9Ô0°Ô1DÑEÔEÐEÐnrˆÔð Ô"Ð.õ % VÔ%8¸&Ô:LÑMÔMÐMàð 	ÔÐÐr_   c                 ó   — | j         j        S r�   ©Úlanguage_modelr{  ©r‰   s    r]   Úget_per_layer_input_embeddingsz*Gemma4Model.get_per_layer_input_embeddingsf  s   € ØÔ"Ô9Ð9r_   c                 ó   — || j         _        d S r�   rý  ©r‰   r´  s     r]   Úset_per_layer_input_embeddingsz*Gemma4Model.set_per_layer_input_embeddingsi  s   € Ø5:ˆÔÔ2Ð2Ð2r_   zOProjects the last hidden state from the vision model into language model space.rm  Nrk  Úimage_position_idsrT  rJ   c                 ój   —  | j         d||dœ|¤Ž}|j        }|                      |¬¦  «        |_        |S )zÝ
        image_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`, *optional*):
            The patch positions as (x, y) coordinates in the image. Padding patches are indicated by (-1, -1).
        ©rk  r^  ©rE   rS   )r÷  rí  rø  Úpooler_output)r‰   rk  r  rT  Úvision_outputsrí  s         r]   Úget_image_featureszGemma4Model.get_image_featuresl  s]   € ð +˜Ô*ð 
Ø%Ø1ð
ð 
ð ð
ð 
ˆð
 +Ô<ÐØ'+×'8Ò'8ÐGXÐ'8Ñ'YÔ'YˆÔ$ØÐr_   zQProjects the last hidden state from the vision encoder into language model space.Úpixel_values_videosÚvideo_position_idsc                 óÂ   — |                      dd¦  «        }|                      dd¦  «        } | j        d||dœ|¤Ž}|j        }|                      |¬¦  «        |_        |S )a9  
        video_position_ids (`torch.LongTensor` of shape `(num_videos, num_frames, max_patches, 2)`, *optional*):
            2D patch position coordinates from the video processor, with `(-1, -1)` indicating padding.
            Passed through to the vision encoder for positional embedding computation.
        r   r?   r  r  rS   )Úflattenr÷  rí  rø  r  )r‰   r  r  rT  r	  rí  s         r]   Úget_video_featureszGemma4Model.get_video_features�  s‰   € ð 2×9Ò9¸!¸QÑ?Ô?ÐØ/×7Ò7¸¸1Ñ=Ô=ÐØ*˜Ô*ð 
Ø,Ø1ð
ð 
ð ð
ð 
ˆð
 +Ô<ÐØ'+×'8Ò'8ÐGXÐ'8Ñ'YÔ'YˆÔ$ØÐr_   r„  rE   c                 óÀ  — |�2|| j         j        k    }|| j         j        k    }|| j         j        k    }�n&| |                      ¦   «         t          j        | j         j        t
          j        |j        ¬¦  «        ¦  «        k     	                    d¦  «        }| |                      ¦   «         t          j        | j         j        t
          j        |j        ¬¦  «        ¦  «        k     	                    d¦  «        }| |                      ¦   «         t          j        | j         j        t
          j        |j        ¬¦  «        ¦  «        k     	                    d¦  «        }|||fS )a‹  
        Obtains mask for multimodal placeholders (replaced by soft tokens) and hard text tokens.

        Masks will be obtained from `mm_token_type_ids`, `input_ids`, or `inputs_embeds` as available and in that
        precedence order. If passing `input_ids` or `inputs_embeds`, the image mask will be derived using
        `config.image_token_id`. Same goes for audio and video masks

        Args:
            input_ids: A tensor containing the hard token IDs from the text tokenizer.
            inputs_embeds: A tensor containing the embeddings for all hard text tokens.

        Returns:
            image_mask, video_mask, audio_mask
        N)r¶   r±   r¯   )
rD   Úimage_token_idÚvideo_token_idÚaudio_token_idÚget_input_embeddingsrk   r‡   r~  r±   r  )r‰   r„  rE   Úspecial_image_maskÚspecial_video_maskÚspecial_audio_masks         r]   Úget_placeholder_maskz Gemma4Model.get_placeholder_mask™  sP  € ð& Ð Ø!*¨d¬kÔ.HÒ!HÐØ!*¨d¬kÔ.HÒ!HÐØ!*¨d¬kÔ.HÒ!HÐÑð Ø.�4×,Ò,Ñ.Ô.Ý”L ¤Ô!;Å5Ä:ÐVcÔVjÐkÑkÔkñô ò÷ Šc�"‰gŒgð ð Ø.�4×,Ò,Ñ.Ô.Ý”L ¤Ô!;Å5Ä:ÐVcÔVjÐkÑkÔkñô ò÷ Šc�"‰gŒgð ð Ø.�4×,Ò,Ñ.Ô.Ý”L ¤Ô!;Å5Ä:ÐVcÔVjÐkÑkÔkñô ò÷ Šc�"‰gŒgð ð "Ð#5Ð7IÐIÐIr_   r   rF   r!  rH   rG   rç  r�  rŠ  c                 ó@  — |du |
duz  rt          d¦  «        ‚|�|�t          d¦  «        ‚|                      ||
¦  «        \  }}}||z  |z  }d}|
€V|                     ¦   «         }t          j        || j        j        j        |¦  «        } |                      ¦   «         |¦  «        }
|€®| j         	                    ¦   «         j
        r�| j        j        j        | j        j        j        dd…f         }|                     |
j        ¦  «        }t          j        |d         |                     ddd¦  «        |
¦  «        }| j                             ||¦  «        }|��&|                      ||d¬¦  «        j        }|                     |
j        |
j        ¦  «        }|                     ¦   «         }|                     d¦  «                             |
¦  «                             |
j        ¦  «        }t1          |
|                              ¦   «         |                     ¦   «         k    d	|› d
|j        d         › �¦  «         |
                     |                     |
j        ¦  «        |                     |
j        ¦  «        ¦  «        }
|��&|                      ||d¬¦  «        j        }|                     |
j        |
j        ¦  «        }|                     ¦   «         }|                     d¦  «                             |
¦  «                             |
j        ¦  «        }t1          |
|                              ¦   «         |                     ¦   «         k    d|› d
|j        d         › �¦  «         |
                     |                     |
j        ¦  «        |                     |
j        ¦  «        ¦  «        }
|��@|��=|                      ||d¬¦  «        }|j        }|j        }||                     |j        ¦  «                 }|                     ¦   «         }|                     d¦  «                             |
¦  «                             |
j        ¦  «        }t1          |
|                              ¦   «         |                     ¦   «         k    d|› d
|j        d         |j        d         z  › �¦  «         |
                     |                     |
j        ¦  «        |                     |
j        ¦  «        ¦  «        }
|€V|�|                     ¦   «         nd}t          j         |
j        d         |
j        ¬¦  «        |z   }|                     d¦  «        }tC          |x} tD          ¦  «        sx| j         	                    ¦   «         |
|||dœ}!| j         	                    ¦   «         }"|"j#        dk    }#|#r'|	�%tI          |	|
j        ¬¦  «        }$tK          dd|$i|!¤Ž} ntM          di |!¤Ž}  | j        d|| |||
|ddœ|¤Ž}%tO          |%j(        |%j)        |%j*        |%j+        |�|nd|�|nd|%j,        ¬¦  «        S )áK  
        input_features_mask (`torch.FloatTensor]` of shape `(num_images, seq_length)`):
            The attention mask for the input audio.
        image_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`, *optional*):
            2D patch position coordinates from the image processor, with `(-1, -1)` indicating padding.
            Passed through to the vision encoder for positional embedding computation.
        video_position_ids (`torch.LongTensor` of shape `(num_videos, num_frames, max_patches, 2)`, *optional*):
            2D patch position coordinates from the video processor, with `(-1, -1)` indicating padding.
            Passed through to the vision encoder for positional embedding computation.
        per_layer_inputs (`torch.Tensor`, *optional*):
            Pre-computed per-layer input text embeddings of shape `(batch_size, sequence_length, num_hidden_layers,
            hidden_size_per_layer_input)`. When provided, these are used directly instead of being computed from `input_ids`
            via `get_per_layer_inputs()` in the text model. If calling the `forward` with `inputs_embeds` instead of `input_ids`,
            you should probably precompute them and forward them along `inputs_embeds`, otherwise recomputing them needs
            to reverse the main embedding, which is expensive.
        Nr�  r�  r²   r?   r¯   T)Úreturn_dictz6Image features and image tokens do not match, tokens: z, features: r   z6Video features and video tokens do not match, tokens: z6Audio features and audio tokens do not match, tokens: r°   rL   ÚvisionrI   )rŠ  rF   rH   rG   rE   r�  r  )rí  rG   r‹   rr  Úimage_hidden_statesÚaudio_hidden_statesrb   rS   )-r{  r  r	  rk   rf  rD   rÎ  Úpad_token_idr  Úget_text_configr<  rþ  r‡  r  r·   r±   rç   r‰  r
  r  r¶   r2  r©   Ú	expand_asr$   ÚnumelrØ   Úmasked_scatterr  Úget_audio_featuresrF   r’  r¨   r´  rh   r  rñ  r^   r   ra   rí  rG   r‹   rr  rb   )&r‰   r„  rk  r  r   rF   r!  rH   rG   rç  rE   r�  r  r  rŠ  rT  Ú
image_maskÚ
video_maskÚ
audio_maskÚmultimodal_maskÚllm_input_idsÚpad_embeddingÚllm_inputs_embedsÚimage_featuresÚn_image_tokensÚvideo_featuresÚn_video_tokensÚaudio_outputÚaudio_featuresÚaudio_mask_from_encoderÚn_audio_tokensr•  r–  rU   rÎ  Ú	use_bidirrI   r¥  s&                                         r]   r�   zGemma4Model.forwardÆ  s©  € ðJ ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ð%5Ð%AÝÐ[Ñ\Ô\Ð\à-1×-FÒ-FÀyÐR_Ñ-`Ô-`Ñ*ˆ
�J 
Ø$ zÑ1°JÑ>ˆð ˆØÐ Ø%ŸOšOÑ-Ô-ˆMÝ!œK¨¸¼Ô9PÔ9]Ð_lÑmÔmˆMØ7˜D×5Ò5Ñ7Ô7¸ÑFÔFˆMàÐ#¨¬×(CÒ(CÑ(EÔ(EÔ(aÐ#Ø Ô/Ô<ÔCÀDÄKÔD[ÔDhÐjkÐjkÐjkÐDkÔlˆMØ-×0Ò0°Ô1EÑFÔFˆOÝ %¤¨O¸IÔ,FÈ×HZÒHZÐ[\Ð^_ÐacÑHdÔHdÐfsÑ tÔ tÐØ#Ô2×GÒGÈÐWhÑiÔiÐð Ñ#Ø!×4Ò4°\ÐCUÐcgÐ4ÑhÔhÔvˆNØ+×.Ò.¨}Ô/CÀ]ÔEXÑYÔYˆNð (Ÿ^š^Ñ-Ô-ˆNØ#×-Ò-¨bÑ1Ô1×;Ò;¸MÑJÔJ×MÒMÈmÔNbÑcÔcˆJÝ"Ø˜jÔ)×/Ò/Ñ1Ô1°^×5IÒ5IÑ5KÔ5KÒKð.Èð .ð .Ø"Ô(¨Ô+ð.ð .ñô ð ð *×8Ò8Ø—’˜mÔ2Ñ3Ô3°^×5FÒ5FÀ}ÔG[Ñ5\Ô5\ñô ˆMð Ñ*Ø!×4Ò4Ø#Ð%7ÀTð 5ñ ô äð ð ,×.Ò.¨}Ô/CÀ]ÔEXÑYÔYˆNð (Ÿ^š^Ñ-Ô-ˆNØ#×-Ò-¨bÑ1Ô1×;Ò;¸MÑJÔJ×MÒMÈmÔNbÑcÔcˆJÝ"Ø˜jÔ)×/Ò/Ñ1Ô1°^×5IÒ5IÑ5KÔ5KÒKð.Èð .ð .Ø"Ô(¨Ô+ð.ð .ñô ð ð *×8Ò8Ø—’˜mÔ2Ñ3Ô3°^×5FÒ5FÀ}ÔG[Ñ5\Ô5\ñô ˆMð
 Ñ%Ð*=Ñ*IØ×2Ò2°>ÐCVÐdhÐ2ÑiÔiˆLØ)Ô7ˆNØ&2Ô&AÐ#ð
 ,Ð,C×,FÒ,FÀ~ÔG\Ñ,]Ô,]Ô^ˆNà'Ÿ^š^Ñ-Ô-ˆNØ#×-Ò-¨bÑ1Ô1×;Ò;¸MÑJÔJ×MÒMÈmÔNbÑcÔcˆJÝ"Ø˜jÔ)×/Ò/Ñ1Ô1°^×5IÒ5IÑ5KÔ5KÒKðHÈð Hð HØ"Ô(¨Ô+¨nÔ.BÀ1Ô.EÑEðHð Hñô ð ð *×8Ò8Ø—’˜mÔ2Ñ3Ô3°^×5FÒ5FÀ}ÔG[Ñ5\Ô5\ñô ˆMð
 ÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå°Ð?Ð-ÅÑFÔFð 	Oàœ+×5Ò5Ñ7Ô7Ø!.Ø"0Ø#2Ø ,ðð ˆKð œ+×5Ò5Ñ7Ô7ˆKØ#Ô?À8ÒKˆIàð 	OÐ.Ð:Ý%DÐEVÐ_lÔ_sÐ%tÑ%tÔ%tÐ"Ý&Cð 'ð 'Ø'9ð'à!ð'ð 'Ð#Ð#õ '@Ð&NÐ&NÀ+Ð&NÐ&NÐ#à%�$Ô%ð 	
Ø-Ø.Ø%Ø+Ø'ØØð	
ð 	
ð ð	
ð 	
ˆõ )Ø%Ô7Ø#Ô3Ø!Ô/ØÔ)Ø2>Ð2J  ÐPTØ2@Ð2L  ÐRVØ$Ô5ð
ñ 
ô 
ð 	
r_   zPProjects the last hidden state from the audio encoder into language model space.c                 ó”   — | j         €t          d¦  «        ‚ | j         ||fddi|¤Ž}|                      |j        ¬¦  «        |_        |S )a0  
        input_features (`torch.FloatTensor]` of shape `(num_images, seq_length, num_features)`):
            The tensors corresponding to the input audio.
        input_features_mask (`torch.FloatTensor]` of shape `(num_images, seq_length)`):
            The attention mask for the input audio.
        Nz•Audio features were requested, but the model was initialized without an audio_config. Cannot process audio without an audio tower and audio embedder.r  Tr  )rú  r{  rû  rí  r  )r‰   r   r!  rT  Úaudio_outputss        r]   r$  zGemma4Model.get_audio_featureso  sm   € ð ÔÐ#ÝðRñô ð ð
 )˜Ô(¨Ð9LÐiÐiÐZ^ÐiÐbhÐiÐiˆØ&*×&6Ò&6À]ÔEdÐ&6Ñ&eÔ&eˆÔ#àÐr_   r�   r  )NNNNNNNNNNNNNN)rd   re   rf   rA   r‚   r   r  r!   r    rk   rá  rÖ  r   r   r   r
  r  rj   rt   r  r&   rl   r   r—  ra   r�   rs   r$  r‘   r’   s   @r]   ró  ró  P  s¼  ø€ € € € € ð
˜|ð 
ð 
ð 
ð 
ð 
ð 
ð:ð :ð :ð;ð ;ð ;ð Ø€^Ð!rÐsÑsÔsð 7;ðð àÔ'ðð "Ô,¨tÑ3ðð Ð+Ô,ð	ð
 
$ðð ð ñ tÔsñ Ôðð& Ø€^Ð!tÐuÑuÔuð 7;ðð à"Ô.ðð "Ô,¨tÑ3ðð Ð+Ô,ð	ð
 
$ðð ð ñ vÔuñ Ôðð0 .2Ø26ð+Jð +JàÔ# dÑ*ð+Jð Ô(¨4Ñ/ð+Jð 
ˆuÔ Ô!1°5Ô3CÐCÔ	Dð	+Jð +Jð +Jð +JðZ  ØØð .2Ø15Ø8<Ø37Ø.2Ø37Ø04Ø(,Ø59Ø26Ø!%Ø6:Ø6:Ø04ðd
ð d
àÔ# dÑ*ðd
ð Ô'¨$Ñ.ðd
ð #Ô.°Ñ5ð	d
ð
 Ô)¨DÑ0ðd
ð œ tÑ+ðd
ð #œ\¨DÑ0ðd
ð Ô&¨Ñ-ðd
ð  ™ðd
ð !Ô+¨dÑ2ðd
ð Ô(¨4Ñ/ðd
ð ˜$‘;ðd
ð "Ô,¨tÑ3ðd
ð "Ô,¨tÑ3ðd
ð  œ,¨Ñ-ðd
ð  Ð+Ô,ð!d
ð" 
#ð#d
ð d
ð d
ñ „^ñ Ôñ  Ôðd
ðL Ø€^Ð!sÐtÑtÔtðàœðð #œ\ðð Ð+Ô,ð	ð
 
Ð'Ñ	'ðð ð ñ uÔtñ Ôðð ð ð ð r_   ró  z…
    The base Gemma 4 model comprising a vision backbone, an audio backbone, a language model, and a language modeling
    head.
    c            %       ó†  ‡ — e Zd ZdZd„ Zd„ Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d dej        dz  dej        dz  dej        dz  d	ej        dz  d
ej	        dz  dej	        dz  dej        dz  dej        dz  dej        dz  de
dz  dej        dz  dej        dz  dej        dz  dedz  deej	        z  dej	        dz  dee         def$d„Ze	 d!dej        dej        dz  dee         fd„¦   «         Ze	 	 d"dedej	        d
ej	        dz  de
dz  dej	        dz  dej	        dz  dedz  defd„¦   «         Z	 	 	 	 	 	 	 	 	 	 	 	 	 d#ˆ fd„	Zˆ xZS )$ÚGemma4ForConditionalGenerationrš  c                 ó4   — | j                              ¦   «         S r�   )rš  r   rÿ  s    r]   r   z=Gemma4ForConditionalGeneration.get_per_layer_input_embeddings’  s   € ØŒz×8Ò8Ñ:Ô:Ð:r_   c                 ó:   — | j                              |¦  «         d S r�   )rš  r  r  s     r]   r  z=Gemma4ForConditionalGeneration.set_per_layer_input_embeddings•  s   € ØŒ
×1Ò1°%Ñ8Ô8Ð8Ð8Ð8r_   Nr   r„  rk  r  r   rF   r!  rH   r  r  rG   rç  rE   r›  r�  rœ  rŠ  rT  rJ   c           
      ód  —  | j         di d|“d|“d|“d|“d|“d|“d|“d|
“d	|“d
|“d|“d|“d|“d|“d|	“dd“|¤Ž}|j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }| j                             ¦   «         j        x}�||z  }t          j
        |¦  «        }||z  }d}|�, | j        ||| j                             ¦   «         j        fi |¤Ž}t          |||j        |j        |j        |j        |j        |j        ¬¦  «        S )r  r„  rk  r  r   rF   r!  rH   rG   rç  rE   rŠ  r›  r�  r  r  r  TN)rž  rŸ  rG   r‹   rr  r  r  rb   rS   )rš  rí  r´  r�   r   r¡  rD   r   r¢  rk   rñ   r£  r¤  ro   rG   r‹   rr  r  r  rb   )r‰   r„  rk  r  r   rF   r!  rH   r  r  rG   rç  rE   r›  r�  rœ  rŠ  rT  r¥  r‹   r¦  rŸ  r¢  rž  s                           r]   r�   z&Gemma4ForConditionalGeneration.forward˜  sû  € ðH �$”*ð 
ð 
ð 
Ø�ið
à%˜ð
ð !4Ð 3ð
ð *˜>ð	
ð
 *˜>ð
ð !4Ð 3ð
ð &˜ð
ð ,˜Oð
ð 0Ð/ð
ð (˜-ð
ð .Ð-ð
ð �6ð
ð  �ið
ð  2Ð1ð
ð  2Ð1ð
ð  ˜Øð#
ð 
ˆð(  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆØ'+¤{×'BÒ'BÑ'DÔ'DÔ'\Ð\Ð#ÐiØÐ5Ñ5ˆFÝ”Z Ñ'Ô'ˆFØÐ5Ñ5ˆFàˆØÐØ%�4Ô% f¨f°d´k×6QÒ6QÑ6SÔ6SÔ6^ÐiÐiÐbhÐiÐiˆDå+ØØØ#Ô3Ø!Ô/ØÔ)Ø 'Ô ;Ø 'Ô ;Ø$Ô5ð	
ñ 	
ô 	
ð 		
r_   c                 ó*   —  | j         j        ||fi |¤ŽS )a-  
        image_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`, *optional*):
            2D patch position coordinates from the image processor, with `(-1, -1)` indicating padding.
            Passed through to the vision encoder for positional embedding computation.
        )rš  r
  )r‰   rk  r  rT  s       r]   r
  z1Gemma4ForConditionalGeneration.get_image_featuresè  s%   € ð -ˆtŒzÔ,¨\Ð;MÐXÐXÐQWÐXÐXÐXr_   FrD   Úis_first_iterationc                 óð   — |                       ¦   «         ||||dœ}|                       ¦   «         }	t          |	dd ¦  «        dk    }
|
r&|�$t          ||j        ¬¦  «        }t	          dd|i|¤ŽS t          di |¤ŽS )NrL   r  r  r°   rI   rS   )r   r§  rñ  r±   r^   r   )rD   rE   rF   rG   rH   rç  r=  rT  rU   rÎ  r4  rI   s               r]   r   z8Gemma4ForConditionalGeneration.create_masks_for_generateö  s¹   € ð ×,Ò,Ñ.Ô.Ø*Ø,Ø.Ø(ð
ð 
ˆð ×,Ò,Ñ.Ô.ˆÝ˜KÐ)FÈÑMÔMÐQYÒYˆ	àð 	Ð*Ð6Ý!@ÐARÐ[hÔ[oÐ!pÑ!pÔ!pÐÝ0ð ð Ø#5ðàðð ð õ
 )Ð7Ð7¨;Ð7Ð7Ð7r_   Tc                 óº   •—  t          ¦   «         j        |f|||||||
|dœ|¤Ž}|s|s||d<   ||d<   ||d<   |	|d<   nd |d<   |s|                     dd ¦  «        }|S )N)rG   rE   rF   rH   r�  rœ  rÙ  r=  rk  r  r   r!  rç  rŠ  )r�   Úprepare_inputs_for_generationr“  )r‰   r„  rG   rE   rH   rk  r  r   rF   r!  rÙ  r�  rœ  r›  r=  rT  Úmodel_inputsrX   rŠ   s                     €r]   r@  z<Gemma4ForConditionalGeneration.prepare_inputs_for_generation	  s¾   ø€ ð& =•u‘w”wÔ<Øð
à+Ø'Ø)Ø%ØØ)Ø)Ø1ð
ð 
ð ð
ð 
ˆð ð 	5 Yð 	5Ø+7ˆL˜Ñ(Ø2EˆLÐ.Ñ/Ø-;ˆLÐ)Ñ*Ø2EˆLÐ.Ñ/Ð/ð 15ˆLÐ,Ñ-ð "ð 	;Ø× Ò Ð!3°TÑ:Ô:ˆAàÐr_   )NNNNNNNNNNNNNNr   Nr�   )NF)NNNNNNNNNTNNF)rd   re   rf   r§  r   r  rk   rÖ  rá  rl   r   r—  r�   r   r   ro   r�   r    r
  rÁ  r   rh   r   r@  r‘   r’   s   @r]   r8  r8  ‰  s  ø€ € € € € ð  Ðð;ð ;ð ;ð9ð 9ð 9ð
 .2Ø15Ø8<Ø37Ø.2Ø37Ø04Ø6:Ø6:Ø(,Ø59Ø26Ø*.Ø!%Ø-.Ø04ð#N
ð N
àÔ# dÑ*ðN
ð Ô'¨$Ñ.ðN
ð #Ô.°Ñ5ð	N
ð
 Ô)¨DÑ0ðN
ð œ tÑ+ðN
ð #œ\¨DÑ0ðN
ð Ô&¨Ñ-ðN
ð "Ô,¨tÑ3ðN
ð "Ô,¨tÑ3ðN
ð  ™ðN
ð !Ô+¨dÑ2ðN
ð Ô(¨4Ñ/ðN
ð Ô  4Ñ'ðN
ð ˜$‘;ðN
ð  ˜eœlÑ*ð!N
ð"  œ,¨Ñ-ð#N
ð$ Ð+Ô,ð%N
ð& 
&ð'N
ð N
ð N
ð N
ð` ð 7;ðYð YàÔ'ðYð "Ô,¨tÑ3ðYð Ð+Ô,ð	Yð Yð Yñ „^ðYð ð 26Ø*/ð8ð 8Ø ð8à”|ð8ð œ tÑ+ð8ð  ™ð	8ð
 ”l TÑ)ð8ð !œ<¨$Ñ.ð8ð ! 4™Kð8ð 
ð8ð 8ð 8ñ „\ð8ðB ØØØØ ØØØ ØØØØØ ð.ð .ð .ð .ð .ð .ð .ð .ð .ð .r_   r8  )r©  r™  r8  ró  rN  rp  rc  )r*   )�r¤   Úcollectionsr   Úcollections.abcr   Údataclassesr   Ú	functoolsr   rk   r   Útorch.nnr   rÙ   Ú r
   rW  Úactivationsr   Úcache_utilsr   r   Úconfiguration_utilsr   Úmasking_utilsr   r   r   r   r   r   r   r   Úmodeling_flash_attention_utilsr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r    r!   r"   r#   r$   Úutils.genericr%   r&   r'   Úutils.output_capturingr(   r)   Úauto.modeling_autor+   Úgemma3.modeling_gemma3r,   r-   r.   r/   r0   r1   r2   Úgemma3n.modeling_gemma3nr3   r4   r5   r6   r7   r8   r9   r:   r;   Úllama.modeling_llamar<   Úmixtral.modeling_mixtralr=   Ú0moonshine_streaming.modeling_moonshine_streamingr>   Úconfiguration_gemma4r@   rA   rB   rC   Ú
get_loggerrd   Úloggerrl   rh   r^   ra   ro   rq   rs   r  rv   r”   r˜   r¿   r  r  r$  ÚConv1dr6  r=  rK  rW  rq  r‹  r�   r¡  r£  rÃ  rØ  rã  rð  r÷  r  r"  r)  r8  rL  rN  rp  r™  r©  rc  rÌ  ræ  r±   rñ  ró  r8  Ú__all__rS   r_   r]   ú<module>r_     s]  ðð €€€Ø  Ð  Ð  Ð  Ð  Ð  Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !Ø %Ð %Ð %Ð %Ð %Ð %à €€€Ø Ð Ð Ð Ð Ð Ø $Ð $Ð $Ð $Ð $Ð $à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3ð	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð CÐ BÐ BÐ BÐ BÐ BØ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ^Ð ]Ð ]Ð ]Ð ]Ð ]Ð ]Ð ]Ð ]Ð ]Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ *Ð *Ð *Ð *Ð *Ð *ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 8Ð 7Ð 7Ð 7Ð 7Ð 7Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø [Ð [Ð [Ð [Ð [Ð [Ø gÐ gÐ gÐ gÐ gÐ gÐ gÐ gÐ gÐ gÐ gÐ gð ÐÑÔð 	Øð 
ˆÔ	˜HÑ	%Ô	%€ð4Øð4à”<ð4ð ”L 4Ñ'ð4ð ˜T‘\ð	4ð
 ”, Ñ%ð4ð œð4ð 
ð4ð 4ð 4ð 4ðnQð Qð Qð Qð QÐ :ñ Qô Qð Qð*Qð Qð Qð Qð QÐ#@ñ Qô Qð Qð2 ð
Qð 
Qð 
Qð 
Qð 
QÐ$;ñ 
Qô 
Qñ „ð
Qð Ø
ð3ð 3ð 3ð 3ð 3Ð7ñ 3ô 3ñ „ñ „ð3ðð ð ð ð ˜BœIñ ô ð ð:	ð 	ð 	ð 	ð 	�Nñ 	ô 	ð 	ð7ð 7ð 7ð 7ð 7 r¤yñ 7ô 7ð 7ð>i)ð i)ð i)ð i)ð i)˜2œ9ñ i)ô i)ð i)ðX#ð #ð #ð #ð #¨b¬iñ #ô #ð #ð8;ð ;ð ;ð ;ð ;¨¬ñ ;ô ;ð ;ð< ð  ð  ð  ð  ˜RœYñ  ô  ð  ðH"ð "ð "ð "ð "˜bœiñ "ô "ð "ðB&ð &ð &ð &ð &˜RœYñ &ô &ð &ðR0ð 0ð 0ð 0ð 0�r”yñ 0ô 0ð 0ðl*3ð *3ð *3ð *3ð *3 ¤	ñ *3ô *3ð *3ðZ@0ð @0ð @0ð @0ð @0˜œñ @0ô @0ð @0ðFað að að að a�iñ aô að að ð5&ð 5&Ø„|ð5&à	Œð5&ð 
Œð5&ð ”,ð	5&ð
 ð5&ð „\ð5&ð 5&ð 5&ð 5&ðp:ð :ð :ð :ð :Ð"6ñ :ô :ð :ðz ð:)ð :)ð :)ð :)ð :)˜Oñ :)ô :)ñ Ôð:)ð|"ð "ð "ð "ð "Ð1ñ "ô "ð "ðJ)Hð )Hð )Hð )Hð )H˜"œ)ñ )Hô )Hð )Hð^^ð ^ð ^ð ^ð ^�Iñ ^ô ^ð ^ðUð Uð Uð Uð UÐ 5ñ Uô Uð UðDq)ð q)ð q)ð q)ð q)˜"œ)ñ q)ô q)ð q)ðh7ð 7ð 7ð 7ð 7˜ñ 7ô 7ð 7ð"@ð "@ð "@ð "@ð "@�r”yñ "@ô "@ð "@ðJOð Oð Oð Oð OÐ/ñ Oô Oð Oðd	ð 	ð 	ð 	ð 	Ð$Añ 	ô 	ð 	ð=)ð =)ð =)ð =)ð =)Ð2ñ =)ô =)ð =)ð@ €Ð`ÐaÑaÔaðW
ð W
ð W
ð W
ð W
�oñ W
ô W
ñ bÔaðW
ðt €Ð]Ð^Ñ^Ô^ðJ
ð J
ð J
ð J
ð J
Ð)ñ J
ô J
ñ _Ô^ðJ
ðZScð Scð Scð Scð ScÐ,ñ Scô Scð ScðlAHð AHð AHð AHð AHÐ-ñ AHô AHð AHðH6ð 6ð 6ð 6ð 6Ð8ñ 6ô 6ð 6ð>Ø”L 4Ñ'ðà”\ DÑ(ðð ��_ðð ð ð ð:	°u´|ð 	ÈUÌ\ð 	Ð^cÔ^jð 	ð 	ð 	ð 	ð €ððñ ô ðpð pð pð pð p�,ñ pô pñô ðpðf	 €ððñ ô ðtð tð tð tð tÐ%Dñ tô tñô ðtðnð ð €€€r_   