§
    ‚Štj"ë ã                   óî	  — d dl Z d dlmZ d dlmZ d dlmZ d dlmZ d dl	m
Z
 d dlZd dlmZ d dlmZ d	d
lmZ d	dlmZ d	dlmZmZ d	dlmZ d	dlmZ d	dlmZ d	dlmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& d	dl'm(Z( d	dl)m*Z* d	dl+m,Z,m-Z- d	dl.m/Z/m0Z0 d	dl1m2Z2m3Z3 d	dl4m5Z5 d	dl6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z< d	dl=m>Z>m?Z? d	dl@mAZAmBZB ddlCmDZD ddlEmFZFmGZGmHZHmIZI  e;¦   «         rd dlJmKZK  e9d¬ ¦  «        e G d!„ d"e,¦  «        ¦   «         ¦   «         ZL e9d#¬ ¦  «        e G d$„ d%e7¦  «        ¦   «         ¦   «         ZMe G d&„ d'e,¦  «        ¦   «         ZNe9e G d(„ d)e-¦  «        ¦   «         ¦   «         ZO G d*„ d+ejP        ¦  «        ZQ G d,„ d-ejP        ¦  «        ZR G d.„ d/ejP        ¦  «        ZS G d0„ d1ejP        ¦  «        ZT G d2„ d3ejP        ¦  «        ZU G d4„ d5ejP        ¦  «        ZV G d6„ d7ejP        ¦  «        ZW G d8„ d9ejX        ¦  «        ZY G d:„ d;ejP        ¦  «        ZZ G d<„ d=ejP        ¦  «        Z[ G d>„ d?ejP        ¦  «        Z\ G d@„ dAejP        ¦  «        Z] G dB„ dCejP        ¦  «        Z^ G dD„ dEejP        ¦  «        Z_dF„ Z`d�dGeja        dHeja        dIeja        dJebfdK„ZcdLeja        dMebdNeja        fdO„Zd	 	 	 d�dQejP        dReja        dSeja        dTeja        dUeja        dz  dVeeebz  dWeedz  dXeedz  dNefeja        eja        f         fdY„Zg	 d‘dGeja        dHeja        dIeja        dZeja        dJebdNeja        fd[„Zh G d\„ d]ejP        ¦  «        Zi G d^„ d_e*¦  «        Zj G d`„ daejP        ¦  «        Zk G db„ dcejP        ¦  «        Zl G dd„ deejP        ¦  «        Zm G df„ dgejP        ¦  «        Zne G dh„ diejP        ¦  «        ¦   «         Zo G dj„ dkejP        ¦  «        Zp G dl„ dme*¦  «        Zq G dn„ doejr        ¦  «        Zse9 G dp„ dqe3¦  «        ¦   «         Zt e9dr¬ ¦  «         G ds„ dtet¦  «        ¦   «         Zu e9du¬ ¦  «         G dv„ dwete¦  «        ¦   «         Zvdxefebebf         dNefdy„Zw G dz„ d{et¦  «        Zx G d|„ d}et¦  «        Zy G d~„ dejP        ¦  «        Zzd€ed�eja        dUeja        dz  d‚edz  dZeja        dz  dƒeja        dNe{fd„„Z|d…eja        d†ej}        dNeja        fd‡„Z~ e9dˆ¬ ¦  «         G d‰„ dŠet¦  «        ¦   «         Z e9d‹¬ ¦  «         G dŒ„ d�ete¦  «        ¦   «         Z€g dŽ¢Z�dS )’é    N)ÚUserDict)ÚCallable)Ú	dataclass)Úcached_property)ÚOptional)Únn)Ú
functionalé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚPreTrainedConfig)ÚGenerationMixin)Úuse_experts_implementation)Ú_preprocess_mask_argumentsÚblockwise_overlayÚcreate_bidirectional_maskÚcreate_causal_maskÚcreate_masks_for_generateÚ!create_sliding_window_causal_maskÚmaybe_pad_block_sequence_idsÚsliding_window_overlay)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPooling)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_accelerate_availableÚtorch_compilable_check)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Ú	AutoModelé   )ÚGemma4AudioConfigÚGemma4ConfigÚGemma4TextConfigÚGemma4VisionConfig)Úadd_hook_to_modulezK
    Base class for Gemma4 outputs, with hidden states and attentions.
    ©Úcustom_introc                   ó˜   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
eeej        ej        f         f         dz  ed<   dS )ÚGemma4ModelOutputWithPasta¯  
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    audio_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        audio_hidden_states of the model produced by the audio encoder and after projecting the last hidden state.
    shared_kv_states (`dict`, *optional*):
        Dictionary mapping layer type strings to tuples of (key_states, value_states) tensors.
        Used to pass shared KV states between layers during KV sharing.
    NÚimage_hidden_statesÚaudio_hidden_statesÚshared_kv_states)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r9   ÚtorchÚFloatTensorÚ__annotations__r:   r;   ÚdictÚstrÚtupleÚTensor© ó    úh/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/gemma4/modeling_gemma4.pyr8   r8   H   s~   € € € € € € ðð ð" 59Ð˜Ô*¨TÑ1Ð8Ð8Ñ8à48Ð˜Ô*¨TÑ1Ð8Ð8Ñ8àLPÐ�d˜3  e¤l°E´LÐ&@Ô AÐAÔBÀTÑIÐPÐPÑPÐPÐPrH   r8   zR
    Base class for Gemma4 causal language model (or autoregressive) outputs.
    c                   ó<  — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZej        dz  ed<   dZej        dz  ed	<   dZeeeej        ej        f         f         dz  ed
<   dS )ÚGemma4CausalLMOutputWithPasta  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.text_config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder after projecting last hidden state.
    audio_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        audio_hidden_states of the model produced by the audio encoder and after projecting the last hidden state.
    shared_kv_states (`dict`, *optional*):
        Dictionary mapping layer type strings to tuples of (key_states, value_states) tensors.
        Used to pass shared KV states between layers during KV sharing.
    NÚlossÚlogitsÚpast_key_valuesÚhidden_statesÚ
attentionsr9   r:   r;   )r<   r=   r>   r?   rL   r@   rA   rB   rM   rN   r   rO   rE   rP   r9   r:   r;   rC   rD   rF   rG   rH   rI   rK   rK   g   sü   € € € € € € ðð ð* &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø48Ð˜Ô*¨TÑ1Ð8Ð8Ñ8à48Ð˜Ô*¨TÑ1Ð8Ð8Ñ8àLPÐ�d˜3  e¤l°E´LÐ&@Ô AÐAÔBÀTÑIÐPÐPÑPÐPÐPrH   rK   c                   ó\   — e Zd ZU dZdZeeeej	        ej	        f         f         dz  e
d<   dS )ÚGemma4TextModelOutputWithPasta9  
    BaseModelOutputWithPast extended with shared_kv_states for KV sharing.

    Args:
        shared_kv_states (`dict`, *optional*):
            Dictionary mapping layer type strings to tuples of (key_states, value_states) tensors.
            Used to pass shared KV states between layers during KV sharing.
    Nr;   )r<   r=   r>   r?   r;   rC   rD   rE   r@   rF   rB   rG   rH   rI   rR   rR   �   sN   € € € € € € ðð ð MQÐ�d˜3  e¤l°E´LÐ&@Ô AÐAÔBÀTÑIÐPÐPÑPÐPÐPrH   rR   c                   ó2   — e Zd ZU dZdZej        dz  ed<   dS )ÚGemma4AudioModelOutputz§
    attention_mask (`torch.BoolTensor`, *optional*):
        A torch.BoolTensor of shape `(batch_size, num_frames)`. True for valid positions, False for padding.
    NÚattention_mask)r<   r=   r>   r?   rU   r@   Ú
BoolTensorrB   rG   rH   rI   rT   rT   �   s6   € € € € € € ðð ð
 /3€N�EÔ$ tÑ+Ð2Ð2Ñ2Ð2Ð2rH   rT   c                   óZ   ‡ — e Zd Zdeez  dededdfˆ fd„Zdej        dej        fd„Z	ˆ xZ
S )	ÚGemma4ClippableLinearÚconfigÚin_featuresÚout_featuresÚreturnNc                 óV  •— t          ¦   «                              ¦   «          |j        | _        t          j        ||d¬¦  «        | _        | j        rØ|                      dt          j        t          d¦  «         ¦  «        ¦  «         |                      dt          j        t          d¦  «        ¦  «        ¦  «         |                      dt          j        t          d¦  «         ¦  «        ¦  «         |                      dt          j        t          d¦  «        ¦  «        ¦  «         d S d S )NF©ÚbiasÚ	input_minÚinfÚ	input_maxÚ
output_minÚ
output_max)
ÚsuperÚ__init__Úuse_clipped_linearsr   ÚLinearÚlinearÚregister_bufferr@   ÚtensorÚfloat)ÚselfrY   rZ   r[   Ú	__class__s       €rI   rf   zGemma4ClippableLinear.__init__©   sø   ø€ õ 	‰Œ×ÒÑÔÐØ#)Ô#=ˆÔ Ý”i ¨\ÀÐFÑFÔFˆŒàÔ#ð 	KØ× Ò  ­e¬l½EÀ%¹L¼L¸=Ñ.IÔ.IÑJÔJÐJØ× Ò  ­e¬l½5À¹<¼<Ñ.HÔ.HÑIÔIÐIØ× Ò  ­u¬|½UÀ5¹\¼\¸MÑ/JÔ/JÑKÔKÐKØ× Ò  ­u¬|½EÀ%¹L¼LÑ/IÔ/IÑJÔJÐJÐJÐJð		Kð 	KrH   rO   c                 óÌ   — | j         r t          j        || j        | j        ¦  «        }|                      |¦  «        }| j         r t          j        || j        | j        ¦  «        }|S ©N)rg   r@   Úclampr`   rb   ri   rc   rd   )rm   rO   s     rI   ÚforwardzGemma4ClippableLinear.forward¹   s_   € ØÔ#ð 	WÝ!œK¨°t´~ÀtÄ~ÑVÔVˆMàŸš MÑ2Ô2ˆàÔ#ð 	YÝ!œK¨°t´ÈÌÑXÔXˆMàÐrH   )r<   r=   r>   r3   r0   Úintrf   r@   rF   rr   Ú__classcell__©rn   s   @rI   rX   rX   ¨   s™   ø€ € € € € ðKà"Ð%6Ñ6ðKð ðKð ð	Kð
 
ðKð Kð Kð Kð Kð Kð 	 U¤\ð 	°e´lð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	rH   rX   c                   óh   ‡ — e Zd Zddededefˆ fd„Zdej        fd„Z	dej        d	ej        fd
„Z
ˆ xZS )ÚGemma4RMSNormç�íµ ÷Æ°>TÚdimÚepsÚ
with_scalec                 óÐ   •— t          ¦   «                              ¦   «          || _        || _        | j        r/t	          j        t          j        |¦  «        d¬¦  «        | _        d S d S )NT)Úrequires_grad)	re   rf   rz   r{   r   Ú	Parameterr@   ÚonesÚweight)rm   ry   rz   r{   rn   s       €rI   rf   zGemma4RMSNorm.__init__Æ   s`   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ$ˆŒàŒ?ð 	LÝœ,¥u¤z°#¡¤ÀdÐKÑKÔKˆDŒKˆKˆKð	Lð 	LrH   rO   c                 ó–   — |                      d¦  «                             dd¬¦  «        | j        z   }|t          j         |d¦  «        z  S )Nr-   éÿÿÿÿT)Úkeepdimç      à¿)ÚpowÚmeanrz   r@   )rm   rO   Úmean_squareds      rI   Ú_normzGemma4RMSNorm._normÎ   sF   € Ø$×(Ò(¨Ñ+Ô+×0Ò0°¸TÐ0ÑBÔBÀTÄXÑMˆà�uœy¨°tÑ<Ô<Ñ<Ð<rH   r\   c                 óÀ   — |                       |                     ¦   «         ¦  «        }| j        r|| j                             ¦   «         z  }|                     |¦  «        S rp   )rˆ   rl   r{   r€   Útype_as)rm   rO   Únormed_outputs      rI   rr   zGemma4RMSNorm.forwardÓ   sV   € ØŸ
š
 =×#6Ò#6Ñ#8Ô#8Ñ9Ô9ˆØŒ?ð 	@Ø)¨D¬K×,=Ò,=Ñ,?Ô,?Ñ?ˆMØ×$Ò$ ]Ñ3Ô3Ð3rH   )rx   T)r<   r=   r>   rs   rl   Úboolrf   r@   rF   rˆ   rr   rt   ru   s   @rI   rw   rw   Å   s¤   ø€ € € € € ðLð L˜Cð L eð LÀð Lð Lð Lð Lð Lð Lð= 5¤<ð =ð =ð =ð =ð
4 U¤\ð 4°e´lð 4ð 4ð 4ð 4ð 4ð 4ð 4ð 4rH   rw   c                   óŒ   ‡ — e Zd ZU dZej        ed<   defˆ fd„Z ej	        ¦   «         dej        dej        fd„¦   «         Z
ˆ xZS )Ú Gemma4AudioRelPositionalEncodingzíSinusoidal relative positional encoding for the audio encoder.

    Produces position embeddings of shape [1, context_size // 2 + 1, hidden_size] with
    concatenated [sin..., cos...] layout matching the original Gemma4 convention.
    Úinv_timescalesrY   c                 óæ  •— t          ¦   «                              ¦   «          |j        | _        |j        |j        z   dz
  |j        z   | _        d}d}| j        dz  }t          j        ||z  ¦  «        t          |dz
  d¦  «        z  }|t          j        t          j        |¦  «        | z  ¦  «        z  }|                      d|                     d¦  «                             d¦  «        d¬¦  «         d S )	Nr/   ç      ð?ç     ˆÃ@r-   r�   r   F©Ú
persistent)re   rf   Úhidden_sizeÚattention_chunk_sizeÚattention_context_leftÚattention_context_rightÚcontext_sizeÚmathÚlogÚmaxr@   ÚexpÚarangerj   Ú	unsqueeze)rm   rY   Úmin_timescaleÚmax_timescaleÚnum_timescalesÚlog_timescale_incrementr�   rn   s          €rI   rf   z)Gemma4AudioRelPositionalEncoding.__init__ã   só   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔàÔ'¨&Ô*GÑGÈ!ÑKÈfÔNlÑlð 	Ôð ˆØˆØÔ)¨QÑ.ˆÝ"&¤(¨=¸=Ñ+HÑ"IÔ"IÍCÐP^ÐabÑPbÐdeÑLfÔLfÑ"fÐØ&­¬µ5´<ÀÑ3OÔ3OÐSjÐRjÑ3jÑ)kÔ)kÑkˆØ×ÒÐ-¨~×/GÒ/GÈÑ/JÔ/J×/TÒ/TÐUVÑ/WÔ/WÐdiÐÑjÔjÐjÐjÐjrH   rO   r\   c                 óP  — t          j        | j        dz  dd|j        ¬¦  «        }|d         }|| j                             |j        ¬¦  «        z  }t          j        t          j        |¦  «        t          j        |¦  «        gd¬¦  «        }|                     |j	        ¬¦  «        S )Nr-   r‚   ©Údevice©.N©ry   ©Údtype)
r@   rž   r™   r¦   r�   ÚtoÚcatÚsinÚcosrª   )rm   rO   Úposition_idsÚscaled_timeÚ	pos_embeds        rI   rr   z(Gemma4AudioRelPositionalEncoding.forwardð   s•   € å”| DÔ$5¸Ñ$:¸BÀÈ=ÔK_Ð`Ñ`Ô`ˆØ# IÔ.ˆØ" TÔ%8×%;Ò%;À=ÔCWÐ%;Ñ%XÔ%XÑXˆÝ”I�uœy¨Ñ5Ô5µu´yÀÑ7MÔ7MÐNÐTVÐWÑWÔWˆ	Ø�|Š| -Ô"5ˆ|Ñ6Ô6Ð6rH   )r<   r=   r>   r?   r@   rF   rB   r0   rf   Úno_gradrr   rt   ru   s   @rI   rŽ   rŽ   Ú   s¢   ø€ € € € € € ðð ð ”LÐ Ð Ñ ðkÐ0ð kð kð kð kð kð kð €U„]�_„_ð7 U¤\ð 7°e´lð 7ð 7ð 7ñ „_ð7ð 7ð 7ð 7ð 7rH   rŽ   c                   óò   ‡ — e Zd ZdZdedefˆ fd„Zdej        dej        fd„Z	dej        dej        fd„Z
d	ej        dej        fd
„Z	 ddej        dej        dej        dz  deej        df         fd„Zˆ xZS )ÚGemma4AudioAttentionz3Chunked local attention with relative position biasrY   Ú	layer_idxc                 óJ  •— t          ¦   «                              ¦   «          || _        || _        |j        | _        |j        |j        z  | _        |j        | _	        | j        dz  t          j        d¦  «        z  | _        t          j        dt          j        z   ¦  «        t          j        d¦  «        z  | _        |j        | _        |j        dz
  | _        |j        | _        | j        | j        z   | j        z   | _        t-          ||j        | j	        | j        z  ¦  «        | _        t-          ||j        | j	        | j        z  ¦  «        | _        t-          ||j        | j	        | j        z  ¦  «        | _        t-          ||j        |j        ¦  «        | _        t7          j        |j        | j	        | j        z  d¬¦  «        | _        t7          j        t?          j         | j        ¦  «        ¦  «        | _!        |  "                    dt?          j#        | j        ¦  «        d¬¦  «         d S )Nr„   r-   r/   Fr^   Úsoftcapr“   )$re   rf   rY   rµ   Úattention_logit_capÚattention_logits_soft_capr•   Únum_attention_headsÚhead_dimÚ	num_headsrš   r›   Úq_scaleÚeÚk_scaler–   Ú
chunk_sizer—   Úmax_past_horizonr˜   Úmax_future_horizonr™   rX   Úq_projÚk_projÚv_projÚpostr   rh   Úrelative_k_projr~   r@   ÚzerosÚper_dim_scalerj   rk   ©rm   rY   rµ   rn   s      €rI   rf   zGemma4AudioAttention.__init__ü   s»  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒØ)/Ô)CˆÔ&ØÔ*¨fÔ.HÑHˆŒØÔ3ˆŒàœ tÑ+­t¬x¸©{¬{Ñ:ˆŒÝ”x ¥D¤F¡
Ñ+Ô+­d¬h°q©k¬kÑ9ˆŒà Ô5ˆŒØ &Ô =ÀÑ AˆÔØ"(Ô"@ˆÔØ œO¨dÔ.CÑCÀdÔF]Ñ]ˆÔå+¨F°FÔ4FÈÌÐY]ÔYfÑHfÑgÔgˆŒÝ+¨F°FÔ4FÈÌÐY]ÔYfÑHfÑgÔgˆŒÝ+¨F°FÔ4FÈÌÐY]ÔYfÑHfÑgÔgˆŒÝ)¨&°&Ô2DÀfÔFXÑYÔYˆŒ	å!œy¨Ô);¸T¼^ÈdÌmÑ=[ÐbgÐhÑhÔhˆÔÝœ\­%¬+°d´mÑ*DÔ*DÑEÔEˆÔà×Ò˜Y­¬°TÔ5SÑ(TÔ(TÐafÐÑgÔgÐgÐgÐgrH   rO   r\   c           	      óô   — |j         \  }}}}|| j        z   dz
  | j        z  }|| j        z  |z
  }t          j        |ddddd|f¦  «        }|                     ||| j        ||¦  «                             ¦   «         S )z€Splits a `(batch_size, seq_len, num_heads, head_dim)` tensor into non-overlapping blocks of `chunk_size` along the sequence dim.r/   r   )ÚshaperÀ   ÚFÚpadÚreshapeÚ
contiguous)rm   rO   Ú
batch_sizeÚseq_lenr¼   r»   Ú
num_blocksrÎ   s           rI   Ú_convert_to_blockz&Gemma4AudioAttention._convert_to_block  s…   € à3@Ô3FÑ0ˆ
�G˜Y¨Ø ¤Ñ/°!Ñ3¸¼ÑGˆ
Ø˜4œ?Ñ*¨WÑ4ˆÝœ˜m¨a°°A°q¸!¸SÐ-AÑBÔBˆØ×$Ò$ Z°¸T¼_ÈiÐYaÑbÔb×mÒmÑoÔoÐorH   c           
      ó  — |j         \  }}}}t          j        |dddd| j        | j        | j        z   dz
  f¦  «        }|                     d| j        | j        ¦  «        }t          j	        |dd¦  «        }| 
                    ¦   «         S )z`Extracts overlapping context windows of `context_size` for every block, strided by `chunk_size`.r   r/   r‚   r-   )rÌ   rÍ   rÎ   rÁ   rÂ   rÀ   Úunfoldr™   r@   ÚmovedimrÐ   )rm   rO   rÑ   rÒ   r¼   r»   s         rI   Ú_extract_block_contextz+Gemma4AudioAttention._extract_block_context  sŽ   € à3@Ô3FÑ0ˆ
�G˜Y¨ÝœØ˜A˜q ! Q¨Ô(=¸tÔ?VÐY]ÔYhÑ?hÐklÑ?lÐmñ
ô 
ˆð &×,Ò,¨Q°Ô0AÀ4Ä?ÑSÔSˆÝœ m°R¸Ñ;Ô;ˆØ×'Ò'Ñ)Ô)Ð)rH   Úxc                 óð   — |j         \  }}}}}| j        }t          j        |d|dz   |z
  f¦  «        }|                     |||||dz   z  ¦  «        }|dd||z  …f         }|                     |||||¦  «        S )zjRelative position shift for blocked attention. See appendix B of https://huggingface.co/papers/1901.02860.r   r/   .N)rÌ   r™   rÍ   rÎ   Úview)rm   rÙ   rÑ   r¼   rÓ   Ú
block_sizeÚposition_lengthr™   s           rI   Ú
_rel_shiftzGemma4AudioAttention._rel_shift(  s’   € àIJÌÑFˆ
�I˜z¨:°ØÔ(ˆÝŒE�!�a˜¨Ñ)¨OÑ;Ð<Ñ=Ô=ˆØ�FŠF�:˜y¨*°jÀLÐSTÑDTÑ6UÑVÔVˆØˆcÐ.�Z ,Ñ.Ð.Ð.Ô/ˆØ�vŠv�j )¨Z¸À\ÑRÔRÐRrH   NÚposition_embeddingsrU   c                 óö  — |j         \  }}}||| j        | j        f}|                      |¦  «                             ¦   «                              |¦  «        }|                      |¦  «                             ¦   «                              |¦  «        }	|                      |¦  «                             ¦   «                              |¦  «        }
|| j        z  t          j
        | j        ¦  «        z  }|	| j        z  }	|                      |¦  «        }|                      |	¦  «        }	|                      |
¦  «        }
|j         d         }|                      |¦  «        }|                     d| j        | j        ¦  «        }|                     |j        ¬¦  «        }|                     ddddd¦  «        }||	                     ddddd¦  «        z  }|                     || j        d| j        ¦  «        }||                     ddd¦  «        z  }|                     || j        || j        d¦  «        }|                      |¦  «        }||z   }|| j        z  }t/          j        |¦  «        }|| j        z  }|�2|                     |                     ¦   «         | j        j        ¦  «        }t          j        |dt.          j        ¬¦  «                             |
j        ¦  «        }||
                     ddddd¦  «        z  }|                     ddddd¦  «                             ||| j        z  d¦  «        }|d d …d |…f                              ¦   «         }|                       |                     |j        ¦  «        ¦  «        }||fS )	Nr/   r‚   r©   r   r
   r-   é   ©ry   rª   )!rÌ   r¼   r»   rÃ   rl   rÛ   rÄ   rÅ   r½   rÍ   ÚsoftplusrÉ   r¿   rÔ   rØ   rÇ   r«   rª   ÚpermuterÏ   rÀ   rÞ   r·   r@   ÚtanhÚmasked_fillÚlogical_notrY   Úattention_invalid_logits_valueÚsoftmaxÚfloat32rÐ   rÆ   )rm   rO   rß   rU   rÑ   Ú
seq_lengthÚ_Úhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesrÓ   Úrelative_key_statesÚqueriesÚ	matrix_acÚqueries_flatÚ	matrix_bdÚattn_weightsÚattn_outputs                      rI   rr   zGemma4AudioAttention.forward1  s6  € ð %2Ô$7Ñ!ˆ
�J Ø" J°´ÀÄÐNˆà—{’{ =Ñ1Ô1×7Ò7Ñ9Ô9×>Ò>¸|ÑLÔLˆØ—[’[ Ñ/Ô/×5Ò5Ñ7Ô7×<Ò<¸\ÑJÔJˆ
Ø—{’{ =Ñ1Ô1×7Ò7Ñ9Ô9×>Ò>¸|ÑLÔLˆà# d¤lÑ2µQ´ZÀÔ@RÑ5SÔ5SÑSˆØ $¤,Ñ.ˆ
à×-Ò-¨lÑ;Ô;ˆØ×0Ò0°Ñ<Ô<ˆ
Ø×2Ò2°<Ñ@Ô@ˆØ!Ô'¨Ô*ˆ
à"×2Ò2Ð3FÑGÔGÐØ1×6Ò6°r¸4¼>È4Ì=ÑYÔYÐØ1×4Ò4¸<Ô;MÐ4ÑNÔNÐà×&Ò& q¨!¨Q°°1Ñ5Ô5ˆØ˜j×0Ò0°°A°q¸!¸QÑ?Ô?Ñ?ˆ	à—’ z°4´>À2ÀtÄ}ÑUÔUˆØ Ð#6×#>Ò#>¸qÀ!ÀQÑ#GÔ#GÑGˆ	Ø×%Ò% j°$´.À*ÈdÌoÐ_aÑbÔbˆ	Ø—O’O IÑ.Ô.ˆ	à  9Ñ,ˆØ# d¤lÑ2ˆÝ”z ,Ñ/Ô/ˆØ# d¤lÑ2ˆàÐ%Ø'×3Ò3Ø×*Ò*Ñ,Ô,¨d¬kÔ.Xñô ˆLõ ”y °2½U¼]ÐKÑKÔK×NÒNÈ|ÔOaÑbÔbˆØ" \×%9Ò%9¸!¸QÀÀ1ÀaÑ%HÔ%HÑHˆØ!×)Ò)¨!¨Q°°1°aÑ8Ô8×@Ò@ÀÈZÐZ^ÔZiÑMiÐkmÑnÔnˆØ! ! ! ! [ j [ .Ô1×<Ò<Ñ>Ô>ˆØ—i’i §¢¨}Ô/BÑ CÔ CÑDÔDˆà˜LÐ(Ð(rH   rp   )r<   r=   r>   r?   r0   rs   rf   r@   rF   rÔ   rØ   rÞ   rV   rE   rr   rt   ru   s   @rI   r´   r´   ù   s4  ø€ € € € € Ø=Ð=ðhÐ0ð h¸Sð hð hð hð hð hð hð4p¨u¬|ð pÀÄð pð pð pð pð*°E´Lð *ÀUÄ\ð *ð *ð *ð *ðS˜EœLð S¨U¬\ð Sð Sð Sð Sð 37ð	1)ð 1)à”|ð1)ð #œ\ð1)ð Ô(¨4Ñ/ð	1)ð
 
ˆuŒ|˜TÐ!Ô	"ð1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)rH   r´   c                   óJ   ‡ — e Zd Zˆ fd„Zddej        dej        dz  fd„Zˆ xZS )Ú'Gemma4AudioSubSampleConvProjectionLayerc                 óð   •— t          ¦   «                              ¦   «          t          j        ||dddd¬¦  «        | _        t          j        ||dd¬¦  «        | _        t          j        ¦   «         | _        d S )N)r
   r
   )r-   r-   r/   F)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingr_   T)rz   Úelementwise_affiner_   )	re   rf   r   ÚConv2dÚconvÚ	LayerNormÚnormÚReLUÚact)rm   rû   rü   Únorm_epsrn   s       €rI   rf   z0Gemma4AudioSubSampleConvProjectionLayer.__init__f  sr   ø€ Ý‰Œ×ÒÑÔÐÝ”IØ#Ø%ØØØØð
ñ 
ô 
ˆŒ	õ ”L °8ÐPTÐ[`ÐaÑaÔaˆŒ	Ý”7‘9”9ˆŒˆˆrH   NrO   Úmaskc           
      óÆ  — |�.|                      |j        ¬¦  «        }||d d …d d d …d f         z  }|                      |                      | j        j        j        ¦  «        ¦  «        }|                      |                      |                     dddd¦  «        ¦  «                             dddd¦  «                             ¦   «         ¦  «        }|�|d d …d d d…f         }||fS )Nr¥   r   r-   r
   r/   )	r«   r¦   r  r€   rª   r  r  rä   rÐ   )rm   rO   r  s      rI   rr   z/Gemma4AudioSubSampleConvProjectionLayer.forwards  sâ   € ØÐØ—7’7 -Ô"6�7Ñ7Ô7ˆDØ)¨D°°°°D¸!¸!¸!¸TÐ1AÔ,BÑBˆMàŸ	š	 -×"2Ò"2°4´9Ô3CÔ3IÑ"JÔ"JÑKÔKˆØŸš §¢¨=×+@Ò+@ÀÀAÀqÈ!Ñ+LÔ+LÑ!MÔ!M×!UÒ!UÐVWÐYZÐ\]Ð_`Ñ!aÔ!a×!lÒ!lÑ!nÔ!nÑoÔoˆàÐØ˜˜˜˜3˜3˜Q˜3˜”<ˆDà˜dÐ"Ð"rH   rp   )r<   r=   r>   rf   r@   rF   rr   rt   ru   s   @rI   rù   rù   e  sh   ø€ € € € € ðð ð ð ð ð#ð # U¤\ð #¸¼ÈÑ9Lð #ð #ð #ð #ð #ð #ð #ð #rH   rù   c            	       óz   ‡ — e Zd Zdefˆ fd„Z	 ddej        dej        dz  deej        ej        f         fd„Zˆ xZ	S )	Ú"Gemma4AudioSubSampleConvProjectionrY   c                 óx  •— t          ¦   «                              ¦   «          t          d|j        d         |j        ¬¦  «        | _        t          |j        d         |j        d         |j        ¬¦  «        | _        |j        d         dz  |j        d         z  }t          j        ||j	        d¬¦  «        | _
        d S )Nr/   r   )rû   rü   r  rá   Fr^   )re   rf   rù   Úsubsampling_conv_channelsÚrms_norm_epsÚlayer0Úlayer1r   rh   r•   Úinput_proj_linear)rm   rY   Úproj_input_dimrn   s      €rI   rf   z+Gemma4AudioSubSampleConvProjection.__init__‚  s¼   ø€ Ý‰Œ×ÒÑÔÐÝ=ØØÔ9¸!Ô<ØÔ(ð
ñ 
ô 
ˆŒõ
 >ØÔ8¸Ô;ØÔ9¸!Ô<ØÔ(ð
ñ 
ô 
ˆŒð
 !Ô:¸1Ô=ÀÑBÀfÔFfÐghÔFiÑiˆÝ!#¤¨>¸6Ô;MÐTYÐ!ZÑ!ZÔ!ZˆÔÐÐrH   NÚinput_featuresÚinput_features_maskr\   c                 óT  — |                      d¦  «        }|                      ||¦  «        \  }}|                      ||¦  «        \  }}|j        \  }}}}|                     dddd¦  «                             ¦   «                              ||d¦  «        }|                      |¦  «        |fS )Nr/   r   r-   r
   r‚   )rŸ   r  r  rÌ   rä   rÐ   rÏ   r  )rm   r  r  rO   r  rÑ   rì   rÒ   s           rI   rr   z*Gemma4AudioSubSampleConvProjection.forward‘  s«   € ð
 '×0Ò0°Ñ3Ô3ˆØ"Ÿkšk¨-Ð9LÑMÔMÑˆ�tØ"Ÿkšk¨-¸Ñ>Ô>Ñˆ�tà$1Ô$7Ñ!ˆ
�A�w Ø%×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔF×NÒNÈzÐ[bÐdfÑgÔgˆØ×%Ò% mÑ4Ô4°dÐ:Ð:rH   rp   )
r<   r=   r>   r0   rf   r@   rF   rE   rr   rt   ru   s   @rI   r  r  �  s�   ø€ € € € € ð[Ð0ð [ð [ð [ð [ð [ð [ð$ 48ð;ð ;àœð;ð #œ\¨DÑ0ð;ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð	;ð ;ð ;ð ;ð ;ð ;ð ;ð ;rH   r  c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚGemma4AudioFeedForwardrY   c                 ó¤  •— t          ¦   «                              ¦   «          || _        t          ||j        |j        dz  ¦  «        | _        t          ||j        dz  |j        ¦  «        | _        t          |j        ¦  «        | _        t          |j        ¦  «        | _	        t          |j                 | _        |j        | _        |j        | _        d S )Nrá   )re   rf   rY   rX   r•   Úffw_layer_1Úffw_layer_2rw   Úpre_layer_normÚpost_layer_normr   Ú
hidden_actÚact_fnÚgradient_clippingÚresidual_weightÚpost_layer_scale©rm   rY   rn   s     €rI   rf   zGemma4AudioFeedForward.__init__   s±   ø€ Ý‰Œ×ÒÑÔÐØˆŒå0°¸Ô9KÈVÔM_ÐbcÑMcÑdÔdˆÔÝ0°¸Ô9KÈaÑ9OÐQWÔQcÑdÔdˆÔå+¨FÔ,>Ñ?Ô?ˆÔÝ,¨VÔ-?Ñ@Ô@ˆÔÝ˜VÔ.Ô/ˆŒà!'Ô!9ˆÔØ &Ô 6ˆÔÐÐrH   rO   r\   c                 ó¸  — t          | j        t          j        |j        ¦  «        j        ¦  «        }|}t          j        || |¦  «        }|                      |¦  «        }|                      |¦  «        }|  	                    |¦  «        }|  
                    |¦  «        }t          j        || |¦  «        }|                      |¦  «        }|| j        z  }||z  }|S rp   )Úminr  r@   Úfinforª   rœ   rq   r  r  r  r  r  r!  )rm   rO   r  Úresiduals       rI   rr   zGemma4AudioFeedForward.forward®  sÔ   € å Ô 6½¼ÀMÔDWÑ8XÔ8XÔ8\Ñ]Ô]Ðà ˆÝœ MÐ4EÐ3EÐGXÑYÔYˆØ×+Ò+¨MÑ:Ô:ˆà×(Ò(¨Ñ7Ô7ˆØŸš MÑ2Ô2ˆØ×(Ò(¨Ñ7Ô7ˆåœ MÐ4EÐ3EÐGXÑYÔYˆØ×,Ò,¨]Ñ;Ô;ˆØ˜Ô.Ñ.ˆØ˜Ñ!ˆàÐrH   ©	r<   r=   r>   r0   rf   r@   rF   rr   rt   ru   s   @rI   r  r  Ÿ  sk   ø€ € € € € ð7Ð0ð 7ð 7ð 7ð 7ð 7ð 7ð U¤\ð °e´lð ð ð ð ð ð ð ð rH   r  c                   óR   ‡ — e Zd Zed„ ¦   «         Zdej        dej        fˆ fd„Zˆ xZS )ÚGemma4AudioCausalConv1dc                 ód   — | j         d         dz
  | j        d         z  dz   }|| j        d         z
  S )Nr   r/   )rý   Údilationrþ   )rm   Úeffective_kernel_sizes     rI   Úleft_padz Gemma4AudioCausalConv1d.left_padÑ  s7   € à!%Ô!1°!Ô!4°qÑ!8¸D¼MÈ!Ô<LÑ LÈqÑ PÐØ$ t¤{°1¤~Ñ5Ð5rH   rÙ   r\   c                 ó”   •— t           j                             || j        df¦  «        }t	          ¦   «                              |¦  «        S ©Nr   )r   r	   rÎ   r-  re   rr   )rm   rÙ   rn   s     €rI   rr   zGemma4AudioCausalConv1d.forwardÖ  s9   ø€ õ ŒM×Ò˜a $¤-°Ð!3Ñ4Ô4ˆå‰wŒw�Š˜qÑ!Ô!Ð!rH   )	r<   r=   r>   r   r-  r@   rF   rr   rt   ru   s   @rI   r)  r)  Ã  sn   ø€ € € € € ð ð6ð 6ñ „_ð6ð"àŒ<ð"ð 
Œð	"ð "ð "ð "ð "ð "ð "ð "ð "ð "rH   r)  c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚGemma4AudioLightConv1drY   c                 ó   •— t          ¦   «                              ¦   «          || _        t          ||j        |j        dz  ¦  «        | _        t          ||j        |j        ¦  «        | _        t          |j        |j        |j        |j        d¬¦  «        | _	        t          |j        |j        d¬¦  «        | _        t          |j        |j        d¬¦  «        | _        t          |j                 | _        |j        | _        d S )Nr-   F)rû   rü   rý   Úgroupsr_   T©rz   r{   )re   rf   rY   rX   r•   Úlinear_startÚ
linear_endr)  Úconv_kernel_sizeÚdepthwise_conv1drw   r  r  Ú	conv_normr   r  r  r  r"  s     €rI   rf   zGemma4AudioLightConv1d.__init__å  sè   ø€ Ý‰Œ×ÒÑÔÐØˆŒå1°&¸&Ô:LÈfÔN`ÐcdÑNdÑeÔeˆÔÝ/°¸Ô8JÈFÔL^Ñ_Ô_ˆŒÝ 7ØÔ*ØÔ+ØÔ/ØÔ%Øð!
ñ !
ô !
ˆÔõ ,¨FÔ,>ÀFÔDWÐdhÐiÑiÔiˆÔÝ& vÔ'9¸vÔ?RÐ_cÐdÑdÔdˆŒÝ˜VÔ.Ô/ˆŒà!'Ô!9ˆÔÐÐrH   rO   r\   c                 ó2  — |}|                       |¦  «        }|                      |¦  «        }t          j                             |d¬¦  «        }|                      |                     dd¦  «        ¦  «                             dd¦  «        }t          | j        t          j
        |j        ¦  «        j        ¦  «        }t          j        || |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z  }|S )Nr‚   r¨   r/   r-   )r  r5  r   r	   Úglur8  Ú	transposer$  r  r@   r%  rª   rœ   rq   r9  r  r6  )rm   rO   r&  r  s       rI   rr   zGemma4AudioLightConv1d.forwardù  sü   € Ø ˆà×+Ò+¨MÑ:Ô:ˆØ×)Ò)¨-Ñ8Ô8ˆÝœ×)Ò)¨-¸RÐ)Ñ@Ô@ˆà×-Ò-¨m×.EÒ.EÀaÈÑ.KÔ.KÑLÔL×VÒVÐWXÐZ[Ñ\Ô\ˆõ   Ô 6½¼ÀMÔDWÑ8XÔ8XÔ8\Ñ]Ô]ÐÝœ MÐ4EÐ3EÐGXÑYÔYˆØŸš }Ñ5Ô5ˆàŸš MÑ2Ô2ˆØŸš¨Ñ6Ô6ˆØ˜Ñ!ˆØÐrH   r'  ru   s   @rI   r1  r1  ä  sk   ø€ € € € € ð:Ð0ð :ð :ð :ð :ð :ð :ð( U¤\ð °e´lð ð ð ð ð ð ð ð rH   r1  c            
       ó~   ‡ — e Zd Zdedefˆ fd„Zdej        dej        dz  dej        de	e
         d	ej        f
d
„Zˆ xZS )ÚGemma4AudioLayerrY   rµ   c                 ó¦  •— t          ¦   «                              ¦   «          || _        t          |¦  «        | _        t          |¦  «        | _        t          ||¦  «        | _        t          |¦  «        | _	        t          |j        ¦  «        | _        t          |j        ¦  «        | _        t          |j        ¦  «        | _        |j        | _        d S rp   )re   rf   rY   r  Úfeed_forward1Úfeed_forward2r´   Ú	self_attnr1  Úlconv1drw   r•   Únorm_pre_attnÚnorm_post_attnÚnorm_outr  rÊ   s      €rI   rf   zGemma4AudioLayer.__init__  s©   ø€ Ý‰Œ×ÒÑÔÐØˆŒå3°FÑ;Ô;ˆÔÝ3°FÑ;Ô;ˆÔÝ-¨f°iÑ@Ô@ˆŒÝ-¨fÑ5Ô5ˆŒå*¨6Ô+=Ñ>Ô>ˆÔÝ+¨FÔ,>Ñ?Ô?ˆÔÝ% fÔ&8Ñ9Ô9ˆŒà!'Ô!9ˆÔÐÐrH   rO   rU   Nrß   Úkwargsr\   c                 óF  — t          | j        t          j        | j        j        j        ¦  «        j        ¦  «        }|                      |¦  «        }|}t          j	        || |¦  «        }|                      |¦  «        }|  
                    |||¬¦  «        \  }}t          j	        || |¦  «        }|                      |¦  «        }||z  }|                      |¦  «        }|                      |¦  «        }t          j	        || |¦  «        }|                      |¦  «        }|S )N)rO   rß   rU   )r$  r  r@   r%  rD  r€   rª   rœ   r@  rq   rB  rE  rC  rA  rF  )rm   rO   rU   rß   rG  r  r&  rì   s           rI   rr   zGemma4AudioLayer.forward  s   € õ   Ô 6½¼ÀDÔDVÔD]ÔDcÑ8dÔ8dÔ8hÑiÔiÐà×*Ò*¨=Ñ9Ô9ˆØ ˆåœ MÐ4EÐ3EÐGXÑYÔYˆØ×*Ò*¨=Ñ9Ô9ˆàŸ>š>Ø'Ø 3Ø)ð *ñ 
ô 
Ñˆ�qõ œ MÐ4EÐ3EÐGXÑYÔYˆØ×+Ò+¨MÑ:Ô:ˆØ˜Ñ!ˆàŸš ]Ñ3Ô3ˆØ×*Ò*¨=Ñ9Ô9ˆåœ MÐ4EÐ3EÐGXÑYÔYˆØŸš mÑ4Ô4ˆàÐrH   )r<   r=   r>   r0   rs   rf   r@   rF   rV   r"   r$   rr   rt   ru   s   @rI   r>  r>    s¤   ø€ € € € € ð:Ð0ð :¸Sð :ð :ð :ð :ð :ð :ð à”|ð ð Ô(¨4Ñ/ð ð #œ\ð	 ð
 Ð+Ô,ð ð 
Œð ð  ð  ð  ð  ð  ð  ð  rH   r>  c                   ó–   ‡ — e Zd Zdefˆ fd„Zdej        dej        dej        fd„Zdej        dej        dej        dej        fd„Zˆ xZ	S )	ÚGemma4VisionPatchEmbedderrY   c                 ód  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        t          j        d| j        dz  z  | j        d¬¦  «        | _        t          j	        t          j        d| j        | j        ¦  «        ¦  «        | _        d S )Nr
   r-   Fr^   )re   rf   rY   r•   Ú
patch_sizeÚposition_embedding_sizer   rh   Ú
input_projr~   r@   r   Úposition_embedding_tabler"  s     €rI   rf   z"Gemma4VisionPatchEmbedder.__init__D  s•   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ Ô+ˆŒØ'-Ô'EˆÔ$åœ) A¨¬¸Ñ(:Ñ$:¸DÔ<LÐSXÐYÑYÔYˆŒÝ(*¬µU´ZÀÀ4ÔC_ÐaeÔaqÑ5rÔ5rÑ(sÔ(sˆÔ%Ð%Ð%rH   Úpixel_position_idsÚpadding_positionsr\   c                 ó&  — |                      d¬¦  «        }t          j        |d         | j        d         ¦  «        }t          j        |d         | j        d         ¦  «        }||z   }t	          j        |                     d¦  «        d|¦  «        }|S )ak  Compute 2-D patch position embeddings via embedding lookup.

        ``pixel_position_ids`` has shape ``(batch, num_patches, 2)`` where the
        last dimension holds (x, y) indices into ``position_embedding_table``
        (shape ``(2, position_embedding_size, hidden_size)``).  The result is the
        sum of the x- and y-embeddings for each patch.
        r   ©r$  ©.r   ©.r/   r/   r‚   ç        )rq   rÍ   Ú	embeddingrO  r@   ÚwhererŸ   )rm   rP  rQ  Úclamped_positionsÚx_embÚy_embrß   s          rI   Ú_position_embeddingsz.Gemma4VisionPatchEmbedder._position_embeddingsN  s‘   € ð /×4Ò4¸Ð4Ñ;Ô;Ðõ ”Ð-¨fÔ5°tÔ7TÐUVÔ7WÑXÔXˆÝ”Ð-¨fÔ5°tÔ7TÐUVÔ7WÑXÔXˆØ# e™mÐÝ#œkÐ*;×*EÒ*EÀbÑ*IÔ*IÈ3ÐPcÑdÔdÐØ"Ð"rH   Úpixel_valuesc                 óÌ   — d|dz
  z  }| j         j        j        x}j        r|                     |¦  «        }|                       |¦  «        }|                      ||¦  «        }||z   S )Nr-   ç      à?)rN  r€   rª   Úis_floating_pointr«   r\  )rm   r]  rP  rQ  Útarget_dtyperO   rß   s          rI   rr   z!Gemma4VisionPatchEmbedder.forwardd  sq   € ð ˜L¨3Ñ.Ñ/ˆØ œOÔ2Ô8Ð8ˆLÔKð 	9Ø'Ÿ?š?¨<Ñ8Ô8ˆLØŸš¨Ñ5Ô5ˆØ"×7Ò7Ð8JÐL]Ñ^Ô^ÐØÐ2Ñ2Ð2rH   )
r<   r=   r>   r3   rf   r@   rF   r\  rr   rt   ru   s   @rI   rJ  rJ  C  s¼   ø€ € € € € ðtÐ1ð tð tð tð tð tð tð#°u´|ð #ÐX]ÔXdð #ÐinÔiuð #ð #ð #ð #ð,	3Ø!œLð	3Ø>C¼lð	3Ø_dÔ_kð	3à	Œð	3ð 	3ð 	3ð 	3ð 	3ð 	3ð 	3ð 	3rH   rJ  c                   óà   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        dede	ej        ej        f         fd„Z
	 ddej        dej        d
ej        ded	z  de	ej        ej        f         f
d„Zˆ xZS )ÚGemma4VisionPoolera[  Spatial pooling and ``sqrt(hidden_size)`` scaling for vision encodings.

    The scaling expands the activation magnitude, which can exceed the float16 range, so it is
    computed in float32 and the pooled features are returned in float32. The caller
    (``Gemma4VisionModel.forward``) standardizes them and casts back to the working dtype.
    rY   c                 ó~   •— t          ¦   «                              ¦   «          |j        | _        | j        dz  | _        d S )Nr_  )re   rf   r•   Úroot_hidden_sizer"  s     €rI   rf   zGemma4VisionPooler.__init__x  s:   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ $Ô 0°#Ñ 5ˆÔÐÐrH   rO   rP  Úlengthr\   c                 óÜ  — |j         d         }t          ||z  dz  ¦  «        }|dz  }||z  |k    r$t          d|j         › d|› d|›d|›d|› d	�¦  «        ‚|                     d
¬¦  «        }|d                              dd¬¦  «        d
         dz   }t          j        ||d¬¦  «        }	|	d         ||z  |	d         z  z   }	t          j        |	 	                    ¦   «         |¦  «         
                    ¦   «         |z  }
|
                     dd¦  «        | 
                    ¦   «         z  }t          j        |
d
k                         d¬¦  «        ¦  «        }|                     |j        ¦  «        |fS )zÙ
        2D spatial pooling according to patch positions.
        Pools the input tokens by averaging patches within a `k^2` grid, where `k` is determined by the ratio between
        input and output lengths
        r/   r_  r-   zCannot pool z to z: k=z^2 times length=z	 must be ú.r   rS  rT  r‚   T©ry   rƒ   Úfloor)Úrounding_moderU  r¨   )rÌ   rs   Ú
ValueErrorrq   rœ   r@   ÚdivrÍ   Úone_hotÚlongrl   r<  rç   Úallr«   rª   )rm   rO   rP  rf  Úinput_seq_lenÚkÚ	k_squaredrY  Úmax_xÚkernel_idxsÚweightsÚoutputr  s                rI   Ú_avg_pool_by_positionsz)Gemma4VisionPooler._avg_pool_by_positions}  sŒ  € ð &Ô+¨AÔ.ˆÝ� &Ñ(¨SÑ0Ñ1Ô1ˆØ�q‘Dˆ	Ø�vÑ Ò.Ð.ÝØq˜}Ô2ÐqÐq¸ÐqÐqÀ!ÐqÐqÈvÐqÐqÐanÐqÐqÐqñô ð ð /×4Ò4¸Ð4Ñ;Ô;ÐØ! &Ô)×-Ò-°"¸dÐ-ÑCÔCÀAÔFÈÑJˆÝ”iÐ 1°1ÀGÐLÑLÔLˆØ! &Ô)¨U°a©Z¸;ÀvÔ;NÑ,NÑNˆÝ”)˜K×,Ò,Ñ.Ô.°Ñ7Ô7×=Ò=Ñ?Ô?À)ÑKˆØ×"Ò" 1 aÑ(Ô(¨=×+>Ò+>Ñ+@Ô+@Ñ@ˆÝÔ  '¨Q¢,×!3Ò!3¸Ð!3Ñ!:Ô!:Ñ;Ô;ˆØ�yŠy˜Ô,Ñ-Ô-¨tÐ3Ð3rH   NrQ  Úoutput_lengthc                 óN  — ||j         d         k    r!t          d|› d|j         d         › d�¦  «        ‚|                     |                     d¦  «        d¦  «        }|j         d         |k    r|                      |||¦  «        \  }}|                     ¦   «         | j        z  }||fS )Nr/   z*Cannot output more soft tokens (requested z) than there are patches (z9). Change the value of `num_soft_tokens` when processing.r‚   rV  )rÌ   rl  ræ   rŸ   rx  rl   re  )rm   rO   rP  rQ  ry  s        rI   rr   zGemma4VisionPooler.forward˜  sÞ   € ð ˜=Ô.¨qÔ1Ò1Ð1Ýðg¸]ð gð gØ"Ô(¨Ô+ðgð gð gñô ð ð
 &×1Ò1Ð2C×2MÒ2MÈbÑ2QÔ2QÐSVÑWÔWˆàÔ˜qÔ! ]Ò2Ð2Ø/3×/JÒ/JØÐ1°=ñ0ô 0Ñ,ˆMÐ,ð &×+Ò+Ñ-Ô-°Ô0EÑEˆØÐ/Ð/Ð/rH   rp   )r<   r=   r>   r?   r3   rf   r@   rF   rs   rE   rx  rr   rt   ru   s   @rI   rc  rc  p  sü   ø€ € € € € ðð ð6Ð1ð 6ð 6ð 6ð 6ð 6ð 6ð
4Ø"œ\ð4Ø?D¼|ð4ØUXð4à	ˆuŒ|˜Uœ\Ð)Ô	*ð4ð 4ð 4ð 4ð@ %)ð0ð 0à”|ð0ð "œLð0ð !œ<ð	0ð
 ˜T‘zð0ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð0ð 0ð 0ð 0ð 0ð 0ð 0ð 0rH   rc  c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚGemma4VisionMLPrY   c                 ót  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        t          || j        | j        ¦  «        | _        t          || j        | j        ¦  «        | _        t          || j        | j        ¦  «        | _        t          |j
                 | _        d S rp   )re   rf   rY   r•   Úintermediate_sizerX   Ú	gate_projÚup_projÚ	down_projr   Úhidden_activationr  r"  s     €rI   rf   zGemma4VisionMLP.__init__´  s”   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔÝ.¨v°tÔ7GÈÔI_Ñ`Ô`ˆŒÝ,¨V°TÔ5EÀtÔG]Ñ^Ô^ˆŒÝ.¨v°tÔ7MÈtÔO_Ñ`Ô`ˆŒÝ˜VÔ5Ô6ˆŒˆˆrH   c                 ó¨   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        }|S rp   ©r�  r  r  r€  ©rm   rÙ   r�  s      rI   rr   zGemma4VisionMLP.forward¾  óA   € Ø—N’N 4§;¢;¨t¯~ª~¸aÑ/@Ô/@Ñ#AÔ#AÀDÇLÂLÐQRÁOÄOÑ#SÑTÔTˆ	ØÐrH   )r<   r=   r>   r3   rf   rr   rt   ru   s   @rI   r|  r|  ³  sT   ø€ € € € € ð7Ð1ð 7ð 7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð rH   r|  c                   óØ   ‡ — e Zd ZU ej        ed<   ddefˆ fd„Ze	 	 	 ddedz  dej	        dz  de
dz  dedef         fd	„¦   «         Z ej        ¦   «         ed
„ ¦   «         ¦   «         Zˆ xZS )ÚGemma4VisionRotaryEmbeddingÚinv_freqNrY   c                 ó²  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        |                      d|d¬¦  «         |                      d|                     ¦   «         d¬¦  «         d S )NÚ	rope_typeÚdefaultr‰  Fr“   Úoriginal_inv_freq)re   rf   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrY   Úrope_parametersr‹  Úcompute_default_rope_parametersr   Úattention_scalingrj   Úclone)rm   rY   r¦   Úrope_init_fnr‰  rn   s        €rI   rf   z$Gemma4VisionRotaryEmbedding.__init__Æ  sÊ   ø€ Ý‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(à×Ò˜Z¨¸eÐÑDÔDÐDØ×ÒÐ0°(·.².Ñ2BÔ2BÈuÐÑUÔUÐUÐUÐUrH   r¦   rÒ   r\   útorch.Tensorc                 ó  — | j         d         }t          | dd¦  «        p| j        | j        z  }|dz  }d}d|t	          j        d|dt          j        ¬¦  «                             |t          j        ¬¦  «        |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetar»   Nr-   r‘   r   r©   ©r¦   rª   ©	r‘  Úgetattrr•   rº   r@   rž   Úint64r«   rl   )rY   r¦   rÒ   Úbasery   Úspatial_dimÚattention_factorr‰  s           rI   r’  z;Gemma4VisionRotaryEmbedding.compute_default_rope_parametersÖ  s™   € ð& Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆð ˜Q‘hˆàÐØØÝ”˜Q ¨Qµe´kÐBÑBÔB×EÒEÈVÕ[`Ô[fÐEÑgÔgÐjuÑuñwñ
ˆð Ð)Ð)Ð)rH   c                 óH  — | j         d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}g g }}t          d¦  «        D �]}|d d …d d …|f         }|d d …d d d …f                              ¦   «         }	t          |d¬¦  «        5  |                     ¦   «         |	                     ¦   «         z                       dd¦  «        }
t          j        |
|
fd¬	¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }d d d ¦  «         n# 1 swxY w Y   |                     |¦  «         |                     |¦  «         �Œt          j        |d¬	¦  «                             |j        ¬
¦  «        }t          j        |d¬	¦  «                             |j        ¬
¦  «        }||fS )Nr   r‚   r/   ÚmpsÚcpur-   F©Údevice_typeÚenabledr¨   r©   )r‰  rl   ÚexpandrÌ   r«   r¦   Ú
isinstanceÚtyperD   Úranger)   r<  r@   r¬   r®   r“  r­   Úappendrª   )rm   rÙ   r¯   Úinv_freq_expandedr¤  Úall_cosÚall_sinÚiÚdim_position_idsÚdim_position_ids_expandedÚfreqsÚembr®   r­   s                 rI   rr   z#Gemma4VisionRotaryEmbedding.forwardù  sC  € ð !œM¨$°°°°4¨-Ô8×>Ò>Ñ@Ô@×GÒGÈÔHZÐ[\ÔH]Ð_aÐcdÑeÔe×hÒhÐijÔiqÑrÔrÐÝ'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆð ˜r�ˆÝ�q‘”ð 
	 ñ 
	 ˆAØ+¨A¨A¨A¨q¨q¨q°!¨GÔ4ÐØ(8¸¸¸¸DÀ!À!À!¸Ô(D×(JÒ(JÑ(LÔ(LÐ%å¨KÀÐGÑGÔGð 9ð 9Ø*×0Ò0Ñ2Ô2Ð5N×5TÒ5TÑ5VÔ5VÑV×aÒaÐbcÐefÑgÔg�Ý”i ¨ °BÐ7Ñ7Ô7�Ø—g’g‘i”i $Ô"8Ñ8�Ø—g’g‘i”i $Ô"8Ñ8�ð	9ð 9ð 9ñ 9ô 9ð 9ð 9ð 9ð 9ð 9ð 9øøøð 9ð 9ð 9ð 9ð
 �NŠN˜3ÑÔÐØ�NŠN˜3ÑÔÐÑåŒi˜ RÐ(Ñ(Ô(×+Ò+°!´'Ð+Ñ:Ô:ˆÝŒi˜ RÐ(Ñ(Ô(×+Ò+°!´'Ð+Ñ:Ô:ˆØ�Cˆxˆs   Ã1BFÆF	ÆF	rp   ©NNN)r<   r=   r>   r@   rF   rB   r3   rf   Ústaticmethodr¦   rs   rE   rl   r’  r²   r   rr   rt   ru   s   @rI   rˆ  rˆ  Ã  sý   ø€ € € € € € ØŒlÐÐÑðVð VÐ1ð Vð Vð Vð Vð Vð Vð  à,0Ø&*Ø"ð *ð  *Ø" TÑ)ð *à”˜tÑ#ð *ð �t‘ð *ð 
ˆ~˜uÐ$Ô	%ð	 *ð  *ð  *ñ „\ð *ðD €U„]�_„_Øðð ñ Ôñ „_ðð ð ð ð rH   rˆ  c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr‚   r-   r¨   )rÌ   r@   r¬   )rÙ   Úx1Úx2s      rI   Úrotate_halfr¸    s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'rH   rÙ   r®   r­   Úunsqueeze_dimc                 ó†   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   S )a\  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        x (`torch.Tensor`): The tensor to embed.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )rŸ   r¸  ©rÙ   r®   r­   r¹  s       rI   Úapply_rotary_pos_embr¼    s@   € ð" �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�‰G� A™œ¨Ñ,Ñ-Ð-rH   rO   Ún_repr\   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r/   N)rÌ   r¦  rÏ   )rO   r½  ÚbatchÚnum_key_value_headsÚslenr»   s         rI   Ú	repeat_kvrÂ  /  s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTrH   rV  ÚmoduleÚqueryÚkeyÚvaluerU   ÚdropoutÚscalingr·   c                 ól  — |€
| j         dz  }t          || j        ¦  «        }	t          || j        ¦  «        }
t          j        ||	                     dd¦  «        ¦  «        |z  }|�||z  }t          j        |¦  «        }||z  }|�||z   }t          j         	                    |dt          j
        ¬¦  «                             |j        ¦  «        }t          j                             ||| j        ¬¦  «        }t          j        ||
¦  «        }|                     dd¦  «                             ¦   «         }||fS )Nr„   r-   r
   r‚   râ   )ÚpÚtrainingr/   )r»   rÂ  Únum_key_value_groupsr@   Úmatmulr<  rå   r   r	   ré   rê   r«   rª   rÇ  rË  rÐ   )rÃ  rÄ  rÅ  rÆ  rU   rÇ  rÈ  r·   rG  rï   rð   rö   r÷   s                rI   Úeager_attention_forwardrÎ  ;  s%  € ð €Ø”/ 4Ñ'ˆå˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LàÐØ# gÑ-ˆÝ”z ,Ñ/Ô/ˆØ# gÑ-ˆØÐ!Ø# nÑ4ˆõ ”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€KØ˜Ð$Ð$rH   r¯   c           	      ó�  ‡‡
‡‡— |j         d         }| j         d         }d|d|z  z  z  }|dk    rt          d|› d|› d|› d�¦  «        ‚|g|z  }t          j        | |d¬¦  «        Št          j        ||d¬¦  «        Š
t          j        ||d¬¦  «        Šˆ
ˆˆˆfd	„t	          |¦  «        D ¦   «         }	t          j        |	d¬¦  «        S )
ak  Applies multidimensional RoPE to inputs.

    Args:
        x (`torch.Tensor`): The tensor to embed.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            If position_ids.ndim + 2 == x.ndim, then this function passes through to `apply_rotary_pos_emb()`.
            Otherwise, position_ids is used to split the inputs, x, into multiple pieces, where each piece is fed to
            `apply_rotary_pos_emb()`, and then concatenated back together.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.

    Returns:
      Tensor of shape [B, L, N, H] with RoPE applied.
    r‚   r-   r   zEInvalid configuration: num_rotated_channels_per_dim must be > 0, got z (num_input_channels=z, ndim=ú)r¨   c                 óZ   •— g | ]'}t          ‰|         ‰|         ‰|         ‰¬ ¦  «        ‘Œ(S )r»  )r¼  )Ú.0rr  Ú	cos_partsÚ	sin_partsr¹  Úx_partss     €€€€rI   ú
<listcomp>z/apply_multidimensional_rope.<locals>.<listcomp>‰  sR   ø€ ð ð ð ð õ 	Ø�aŒjØ˜!”Ø˜!”Ø'ð		
ñ 	
ô 	
ðð ð rH   )rÌ   rl  r@   Úsplitr©  r¬   )rÙ   r®   r­   r¯   r¹  ÚndimÚnum_input_channelsÚnum_rotated_channels_per_dimÚsplit_sizesÚy_partsrÓ  rÔ  rÕ  s       `     @@@rI   Úapply_multidimensional_roperÝ  ]  s+  øøøø€ ð8 Ô˜bÔ!€DØœ œÐØ#$Ð(:¸qÀ4¹xÑ(HÑ#IÐ à# qÒ(Ð(ÝðØ,ðð ØCUðð àðð ð ñ
ô 
ð 	
ð 0Ð0°4Ñ7€KÝŒk˜!˜[¨bÐ1Ñ1Ô1€GÝ”˜C °"Ð5Ñ5Ô5€IÝ”˜C °"Ð5Ñ5Ô5€Iðð ð ð ð ð ð õ �t‘”ðñ ô €Gõ Œ9�W "Ð%Ñ%Ô%Ð%rH   c                   óÜ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej        dej        dej        dz  d	ej	        dz  d
e
e         deej        ej        dz  eej                 dz  f         fd„Zˆ xZS )ÚGemma4VisionAttentionú=Multi-headed attention from 'Attention Is All You Need' paperrY   rµ   c                 óZ  •— t          ¦   «                              ¦   «          t          |d¦  «        r|j        |         nd | _        || _        || _        t          |d|j        |j	        z  ¦  «        | _
        |j	        |j        z  | _        d| _        | j        j        | _        d| _        t!          ||j        |j	        | j
        z  ¦  «        | _        t!          ||j        |j        | j
        z  ¦  «        | _        t!          ||j        |j        | j
        z  ¦  «        | _        t!          ||j	        | j
        z  |j        ¦  «        | _        t+          |j
        |j        ¬¦  «        | _        t+          |j
        |j        ¬¦  «        | _        t+          | j
        |j        d¬¦  «        | _        d S )NÚlayer_typesr»   r‘   F©ry   rz   r4  )re   rf   Úhasattrrâ  Ú
layer_typerY   rµ   r›  r•   rº   r»   rÀ  rÌ  rÈ  Úattention_dropoutÚ	is_causalrX   rÃ   rÄ   rÅ   Úo_projrw   r  Úq_normÚk_normÚv_normrÊ   s      €rI   rf   zGemma4VisionAttention.__init__˜  sr  ø€ Ý‰Œ×ÒÑÔÐÝ;BÀ6È=Ñ;YÔ;YÐc˜&Ô,¨YÔ7Ð7Ð_cˆŒØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!ØˆŒØ!%¤Ô!>ˆÔØˆŒÝ+¨F°FÔ4FÈÔHbÐeiÔerÑHrÑsÔsˆŒÝ+¨F°FÔ4FÈÔHbÐeiÔerÑHrÑsÔsˆŒÝ+¨F°FÔ4FÈÔHbÐeiÔerÑHrÑsÔsˆŒÝ+¨F°FÔ4NÐQUÔQ^Ñ4^Ð`fÔ`rÑsÔsˆŒå#¨¬¸VÔ=PÐQÑQÔQˆŒÝ#¨¬¸VÔ=PÐQÑQÔQˆŒÝ# D¤M°vÔ7JÐW\Ð]Ñ]Ô]ˆŒˆˆrH   NrO   rß   rU   r¯   rG  r\   c                 óˆ  — |j         d d…         }g |¢d‘| j        ‘R }|\  }}	|                      |¦  «                             |¦  «        }
|                      |
¦  «        }
t          |
||	|¦  «        }
|
                     dd¦  «        }
|                      |¦  «                             |¦  «        }|                      |¦  «        }t          |||	|¦  «        }|                     dd¦  «        }|  	                    |¦  «                             |¦  «        }|  
                    |¦  «        }|                     dd¦  «        }t          j        | j        j        t          ¦  «        } || |
|||f| j        r| j        nd| j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr‚   r/   r-   rV  )rÇ  rÈ  )rÌ   r»   rÃ   rÛ   ré  rÝ  r<  rÄ   rê  rÅ   rë  r    Úget_interfacerY   Ú_attn_implementationrÎ  rË  ræ  rÈ  rÏ   rÐ   rè  )rm   rO   rß   rU   r¯   rG  Úinput_shaperí   r®   r­   rî   rï   rð   Úattention_interfacer÷   rö   s                   rI   rr   zGemma4VisionAttention.forward«  sæ  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà&‰ˆˆSà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆØ—{’{ <Ñ0Ô0ˆÝ2°<ÀÀcÈ<ÑXÔXˆØ#×-Ò-¨a°Ñ3Ô3ˆà—[’[ Ñ/Ô/×4Ò4°\ÑBÔBˆ
Ø—[’[ Ñ,Ô,ˆ
Ý0°¸SÀ#À|ÑTÔTˆ
Ø×)Ò)¨!¨QÑ/Ô/ˆ
à—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆØ—{’{ <Ñ0Ô0ˆØ#×-Ò-¨a°Ñ3Ô3ˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð /3¬mÐD�DÔ*Ð*ÀØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(rH   r³  )r<   r=   r>   r?   r3   rs   rf   r@   rF   Ú
LongTensorr"   r$   rE   rr   rt   ru   s   @rI   rß  rß  •  sò   ø€ € € € € ØGÐGð^Ð1ð ^¸cð ^ð ^ð ^ð ^ð ^ð ^ð, -1Ø.2Ø04ð,)ð ,)à”|ð,)ð #œ\ð,)ð œ tÑ+ð	,)ð
 Ô&¨Ñ-ð,)ð Ð+Ô,ð,)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð,)ð ,)ð ,)ð ,)ð ,)ð ,)ð ,)ð ,)rH   rß  c                   óÔ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej        dej        dej        dz  dej        dz  d	e	e
         d
eej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚGemma4VisionEncoderLayerrY   rµ   c                 óÐ  •— t          ¦   «                              ¦   «          || _        |j        | _        || _        t          ||¬¦  «        | _        t          |¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        d S )N©rY   rµ   ©rz   )re   rf   rY   r•   rµ   rß  rB  r|  Úmlprw   r  Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormrÊ   s      €rI   rf   z!Gemma4VisionEncoderLayer.__init__Û  sÇ   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ"ˆŒÝ.°fÈ	ÐRÑRÔRˆŒÝ" 6Ñ*Ô*ˆŒÝ,¨TÔ-=À6ÔCVÐWÑWÔWˆÔÝ(5°dÔ6FÈFÔL_Ð(`Ñ(`Ô(`ˆÔ%Ý)6°tÔ7GÈVÔM`Ð)aÑ)aÔ)aˆÔ&Ý*7¸Ô8HÈfÔNaÐ*bÑ*bÔ*bˆÔ'Ð'Ð'rH   NrO   rß   rU   r¯   rG  r\   c                 ó  — |}|                       |¦  «        } | j        d||||dœ|¤Ž\  }}|                      |¦  «        }||z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z   }|S )N)rO   rß   rU   r¯   rG   )rø  rB  rù  rú  r÷  rû  )rm   rO   rß   rU   r¯   rG  r&  rì   s           rI   rr   z Gemma4VisionEncoderLayer.forwardç  s¿   € ð !ˆà×,Ò,¨]Ñ;Ô;ˆà)˜4œ>ð 
Ø'Ø 3Ø)Ø%ð	
ð 
ð
 ð
ð 
Ñˆ�qð ×5Ò5°mÑDÔDˆØ  =Ñ0ˆà ˆØ×6Ò6°}ÑEÔEˆØŸš Ñ/Ô/ˆØ×7Ò7¸ÑFÔFˆØ  =Ñ0ˆàÐrH   r³  )r<   r=   r>   r3   rs   rf   r@   rF   rñ  r"   r$   rE   rA   rr   rt   ru   s   @rI   ró  ró  Ú  sî   ø€ € € € € ð
cÐ1ð 
c¸cð 
cð 
cð 
cð 
cð 
cð 
cð -1Ø.2Ø04ðð à”|ðð #œ\ðð œ tÑ+ð	ð
 Ô&¨Ñ-ðð Ð+Ô,ðð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uðð ð ð ð ð ð ð rH   ró  c                   ót   ‡ — e Zd Zdefˆ fd„Z	 d
dej        dej        dej        dz  dee	         de
f
d	„Zˆ xZS )ÚGemma4VisionEncoderrY   c                 ó  •‡— t          ¦   «                              ¦   «          ‰| _        ‰j        | _        t          ‰¦  «        | _        t          j        ˆfd„t          | j        ¦  «        D ¦   «         ¦  «        | _
        d S )Nc                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS )rõ  )ró  )rÒ  r®  rY   s     €rI   rÖ  z0Gemma4VisionEncoder.__init__.<locals>.<listcomp>  s'   ø€ ÐbÐbÐbÀaÕ%¨V¸qÐAÑAÔAÐbÐbÐbrH   )re   rf   rY   Únum_hidden_layersÚ
num_layersrˆ  Ú
rotary_embr   Ú
ModuleListr©  Úlayersr"  s    `€rI   rf   zGemma4VisionEncoder.__init__  st   øø€ Ý‰Œ×ÒÑÔÐØˆŒØ Ô2ˆŒÝ5°fÑ=Ô=ˆŒÝ”mØbÐbÐbÐbÍ5ÐQUÔQ`ÑKaÔKaÐbÑbÔbñ
ô 
ˆŒˆˆrH   NÚinputs_embedsrU   rP  rG  r\   c                 óÒ   — t          | j        ||¬¦  «        }|}|                      ||¦  «        }| j        d| j        j        …         D ]} ||f|||dœ|¤Ž}Œt          |¬¦  «        S )z�
        pixel_position_ids (torch.Tensor):
            Patch positions as (x, y) coordinates in the image as [batch, num_patches, 2].
        )rY   r  rU   N)rU   rß   r¯   ©Úlast_hidden_state)r   rY   r  r  r  r   )rm   r  rU   rP  rG  rO   rß   Údecoder_layers           rI   rr   zGemma4VisionEncoder.forward  s¨   € õ 3Ø”;Ø'Ø)ð
ñ 
ô 
ˆð &ˆØ"Ÿošo¨mÐ=OÑPÔPÐð "œ[Ð)H¨4¬;Ô+HÐ)HÔIð 	ð 	ˆMØ)˜MØðà-Ø$7Ø/ð	ð ð
 ðð ˆMˆMõ '¸ÐGÑGÔGÐGrH   rp   )r<   r=   r>   r3   rf   r@   rF   rñ  r"   r$   r   rr   rt   ru   s   @rI   rþ  rþ    s³   ø€ € € € € ð
Ð1ð 
ð 
ð 
ð 
ð 
ð 
ð 7;ð	Hð Hà”|ðHð œðHð "Ô,¨tÑ3ð	Hð
 Ð+Ô,ðHð 
!ðHð Hð Hð Hð Hð Hð Hð HrH   rþ  c                   ó.   ‡ — e Zd Zdedefˆ fd„Zd„ Zˆ xZS )ÚGemma4TextMLPrY   rµ   c                 óö  •— t          ¦   «                              ¦   «          |j        |j        z
  }||cxk    odk    nc }|j        o|}|| _        |j        | _        |j        |rdndz  | _        t          j	        | j        | j        d¬¦  «        | _
        t          j	        | j        | j        d¬¦  «        | _        t          j	        | j        | j        d¬¦  «        | _        t          |j                 | _        d S )Nr   r-   r/   Fr^   )re   rf   r  Únum_kv_shared_layersÚuse_double_wide_mlprY   r•   r~  r   rh   r  r€  r�  r   r‚  r  )rm   rY   rµ   Úfirst_kv_shared_layer_idxÚis_kv_shared_layerr  rn   s         €rI   rf   zGemma4TextMLP.__init__3  sù   ø€ Ý‰Œ×ÒÑÔÐØ$*Ô$<¸vÔ?ZÑ$ZÐ!Ø&Ð*CÐGÐGÒGÐGÀaÒGÐGÐGÐGÐØ$Ô8ÐOÐ=OÐØˆŒØ!Ô-ˆÔØ!'Ô!9ÐBUÐ=\¸Q¸QÐ[\Ñ!]ˆÔÝœ 4Ô#3°TÔ5KÐRWÐXÑXÔXˆŒÝ”y Ô!1°4Ô3IÐPUÐVÑVÔVˆŒÝœ 4Ô#9¸4Ô;KÐRWÐXÑXÔXˆŒÝ˜VÔ5Ô6ˆŒˆˆrH   c                 ó¨   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        }|S rp   r„  r…  s      rI   rr   zGemma4TextMLP.forward@  r†  rH   )r<   r=   r>   r2   rs   rf   rr   rt   ru   s   @rI   r  r  2  s[   ø€ € € € € ð7Ð/ð 7¸Cð 7ð 7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð rH   r  c                   óâ   ‡ — e Zd ZU ej        ed<   ddefˆ fd„Ze	 	 	 	 ddedz  de	d         de
dz  dedz  d	ed
ef         f
d„¦   «         Z ej        ¦   «         edd„¦   «         ¦   «         Zˆ xZS )ÚGemma4TextRotaryEmbeddingr‰  NrY   c                 ó†  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        t          |j        ¦  «        | _        i | _        i | _	        | j        D ]Ñ}| j        j
        |         }|€Œ|d         x}dk    rt          |         }n| j        }|| j        |<   || j	        |<   ||dœ}|dk    r|dk    rd|d<    || j        fi |¤Ž\  }}	|                      |› d�|d	¬
¦  «         |                      |› d�|                     ¦   «         d	¬
¦  «         t          | |› d�|	¦  «         ŒÒd S )Nr‹  rŒ  )r¦   rå  Úfull_attentionÚproportionalÚglobal_head_dimÚhead_dim_keyÚ	_inv_freqFr“   Ú_original_inv_freqÚ_attention_scaling)re   rf   rŽ  r�  r�  rY   Úsetrâ  Úrope_init_fnsr‹  r‘  r   r’  rj   r”  Úsetattr)rm   rY   r¦   rå  Úrope_paramsr‹  r•  Úrope_init_fn_kwargsÚcurr_inv_freqÚcurr_attention_scalingrn   s             €rI   rf   z"Gemma4TextRotaryEmbedding.__init__H  s–  ø€ Ý‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒÝ˜vÔ1Ñ2Ô2ˆÔØSUˆÔØ)+ˆŒàÔ*ð 	Uð 	UˆJØœ+Ô5°jÔAˆKØÐ"Øà(¨Ô5Ð5�	¸)ÒCÐCÝ2°9Ô=��à#ÔC�à-9ˆDÔ˜zÑ*Ø)2ˆDŒN˜:Ñ&à-3À:Ð"NÐ"NÐØÐ-Ò-Ð-°)¸~Ò2MÐ2MØ6GÐ# NÑ3à4@°LÀÄÐ4dÐ4dÐPcÐ4dÐ4dÑ1ˆMÐ1Ø× Ò  JÐ!9Ð!9Ð!9¸=ÐUZÐ Ñ[Ô[Ð[Ø× Ò  JÐ!BÐ!BÐ!BÀM×DWÒDWÑDYÔDYÐfkÐ ÑlÔlÐlÝ�D˜ZÐ;Ð;Ð;Ð=SÑTÔTÐTÐTð)	Uð 	UrH   r¦   ztorch.devicerÒ   rå  r\   r–  c                 ó  — | j         |         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «                             |t          j        ¬¦  «        |z  z  z  }||fS )	a|  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        r˜  r»   Nr‘   r   r-   r©   r™  rš  )rY   r¦   rÒ   rå  r�  ry   rŸ  r‰  s           rI   r’  z9Gemma4TextRotaryEmbedding.compute_default_rope_parametersh  s‘   € ð2 Ô% jÔ1°,Ô?ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐð Ø•U”\ ! S¨!µ5´;Ð?Ñ?Ô?×BÒBÈ&ÕX]ÔXcÐBÑdÔdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)rH   c                 ó|  — t          | |› d�¦  «        }t          | |› d�¦  «        }|d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬	¦  «        5  |                     ¦   «         |                     ¦   «         z   
                    dd
¦  «        }	t          j        |	|	fd¬¦  «        }
|
                     ¦   «         |z  }|
                     ¦   «         |z  }d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |                     |j        ¬¦  «        fS )Nr  r  r   r‚   r/   r¡  r¢  Fr£  r-   r¨   r©   )r›  rl   r¦  rÌ   r«   r¦   r§  r¨  rD   r)   r<  r@   r¬   r®   r­   rª   )rm   rÙ   r¯   rå  r‰  r“  r«  Úposition_ids_expandedr¤  r±  r²  r®   r­   s                rI   rr   z!Gemma4TextRotaryEmbedding.forwardŒ  sâ  € õ ˜4 JÐ!9Ð!9Ð!9Ñ:Ô:ˆÝ# D¨ZÐ*KÐ*KÐ*KÑLÔLÐà$ T¨1¨1¨1¨d ]Ô3×9Ò9Ñ;Ô;×BÒBÀ<ÔCUÐVWÔCXÐZ\Ð^_Ñ`Ô`×cÒcÐdeÔdlÑmÔmÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	0ð 	0Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)Ð/Ñ/ˆCØ—'’'‘)”)Ð/Ñ/ˆCð		0ð 	0ð 	0ñ 	0ô 	0ð 	0ð 	0ð 	0ð 	0ð 	0ð 	0øøøð 	0ð 	0ð 	0ð 	0ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   Ã-BE=Å=FÆF©NN)NNNNrp   )r<   r=   r>   r@   rF   rB   r2   rf   r´  r   rs   rD   rE   rl   r’  r²   r   rr   rt   ru   s   @rI   r  r  E  s  ø€ € € € € € ØŒlÐÐÑðUð UÐ/ð Uð Uð Uð Uð Uð Uð@ à*.Ø+/Ø"Ø!%ð	!*ð !*Ø  4Ñ'ð!*à˜Ô(ð!*ð �t‘ð!*ð ˜$‘Jð	!*ð
 
ˆ~˜uÐ$Ô	%ð!*ð !*ð !*ñ „\ð!*ðF €U„]�_„_Øð<ð <ð <ñ Ôñ „_ð<ð <ð <ð <ð <rH   r  c                   óè   ‡ — e Zd ZdZdedefˆ fd„Z	 ddej        dej        dej        dz  d	e	e
eej        ej        f         f         d
edz  dee         deej        ej        dz  f         fd„Zˆ xZS )ÚGemma4TextAttentionrà  rY   rµ   c                 ó†  •— t          ¦   «                              ¦   «          t          |d¦  «        r|j        |         nd | _        || _        || _        | j        dk    | _        | j        r|j        nd | _        | j        s|j	        r|j	        n|j
        | _
        |j        o| j         | _        | j        r|j        n|j        }|j        |z  | _        d| _        | j        j        | _        |j        dk    | _        | j        j        t-          | j        dd¦  «        z
  }||cxk    odk    nc | _        |j        d |…         }| j         o?|t1          |¦  «        dz
  |d d d…                              |j        |         ¦  «        z
  k    | _        t7          j        |j        |j        | j
        z  |j        ¬	¦  «        | _        tA          | j
        |j!        ¬
¦  «        | _"        | j        s¦tA          | j
        |j!        ¬
¦  «        | _#        tA          | j
        |j!        d¬¦  «        | _$        t7          j        |j        || j
        z  |j        ¬	¦  «        | _%        | j        s)t7          j        |j        || j
        z  |j        ¬	¦  «        nd | _&        t7          j        |j        | j
        z  |j        |j        ¬	¦  «        | _'        d S )Nrâ  Úsliding_attentionr‘   rp  r  r   r/   r‚   r^   rã  Fr4  )(re   rf   rä  râ  rå  rY   rµ   Ú
is_slidingÚsliding_windowr  r»   Úattention_k_eq_vÚuse_alternative_attentionÚnum_global_key_value_headsrÀ  rº   rÌ  rÈ  ræ  Úuse_bidirectional_attentionrç  r  r›  r  ÚlenÚindexÚstore_full_length_kvr   rh   r•   Úattention_biasrÃ   rw   r  ré  rê  rë  rÄ   rÅ   rè  )rm   rY   rµ   rÀ  r  Úprev_layersrn   s         €rI   rf   zGemma4TextAttention.__init__¢  sÝ  ø€ Ý‰Œ×ÒÑÔÐÝ;BÀ6È=Ñ;YÔ;YÐc˜&Ô,¨YÔ7Ð7Ð_cˆŒØˆŒØ"ˆŒØœ/Ð-@Ò@ˆŒØ7;´ÐP˜fÔ3Ð3ÈDˆÔà6:´oÐuÈ&ÔJ`Ðu˜Ô.Ð.ÐflÔfuˆŒØ)/Ô)@Ð)XÈÌÐEXˆÔ&à15Ô1OÐoˆFÔ-Ð-ÐU[ÔUoð 	ð %+Ô$>ÐBUÑ$UˆÔ!ØˆŒØ!%¤Ô!>ˆÔØÔ;¸uÒDˆŒð %)¤KÔ$AÅGÈDÌKÐYoÐqrÑDsÔDsÑ$sÐ!Ø"+Ð/HÐ"MÐ"MÒ"MÐ"MÈAÒ"MÐ"MÐ"MÐ"MˆÔØÔ(Ð)CÐ*CÐ)CÔDˆØ(,Ô(?Ð$?ð %/ÀIÕQTÐU`ÑQaÔQaÐdeÑQeÐhsØˆDˆbˆDôi
ç
Š%�Ô" 9Ô-Ñ
.Ô
.ñR/ò E/ˆÔ!õ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ $¨¬¸6Ô;NÐOÑOÔOˆŒð Ô&ð 	Ý'¨D¬M¸vÔ?RÐSÑSÔSˆDŒKÝ'¨¬¸6Ô;NÐ[`ÐaÑaÔaˆDŒKåœ)ØÔ"Ð$7¸$¼-Ñ$GÈfÔNcðñ ô ˆDŒKð
 Ô5ð•”	˜&Ô,Ð.AÀDÄMÑ.QÐX^ÔXmÐnÑnÔnÐnàð ŒKõ ”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒˆˆrH   NrO   rß   rU   r;   rN   rG  r\   c                 ó¸  — |j         d d…         }g |¢d‘| j        ‘R }|\  }	}
|                      |¦  «                             |¦  «        }|                      |¦  «        }t          ||	|
d¬¦  «        }|                     dd¦  «        }| j        rE|| j                 \  }}| 	                    |j
        ¦  «        }| 	                    |j
        ¦  «        }nÂ|                      |¦  «                             |¦  «        }| j        �(|                      |¦  «                             |¦  «        n|}|                      |¦  «        }t          ||	|
d¬¦  «        }|                     dd¦  «        }|                      |¦  «        }|                     dd¦  «        }|�&| j        s|                     ||| j        ¦  «        \  }}| j        r||f|| j        <   t%          j        | j        j        t,          ¦  «        } || ||||f| j        r| j        nd| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr‚   r-   )r¹  r/   rV  )rÇ  rÈ  r-  )rÌ   r»   rÃ   rÛ   ré  r¼  r<  r  rå  r«   r¦   rÄ   rÅ   rê  rë  Úupdaterµ   r4  r    rí  rY   rî  rÎ  rË  ræ  rÈ  r-  rÏ   rÐ   rè  )rm   rO   rß   rU   r;   rN   rG  rï  rí   r®   r­   rî   rï   rð   rð  r÷   rö   s                    rI   rr   zGemma4TextAttention.forwardÓ  s•  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà&‰ˆˆSà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆØ—{’{ <Ñ0Ô0ˆÝ+¨L¸#¸sÐRSÐTÑTÔTˆØ#×-Ò-¨a°Ñ3Ô3ˆð
 Ô"ð 	8Ø'7¸¼Ô'HÑ$ˆJ˜à#Ÿš |Ô':Ñ;Ô;ˆJØ'Ÿ?š?¨<Ô+>Ñ?Ô?ˆLˆLàŸš ]Ñ3Ô3×8Ò8¸ÑFÔFˆJØLPÌKÐLc˜4Ÿ;š; }Ñ5Ô5×:Ò:¸<ÑHÔHÐHÐisˆLàŸš ZÑ0Ô0ˆJÝ-¨j¸#¸sÐRSÐTÑTÔTˆJØ#×-Ò-¨a°Ñ3Ô3ˆJàŸ;š; |Ñ4Ô4ˆLØ'×1Ò1°!°QÑ7Ô7ˆLàÐ&¨tÔ/FÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜ØÔ$ð 	IØ0:¸LÐ0HÐ˜Tœ_Ñ-å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð /3¬mÐD�DÔ*Ð*ÀØ”LØÔ.ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(rH   rp   )r<   r=   r>   r?   r2   rs   rf   r@   rF   rC   rD   rE   r   r"   r   rr   rt   ru   s   @rI   r)  r)  Ÿ  sõ   ø€ € € € € ØGÐGð/
Ð/ð /
¸Cð /
ð /
ð /
ð /
ð /
ð /
ðn )-ð=)ð =)à”|ð=)ð #œ\ð=)ð œ tÑ+ð	=)ð
 ˜s E¨%¬,¸¼Ð*DÔ$EÐEÔFð=)ð  ™ð=)ð Ð-Ô.ð=)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð=)ð =)ð =)ð =)ð =)ð =)ð =)ð =)rH   r)  c                   óh   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        dej        dej        fd„Zˆ xZ	S )	ÚGemma4TextExpertsz2Collection of expert weights stored as 3D tensors.rY   c                 ó´  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          j        t          j
        | j        d| j        z  | j        ¦  «        ¦  «        | _        t          j        t          j
        | j        | j        | j        ¦  «        ¦  «        | _        t          |j                 | _        d S )Nr-   )re   rf   Únum_expertsr•   Ú
hidden_dimÚmoe_intermediate_sizeÚintermediate_dimr   r~   r@   ÚemptyÚgate_up_projr�  r   r‚  r  r"  s     €rI   rf   zGemma4TextExperts.__init__  s£   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ Ô,ˆŒØ &Ô <ˆÔÝœL­¬°TÔ5EÀqÈ4ÔK`ÑG`ÐbfÔbqÑ)rÔ)rÑsÔsˆÔÝœ¥e¤k°$Ô2BÀDÄOÐUYÔUjÑ&kÔ&kÑlÔlˆŒÝ˜VÔ5Ô6ˆŒˆˆrH   rO   Útop_k_indexÚtop_k_weightsr\   c                 ó€  — t          j        |¦  «        }t          j        ¦   «         5  t           j        j                             || j        ¬¦  «        }|                     ddd¦  «        }t          j        | 	                    d¬¦  «        d¦  «         
                    ¦   «         }d d d ¦  «         n# 1 swxY w Y   |D ]þ}|d         }|| j        k    rŒt          j        ||         ¦  «        \  }}	||	         }
t          j                             |
| j        |         ¦  «                             dd¬¦  «        \  }}|                      |¦  «        |z  }t          j                             || j        |         ¦  «        }|||	|d f         z  }|                     d|	|                     |j        ¦  «        ¦  «         Œÿ|S )N)Únum_classesr-   r/   r   )r‚   éþÿÿÿr¨   r‚   )r@   Ú
zeros_liker²   r   r	   rn  r<  rä   ÚgreaterÚsumÚnonzerorX  ri   rA  Úchunkr  r�  Ú
index_add_r«   rª   )rm   rO   rB  rC  Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess                 rI   rr   zGemma4TextExperts.forward   sø  € õ $Ô.¨}Ñ=Ô=ÐÝŒ]‰_Œ_ð 	Sð 	SÝœ(Ô-×5Ò5°kÈtÔO_Ð5Ñ`Ô`ˆKØ%×-Ò-¨a°°AÑ6Ô6ˆKÝœ {§¢¸8 Ñ'DÔ'DÀaÑHÔH×PÒPÑRÔRˆJð	Sð 	Sð 	Sñ 	Sô 	Sð 	Sð 	Sð 	Sð 	Sð 	Sð 	Søøøð 	Sð 	Sð 	Sð 	Sð
 %ð 
	nð 
	nˆJØ# AœˆJØ˜TÔ-Ò-Ð-ØÝ#(¤;¨{¸:Ô/FÑ#GÔ#GÑ ˆI�yØ)¨)Ô4ˆMÝ”}×+Ò+¨M¸4Ô;LÈZÔ;XÑYÔY×_Ò_Ð`aÐgiÐ_ÑjÔj‰HˆD�"Ø$(§K¢K°Ñ$5Ô$5¸Ñ$:Ð!Ý$&¤M×$8Ò$8Ð9NÐPTÔP^Ð_iÔPjÑ$kÔ$kÐ!Ø$9¸MÈ)ÐU^Ð`dÐJdÔ<eÑ$eÐ!Ø×*Ò*¨1¨iÐ9N×9QÒ9QÐReÔRkÑ9lÔ9lÑmÔmÐmÐmà"Ð"s   ¨A>B2Â2B6Â9B6)
r<   r=   r>   r?   r2   rf   r@   rF   rr   rt   ru   s   @rI   r:  r:    s�   ø€ € € € € à<Ð<ð7Ð/ð 7ð 7ð 7ð 7ð 7ð 7ð#à”|ð#ð ”\ð#ð ”|ð	#ð
 
Œð#ð #ð #ð #ð #ð #ð #ð #rH   r:  c                   ób   ‡ — e Zd Zdefˆ fd„Zdej        deej        ej        f         fd„Zˆ xZ	S )ÚGemma4TextRouterrY   c                 óò  •— t          ¦   «                              ¦   «          || _        |j        | _        | j        dz  | _        |j        | _        t          | j        | j        d¬¦  «        | _        t          j
        |j        |j        d¬¦  «        | _        t          j        t          j        | j        ¦  «        ¦  «        | _        t          j        t          j        |j        ¦  «        ¦  «        | _        d S )Nr„   Fr4  r^   )re   rf   rY   r•   Úscalar_root_sizer  rz   rw   r  r   rh   r<  Úprojr~   r@   r   ÚscaleÚper_expert_scaler"  s     €rI   rf   zGemma4TextRouter.__init__<  sÁ   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ $Ô 0°$Ñ 6ˆÔØÔ&ˆŒå! $Ô"2¸¼ÈUÐSÑSÔSˆŒ	Ý”I˜fÔ0°&Ô2DÈ5ÐQÑQÔQˆŒ	Ý”\¥%¤*¨TÔ-=Ñ">Ô">Ñ?Ô?ˆŒ
Ý "¤­U¬Z¸Ô8JÑ-KÔ-KÑ LÔ LˆÔÐÐrH   rO   r\   c                 óx  — |                       |¦  «        }|| j        z  | j        z  }|                      |¦  «        }t          j                             |dt          j        ¬¦  «        }t          j	        || j
        j        d¬¦  «        \  }}||                     dd¬¦  «        z  }|| j        |         z  }|||fS )Nr‚   râ   )rr  ry   Tri  )r  r\  rZ  r[  r   r	   ré   r@   rê   ÚtopkrY   Útop_k_expertsrI  r]  )rm   rO   Úexpert_scoresÚrouter_probabilitiesrC  rB  s         rI   rr   zGemma4TextRouter.forwardH  sÄ   € ØŸ	š	 -Ñ0Ô0ˆØ%¨¬
Ñ2°TÔ5JÑJˆàŸ	š	 -Ñ0Ô0ˆå!œ}×4Ò4°]ÈÕRWÔR_Ð4Ñ`Ô`Ðõ &+¤ZØ ØŒkÔ'Øð&
ñ &
ô &
Ñ"ˆ�{ð 	˜×*Ò*¨r¸4Ð*Ñ@Ô@Ñ@ˆð &¨Ô(=¸kÔ(JÑJˆà# ]°KÐ?Ð?rH   )
r<   r=   r>   r2   rf   r@   rF   rE   rr   rt   ru   s   @rI   rX  rX  ;  sˆ   ø€ € € € € ð
MÐ/ð 
Mð 
Mð 
Mð 
Mð 
Mð 
Mð@ U¤\ð @°e¸E¼LÈ%Ì,Ð<VÔ6Wð @ð @ð @ð @ð @ð @ð @ð @rH   rX  c                   óì   ‡ — e Zd Zdeez  defˆ fd„Z	 	 	 	 	 	 ddej        dej        de	e
eej        ej        f         f         dz  dej        d	ej        dz  d
ej        dz  dedz  dej        fd„Zˆ xZS )ÚGemma4TextDecoderLayerrY   rµ   c                 óˆ  •— t          ¦   «                              ¦   «          || _        |j        | _        || _        t          ||¬¦  «        | _        t          ||¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        |                      dt!          j        d¦  «        ¦  «         |j        | _        | j        rƒt&          |j                 | _        t-          j        | j        | j        d¬¦  «        | _        t-          j        | j        | j        d¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        |j        | _        | j        rŠt9          |¦  «        | _        t=          |¦  «        | _        t          | j        |j
        ¬¦  «        | _         t          | j        |j
        ¬¦  «        | _!        t          | j        |j
        ¬¦  «        | _"        d S d S )Nrõ  rö  Úlayer_scalarr/   Fr^   )#re   rf   rY   r•   rµ   r)  rB  r  r÷  rw   r  rø  rù  rú  rû  rj   r@   r   Úhidden_size_per_layer_inputr   r‚  r  r   rh   Úper_layer_input_gateÚper_layer_projectionÚpost_per_layer_input_normÚenable_moe_blockrX  Úrouterr:  ÚexpertsÚpost_feedforward_layernorm_1Úpost_feedforward_layernorm_2Úpre_feedforward_layernorm_2rÊ   s      €rI   rf   zGemma4TextDecoderLayer.__init__a  sÿ  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ"ˆŒÝ,°FÀiÐPÑPÔPˆŒÝ  ¨Ñ3Ô3ˆŒÝ,¨TÔ-=À6ÔCVÐWÑWÔWˆÔÝ(5°dÔ6FÈFÔL_Ð(`Ñ(`Ô(`ˆÔ%Ý)6°tÔ7GÈVÔM`Ð)aÑ)aÔ)aˆÔ&Ý*7¸Ô8HÈfÔNaÐ*bÑ*bÔ*bˆÔ'Ø×Ò˜^­U¬Z¸©]¬]Ñ;Ô;Ð;à+1Ô+MˆÔ(ØÔ+ð 	fÝ  Ô!9Ô:ˆDŒKÝ(*¬	°$Ô2BÀDÔDdÐkpÐ(qÑ(qÔ(qˆDÔ%Ý(*¬	°$Ô2RÐTXÔTdÐkpÐ(qÑ(qÔ(qˆDÔ%Ý-:¸4Ô;KÐQWÔQdÐ-eÑ-eÔ-eˆDÔ*à &Ô 7ˆÔØÔ ð 	hÝ*¨6Ñ2Ô2ˆDŒKÝ,¨VÑ4Ô4ˆDŒLÝ0=¸dÔ>NÐTZÔTgÐ0hÑ0hÔ0hˆDÔ-Ý0=¸dÔ>NÐTZÔTgÐ0hÑ0hÔ0hˆDÔ-Ý/<¸TÔ=MÐSYÔSfÐ/gÑ/gÔ/gˆDÔ,Ð,Ð,ð	hð 	hrH   NrO   Úper_layer_inputr;   rß   rU   r¯   rN   r\   c           
      óp  — |}	|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}
|                      |¦  «        }|	|z   }|}	|                      |¦  «        }|                      |¦  «        }| j        r¯|                      |¦  «        }|	                     d|	j        d         ¦  «        }|  	                    |¦  «        \  }
}}|  
                    |¦  «        }|                      |||¦  «        }|                     |	j        ¦  «        }|                      |¦  «        }||z   }|                      |¦  «        }|	|z   }| j        r`|}	|                      |¦  «        }|                      |¦  «        }||z  }|                      |¦  «        }|                      |¦  «        }|	|z   }|| j        z  }|S )N)rO   rß   rU   r;   r¯   rN   r‚   rG   )rø  rB  rù  rú  r÷  rk  rn  rÏ   rÌ   rl  rp  rm  ro  rû  rg  rh  r  ri  rj  rf  )rm   rO   rq  r;   rß   rU   r¯   rN   rG  r&  rì   Úhidden_states_1Úhidden_states_flatrC  rB  Úhidden_states_2s                   rI   rr   zGemma4TextDecoderLayer.forward}  só  € ð !ˆà×,Ò,¨]Ñ;Ô;ˆØ)˜4œ>ð 
Ø'Ø 3Ø)Ø-Ø%Ø+ð
ð 
ð ð
ð 
Ñˆ�qð ×5Ò5°mÑDÔDˆØ  =Ñ0ˆà ˆØ×6Ò6°}ÑEÔEˆØŸš Ñ/Ô/ˆàÔ ð 	>Ø"×?Ò?ÀÑNÔNˆOð "*×!1Ò!1°"°h´nÀRÔ6HÑ!IÔ!IÐØ,0¯KªKÐ8JÑ,KÔ,KÑ)ˆAˆ}˜kØ"×>Ò>Ð?QÑRÔRˆOØ"Ÿlšl¨?¸KÈÑWÔWˆOØ-×5Ò5°h´nÑEÔEˆOØ"×?Ò?ÀÑPÔPˆOð ,¨oÑ=ˆMà×7Ò7¸ÑFÔFˆØ  =Ñ0ˆàÔ+ð 	5Ø$ˆHØ ×5Ò5°mÑDÔDˆMØ ŸKšK¨Ñ6Ô6ˆMØ)¨OÑ;ˆMØ ×5Ò5°mÑDÔDˆMØ ×:Ò:¸=ÑIÔIˆMØ$ }Ñ4ˆMà˜Ô*Ñ*ˆØÐrH   )NNNNNN)r<   r=   r>   r2   r3   rs   rf   r@   rF   rC   rD   rE   rñ  r   rr   rt   ru   s   @rI   rd  rd  `  s  ø€ € € € € ðhÐ/Ð2DÑDð hÐQTð hð hð hð hð hð hð> )-ØPTØ,0Ø.2Ø04Ø(,ð9ð 9à”|ð9ð œð9ð ˜s E¨%¬,¸¼Ð*DÔ$EÐEÔFÈÑMð	9ð
 #œ\ð9ð œ tÑ+ð9ð Ô&¨Ñ-ð9ð  ™ð9ð 
Œð9ð 9ð 9ð 9ð 9ð 9ð 9ð 9rH   rd  c            	       óP   ‡ — e Zd ZdZd
dedededefˆ fd„Zdej        fˆ fd	„Z	ˆ xZ
S )ÚGemma4TextScaledWordEmbeddingz\
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    r‘   Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scalec                 ó°   •— t          ¦   «                              |||¦  «         || _        |                      dt	          j        |¦  «        d¬¦  «         d S )Nr{  Fr“   )re   rf   Úscalar_embed_scalerj   r@   rk   )rm   rx  ry  rz  r{  rn   s        €rI   rf   z&Gemma4TextScaledWordEmbedding.__init__¾  sS   ø€ Ý‰Œ×Ò˜¨¸ÑDÔDÐDØ"-ˆÔØ×Ò˜]­E¬L¸Ñ,EÔ,EÐRWÐÑXÔXÐXÐXÐXrH   Ú	input_idsc                 ó�   •— t          ¦   «                              |¦  «        | j                             | j        j        ¦  «        z  S rp   )re   rr   r{  r«   r€   rª   )rm   r~  rn   s     €rI   rr   z%Gemma4TextScaledWordEmbedding.forwardÃ  s4   ø€ Ý‰wŒw�Š˜yÑ)Ô)¨DÔ,<×,?Ò,?ÀÄÔ@QÑ,RÔ,RÑRÐRrH   )r‘   )r<   r=   r>   r?   rs   rl   rf   r@   rF   rr   rt   ru   s   @rI   rw  rw  ¹  s¦   ø€ € € € € ðð ðYð Y sð Y¸3ð YÈSð YÐ_dð Yð Yð Yð Yð Yð Yð
S ¤ð Sð Sð Sð Sð Sð Sð Sð Sð Sð SrH   rw  c            	       óö   ‡ — e Zd ZU eed<   dZdZg d¢ZddgZdZ	dZ
dZdZdZdZdZ ej        ¦   «         ˆ fd	„¦   «         Zd
„ Zd„ Z	 	 	 ddedz  dedz  dedej        fˆ fd„Z	 	 	 ddedz  dedz  defd„Zˆ xZS )ÚGemma4PreTrainedModelrY   ÚmodelT)rd  ró  rJ  r>  rN   r;   N)ÚimageÚtextÚvideoÚaudioc                 óf	  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rt	          j        |j        ¦  «         d S t          |t          ¦  «        r¨d}d}|j        dz  }t          j
        ||z  ¦  «        t          |dz
  d¦  «        z  }|t          j        t          j        |¦  «        | z  ¦  «        z  }t	          j        |j        |                     d¦  «                             d¦  «        ¦  «         d S t          |t$          ¦  «        r:t	          j        |j        |j        ¦  «         t	          j        |j        ¦  «         d S t          |t0          ¦  «        rž|j                             ¦   «         D ]‚\  }}d|i}	|dk    r|j        |         dk    rd	|	d
<    ||j        fi |	¤Ž\  }
}t	          j        t;          ||› d�¦  «        |
¦  «         t	          j        t;          ||› d�¦  «        |
¦  «         Œƒd S t          |t<          ¦  «        rm|j        dk    rt>          |j                 n|j         } ||j        ¦  «        \  }}t	          j        |j!        |¦  «         t	          j        |j"        |¦  «         d S t          |tF          ¦  «        r!t	          j        |j$        |j%        ¦  «         d S t          |tL          ¦  «        r4t	          j        |j'        ¦  «         t	          j        |j(        ¦  «         d S t          |tR          ¦  «        rF| j        j*        }t	          j+        |j,        d|¬¦  «         t	          j+        |j-        d|¬¦  «         d S t          |t\          ¦  «        rt	          j        |j/        ¦  «         d S t          |t`          ¦  «        r§|j1        r t	          j        |j2        tg          d¦  «         ¦  «         t	          j        |j4        tg          d¦  «        ¦  «         t	          j        |j5        tg          d¦  «         ¦  «         t	          j        |j6        tg          d¦  «        ¦  «         d S t          |tn          ¦  «        r@|j        j8        r6t	          j        |j9        ¦  «         t	          j        |j:        ¦  «         d S d S d S )Nr‘   r’   r-   r/   r   rå  r  r  r  r  r  r  rŒ  rV  )r†   Ústdra   );re   Ú_init_weightsr§  rJ  ÚinitÚones_rO  rŽ   r•   rš   r›   rœ   r@   r�   rž   Úcopy_r�   rŸ   r´   Ú	constant_r·   r¹   Úzeros_rÉ   r  r  Úitemsr‹  rY   r›  rˆ  r   r’  r‰  r�  rw  r{  r}  rX  r\  r]  r:  Úinitializer_rangeÚnormal_rA  r�  rd  rf  rX   rg   r`   rl   rb   rc   rd   ÚGemma4VisionModelÚstandardizeÚstd_biasÚ	std_scale)rm   rÃ  r    r¡   r¢   r£   r�   rå  r•  r!  r"  rì   Úrope_fnÚbuffer_valuerˆ  rn   s                  €rI   r‰  z#Gemma4PreTrainedModel._init_weightsÜ  sŽ  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ7Ñ8Ô8ð 0	)ÝŒJ�vÔ6Ñ7Ô7Ð7Ð7Ð7Ý˜Õ @ÑAÔAð .	)ØˆMØ#ˆMØ#Ô/°1Ñ4ˆNÝ&*¤h¨}¸}Ñ/LÑ&MÔ&MÕPSÐTbÐefÑTfÐhiÑPjÔPjÑ&jÐ#Ø*­U¬Yµu´|ÀNÑ7SÔ7SÐWnÐVnÑ7nÑ-oÔ-oÑoˆNÝŒJ�vÔ,¨n×.FÒ.FÀqÑ.IÔ.I×.SÒ.SÐTUÑ.VÔ.VÑWÔWÐWÐWÐWÝ˜Õ 4Ñ5Ô5ð '	)ÝŒN˜6œ>¨6Ô+KÑLÔLÐLÝŒK˜Ô,Ñ-Ô-Ð-Ð-Ð-Ý˜Õ 9Ñ:Ô:ð $	)Ø,2Ô,@×,FÒ,FÑ,HÔ,Hð ^ð ^Ñ(�
˜LØ'3°ZÐ&@Ð#ØÐ!1Ò1Ð1°fÔ6FÀzÔ6RÐVdÒ6dÐ6dØ:KÐ'¨Ñ7à#/ <°´Ð#UÐ#UÐATÐ#UÐ#UÑ �˜qÝ”
�7 6¨jÐ+CÐ+CÐ+CÑDÔDÀmÑTÔTÐTÝ”
�7 6¨jÐ+LÐ+LÐ+LÑMÔMÈ}Ñ]Ô]Ð]Ð]ð^ð ^õ ˜Õ ;Ñ<Ô<ð 	)ð Ô# yÒ0Ð0õ $ FÔ$4Ô5Ð5àÔ;ð ð
 &˜g f¤mÑ4Ô4‰OˆL˜!ÝŒJ�v”¨Ñ5Ô5Ð5ÝŒJ�vÔ/°Ñ>Ô>Ð>Ð>Ð>Ý˜Õ =Ñ>Ô>ð 	)ÝŒN˜6Ô-¨vÔ/HÑIÔIÐIÐIÐIÝ˜Õ 0Ñ1Ô1ð 	)ÝŒJ�v”|Ñ$Ô$Ð$ÝŒJ�vÔ.Ñ/Ô/Ð/Ð/Ð/Ý˜Õ 1Ñ2Ô2ð 	)Ø”+Ô/ˆCÝŒL˜Ô,°3¸CÐ@Ñ@Ô@Ð@ÝŒL˜Ô)°¸Ð=Ñ=Ô=Ð=Ð=Ð=Ý˜Õ 6Ñ7Ô7ð 		)ÝŒJ�vÔ*Ñ+Ô+Ð+Ð+Ð+Ý˜Õ 5Ñ6Ô6ð 	)¸6Ô;Uð 	)ÝŒN˜6Ô+­e°E©l¬l¨]Ñ;Ô;Ð;ÝŒN˜6Ô+­U°5©\¬\Ñ:Ô:Ð:ÝŒN˜6Ô,­u°U©|¬|¨mÑ<Ô<Ð<ÝŒN˜6Ô,­e°E©l¬lÑ;Ô;Ð;Ð;Ð;Ý˜Õ 1Ñ2Ô2ð 	)°v´}Ô7Pð 	)ÝŒK˜œÑ(Ô(Ð(ÝŒJ�vÔ'Ñ(Ô(Ð(Ð(Ð(ð	)ð 	)ð 	)ð 	)rH   c                 ó   — | j         j        S rp   ©Ú
base_modelÚembed_tokens_per_layer©rm   s    rI   Úget_per_layer_input_embeddingsz4Gemma4PreTrainedModel.get_per_layer_input_embeddings  s   € ØŒÔ5Ð5rH   c                 ó   — || j         _        d S rp   r™  ©rm   rÆ  s     rI   Úset_per_layer_input_embeddingsz4Gemma4PreTrainedModel.set_per_layer_input_embeddings  s   € Ø16ˆŒÔ.Ð.Ð.rH   Únew_num_tokensÚpad_to_multiple_ofÚmean_resizingr\   c                 ó~   •— t          ¦   «                              |||¬¦  «        }|                      |||¦  «         |S )N)r¡  r¢  r£  )re   Úresize_token_embeddingsÚ_resize_per_layer_embeddings)rm   r¡  r¢  r£  r  rn   s        €rI   r¥  z-Gemma4PreTrainedModel.resize_token_embeddings  sM   ø€ õ ™œ×7Ò7Ø)Ø1Ø'ð 8ñ 
ô 
ˆð
 	×)Ò)¨.Ð:LÈmÑ\Ô\Ð\ØÐrH   c                 óš  — | j         | j                             ¦   «         _        | j                             ¦   «         j        r‰|                      ¦   «         }|                      ||||¦  «        }t          |d¦  «        r|j        }t          ||¦  «         | 
                    |j        j        ¦  «         |                      |¦  «         d S d S )NÚ_hf_hook)Ú
vocab_sizerY   Úget_text_configÚvocab_size_per_layer_inputrg  r�  Ú_get_resized_embeddingsrä  r¨  r4   Úrequires_grad_r€   r}   r   )rm   r¡  r¢  r£  r›  Únew_embeddings_per_layerÚhooks          rI   r¦  z2Gemma4PreTrainedModel._resize_per_layer_embeddings%  sÛ   € ð DHÄ?ˆŒ×#Ò#Ñ%Ô%Ô@ØŒ;×&Ò&Ñ(Ô(ÔDð 		JØ%)×%HÒ%HÑ%JÔ%JÐ"Ø'+×'CÒ'CØ&¨Ð8JÈMñ(ô (Ð$õ Ð-¨zÑ:Ô:ð CØ-Ô6�Ý"Ð#;¸TÑBÔBÐBØ$×3Ò3Ð4JÔ4QÔ4_Ñ`Ô`Ð`Ø×/Ò/Ð0HÑIÔIÐIÐIÐIð		Jð 		JrH   )NNT)r<   r=   r>   r1   rB   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendÚ_can_record_outputsÚinput_modalitiesr@   r²   r‰  r�  r   rs   rŒ   r   Ú	Embeddingr¥  r¦  rt   ru   s   @rI   r�  r�  Ç  s~  ø€ € € € € € àÐÐÑØÐØ&*Ð#ðð ð Ðð $5Ð6HÐ"IÐØÐØ€NØÐà!ÐØ"&ÐØÐØ:Ðà€U„]�_„_ð2)ð 2)ð 2)ð 2)ñ „_ð2)ðh6ð 6ð 6ð7ð 7ð 7ð
 &*Ø)-Ø"ð	ð à˜d™
ðð   $™Jðð ð	ð
 
Œðð ð ð ð ð ð  &*Ø)-Ø"ð	Jð Jà˜d™
ðJð   $™JðJð ð	Jð Jð Jð Jð Jð Jð Jð JrH   r�  zAThe base Gemma 4 language model without a language modeling head.c                   ó¤  ‡ — e Zd ZU eed<   dZ eed¬¦  «        ee	dœZ
defˆ fd„Zeee	 	 	 	 	 	 	 ddej        dz  d	ej        dz  d
ej        dz  dedz  dej        dz  dej        dz  dedz  dee         defd„¦   «         ¦   «         ¦   «         Zdej        dz  dej        dz  dej        fd„Z	 ddej        dej        dz  dej        fd„Zˆ xZS )ÚGemma4TextModelrY   )r„  r   )r3  )Úrouter_logitsrO   rP   c                 ó0  •‡‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          ‰j        ‰j        | j        | j        j        dz  ¬¦  «        | _        t          j
        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t#          ‰¦  «        | _        d| _        t)          | j        j        ¦  «        | _        ‰j        | _        | j        r›t          ‰j        ‰j        ‰j        z  | j        ‰j        dz  ¬¦  «        | _        d| _        t          j        ‰j        ‰j        ‰j        z  d¬¦  «        | _        ‰j        dz  | _        t          ‰j        ‰j        ¬¦  «        | _        g | _        tA          | j        ¦  «        D ]7\  Š}|j!        j"        r&| j         #                    ˆfd	„d
D ¦   «         ¦  «         Œ8|  $                    ¦   «          d S )Nr_  )r{  c                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS rG   )rd  ©rÒ  rµ   rY   s     €rI   rÖ  z,Gemma4TextModel.__init__.<locals>.<listcomp>L  s$   ø€ ÐhÐhÐh¸9Õ# F¨IÑ6Ô6ÐhÐhÐhrH   rö  FgÍ;fž æ?r^   r„   c                 ó    •— g | ]
}d ‰› d|› �‘ŒS )zlayers.z.self_attn.rG   )rÒ  Únamer®  s     €rI   rÖ  z,Gemma4TextModel.__init__.<locals>.<listcomp>l  s*   ø€ ÐiÐiÐi¸Ð3˜qÐ3Ð3¨TÐ3Ð3ÐiÐiÐirH   )rÄ   rÅ   rê  rë  )%re   rf   Úpad_token_idrz  r©  rw  r•   rY   Úembed_tokensr   r  r©  r  r  rw   r  r  r  r  Úgradient_checkpointingr  râ  Úunique_layer_typesrg  r«  r›  Úper_layer_input_scalerh   Úper_layer_model_projectionÚ per_layer_model_projection_scaleÚper_layer_projection_normÚ"_keys_to_ignore_on_load_unexpectedÚ	enumeraterB  r  ÚextendÚ	post_init)rm   rY   Úlayerr®  rn   s    ` @€rI   rf   zGemma4TextModel.__init__B  s&  øøø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒõ :ØÔ˜vÔ1°4Ô3CÐQUÔQ\ÔQhÐjmÑQmð
ñ 
ô 
ˆÔõ ”mØhÐhÐhÐhÍÈfÔNfÑHgÔHgÐhÑhÔhñ
ô 
ˆŒõ " &Ô"4¸&Ô:MÐNÑNÔNˆŒ	Ý3°FÑ;Ô;ˆŒØ&+ˆÔ#Ý"% d¤kÔ&=Ñ">Ô">ˆÔð
 ,2Ô+MˆÔ(ØÔ+ð 	xÝ*GØÔ1ØÔ(¨6Ô+MÑMØÔ Ø"Ô>ÀÑCð	+ñ +ô +ˆDÔ'ð *3ˆDÔ&Ý.0¬iØÔ"ØÔ(¨6Ô+MÑMØð/ñ /ô /ˆDÔ+ð
 5;Ô4FÈÑ4LˆDÔ1Ý-:¸6Ô;]ÐciÔcvÐ-wÑ-wÔ-wˆDÔ*ð 35ˆÔ/Ý! $¤+Ñ.Ô.ð 	ð 	‰HˆAˆuØŒÔ1ð ØÔ7×>Ò>ØiÐiÐiÐiÐ@hÐiÑiÔiñô ð øð
 	�ŠÑÔÐÐÐrH   Nr~  rU   r¯   rN   r  Úper_layer_inputsÚ	use_cacherG  r\   c           
      óZ  — |du |duz  rt          d¦  «        ‚|�|�t          d¦  «        ‚|�|                      |¦  «        }| j        r.|€|                      ||¦  «        }|                      ||¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}	t          j	        |j
        d         |j        ¬¦  «        |	z   }|                     d¦  «        }t          |x}
t          ¦  «        s&| j        ||||dœ}t          di |¤Žt!          di |¤Žd	œ}
|}i }| j        D ]}|                      |||¦  «        ||<   Œ|                     d
t)          ¦   «         ¦  «        }t+          | j        d| j        j        …         ¦  «        D ]W\  }}|�|dd…dd…|dd…f         nd} |||f||| j        j        |                  |
| j        j        |                  ||dœ|¤Ž}ŒX|                      |¦  «        }t5          |||                     dd¦  «        r|nd¬¦  «        S )ak  
        per_layer_inputs (`torch.Tensor`, *optional*):
            Pre-computed per-layer input text embeddings of shape `(batch_size, sequence_length, num_hidden_layers,
            hidden_size_per_layer_input)`. When provided, these are used directly instead of being computed from `input_ids`
            via `get_per_layer_inputs()` in the text model. If calling the `forward` with `inputs_embeds` instead of `input_ids`,
            you should probably precompute them and forward them along `inputs_embeds`, otherwise recomputing them needs
            to reverse the main embedding, which is expensive.
        Nú:You must specify exactly one of input_ids or inputs_embedsú<You cannot specify per_layer_inputs if input_ids is provided©rY   r   r/   r¥   ©rY   r  rU   rN   r¯   ©r  r+  r;   )r;   rß   rU   r¯   rN   Úreturn_shared_kv_statesF)r	  rN   r;   rG   )rl  rÅ  rg  Úget_per_layer_inputsÚproject_per_layer_inputsr   rY   Úget_seq_lengthr@   rž   rÌ   r¦   rŸ   r§  rC   r   r   rÇ  r  Úpopr   rÍ  r  r  râ  r  rR   Úget)rm   r~  rU   r¯   rN   r  rÑ  rÒ  rG  Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsrO   rß   rå  r;   r®  r
  rq  s                      rI   rr   zGemma4TextModel.forwardr  sã  € ð, ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ð%5Ð%AÝÐ[Ñ\Ô\Ð\àÐ Ø ×-Ò-¨iÑ8Ô8ˆMàÔ+ð 	^ØÐ'Ø#'×#<Ò#<¸YÈÑ#VÔ#VÐ Ø#×<Ò<¸]ÐL\Ñ]Ô]Ðàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLõ °Ð?Ð-ÅÑFÔFð 	ð œ+Ø!.Ø"0Ø#2Ø ,ðð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ%FÐ%UÐ%UÈÐ%UÐ%Uð#ð #Ðð &ˆØ ÐØÔ1ð 	gð 	gˆJØ.2¯oªo¸mÈ\Ð[eÑ.fÔ.fÐ 
Ñ+Ð+ð
 "Ÿ:š:Ð&8½(¹*¼*ÑEÔEÐõ !*¨$¬+Ð6U¸¼Ô8UÐ6UÔ*VÑ WÔ Wð 	ð 	ÑˆAˆ}Ø>NÐ>ZÐ.¨q¨q¨q°!°!°!°Q¸¸¸¨zÔ:Ð:Ð`dˆOà)˜MØØð	ð "2Ø$7¸¼Ô8OÐPQÔ8RÔ$SØ2°4´;Ô3JÈ1Ô3MÔNØ)Ø /ð	ð 	ð ð	ð 	ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå,Ø+Ø+Ø17·²Ð<UÐW\Ñ1]Ô1]ÐgÐ-Ð-Ðcgð
ñ 
ô 
ð 	
rH   c                 óh  — | j         st          d| j        › �¦  «        ‚|€Ût          j        ¦   «         5  |dd…dd…ddd…f         | j        j        dddd…dd…f         | j        j        dz  z  k                         d¬¦  «         	                    ¦   «         dd…df         }	 | 
                    |j        dd…         ¦  «        }n# t          $ r t          d¦  «        ‚w xY w	 ddd¦  «         n# 1 swxY w Y    |                      |¦  «        j        g |j        ¢| j        j        ‘| j         ‘R Ž S )a  Compute the token-identity component of Per-Layer Embeddings (PLE).

        Looks up `input_ids` in `embed_tokens_per_layer` (a scaled embedding that multiplies
        by `sqrt(hidden_size_per_layer_input)`) and reshapes the packed output from
        `[batch, seq, num_hidden_layers * hidden_size_per_layer_input]` to
        `[batch, seq, num_hidden_layers, hidden_size_per_layer_input]`.

        If only `inputs_embeds` is provided (no `input_ids`), reverses the main embedding
        to recover `input_ids` for the PLE lookup.
        z}Attempting to call get_per_layer_inputs() from a model initialized with a config that does not support per-layer embeddings. Nr_  r
   r¨   r-   a)  It seems like you tried to call `forward` from `inputs_embeds` without providing `input_ids`, and that the `inputs_embeds` you provided do not exactly match the embedding weights. Since Gemma4 needs to reverse the embedding to compute another embedding, make sure you provide exact `inputs_embeds`)rg  ÚRuntimeErrorrY   r@   r²   rÅ  r€   r•   rp  rJ  rÛ   rÌ   r›  rÏ   r  )rm   r~  r  s      rI   rÚ  z$Gemma4TextModel.get_per_layer_inputsÐ  sÉ  € ð Ô/ð 	Ýð8Ø*.¬+ð8ð 8ñô ð ð ÐÝ”‘”ð ð ð & a a a¨¨¨¨D°!°!°! mÔ4ØÔ,Ô3°D¸$ÀÀÀÀ1À1À1Ð4DÔEÈÌÔH_ÐadÑHdÑdòe÷ ’S˜Q�S‘Z”Zß’W‘Y”Y˜q˜q˜q !˜tô%ð ðØ )§¢¨}Ô/BÀ2ÀAÀ2Ô/FÑ GÔ G�I�IøÝ#ð ð ð Ý&ðrñô ð ðøøøð ðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð$ >ˆt×*Ò*¨9Ñ5Ô5Ô=ð 
ØŒ_ð
àŒKÔ)ð
ð Ô,ð
ð 
ð 
ð 	
s*   ´A/C1Â$"CÃC1ÃC!Ã!C1Ã1C5Ã8C5c                 ó  — | j         st          d| j        › �¦  «        ‚|                      |¦  «        | j        z  } |j        g |j        dd…         ¢| j        j        ‘| j         ‘R Ž }|                      |¦  «        }|€|S ||z   | j	        z  S )aŒ  Compute the context-aware component of PLE and combine with token-identity.

        Projects `inputs_embeds` through `per_layer_model_projection` (Linear), scales by
        `1/sqrt(hidden_size)`, reshapes to `[batch, seq, num_layers, ple_dim]`, and normalizes
        with `per_layer_projection_norm` (RMSNorm).

        If `per_layer_inputs` (the token-identity component from `get_per_layer_inputs()`)
        is provided, combines both: `(context_projection + token_identity) * (1/sqrt(2))`.
        If `per_layer_inputs` is None (e.g. for multimodal inputs where input_ids are not
        available), returns just the context projection.
        z�Attempting to call project_per_layer_inputs() from a model initialized with a config that does not support per-layer embeddings. Nr‚   )
rg  rã  rY   rÉ  rÊ  rÏ   rÌ   r  rË  rÈ  )rm   r  rÑ  ri  s       rI   rÛ  z(Gemma4TextModel.project_per_layer_inputsü  s×   € ð  Ô/ð 	Ýð@Ø26´+ð@ð @ñô ð ð
  $×>Ò>¸}ÑMÔMÐPTÔPuÑuÐØ;Ð3Ô;ð  
ØÔ   " Ô%ð 
àŒKÔ)ð 
ð Ô,ð 
ð  
ð  
Ðð
  $×=Ò=Ð>RÑSÔSÐàÐ#Ø'Ð'à$Ð'7Ñ7¸4Ô;UÑUÐUrH   )NNNNNNNrp   )r<   r=   r>   r2   rB   rº  r+   rX  rd  r)  r¹  rf   r*   r,   r%   r@   rñ  rF   r   rA   rŒ   r"   r$   rR   rr   rÚ  rÛ  rt   ru   s   @rI   r½  r½  8  sü  ø€ € € € € € àÐÐÑØ Ðà'˜Ð(8ÀÐBÑBÔBØ/Ø)ðð Ðð.Ð/ð .ð .ð .ð .ð .ð .ð`  ØØð .2Ø.2Ø04Ø(,Ø26Ø04Ø!%ðY
ð Y
àÔ# dÑ*ðY
ð œ tÑ+ðY
ð Ô&¨Ñ-ð	Y
ð
  ™ðY
ð Ô(¨4Ñ/ðY
ð  œ,¨Ñ-ðY
ð ˜$‘;ðY
ð Ð+Ô,ðY
ð 
'ðY
ð Y
ð Y
ñ „^ñ „_ñ  ÔðY
ðv*
¨e¬l¸TÑ.Að *
ÐRWÔR^ÐaeÑReð *
ÐjoÔjvð *
ð *
ð *
ð *
ð^ 15ð!Vð !Và”|ð!Vð  œ,¨Ñ-ð!Vð 
Œð	!Vð !Vð !Vð !Vð !Vð !Vð !Vð !VrH   r½  z>The base Gemma 4 language model with a language modeling head.c                   óD  ‡ — e Zd ZU ddiZddiZddgdgfiZeed<   dZdefˆ fd	„Z	e
e	 	 	 	 	 	 	 	 	 ddej        d
z  dej        d
z  dej        d
z  ded
z  dej        d
z  dej        d
z  ded
z  deej        z  dej        d
z  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚGemma4ForCausalLMúlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrO   rM   rY   r‚  c                 óþ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S ©NFr^   )
re   rf   r½  r‚  r©  r   rh   r•   rè  rÏ  r"  s     €rI   rf   zGemma4ForCausalLM.__init__(  sj   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý$ VÑ,Ô,ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒð 	�ŠÑÔÐÐÐrH   Nr   r~  rU   r¯   rN   r  ÚlabelsrÒ  Úlogits_to_keeprÑ  rG  r\   c
                 óÎ  —  | j         d||||||	|dœ|
¤Ž}|j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }| j        j        �2|| j        j        z  }t          j	        |¦  «        }|| j        j        z  }d}|� | j
        ||| j        fi |
¤Ž}t          |||j        |j        |j        |j        ¬¦  «        S )aí  
        per_layer_inputs (`torch.Tensor`, *optional*):
            Pre-computed per-layer input text embeddings of shape `(batch_size, sequence_length, num_hidden_layers,
            hidden_size_per_layer_input)`. When provided, these are used directly instead of being computed from `input_ids`
            via `get_per_layer_inputs()` in the text model. If calling the `forward` with `inputs_embeds` instead of `input_ids`,
            you should probably precompute them and forward them along `inputs_embeds`, otherwise recomputing them needs
            to reverse the main embedding, which is expensive.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, Gemma4ForCausalLM

        >>> model = Gemma4ForCausalLM.from_pretrained("google/gemma-4-E2B-it")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-4-E2B-it")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```)r~  rU   r¯   rN   r  rÑ  rÒ  N)rL   rM   rN   rO   rP   r;   rG   )r‚  r	  r§  rs   Úslicerè  rY   Úfinal_logit_softcappingr@   rå   Úloss_functionr©  rK   rN   rO   rP   r;   )rm   r~  rU   r¯   rN   r  rì  rÒ  rí  rÑ  rG  ÚoutputsrO   Úslice_indicesrM   rL   s                   rI   rr   zGemma4ForCausalLM.forward1  s/  € ðP 2<°´ð 	2
ØØ)Ø%Ø+Ø'Ø-Øð	2
ð 	2
ð ð	2
ð 	2
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆØŒ;Ô.Ð:Ø˜dœkÔAÑAˆFÝ”Z Ñ'Ô'ˆFØ˜dœkÔAÑAˆFàˆØÐØ%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDå+ØØØ#Ô3Ø!Ô/ØÔ)Ø$Ô5ð
ñ 
ô 
ð 	
rH   )	NNNNNNNr   N)r<   r=   r>   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr2   rB   r°  rf   r&   r%   r@   rñ  rF   r   rA   rŒ   rs   r"   r$   rK   rr   rt   ru   s   @rI   ræ  ræ     s”  ø€ € € € € € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐÐÑØÐðÐ/ð ð ð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.Ø04ðE
ð E
àÔ# dÑ*ðE
ð œ tÑ+ðE
ð Ô&¨Ñ-ð	E
ð
  ™ðE
ð Ô(¨4Ñ/ðE
ð Ô  4Ñ'ðE
ð ˜$‘;ðE
ð ˜eœlÑ*ðE
ð  œ,¨Ñ-ðE
ð Ð+Ô,ðE
ð 
&ðE
ð E
ð E
ñ „^ñ ÔðE
ð E
ð E
ð E
ð E
rH   ræ  r-  c           
      óZ   ‡ — dt           dt           dt           dt           dt          f
ˆ fd„}|S )zL
    This creates uni/bidirectional attention mask with sliding window.
    Ú	batch_idxÚhead_idxÚq_idxÚkv_idxr\   c                 óX   •— ‰	\  }}||z
  }|dk    ||k     z  }|dk     | |k     z  }||z  S r/  rG   )
rø  rù  rú  rû  Úleft_window_sizeÚright_window_sizeÚdistÚ	left_maskÚ
right_maskr-  s
            €rI   Ú
inner_maskz0sliding_window_mask_function.<locals>.inner_mask€  sM   ø€ Ø.<Ñ+ÐÐ+à�v‰~ˆØ˜Q’Y 4Ð*:Ò#:Ñ;ˆ	Ø˜Q’h D 5Ð+<Ò#<Ñ=ˆ
Ø˜:Ñ%Ð%rH   )rs   rŒ   )r-  r  s   ` rI   Úsliding_window_mask_functionr  {  sL   ø€ ð
&�cð &­Sð &½ð &Åcð &Ídð &ð &ð &ð &ð &ð &ð ÐrH   c                   ó  ‡ — e Zd ZU dZeed<   dZdZee	dœZ
defˆ fd„Zdej        dej        fd	„Zee ed
¬¦  «        	 ddej        dej        dz  dee         deej        ej        f         fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚGemma4AudioModelznAn audio encoder based on the [Universal Speech Model](https://huggingface.co/papers/2303.01037) architecture.rY   r  zmodel.audio_tower©rO   rP   c                 óŒ  •‡— t          ¦   «                              ‰¦  «         ‰| _        t          ‰¦  «        | _        t          ‰¦  «        | _        t          j        ˆfd„t          ‰j
        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j        ‰j        d¬¦  «        | _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS rG   )r>  rÁ  s     €rI   rÖ  z-Gemma4AudioModel.__init__.<locals>.<listcomp>�  s$   ø€ ÐbÐbÐb°YÕ˜f iÑ0Ô0ÐbÐbÐbrH   Tr^   )re   rf   rY   r  Úsubsample_conv_projectionrŽ   Úrel_pos_encr   r  r©  r  r  rh   r•   Úoutput_proj_dimsÚoutput_projrÏ  r"  s    `€rI   rf   zGemma4AudioModel.__init__–  s°   øø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒå)KÈFÑ)SÔ)SˆÔ&Ý;¸FÑCÔCˆÔÝ”mØbÐbÐbÐbÅ%ÈÔH`ÑBaÔBaÐbÑbÔbñ
ô 
ˆŒõ œ9 VÔ%7¸Ô9PÐW[Ð\Ñ\Ô\ˆÔà�ŠÑÔÐÐÐrH   Úmask_4dr\   c                 óR  — |j         \  }}}}|j        }| j        j        }| j        j        dz
  }| j        j        }||z   dz
  |z  }	|	|z  }
|
|z
  }t          j        |d|d|fd¬¦  «        }|                     |d|	||
¦  «        }t          j        |||fd¬¦  «        }t          j
        |	|¬¦  «        |z  }t          j
        ||z   |z   |¬¦  «        }|dd…df         |ddd…f         z   }|dddd…ddd…f                              |dd|d¦  «        }|                     d|¦  «        S )zÜ
        Convert a standard 4D attention mask `[batch_size, 1, seq_len, seq_len]` to the 5D blocked format
        `[batch_size, 1, num_blocks, chunk_size, context_size]` expected by the chunked local attention,
        r/   r   F)rÆ  r¥   Nr‚   )rÌ   r¦   rY   r–   r—   r˜   rÍ   rÎ   rÏ   r@   rž   r¦  Úgather)rm   r  rÑ   rì   rÒ   r¦   rÀ   rÁ   rÂ   rÓ   Úpadded_seq_lenÚ
pad_amountÚmask_5dÚblock_startsÚoffsetsÚ
kv_indicess                   rI   Ú_convert_4d_mask_to_blocked_5dz/Gemma4AudioModel._convert_4d_mask_to_blocked_5d£  sd  € ð
 %,¤MÑ!ˆ
�A�w Ø”ˆà”[Ô5ˆ
Øœ;Ô=ÀÑAÐØ!œ[Ô@Ðà 
Ñ*¨QÑ.°:Ñ=ˆ
Ø# jÑ0ˆØ# gÑ-ˆ
å”%˜ ! Z°°JÐ!?ÀuÐMÑMÔMˆØ—/’/ *¨a°¸ZÈÑXÔXˆÝ”%˜Ð"2Ð4FÐ!GÈuÐUÑUÔUˆå”| J°vÐ>Ñ>Ô>ÀÑKˆÝ”,˜zÐ,<Ñ<Ð?QÑQÐZ`ÐaÑaÔaˆØ! ! ! ! T 'Ô*¨W°T¸1¸1¸1°WÔ-=Ñ=ˆ
Ø  d¨A¨A¨A¨t°Q°Q°QÐ 6Ô7×>Ò>¸zÈ1ÈbÐR\Ð^`ÑaÔaˆ
à�~Š~˜b *Ñ-Ô-Ð-rH   z&Encodes audio features to soft tokens.r5   NrU   rG  c           	      ó¤  — |                       ||¦  «        \  }}|                      |¦  «        }t          | j        ||t	          | j        j        dz
  | j        j        f¦  «        ¬¦  «        }|�|                      |¦  «        }| j        d | j        j	        …         D ]} ||f||dœ|¤Ž}Œ|  
                    |¦  «        }t          ||¬¦  «        S )Nr/   )rY   r  rU   Úand_mask_function)rU   rß   )r	  rU   )r	  r
  r   rY   r  r—   r˜   r  r  r  r  rT   )rm   r  rU   rG  rO   Úoutput_maskrß   Úencoder_layers           rI   rr   zGemma4AudioModel.forward¾  s  € ð &*×%CÒ%CÀNÐTbÑ%cÔ%cÑ"ˆ�{Ø"×.Ò.¨}Ñ=Ô=Ðå2Ø”;Ø'Ø&Ý:Ø”Ô3°aÑ7¸¼Ô9\Ð]ñô ð	
ñ 
ô 
ˆð Ð%Ø!×@Ò@ÀÑPÔPˆNà!œ[Ð)H¨4¬;Ô+HÐ)HÔIð 	ð 	ˆMØ)˜MØðà-Ø$7ðð ð ð	ð ˆMˆMð ×(Ò(¨Ñ7Ô7ˆÝ%¸ÐVaÐbÑbÔbÐbrH   rp   )r<   r=   r>   r?   r0   rB   Úmain_input_namer°  r>  r´   r¹  rf   r@   rF   r  r*   r,   r%   r"   r$   rE   rV   rr   rt   ru   s   @rI   r  r  ‹  s=  ø€ € € € € € ØxÐxàÐÐÑØ&€OØ+Ðà)Ø*ðð Ðð
Ð0ð ð ð ð ð ð ð.°e´lð .ÀuÄ|ð .ð .ð .ð .ð6  ØØ€^Ð!IÐJÑJÔJð /3ðcð càœðcð œ tÑ+ðcð Ð+Ô,ð	cð
 
ˆuŒ|˜UÔ-Ð-Ô	.ðcð cð cñ KÔJñ „_ñ  Ôðcð cð cð cð crH   r  c                   ó²   ‡ — e Zd ZdZeZeedœZdefˆ fd„Z	e
e ed¬¦  «        dej        dej        d	ee         d
efd„¦   «         ¦   «         ¦   «         Zˆ xZS )r’  zThe Gemma 4 Vision Encoder.r  rY   c                 óÊ  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¦  «        | _        t          |¦  «        | _        | j        j	        rd|  
                    dt          j        | j        j        ¦  «        ¦  «         |  
                    dt          j        | j        j        ¦  «        ¦  «         |                      ¦   «          d S )Nr”  r•  )re   rf   rJ  Úpatch_embedderrþ  Úencoderrc  ÚpoolerrY   r“  rj   r@   r@  r•   rÏ  r"  s     €rI   rf   zGemma4VisionModel.__init__ê  s·   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý7¸Ñ?Ô?ˆÔÝ*¨6Ñ2Ô2ˆŒÝ(¨Ñ0Ô0ˆŒàŒ;Ô"ð 	TØ× Ò  ­U¬[¸¼Ô9PÑ-QÔ-QÑRÔRÐRØ× Ò  ­e¬k¸$¼+Ô:QÑ.RÔ.RÑSÔSÐSà�ŠÑÔÐÐÐrH   z1Encodes image pixels to soft tokens from patches.r5   r]  rP  rG  r\   c                 óð  — | j         j        }|j        d         ||z  z  }|dk                         d¬¦  «        }|                      |||¦  «        } | j        d|| |dœ|¤Ž}|                      |j        |||¬¦  «        \  }	}
|	|
         }	| j         j        r6|	| j	         
                    ¦   «         z
  | j         
                    ¦   «         z  }	|	                     |j        ¦  «        }	t          |	¬¦  «        S )aà  
        pixel_values (`torch.FloatTensor` or `list[torch.FloatTensor]`):
            The images to encode. Either a single `[batch, channels, height, width]` tensor
            (all images same size) or a list of `[1, channels, height, width]` tensors (different sizes).
        pixel_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`):
            The patch positions as (x, y) coordinates in the image. Padding patches are indicated by (-1, -1).
        rF  r‚   r¨   )r  rU   rP  )rO   rP  rQ  ry  r  rG   )rY   Úpooling_kernel_sizerÌ   rp  r  r  r   r	  r“  r”  rl   r•  r«   rª   r   )rm   r]  rP  rG  r"  ry  rQ  r  rw  rO   Úpooler_masks              rI   rr   zGemma4VisionModel.forwardö  s,  € ð  #œkÔ=ÐØ$Ô*¨2Ô.Ð3FÐI\Ñ3\Ñ]ˆà/°2Ò5×:Ò:¸rÐ:ÑBÔBÐØ×+Ò+¨LÐ:LÐN_Ñ`Ô`ˆØ�”ð 
Ø'Ø-Ð-Ø1ð
ð 
ð ð	
ð 
ˆð &*§[¢[Ø Ô2Ø1Ø/Ø'ð	 &1ñ &
ô &
Ñ"ˆ�{ð & kÔ2ˆð Œ;Ô"ð 	]Ø*¨T¬]×-@Ò-@Ñ-BÔ-BÑBÀdÄn×FZÒFZÑF\ÔF\Ñ\ˆMØ%×(Ò(¨Ô)<Ñ=Ô=ˆå&¸ÐGÑGÔGÐGrH   )r<   r=   r>   r?   r3   rY   ró  rß  r¹  rf   r*   r,   r%   r@   rA   rñ  r"   r$   r   rr   rt   ru   s   @rI   r’  r’  á  sæ   ø€ € € € € Ø%Ð%à€Fà1Ø+ðð Ðð

Ð1ð 
ð 
ð 
ð 
ð 
ð 
ð  ØØ€^Ð!TÐUÑUÔUð)HàÔ'ð)Hð "Ô,ð)Hð Ð+Ô,ð	)Hð
 
!ð)Hð )Hð )Hñ VÔUñ „_ñ  Ôð)Hð )Hð )Hð )Hð )HrH   r’  c                   óV   ‡ — e Zd ZdZdeez  defˆ fd„Zdej	        dej	        fd„Z
ˆ xZS )ÚGemma4MultimodalEmbedderzQEmbeds token ids or soft tokens for multimodal content into language model space.Úmultimodal_configÚtext_configc                 ó<  •— t          ¦   «                              ¦   «          t          |d|j        ¦  «        | _        |j        | _        |j        | _        t          j	        | j        | j        d¬¦  «        | _
        t          | j        | j        d¬¦  «        | _        d S )Nr  Fr^   r4  )re   rf   r›  r•   Úmultimodal_hidden_sizer  rz   Útext_hidden_sizer   rh   Úembedding_projectionrw   Úembedding_pre_projection_norm)rm   r&  r'  rn   s      €rI   rf   z!Gemma4MultimodalEmbedder.__init__(  s�   ø€ õ
 	‰Œ×ÒÑÔÐå&-Ð.?ÐASÐUfÔUrÑ&sÔ&sˆÔ#Ø$Ô1ˆŒØ +Ô 7ˆÔÝ$&¤I¨dÔ.IÈ4ÔK`ÐglÐ$mÑ$mÔ$mˆÔ!Ý-:¸4Ô;VÐ\`Ô\dÐqvÐ-wÑ-wÔ-wˆÔ*Ð*Ð*rH   r  r\   c                 óV   — |                       |¦  «        }|                      |¦  «        S )a:  Embeds token ids or soft tokens for multimodal content into language model space.
        Args:
            inputs_embeds: A torch.Tensor containing the soft tokens to embed.
        Returns:
            A torch.Tensor of embeddings with shape `[batch_size, seq_len, self.config.text_config.hidden_size]`.
        )r,  r+  )rm   r  Úembs_normeds      rI   rr   z Gemma4MultimodalEmbedder.forward5  s+   € ð ×8Ò8¸ÑGÔGˆØ×(Ò(¨Ñ5Ô5Ð5rH   )r<   r=   r>   r?   r0   r3   r2   rf   r@   rF   rr   rt   ru   s   @rI   r%  r%  %  s‰   ø€ € € € € Ø[Ð[ðxà,Ð/AÑAðxð &ðxð xð xð xð xð xð6 U¤\ð 6°e´lð 6ð 6ð 6ð 6ð 6ð 6ð 6ð 6rH   r%  rY   r  rN   Úblock_sequence_idsc                 óò   — | ||||dœ}t          di |¤Ž}t          di |¤ddi¤Ž\  }}	}	}	}
}	}|r|}nt          |||
|¦  «        }t          di |¤t          |¦  «        t	          | j        ¦  «        dœ¤Ž}||dœS )a‹  Create full_attention and sliding_attention masks with correct composition.

    For global (full attention) layers:  causal only (no bidirectional)
    For local (sliding window) layers:  AND(sliding_window, OR(causal, blockwise))

    Unlike Gemma 3 (which applies bidirectional attention on all layers), Gemma 4
    explicitly disables bidirectional attention on global attention layers.
    r×  rµ   r   )Úor_mask_functionr  rØ  rG   )r   r   r   r   r   r-  )rY   r  rU   rN   r¯   r/  rá  Ú	full_maskÚ
early_exitrì   Ú	kv_lengthÚ	kv_offsetÚpadded_block_sequence_idsÚsliding_masks                 rI   Úcreate_masks_for_vision_modelr8  @  sþ   € ð" Ø&Ø(Ø*Ø$ðð €Kõ #Ð1Ð1 [Ð1Ð1€Iõ 4Nð 4ð 4Ø
ð4ð 4àð4ð 4ð 4Ñ0€J��1�a˜ A yð ð 
Ø$6Ð!Ð!å$@Ø °	¸9ñ%
ô %
Ð!õ &ð ð Ø
ðå*Ð+DÑEÔEÝ0°Ô1FÑGÔGðð ð ð €Lð $Ø)ðð ð rH   Úmm_token_type_idsr¦   c                 ó  — |                       |¦  «        } | dk    | dk    z  }t          j        |dd¬¦  «        }d|d<   || z  }t          j        |                     ¦   «         d¬¦  «        dz
  }t          j        ||d¦  «        }|S )Nr/   r-   r‚   )ÚshiftsÚdimsFrT  r¨   )r«   r@   ÚrollÚcumsumrs   rX  )r9  r¦   Ú	is_visionÚis_prev_visionÚnew_vision_startsÚvision_group_idsr/  s          rI   Úget_block_sequence_ids_for_maskrC  w  sš   € Ø)×,Ò,¨VÑ4Ô4Ðà" aÒ'Ð,=ÀÒ,BÑC€IÝ”Z 	°!¸"Ð=Ñ=Ô=€NØ"€N�6ÑØ! ^ OÑ3ÐÝ”|Ð$5×$9Ò$9Ñ$;Ô$;ÀÐCÑCÔCÀaÑGÐÝœ YÐ0@À"ÑEÔEÐØÐrH   zŒ
    The base Gemma 4 model comprising a vision backbone, an audio backbone, and a language model without a
    language modeling head.
    c            $       ój  ‡ — e Zd ZdZdefˆ fd„Ze ed¬¦  «        	 d dej	        dej
        dz  d	ee         d
efd„¦   «         ¦   «         Z	 	 d!dej
        dz  dej	        dz  d
eej        ej        ej        f         fd„Zeee	 	 	 	 	 	 	 	 	 	 	 	 	 	 d"dej
        dz  dej	        dz  dej	        dz  dej	        dz  dej        dz  dej        dz  dej
        dz  dedz  dej
        dz  dej	        dz  dedz  dej
        dz  dej
        dz  dej        dz  d	ee         d
ef d„¦   «         ¦   «         ¦   «         Zd„ Zd„ Ze ed¬¦  «        dej        dej        d	ee         d
eez  fd„¦   «         ¦   «         Ze ed¬¦  «        	 d dej	        dej
        dz  d	ee         d
efd„¦   «         ¦   «         Zˆ xZS )#ÚGemma4ModelFrY   c                 ó4  •— t          ¦   «                              |¦  «         |j        �t          j        |j        ¦  «        nd | _        |j        j        | _        t          j        |j        ¬¦  «        }|| _        |j        j	        | _	        |j
        �t          j        |j
        ¦  «        nd | _        |j        �t          |j        |j        ¦  «        nd | _        |j
        �t          |j
        |j        ¦  «        nd | _        |                      ¦   «          d S )NrÖ  )re   rf   Úvision_configr.   Úfrom_configÚvision_towerr'  r©  Úlanguage_modelr«  Úaudio_configÚaudio_towerr%  Úembed_visionÚembed_audiorÏ  )rm   rY   rJ  rn   s      €rI   rf   zGemma4Model.__init__�  s  ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØKQÔK_ÐKk�IÔ1°&Ô2FÑGÔGÐGÐquˆÔØ Ô,Ô7ˆŒå"Ô.°fÔ6HÐIÑIÔIˆØ,ˆÔØ*0Ô*<Ô*WˆÔ'ØIOÔI\ÐIh�9Ô0°Ô1DÑEÔEÐEÐnrˆÔð Ô#Ð/õ % VÔ%9¸6Ô;MÑNÔNÐNàð 	Ôð Ô"Ð.õ % VÔ%8¸&Ô:LÑMÔMÐMàð 	Ôð
 	�ŠÑÔÐÐÐrH   zOProjects the last hidden state from the vision model into language model space.r5   Nr]  Úimage_position_idsrG  r\   c                 ój   —  | j         d||dœ|¤Ž}|j        }|                      |¬¦  «        |_        |S )zÝ
        image_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`, *optional*):
            The patch positions as (x, y) coordinates in the image. Padding patches are indicated by (-1, -1).
        ©r]  rP  ©r  rG   )rI  r	  rM  Úpooler_output)rm   r]  rO  rG  Úvision_outputsr	  s         rI   Úget_image_featureszGemma4Model.get_image_features¢  s]   € ð +˜Ô*ð 
Ø%Ø1ð
ð 
ð ð
ð 
ˆð
 +Ô<ÐØ'+×'8Ò'8ÐGXÐ'8Ñ'YÔ'YˆÔ$ØÐrH   r~  r  c                 óÀ  — |�2|| j         j        k    }|| j         j        k    }|| j         j        k    }�n&| |                      ¦   «         t          j        | j         j        t
          j        |j        ¬¦  «        ¦  «        k     	                    d¦  «        }| |                      ¦   «         t          j        | j         j        t
          j        |j        ¬¦  «        ¦  «        k     	                    d¦  «        }| |                      ¦   «         t          j        | j         j        t
          j        |j        ¬¦  «        ¦  «        k     	                    d¦  «        }|||fS )a‹  
        Obtains mask for multimodal placeholders (replaced by soft tokens) and hard text tokens.

        Masks will be obtained from `mm_token_type_ids`, `input_ids`, or `inputs_embeds` as available and in that
        precedence order. If passing `input_ids` or `inputs_embeds`, the image mask will be derived using
        `config.image_token_id`. Same goes for audio and video masks

        Args:
            input_ids: A tensor containing the hard token IDs from the text tokenizer.
            inputs_embeds: A tensor containing the embeddings for all hard text tokens.

        Returns:
            image_mask, video_mask, audio_mask
        N)rª   r¦   r‚   )
rY   Úimage_token_idÚvideo_token_idÚaudio_token_idÚget_input_embeddingsr@   rk   ro  r¦   rp  )rm   r~  r  Úspecial_image_maskÚspecial_video_maskÚspecial_audio_masks         rI   Úget_placeholder_maskz Gemma4Model.get_placeholder_mask·  sP  € ð& Ð Ø!*¨d¬kÔ.HÒ!HÐØ!*¨d¬kÔ.HÒ!HÐØ!*¨d¬kÔ.HÒ!HÐÑð Ø.�4×,Ò,Ñ.Ô.Ý”L ¤Ô!;Å5Ä:ÐVcÔVjÐkÑkÔkñô ò÷ Šc�"‰gŒgð ð Ø.�4×,Ò,Ñ.Ô.Ý”L ¤Ô!;Å5Ä:ÐVcÔVjÐkÑkÔkñô ò÷ Šc�"‰gŒgð ð Ø.�4×,Ò,Ñ.Ô.Ý”L ¤Ô!;Å5Ä:ÐVcÔVjÐkÑkÔkñô ò÷ Šc�"‰gŒgð ð "Ð#5Ð7IÐIÐIrH   Úpixel_values_videosr  rU   r  r¯   rN   r9  rÒ  Úvideo_position_idsrÑ  c                 ó@  — |du |
duz  rt          d¦  «        ‚|�|�t          d¦  «        ‚|                      ||
¦  «        \  }}}||z  |z  }d}|
€V|                     ¦   «         }t          j        || j        j        j        |¦  «        } |                      ¦   «         |¦  «        }
|€®| j         	                    ¦   «         j
        r�| j        j        j        | j        j        j        dd…f         }|                     |
j        ¦  «        }t          j        |d         |                     ddd¦  «        |
¦  «        }| j                             ||¦  «        }|��&|                      ||d¬¦  «        j        }|                     |
j        |
j        ¦  «        }|                     ¦   «         }|                     d¦  «                             |
¦  «                             |
j        ¦  «        }t1          |
|                              ¦   «         |                     ¦   «         k    d	|› d
|j        d         › �¦  «         |
                     |                     |
j        ¦  «        |                     |
j        ¦  «        ¦  «        }
|��&|                      ||d¬¦  «        j        }|                     |
j        |
j        ¦  «        }|                     ¦   «         }|                     d¦  «                             |
¦  «                             |
j        ¦  «        }t1          |
|                              ¦   «         |                     ¦   «         k    d|› d
|j        d         › �¦  «         |
                     |                     |
j        ¦  «        |                     |
j        ¦  «        ¦  «        }
|��@|��=|                      ||d¬¦  «        }|j        }|j        }||                     |j        ¦  «                 }|                     ¦   «         }|                     d¦  «                             |
¦  «                             |
j        ¦  «        }t1          |
|                              ¦   «         |                     ¦   «         k    d|› d
|j        d         |j        d         z  › �¦  «         |
                     |                     |
j        ¦  «        |                     |
j        ¦  «        ¦  «        }
|€V|�|                     ¦   «         nd}t          j         |
j        d         |
j        ¬¦  «        |z   }|                     d¦  «        }tC          |x} tD          ¦  «        sx| j         	                    ¦   «         |
|||dœ}!| j         	                    ¦   «         }"|"j#        dk    }#|#r'|	�%tI          |	|
j        ¬¦  «        }$tK          dd|$i|!¤Ž} ntM          di |!¤Ž}  | j        d|| |||
|ddœ|¤Ž}%tO          |%j(        |%j)        |%j*        |%j+        |�|nd|�|nd|%j,        ¬¦  «        S )áK  
        input_features_mask (`torch.FloatTensor]` of shape `(num_images, seq_length)`):
            The attention mask for the input audio.
        image_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`, *optional*):
            2D patch position coordinates from the image processor, with `(-1, -1)` indicating padding.
            Passed through to the vision encoder for positional embedding computation.
        video_position_ids (`torch.LongTensor` of shape `(num_videos, num_frames, max_patches, 2)`, *optional*):
            2D patch position coordinates from the video processor, with `(-1, -1)` indicating padding.
            Passed through to the vision encoder for positional embedding computation.
        per_layer_inputs (`torch.Tensor`, *optional*):
            Pre-computed per-layer input text embeddings of shape `(batch_size, sequence_length, num_hidden_layers,
            hidden_size_per_layer_input)`. When provided, these are used directly instead of being computed from `input_ids`
            via `get_per_layer_inputs()` in the text model. If calling the `forward` with `inputs_embeds` instead of `input_ids`,
            you should probably precompute them and forward them along `inputs_embeds`, otherwise recomputing them needs
            to reverse the main embedding, which is expensive.
        NrÔ  rÕ  r§   r/   r‚   T)Úreturn_dictz6Image features and image tokens do not match, tokens: z, features: r   z6Video features and video tokens do not match, tokens: z6Audio features and audio tokens do not match, tokens: r¥   r×  Úvisionr/  )rÑ  rU   r¯   rN   r  rÒ  rc  )r	  rN   rO   rP   r9   r:   r;   rG   )-rl  r^  r”  r@   rX  rY   r'  rÄ  rZ  rª  rg  rJ  rÅ  r€   r«   r¦   rÛ   rÚ  rU  rS  rª   rI  rŸ   Ú	expand_asr(   ÚnumelrÌ   Úmasked_scatterÚget_video_featuresÚget_audio_featuresrU   rÜ  rž   r§  rC   r1  rC  r8  r   r8   r	  rN   rO   rP   r;   )&rm   r~  r]  r_  r  rU   r  r¯   rN   r9  r  rÒ  rO  r`  rÑ  rG  Ú
image_maskÚ
video_maskÚ
audio_maskÚmultimodal_maskÚllm_input_idsÚpad_embeddingÚllm_inputs_embedsÚimage_featuresÚn_image_tokensÚvideo_featuresÚn_video_tokensÚaudio_outputÚaudio_featuresÚaudio_mask_from_encoderÚn_audio_tokensrß  rà  rá  r'  Ú	use_bidirr/  rò  s&                                         rI   rr   zGemma4Model.forwardä  s©  € ðJ ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ð%5Ð%AÝÐ[Ñ\Ô\Ð\à-1×-FÒ-FÀyÐR_Ñ-`Ô-`Ñ*ˆ
�J 
Ø$ zÑ1°JÑ>ˆð ˆØÐ Ø%ŸOšOÑ-Ô-ˆMÝ!œK¨¸¼Ô9PÔ9]Ð_lÑmÔmˆMØ7˜D×5Ò5Ñ7Ô7¸ÑFÔFˆMàÐ#¨¬×(CÒ(CÑ(EÔ(EÔ(aÐ#Ø Ô/Ô<ÔCÀDÄKÔD[ÔDhÐjkÐjkÐjkÐDkÔlˆMØ-×0Ò0°Ô1EÑFÔFˆOÝ %¤¨O¸IÔ,FÈ×HZÒHZÐ[\Ð^_ÐacÑHdÔHdÐfsÑ tÔ tÐØ#Ô2×GÒGÈÐWhÑiÔiÐð Ñ#Ø!×4Ò4°\ÐCUÐcgÐ4ÑhÔhÔvˆNØ+×.Ò.¨}Ô/CÀ]ÔEXÑYÔYˆNð (Ÿ^š^Ñ-Ô-ˆNØ#×-Ò-¨bÑ1Ô1×;Ò;¸MÑJÔJ×MÒMÈmÔNbÑcÔcˆJÝ"Ø˜jÔ)×/Ò/Ñ1Ô1°^×5IÒ5IÑ5KÔ5KÒKð.Èð .ð .Ø"Ô(¨Ô+ð.ð .ñô ð ð *×8Ò8Ø—’˜mÔ2Ñ3Ô3°^×5FÒ5FÀ}ÔG[Ñ5\Ô5\ñô ˆMð Ñ*Ø!×4Ò4Ø#Ð%7ÀTð 5ñ ô äð ð ,×.Ò.¨}Ô/CÀ]ÔEXÑYÔYˆNð (Ÿ^š^Ñ-Ô-ˆNØ#×-Ò-¨bÑ1Ô1×;Ò;¸MÑJÔJ×MÒMÈmÔNbÑcÔcˆJÝ"Ø˜jÔ)×/Ò/Ñ1Ô1°^×5IÒ5IÑ5KÔ5KÒKð.Èð .ð .Ø"Ô(¨Ô+ð.ð .ñô ð ð *×8Ò8Ø—’˜mÔ2Ñ3Ô3°^×5FÒ5FÀ}ÔG[Ñ5\Ô5\ñô ˆMð
 Ñ%Ð*=Ñ*IØ×2Ò2°>ÐCVÐdhÐ2ÑiÔiˆLØ)Ô7ˆNØ&2Ô&AÐ#ð
 ,Ð,C×,FÒ,FÀ~ÔG\Ñ,]Ô,]Ô^ˆNà'Ÿ^š^Ñ-Ô-ˆNØ#×-Ò-¨bÑ1Ô1×;Ò;¸MÑJÔJ×MÒMÈmÔNbÑcÔcˆJÝ"Ø˜jÔ)×/Ò/Ñ1Ô1°^×5IÒ5IÑ5KÔ5KÒKðHÈð Hð HØ"Ô(¨Ô+¨nÔ.BÀ1Ô.EÑEðHð Hñô ð ð *×8Ò8Ø—’˜mÔ2Ñ3Ô3°^×5FÒ5FÀ}ÔG[Ñ5\Ô5\ñô ˆMð
 ÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå°Ð?Ð-ÅÑFÔFð 	Oàœ+×5Ò5Ñ7Ô7Ø!.Ø"0Ø#2Ø ,ðð ˆKð œ+×5Ò5Ñ7Ô7ˆKØ#Ô?À8ÒKˆIàð 	OÐ.Ð:Ý%DÐEVÐ_lÔ_sÐ%tÑ%tÔ%tÐ"Ý&Cð 'ð 'Ø'9ð'à!ð'ð 'Ð#Ð#õ '@Ð&NÐ&NÀ+Ð&NÐ&NÐ#à%�$Ô%ð 	
Ø-Ø.Ø%Ø+Ø'ØØð	
ð 	
ð ð	
ð 	
ˆõ )Ø%Ô7Ø#Ô3Ø!Ô/ØÔ)Ø2>Ð2J  ÐPTØ2@Ð2L  ÐRVØ$Ô5ð
ñ 
ô 
ð 	
rH   c                 ó   — | j         j        S rp   ©rJ  r›  rœ  s    rI   r�  z*Gemma4Model.get_per_layer_input_embeddings�	  s   € ØÔ"Ô9Ð9rH   c                 ó   — || j         _        d S rp   r{  rŸ  s     rI   r   z*Gemma4Model.set_per_layer_input_embeddings�	  s   € Ø5:ˆÔÔ2Ð2Ð2rH   zPProjects the last hidden state from the audio encoder into language model space.c                 ó”   — | j         €t          d¦  «        ‚ | j         ||fddi|¤Ž}|                      |j        ¬¦  «        |_        |S )a0  
        input_features (`torch.FloatTensor]` of shape `(num_images, seq_length, num_features)`):
            The tensors corresponding to the input audio.
        input_features_mask (`torch.FloatTensor]` of shape `(num_images, seq_length)`):
            The attention mask for the input audio.
        Nz•Audio features were requested, but the model was initialized without an audio_config. Cannot process audio without an audio tower and audio embedder.rc  TrR  )rL  rl  rN  r	  rS  )rm   r  r  rG  Úaudio_outputss        rI   ri  zGemma4Model.get_audio_features“	  sm   € ð ÔÐ#ÝðRñô ð ð
 )˜Ô(¨Ð9LÐiÐiÐZ^ÐiÐbhÐiÐiˆØ&*×&6Ò&6À]ÔEdÐ&6Ñ&eÔ&eˆÔ#àÐrH   zQProjects the last hidden state from the vision encoder into language model space.c                 óÂ   — |                      dd¦  «        }|                      dd¦  «        } | j        d||dœ|¤Ž}|j        }|                      |¬¦  «        |_        |S )a9  
        video_position_ids (`torch.LongTensor` of shape `(num_videos, num_frames, max_patches, 2)`, *optional*):
            2D patch position coordinates from the video processor, with `(-1, -1)` indicating padding.
            Passed through to the vision encoder for positional embedding computation.
        r   r/   rQ  rR  rG   )ÚflattenrI  r	  rM  rS  )rm   r_  r`  rG  rT  r	  s         rI   rh  zGemma4Model.get_video_features¬	  s‰   € ð 2×9Ò9¸!¸QÑ?Ô?ÐØ/×7Ò7¸¸1Ñ=Ô=ÐØ*˜Ô*ð 
Ø,Ø1ð
ð 
ð ð
ð 
ˆð
 +Ô<ÐØ'+×'8Ò'8ÐGXÐ'8Ñ'YÔ'YˆÔ$ØÐrH   rp   r'  )NNNNNNNNNNNNNN)r<   r=   r>   Úaccepts_loss_kwargsr1   rf   r&   r%   r@   rA   rñ  r"   r$   r   rU  rE   rV   r^  r*   rF   r   rŒ   r8   rr   r�  r   rT   ri  rh  rt   ru   s   @rI   rE  rE  ƒ  sÄ  ø€ € € € € ð  Ðð˜|ð ð ð ð ð ð ð* Ø€^Ð!rÐsÑsÔsð 7;ðð àÔ'ðð "Ô,¨tÑ3ðð Ð+Ô,ð	ð
 
$ðð ð ñ tÔsñ Ôðð* .2Ø26ð+Jð +JàÔ# dÑ*ð+Jð Ô(¨4Ñ/ð+Jð 
ˆuÔ Ô!1°5Ô3CÐCÔ	Dð	+Jð +Jð +Jð +JðZ  ØØð .2Ø15Ø8<Ø37Ø.2Ø37Ø04Ø(,Ø59Ø26Ø!%Ø6:Ø6:Ø04ðd
ð d
àÔ# dÑ*ðd
ð Ô'¨$Ñ.ðd
ð #Ô.°Ñ5ð	d
ð
 Ô)¨DÑ0ðd
ð œ tÑ+ðd
ð #œ\¨DÑ0ðd
ð Ô&¨Ñ-ðd
ð  ™ðd
ð !Ô+¨dÑ2ðd
ð Ô(¨4Ñ/ðd
ð ˜$‘;ðd
ð "Ô,¨tÑ3ðd
ð "Ô,¨tÑ3ðd
ð  œ,¨Ñ-ðd
ð  Ð+Ô,ð!d
ð" 
#ð#d
ð d
ð d
ñ „^ñ Ôñ  Ôðd
ðL:ð :ð :ð;ð ;ð ;ð Ø€^Ð!sÐtÑtÔtðàœðð #œ\ðð Ð+Ô,ð	ð
 
Ð'Ñ	'ðð ð ñ uÔtñ Ôðð. Ø€^Ð!tÐuÑuÔuð 7;ðð à"Ô.ðð "Ô,¨tÑ3ðð Ð+Ô,ð	ð
 
$ðð ð ñ vÔuñ Ôðð ð ð ð rH   rE  z…
    The base Gemma 4 model comprising a vision backbone, an audio backbone, a language model, and a language modeling
    head.
    c            '       óÂ  ‡ — e Zd ZddiZdZdZdefˆ fd„Ze	 d#de	j
        d	e	j        dz  d
ee         fd„¦   «         Zee	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d$de	j        dz  de	j
        dz  de	j
        dz  de	j
        dz  de	j        dz  de	j        dz  de	j        dz  d	e	j        dz  de	j        dz  dedz  de	j        dz  de	j
        dz  de	j        dz  dedz  dee	j        z  de	j        dz  d
ee         def$d„¦   «         ¦   «         Z	 	 	 	 	 	 	 	 	 	 	 	 	 d%ˆ fd„	Zd„ Zd „ Ze	 	 d&dede	j        de	j        dz  dedz  de	j        dz  de	j        dz  d!edz  defd"„¦   «         Zˆ xZS )'ÚGemma4ForConditionalGenerationrç  z(model.language_model.embed_tokens.weightFr‚  rY   c                 óú   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        j        |j        j        d¬¦  «        | _	        |  
                    ¦   «          d S rë  )re   rf   rE  r‚  r   rh   r'  r•   r©  rè  rÏ  r"  s     €rI   rf   z'Gemma4ForConditionalGeneration.__init__Ð	  se   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý  Ñ(Ô(ˆŒ
Ý”y Ô!3Ô!?ÀÔASÔA^ÐejÐkÑkÔkˆŒØ�ŠÑÔÐÐÐrH   Nr]  rO  rG  c                 ó*   —  | j         j        ||fi |¤ŽS )a-  
        image_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`, *optional*):
            2D patch position coordinates from the image processor, with `(-1, -1)` indicating padding.
            Passed through to the vision encoder for positional embedding computation.
        )r‚  rU  )rm   r]  rO  rG  s       rI   rU  z1Gemma4ForConditionalGeneration.get_image_featuresÖ	  s%   € ð -ˆtŒzÔ,¨\Ð;MÐXÐXÐQWÐXÐXÐXrH   r   r~  r_  r  rU   r  r¯   r`  rN   r9  r  rì  rÒ  rí  rÑ  r\   c           
      ód  —  | j         di d|“d|“d|“d|“d|“d|“d|“d|
“d	|“d
|“d|“d|“d|“d|“d|	“dd“|¤Ž}|j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }| j                             ¦   «         j        x}�||z  }t          j
        |¦  «        }||z  }d}|�, | j        ||| j                             ¦   «         j        fi |¤Ž}t          |||j        |j        |j        |j        |j        |j        ¬¦  «        S )rb  r~  r]  r_  r  rU   r  r¯   rN   r9  r  rÑ  rì  rÒ  rO  r`  rc  TN)rL   rM   rN   rO   rP   r9   r:   r;   rG   )r‚  r	  r§  rs   rï  rè  rY   rª  rð  r@   rå   rñ  r©  rK   rN   rO   rP   r9   r:   r;   )rm   r~  r]  r_  r  rU   r  r¯   rO  r`  rN   r9  r  rì  rÒ  rí  rÑ  rG  rò  rO   ró  rM   rð  rL   s                           rI   rr   z&Gemma4ForConditionalGeneration.forwardä	  sû  € ðL �$”*ð 
ð 
ð 
Ø�ið
à%˜ð
ð !4Ð 3ð
ð *˜>ð	
ð
 *˜>ð
ð !4Ð 3ð
ð &˜ð
ð ,˜Oð
ð 0Ð/ð
ð (˜-ð
ð .Ð-ð
ð �6ð
ð  �ið
ð  2Ð1ð
ð  2Ð1ð
ð  ˜Øð#
ð 
ˆð(  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆØ'+¤{×'BÒ'BÑ'DÔ'DÔ'\Ð\Ð#ÐiØÐ5Ñ5ˆFÝ”Z Ñ'Ô'ˆFØÐ5Ñ5ˆFàˆØÐØ%�4Ô% f¨f°d´k×6QÒ6QÑ6SÔ6SÔ6^ÐiÐiÐbhÐiÐiˆDå+ØØØ#Ô3Ø!Ô/ØÔ)Ø 'Ô ;Ø 'Ô ;Ø$Ô5ð	
ñ 	
ô 	
ð 		
rH   Tc                 óº   •—  t          ¦   «         j        |f|||||||
|dœ|¤Ž}|s|s||d<   ||d<   ||d<   |	|d<   nd |d<   |s|                     dd ¦  «        }|S )N)rN   r  rU   r¯   rÒ  rí  Útoken_type_idsÚis_first_iterationr]  r_  r  r  r9  rÑ  )re   Úprepare_inputs_for_generationrÝ  )rm   r~  rN   r  r¯   r]  r_  r  rU   r  rˆ  rÒ  rí  rì  r‰  rG  Úmodel_inputsrì   rn   s                     €rI   rŠ  z<Gemma4ForConditionalGeneration.prepare_inputs_for_generation6
  s¾   ø€ ð& =•u‘w”wÔ<Øð
à+Ø'Ø)Ø%ØØ)Ø)Ø1ð
ð 
ð ð
ð 
ˆð ð 	5 Yð 	5Ø+7ˆL˜Ñ(Ø2EˆLÐ.Ñ/Ø-;ˆLÐ)Ñ*Ø2EˆLÐ.Ñ/Ð/ð 15ˆLÐ,Ñ-ð "ð 	;Ø× Ò Ð!3°TÑ:Ô:ˆAàÐrH   c                 ó4   — | j                              ¦   «         S rp   )r‚  r�  rœ  s    rI   r�  z=Gemma4ForConditionalGeneration.get_per_layer_input_embeddingsf
  s   € ØŒz×8Ò8Ñ:Ô:Ð:rH   c                 ó:   — | j                              |¦  «         d S rp   )r‚  r   rŸ  s     rI   r   z=Gemma4ForConditionalGeneration.set_per_layer_input_embeddingsi
  s   € ØŒ
×1Ò1°%Ñ8Ô8Ð8Ð8Ð8rH   r‰  c                 óð   — |                       ¦   «         ||||dœ}|                       ¦   «         }	t          |	dd ¦  «        dk    }
|
r&|�$t          ||j        ¬¦  «        }t	          dd|i|¤ŽS t          di |¤ŽS )Nr×  r1  rd  r¥   r/  rG   )rª  r›  rC  r¦   r8  r   )rY   r  rU   rN   r¯   r9  r‰  rG  rá  r'  ry  r/  s               rI   r   z8Gemma4ForConditionalGeneration.create_masks_for_generatel
  s¹   € ð ×,Ò,Ñ.Ô.Ø*Ø,Ø.Ø(ð
ð 
ˆð ×,Ò,Ñ.Ô.ˆÝ˜KÐ)FÈÑMÔMÐQYÒYˆ	àð 	Ð*Ð6Ý!@ÐARÐ[hÔ[oÐ!pÑ!pÔ!pÐÝ0ð ð Ø#5ðàðð ð õ
 )Ð7Ð7¨;Ð7Ð7Ð7rH   rp   )NNNNNNNNNNNNNNr   N)NNNNNNNNNTNNF)NF)r<   r=   r>   rô  r�  r°  r1   rf   r%   r@   rA   rñ  r"   r$   rU  r&   rF   r   rŒ   rs   rK   rr   rŠ  r�  r   r´  r   rC   r   rt   ru   s   @rI   rƒ  rƒ  Å	  sa  ø€ € € € € ð +Ð,VÐWÐØÐØÐð˜|ð ð ð ð ð ð ð ð 7;ðYð YàÔ'ðYð "Ô,¨tÑ3ðYð Ð+Ô,ð	Yð Yð Yñ „^ðYð Øð .2Ø15Ø8<Ø37Ø.2Ø37Ø04Ø6:Ø6:Ø(,Ø59Ø26Ø*.Ø!%Ø-.Ø04ð#N
ð N
àÔ# dÑ*ðN
ð Ô'¨$Ñ.ðN
ð #Ô.°Ñ5ð	N
ð
 Ô)¨DÑ0ðN
ð œ tÑ+ðN
ð #œ\¨DÑ0ðN
ð Ô&¨Ñ-ðN
ð "Ô,¨tÑ3ðN
ð "Ô,¨tÑ3ðN
ð  ™ðN
ð !Ô+¨dÑ2ðN
ð Ô(¨4Ñ/ðN
ð Ô  4Ñ'ðN
ð ˜$‘;ðN
ð  ˜eœlÑ*ð!N
ð"  œ,¨Ñ-ð#N
ð$ Ð+Ô,ð%N
ð& 
&ð'N
ð N
ð N
ñ „^ñ ÔðN
ðf ØØØØ ØØØ ØØØØØ ð.ð .ð .ð .ð .ð .ð`;ð ;ð ;ð9ð 9ð 9ð ð 26Ø*/ð8ð 8Ø ð8à”|ð8ð œ tÑ+ð8ð  ™ð	8ð
 ”l TÑ)ð8ð !œ<¨$Ñ.ð8ð ! 4™Kð8ð 
ð8ð 8ð 8ñ „\ð8ð 8ð 8ð 8ð 8rH   rƒ  )r  ræ  rƒ  rE  r�  r½  r’  )r/   )rV  NN)r-   )‚rš   Úcollectionsr   Úcollections.abcr   Údataclassesr   Ú	functoolsr   Útypingr   r@   r   Útorch.nnr	   rÍ   Ú r   rŠ  Úactivationsr   Úcache_utilsr   r   Úconfiguration_utilsr   Ú
generationr   Úintegrationsr   Úmasking_utilsr   r   r   r   r   r   r   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr    r!   Úprocessing_utilsr"   Úutilsr#   r$   r%   r&   r'   r(   Úutils.genericr)   r*   Úutils.output_capturingr+   r,   Úauto.modeling_autor.   Úconfiguration_gemma4r0   r1   r2   r3   Úaccelerate.hooksr4   r8   rK   rR   rT   ÚModulerX   rw   rŽ   r´   rù   r  r  ÚConv1dr)  r1  r>  rJ  rc  r|  rˆ  r¸  rF   rs   r¼  rÂ  rl   rE   rÎ  rÝ  rß  ró  rþ  r  r  r)  r:  rX  rd  r»  rw  r�  r½  ræ  r  r  r’  r%  rC   r8  r¦   rC  rE  rƒ  Ú__all__rG   rH   rI   ú<module>r«     sI  ðð* €€€Ø  Ð  Ð  Ð  Ð  Ð  Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !Ø %Ð %Ð %Ð %Ð %Ð %Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ø $Ð $Ð $Ð $Ð $Ð $à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø )Ð )Ð )Ð )Ð )Ð )Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6ð	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð CÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø SÐ SÐ SÐ SÐ SÐ SÐ SÐ SØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð HÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ *Ð *Ð *Ð *Ð *Ð *Ø gÐ gÐ gÐ gÐ gÐ gÐ gÐ gÐ gÐ gÐ gÐ gð ÐÑÔð 4Ø3Ð3Ð3Ð3Ð3Ð3ð €ððñ ô ð
 ðQð Qð Qð Qð QÐ 7ñ Qô Qñ „ñô ðQð2 €ððñ ô ð
 ðQð Qð Qð Qð Q ;ñ Qô Qñ „ñô ðQðD ð
Qð 
Qð 
Qð 
Qð 
QÐ$;ñ 
Qô 
Qñ „ð
Qð Ø
ð3ð 3ð 3ð 3ð 3Ð7ñ 3ô 3ñ „ñ „ð3ðð ð ð ð ˜BœIñ ô ð ð:4ð 4ð 4ð 4ð 4�B”Iñ 4ô 4ð 4ð*7ð 7ð 7ð 7ð 7 r¤yñ 7ô 7ð 7ð>i)ð i)ð i)ð i)ð i)˜2œ9ñ i)ô i)ð i)ðX#ð #ð #ð #ð #¨b¬iñ #ô #ð #ð8;ð ;ð ;ð ;ð ;¨¬ñ ;ô ;ð ;ð< ð  ð  ð  ð  ˜RœYñ  ô  ð  ðH"ð "ð "ð "ð "˜bœiñ "ô "ð "ðB&ð &ð &ð &ð &˜RœYñ &ô &ð &ðR0ð 0ð 0ð 0ð 0�r”yñ 0ô 0ð 0ðl*3ð *3ð *3ð *3ð *3 ¤	ñ *3ô *3ð *3ðZ@0ð @0ð @0ð @0ð @0˜œñ @0ô @0ð @0ðFð ð ð ð �b”iñ ô ð ð Lð Lð Lð Lð L "¤)ñ Lô Lð Lð^(ð (ð (ð.ð .˜EœLð .¨u¬|ð .À%Ä,ð .Ð_bð .ð .ð .ð .ð,	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð$ Ø Ø ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �S‰[ð%ð �T‰\ð%ð �T‰\ð%ð ˆ5Œ<˜œÐ%Ô&ð%ð %ð %ð %ðN ð5&ð 5&Ø„|ð5&à	Œð5&ð 
Œð5&ð ”,ð	5&ð
 ð5&ð „\ð5&ð 5&ð 5&ð 5&ðpB)ð B)ð B)ð B)ð B)˜BœIñ B)ô B)ð B)ðJ)ð )ð )ð )ð )Ð9ñ )ô )ð )ðX)Hð )Hð )Hð )Hð )H˜"œ)ñ )Hô )Hð )HðXð ð ð ð �B”Iñ ô ð ð&W<ð W<ð W<ð W<ð W< ¤	ñ W<ô W<ð W<ðtq)ð q)ð q)ð q)ð q)˜"œ)ñ q)ô q)ð q)ðh ð$#ð $#ð $#ð $#ð $#˜œ	ñ $#ô $#ñ Ôð$#ðN"@ð "@ð "@ð "@ð "@�r”yñ "@ô "@ð "@ðJVð Vð Vð Vð VÐ7ñ Vô Vð VðrSð Sð Sð Sð S B¤Lñ Sô Sð Sð ðmJð mJð mJð mJð mJ˜Oñ mJô mJñ „ðmJð` €Ð`ÐaÑaÔaðdVð dVð dVð dVð dVÐ+ñ dVô dVñ bÔaðdVðN €Ð]Ð^Ñ^Ô^ðW
ð W
ð W
ð W
ð W
Ð-¨ñ W
ô W
ñ _Ô^ðW
ðt°°s¸C°x´ð ÀXð ð ð ð ð Scð Scð Scð Scð ScÐ,ñ Scô Scð ScðlAHð AHð AHð AHð AHÐ-ñ AHô AHð AHðH6ð 6ð 6ð 6ð 6˜rœyñ 6ô 6ð 6ð64Øð4à”<ð4ð ”L 4Ñ'ð4ð ˜T‘\ð	4ð
 ”, Ñ%ð4ð œð4ð 
ð4ð 4ð 4ð 4ðn	°u´|ð 	ÈUÌ\ð 	Ð^cÔ^jð 	ð 	ð 	ð 	ð €ððñ ô ðyð yð yð yð yÐ'ñ yô yñô ðyðx	 €ððñ ô ð~8ð ~8ð ~8ð ~8ð ~8Ð%:¸Oñ ~8ô ~8ñô ð~8ðBð ð €€€rH   