§
    ‚Štj"¾ ã                   óR  — d dl Z d dlmZ d dlmZmZ d dlmZ d dlm	Z	 d dl
Z
d dlmZ d dlmc mZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZ ddlmZ ddlm Z m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1 ddl2m3Z3m4Z4 ddl5m6Z6 ddl7m8Z8 ddl9m:Z:m;Z;m<Z<m=Z=  e0¦   «         rd dl>m?Z? e.e G d„ de!¦  «        ¦   «         ¦   «         Z@ e.d¬¦  «        e G d„ de ¦  «        ¦   «         ¦   «         ZA e.d¬¦  «        e G d „ d!e,¦  «        ¦   «         ¦   «         ZB G d"„ d#ejC        ¦  «        ZD G d$„ d%ejC        ¦  «        ZE G d&„ d'ejC        ¦  «        ZF G d(„ d)ejC        ¦  «        ZG G d*„ d+ejC        ¦  «        ZH G d,„ d-ejC        ¦  «        ZI G d.„ d/ejC        ¦  «        ZJ G d0„ d1ejC        ¦  «        ZK G d2„ d3ejC        ¦  «        ZL G d4„ d5ejC        ¦  «        ZM G d6„ d7ejN        ¦  «        ZO G d8„ d9ejC        ¦  «        ZP G d:„ d;ejC        ¦  «        ZQ G d<„ d=ejC        ¦  «        ZRd>„ ZSd?e
jT        d@eUdAe
jT        fdB„ZV	 	 	 dkdDejC        dEe
jT        dFe
jT        dGe
jT        dHe
jT        dz  dIeWeUz  dJeWdz  dKeWdz  dAeXe
jT        e
jT        f         fdL„ZYdldMe
jT        dNe
jT        dOe
jT        dPeUfdQ„ZZ G dR„ dSejC        ¦  «        Z[ G dT„ dUe¦  «        Z\e. G dV„ dWe(¦  «        ¦   «         Z] G dX„ dYe]¦  «        Z^ G dZ„ d[ejC        ¦  «        Z_ e.d\¬¦  «         G d]„ d^e]¦  «        ¦   «         Z` e.d_¬¦  «         G d`„ dae]e¦  «        ¦   «         Za G db„ dcejC        ¦  «        Zb e.dd¬¦  «         G de„ dfe]¦  «        ¦   «         Zc e.dg¬¦  «         G dh„ die]e¦  «        ¦   «         Zdg dj¢ZedS )mé    N)ÚUserDict)ÚCallableÚSequence)Ú	dataclass)ÚOptionalé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_accelerate_availableÚtorch_compilable_check)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )Ú	AutoModelé   )ÚGemma3nAudioConfigÚGemma3nConfigÚGemma3nTextConfigÚGemma3nVisionConfig)Úadd_hook_to_modulec                   ó2   — e Zd ZU dZdZej        dz  ed<   dS )ÚGemma3nAudioEncoderModelOutputzy
    audio_mel_mask (`torch.BoolTensor`, *optional*):
        A torch.BoolTensor of shape `(batch_size, num_frames)`
    NÚaudio_mel_mask)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r,   ÚtorchÚ
BoolTensorÚ__annotations__© ó    új/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/gemma3n/modeling_gemma3n.pyr+   r+   ;   s6   € € € € € € ðð ð
 /3€N�EÔ$ tÑ+Ð2Ð2Ñ2Ð2Ð2r5   r+   zL
    Base class for Gemma3n outputs, with hidden states and attentions.
    ©Úcustom_introc                   óP   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dS )ÚGemma3nModelOutputWithPastaÝ  
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    audio_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        audio_hidden_states of the model produced by the audio encoder and after projecting the last hidden state.
    NÚimage_hidden_statesÚaudio_hidden_states)	r-   r.   r/   r0   r;   r1   ÚFloatTensorr3   r<   r4   r5   r6   r:   r:   F   sP   € € € € € € ðð ð 59Ð˜Ô*¨TÑ1Ð8Ð8Ñ8à48Ð˜Ô*¨TÑ1Ð8Ð8Ñ8Ð8Ð8r5   r:   zS
    Base class for Gemma3n causal language model (or autoregressive) outputs.
    c                   óô   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZej        dz  ed<   dZej        dz  ed	<   dS )
ÚGemma3nCausalLMOutputWithPastaF  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.text_config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder after projecting last hidden state.
    audio_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        audio_hidden_states of the model produced by the audio encoder and after projecting the last hidden state.
    NÚlossÚlogitsÚpast_key_valuesÚhidden_statesÚ
attentionsr;   r<   )r-   r.   r/   r0   r@   r1   r=   r3   rA   rB   r   rC   ÚtuplerD   r;   r<   r4   r5   r6   r?   r?   `   sÎ   € € € € € € ðð ð$ &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø48Ð˜Ô*¨TÑ1Ð8Ð8Ñ8à48Ð˜Ô*¨TÑ1Ð8Ð8Ñ8Ð8Ð8r5   r?   c                   óh   ‡ — e Zd Zddededefˆ fd„Zdej        fd„Z	dej        d	ej        fd
„Z
ˆ xZS )ÚGemma3nRMSNormç�íµ ÷Æ°>TÚdimÚepsÚ
with_scalec                 óÐ   •— t          ¦   «                              ¦   «          || _        || _        | j        r/t	          j        t          j        |¦  «        d¬¦  «        | _        d S d S )NT)Úrequires_grad)	ÚsuperÚ__init__rJ   rK   ÚnnÚ	Parameterr1   ÚonesÚweight)ÚselfrI   rJ   rK   Ú	__class__s       €r6   rO   zGemma3nRMSNorm.__init__„   s`   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ$ˆŒàŒ?ð 	LÝœ,¥u¤z°#¡¤ÀdÐKÑKÔKˆDŒKˆKˆKð	Lð 	Lr5   rC   c                 ó–   — |                      d¦  «                             dd¬¦  «        | j        z   }|t          j         |d¦  «        z  S )Nr"   éÿÿÿÿT)Úkeepdimç      à¿)ÚpowÚmeanrJ   r1   )rT   rC   Úmean_squareds      r6   Ú_normzGemma3nRMSNorm._normŒ   sF   € Ø$×(Ò(¨Ñ+Ô+×0Ò0°¸TÐ0ÑBÔBÀTÄXÑMˆà�uœy¨°tÑ<Ô<Ñ<Ð<r5   Úreturnc                 óÀ   — |                       |                     ¦   «         ¦  «        }| j        r|| j                             ¦   «         z  }|                     |¦  «        S ©N)r]   ÚfloatrK   rS   Útype_as)rT   rC   Únormed_outputs      r6   ÚforwardzGemma3nRMSNorm.forward‘   sV   € ØŸ
š
 =×#6Ò#6Ñ#8Ô#8Ñ9Ô9ˆØŒ?ð 	@Ø)¨D¬K×,=Ò,=Ñ,?Ô,?Ñ?ˆMØ×$Ò$ ]Ñ3Ô3Ð3r5   )rH   T)r-   r.   r/   Úintra   ÚboolrO   r1   ÚTensorr]   rd   Ú__classcell__©rU   s   @r6   rG   rG   ƒ   s¤   ø€ € € € € ðLð L˜Cð L eð LÀð Lð Lð Lð Lð Lð Lð= 5¤<ð =ð =ð =ð =ð
4 U¤\ð 4°e´lð 4ð 4ð 4ð 4ð 4ð 4ð 4ð 4r5   rG   c                   óÄ   ‡ — e Zd Zdefˆ fd„Zdej        dej        dej        fd„Zdej        de	d	e	d
e	de	de	de	dej        fd„Z
dej        dej        dej        fd„Zˆ xZS )Ú%Gemma3nAudioRelativePositionEmbeddingÚconfigc                 ó$  •— t          ¦   «                              ¦   «          || _        | j        j        | _        | j        j        | _        | j        | j        z  | _        t          d| j        j	        dz
  ¦  «        | _
        | j        j        | _        t          j        | j        | j        | j        z  d¬¦  «        | _        d}d}| j        dz  }t!          j        t%          |¦  «        t%          |¦  «        z  ¦  «        t          |dz
  d¦  «        z  }|t'          j        t'          j        |¦  «        | z  ¦  «        z  }|                      d|                     ¦   «                              d¦  «                             d¦  «        d¬	¦  «         d S )
Nr   r$   F©Úbiasç      ð?ç     ˆÃ@r"   Úinv_timescales©Ú
persistent)rN   rO   rl   Úconf_num_attention_headsÚ	num_headsÚhidden_sizeÚchannelsÚhead_dimÚmaxÚconf_attention_context_leftÚmax_backwardÚconf_attention_context_rightÚmax_forwardrP   ÚLinearÚpos_projÚmathÚlogra   r1   ÚexpÚarangeÚregister_bufferÚ	unsqueeze)rT   rl   Úmin_timescaleÚmax_timescaleÚnum_timescalesÚlog_timescale_incrementrr   rU   s          €r6   rO   z.Gemma3nAudioRelativePositionEmbedding.__init__œ   sg  ø€ Ý‰Œ×ÒÑÔÐØˆŒàœÔ=ˆŒØœÔ/ˆŒØœ¨¬Ñ7ˆŒÝ  4¤;Ô#JÈQÑ#NÑOÔOˆÔØœ;ÔCˆÔåœ	 $¤-°´À$Ä-Ñ1OÐV[Ð\Ñ\Ô\ˆŒàˆØˆØœ¨!Ñ+ˆÝ"&¤(­5°Ñ+?Ô+?Å%ÈÑBVÔBVÑ+VÑ"WÔ"WÕZ]Ð^lÐopÑ^pÐrsÑZtÔZtÑ"tÐØ&­¬µ5´<ÀÑ3OÔ3OÐSjÐRjÑ3jÑ)kÔ)kÑkˆØ×ÒØØ× Ò Ñ"Ô"×,Ò,¨QÑ/Ô/×9Ò9¸!Ñ<Ô<Øð 	ñ 	
ô 	
ð 	
ð 	
ð 	
r5   ÚpositionÚdtyper^   c                 óN  — |                      ¦   «                              d¦  «        }|| j                             |j        t
          j        ¬¦  «        z  }t          j        t          j        |¦  «        t          j	        |¦  «        gd¬¦  «        }| 
                    |¦  «        S )NrW   ©ÚdevicerŒ   ©rI   )ra   r†   rr   Útor�   r1   Úfloat32ÚcatÚsinÚcosÚtype)rT   r‹   rŒ   Úscaled_timeÚtiming_signals        r6   Ú_get_timing_signal_1d_posz?Gemma3nAudioRelativePositionEmbedding._get_timing_signal_1d_pos³   s‡   € Ø—>’>Ñ#Ô#×-Ò-¨bÑ1Ô1ˆØ Ô!4×!7Ò!7¸x¼ÕV[ÔVcÐ!7Ñ!dÔ!dÑdˆÝœ	¥5¤9¨[Ñ#9Ô#9½5¼9À[Ñ;QÔ;QÐ"RÐXZÐ[Ñ[Ô[ˆØ×!Ò! %Ñ(Ô(Ð(r5   Úterm_bd_before_shiftÚ
batch_sizerv   Únum_query_blocksÚquery_block_sizeÚkey_context_sizeÚmax_span_plus_1c                 óþ   — |dz   |z
  }d|f}	t           j                             ||	¦  «        }
|
                     |||||dz   z  f¦  «        }|dd…dd…dd…d||z  …f         }|                     |||||f¦  «        }|S )aZ  Performs the relative shift.

        Args:
          term_bd_before_shift: Tensor of shape [B, N, U, W, F_span]. batch_size
            (B), num_heads (N), num_query_blocks (U), query_block_size (W),
            key_context_size (C = W+L+R), max_span_plus_1 (F_span = L+R+1).

        Returns:
          Tensor of shape [B, N, U, W, C].
        r$   r   N)rP   Ú
functionalÚpadÚreshape)rT   rš   r›   rv   rœ   r�   rž   rŸ   Úpad_amount_last_dimÚpadding_tupleÚterm_bd_paddedÚterm_bd_reshapedÚterm_bd_slicedÚterm_bd_shifteds                 r6   Ú_relative_shiftz5Gemma3nAudioRelativePositionEmbedding._relative_shift¹   sÎ   € ð4  0°!Ñ3°ÑFÐð Ð/Ð0ˆåœ×*Ò*Ð+?ÀÑOÔOˆð
 *×1Ò1àØØ Ø Ð$4°qÑ$8Ñ9ð	ñ
ô 
Ðð *¨!¨!¨!¨Q¨Q¨Q°°°Ð3XÐ5EÐHXÑ5XÐ3XÐ*XÔYˆð )×0Ò0àØØ Ø Ø ðñ
ô 
ˆð Ðr5   ÚqueriesÚkeysc           	      óX  — |j         \  }}}}}|j         \  }}}	}}t          j        | j        | j         dz
  d|j        ¬¦  «                             d¦  «        }
|
j         d         }|                      |
|j        ¬¦  «        }|  	                    |¦  «        }| 
                    d|| j        | j        ¦  «                             d¦  «        }|                     ddddd¦  «        }|                     ddddd¦  «        }t          j        ||¦  «        }|                     ddddd¦  «        }|                     ddd¦  «        }| 
                    ||||z  |¦  «        }t          j        ||¦  «        }| 
                    |||||¦  «        }|                      ||||||	|¦  «        }||z   S )	Nr$   rW   ©r�   r   ©rŒ   r   r"   é   )Úshaper1   r„   r|   r~   r�   r†   r™   rŒ   r€   r£   rv   ry   ÚsqueezeÚpermuteÚmatmulrª   )rT   r«   r¬   r›   rœ   r�   rv   ry   Ú_rž   Úpos_indicesrŸ   Úsin_emb_timing_signalÚprojected_sin_embÚsin_embÚ	queries_pÚkeys_p_tÚterm_acÚ
q_permutedÚ
s_permutedÚ
q_reshapedÚterm_bd_unshifed_matmulÚterm_bd_unshifedr©   s                           r6   rd   z-Gemma3nAudioRelativePositionEmbedding.forwardö   sò  € ð OVÌmÑKˆ
Ð$Ð&6¸	À8Ø'+¤zÑ$ˆˆ1Ð  1õ ”l 4Ô#4°tÔ7GÐ6GÈ!Ñ6KÈRÐX_ÔXfÐgÑgÔg×qÒqØñ
ô 
ˆð &Ô+¨AÔ.ˆà $× >Ò >Ø˜wœ}ð !?ñ !
ô !
Ðð
 !ŸMšMÐ*?Ñ@Ô@Ðà#×+Ò+¨A¨ÀÄÐPTÔP]Ñ^Ô^×fÒfØñ
ô 
ˆð —O’O A q¨!¨Q°Ñ2Ô2ˆ	Ø—<’<  1 a¨¨AÑ.Ô.ˆÝ”,˜y¨(Ñ3Ô3ˆð —_’_ Q¨¨1¨a°Ñ3Ô3ˆ
ð —_’_ Q¨¨1Ñ-Ô-ˆ
ð  ×'Ò'¨
°IÐ?OÐRbÑ?bÐdlÑmÔmˆ
õ
 #(¤,¨z¸:Ñ"FÔ"FÐð 3×:Ò:ØØØØØñ
ô 
Ðð ×.Ò.ØØØØØØØñ
ô 
ˆð ˜Ñ(Ð(r5   )r-   r.   r/   r%   rO   r1   rg   rŒ   r™   re   rª   rd   rh   ri   s   @r6   rk   rk   ›   s  ø€ € € € € ð
Ð1ð 
ð 
ð 
ð 
ð 
ð 
ð.)°%´,ð )ÀuÄ{ð )ÐW\ÔWcð )ð )ð )ð )ð;à#œlð;ð ð;ð ð	;ð
 ð;ð ð;ð ð;ð ð;ð 
Œð;ð ;ð ;ð ;ðzL)˜uœ|ð L)°5´<ð L)ÀEÄLð L)ð L)ð L)ð L)ð L)ð L)ð L)ð L)r5   rk   c                   óÐ   ‡ — e Zd Zdefˆ fd„Zd„ Zdej        dededej        fd„Z	d	ej        dej        fd
„Z
d	ej        dej        fd„Zd	ej        dej        dej        fd„Zˆ xZS )ÚGemma3nAudioAttentionrl   c                 ó   •— t          ¦   «                              ¦   «          || _        | j        j        | _        | j        j        | _        | j        | j        z  | _        | j        j        | _        | j        j	        | _
        t          d| j        j        dz
  ¦  «        | _        | j        j        | _        | j        | j        z   | j
        z   | _        t#          |¦  «        | _        t'          j        t+          j        | j        f¦  «        ¦  «        | _        t'          j        | j        | j        | j        z  d¬¦  «        | _        t'          j        | j        | j        | j        z  d¬¦  «        | _        t'          j        | j        | j        | j        z  d¬¦  «        | _        | j        dz  }dt*          j        j                             t+          j        d¦  «        ¦  «        z  }|                      d||z                        ¦   «          !                    ¦   «         d¬	¦  «         |  "                    ¦   «         }|                      d
|d¬	¦  «         |                      dt+          j        | j        ¦  «         #                    ¦   «         d¬	¦  «         d S )Nr   r$   Frn   rY   rp   ç        Úq_scalers   Úlocal_causal_valid_maskÚsoftcap)$rN   rO   rl   ru   rv   rw   ry   Úconf_attention_chunk_sizeÚ
chunk_sizer}   Úmax_future_horizonrz   r{   Úmax_past_horizonÚconf_attention_logit_capÚattention_logits_soft_capÚcontext_sizerk   Úrelative_position_embeddingrP   rQ   r1   ÚzerosÚper_dim_scaler   Úq_projÚk_projÚv_projr¡   ÚsoftplusÚtensorr…   ÚcloneÚdetachÚcreate_local_causal_valid_maskra   )rT   rl   rÆ   Úr_softplus_0rÇ   rU   s        €r6   rO   zGemma3nAudioAttention.__init__F  s-  ø€ Ý‰Œ×ÒÑÔÐØˆŒàœÔ=ˆŒØœ;Ô2ˆÔØÔ(¨D¬NÑ:ˆŒàœ+Ô?ˆŒØ"&¤+Ô"JˆÔÝ # A t¤{Ô'NÐQRÑ'RÑ SÔ SˆÔØ)-¬Ô)MˆÔ&Ø œO¨dÔ.CÑCÀdÔF]Ñ]ˆÔå+PÐQWÑ+XÔ+XˆÔ(Ýœ\­%¬+°t´}Ð6FÑ*GÔ*GÑHÔHˆÔå”i Ô 0°$´.À4Ä=Ñ2PÐW\Ð]Ñ]Ô]ˆŒÝ”i Ô 0°$´.À4Ä=Ñ2PÐW\Ð]Ñ]Ô]ˆŒÝ”i Ô 0°$´.À4Ä=Ñ2PÐW\Ð]Ñ]Ô]ˆŒà”- Ñ%ˆØ�UœXÔ0×9Ò9½%¼,ÀsÑ:KÔ:KÑLÔLÑLˆØ×Ò˜Y¨°<Ñ)?×(FÒ(FÑ(HÔ(H×(OÒ(OÑ(QÔ(QÐ^cÐÑdÔdÐdà"&×"EÒ"EÑ"GÔ"GÐØ×ÒÐ6Ð8OÐ\aÐÑbÔbÐbà×ÒØÝŒL˜Ô7Ñ8Ô8×>Ò>Ñ@Ô@Øð 	ñ 	
ô 	
ð 	
ð 	
ð 	
r5   c                 ó’  — t          j        t          j        | j        | j        ft           j        ¬¦  «        d¬¦  «        j        }t          j        t          j        | j        | j        ft           j        ¬¦  «        | j        | j        z   ¬¦  «        }t          j        | j        | j        ft           j        ¬¦  «        }||z  |z  }|S )Nr¯   r   )Údiagonal)	r1   ÚtrilrR   rÏ   rÊ   rf   ÚTrÌ   rË   )rT   Úlower_causal_maskÚupper_causal_maskrÇ   s       r6   rÚ   z4Gemma3nAudioAttention.create_local_causal_valid_maskh  sÃ   € Ý!œJÝŒJ˜Ô)¨4¬?Ð;Å5Ä:ÐNÑNÔNØð
ñ 
ô 
ô ð 	õ "œJÝŒJ˜œ¨Ô):Ð;Å5Ä:ÐNÑNÔNØÔ*¨TÔ-DÑDð
ñ 
ô 
Ðõ #(¤*¨d¬o¸tÔ?PÐ-QÕY^ÔYcÐ"dÑ"dÔ"dÐØ"9Ð<MÑ"MÐPaÑ"aÐØ&Ð&r5   ÚxÚpad_leftÚ	pad_rightr^   c                 ó´   — |j         ^}}}|                     ||g|¢R ¦  «        }|                     ||g|¢R ¦  «        }t          j        |||gd¬¦  «        }|S )Nr$   r�   )r±   Ú	new_zerosr1   r“   )	rT   râ   rã   rä   Úbatchrµ   Ú
tail_shapeÚleftÚrights	            r6   Ú	_pad_dim1zGemma3nAudioAttention._pad_dim1u  sl   € Ø !¤Ðˆˆq�:Ø�{Š{˜E 8Ð9¨jÐ9Ð9Ñ:Ô:ˆØ—’˜U IÐ;°
Ð;Ð;Ñ<Ô<ˆÝŒI�t˜Q Ð&¨AÐ.Ñ.Ô.ˆØˆr5   rC   c                 ó$  — |j         }|dd…         \  }}|| j        z   dz
  | j        z  }|| j        z  |z
  x}dk    r|                      |d|¦  «        }||| j        f|dd…         z   }|                     |¦  «                             ¦   «         }|S )aE  Turns a sequence to non overlapping blocks.

        Args:
            hidden_states: a tensor of [batch, time, ...].

        Returns:
            A tensor of [batch, num_blocks, block_size, ...], with necessary
            paddings,
            where output[:, i, ...] are x[:, i*block_size:(i+1)*block_size, ...].
        Nr"   r$   r   )r±   rÊ   rë   r£   Ú
contiguous)rT   rC   r±   ÚbÚtÚ
num_blocksÚpadding_lenÚpermute_dimss           r6   Ú_convert_to_blockz'Gemma3nAudioAttention._convert_to_block|  s¦   € ð Ô#ˆØ�R�a�RŒy‰ˆˆ1Ø˜$œ/Ñ)¨AÑ-°$´/ÑAˆ
à%¨¬Ñ7¸!Ñ;Ð;ˆK¸qÒ@Ð@Ø ŸNšN¨=¸!¸[ÑIÔIˆMà˜: t¤Ð7¸%ÀÀÀ¼)ÑCˆØ%×-Ò-¨lÑ;Ô;×FÒFÑHÔHˆØÐr5   c                 ó0  — | j         }| j        | j        z   dz
  }|                      |||¦  «        }| j        }| j        }|                     d||¬¦  «        }|j        dk    r"|j        dk    rt          j        |dd¬¦  «        }| 	                    ¦   «         S )aã  Extracts temporal context for every block.

        Args:
            hidden_states: a tensor of [batch, time, ...].

        Returns:
            A tensor of [batch, num_blocks, context_size, ...], with necessary
            paddings,
            where context_size = block_size + left_context + right_context,
            and output[:, i, ...] are x[:, start-left_context:end+right_context,
            ...],
            start = i * block_size, end = (i + 1) * block_size.
        r$   )Ú	dimensionÚsizeÚstepr"   r   rW   )ÚsourceÚdestination)
rÌ   rË   rÊ   rë   rÏ   ÚunfoldÚndimr1   Úmovedimrí   )rT   rC   rã   rä   Ú	frame_lenÚ
frame_stepÚ
x_unfoldeds          r6   Ú_extract_block_contextz,Gemma3nAudioAttention._extract_block_context’  s¨   € ð Ô(ˆð Ô+¨d¬oÑ=ÀÑAˆ	ØŸš }°hÀ	ÑJÔJˆàÔ%ˆ	Ø”_ˆ
ð #×)Ò)°A¸IÈJÐ)ÑWÔWˆ
ð Ô Ò!Ð! j¤o¸Ò&9Ð&9õ œ z¸"È!ÐLÑLÔLˆJà×$Ò$Ñ&Ô&Ð&r5   Úmaskc                 óò  — g |j         d d…         ¢| j        ‘| j        ‘R }|                      |¦  «                             |¦  «                             ¦   «         }|                      |¦  «                             |¦  «                             ¦   «         }|                      |¦  «                             |¦  «                             ¦   «         }t          j	        j
                             | j        ¦  «        }ddd| j        f}|                     |¦  «        }	|| j        z  |	z  }|j         d d…         \  }
}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|j         d         }| }|                      |¦  «        }|j        dk    r@|j         d         |j         d         z  | j        k    r|                     |
|| j        ¦  «        }|j         |
|| j        fk    r&t'          d|j         › d|
› d|› d| j        › d	�	¦  «        ‚|                     d¦  «                             d
¦  «        }| j                             d¦  «                             d¦  «                             d¦  «        }t          j        ||                     |j        ¦  «        ¦  «        }|                      ||¦  «        }| j                             |j        ¦  «        }||z  }t          j        |¦  «        }||z  }t          j        ||t          j        |j        ¦  «        j        ¦  «        }t          j	        j
                              |dt          j!        ¬¦  «                             |j        ¬¦  «        }|j         \  }}}}}|j         d         }| "                    ddddd¦  «                             d||¦  «        }| "                    ddddd¦  «                             d||¦  «        }t          j#        ||¦  «        } |                      |||||¦  «         "                    ddddd¦  «        }!|!                     |
|| j$        z  | j        | j        f¦  «        }!|!d d …d |…f         }!|!S )NrW   r$   r"   r°   r   z%Shape of extracted_valid_mask_blocks z	 is not (z, z) after potential reshape.éþÿÿÿr   ©rI   rŒ   r¯   )%r±   rv   ry   rÓ   r£   rí   rÔ   rÕ   r1   rP   r¡   rÖ   rÒ   ÚviewrÆ   ró   r   rû   rÏ   Ú
ValueErrorr†   rÇ   Úlogical_andr‘   r�   rÐ   rÈ   ÚtanhÚwhereÚfinforŒ   ÚminÚsoftmaxr’   r³   ÚbmmrÊ   )"rT   rC   r  Ú	qkv_shapeÚquery_statesÚ
key_statesÚvalue_statesÚper_dim_scale_spÚbroadcast_shapeÚper_dim_scale_sp_broadcastr›   Úq_timeÚquery_blocksÚ
key_blocksÚvalue_blocksrœ   Úoriginal_valid_maskÚextracted_valid_mask_blocksÚcondition_from_input_validityÚcondition_from_causalityÚfinal_condition_for_whererA   Úsoftcap_valÚprobabilitiesÚb_dimÚn_dimÚu_dimÚw_dimÚc_dimÚh_dimÚprob_bunÚv_bunÚ
result_bmmÚcontext_vectorss"                                     r6   rd   zGemma3nAudioAttention.forwardÂ  sž  € àN�mÔ)¨#¨2¨#Ô.ÐN°´ÐNÀÄÐNÐNˆ	Ø—{’{ =Ñ1Ô1×9Ò9¸)ÑDÔD×OÒOÑQÔQˆØ—[’[ Ñ/Ô/×7Ò7¸	ÑBÔB×MÒMÑOÔOˆ
Ø—{’{ =Ñ1Ô1×9Ò9¸)ÑDÔD×OÒOÑQÔQˆå œ8Ô.×7Ò7¸Ô8JÑKÔKÐà˜a  D¤MÐ2ˆØ%5×%:Ò%:¸?Ñ%KÔ%KÐ"Ø# d¤lÑ2Ð5OÑOˆà)Ô/°°°Ô3Ñˆ
�Fà×-Ò-¨lÑ;Ô;ˆØ×0Ò0°Ñ<Ô<ˆ
Ø×2Ò2°<Ñ@Ô@ˆØ'Ô-¨aÔ0Ðð  $˜eÐð '+×&AÒ&AÐBUÑ&VÔ&VÐ#ð (Ô,°Ò1Ð1Ø+Ô1°!Ô4Ð7RÔ7XÐYZÔ7[Ñ[Ð_cÔ_pÒpÐpà*E×*MÒ*MØÐ,¨dÔ.?ñ+ô +Ð'ð 'Ô,ØØØÔð1
ò 
ð 
õ
 ðVØ/Ô5ðVð VØ@JðVð Và$ðVð Và(,Ô(9ðVð Vð Vñô ð ð )D×(MÒ(MÈaÑ(PÔ(P×(ZÒ(ZÐ[]Ñ(^Ô(^Ð%ð $(Ô#?×#IÒ#IÈ!Ñ#LÔ#L×#VÒ#VÐWXÑ#YÔ#Y×#cÒ#cÐdeÑ#fÔ#fÐ õ
 %*Ô$5Ø)Ø$×'Ò'Ð(EÔ(LÑMÔMñ%
ô %
Ð!ð ×1Ò1°,À
ÑKÔKˆð ”l—o’o f¤mÑ4Ô4ˆØ˜+Ñ%ˆÝ”˜FÑ#Ô#ˆØ˜+Ñ%ˆõ ”Ð6¸ÅÄÈFÌLÑ@YÔ@YÔ@]Ñ^Ô^ˆÝœÔ+×3Ò3°FÀÍ%Ì-Ð3ÑXÔX×[Ò[ÐbnÔbtÐ[ÑuÔuˆð -:Ô,?Ñ)ˆˆu�e˜U EØÔ" 2Ô&ˆØ ×(Ò(¨¨A¨q°!°QÑ7Ô7×?Ò?ÀÀEÈ5ÑQÔQˆØ×$Ò$ Q¨¨1¨a°Ñ3Ô3×;Ò;¸BÀÀuÑMÔMˆÝ”Y˜x¨Ñ/Ô/ˆ
Ø$×,Ò,¨U°E¸5À%ÈÑOÔO×WÒWÐXYÐ[\Ð^_ÐabÐdeÑfÔfˆØ)×1Ò1àØ  4¤?Ñ2Ø”Ø”ð	ñ
ô 
ˆð *¨!¨!¨!¨W¨f¨W¨*Ô5ˆàÐr5   )r-   r.   r/   r%   rO   rÚ   r1   rg   re   rë   ró   r   r2   rd   rh   ri   s   @r6   rÃ   rÃ   E  s  ø€ € € € € ð 
Ð1ð  
ð  
ð  
ð  
ð  
ð  
ðD'ð 'ð 'ð˜5œ<ð °3ð À3ð È5Ì<ð ð ð ð ð¨u¬|ð ÀÄð ð ð ð ð,.'°E´Lð .'ÀUÄ\ð .'ð .'ð .'ð .'ð`d U¤\ð d¸Ô9Ið dÈeÌlð dð dð dð dð dð dð dð dr5   rÃ   c                   ód   ‡ — e Zd ZdZ	 d
dedee         defˆ fd„Zdej	        dej	        fd	„Z
ˆ xZS )ÚGemma3nAudioCumulativeGroupNormaè  Applies Group Normalization cumulatively over the time dimension.

    This layer normalizes the input by calculating the mean and variance
    cumulatively over the time dimension (dim 1). The statistics are computed
    over all feature dimensions (specified by `feature_dims` and `num_channels`)
    for elements marked as valid by the optional `mask`.

    If a `mask` is provided (True for valid, False for invalid/padded),
    invalid time steps do not contribute to the statistics calculation, and
    their corresponding output values are zeroed out.

    Scale and bias, if enabled, are applied per-channel (last dimension).
    This behavior is similar to JAX's `GroupNormalization` with `num_groups=1`
    and `cumulative=True`.
    çü©ñÒMbP?Únum_channelsÚfeature_dimsrJ   c           	      óV  •— t          ¦   «                              ¦   «          || _        t          |¦  «        | _        || _        t          j        t          j	        |¦  «        ¦  «        | _
        t          t          ddt          | j        ¦  «        z   dz   ¦  «        ¦  «        | _        d S )Nr"   r$   )rN   rO   r-  rE   r.  rJ   rP   rQ   r1   rR   rS   ÚrangeÚlenÚreduction_axes)rT   r-  r.  rJ   rU   s       €r6   rO   z(Gemma3nAudioCumulativeGroupNorm.__init__:  sŽ   ø€ õ 	‰Œ×ÒÑÔÐØ(ˆÔÝ! ,Ñ/Ô/ˆÔØˆŒõ ”l¥5¤:¨lÑ#;Ô#;Ñ<Ô<ˆŒõ $¥E¨!¨Qµ°TÔ5FÑ1GÔ1GÑ-GÈ!Ñ-KÑ$LÔ$LÑMÔMˆÔÐÐr5   rC   r^   c                 óÈ  — | j         | j        fz   }|j        dd…         |k    r"t          d|j        dd…         › d|› �¦  «        ‚|j        }t
          j        }|                     |¦  «        }t          j        ||¬¦  «        }t          j	        || j
        d¬¦  «        }t          j        |d¬	¦  «        }t          j	        || j
        d¬¦  «        }	t          j        |	d¬	¦  «        }
t          j        |
d
¬¦  «        }||z  }||z
                       d¦  «        }t          j	        || j
        d¬¦  «        }t          j        |d¬	¦  «        }||z  }||z
  t          j        || j        z   ¦  «        z  }| j                             |¦  «        }dg|                     ¦   «         dz
  z  | j        gz   }||                     |¦  «        z  }||z  }|                     |¦  «        S )zÞApplies cumulative group norm, optionally using a mask.

        Args:
          hidden_states: Input tensor, shape [B, T, *feature_dims, C].

        Returns:
          Normalized tensor with the same shape as x.
        r"   NzInput tensor shape suffix z> does not match expected suffix (feature_dims + num_channels) r¯   T©rI   rX   r$   r�   rp   )r  )r.  r-  r±   r  rŒ   r1   r’   r‘   Ú	ones_likeÚsumr2  ÚcumsumÚclamprZ   ÚrsqrtrJ   rS   rI   r  )rT   rC   Úexpected_input_suffixÚinput_dtypeÚ
calc_dtypeÚx_calcÚ	mask_calcÚsum_values_at_tÚcum_sum_valuesÚelements_in_group_at_tÚcum_count_elementsÚsafe_cum_count_elementsÚcum_meanÚsquared_diff_from_meanÚsum_sq_diff_at_tÚcum_sum_sq_diffÚcum_varianceÚnormalized_xÚscaleÚscale_view_shapeÚfinal_outputs                        r6   rd   z'Gemma3nAudioCumulativeGroupNorm.forwardL  s   € ð !%Ô 1°TÔ5FÐ4HÑ HÐØÔ˜q˜r˜rÔ"Ð&;Ò;Ð;ÝðQ¨]Ô-@ÀÀÀÔ-Dð Qð QØ9NðQð Qñô ð ð
 $Ô)ˆå”]ˆ
Ø×!Ò! *Ñ-Ô-ˆõ ”O F°*Ð=Ñ=Ô=ˆ	õ  œ) F°Ô0CÈTÐRÑRÔRˆåœ o¸1Ð=Ñ=Ô=ˆõ "'¤¨9¸$Ô:MÐW[Ð!\Ñ!\Ô!\Ðå"œ\Ð*@ÀaÐHÑHÔHÐå"'¤+Ð.@ÀcÐ"JÑ"JÔ"JÐð "Ð$;Ñ;ˆð
 #)¨8Ñ"3×!8Ò!8¸Ñ!;Ô!;ÐÝ œ9Ð%;ÀÔATÐ^bÐcÑcÔcÐõ  œ,Ð'7¸QÐ?Ñ?Ô?ˆð 'Ð)@Ñ@ˆð  Ñ)­U¬[¸ÈÌÑ9PÑ-QÔ-QÑQˆð ”—’˜zÑ*Ô*ˆà˜3 -×"3Ò"3Ñ"5Ô"5¸Ñ"9Ñ:¸dÔ>OÐ=PÑPÐØ# e§j¢jÐ1AÑ&BÔ&BÑBˆð $ iÑ/ˆà�Š˜{Ñ+Ô+Ð+r5   )r,  )r-   r.   r/   r0   re   r   ra   rO   r1   rg   rd   rh   ri   s   @r6   r+  r+  )  s«   ø€ € € € € ðð ð( ð	Nð NàðNð ˜s”mðNð ð	Nð Nð Nð Nð Nð Nð$G, U¤\ð G,°e´lð G,ð G,ð G,ð G,ð G,ð G,ð G,ð G,r5   r+  c                   óp   ‡ — e Zd ZdZ	 ddedededeeeeef         fˆ fd„Zdej	        d	ej	        fd
„Z
ˆ xZS )ÚGemma3nAudioSSCPConvBlockzÙA single convolution block for the SubSampleConvProjection.

    This block consists of a 2D convolution, followed by CumulativeGroupNorm,
    and a ReLU activation. It handles manual padding for the convolution.
    ©r   r   r   r   rl   ÚidxÚinput_freq_dimÚmanual_paddingc                 ó"  •— t          ¦   «                              ¦   «          || _        || _        |dk    rdn| j        j        |dz
           }| j        j        |         }| j        j        |         \  }}| j        j        |         \  }	}
t          j        ||||f|	|
fdd¬¦  «        | _	        || j        d         z   | j        d         z   }||z
  |
z  dz   }t          ||f| j        j        ¬¦  «        | _        t          j        ¦   «         | _        d S )Nr   r$   )r   r   F)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingro   )r-  r.  rJ   )rN   rO   rl   rR  Ússcp_conv_channel_sizeÚsscp_conv_kernel_sizeÚsscp_conv_stride_sizerP   ÚConv2dÚconvr+  Ússcp_conv_group_norm_epsÚnormÚReLUÚ
activation)rT   rl   rP  rQ  rR  rT  rU  Úkernel_hÚkernel_wÚstride_hÚstride_wÚf_in_paddedÚ
f_out_convrU   s                €r6   rO   z"Gemma3nAudioSSCPConvBlock.__init__�  s+  ø€ õ 	‰Œ×ÒÑÔÐØˆŒØ,ˆÔð  !š8˜8�a�a¨¬Ô)KÈCÐRSÉGÔ)TˆØ”{Ô9¸#Ô>ˆØ!œ[Ô>¸sÔCÑˆ�(Ø!œ[Ô>¸sÔCÑˆ�(å”IØ#Ø%àØðð ˜hÐ'ØØð

ñ 

ô 

ˆŒ	ð % tÔ':¸1Ô'=Ñ=ÀÔ@SÐTUÔ@VÑVˆØ! HÑ,°Ñ9¸AÑ=ˆ
å3Ø%Ø$˜Ø”Ô4ð
ñ 
ô 
ˆŒ	õ œ'™)œ)ˆŒˆˆr5   Úaudio_encodingsr^   c                 ó¦  — t          j        || j        dd¬¦  «                             | j        j        j        ¦  «        }|                      |¦  «        }|                     dddd¦  «                             ¦   «         }|  	                    |¦  «        }|                     dddd¦  «                             ¦   «         }|  
                    |¦  «        S )NÚconstantrÅ   )ÚmodeÚvaluer   r"   r   r$   )ÚFr¢   rR  r‘   r]  rS   rŒ   r³   rí   r_  ra  )rT   rh  Úaudio_encodings_paddedÚaudio_encodings_convÚ
x_for_normÚx_normedÚaudio_encodings_normeds          r6   rd   z!Gemma3nAudioSSCPConvBlock.forwardÈ  sÃ   € õ "#¤ ¸Ô8KÐR\ÐdgÐ!hÑ!hÔ!h×!kÒ!kØŒIÔÔ"ñ"
ô "
Ðð
  $ŸyšyÐ)?Ñ@Ô@Ðð *×1Ò1°!°Q¸¸1Ñ=Ô=×HÒHÑJÔJˆ
Ø—9’9˜ZÑ(Ô(ˆà!)×!1Ò!1°!°Q¸¸1Ñ!=Ô!=×!HÒ!HÑ!JÔ!JÐØ�ŠÐ5Ñ6Ô6Ð6r5   )rO  )r-   r.   r/   r0   r%   re   rE   rO   r1   rg   rd   rh   ri   s   @r6   rN  rN  –  s­   ø€ € € € € ðð ð 5Að)$ð )$à"ð)$ð ð)$ð ð	)$ð
 ˜c 3¨¨SÐ0Ô1ð)$ð )$ð )$ð )$ð )$ð )$ðV7 u¤|ð 7¸¼ð 7ð 7ð 7ð 7ð 7ð 7ð 7ð 7r5   rN  c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )Ú#Gemma3nAudioSubSampleConvProjectionrl   c                 ó’  •— t          ¦   «                              ¦   «          || _        |j        }g }g }t	          d¦  «        D ]r}|j        |         \  }}|j        |         \  }}	d}
|dz
  }d}d}|||
|f}|                     |¦  «         ||z   |z   }||z
  |	z  dz   }|                     |¦  «         |}Œst          d|j        ||d         ¬¦  «        | _	        t          d|d         ||d         ¬¦  «        | _
        |j        d         }|d         }||z  | _        t          j        | j        | j        j        d¬¦  «        | _        d S )Nr"   r   r$   )rP  rQ  rl   rR  rW   Frn   )rN   rO   rl   Úinput_feat_sizer0  rZ  r[  ÚappendrN  Úconv_0Úconv_1rY  Úinput_proj_in_featuresrP   r   rw   Úinput_proj_linear)rT   rl   Úcurrent_f_for_block_inputÚcalculated_block_paddingÚcalculated_f_out_dimsÚirb  rc  rd  re  Ú	pad_t_topÚpad_t_bottomÚ
pad_f_leftÚpad_f_rightÚmanual_padding_tuplerf  Úf_out_after_convÚfinal_c_outÚfinal_f_outrU   s                      €r6   rO   z,Gemma3nAudioSubSampleConvProjection.__init__Ü  s¡  ø€ Ý‰Œ×ÒÑÔÐØˆŒà$*Ô$:Ð!Ø#%Ð Ø "Ðå�q‘”ð 	9ð 	9ˆAØ!'Ô!=¸aÔ!@ÑˆH�hØ!'Ô!=¸aÔ!@ÑˆH�hð ˆIØ# a™<ˆLð ˆJØˆKð ØØØð	$Ð ð %×+Ò+Ð,@ÑAÔAÐAð 4°jÑ@À;ÑNˆKØ +¨hÑ 6¸8ÑCÀaÑGÐØ!×(Ò(Ð)9Ñ:Ô:Ð:Ø(8Ð%Ð%å/ØØ!Ô1ØØ3°AÔ6ð	
ñ 
ô 
ˆŒõ 0ØØ0°Ô3ØØ3°AÔ6ð	
ñ 
ô 
ˆŒð Ô3°BÔ7ˆØ+¨BÔ/ˆØ&1°KÑ&?ˆÔ#Ý!#¤¨4Ô+FÈÌÔH_ÐfkÐ!lÑ!lÔ!lˆÔÐÐr5   rh  r^   c                 óN  — |                      d¦  «        }|                      |¦  «        }|                      |¦  «        }|j        \  }}}}|                     dddd¦  «                             ¦   «         }|                     ||||z  ¦  «        }	|                      |	¦  «        }
|
S )Nr$   r   r"   r   )r†   rx  ry  r±   r³   rí   r  r{  )rT   rh  Úaudio_encodings_reshapedrâ   rî   Úc_outÚt_outÚf_outÚ
x_permutedÚoutput_flattenedÚoutputs              r6   rd   z+Gemma3nAudioSubSampleConvProjection.forward  sŸ   € ð $3×#<Ò#<¸QÑ#?Ô#?Ð Ø�KŠKÐ0Ñ1Ô1ˆØ�KŠK˜‰NŒNˆà!"¤Ñˆˆ5�%˜à—Y’Y˜q ! Q¨Ñ*Ô*×5Ò5Ñ7Ô7ˆ
Ø%Ÿ?š?¨1¨e°U¸U±]ÑCÔCÐØ×'Ò'Ð(8Ñ9Ô9ˆØˆr5   ©	r-   r.   r/   r%   rO   r1   rg   rd   rh   ri   s   @r6   rt  rt  Û  ss   ø€ € € € € ð7mÐ1ð 7mð 7mð 7mð 7mð 7mð 7mðr u¤|ð ¸¼ð ð ð ð ð ð ð ð r5   rt  c                   óV   ‡ — e Zd Zdefˆ fd„Zdej        dej        dej        fd„Zˆ xZ	S )ÚGemma3nAudioConformerAttentionrl   c                 óÖ  •— t          ¦   «                              ¦   «          || _        | j        j        | _        |                      dt          j        | j        j        ¦  «        d¬¦  «         t          | j        j        ¦  «        | _
        t          |¦  «        | _        t          j        | j        | j        j        d¬¦  «        | _        t          | j        j        ¦  «        | _        d S )NÚgradient_clippingFrs   rn   )rN   rO   rl   rw   Úpost_in_featuresr…   r1   r×   r”  rG   Úpre_attn_normrÃ   ÚattnrP   r   ÚpostÚ	post_norm©rT   rl   rU   s     €r6   rO   z'Gemma3nAudioConformerAttention.__init__%  s¶   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ $¤Ô 7ˆÔØ×ÒÐ0µ%´,¸t¼{Ô?\Ñ2]Ô2]ÐjoÐÑpÔpÐpÝ+¨D¬KÔ,CÑDÔDˆÔÝ)¨&Ñ1Ô1ˆŒ	Ý”I˜dÔ3°T´[Ô5LÐSXÐYÑYÔYˆŒ	Ý'¨¬Ô(?Ñ@Ô@ˆŒˆˆr5   rh  r,   r^   c                 ó†  — |}t          j        || j         | j        ¦  «        }|                      |¦  «        }|                      ||¦  «        }|j        \  }}}}	|                     ||||	z  ¦  «        }
|                      |
¦  «        }t          j        || j         | j        ¦  «        }||                      |¦  «        z   S r`   )	r1   r8  r”  r–  r—  r±   r£   r˜  r™  )rT   rh  r,   Úaudio_encodings_input_to_attnÚaudio_encodings_normÚaudio_encodings_attn_outrî   rï   rv   ry   r‰  s              r6   rd   z&Gemma3nAudioConformerAttention.forward/  sÅ   € Ø(7Ð%Ýœ+ o¸Ô8NÐ7NÐPTÔPfÑgÔgˆØ#×1Ò1°/ÑBÔBÐà#'§9¢9Ð-AÀ>Ñ#RÔ#RÐ ð %=Ô$BÑ!ˆˆ1ˆi˜Ø#;×#CÒ#CÀAÀqÈ)ÐV^ÑJ^Ñ#_Ô#_Ð àŸ)š)Ð$<Ñ=Ô=ˆÝœ+ o¸Ô8NÐ7NÐPTÔPfÑgÔgˆØ,¨t¯~ª~¸oÑ/NÔ/NÑNÐNr5   ©
r-   r.   r/   r%   rO   r1   rg   r2   rd   rh   ri   s   @r6   r’  r’  $  s‰   ø€ € € € € ðAÐ1ð Að Að Að Að Að AðO u¤|ð OÀUÔEUð OÐZ_ÔZfð Oð Oð Oð Oð Oð Oð Oð Or5   r’  c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )Ú Gemma3nAudioConformerFeedForwardrl   c                 ó$  •— t          ¦   «                              ¦   «          || _        |                      dt	          j        | j        j        ¦  «        d¬¦  «         t          | j        j        ¦  «        | _	        t          j        | j        j        | j        j        dz  d¬¦  «        | _        t          j        | j        j        dz  | j        j        d¬¦  «        | _        t          | j        j        ¦  «        | _        | j        j        | _        d S )Nr”  Frs   r°   rn   )rN   rO   rl   r…   r1   r×   r”  rG   rw   Úpre_layer_normrP   r   Úffw_layer_1Úffw_layer_2Úpost_layer_normÚconf_residual_weightÚpost_layer_scalerš  s     €r6   rO   z)Gemma3nAudioConformerFeedForward.__init__A  sÜ   ø€ Ý‰Œ×ÒÑÔÐØˆŒà×ÒÐ0µ%´,¸t¼{Ô?\Ñ2]Ô2]ÐjoÐÑpÔpÐpå,¨T¬[Ô-DÑEÔEˆÔÝœ9 T¤[Ô%<¸d¼kÔ>UÐXYÑ>YÐ`eÐfÑfÔfˆÔÝœ9 T¤[Ô%<¸qÑ%@À$Ä+ÔBYÐ`eÐfÑfÔfˆÔÝ-¨d¬kÔ.EÑFÔFˆÔØ $¤Ô @ˆÔÐÐr5   rh  r^   c                 óŠ  — |}t          j        || j         | j        ¦  «        }|                      |¦  «        }|                      |¦  «        }t
          j                             |¦  «        }|                      |¦  «        }t          j        || j         | j        ¦  «        }|  	                    |¦  «        }||| j
        z  z   S r`   )r1   r8  r”  r£  r¤  rP   r¡   Úsilur¥  r¦  r¨  )rT   rh  Úresiduals      r6   rd   z(Gemma3nAudioConformerFeedForward.forwardM  sµ   € Ø"ˆÝœ+ o¸Ô8NÐ7NÐPTÔPfÑgÔgˆØ×-Ò-¨oÑ>Ô>ˆØ(,×(8Ò(8¸Ñ(IÔ(IˆÝœ-×,Ò,¨_Ñ=Ô=ˆØ(,×(8Ò(8¸Ñ(IÔ(IˆÝœ+ o¸Ô8NÐ7NÐPTÔPfÑgÔgˆØ×.Ò.¨Ñ?Ô?ˆØ˜?¨TÔ-BÑBÑCÐCr5   r�  ri   s   @r6   r¡  r¡  @  s|   ø€ € € € € ð
AÐ1ð 
Að 
Að 
Að 
Að 
Að 
Að	D u¤|ð 	D¸¼ð 	Dð 	Dð 	Dð 	Dð 	Dð 	Dð 	Dð 	Dr5   r¡  c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )Ú Gemma3nAudioConformerLightConv1drl   c           	      óä  •— t          ¦   «                              ¦   «          || _        t          | j        j        | j        j        ¬¦  «        | _        t          j        | j        j        | j        j        dz  d¬¦  «        | _	        t          j
        | j        j        | j        j        | j        j        dd| j        j        d¬¦  «        | _        |                      dt          j        | j        j        ¦  «        d¬	¦  «         t          | j        j        | j        j        ¬¦  «        | _        t          j        | j        j        | j        j        d¬¦  «        | _        | j        j        dz
  | _        d S )
N©rJ   r"   Frn   r$   r   )rT  rU  rV  rW  rX  Úgroupsro   r”  rs   )rN   rO   rl   rG   rw   Úrms_norm_epsr£  rP   r   Úlinear_startÚConv1dÚconf_conv_kernel_sizeÚdepthwise_conv1dr…   r1   r×   r”  Ú	conv_normÚ
linear_endÚcausal_paddingrš  s     €r6   rO   z)Gemma3nAudioConformerLightConv1d.__init__Z  s2  ø€ Ý‰Œ×ÒÑÔÐØˆŒå,¨T¬[Ô-DÈ$Ì+ÔJbÐcÑcÔcˆÔÝœI d¤kÔ&=¸t¼{Ô?VÐYZÑ?ZÐafÐgÑgÔgˆÔÝ "¤	ØœÔ/ØœÔ0ØœÔ9ØØØ”;Ô*Øð!
ñ !
ô !
ˆÔð 	×ÒÐ0µ%´,¸t¼{Ô?\Ñ2]Ô2]ÐjoÐÑpÔpÐpÝ'¨¬Ô(?ÀTÄ[ÔE]Ð^Ñ^Ô^ˆŒÝœ) D¤KÔ$;¸T¼[Ô=TÐ[`ÐaÑaÔaˆŒà"œkÔ?À!ÑCˆÔÐÐr5   rh  r^   c                 óF  — |}|                       |¦  «        }|                      |¦  «        }t          j        j                             |d¬¦  «        }|                     ddd¦  «        }t          j        || j	        df¦  «        }|  
                    |¦  «        }|                     ddd¦  «        }t          j        || j         | j        ¦  «        }|                      |¦  «        }t          j                             |¦  «        }|                      |¦  «        }||z   }|S )NrW   r�   r   r"   r$   )r£  r²  r1   rP   r¡   Úglur³   rm  r¢   r¸  rµ  r8  r”  r¶  rª  r·  )rT   rh  Úaudio_encodings_residualÚaudio_encodings_permutedÚaudio_encodings_permuted_paddedr�  s         r6   rd   z(Gemma3nAudioConformerLightConv1d.forwardo  s  € Ø#2Ð à×-Ò-¨oÑ>Ô>ˆØ×+Ò+¨OÑ<Ô<ˆÝœ(Ô-×1Ò1°/ÀrÐ1ÑJÔJˆà#2×#:Ò#:¸1¸aÀÑ#CÔ#CÐ å*+¬%Ð0HÈ4ÔK^Ð`aÐJbÑ*cÔ*cÐ'Ø×/Ò/Ð0OÑPÔPˆà)×1Ò1°!°Q¸Ñ:Ô:ˆÝœ+ o¸Ô8NÐ7NÐPTÔPfÑgÔgˆØŸ.š.¨Ñ9Ô9ˆÝœ-×,Ò,¨_Ñ=Ô=ˆØŸ/š/¨/Ñ:Ô:ˆØ Ð#;Ñ;ˆØˆr5   r�  ri   s   @r6   r­  r­  Y  sr   ø€ € € € € ðDÐ1ð Dð Dð Dð Dð Dð Dð* u¤|ð ¸¼ð ð ð ð ð ð ð ð r5   r­  c                   óV   ‡ — e Zd Zdefˆ fd„Zdej        dej        dej        fd„Zˆ xZ	S )ÚGemma3nAudioConformerBlockrl   c                 óÂ  •— t          ¦   «                              ¦   «          || _        t          | j        ¦  «        | _        t          | j        ¦  «        | _        t          | j        ¦  «        | _        t          | j        ¦  «        | _	        |  
                    dt          j        | j        j        ¦  «        d¬¦  «         t          | j        j        ¦  «        | _        d S )Nr”  Frs   )rN   rO   rl   r¡  Úffw_layer_startr’  Ú	attentionr­  Úlconv1dÚffw_layer_endr…   r1   r×   r”  rG   rw   r_  rš  s     €r6   rO   z#Gemma3nAudioConformerBlock.__init__…  sª   ø€ Ý‰Œ×ÒÑÔÐØˆŒå?ÀÄÑLÔLˆÔÝ7¸¼ÑDÔDˆŒÝ7¸¼ÑDÔDˆŒÝ=¸d¼kÑJÔJˆÔØ×ÒÐ0µ%´,¸t¼{Ô?\Ñ2]Ô2]ÐjoÐÑpÔpÐpÝ" 4¤;Ô#:Ñ;Ô;ˆŒ	ˆ	ˆ	r5   rh  r,   r^   c                 ó‚  — |                       |¦  «        }|                      ||¦  «        }| }||                     d¦  «                             |j        ¦  «        z  }|                      |¦  «        }|                      |¦  «        }t          j        || j	         | j	        ¦  «        }|  
                    |¦  «        }|S )NrW   )rÁ  rÂ  r†   r‘   rŒ   rÃ  rÄ  r1   r8  r”  r_  )rT   rh  r,   Úvalidity_mask_for_lconvÚaudio_encodings_for_lconv_inputr�  s         r6   rd   z"Gemma3nAudioConformerBlock.forward�  s½   € Ø×.Ò.¨Ñ?Ô?ˆØŸ.š.¨¸.ÑIÔIˆØ#1 /ÐØ*9Ð<S×<]Ò<]Ð^`Ñ<aÔ<a×<dÒ<dØÔ!ñ=
ô =
ñ +
Ð'ð Ÿ,š,Ð'FÑGÔGˆà×,Ò,¨_Ñ=Ô=ˆÝœ+ o¸Ô8NÐ7NÐPTÔPfÑgÔgˆØ—’˜?Ñ+Ô+ˆØˆr5   rŸ  ri   s   @r6   r¿  r¿  „  sw   ø€ € € € € ð	<Ð1ð 	<ð 	<ð 	<ð 	<ð 	<ð 	<ð u¤|ð ÀUÔEUð ÐZ_ÔZfð ð ð ð ð ð ð ð r5   r¿  c            	       óP   ‡ — e Zd ZdZd
dedededefˆ fd„Zdej        fˆ fd	„Z	ˆ xZ
S )ÚGemma3nTextScaledWordEmbeddingz\
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    rp   Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scalec                 ó°   •— t          ¦   «                              |||¦  «         || _        |                      dt	          j        |¦  «        d¬¦  «         d S )NrÍ  Frs   )rN   rO   Úscalar_embed_scaler…   r1   r×   )rT   rÊ  rË  rÌ  rÍ  rU   s        €r6   rO   z'Gemma3nTextScaledWordEmbedding.__init__¤  sS   ø€ Ý‰Œ×Ò˜¨¸ÑDÔDÐDØ"-ˆÔØ×Ò˜]­E¬L¸Ñ,EÔ,EÐRWÐÑXÔXÐXÐXÐXr5   Ú	input_idsc                 ó�   •— t          ¦   «                              |¦  «        | j                             | j        j        ¦  «        z  S r`   )rN   rd   rÍ  r‘   rS   rŒ   )rT   rÐ  rU   s     €r6   rd   z&Gemma3nTextScaledWordEmbedding.forward©  s4   ø€ Ý‰wŒw�Š˜yÑ)Ô)¨DÔ,<×,?Ò,?ÀÄÔ@QÑ,RÔ,RÑRÐRr5   )rp   )r-   r.   r/   r0   re   ra   rO   r1   rg   rd   rh   ri   s   @r6   rÉ  rÉ  Ÿ  s¦   ø€ € € € € ðð ðYð Y sð Y¸3ð YÈSð YÐ_dð Yð Yð Yð Yð Yð Yð
S ¤ð Sð Sð Sð Sð Sð Sð Sð Sð Sð Sr5   rÉ  c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚGemma3nTextLaurelBlockz Learned Augmented Residual Layerrl   c                 ój  •— t          ¦   «                              ¦   «          || _        t          j        | j        j        | j        j        d¬¦  «        | _        t          j        | j        j        | j        j        d¬¦  «        | _        t          | j        j        | j        j
        ¬¦  «        | _        d S )NFrn   r¯  )rN   rO   rl   rP   r   rw   Úlaurel_rankÚlinear_leftÚlinear_rightrG   r±  Úpost_laurel_normrš  s     €r6   rO   zGemma3nTextLaurelBlock.__init__°  s�   ø€ Ý‰Œ×ÒÑÔÐØˆŒåœ9 T¤[Ô%<¸d¼kÔ>UÐ\aÐbÑbÔbˆÔÝœI d¤kÔ&=¸t¼{Ô?VÐ]bÐcÑcÔcˆÔÝ .¨t¬{Ô/FÈDÌKÔLdÐ eÑ eÔ eˆÔÐÐr5   rC   r^   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }||z   S r`   )rÖ  r×  rØ  )rT   rC   Úlaurel_hidden_statesÚnormed_laurel_hidden_statess       r6   rd   zGemma3nTextLaurelBlock.forward¸  sL   € Ø-1×-=Ò-=¸mÑ-LÔ-LÐØ-1×->Ò->Ð?SÑ-TÔ-TÐØ&*×&;Ò&;Ð<PÑ&QÔ&QÐ#ØÐ:Ñ:Ð:r5   )
r-   r.   r/   r0   r'   rO   r1   rg   rd   rh   ri   s   @r6   rÓ  rÓ  ­  sx   ø€ € € € € Ø*Ð*ðfÐ0ð fð fð fð fð fð fð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ð ;ð ;ð ;ð ;r5   rÓ  c                   ór   ‡ — e Zd Zd
dedefˆ fd„Zdej        dej        fd„Zdej        dej        fd	„Z	ˆ xZ
S )ÚGemma3nTextMLPr   rl   Ú	layer_idxc                 óÈ  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        |         | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _	        t          |j                 | _        |j        |         | _        d S ©NFrn   )rN   rO   rl   rw   Úintermediate_sizerP   r   Ú	gate_projÚup_projÚ	down_projr
   Úhidden_activationÚact_fnÚactivation_sparsity_patternÚactivation_sparsity©rT   rl   rÞ  rU   s      €r6   rO   zGemma3nTextMLP.__init__À  s½   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9¸)Ô!DˆÔÝœ 4Ô#3°TÔ5KÐRWÐXÑXÔXˆŒÝ”y Ô!1°4Ô3IÐPUÐVÑVÔVˆŒÝœ 4Ô#9¸4Ô;KÐRWÐXÑXÔXˆŒÝ˜VÔ5Ô6ˆŒØ#)Ô#EÀiÔ#PˆÔ Ð Ð r5   rC   r^   c                 óô   — |                       |¦  «        }| j        dk    r|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      ||z  ¦  «        }|S )NrÅ   )râ  rè  Ú_gaussian_topkræ  rã  rä  )rT   rC   râ  Úactivationsrã  rä  s         r6   rd   zGemma3nTextMLP.forwardË  sr   € Ø—N’N =Ñ1Ô1ˆ	ØÔ# cÒ)Ð)Ø×+Ò+¨IÑ6Ô6ˆIØ—k’k )Ñ,Ô,ˆØ—,’,˜}Ñ-Ô-ˆØ—N’N ;°Ñ#8Ñ9Ô9ˆ	ØÐr5   Úinputsc                 ó²  — t          j        | j        t           j        |j        ¬¦  «        }t           j        j                             dd¦  «        }|                     |¦  «        }| 	                    |j
        ¦  «        }t          j        |dd¬¦  «        }t          j        |ddd¬¦  «        }|||z  z   }t          j                             ||z
  ¦  «        S )	N©rŒ   r�   r   r$   rW   Tr4  F)rI   rX   Úunbiased)r1   r×   rè  r’   r�   ÚdistributionsÚnormalÚNormalÚicdfr–   rŒ   r[   ÚstdrP   r¡   Úrelu)rT   rí  Útarget_sparsity_tensorÚnormal_distÚstd_multiplierÚinputs_meanÚ
inputs_stdÚcutoff_xs           r6   rë  zGemma3nTextMLP._gaussian_topkÔ  sÀ   € Ý!&¤¨dÔ.FÍeÌmÐdjÔdqÐ!rÑ!rÔ!rÐõ Ô)Ô0×7Ò7¸¸1Ñ=Ô=ˆØ'2×'7Ò'7Ð8NÑ'OÔ'OˆØ'×,Ò,¨V¬\Ñ:Ô:ˆÝ”j ¨R¸Ð>Ñ>Ô>ˆÝ”Y˜v¨2°tÀeÐLÑLÔLˆ
Ø ¨nÑ!<Ñ<ˆÝŒ}×!Ò! &¨8Ñ"3Ñ4Ô4Ð4r5   )r   )r-   r.   r/   r'   re   rO   r1   rg   rd   rë  rh   ri   s   @r6   rÝ  rÝ  ¿  s¦   ø€ € € € € ð	Qð 	QÐ0ð 	Q¸Sð 	Qð 	Qð 	Qð 	Qð 	Qð 	Qð U¤\ð °e´lð ð ð ð ð5 U¤\ð 5°e´lð 5ð 5ð 5ð 5ð 5ð 5ð 5ð 5r5   rÝ  c                   óê   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zdej        dej        fd„Z	d	ej        d
ej        dej        fd„Z
dej        dej        fd„Zdej        dej        fd„Zˆ xZS )ÚGemma3nTextAltUpa�  Alternating Updates (AltUp)

    The AltUp module wraps transformer layers. The `predict` step modifies the
    input to the transformer layer, and the `correct` step propagates the output
    of the transformer layer to the sparsely updated dimensions.

    See more in the research paper:

    https://proceedings.neurips.cc/paper_files/paper/2023/file/f2059277ac6ce66e7e5543001afa8bb5-Paper-Conference.pdf
    rl   c                 ó¨  •— t          ¦   «                              ¦   «          || _        t          j        t          j        | j        j        ¦  «        ¦  «        | _        t          j	        | j        j
        | j        j
        d¬¦  «        | _        t          j	        | j        j
        | j        j
        dz  d¬¦  «        | _        t          j	        | j        j        | j        j
        d¬¦  «        | _        t          | j        j        | j        j        ¬¦  «        | _        |                      dt          j        | j        j        dz  ¦  «        d¬¦  «         d S )NFrn   r"   r¯  Úrouter_input_scaleç      ð¿rs   )rN   rO   rl   rP   rQ   r1   rÑ   rw   Úcorrect_output_scaler   Úaltup_num_inputsÚcorrection_coefsÚprediction_coefsÚmodality_routerrG   r±  Úrouter_normr…   r×   rš  s     €r6   rO   zGemma3nTextAltUp.__init__ñ  s
  ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ$&¤Lµ´¸T¼[Ô=TÑ1UÔ1UÑ$VÔ$VˆÔ!Ý "¤	¨$¬+Ô*FÈÌÔHdÐkpÐ qÑ qÔ qˆÔÝ "¤	¨$¬+Ô*FÈÌÔHdÐfgÑHgÐnsÐ tÑ tÔ tˆÔÝ!œy¨¬Ô)@À$Ä+ÔB^ÐejÐkÑkÔkˆÔÝ)¨$¬+Ô*AÀtÄ{ÔG_Ð`Ñ`Ô`ˆÔØ×ÒÐ1µ5´<ÀÄÔ@WÐY]Ñ@]Ñ3^Ô3^ÐkpÐÑqÔqÐqÐqÐqr5   râ   r^   c                 óØ   — |                       |¦  «        | j        z  }|                      |¦  «        }t          j        |                     ¦   «         ¦  «                             |¦  «        S r`   )r  r   r  r1   r  ra   rb   )rT   râ   Úrouter_inputsÚrouteds       r6   Úcompute_router_modalitiesz*Gemma3nTextAltUp.compute_router_modalitiesû  sV   € Ø×(Ò(¨Ñ+Ô+¨dÔ.EÑEˆØ×%Ò% mÑ4Ô4ˆÝŒz˜&Ÿ,š,™.œ.Ñ)Ô)×1Ò1°!Ñ4Ô4Ð4r5   rC   c                 óz  — |                       || j        j                 ¦  «        }| j        rF| j        j        �:| j        j        j                             | j        j         | j        j        ¦  «          |                      |¦  «        j	        g |j
        dd…         ¢| j        j        ‘| j        j        ‘R Ž                      dddd¦  «        }t          j        |                     dddd¦  «        |¦  «        }|                     dddd¦  «        }||z  }|                     ¦   «                              |¦  «        S )aµ  Predicts the output of a layer using a trainable map.

        Args:
            hidden_states: A 4D tensor of shape `[num_altup_inputs, batch_size, num_tokens, hidden_size]` derived by
                stacking the input embeddings and preprocessing the last `num_altup_inputs - 1` matrices.

        Returns:
            A 4D tensor of shape `[num_altup_inputs, batch_size, num_tokens, hidden_size]` containing the predictions.
        NrW   r   r$   r   r"   )r  rl   Úaltup_active_idxÚtrainingÚaltup_coef_clipr  rS   ÚdataÚclamp_r£   r±   r  r³   r1   r´   rí   rb   )rT   rC   Ú
modalitiesÚ	all_coefsÚpredictionss        r6   ÚpredictzGemma3nTextAltUp.predict   sA  € ð ×3Ò3°MÀ$Ä+ÔB^Ô4_Ñ`Ô`ˆ
àŒ=ð 	p˜Tœ[Ô8ÐDØÔ!Ô(Ô-×4Ò4°d´kÔ6QÐ5QÐSWÔS^ÔSnÑoÔoÐoðˆD×!Ò! *Ñ-Ô-ÜðiØ Ô& s¨ sÔ+ðiØ-1¬[Ô-IðiØKOÌ;ÔKgðið ið içŠW�Q˜˜1˜aÑ Ô ð 	õ ”l =×#8Ò#8¸¸A¸qÀ!Ñ#DÔ#DÀiÑPÔPˆØ!×)Ò)¨!¨Q°°1Ñ5Ô5ˆØ�}Ñ$ˆØ×%Ò%Ñ'Ô'×/Ò/°Ñ>Ô>Ð>r5   r  Ú	activatedc                 ó†  — |                       |¦  «        }||| j        j                 z
  }|                     | j        j        ddd¦  «        }| j        rl| j        j        �`| j        j         	                    | j        j         | j        j        ¦  «        }t          j        j                             ||d¬¦  «        dz   }n|                      |¦  «        dz   }|                     ddd¦  «                             d¦  «        }t          j        ||¦  «        }||z  }|                     ¦   «                              |¦  «        S )a_  Corrects the predictions relative to the

        Args:
            predictions: A 4D tensor of shape `[num_altup_inputs, batch_size, num_tokens, hidden_size]` derived by
                stacking the input embeddings and preprocessing the last `num_altup_inputs - 1` matrices.
            activated: A 3D tensor of shape `[batch_size, num_tokens, hidden_size]` containing the activated inputs.

        Returns:
            A 4D tensor of shape `[num_altup_inputs, batch_size, num_tokens, hidden_size]` correcting the original
                predictions relative to the activated input embeddings.
        r$   Nrn   rp   r"   r   rW   )r  rl   r  Úrepeatr  r  r  r  rS   r8  r1   rP   r¡   Úlinearr³   r†   Úmulrí   rb   )rT   r  r  r  Ú
innovationrS   r  Ú	correcteds           r6   ÚcorrectzGemma3nTextAltUp.correct  s)  € ð ×3Ò3°IÑ>Ô>ˆ
Ø ¨T¬[Ô-IÔ!JÑJˆ
Ø×&Ò& t¤{Ô'CÀQÈÈ1ÑMÔMˆ
àŒ=ð 	@˜Tœ[Ô8ÐDØÔ*Ô1×7Ò7¸¼Ô9TÐ8TÐVZÔVaÔVqÑrÔrˆFÝœÔ+×2Ò2°:¸vÈDÐ2ÑQÔQÐTWÑWˆIˆIà×-Ò-¨jÑ9Ô9¸CÑ?ˆIð
 ×%Ò% a¨¨AÑ.Ô.×8Ò8¸Ñ<Ô<ˆ	å”I˜j¨)Ñ4Ô4ˆ	Ø�[Ñ ˆ	Ø×#Ò#Ñ%Ô%×-Ò-¨iÑ8Ô8Ð8r5   r  c                 ól   — |                      | j        ¦  «        | j        z                        |¦  «        S )a	  
        This is only defined as the `forward` so that accelerate hooks can move correctly `correct_output_scale`
        (which is a nn.Parameter, not a Module) between devices when offloading. It is otherwise only used in
        `scale_corrected_output`
        )rb   r  ©rT   r  s     r6   rd   zGemma3nTextAltUp.forward;  s2   € ð ×!Ò! $Ô";Ñ<Ô<¸tÔ?XÑX×aÒaÐbkÑlÔlÐlr5   c                 ó,   — |                       |¦  «        S )zMScales the provided 3D tensor of shape [batch_size, num_tokens, hidden_size].)rd   r  s     r6   Úscale_corrected_outputz'Gemma3nTextAltUp.scale_corrected_outputC  s   € à�|Š|˜IÑ&Ô&Ð&r5   )r-   r.   r/   r0   r'   rO   r1   rg   r  r  r  rd   r!  rh   ri   s   @r6   rþ  rþ  å  s#  ø€ € € € € ð	ð 	ðrÐ0ð rð rð rð rð rð rð5¨5¬<ð 5¸E¼Lð 5ð 5ð 5ð 5ð
? U¤\ð ?°e´lð ?ð ?ð ?ð ?ð89 5¤<ð 9¸E¼Lð 9ÈUÌ\ð 9ð 9ð 9ð 9ð>m ¤ð m°%´,ð mð mð mð mð'°´ð 'ÀÄð 'ð 'ð 'ð 'ð 'ð 'ð 'ð 'r5   rþ  c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..NrW   r"   r�   )r±   r1   r“   )râ   Úx1Úx2s      r6   Úrotate_halfr%  H  s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r5   rC   Ún_repr^   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r$   N)r±   Úexpandr£   )rC   r&  rç   Únum_key_value_headsÚslenry   s         r6   Ú	repeat_kvr+  O  s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr5   rÅ   ÚmoduleÚqueryÚkeyrl  Úattention_maskÚdropoutÚscalingrÈ   c                 ól  — |€
| j         dz  }t          || j        ¦  «        }	t          || j        ¦  «        }
t          j        ||	                     dd¦  «        ¦  «        |z  }|�||z  }t          j        |¦  «        }||z  }|�||z   }t          j         	                    |dt          j
        ¬¦  «                             |j        ¦  «        }t          j                             ||| j        ¬¦  «        }t          j        ||
¦  «        }|                     dd¦  «                             ¦   «         }||fS )NrY   r"   r   rW   r  )Úpr  r$   )ry   r+  Únum_key_value_groupsr1   r´   Ú	transposer  rP   r¡   r  r’   r‘   rŒ   r0  r  rí   )r,  r-  r.  rl  r/  r0  r1  rÈ   Úkwargsr  r  Úattn_weightsÚattn_outputs                r6   Úeager_attention_forwardr9  [  s%  € ð €Ø”/ 4Ñ'ˆå˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LàÐØ# gÑ-ˆÝ”z ,Ñ/Ô/ˆØ# gÑ-ˆØÐ!Ø# nÑ4ˆõ ”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€KØ˜Ð$Ð$r5   râ   r•   r”   Úunsqueeze_dimc                 ó†   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   S )a\  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        x (`torch.Tensor`): The tensor to embed.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )r†   r%  )râ   r•   r”   r:  s       r6   Úapply_rotary_pos_embr<  }  s@   € ð" �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�‰G� A™œ¨Ñ,Ñ-Ð-r5   c                   óì   ‡ — e Zd Zdedefˆ fd„Z	 	 ddej        dej        dej        dz  dedz  d	e	ee
ej        ej        f         f         dz  d
ee         de
ej        ej        dz  f         fd„Zˆ xZS )ÚGemma3nTextAttentionrl   rÞ  c                 ó¼  •— t          ¦   «                              ¦   «          || _        || _        t	          |d¦  «        r|j        |         nd | _        | j        dk    | _        | j        r|j        nd | _        t          |d|j
        |j        z  ¦  «        | _        |j        |j        z  | _        d| _        | j        j        | _        d| _        | j        j        | j        j        z
  }||cxk    odk    nc | _        |j        d |…         }| j        rIt+          |¦  «        dz
  |d d d…                              |j        |         ¦  «        z
  | _        d	| _        nLd | _        |t+          |¦  «        dz
  |d d d…                              |j        |         ¦  «        z
  k    | _        t3          j        |j
        |j        | j        z  |j        ¬
¦  «        | _        t;          |j        |j        ¬¦  «        | _        | j        s§t3          j        |j
        |j        | j        z  |j        ¬
¦  «        | _         t3          j        |j
        |j        | j        z  |j        ¬
¦  «        | _!        t;          |j        |j        ¬¦  «        | _"        t;          |j        |j        d	¬¦  «        | _#        t3          j        |j        | j        z  |j
        |j        ¬
¦  «        | _$        d S )NÚlayer_typesÚsliding_attentionry   rp   Tr   r$   rW   Frn   )rI   rJ   )rI   rJ   rK   )%rN   rO   rl   rÞ  Úhasattrr@  Ú
layer_typeÚ
is_slidingÚsliding_windowÚgetattrrw   Únum_attention_headsry   r)  r4  r1  Úattention_dropoutÚ	is_causalÚnum_hidden_layersÚnum_kv_shared_layersÚis_kv_shared_layerr1  ÚindexÚkv_shared_layer_indexÚstore_full_length_kvrP   r   Úattention_biasrÓ   rG   r±  Úq_normrÔ   rÕ   Úk_normÚv_normÚo_proj)rT   rl   rÞ  Úfirst_kv_shared_layer_idxÚprev_layersrU   s        €r6   rO   zGemma3nTextAttention.__init__”  sÝ  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒÝ;BÀ6È=Ñ;YÔ;YÐc˜&Ô,¨YÔ7Ð7Ð_cˆŒØœ/Ð-@Ò@ˆŒØ7;´ÐP˜fÔ3Ð3ÈDˆÔå ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!ØˆŒØ!%¤Ô!>ˆÔØˆŒà$(¤KÔ$AÀDÄKÔDdÑ$dÐ!Ø"+Ð/HÐ"LÐ"LÒ"LÐ"LÈ1Ò"LÐ"LÐ"LÐ"LˆÔØÔ(Ð)CÐ*CÐ)CÔDˆØÔ"ð 		å),¨[Ñ)9Ô)9¸AÑ)=ÀÈDÈDÈbÈDÔ@Q×@WÒ@WÐX^ÔXjÐktÔXuÑ@vÔ@vÑ)vˆDÔ&Ø(-ˆDÔ%Ð%à)-ˆDÔ&à(1µS¸Ñ5EÔ5EÈÑ5IÈKÐX\ÐX\ÐZ\ÐX\ÔL]×LcÒLcØÔ" 9Ô-ñMô Mñ 6ò )ˆDÔ%õ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ %¨¬¸fÔ>QÐRÑRÔRˆŒð Ô&ð 	iÝœ)ØÔ" FÔ$>ÀÄÑ$NÐU[ÔUjðñ ô ˆDŒKõ œ)ØÔ" FÔ$>ÀÄÑ$NÐU[ÔUjðñ ô ˆDŒKõ )¨V¬_À&ÔBUÐVÑVÔVˆDŒKÝ(¨V¬_À&ÔBUÐbgÐhÑhÔhˆDŒKå”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒˆˆr5   NrC   Úposition_embeddingsr/  rB   Úshared_kv_statesr6  r^   c                 ó°  — |j         d d…         }g |¢d‘| j        j        ‘R }|\  }	}
|                      |¦  «                             |¦  «        }|                      |¦  «        }t          ||	|
d¬¦  «        }|                     dd¦  «        }| j        rE|| j	                 \  }}| 
                    |j        ¦  «        }| 
                    |j        ¦  «        }n¹|                      |¦  «                             |¦  «        }|                      |¦  «        }t          ||	|
d¬¦  «        }|                     dd¦  «        }|                      |¦  «                             |¦  «        }|                      |¦  «        }|                     dd¦  «        }|�&| j        s|                     ||| j        ¦  «        \  }}| j        r||f|| j        <   t'          j        | j        j        t,          ¦  «        } || ||||f| j        r| j        nd| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )NrW   r"   )r:  r$   rÅ   )r0  r1  rE  )r±   rl   ry   rÓ   r  rQ  r<  r5  rL  rN  r‘   r�   rÔ   rR  rÕ   rS  ÚupdaterÞ  rO  r   Úget_interfaceÚ_attn_implementationr9  r  rH  r1  rE  r£   rí   rT  )rT   rC   rW  r/  rB   rX  r6  Úinput_shapeÚhidden_shaper•   r”   r  r  r  Úattention_interfacer8  r7  s                    r6   rd   zGemma3nTextAttention.forwardÄ  sŒ  € ð $Ô)¨#¨2¨#Ô.ˆØ?˜Ð? bÐ?¨$¬+Ô*>Ð?Ð?ˆà&‰ˆˆSØ—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆØ—{’{ <Ñ0Ô0ˆÝ+¨L¸#¸sÐRSÐTÑTÔTˆØ#×-Ò-¨a°Ñ3Ô3ˆð
 Ô"ð 	8Ø'7¸Ô8RÔ'SÑ$ˆJ˜à#Ÿš |Ô':Ñ;Ô;ˆJØ'Ÿ?š?¨<Ô+>Ñ?Ô?ˆLˆLàŸš ]Ñ3Ô3×8Ò8¸ÑFÔFˆJØŸš ZÑ0Ô0ˆJÝ-¨j¸#¸sÐRSÐTÑTÔTˆJØ#×-Ò-¨a°Ñ3Ô3ˆJàŸ;š; }Ñ5Ô5×:Ò:¸<ÑHÔHˆLØŸ;š; |Ñ4Ô4ˆLØ'×1Ò1°!°QÑ7Ô7ˆLàÐ&¨tÔ/FÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜ØÔ$ð 	HØ/9¸<Ð/GÐ˜Tœ^Ñ,å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð /3¬mÐD�DÔ*Ð*ÀØ”LØÔ.ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r5   ©NN)r-   r.   r/   r'   re   rO   r1   rg   r   ÚdictrE   r   r   rd   rh   ri   s   @r6   r>  r>  “  s÷   ø€ € € € € ð.
Ð0ð .
¸Sð .
ð .
ð .
ð .
ð .
ð .
ðj )-ØPTð;)ð ;)à”|ð;)ð #œ\ð;)ð œ tÑ+ð	;)ð
  ™ð;)ð ˜s E¨%¬,¸¼Ð*DÔ$EÐEÔFÈÑMð;)ð Ð+Ô,ð;)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð;)ð ;)ð ;)ð ;)ð ;)ð ;)ð ;)ð ;)r5   r>  c                   ó0  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 ddej        dej        dej        deee	ej        ej        f         f         dz  d	ej        dz  d
ej
        dz  dedz  dee         de	ej        e	ej        ej        f         dz  f         fd„Zˆ xZS )ÚGemma3nTextDecoderLayerrl   rÞ  c                 ó@  •— t          ¦   «                              ¦   «          || _        |j        | _        || _        t          ||¦  «        | _        t          ||¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        |j        | _        t           |j                 | _        t'          |¦  «        | _        t+          |¦  «        | _        t/          j        | j        | j        d¬¦  «        | _        t/          j        | j        | j        d¬¦  «        | _        t          | j        |j
        ¬¦  «        | _        d S )N)rÞ  r¯  Frn   )rN   rO   rl   rw   rÞ  r>  Ú	self_attnrÝ  ÚmlprG   r±  Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormÚhidden_size_per_layer_inputr
   rå  ræ  rþ  ÚaltuprÓ  ÚlaurelrP   r   Úper_layer_input_gateÚper_layer_projectionÚpost_per_layer_input_normré  s      €r6   rO   z Gemma3nTextDecoderLayer.__init__  s]  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ"ˆŒÝ-¨f°iÑ@Ô@ˆŒÝ! &°IÐ>Ñ>Ô>ˆŒÝ-¨dÔ.>ÀFÔDWÐXÑXÔXˆÔÝ(6°tÔ7GÈVÔM`Ð(aÑ(aÔ(aˆÔ%Ý)7¸Ô8HÈfÔNaÐ)bÑ)bÔ)bˆÔ&Ý*8¸Ô9IÈvÔObÐ*cÑ*cÔ*cˆÔ'à+1Ô+MˆÔ(Ý˜VÔ5Ô6ˆŒå% fÑ-Ô-ˆŒ
Ý,¨VÑ4Ô4ˆŒÝ$&¤I¨dÔ.>ÀÔ@`ÐglÐ$mÑ$mÔ$mˆÔ!Ý$&¤I¨dÔ.NÐPTÔP`ÐglÐ$mÑ$mÔ$mˆÔ!Ý)7¸Ô8HÈfÔNaÐ)bÑ)bÔ)bˆÔ&Ð&Ð&r5   NrC   rW  Úper_layer_inputrX  r/  Úposition_idsrB   r6  r^   c           
      ó�  — | j                              |¦  «        }	|	| j        j                 }
|                      |
¦  «        }|                      |¦  «        } | j        d||||||dœ|¤Ž\  }}|                      |¦  «        }|
|z   }||z   t          j	        d¦  «        z  }|  
                    |¦  «        }|                      |¦  «        }|                      |¦  «        }||z   }| j                              |	|¦  «        }|| j        j                                      ¦   «         }| j        j        r| j                              |¦  «        }|                      |¦  «        }|                      |¦  «        }t'          j        ||¦  «        }|                      |¦  «        }|                      |¦  «        }|dd …xx         |z  cc<   |S )N)rC   r/  rX  rr  rW  rB   r"   r$   r4   )rl  r  rl   r  rg  rm  re  rh  r�   Úsqrtri  rf  rj  r  rØ   Úaltup_correct_scaler!  rn  ræ  r1   Úmultiplyro  rp  )rT   rC   rW  rq  rX  r/  rr  rB   r6  r  Úactive_predictionÚactive_prediction_normedÚlaurel_outputr—  rµ   Ú
attn_gatedÚattn_laurelÚ	attn_normÚattn_ffwÚattn_ffw_normÚattn_ffw_laurel_gatedÚcorrected_predictionsÚfirst_predictions                          r6   rd   zGemma3nTextDecoderLayer.forward  sï  € ð ”j×(Ò(¨Ñ7Ô7ˆØ'¨¬Ô(DÔEÐà#'×#7Ò#7Ð8IÑ#JÔ#JÐ ØŸšÐ$<Ñ=Ô=ˆà �$”.ð 
Ø2Ø)Ø-Ø%Ø 3Ø+ð
ð 
ð ð
ð 
‰ˆˆað ×,Ò,¨TÑ2Ô2ˆà&¨Ñ-ˆ
Ø! MÑ1µT´Y¸q±\´\ÑAˆà×2Ò2°;Ñ?Ô?ˆ	Ø—8’8˜IÑ&Ô&ˆØ×7Ò7¸ÑAÔAˆØ +¨mÑ ;ÐØ $¤
× 2Ò 2°;Ð@UÑ VÔ VÐà0°´Ô1MÔN×TÒTÑVÔVÐØŒ;Ô*ð 	SØ#œz×@Ò@ÐAQÑRÔRÐð  ×4Ò4Ð5EÑFÔFÐØŸ;š;Ð'7Ñ8Ô8ÐÝ œ>Ð*:¸OÑLÔLÐð  ×4Ò4Ð5EÑFÔFÐØ×9Ò9Ð:JÑKÔKÐØ˜a˜b˜bÐ!Ð!Ô!Ð%5Ñ5Ð!Ð!Ñ!à$Ð$r5   )NNNNNN)r-   r.   r/   r'   re   rO   r1   rg   ra  rE   Ú
LongTensorr   r   r   r=   rd   rh   ri   s   @r6   rc  rc    s6  ø€ € € € € ðcÐ0ð c¸Sð cð cð cð cð cð cð0 -1Ø(,ØPTØ.2Ø04Ø(,ð3%ð 3%à”|ð3%ð #œ\ð3%ð œð	3%ð
 ˜s E¨%¬,¸¼Ð*DÔ$EÐEÔFÈÑMð3%ð œ tÑ+ð3%ð Ô&¨Ñ-ð3%ð  ™ð3%ð Ð+Ô,ð3%ð 
ˆuŒ|˜U 5Ô#4°eÔ6GÐ#GÔHÈ4ÑOÐOÔ	Pð3%ð 3%ð 3%ð 3%ð 3%ð 3%ð 3%ð 3%r5   rc  c            	       óú   ‡ — e Zd ZU eed<   dZdZdgZddgZdZ	dZ
dZdZdZeedœZdZ ej        ¦   «         ˆ fd	„¦   «         Zd
„ Zd„ Z	 	 	 ddedz  dedz  dedej        fˆ fd„Z	 	 	 ddedz  dedz  defd„Zˆ xZS )ÚGemma3nPreTrainedModelrl   ÚmodelTrc  rB   rX  )rC   rD   )ÚimageÚtextÚaudioc                 óè  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rt	          j        |j        ¦  «         �nit          |t          ¦  «        rÆt	          j        |j	        ¦  «         |j
        dz  }dt          j        j                             t          j        d¦  «        ¦  «        z  }t	          j        |j        ||z  ¦  «         t	          j        |j        |j        ¦  «         t	          j        |j        |                     ¦   «         ¦  «         �nŽt          |t.          ¦  «        r!t	          j        |j        |j        ¦  «         �nXt          |t4          ¦  «        rBt	          j        |j        ¦  «         t	          j        |j        | j        j        dz  ¦  «         �nt          |t>          ¦  «        rÕd\  }}|j         dz  }tC          j"        tG          |¦  «        tG          |¦  «        z  ¦  «        tI          |dz
  d¦  «        z  }|t          j%        t          j&        |¦  «        | z  ¦  «        z  }t	          j        |j'        | #                    ¦   «          (                    d¦  «         (                    d¦  «        ¦  «         �nt          |tR          ¦  «        rRt	          j        |j*        | j        dz  ¦  «         t	          j        |j+        dtC          j,        d	¦  «        z  ¦  «         n°t          |tZ          ¦  «        r›|j.        D ]“}	|j/        }
|j0        |	         d
k    rtb          |j0        |	                  }
 |
|j        |	¬¦  «        \  }}t	          j        te          ||	› d�¦  «        |¦  «         t	          j        te          ||	› d�¦  «        |¦  «         Œ”tg          |d¦  «        r&t	          j        |j4        | j        j4        ¦  «         d S d S )NrY   rp   rÅ   r  )rp   rq   r"   r$   r   ç       @Údefault©rC  Ú	_inv_freqÚ_original_inv_freqr”  )5rN   Ú_init_weightsÚ
isinstancer+  ÚinitÚones_rS   rÃ   Úzeros_rÒ   ry   r1   rP   r¡   rÖ   r×   Úcopy_rÆ   Ú	constant_rÈ   rÎ   rÇ   rÚ   rÉ  rÍ  rÏ  rþ  r  r   rl   rw   rk   rx   r�   r‚   ra   rz   rƒ   r„   rr   r†   ÚGemma3nTextModelÚper_layer_projection_scaleÚper_layer_input_scalert  ÚGemma3nRotaryEmbeddingr@  Úcompute_default_rope_parametersÚ	rope_typer   rF  rB  r”  )rT   r,  rÆ   rÛ   r‡   rˆ   r‰   rŠ   rr   rC  Úrope_init_fnÚcurr_inv_freqrµ   rU   s                €r6   r�  z$Gemma3nPreTrainedModel._init_weightsa  s•  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ=Ñ>Ô>ð  	^ÝŒJ�v”}Ñ%Ô%Ð%Ñ%Ý˜Õ 5Ñ6Ô6ð 	^ÝŒK˜Ô,Ñ-Ô-Ð-Ø”o tÑ+ˆGØ¥¤Ô!4×!=Ò!=½e¼lÈ3Ñ>OÔ>OÑ!PÔ!PÑPˆLÝŒJ�v”~ w°Ñ'=Ñ>Ô>Ð>ÝŒN˜6œ>¨6Ô+KÑLÔLÐLÝŒJ�vÔ5°v×7\Ò7\Ñ7^Ô7^Ñ_Ô_Ð_Ñ_Ý˜Õ >Ñ?Ô?ð 	^ÝŒN˜6Ô-¨vÔ/HÑIÔIÐIÑIÝ˜Õ 0Ñ1Ô1ð 	^ÝŒK˜Ô3Ñ4Ô4Ð4ÝŒN˜6Ô4°d´kÔ6MÈtÑ6SÑTÔTÐTÑTÝ˜Õ EÑFÔFð 	^Ø+5Ñ(ˆM˜=Ø#œ_°Ñ1ˆNÝ&*¤h­u°]Ñ/CÔ/CÅeÈMÑFZÔFZÑ/ZÑ&[Ô&[Õ^aØ Ñ" Añ_ô _ñ 'Ð#ð +­U¬Yµu´|ÀNÑ7SÔ7SÐWnÐVnÑ7nÑ-oÔ-oÑoˆNÝŒJ�vÔ,¨n×.BÒ.BÑ.DÔ.D×.NÒ.NÈqÑ.QÔ.Q×.[Ò.[Ð\]Ñ.^Ô.^Ñ_Ô_Ð_Ñ_Ý˜Õ 0Ñ1Ô1ð 
	^ÝŒN˜6Ô<¸dÔ>NÐPTÑ>TÑUÔUÐUÝŒN˜6Ô7¸½T¼YÀs¹^¼^Ñ9KÑLÔLÐLÐLÝ˜Õ 6Ñ7Ô7ð 	^Ø$Ô0ð ^ð ^�
Ø%ÔE�ØÔ# JÔ/°9Ò<Ð<Ý#6°vÔ7GÈ
Ô7SÔ#T�LØ#/ <°´È*Ð#UÑ#UÔ#UÑ �˜qÝ”
�7 6¨jÐ+CÐ+CÐ+CÑDÔDÀmÑTÔTÐTÝ”
�7 6¨jÐ+LÐ+LÐ+LÑMÔMÈ}Ñ]Ô]Ð]Ð]å�6Ð.Ñ/Ô/ð 	TÝŒN˜6Ô3°T´[Ô5RÑSÔSÐSÐSÐSð	Tð 	Tr5   c                 ó   — | j         j        S r`   ©Ú
base_modelÚembed_tokens_per_layer©rT   s    r6   Úget_per_layer_input_embeddingsz5Gemma3nPreTrainedModel.get_per_layer_input_embeddings‰  s   € ØŒÔ5Ð5r5   c                 ó   — || j         _        d S r`   rŸ  ©rT   rl  s     r6   Úset_per_layer_input_embeddingsz5Gemma3nPreTrainedModel.set_per_layer_input_embeddingsŒ  s   € Ø16ˆŒÔ.Ð.Ð.r5   NÚnew_num_tokensÚpad_to_multiple_ofÚmean_resizingr^   c                 ó~   •— t          ¦   «                              |||¬¦  «        }|                      |||¦  «         |S )N)r§  r¨  r©  )rN   Úresize_token_embeddingsÚ_resize_per_layer_embeddings)rT   r§  r¨  r©  Úinputs_embedsrU   s        €r6   r«  z.Gemma3nPreTrainedModel.resize_token_embeddings�  sM   ø€ õ ™œ×7Ò7Ø)Ø1Ø'ð 8ñ 
ô 
ˆð
 	×)Ò)¨.Ð:LÈmÑ\Ô\Ð\ØÐr5   c                 óš  — | j         | j                             ¦   «         _        | j                             ¦   «         j        r‰|                      ¦   «         }|                      ||||¦  «        }t          |d¦  «        r|j        }t          ||¦  «         | 
                    |j        j        ¦  «         |                      |¦  «         d S d S )NÚ_hf_hook)Ú
vocab_sizerl   Úget_text_configÚvocab_size_per_layer_inputrk  r£  Ú_get_resized_embeddingsrB  r¯  r)   Úrequires_grad_rS   rM   r¦  )rT   r§  r¨  r©  r¡  Únew_embeddings_per_layerÚhooks          r6   r¬  z3Gemma3nPreTrainedModel._resize_per_layer_embeddings�  sÛ   € ð DHÄ?ˆŒ×#Ò#Ñ%Ô%Ô@ØŒ;×&Ò&Ñ(Ô(ÔDð 		JØ%)×%HÒ%HÑ%JÔ%JÐ"Ø'+×'CÒ'CØ&¨Ð8JÈMñ(ô (Ð$õ Ð-¨zÑ:Ô:ð CØ-Ô6�Ý"Ð#;¸TÑBÔBÐBØ$×3Ò3Ð4JÔ4QÔ4_Ñ`Ô`Ð`Ø×/Ò/Ð0HÑIÔIÐIÐIÐIð		Jð 		Jr5   )NNT)r-   r.   r/   r&   r3   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrc  r>  Ú_can_record_outputsÚinput_modalitiesr1   Úno_gradr�  r£  r¦  re   rf   rP   Ú	Embeddingr«  r¬  rh   ri   s   @r6   r„  r„  N  s‡  ø€ € € € € € àÐÐÑØÐØ&*Ð#Ø2Ð3ÐØ#4Ð6HÐ"IÐØÐØ€NØÐà!ÐØ"&Ðà0Ø*ðð Ðð 2Ðà€U„]�_„_ð%Tð %Tð %Tð %Tñ „_ð%TðN6ð 6ð 6ð7ð 7ð 7ð
 &*Ø)-Ø"ð	ð à˜d™
ðð   $™Jðð ð	ð
 
Œðð ð ð ð ð ð  &*Ø)-Ø"ð	Jð Jà˜d™
ðJð   $™JðJð ð	Jð Jð Jð Jð Jð Jð Jð Jr5   r„  c                   óš   ‡ — e Zd ZU dZeed<   dZdZdefˆ fd„Ze	e
dej        dej        dee         deez  fd	„¦   «         ¦   «         Zˆ xZS )
ÚGemma3nAudioEncoderzx
    An audio encoder based on the [Universal Speech Model](https://huggingface.co/papers/2303.01037) architecture.
    rl   Ú	audio_melrˆ  c                 ó  •‡— t          ¦   «                              ‰¦  «         ‰| _        t          ‰¦  «        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _	        |  
                    ¦   «          d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r4   )r¿  )Ú.0rµ   rl   s     €r6   ú
<listcomp>z0Gemma3nAudioEncoder.__init__.<locals>.<listcomp>À  s"   ø€ Ð^Ð^Ð^°AÕ'¨Ñ/Ô/Ð^Ð^Ð^r5   )rN   rO   rl   rt  Úsubsample_conv_projectionrP   Ú
ModuleListr0  Úconf_num_hidden_layersÚ	conformerÚ	post_initrš  s    `€r6   rO   zGemma3nAudioEncoder.__init__º  s€   øø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒå)LÈVÑ)TÔ)TˆÔ&ÝœØ^Ð^Ð^Ð^½¸vÔ?\Ñ9]Ô9]Ð^Ñ^Ô^ñ
ô 
ˆŒð 	�ŠÑÔÐÐÐr5   r,   r6  r^   c                 óì  — |                       |¦  «        }|j        d         }d}t          t          | j        j        ¦  «        ¦  «        D ]}|| j        j        |         d         z  }Œt          j        ||j        ¬¦  «        |z  }t          j	        ||j        d         dz
  ¬¦  «        }|j
        dk    r@|j
        dk    r5|                     d¦  «                             |j        d         d¦  «        }nX|j
        |j
        k    rH|j        d         dk    r7|j        d         dk    r&||j        d         k    r|                     d¦  «        }t          j        |d|¦  «        }	| j        D ]}
 |
||	¦  «        }Œ| j        j        dk    r2|dd…dd| j        j        …f         }|	dd…dd| j        j        …f         }	|                     |	                     d¦  «        d¦  «        }t#          ||	¬¦  «        S )	a¬  Encodes a batch of MELs.

        Args:
            audio_mel: a torch.Tensor of shape [batch, num_frames, num_channels,
              mel_bins].

        Returns:
            audio_encodings: a torch.Tensor of shape
                `[batch_size, self.config.audio_soft_tokens_per_image,
                self.config.audio_config.hidden_size]`
            audio_mel_mask: a torch.BoolTensor of shape [batch, num_frames].
        r$   r   r®   )rz   rW   NrÅ   )Úlast_hidden_stater,   )rË  r±   r0  r1  rl   r[  r1   r„   r�   r8  rû   r†   r(  ÚgatherrÎ  Úconf_reduction_factorÚmasked_fillr+   )rT   rÆ  r,   r6  rh  Út_subÚtime_stride_productÚstride_pair_idxÚindicesÚcurrent_maskÚblocks              r6   rd   zGemma3nAudioEncoder.forwardÄ  s!  € ð" ×8Ò8¸ÑCÔCˆð  Ô% aÔ(ˆàÐÝ$¥S¨¬Ô)JÑ%KÔ%KÑLÔLð 	Yð 	YˆOØ 4¤;Ô#DÀ_Ô#UÐVWÔ#XÑXÐÐõ
 ”,˜u¨^Ô-BÐCÑCÔCÐFYÑYˆÝ”+˜g¨>Ô+?ÀÔ+BÀQÑ+FÐGÑGÔGˆð Ô Ò"Ð" w¤|°qÒ'8Ð'8Ø×'Ò'¨Ñ*Ô*×1Ò1°.Ô2FÀqÔ2IÈ2ÑNÔNˆGˆGàÔ 7¤<Ò/Ð/ØÔ$ QÔ'¨1Ò,Ð,Ø”˜aÔ  AÒ%Ð%Ø˜œ qÔ)Ò)Ð)ð ×'Ò'¨Ñ*Ô*ˆGå”| N°A°wÑ?Ô?ˆà”^ð 	Cð 	CˆEØ#˜e O°\ÑBÔBˆOˆOàŒ;Ô,¨qÒ0Ð0Ø-¨a¨a¨aÐ1UÐ1U°D´KÔ4UÐ1UÐ.UÔVˆOà'¨¨¨Ð+OÐ+O¨d¬kÔ.OÐ+OÐ(OÔPˆLà)×5Ò5°l×6LÒ6LÈRÑ6PÔ6PÐRUÑVÔVˆÝ-Ø-Ø'ð
ñ 
ô 
ð 	
r5   )r-   r.   r/   r0   r%   r3   Úmain_input_namerÁ  rO   r    r!   r1   rg   r2   r   r   rE   r+   rd   rh   ri   s   @r6   rÅ  rÅ  °  sÆ   ø€ € € € € € ðð ð ÐÐÑà!€OØÐðÐ1ð ð ð ð ð ð ð  Øð8
Øœð8
Ø7<Ô7Gð8
ØSYÐZlÔSmð8
à	Ð/Ñ	/ð8
ð 8
ð 8
ñ „_ñ  Ôð8
ð 8
ð 8
ð 8
ð 8
r5   rÅ  c                   óà   ‡ — e Zd ZU ej        ed<   defˆ fd„Ze	 	 	 	 ddedz  de	d         de
dz  dedz  d	ed
ef         f
d„¦   «         Z ej        ¦   «         edd„¦   «         ¦   «         Zˆ xZS )r™  Úinv_freqrl   c                 ó€  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        t          t          |j        ¦  «        ¦  «        | _        i | _	        | j        D ]È}| j        j
        |         }|€Œ|d         | j	        |<   | j        }| j	        |         dk    rt          | j	        |                  } || j        |¬¦  «        \  }}|                      |› d�|d¬¦  «         |                      |› d�|                     ¦   «         d¬¦  «         t          | |› d�|¦  «         ŒÉd S )	Nr›  r‹  rŒ  r�  Frs   rŽ  Ú_attention_scaling)rN   rO   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrl   ÚlistÚsetr@  r›  Úrope_parametersrš  r   r…   rØ   Úsetattr)rT   rl   rC  Úrope_paramsrœ  r�  Úcurr_attention_scalingrU   s          €r6   rO   zGemma3nRotaryEmbedding.__init__  s^  ø€ Ý‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!ØˆŒÝ¥ FÔ$6Ñ 7Ô 7Ñ8Ô8ˆÔØˆŒØÔ*ð 	Uð 	UˆJØœ+Ô5°jÔAˆKØÐ"Øà)4°[Ô)AˆDŒN˜:Ñ&Ø%)Ô%IˆLØŒ~˜jÔ)¨YÒ6Ð6Ý2°4´>À*Ô3MÔN�Ø4@°LÀÄÐYcÐ4dÑ4dÔ4dÑ1ˆMÐ1Ø× Ò  JÐ!9Ð!9Ð!9¸=ÐUZÐ Ñ[Ô[Ð[Ø× Ò  JÐ!BÐ!BÐ!BÀM×DWÒDWÑDYÔDYÐfkÐ ÑlÔlÐlÝ�D˜ZÐ;Ð;Ð;Ð=SÑTÔTÐTÐTð	Uð 	Ur5   Nr�   ztorch.deviceÚseq_lenrC  r^   ztorch.Tensorc                 ó  — | j         |         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «                             |t          j        ¬¦  «        |z  z  z  }||fS )	a|  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetary   Nrp   r   r"   r¯   rŽ   )	rå  rF  rw   rG  r1   r„   Úint64r‘   ra   )rl   r�   ré  rC  ÚbaserI   Úattention_factorrÝ  s           r6   rš  z6Gemma3nRotaryEmbedding.compute_default_rope_parameters  s‘   € ð2 Ô% jÔ1°,Ô?ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐð Ø•U”\ ! S¨!µ5´;Ð?Ñ?Ô?×BÒBÈ&ÕX]ÔXcÐBÑdÔdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r5   c                 ó|  — t          | |› d�¦  «        }t          | |› d�¦  «        }|d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬	¦  «        5  |                     ¦   «         |                     ¦   «         z   
                    dd
¦  «        }	t          j        |	|	fd¬¦  «        }
|
                     ¦   «         |z  }|
                     ¦   «         |z  }d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |                     |j        ¬¦  «        fS )Nr�  rß  r   rW   r$   ÚmpsÚcpuF)Údevice_typeÚenabledr"   r�   r¯   )rF  ra   r(  r±   r‘   r�   r�  r–   Ústrr   r5  r1   r“   r•   r”   rŒ   )rT   râ   rr  rC  rÝ  Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedrò  ÚfreqsÚembr•   r”   s                r6   rd   zGemma3nRotaryEmbedding.forward=  sâ  € õ ˜4 JÐ!9Ð!9Ð!9Ñ:Ô:ˆÝ# D¨ZÐ*KÐ*KÐ*KÑLÔLÐà$ T¨1¨1¨1¨d ]Ô3×9Ò9Ñ;Ô;×BÒBÀ<ÔCUÐVWÔCXÐZ\Ð^_Ñ`Ô`×cÒcÐdeÔdlÑmÔmÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	0ð 	0Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)Ð/Ñ/ˆCØ—'’'‘)”)Ð/Ñ/ˆCð		0ð 	0ð 	0ñ 	0ô 	0ð 	0ð 	0ð 	0ð 	0ð 	0ð 	0øøøð 	0ð 	0ð 	0ð 	0ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   Ã-BE=Å=FÆF©NNNNr`   )r-   r.   r/   r1   rg   r3   r'   rO   Ústaticmethodr   re   rô  rE   ra   rš  rÂ  r   rd   rh   ri   s   @r6   r™  r™    s  ø€ € € € € € ØŒlÐÐÑðUÐ0ð Uð Uð Uð Uð Uð Uð* à+/Ø+/Ø"Ø!%ð	!*ð !*Ø! DÑ(ð!*à˜Ô(ð!*ð �t‘ð!*ð ˜$‘Jð	!*ð
 
ˆ~˜uÐ$Ô	%ð!*ð !*ð !*ñ „\ð!*ðF €U„]�_„_Øð<ð <ð <ñ Ôñ „_ð<ð <ð <ð <ð <r5   r™  zBThe base Gemma 3n language model without a language modeling head.c                   ó|  ‡ — e Zd ZU eed<   dZdefˆ fd„Ze ed¬¦  «        e		 	 	 	 	 	 	 dde
j        dz  de
j        dz  d	e
j        dz  d
e
j        dz  dedz  de
j        dz  dedz  dee         defd„¦   «         ¦   «         ¦   «         Zde
j        de
j        fd„Z	 dde
j        de
j        dz  de
j        fd„Zˆ xZS )r–  rl   )r‡  c                 ó¤  •‡ ‡‡— t          ¦   «                              ‰¦  «         ‰j        ‰ _        ‰j        ‰ _        t          ‰j        ‰j        ‰ j        ‰ j        j        dz  ¬¦  «        ‰ _        t          j
        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        ‰ _        t          ‰j        ‰j        ¬¦  «        ‰ _        t#          ‰¦  «        ‰ _        d‰ _        ‰j        ‰ _        ‰j        ‰ _        t          ‰j        ‰j        ‰j        z  ‰ j        ‰j        dz  ¬¦  «        ‰ _        t          j        ‰ j        ‰j        ‰j        z  d¬¦  «        ‰ _        t          ‰j        ‰j        ¬¦  «        ‰ _        t          j
        ˆ fd„t          d‰ j        j        ¦  «        D ¦   «         ¦  «        ‰ _        t          j
        ˆ fd	„t          d‰ j        j        ¦  «        D ¦   «         ¦  «        ‰ _        ‰                      d
t=          j        ‰ j        dz  ¦  «        d¬¦  «         ‰                      dt=          j         t=          j        d¦  «        ¦  «        d¬¦  «         g ‰ _!        tE          ‰ j        ¦  «        D ]7\  Š}|j#        j$        r&‰ j!         %                    ˆfd„dD ¦   «         ¦  «         Œ8‰  &                    ¦   «          d S )Nç      à?)rÍ  c                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r4   )rc  )rÉ  rÞ  rl   s     €r6   rÊ  z-Gemma3nTextModel.__init__.<locals>.<listcomp>_  s$   ø€ ÐiÐiÐi¸IÕ$ V¨YÑ7Ô7ÐiÐiÐir5   r¯  Frn   c                 óR   •— g | ]#}t          j        ‰j        ‰j        d ¬¦  «        ‘Œ$S ©Frn   ©rP   r   rw   ©rÉ  rµ   rT   s     €r6   rÊ  z-Gemma3nTextModel.__init__.<locals>.<listcomp>y  ó0   ø€ ÐwÐwÐwÈ1�RŒY�tÔ'¨Ô)9ÀÐFÑFÔFÐwÐwÐwr5   r$   c                 óR   •— g | ]#}t          j        ‰j        ‰j        d ¬¦  «        ‘Œ$S r  r  r  s     €r6   rÊ  z-Gemma3nTextModel.__init__.<locals>.<listcomp>}  r  r5   r—  rY   rs   r˜  rŠ  c                 ó    •— g | ]
}d ‰› d|› �‘ŒS )zlayers.z.self_attn.r4   )rÉ  Únamer  s     €r6   rÊ  z-Gemma3nTextModel.__init__.<locals>.<listcomp>ˆ  s*   ø€ ÐiÐiÐi¸Ð3˜qÐ3Ð3¨TÐ3Ð3ÐiÐiÐir5   )rÔ   rÕ   rR  rS  )'rN   rO   Úpad_token_idrÌ  r°  rÉ  rw   rl   Úembed_tokensrP   rÌ  r0  rJ  ÚlayersrG   r±  r_  r™  Ú
rotary_embÚgradient_checkpointingrk  r²  r¡  r   Úper_layer_model_projectionÚper_layer_projection_normr  Úaltup_projectionsÚaltup_unembed_projectionsr…   r1   r×   r9  Ú"_keys_to_ignore_on_load_unexpectedÚ	enumeratere  rL  ÚextendrÏ  )rT   rl   Úlayerr  rU   s   `` @€r6   rO   zGemma3nTextModel.__init__U  sØ  øøøø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒõ ;ØÔ˜vÔ1°4Ô3CÐQUÔQ\ÔQhÐjmÑQmð
ñ 
ô 
ˆÔõ ”mØiÐiÐiÐiÍÈvÔOgÑIhÔIhÐiÑiÔiñ
ô 
ˆŒõ # 6Ô#5¸6Ô;NÐOÑOÔOˆŒ	Ý0°Ñ8Ô8ˆŒØ&+ˆÔ#à!Ô-ˆÔØ+1Ô+MˆÔ(å&DØÔ-ØÔ$ vÔ'IÑIØÔØÔ:¸CÑ?ð	'
ñ '
ô '
ˆÔ#õ +-¬)ØÔØÔ$ vÔ'IÑIØð+
ñ +
ô +
ˆÔ'õ *8¸Ô8ZÐ`fÔ`sÐ)tÑ)tÔ)tˆÔ&å!#¤ØwÐwÐwÐwÕPUÐVWÐY]ÔYdÔYuÑPvÔPvÐwÑwÔwñ"
ô "
ˆÔõ *,¬ØwÐwÐwÐwÕPUÐVWÐY]ÔYdÔYuÑPvÔPvÐwÑwÔwñ*
ô *
ˆÔ&ð 	×ÒÐ9½5¼<ÈÔHXÐZ^ÑH^Ñ;_Ô;_ÐlqÐÑrÔrÐrØ×ÒÐ4µe´kÅ%Ä,ÈsÑBSÔBSÑ6TÔ6TÐafÐÑgÔgÐgð 35ˆÔ/Ý! $¤+Ñ.Ô.ð 	ð 	‰HˆAˆuØŒÔ1ð ØÔ7×>Ò>ØiÐiÐiÐiÐ@hÐiÑiÔiñô ð øð
 	�ŠÑÔÐÐÐr5   F)Útie_last_hidden_statesNrÐ  Úper_layer_inputsr/  rr  rB   r­  Ú	use_cacher6  r^   c           	      óf  — |du |duz  rt          d¦  «        ‚|�*|                      |¦  «        }|                      |¦  «        }|                      ||¦  «        }|r|€t	          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}	t          j        |j	        d         |j
        ¬¦  «        |	z   }|                     d¦  «        }t          |x}
t          ¦  «        s&| j        ||||dœ}t          di |¤Žt          di |¤Ždœ}
|}t          j        |d	z  d
d¬¦  «        dz  }t          j        d¦  «        }|g}t%          d| j        j        ¦  «        D ]²} | j        |dz
           |¦  «        }|                     |j        |j
        ¬¦  «        }t          j        |d	z  d
d¬¦  «        }t          j        t          j        ||                     |j
        ¦  «        ¦  «        ¦  «        }||z  |z  }|                     |¦  «         Œ³t          j        |d¬¦  «        }i }t7          | j        j        ¦  «        D ]}|                      |||¦  «        ||<   Œt=          ¦   «         }t?          | j         d| j        j!        …         ¦  «        D ]U\  }}|
| j        j        |                  }|dd…dd…|dd…f         } |||| j        j        |                  |f||||dœ|¤Ž}ŒVt          j        |d         d	z  d
d¬¦  «        dz  }|d         g}t%          d| j        j        ¦  «        D ]¸} | j"        |dz
           ||         ¦  «        }|                     |j        |j
        ¬¦  «        }t          j        |d	z  d
d¬¦  «        }t          j        t          j        ||                     |j
        ¦  «        ¦  «        ¦  «        }||z  |z  }|                     |¦  «         Œ¹t          j        |¦  «        }t          j        |d¬¦  «        }|  #                    |¦  «        }tI          ||¬¦  «        S )zµ
        per_layer_inputs (torch.Tensor, *optional*, defaults to None):
            Pre-computed per-layer embeddings. If None, they are derived from input_ids if provided.
        Nú:You must specify exactly one of input_ids or inputs_embeds©rl   r   r$   r®   )rl   r­  r/  rB   rr  )Úfull_attentionrA  r"   rW   Tr4  rþ  gñhãˆµøä>rï  r�   )rX  r/  rr  rB   )rÑ  rB   r4   )%r  r	  Úget_per_layer_inputsÚproject_per_layer_inputsr   rl   Úget_seq_lengthr1   r„   r±   r�   r†   r�  ra  r   r   r[   r×   r0  r  r  r‘   rŒ   rt  Úmaximumrw  Ústackrä  r@  r  r   r  r
  rJ  r  r_  r   )rT   rÐ  r  r/  rr  rB   r­  r  r6  Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsÚhidden_states_0Útarget_magnitudeÚepsilon_tensorÚtemp_hidden_statesr  Ú
altup_projÚcurrent_hidden_stateÚnew_magnituderC   rW  rC  rX  Údecoder_layerÚcausal_maskrq  Úaltup_unemb_projs                               r6   rd   zGemma3nTextModel.forwardŽ  sÎ  € ð$ ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMØ#×8Ò8¸ÑCÔCÐà×8Ò8¸ÐHXÑYÔYÐàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLõ °Ð?Ð-ÅÑFÔFð 	ð œ+Ø!.Ø"0Ø#2Ø ,ðð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ%FÐ%UÐ%UÈÐ%UÐ%Uð#ð #Ðð (ˆõ !œ: o°qÑ&8¸bÈ$ÐOÑOÔOÐSVÑVÐÝœ dÑ+Ô+ˆà-Ð.ÐÝ�q˜$œ+Ô6Ñ7Ô7ð 	<ð 	<ˆAà6˜Ô/°°A±Ô6°ÑGÔGˆJØ#-§=¢=°Ô7LÐUeÔUl =Ñ#mÔ#mÐ Ý!œJÐ';¸QÑ'>ÀBÐPTÐUÑUÔUˆMÝ!œJ¥u¤}°]ÀN×DUÒDUÐVfÔVmÑDnÔDnÑ'oÔ'oÑpÔpˆMØ#7Ð:JÑ#JÈ]Ñ#ZÐ Ø×%Ò%Ð&:Ñ;Ô;Ð;Ð;åœÐ$6¸AÐ>Ñ>Ô>ˆØ ÐÝ˜dœkÔ5Ñ6Ô6ð 	gð 	gˆJØ.2¯oªo¸mÈ\Ð[eÑ.fÔ.fÐ 
Ñ+Ð+õ
 $™:œ:Ðå )¨$¬+Ð6U¸¼Ô8UÐ6UÔ*VÑ WÔ Wð 	ð 	ÑˆAˆ}Ø-¨d¬kÔ.EÀaÔ.HÔIˆKØ.¨q¨q¨q°!°!°!°Q¸¸¸¨zÔ:ˆOà)˜MØØ# D¤KÔ$;¸AÔ$>Ô?Øð	ð "2Ø*Ø)Ø /ð	ð 	ð ð	ð 	ˆMˆMõ !œ: m°AÔ&6¸!Ñ&;ÀÈTÐRÑRÔRÐVYÑYÐØ+¨AÔ.Ð/ÐÝ�q˜$œ+Ô6Ñ7Ô7ð 	<ð 	<ˆAà-R¨TÔ-KÈAÐPQÉEÔ-RÐS`ÐabÔScÑ-dÔ-dÐØ#3×#6Ò#6¸_Ô=RÐ[kÔ[rÐ#6Ñ#sÔ#sÐ Ý!œJÐ';¸QÑ'>ÀBÐPTÐUÑUÔUˆMÝ!œJ¥u¤}°]ÀN×DUÒDUÐVfÔVmÑDnÔDnÑ'oÔ'oÑpÔpˆMØ#7Ð:JÑ#JÈ]Ñ#ZÐ Ø×%Ò%Ð&:Ñ;Ô;Ð;Ð;åœÐ$6Ñ7Ô7ˆÝœ
 =°aÐ8Ñ8Ô8ˆØŸ	š	 -Ñ0Ô0ˆå&Ø+Ø+ð
ñ 
ô 
ð 	
r5   c                 ór   —  |                       |¦  «        j        g |j        ¢| j        j        ‘| j        ‘R Ž S r`   )r¡  r£   r±   rl   rJ  rk  )rT   rÐ  s     r6   r  z%Gemma3nTextModel.get_per_layer_inputs   sP   € Ø=ˆt×*Ò*¨9Ñ5Ô5Ô=ð 
ØŒ_ð
àŒKÔ)ð
ð Ô,ð
ð 
ð 
ð 	
r5   c                 ó´  — |                       |¦  «        }|| j                             |j        |j        ¬¦  «        z  } |j        g |j        d d…         ¢| j        j        ‘| j	        ‘R Ž }|  
                    |¦  «        }|€|S |j        |j        k    r|dd | j        j        …d d …f         }||z   | j                             |j        |j        ¬¦  «        z  S )Nrï  rW   .)r  r—  r‘   rŒ   r�   r£   r±   rl   rJ  rk  r  r˜  )rT   r­  r  ro  s       r6   r  z)Gemma3nTextModel.project_per_layer_inputs  s(  € ð
 .2×-LÒ-LÈ]Ñ-[Ô-[ÐØ Ô ?× BÒ BØÔ%Ð.BÔ.Ið !Cñ !
ô !
ñ 	
Ðð  <Ð3Ô;ð  
ØÔ   " Ô%ð 
àŒKÔ)ð 
ð Ô,ð 
ð  
ð  
Ðð
  $×=Ò=Ð>RÑSÔSÐàÐ#Ø'Ð'àÔ%Ð)9Ô)?Ò?Ð?à/°Ð5T°t´{Ô7TÐ5TÐVWÐVWÐVWÐ0WÔXÐà$Ð'7Ñ7¸4Ô;U×;XÒ;XØÔ%Ð.BÔ.Ið <Yñ <
ô <
ñ 
ð 	
r5   )NNNNNNNr`   )r-   r.   r/   r'   r3   rÁ  rO   r    r!   r   r1   r‚  rg   r   r=   rf   r   r   r   rd   r  r  rh   ri   s   @r6   r–  r–  P  sÀ  ø€ € € € € € àÐÐÑØ Ðð7Ð0ð 7ð 7ð 7ð 7ð 7ð 7ðr  Ø€_¨EÐ2Ñ2Ô2Øð .2Ø04Ø.2Ø04Ø(,Ø26Ø!%ðm
ð m
àÔ# dÑ*ðm
ð  œ,¨Ñ-ðm
ð œ tÑ+ð	m
ð
 Ô&¨Ñ-ðm
ð  ™ðm
ð Ô(¨4Ñ/ðm
ð ˜$‘;ðm
ð Ð+Ô,ðm
ð 
!ðm
ð m
ð m
ñ „^ñ 3Ô2ñ  Ôðm
ð^
¨eÔ.>ð 
À5Ä<ð 
ð 
ð 
ð 
ð 15ð
ð 
à”|ð
ð  œ,¨Ñ-ð
ð 
Œð	
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r5   r–  z?The base Gemma 3n language model with a language modeling head.c                   ó*  ‡ — e Zd ZU ddiZddiZddgdgfiZeed<   defˆ fd„Ze	e
	 	 	 	 	 	 	 	 ddej        d	z  dej        d	z  dej        d	z  ded	z  dej        d	z  dej        d	z  ded	z  deej        z  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚGemma3nForCausalLMúlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrC   rA   rl   c                 óþ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S rà  )
rN   rO   r–  r…  r°  rP   r   rw   r3  rÏ  rš  s     €r6   rO   zGemma3nForCausalLM.__init__*  sj   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý% fÑ-Ô-ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒð 	�ŠÑÔÐÐÐr5   Nr   rÐ  r/  rr  rB   r­  Úlabelsr  Úlogits_to_keepr6  r^   c	           
      óÀ  —  | j         d||||||dœ|	¤Ž}
|
j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }| j        j        �2|| j        j        z  }t          j	        |¦  «        }|| j        j        z  }d}|� | j
        ||| j        fi |	¤Ž}t          |||
j        |
j        |
j        ¬¦  «        S )a„  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, Gemma3nForCausalLM

        >>> model = Gemma3nForCausalLM.from_pretrained("google/gemma-2-9b")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-9b")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```)rÐ  r/  rr  rB   r­  r  N)r@   rA   rB   rC   rD   r4   )r…  rÑ  r�  re   Úslicer3  rl   Úfinal_logit_softcappingr1   r  Úloss_functionr°  r   rB   rC   rD   )rT   rÐ  r/  rr  rB   r­  r6  r  r7  r6  ÚoutputsrC   Úslice_indicesrA   r@   s                  r6   rd   zGemma3nForCausalLM.forward3  s&  € ð@ ,6¨4¬:ð ,
ØØ)Ø%Ø+Ø'Øð,
ð ,
ð ð,
ð ,
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆØŒ;Ô.Ð:Ø˜dœkÔAÑAˆFÝ”Z Ñ'Ô'ˆFØ˜dœkÔAÑAˆFàˆØÐØ%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDå%ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r5   )NNNNNNNr   )r-   r.   r/   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr'   r3   rO   r   r   r1   r‚  rg   r   r=   rf   re   r   r   r   rd   rh   ri   s   @r6   r1  r1  #  sf  ø€ € € € € € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐÐÑðÐ0ð ð ð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð;
ð ;
àÔ# dÑ*ð;
ð œ tÑ+ð;
ð Ô&¨Ñ-ð	;
ð
  ™ð;
ð Ô(¨4Ñ/ð;
ð Ô  4Ñ'ð;
ð ˜$‘;ð;
ð ˜eœlÑ*ð;
ð Ð+Ô,ð;
ð 
 ð;
ð ;
ð ;
ñ „^ñ Ôð;
ð ;
ð ;
ð ;
ð ;
r5   r1  c                   óv   ‡ — e Zd ZdZdeez  defˆ fd„Z	 	 d
dej	        dz  dej
        dz  dej
        fd	„Zˆ xZS )ÚGemma3nMultimodalEmbedderzQEmbeds token ids or soft tokens for multimodal content into language model space.Úmultimodal_configÚtext_configc                 ó  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        t          j
        | j        | j        ¦  «        | _        t          | j        | j        ¬¦  «        | _        t          | j        | j        ¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          | j        | j        d¬¦  «        | _        d S )Nr¯  Frn   )rJ   rK   )rN   rO   rw   Úmultimodal_hidden_sizer±  rJ   Úvocab_offsetr°  Útext_hidden_sizerP   rÃ  Ú	embeddingrG   Úhard_embedding_normÚsoft_embedding_normr   Úembedding_projectionÚembedding_post_projection_norm)rT   rC  rD  rU   s      €r6   rO   z"Gemma3nMultimodalEmbedder.__init__v  så   ø€ õ
 	‰Œ×ÒÑÔÐà&7Ô&CˆÔ#Ø$Ô1ˆŒØ-Ô:ˆÔØ+Ô6ˆŒØ +Ô 7ˆÔåœ d¤o°tÔ7RÑSÔSˆŒÝ#1°$Ô2MÐSWÔS[Ð#\Ñ#\Ô#\ˆÔ Ý#1°$Ô2MÐSWÔS[Ð#\Ñ#\Ô#\ˆÔ Ý$&¤I¨dÔ.IÈ4ÔK`ÐglÐ$mÑ$mÔ$mˆÔ!Ý.<¸TÔ=RÐX\ÔX`ÐmrÐ.sÑ.sÔ.sˆÔ+Ð+Ð+r5   NrÐ  r­  r^   c                 ó  — |du |duz  rt          d¦  «        ‚|�|                      |¦  «        }n2|                      || j        z
  ¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        S )aå  Embeds token ids or soft tokens for multimodal content into language model space.

        Args:
            input_ids: A torch.LongTensor containing the token ids to embed. Values should be in the range
                `[vocab_offset, vocab_offset + vocab_size)`.
            inputs_embeds: A torch.Tensor containing the soft tokens to embed.

        Returns:
            A torch.Tensor of embeddings with  shape `[batch_size, seq_len, self.config.text_config.hidden_size]`.
        Nr  )r  rK  rI  rG  rJ  rL  rM  )rT   rÐ  r­  Úemb_normÚhard_embÚemb_norm_projs         r6   rd   z!Gemma3nMultimodalEmbedder.forward‰  s™   € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ$Ø×/Ò/°Ñ>Ô>ˆHˆHà—~’~ i°$Ô2CÑ&CÑDÔDˆHØ×/Ò/°Ñ9Ô9ˆHà×1Ò1°(Ñ;Ô;ˆØ×2Ò2°=ÑAÔAÐAr5   r`  )r-   r.   r/   r0   r%   r(   r'   rO   r1   r‚  rg   rd   rh   ri   s   @r6   rB  rB  s  s½   ø€ € € € € Ø[Ð[ðtà-Ð0CÑCðtð 'ðtð tð tð tð tð tð* .2Ø-1ðBð BàÔ# dÑ*ðBð ”| dÑ*ðBð 
Œð	Bð Bð Bð Bð Bð Bð Bð Br5   rB  z�
    The base Gemma 3n model comprising a vision backbone, an audio backbone, and a language model without a
    language modeling head.
    c                   óx  ‡ — e Zd ZdZdefˆ fd„Ze ed¬¦  «        dej	        de
e         deez  fd	„¦   «         ¦   «         Z	 	 	 	 ddej        d
z  dej	        d
z  dej	        d
z  dej	        d
z  fd„Ze	 	 	 	 	 	 	 	 	 	 	 ddej        d
z  dej	        d
z  dej	        d
z  dej        d
z  dej        d
z  dej        d
z  ded
z  dej        d
z  dej	        d
z  dej        d
z  ded
z  de
e         defd„¦   «         Zd„ Zd„ Ze ed¬¦  «        dej        dej        de
e         deez  fd„¦   «         ¦   «         Zˆ xZS ) ÚGemma3nModelFrl   c                 óî  •— t          ¦   «                              |¦  «         t          j        |j        ¬¦  «        | _        |j        j        | _        t          j        |j        ¬¦  «        }|| _        |j        j	        | _	        t          j        |j
        ¦  «        | _        t          |j        |j        ¦  «        | _        t          |j
        |j        ¦  «        | _        |                      ¦   «          d S )Nr  )rN   rO   r#   Úfrom_configÚvision_configÚvision_towerrD  r°  Úlanguage_modelr²  Úaudio_configÚaudio_towerrB  Úembed_visionÚembed_audiorÏ  )rT   rl   rX  rU   s      €r6   rO   zGemma3nModel.__init__¯  sÉ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý%Ô1¸Ô9MÐNÑNÔNˆÔØ Ô,Ô7ˆŒå"Ô.°fÔ6HÐIÑIÔIˆØ,ˆÔØ*0Ô*<Ô*WˆÔ'Ý$Ô0°Ô1DÑEÔEˆÔÝ5°fÔ6JÈFÔL^Ñ_Ô_ˆÔÝ4°VÔ5HÈ&ÔJ\Ñ]Ô]ˆÔØ�ŠÑÔÐÐÐr5   zOProjects the last hidden state from the vision model into language model space.r7   Úpixel_valuesr6  r^   c                 ó:  —  | j         d	|dddœ|¤Ž}|j        }|                     |j        d         | j        j        j        | j        j        ¦  «                             ddd¦  «        }|| j        j        j        dz  z  }|  	                    |¬¦  «        |_
        |S )
NFT)r]  Ú
do_poolingÚreturn_dictr   r"   r$   rþ  ©r­  r4   )rW  rÑ  r£   r±   rl   rV  rw   Úvision_soft_tokens_per_imager³   r[  Úpooler_output)rT   r]  r6  Úvision_outputsrÑ  s        r6   Úget_image_featureszGemma3nModel.get_image_features¼  s¶   € ð +˜Ô*Ðs¸ÐQVÐdhÐsÐsÐlrÐsÐsˆØ*Ô<Ðð .×5Ò5ØÔ# AÔ&ØŒKÔ%Ô1ØŒKÔ4ñ
ô 
÷ Š'�!�Q˜Ñ
Ô
ð	 	ð 	˜Tœ[Ô6ÔBÀCÑGÑGÐØ'+×'8Ò'8ÐGXÐ'8Ñ'YÔ'YˆÔ$àÐr5   NrÐ  r­  Úimage_featuresÚaudio_featuresc           	      ó4  — |€Ç| |                       ¦   «         t          j        | j        j        t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }| |                       ¦   «         t          j        | j        j        t          j        |j        ¬¦  «        ¦  «        k                         d¦  «        }n || j        j        k    }|| j        j        k    }| 	                    ¦   «         }| 
                    d¦  «                             |j        ¦  «        }|�St          ||j        d         z  |                     ¦   «         k    d|› d|j        d         |j        d         z  › �¦  «         | 	                    ¦   «         }| 
                    d¦  «                             |j        ¦  «        }|�St          ||j        d         z  |                     ¦   «         k    d|› d|j        d         |j        d         z  › �¦  «         ||fS )	zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        Nrï  rW   z6Image features and image tokens do not match, tokens: z, features: r   r$   z6Audio features and audio tokens do not match, tokens: )Úget_input_embeddingsr1   r×   rl   Úimage_token_idÚlongr�   ÚallÚaudio_token_idr6  r†   r‘   r   r±   Únumel)	rT   rÐ  r­  rf  rg  Úspecial_image_maskÚspecial_audio_maskÚn_image_tokensÚn_audio_tokenss	            r6   Úget_placeholder_maskz!Gemma3nModel.get_placeholder_maskÒ  sj  € ð ÐØ!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”˜Tœ[Ô7½u¼zÐR_ÔRfÐgÑgÔgñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;ÐàØ.�4×,Ò,Ñ.Ô.Ý”L ¤Ô!;Å5Ä:ÐVcÔVjÐkÑkÔkñô ò÷ Šc�"‰gŒgð Ðð "+¨d¬kÔ.HÒ!HÐØ!*¨d¬kÔ.HÒ!HÐà+×/Ò/Ñ1Ô1ˆØ/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐØÐ%Ý"Ø Ô!4°RÔ!8Ñ8¸N×<PÒ<PÑ<RÔ<RÒRð YÈð  Yð  YÐesÔeyÐz{Ôe|ð  @Nô  @Tð  UVô  @Wñ  fWð  Yð  Yñô ð ð
 ,×/Ò/Ñ1Ô1ˆØ/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐØÐ%Ý"Ø Ô!4°RÔ!8Ñ8¸N×<PÒ<PÑ<RÔ<RÒRð YÈð  Yð  YÐesÔeyÐz{Ôe|ð  @Nô  @Tð  UVô  @Wñ  fWð  Yð  Yñô ð ð
 "Ð#5Ð5Ð5r5   Úinput_featuresr/  Úinput_features_maskrr  rB   Útoken_type_idsr6  r  Ú	lm_kwargsc                 óN  — |du |	duz  rt          d¦  «        ‚|�� |                      ¦   «         |¦  «        }	t          j        |dk    || j        k     ¦  «        }t          j        ||t          j        |¦  «        ¦  «        }| j                             |¦  «        }t          j        || j	        j
        k    || j        j
        k     ¦  «        }| j	        j
        | j	        j        z   dz
  }t          j        |||¦  «                             |	j        ¦  «        }|  	                    |¬¦  «        }|                     |	j        |	j        ¦  «        }|                     d¦  «        }t          j        |||	¦  «        }	|| j        j
        k    }| j        j
        | j        j        z   dz
  }t          j        |||¦  «                             |	j        ¦  «        }|                      |¬¦  «        }|                     |	j        |	j        ¦  «        }|                     d¦  «        }t          j        |||	¦  «        }	nd}|�m|                      |d¬¦  «        j        }|                     |	j        |	j        ¦  «        }|                      ||	|¬	¦  «        \  }}|	                     ||¦  «        }	|��3|��0|                      || d¬¦  «        }|j        }|j        }t          j        | j        dz
  ggt          j        |j        ¬
¦  «        }|                      |¬¦  «        } t          j        |                     d¦  «        | |¦  «        }|j        \  }!}"}#| j        j        |"z
  }$|                      |!|$|#¦  «        }%t          j        ||%fd¬¦  «        }|                     |	j        |	j        ¦  «        }|                      ||	|¬¦  «        \  }}&|	                     |&|¦  «        }	 | j        dd|||||	|ddœ|¤Ž}'t=          |'j        |r|'j         nd|'j!        |'j"        |�|nd|�|nd¬¦  «        S )a}  
        input_features_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Attention mask for `input_features` where non-zero values mark valid audio frames.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Gemma3nForConditionalGeneration

        >>> model = Gemma3nForConditionalGeneration.from_pretrained("google/gemma3n2-3b-mix-224")
        >>> processor = AutoProcessor.from_pretrained("google/gemma3n2-3b-mix-224")

        >>> prompt = "Where is the cat standing?"
        >>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=image, text=prompt,  return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(**inputs,)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Where is the cat standing?\nsnow"
        ```
        Nr  r   r$   )rÐ  rW   T)r`  )r­  rf  rï  r�   )r­  rg  )rÐ  r  r/  rr  rB   r­  r  r`  )rÑ  rB   rC   rD   r;   r<   r4   )#r  ri  r1   r  r²  r	  Ú
zeros_likerX  r  r[  rG  r\  r°  r‘   r�   rŒ   r†   re  rc  rs  Úmasked_scatterÚget_audio_featuresr,   r×   rk  r±   rl   Úaudio_soft_tokens_per_imager(  r“   r:   rÑ  rB   rC   rD   )(rT   rÐ  r]  rt  r/  ru  rr  rB   rv  r­  r6  r  rw  Úper_layer_inputs_maskÚper_layer_inputs_tokensr  Úvision_maskÚdummy_vision_token_idÚvision_input_idsÚvision_embedsÚexpanded_vision_maskÚ
audio_maskÚdummy_audio_token_idÚaudio_input_idsÚaudio_embedsÚexpanded_audio_maskrf  ro  rµ   Úaudio_outputsrg  Úaudio_padding_toksÚaudio_padding_embsÚaudio_batch_sizeÚaudio_seq_lenÚaudio_embed_dimÚextra_padding_tokensÚextra_padding_featuresrp  r<  s(                                           r6   rd   zGemma3nModel.forwardþ  s’  € ð` ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÑ Ø7˜D×5Ò5Ñ7Ô7¸	ÑBÔBˆMõ %*Ô$5°iÀ1²nÀiÐRVÔRqÒFqÑ$rÔ$rÐ!Ý&+¤kÐ2GÈÕTYÔTdÐenÑToÔToÑ&pÔ&pÐ#Ø#Ô2×GÒGÐH_Ñ`Ô`Ðõ  Ô+Ø˜TÔ.Ô;Ò;¸YÈÔIYÔIfÒ=fñô ˆKð %)Ô$5Ô$BÀTÔEVÔEaÑ$aÐdeÑ$eÐ!Ý$œ{¨;¸	ÐCXÑYÔY×\Ò\Ð]jÔ]qÑrÔrÐØ ×-Ò-Ð8HÐ-ÑIÔIˆMØ)×,Ò,¨]Ô-AÀ=ÔCVÑWÔWˆMØ#.×#8Ò#8¸Ñ#<Ô#<Ð Ý!œKÐ(<¸mÈ]Ñ[Ô[ˆMð # dÔ&6Ô&CÒCˆJØ#'Ô#3Ô#@À4ÔCSÔC^Ñ#^ÐabÑ#bÐ Ý#œk¨*°iÐAUÑVÔV×YÒYÐZgÔZnÑoÔoˆOØ×+Ò+°oÐ+ÑFÔFˆLØ'Ÿ?š?¨=Ô+?ÀÔATÑUÔUˆLØ",×"6Ò"6°rÑ":Ô":ÐÝ!œKÐ(;¸\È=ÑYÔYˆMˆMà#Ðð Ð#Ø!×4Ò4°\ÈtÐ4ÑTÔTÔbˆNØ+×.Ò.¨}Ô/CÀ]ÔEXÑYÔYˆNØ$(×$=Ò$=Ø¨À~ð %>ñ %ô %Ñ!Ð ð *×8Ò8Ð9KÈ^Ñ\Ô\ˆMð Ñ%Ð*=Ñ*IØ ×3Ò3°NÐEXÐDXÐfjÐ3ÑkÔkˆMØ*Ô8ˆNØ&Ô5ˆJõ "'¤°´À!Ñ0CÐ/DÐ.EÍUÌZÐ`nÔ`uÐ!vÑ!vÔ!vÐØ!%×!1Ò!1Ð<NÐ!1Ñ!OÔ!OÐÝ"œ[¨×)=Ò)=¸bÑ)AÔ)AÐCUÐWeÑfÔfˆNà?MÔ?SÑ<Ð˜m¨_Ø#'¤;Ô#JÈ]Ñ#ZÐ Ø%7×%>Ò%>Ð?OÐQeÐgvÑ%wÔ%wÐ"å"œY¨Ð8NÐ'OÐUVÐWÑWÔWˆNØ+×.Ò.¨}Ô/CÀ]ÔEXÑYÔYˆNØ$(×$=Ò$=Ø¨À~ð %>ñ %ô %Ñ!ˆAÐ!ð *×8Ò8Ð9KÈ^Ñ\Ô\ˆMà%�$Ô%ð 

ØØ-Ø)Ø%Ø+Ø'ØØð

ð 

ð ð

ð 

ˆõ *Ø%Ô7Ø7@ÐJ˜GÔ3Ð3ÀdØ!Ô/ØÔ)Ø2>Ð2J  ÐPTØ2@Ð2L  ÐRVð
ñ 
ô 
ð 	
r5   c                 ó   — | j         j        S r`   ©rX  r¡  r¢  s    r6   r£  z+Gemma3nModel.get_per_layer_input_embeddings‡  s   € ØÔ"Ô9Ð9r5   c                 ó   — || j         _        d S r`   r’  r¥  s     r6   r¦  z+Gemma3nModel.set_per_layer_input_embeddingsŠ  s   € Ø5:ˆÔÔ2Ð2Ð2r5   zPProjects the last hidden state from the audio encoder into language model space.c                 ól   —  | j         ||fddi|¤Ž}|                      |j        ¬¦  «        }||_        |S )a0  
        input_features (`torch.FloatTensor]` of shape `(num_images, seq_length, num_features)`):
            The tensors corresponding to the input audio.
        input_features_mask (`torch.FloatTensor]` of shape `(num_images, seq_length)`):
            The attention mask for the input audio.
        r`  Tra  )rZ  r\  rÑ  rc  )rT   rt  ru  r6  r‰  r‡  s         r6   r{  zGemma3nModel.get_audio_features�  s]   € ð 9I¸Ô8HØÐ/ð9
ð 9
Ø=Að9
ØEKð9
ð 9
ˆð ×'Ò'°mÔ6UÐ'ÑVÔVˆØ&2ˆÔ#àÐr5   rú  )NNNNNNNNNNN)r-   r.   r/   Úaccepts_loss_kwargsr&   rO   r   r   r1   r=   r   r   rE   r   re  r‚  rs  rg   r   rf   r:   rd   r£  r¦  r+   r{  rh   ri   s   @r6   rS  rS  ¥  sß  ø€ € € € € ð  Ðð˜}ð ð ð ð ð ð ð Ø€^Ð!rÐsÑsÔsðàÔ'ðð Ð+Ô,ðð 
Ð+Ñ	+ð	ð ð ñ tÔsñ Ôðð, .2Ø26Ø37Ø37ð*6ð *6àÔ# dÑ*ð*6ð Ô(¨4Ñ/ð*6ð Ô)¨DÑ0ð	*6ð
 Ô)¨DÑ0ð*6ð *6ð *6ð *6ðX ð .2Ø15Ø37Ø.2Ø37Ø04Ø(,Ø26Ø26Ø*.Ø!%ðF
ð F
àÔ# dÑ*ðF
ð Ô'¨$Ñ.ðF
ð Ô)¨DÑ0ð	F
ð
 œ tÑ+ðF
ð #œ\¨DÑ0ðF
ð Ô&¨Ñ-ðF
ð  ™ðF
ð Ô(¨4Ñ/ðF
ð Ô(¨4Ñ/ðF
ð Ô  4Ñ'ðF
ð ˜$‘;ðF
ð Ð.Ô/ðF
ð 
$ðF
ð F
ð F
ñ ÔðF
ðP:ð :ð :ð;ð ;ð ;ð Ø€^Ð!sÐtÑtÔtðàœðð #œ\ðð Ð+Ô,ð	ð
 
Ð/Ñ	/ðð ð ñ uÔtñ Ôðð ð ð ð r5   rS  z†
    The base Gemma 3n model comprising a vision backbone, an audio backbone, a language model, and a language modeling
    head.
    c                   óÈ  ‡ — e Zd ZddiZdZdefˆ fd„Zedej	        de
e         fd„¦   «         Zee	 	 	 	 	 	 	 	 	 	 	 	 ddej        d	z  dej	        d	z  dej	        d	z  dej        d	z  dej        d	z  dej        d	z  ded	z  dej        d	z  dej	        d	z  dej        d	z  ded	z  deej        z  de
e         defd„¦   «         ¦   «         Z	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd„	Zd„ Zd„ Zˆ xZS )ÚGemma3nForConditionalGenerationr2  z(model.language_model.embed_tokens.weightFrl   c                 óú   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        j        |j        j        d¬¦  «        | _	        |  
                    ¦   «          d S rà  )rN   rO   rS  r…  rP   r   rD  rw   r°  r3  rÏ  rš  s     €r6   rO   z(Gemma3nForConditionalGeneration.__init__®  se   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý! &Ñ)Ô)ˆŒ
Ý”y Ô!3Ô!?ÀÔASÔA^ÐejÐkÑkÔkˆŒØ�ŠÑÔÐÐÐr5   r]  r6  c                 ó(   —  | j         j        |fi |¤ŽS r`   )r…  re  )rT   r]  r6  s      r6   re  z2Gemma3nForConditionalGeneration.get_image_features´  s   € à,ˆtŒzÔ,¨\ÐDÐD¸VÐDÐDÐDr5   Nr   rÐ  rt  r/  ru  rr  rB   rv  r­  r6  r  r7  rw  r^   c                 ó  —  | j         d|||||||||	|
|ddœ|¤Ž}|j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }| j                             ¦   «         j        x}�||z  }t          j
        |¦  «        }||z  }d}|
�, | j        ||
| j                             ¦   «         j        fi |¤Ž}t          |||j        |j        |j        |j        |j        ¬¦  «        S )aŒ  
        input_features_mask (torch.Tensor, *optional*, defaults to None):
            The attention mask for the input audio.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels in
            `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration

        >>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma-3-4b-it")
        >>> processor = AutoProcessor.from_pretrained("google/gemma-3-4b-it")

        >>> messages = [
        ...     {
        ...         "role": "system",
        ...         "content": [
        ...             {"type": "text", "text": "You are a helpful assistant."}
        ...         ]
        ...     },
        ...     {
        ...         "role": "user", "content": [
        ...             {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
        ...             {"type": "text", "text": "Where is the cat standing?"},
        ...         ]
        ...     },
        ... ]

        >>> inputs = processor.apply_chat_template(
        ...     messages,
        ...     tokenizer=True,
        ...     return_dict=True,
        ...     return_tensors="pt",
        ...     add_generation_prompt=True
        ... )
        >>> # Generate
        >>> generate_ids = model.generate(**inputs)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "user\nYou are a helpful assistant.\n\n\n\n\n\nWhere is the cat standing?\nmodel\nBased on the image, the cat is standing in a snowy area, likely outdoors. It appears to"
        ```
        T)rÐ  r]  rt  r/  ru  rr  rB   rv  r­  r6  r  r`  N)r@   rA   rB   rC   rD   r;   r<   r4   )r…  rÑ  r�  re   r9  r3  rl   r±  r:  r1   r  r;  r°  r?   rB   rC   rD   r;   r<   )rT   rÐ  r]  rt  r/  ru  rr  rB   rv  r­  r6  r  r7  rw  r<  rC   r=  rA   r:  r@   s                       r6   rd   z'Gemma3nForConditionalGeneration.forward¸  s^  € ðD �$”*ð 
ØØ%Ø)Ø)Ø 3Ø%Ø+Ø)Ø'ØØØð
ð 
ð ð
ð 
ˆð   Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆØ'+¤{×'BÒ'BÑ'DÔ'DÔ'\Ð\Ð#ÐiØÐ5Ñ5ˆFÝ”Z Ñ'Ô'ˆFØÐ5Ñ5ˆFàˆØÐØ%�4Ô% f¨f°d´k×6QÒ6QÑ6SÔ6SÔ6^ÐlÐlÐbkÐlÐlˆDå,ØØØ#Ô3Ø!Ô/ØÔ)Ø 'Ô ;Ø 'Ô ;ð
ñ 
ô 
ð 	
r5   Tc                 ót   •—  t          ¦   «         j        |f|||||
||	|dœ|¤Ž}|s|
s||d<   ||d<   ||d<   |S )N)rB   r­  r/  rr  r  r7  rv  Úis_first_iterationr]  rt  ru  )rN   Úprepare_inputs_for_generation)rT   rÐ  rB   r­  rr  r]  rt  r/  ru  rv  r  r7  r6  rœ  r6  Úmodel_inputsrU   s                   €r6   r�  z=Gemma3nForConditionalGeneration.prepare_inputs_for_generation!	  s…   ø€ ð$ =•u‘w”wÔ<Øð
à+Ø'Ø)Ø%ØØ)Ø)Ø1ð
ð 
ð ð
ð 
ˆð  ð 	F Yð 	FØ+7ˆL˜Ñ(Ø-;ˆLÐ)Ñ*Ø2EˆLÐ.Ñ/àÐr5   c                 ó4   — | j                              ¦   «         S r`   )r…  r£  r¢  s    r6   r£  z>Gemma3nForConditionalGeneration.get_per_layer_input_embeddingsJ	  s   € ØŒz×8Ò8Ñ:Ô:Ð:r5   c                 ó:   — | j                              |¦  «         d S r`   )r…  r¦  r¥  s     r6   r¦  z>Gemma3nForConditionalGeneration.set_per_layer_input_embeddingsM	  s   € ØŒ
×1Ò1°%Ñ8Ô8Ð8Ð8Ð8r5   )NNNNNNNNNNNr   )NNNNNNNNTNNF)r-   r.   r/   r>  r•  r&   rO   r   r1   r=   r   r   re  r   r‚  rg   r   rf   re   r?   rd   r�  r£  r¦  rh   ri   s   @r6   r—  r—  ¤  sG  ø€ € € € € ð +Ð,VÐWÐØÐð˜}ð ð ð ð ð ð ð ðE¨uÔ/@ð EÈFÐSeÔLfð Eð Eð Eñ „^ðEð Øð .2Ø15Ø37Ø.2Ø37Ø04Ø(,Ø26Ø26Ø*.Ø!%Ø-.ðe
ð e
àÔ# dÑ*ðe
ð Ô'¨$Ñ.ðe
ð Ô)¨DÑ0ð	e
ð
 œ tÑ+ðe
ð #œ\¨DÑ0ðe
ð Ô&¨Ñ-ðe
ð  ™ðe
ð Ô(¨4Ñ/ðe
ð Ô(¨4Ñ/ðe
ð Ô  4Ñ'ðe
ð ˜$‘;ðe
ð ˜eœlÑ*ðe
ð Ð.Ô/ðe
ð 
'ðe
ð e
ð e
ñ „^ñ Ôðe
ðT ØØØØØØ ØØØØØ ð'ð 'ð 'ð 'ð 'ð 'ðR;ð ;ð ;ð9ð 9ð 9ð 9ð 9ð 9ð 9r5   r—  )rÅ  r1  r—  rS  r„  r–  )rÅ   NN)r$   )fr�   Úcollectionsr   Úcollections.abcr   r   Údataclassesr   Útypingr   r1   Útorch.nnrP   Útorch.nn.functionalr¡   rm  Ú r	   r‘  rì  r
   Úcache_utilsr   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   r   Úutils.genericr   r    Úutils.output_capturingr!   Úautor#   Úconfiguration_gemma3nr%   r&   r'   r(   Úaccelerate.hooksr)   r+   r:   r?   ÚModulerG   rk   rÃ   r+  rN  rt  r’  r¡  r­  r¿  rÃ  rÉ  rÓ  rÝ  rþ  r%  rg   re   r+  ra   rE   r9  r<  r>  rc  r„  rÅ  r™  r–  r1  rB  rS  r—  Ú__all__r4   r5   r6   ú<module>r¸     sÐ	  ðð* €€€Ø  Ð  Ð  Ð  Ð  Ð  Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø kÐ kÐ kÐ kÐ kÐ kÐ kÐ kÐ kÐ kØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð HÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø Ð Ð Ð Ð Ð Ø lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lð ÐÑÔð 4Ø3Ð3Ð3Ð3Ð3Ð3ð Ø
ð3ð 3ð 3ð 3ð 3Ð%?ñ 3ô 3ñ „ñ „ð3ð €ððñ ô ð
 ð9ð 9ð 9ð 9ð 9Ð!8ñ 9ô 9ñ „ñô ð9ð( €ððñ ô ð
 ð9ð 9ð 9ð 9ð 9 Kñ 9ô 9ñ „ñô ð9ð:4ð 4ð 4ð 4ð 4�R”Yñ 4ô 4ð 4ð0g)ð g)ð g)ð g)ð g)¨B¬Iñ g)ô g)ð g)ðTað að að að a˜BœIñ aô að aðHj,ð j,ð j,ð j,ð j, b¤iñ j,ô j,ð j,ðZB7ð B7ð B7ð B7ð B7 ¤	ñ B7ô B7ð B7ðJFð Fð Fð Fð F¨"¬)ñ Fô Fð FðROð Oð Oð Oð O R¤Yñ Oô Oð Oð8Dð Dð Dð Dð D r¤yñ Dô Dð Dð2(ð (ð (ð (ð ( r¤yñ (ô (ð (ðVð ð ð ð  ¤ñ ô ð ð6Sð Sð Sð Sð S R¤\ñ Sô Sð Sð;ð ;ð ;ð ;ð ;˜RœYñ ;ô ;ð ;ð$#5ð #5ð #5ð #5ð #5�R”Yñ #5ô #5ð #5ðL`'ð `'ð `'ð `'ð `'�r”yñ `'ô `'ð `'ðF(ð (ð (ð	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð$ Ø Ø ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �S‰[ð%ð �T‰\ð%ð �T‰\ð%ð ˆ5Œ<˜œÐ%Ô&ð%ð %ð %ð %ðD.ð .˜EœLð .¨u¬|ð .À%Ä,ð .Ð_bð .ð .ð .ð .ð,l)ð l)ð l)ð l)ð l)˜2œ9ñ l)ô l)ð l)ð^I%ð I%ð I%ð I%ð I%Ð8ñ I%ô I%ð I%ðX ð^Jð ^Jð ^Jð ^Jð ^J˜_ñ ^Jô ^Jñ „ð^JðBN
ð N
ð N
ð N
ð N
Ð0ñ N
ô N
ð N
ðbL<ð L<ð L<ð L<ð L<˜RœYñ L<ô L<ð L<ð^ €ÐaÐbÑbÔbðO
ð O
ð O
ð O
ð O
Ð-ñ O
ô O
ñ cÔbðO
ðd €Ð^Ð_Ñ_Ô_ðL
ð L
ð L
ð L
ð L
Ð/°ñ L
ô L
ñ `Ô_ðL
ð^/Bð /Bð /Bð /Bð /B ¤	ñ /Bô /Bð /Bðd €ððñ ô ðvð vð vð vð vÐ)ñ vô vñô ðvðr €ððñ ô ðd9ð d9ð d9ð d9ð d9Ð&<¸oñ d9ô d9ñô ðd9ðNð ð €€€r5   