§
    ‚Štj•  ã                   ó>  — d dl Z d dlmZ ddlmZ ddlmZ ddlmZ ddl	m
Z
mZmZmZmZ  ed	¬
¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z G d„ de¦  «        Z G d„ de¦  «        Z G d„ de
¦  «        Z G d„ de¦  «        Z G d„ de¦  «        Zg d¢ZdS )é    N)Ústricté   )ÚCache)Úauto_docstringé   )ÚGemma2Config)ÚGemma2AttentionÚGemma2DecoderLayerÚGemma2ForCausalLMÚ	Gemma2MLPÚGemma2RMSNormzgoogle/vaultgemma-1b)Ú
checkpointc                   ó&   — e Zd ZdZ e¦   «         ZdS )ÚVaultGemmaConfiga(  
    query_pre_attn_scalar (`float`, *optional*, defaults to 256):
        scaling factor used on the attention scores
    final_logit_softcapping (`float`, *optional*, defaults to 30.0):
        scaling factor when applying tanh softcapping on the logits.
    attn_logit_softcapping (`float`, *optional*, defaults to 50.0):
        scaling factor when applying tanh softcapping on the attention scores.

    ```python
    >>> from transformers import VaultGemmaModel, VaultGemmaConfig
    >>> # Initializing a VaultGemma vaultgemma-7b style configuration
    >>> configuration = VaultGemmaConfig()
    >>> # Initializing a model from the vaultgemma-7b style configuration
    >>> model = VaultGemmaModel(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚAttributeErrorÚuse_bidirectional_attention© ó    úo/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/vaultgemma/modular_vaultgemma.pyr   r      s*   € € € € € ðð ð$ #1 .Ñ"2Ô"2ÐÐÐr   r   c                   ó   — e Zd ZdS )ÚVaultGemmaRMSNormN©r   r   r   r   r   r   r   r   1   ó   € € € € € Ø€Dr   r   c                   ó   — e Zd ZdS )ÚVaultGemmaMLPNr   r   r   r   r   r   5   r   r   r   c                   ó,   ‡ — e Zd ZdZdedefˆ fd„Zˆ xZS )ÚVaultGemmaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperÚconfigÚ	layer_idxc                 óV   •— t          ¦   «                              ¦   «          d| _        d S )NT)ÚsuperÚ__init__Ú	is_causal)Úselfr"   r#   Ú	__class__s      €r   r&   zVaultGemmaAttention.__init__<   s$   ø€ Ý‰Œ×ÒÑÔÐØˆŒˆˆr   )r   r   r   r   r   Úintr&   Ú__classcell__©r)   s   @r   r!   r!   9   sR   ø€ € € € € ØGÐGðÐ/ð ¸Cð ð ð ð ð ð ð ð ð ð r   r!   c                   óÞ   ‡ — e Zd Zˆ fd„Z	 	 	 d
dej        deej        ej        f         dej        dz  dej        dz  dedz  deej	        eej	        ej	        f         dz  f         fd	„Z
ˆ xZS )ÚVaultGemmaDecoderLayerc                 óB   •—  t          ¦   «         j        di |¤Ž | `| `d S )Nr   )r%   r&   Úpost_attention_layernormÚpost_feedforward_layernorm)r(   Úsuper_kwargsr)   s     €r   r&   zVaultGemmaDecoderLayer.__init__B   s2   ø€ Ø�‰ŒÔÐ(Ð(˜<Ð(Ð(Ð(ØÐ)ØÐ+Ð+Ð+r   NÚhidden_statesÚposition_embeddingsÚattention_maskÚposition_idsÚpast_key_valuesÚreturnc           	      óÌ   — |}|                       |¦  «        } | j        d|||||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N)r3   r4   r5   r6   r7   r   )Úinput_layernormÚ	self_attnÚpre_feedforward_layernormÚmlp)	r(   r3   r4   r5   r6   r7   ÚkwargsÚresidualÚ_s	            r   ÚforwardzVaultGemmaDecoderLayer.forwardG   sœ   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆà)˜4œ>ð 
Ø'Ø 3Ø)Ø%Ø+ð
ð 
ð ð
ð 
Ñˆ�qð ! =Ñ0ˆà ˆØ×6Ò6°}ÑEÔEˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆàÐr   )NNN)r   r   r   r&   ÚtorchÚTensorÚtupleÚ
LongTensorr   ÚFloatTensorrA   r+   r,   s   @r   r.   r.   A   sÜ   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð /3Ø04Ø(,ðð à”|ðð # 5¤<°´Ð#=Ô>ðð œ tÑ+ð	ð
 Ô&¨Ñ-ðð  ™ðð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uðð ð ð ð ð ð ð r   r.   c                   ó   — e Zd ZdS )ÚVaultGemmaForCausalLMNr   r   r   r   rH   rH   e   r   r   rH   )r   rH   ÚVaultGemmaModelÚVaultGemmaPreTrainedModel)rB   Úhuggingface_hub.dataclassesr   Úcache_utilsr   Úutilsr   Úgemma2.configuration_gemma2r   Úgemma2.modeling_gemma2r	   r
   r   r   r   r   r   r   r!   r.   rH   Ú__all__r   r   r   ú<module>rQ      s»  ðð  €€€Ø .Ð .Ð .Ð .Ð .Ð .à  Ð  Ð  Ð  Ð  Ð  Ø #Ð #Ð #Ð #Ð #Ð #Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uð €Ð1Ð2Ñ2Ô2Øð3ð 3ð 3ð 3ð 3�|ñ 3ô 3ñ „ñ 3Ô2ð3ð,	ð 	ð 	ð 	ð 	˜ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	�Iñ 	ô 	ð 	ðð ð ð ð ˜/ñ ô ð ð!ð !ð !ð !ð !Ð/ñ !ô !ð !ðH	ð 	ð 	ð 	ð 	Ð-ñ 	ô 	ð 	ðð ð €€€r   