
    Ph
                         d dl mZ d dlZddlmZ ddlmZ ddlmZm	Z	  G d d	e      Z
 G d
 de      Z G d de	      Zg dZy)    )OptionalN   )Cache   )Gemma2Config)Gemma2DecoderLayerGemma2ForCausalLMc                       e Zd Zy)VaultGemmaConfigN__name__
__module____qualname__     k/var/www/html/saasai/venv/lib/python3.12/site-packages/transformers/models/vaultgemma/modular_vaultgemma.pyr   r          r   r   c                   T    e Zd Z fdZ	 	 	 	 	 	 ddej
                  deej
                  ej
                  f   deej
                     deej                     dee	   dee
   dee
   d	eej                     d
eej                  eeej                  ej                  f      f   fdZ xZS )VaultGemmaDecoderLayerc                 ,    t        |   di | | `| `y )Nr   )super__init__post_attention_layernormpost_feedforward_layernorm)selfsuper_kwargs	__class__s     r   r   zVaultGemmaDecoderLayer.__init__   s    (<()+r   hidden_statesposition_embeddingsattention_maskposition_idspast_key_valuesoutput_attentions	use_cachecache_positionreturnc	                     |}
| j                  |      } | j                  d||||||||d|	\  }}|
|z   }|}
| j                  |      }| j                  |      }|
|z   }|f}|r||fz  }|S )N)r   r   r    r!   r"   r#   r$   r%   r   )input_layernorm	self_attnpre_feedforward_layernormmlp)r   r   r   r    r!   r"   r#   r$   r%   kwargsresidualself_attn_weightsoutputss                r   forwardzVaultGemmaDecoderLayer.forward#   s     !,,];+94>> 
,
' 3)%+/)
,
 
,
(( !=0 66}E/ =0 ")++Gr   )NNNFFN)r   r   r   r   torchTensortupler   
LongTensorr   boolFloatTensorr0   __classcell__)r   s   @r   r   r      s    , 2637+/,1$)59%||% #5<<#=>% !.	%
 u//0% "%% $D>% D>% !!1!12% 
u  (51B1BEDUDU1U+V"WW	X%r   r   c                       e Zd Zy)VaultGemmaForCausalLMNr   r   r   r   r9   r9   K   r   r   r9   )r   r9   VaultGemmaModelVaultGemmaPreTrainedModel)typingr   r1   cache_utilsr   gemma2.configuration_gemma2r   gemma2.modeling_gemma2r   r	   r   r   r9   __all__r   r   r   <module>rA      sD         6 J	| 	+/ +\	- 	r   