§
    ‚Štj’%  ã                   óv  — d dl Z d dlmZ d dl mZ ddlmZ ddlmZm	Z	 ddl
mZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZ ddlmZmZmZmZmZm Z m!Z!m"Z" ddiZ#dZ$ ej%        e&¦  «        Z' ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z( G d„ dej)        ¦  «        Z* G d„ dej+        ¦  «        Z, G d„ de¦  «        Z- G d„ de"¦  «        Z. G d„ d e¦  «        Z/ G d!„ d"e!¦  «        Z0 G d#„ d$e ¦  «        Z1 G d%„ d&e¦  «        Z2 G d'„ d(e¦  «        Z3 G d)„ d*e¦  «        Z4g d+¢Z5dS ),é    N)Ústrict)Únné   )Úinitialization)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Úcreate_causal_mask)ÚBaseModelOutputWithPast)ÚRopeParameters)ÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚloggingé   )ÚLlamaAttentionÚLlamaForCausalLMÚLlamaForSequenceClassificationÚLlamaForTokenClassificationÚLlamaMLPÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRotaryEmbeddingÚ
vocab_fileztokenizer.modelu   â–�zgoogle/gemma-7b)Ú
checkpointc                   ó¬  — e Zd ZU dZdZdgZddddddddœZdgdgfd	d
gd	gfd	gd	gfdœZdZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZee
d<   dZe	e
d<   dZee
d<   dZee
d <   d!Zee
d"<   d#Ze	d$z  e
d%<   d&Ze	ee	         z  d$z  e
d'<   d(Ze	d$z  e
d)<   d!Zee
d*<   d$Zee z  d$z  e
d+<   d,Z!ee
d-<   d.Z"ee	z  e
d/<   d$Z#ed$z  e
d0<   d$S )1ÚGemmaConfiga  
    use_bidirectional_attention (`bool`, *optional*):
        If True, the model will attend to all text tokens instead of using a causal mask.

    ```python
    >>> from transformers import GemmaModel, GemmaConfig
    >>> # Initializing a Gemma gemma-7b style configuration
    >>> configuration = GemmaConfig()
    >>> # Initializing a model from the gemma-7b style configuration
    >>> model = GemmaModel(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```ÚgemmaÚpast_key_valuesÚcolwiseÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormi è Ú
vocab_sizei   Úhidden_sizei `  Úintermediate_sizeé   Únum_hidden_layersé   Únum_attention_headsÚnum_key_value_headsé   Úhead_dimÚgelu_pytorch_tanhÚ
hidden_acti    Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeç�íµ ÷Æ°>Úrms_norm_epsTÚ	use_cacher   NÚpad_token_idé   Úeos_token_idr   Úbos_token_idÚtie_word_embeddingsÚrope_parametersFÚattention_biasg        Úattention_dropoutÚuse_bidirectional_attention)$Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planr*   ÚintÚ__annotations__r+   r,   r.   r0   r1   r3   r5   Ústrr6   r7   Úfloatr9   r:   Úboolr;   r=   Úlistr>   r?   r@   r   ÚdictrA   rB   rC   © ó    úe/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/gemma/modular_gemma.pyr   r   1   sû  € € € € € € ðð ð €JØ#4Ð"5Ðà%.Ø%.Ø%.Ø%.Ø"+Ø )Ø"+ðð Ðð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ðð Ðð €J�ÐÐÑØ€K�ÐÐÑØ"Ð�sÐ"Ð"Ñ"ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø!Ð˜Ð!Ð!Ñ!Ø€HˆcÐÐÑØ)€J�Ð)Ð)Ñ)Ø#'Ð˜SÐ'Ð'Ñ'Ø#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØ €L�#˜‘*Ð Ð Ñ Ø+,€L�#˜˜Sœ	‘/ DÑ(Ð,Ð,Ñ,Ø €L�#˜‘*Ð Ð Ñ Ø $Ð˜Ð$Ð$Ñ$Ø48€O�^ dÑ*¨TÑ1Ð8Ð8Ñ8Ø €N�DÐ Ð Ñ Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø/3Ð ¨¡Ð3Ð3Ñ3Ð3Ð3rT   r   c            	       óP   ‡ — e Zd ZdZd
dedededefˆ fd„Zdej        fˆ fd	„Z	ˆ xZ
S )ÚGemmaTextScaledWordEmbeddingz\
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    ç      ð?Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scalec                 ó°   •— t          ¦   «                              |||¦  «         || _        |                      dt	          j        |¦  «        d¬¦  «         d S )Nr\   F)Ú
persistent)ÚsuperÚ__init__Úscalar_embed_scaleÚregister_bufferÚtorchÚtensor)ÚselfrY   rZ   r[   r\   Ú	__class__s        €rU   r`   z%GemmaTextScaledWordEmbedding.__init__n   sS   ø€ Ý‰Œ×Ò˜¨¸ÑDÔDÐDØ"-ˆÔØ×Ò˜]­E¬L¸Ñ,EÔ,EÐRWÐÑXÔXÐXÐXÐXrT   r#   c                 ó�   •— t          ¦   «                              |¦  «        | j                             | j        j        ¦  «        z  S ©N)r_   Úforwardr\   ÚtoÚweightÚdtype)re   r#   rf   s     €rU   ri   z$GemmaTextScaledWordEmbedding.forwards   s4   ø€ Ý‰wŒw�Š˜yÑ)Ô)¨DÔ,<×,?Ò,?ÀÄÔ@QÑ,RÔ,RÑRÐRrT   )rX   )rD   rE   rF   rG   rL   rO   r`   rc   ÚTensorri   Ú__classcell__©rf   s   @rU   rW   rW   i   s¦   ø€ € € € € ðð ðYð Y sð Y¸3ð YÈSð YÐ_dð Yð Yð Yð Yð Yð Yð
S ¤ð Sð Sð Sð Sð Sð Sð Sð Sð Sð SrT   rW   c                   ó<   ‡ — e Zd Zddedefˆ fd„Zd„ Zd„ Zd„ Zˆ xZ	S )	ÚGemmaRMSNormr8   ÚdimÚepsc                 ó¬   •— t          ¦   «                              ¦   «          || _        t          j        t          j        |¦  «        ¦  «        | _        d S rh   )r_   r`   rs   r   Ú	Parameterrc   Úzerosrk   )re   rr   rs   rf   s      €rU   r`   zGemmaRMSNorm.__init__x   s?   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”l¥5¤;¨sÑ#3Ô#3Ñ4Ô4ˆŒˆˆrT   c                 ó�   — |t          j        |                     d¦  «                             dd¬¦  «        | j        z   ¦  «        z  S )Nr   éÿÿÿÿT)Úkeepdim)rc   ÚrsqrtÚpowÚmeanrs   )re   Úxs     rU   Ú_normzGemmaRMSNorm._norm}   s8   € Ø•5”;˜qŸušu Q™xœxŸ}š}¨R¸˜}Ñ>Ô>ÀÄÑIÑJÔJÑJÐJrT   c                 ó¸   — |                       |                     ¦   «         ¦  «        }|d| j                             ¦   «         z   z  }|                     |¦  «        S )NrX   )r~   rO   rk   Útype_as)re   r}   Úoutputs      rU   ri   zGemmaRMSNorm.forward€   sL   € Ø—’˜AŸGšG™IœIÑ&Ô&ˆð ˜3 ¤×!2Ò!2Ñ!4Ô!4Ñ4Ñ5ˆØ�~Š~˜aÑ Ô Ð rT   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)Útuplerk   Úshapers   )re   s    rU   Ú
extra_reprzGemmaRMSNorm.extra_repr‡   s%   € Ý˜œÔ)Ñ*Ô*Ð<Ð<°$´(Ð<Ð<Ð<rT   )r8   )
rD   rE   rF   rL   rO   r`   r~   ri   r…   rn   ro   s   @rU   rq   rq   w   s€   ø€ € € € € ð5ð 5˜Cð 5 eð 5ð 5ð 5ð 5ð 5ð 5ð
Kð Kð Kð!ð !ð !ð=ð =ð =ð =ð =ð =ð =rT   rq   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚGemmaMLPc                 ó.  •— t          ¦   «                              |¦  «         t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        d S )NF)Úbias)	r_   r`   r   ÚLinearr+   r,   Ú	gate_projÚup_projÚ	down_proj©re   Úconfigrf   s     €rU   r`   zGemmaMLP.__init__Œ   s|   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ýœ 4Ô#3°TÔ5KÐRWÐXÑXÔXˆŒÝ”y Ô!1°4Ô3IÐPUÐVÑVÔVˆŒÝœ 4Ô#9¸4Ô;KÐRWÐXÑXÔXˆŒˆˆrT   )rD   rE   rF   r`   rn   ro   s   @rU   r‡   r‡   ‹   sA   ø€ € € € € ðYð Yð Yð Yð Yð Yð Yð Yð YrT   r‡   c                   ó   — e Zd ZdS )ÚGemmaRotaryEmbeddingN©rD   rE   rF   rS   rT   rU   r‘   r‘   “   ó   € € € € € Ø€DrT   r‘   c                   ó,   ‡ — e Zd ZdZdedefˆ fd„Zˆ xZS )ÚGemmaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr�   Ú	layer_idxc                 óv   •— t          ¦   «                              ¦   «          t          |dd¦  «         | _        d S )NrC   F)r_   r`   ÚgetattrÚ	is_causal)re   r�   r–   rf   s      €rU   r`   zGemmaAttention.__init__š   s4   ø€ Ý‰Œ×ÒÑÔÐÝ$ VÐ-JÈEÑRÔRÐRˆŒˆˆrT   )rD   rE   rF   rG   r   rL   r`   rn   ro   s   @rU   r•   r•   —   s]   ø€ € € € € ØGÐGðS˜{ð S°sð Sð Sð Sð Sð Sð Sð Sð Sð Sð SrT   r•   c                   ó>   — e Zd Z ej        ¦   «         d„ ¦   «         ZdS )ÚGemmaPreTrainedModelc                 óî   — t          j        | |¦  «         d|j        j        v rt	          j        |j        ¦  «         d S t          |t          ¦  «        r!t	          j	        |j
        |j        ¦  «         d S d S )NÚRMSNorm)r   Ú_init_weightsrf   rD   ÚinitÚzeros_rk   Ú
isinstancerW   Ú	constant_r\   ra   )re   Úmodules     rU   rž   z"GemmaPreTrainedModel._init_weights    s|   € åÔ% d¨FÑ3Ô3Ð3à˜Ô(Ô1Ð1Ð1ÝŒK˜œÑ&Ô&Ð&Ð&Ð&Ý˜Õ <Ñ=Ô=ð 	JÝŒN˜6Ô-¨vÔ/HÑIÔIÐIÐIÐIð	Jð 	JrT   N)rD   rE   rF   rc   Úno_gradrž   rS   rT   rU   r›   r›   Ÿ   s:   € € € € € Ø€U„]�_„_ðJð Jñ „_ðJð Jð JrT   r›   c                   ó²   ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dedz  dej	        dz  d	e
dz  d
ee         defd„Zˆ xZS )Ú
GemmaModelr�   c                 ó²   •— t          ¦   «                              |¦  «         t          |j        |j        | j        | j        j        dz  ¬¦  «        | _        d S )Ng      à?)r\   )r_   r`   rW   r*   r+   r[   r�   r'   rŽ   s     €rU   r`   zGemmaModel.__init__«   sW   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å8ØÔ˜vÔ1°4Ô3CÐQUÔQ\ÔQhÐjmÑQmð
ñ 
ô 
ˆÔÐÐrT   Nr#   r&   Úposition_idsr    r$   r:   ÚkwargsÚreturnc           
      óP  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          | j        ||||¬¦  «        }	|}
|                      |
|¬¦  «        }| j        d | j        j        …         D ]} ||
f|	||||dœ|¤Ž}
Œ|                      |
¦  «        }
t          |
|r|nd ¬	¦  «        S )
Nz:You must specify exactly one of input_ids or inputs_embeds)r�   r   r<   )Údevice)r�   r$   r&   r    r¨   )r¨   )r&   r¨   r    r:   Úposition_embeddings)Úlast_hidden_stater    )Ú
ValueErrorr'   r   r�   Úget_seq_lengthrc   Úaranger„   r¬   Ú	unsqueezer
   Ú
rotary_embr(   r.   r)   r   )re   r#   r&   r¨   r    r$   r:   r©   Úpast_seen_tokensÚcausal_maskr%   r­   Údecoder_layers                rU   ri   zGemmaModel.forward²   s“  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå(Ø”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆð &ˆØ"Ÿošo¨mÈ,˜oÑWÔWÐà!œ[Ð)H¨4¬;Ô+HÐ)HÔIð 		ð 		ˆMØ)˜MØðà*Ø)Ø /Ø#Ø$7ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆÝ&Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
rT   )NNNNNN)rD   rE   rF   r   r`   rc   Ú
LongTensorrm   r   ÚFloatTensorrP   r   r   r   ri   rn   ro   s   @rU   r¦   r¦   ª   sì   ø€ € € € € ð
˜{ð 
ð 
ð 
ð 
ð 
ð 
ð .2Ø.2Ø04Ø(,Ø26Ø!%ð2
ð 2
àÔ# dÑ*ð2
ð œ tÑ+ð2
ð Ô&¨Ñ-ð	2
ð
  ™ð2
ð Ô(¨4Ñ/ð2
ð ˜$‘;ð2
ð Ð+Ô,ð2
ð 
!ð2
ð 2
ð 2
ð 2
ð 2
ð 2
ð 2
ð 2
rT   r¦   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚGemmaForCausalLMc                  ó6   •—  t          ¦   «         j        di | ¤ŽS )a|  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, GemmaForCausalLM

        >>> model = GemmaForCausalLM.from_pretrained("google/gemma-7b")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-7b")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```rS   )r_   ri   )Úsuper_kwargsrf   s    €rU   ri   zGemmaForCausalLM.forwardè   s!   ø€ ð$ �u‰wŒwŒÐ.Ð. Ð.Ð.Ð.rT   )rD   rE   rF   ri   rn   ro   s   @rU   rº   rº   ç   s8   ø€ € € € € ð/ð /ð /ð /ð /ð /ð /ð /ð /rT   rº   c                   ó   — e Zd ZdS )ÚGemmaForSequenceClassificationNr’   rS   rT   rU   r¾   r¾   ý   r“   rT   r¾   c                   ó   — e Zd ZdS )ÚGemmaForTokenClassificationNr’   rS   rT   rU   rÀ   rÀ     r“   rT   rÀ   )r   r¦   rº   r¾   rÀ   r›   )6rc   Úhuggingface_hub.dataclassesr   r   Ú r   rŸ   Úcache_utilsr   r   Úconfiguration_utilsr	   Úmasking_utilsr
   Úmodeling_outputsr   Úmodeling_rope_utilsr   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úllama.modeling_llamar   r   r   r   r   r   r   r   ÚVOCAB_FILES_NAMESÚSPIECE_UNDERLINEÚ
get_loggerrD   Úloggerr   Ú	EmbeddingrW   ÚModulerq   r‡   r‘   r•   r›   r¦   rº   r¾   rÀ   Ú__all__rS   rT   rU   ú<module>rÓ      s¦  ðð" €€€Ø .Ð .Ð .Ð .Ð .Ð .Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø /Ð /Ð /Ð /Ð /Ð /Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø -Ð -Ð -Ð -Ð -Ð -Ø &Ð &Ð &Ð &Ð &Ð &Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @ð	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð "Ð#4Ð5Ð àÐ à	ˆÔ	˜HÑ	%Ô	%€ð €Ð,Ð-Ñ-Ô-Øð34ð 34ð 34ð 34ð 34Ð"ñ 34ô 34ñ „ñ .Ô-ð34ðlSð Sð Sð Sð S 2¤<ñ Sô Sð Sð=ð =ð =ð =ð =�2”9ñ =ô =ð =ð(Yð Yð Yð Yð Yˆxñ Yô Yð Yð	ð 	ð 	ð 	ð 	Ð/ñ 	ô 	ð 	ðSð Sð Sð Sð S�^ñ Sô Sð SðJð Jð Jð Jð JÐ/ñ Jô Jð Jð:
ð :
ð :
ð :
ð :
�ñ :
ô :
ð :
ðz/ð /ð /ð /ð /Ð'ñ /ô /ð /ð,	ð 	ð 	ð 	ð 	Ð%Cñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð"=ñ 	ô 	ð 	ðð ð €€€rT   