§
    ‚Štj�  ã                   óB  — d dl mZ d dlmZ ddlmZmZ ddlmZ ddl	m
Z
mZmZmZ ddlmZ  ed	¬
¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z G d„ de¦  «        Z G d„ de
¦  «        Z G d„ de¦  «        Z G d„ de¦  «        Z G d„ de¦  «        Zg d¢ZdS )é    N)Ústricté   )Úauto_docstringÚcan_return_tupleé   )ÚLlamaConfig)ÚLlamaDecoderLayerÚLlamaForCausalLMÚ
LlamaModelÚLlamaPreTrainedModel)ÚNemotronMLPzinceptionai/Jais-2-8B-Chat)Ú
checkpointc                   ó  — e Zd ZU dddddddœZdZeed<   dZeed<   dZeed	<   d
Z	eed<   dZ
eed<   dZeed<   dZeed<   dZedz  ed<   dZeee         z  dz  ed<   dZeed<   dZeed<    e¦   «         Z e¦   «         ZdS )ÚJais2ConfigÚcolwiseÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proji K Ú
vocab_sizei   Úhidden_sizei h  Úintermediate_sizeé   Únum_attention_headsÚrelu2Ú
hidden_acti    Úmax_position_embeddingsgñhãˆµøä>Úlayer_norm_epsr   NÚbos_token_idiJ Úeos_token_idTÚattention_biasÚmlp_bias)Ú__name__Ú
__module__Ú__qualname__Úbase_model_tp_planr   ÚintÚ__annotations__r   r   r   r   Ústrr   r   Úfloatr   r   Úlistr   Úboolr   ÚAttributeErrorÚrms_norm_epsÚpretraining_tp© ó    úe/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/jais2/modular_jais2.pyr   r      s  € € € € € € ð &/Ø%.Ø%.Ø%.Ø )Ø"+ðð Ðð €J�ÐÐÑØ€K�ÐÐÑØ"Ð�sÐ"Ð"Ñ"Ø!Ð˜Ð!Ð!Ñ!Ø€J�ÐÐÑØ#'Ð˜SÐ'Ð'Ñ'Ø €N�EÐ Ð Ñ Ø €L�#˜‘*Ð Ð Ñ Ø+1€L�#˜˜Sœ	‘/ DÑ(Ð1Ð1Ñ1Ø€N�DÐÐÑØ€HˆdÐÐÑØ!�>Ñ#Ô#€LØ#�^Ñ%Ô%€N€N€Nr.   r   c                   ó   — e Zd ZdS )ÚJais2MLPN©r    r!   r"   r-   r.   r/   r1   r1   9   ó   € € € € € Ø€Dr.   r1   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚJais2DecoderLayerÚconfigÚ	layer_idxc                 óà   •— t          ¦   «                              ||¦  «         t          j        |j        |j        ¬¦  «        | _        t          j        |j        |j        ¬¦  «        | _        d S ©N)Úeps)ÚsuperÚ__init__ÚnnÚ	LayerNormr   r   Úinput_layernormÚpost_attention_layernorm)Úselfr6   r7   Ú	__class__s      €r/   r<   zJais2DecoderLayer.__init__>   s]   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+Ý!œ|¨FÔ,>ÀFÔDYÐZÑZÔZˆÔÝ(*¬°VÔ5GÈVÔMbÐ(cÑ(cÔ(cˆÔ%Ð%Ð%r.   )r    r!   r"   r   r$   r<   Ú__classcell__©rB   s   @r/   r5   r5   =   sW   ø€ € € € € ðd˜{ð d°sð dð dð dð dð dð dð dð dð dð dr.   r5   c                   ó   — e Zd ZdS )ÚJais2PreTrainedModelNr2   r-   r.   r/   rF   rF   D   r3   r.   rF   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )Ú
Jais2Modelr6   c                 ó”   •— t          ¦   «                              |¦  «         t          j        |j        |j        ¬¦  «        | _        d S r9   )r;   r<   r=   r>   r   r   Únorm)rA   r6   rB   s     €r/   r<   zJais2Model.__init__I   s;   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”L Ô!3¸Ô9NÐOÑOÔOˆŒ	ˆ	ˆ	r.   )r    r!   r"   r   r<   rC   rD   s   @r/   rH   rH   H   sO   ø€ € € € € ðP˜{ð Pð Pð Pð Pð Pð Pð Pð Pð Pð Pr.   rH   c                   ó>   ‡ — e Zd Zeeˆ fd„¦   «         ¦   «         Zˆ xZS )ÚJais2ForCausalLMc                 ó6   •—  t          ¦   «         j        di |¤ŽS )aÓ  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, Jais2ForCausalLM

        >>> model = Jais2ForCausalLM.from_pretrained("inceptionai/Jais-2-8B-Chat")
        >>> tokenizer = AutoTokenizer.from_pretrained("inceptionai/Jais-2-8B-Chat")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```r-   )r;   Úforward)rA   Úsuper_kwargsrB   s     €r/   rN   zJais2ForCausalLM.forwardO   s!   ø€ ð( �u‰wŒwŒÐ.Ð. Ð.Ð.Ð.r.   )r    r!   r"   r   r   rN   rC   rD   s   @r/   rL   rL   N   sM   ø€ € € € € ØØð/ð /ð /ð /ñ „^ñ Ôð/ð /ð /ð /ð /r.   rL   )r   rH   rL   rF   )Útorch.nnr=   Úhuggingface_hub.dataclassesr   Úutilsr   r   Úllama.configuration_llamar   Úllama.modeling_llamar	   r
   r   r   Únemotron.modeling_nemotronr   r   r1   r5   rF   rH   rL   Ú__all__r-   r.   r/   ú<module>rW      sî  ðð  Ð Ð Ð Ð Ð Ø .Ð .Ð .Ð .Ð .Ð .à 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3ðð ð ð ð ð ð ð ð ð ð ð ð 5Ð 4Ð 4Ð 4Ð 4Ð 4ð €Ð7Ð8Ñ8Ô8Øð&ð &ð &ð &ð &�+ñ &ô &ñ „ñ 9Ô8ð&ð2	ð 	ð 	ð 	ð 	ˆ{ñ 	ô 	ð 	ðdð dð dð dð dÐ)ñ dô dð dð	ð 	ð 	ð 	ð 	Ð/ñ 	ô 	ð 	ðPð Pð Pð Pð P�ñ Pô Pð Pð/ð /ð /ð /ð /Ð'ñ /ô /ð /ð0ð ð €€€r.   