§
    ‚Štj!  ã                   ó  — d dl Z d dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
mZ ddlmZ ddlmZmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZmZm Z m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z)  G d„ de%¦  «        Z* G d„ de!¦  «        Z+d„ Z, G d„ de'¦  «        Z- G d„ de¦  «        Z. G d„ de¦  «        Z/e G d„ de ¦  «        ¦   «         Z0e G d „ d!e¦  «        ¦   «         Z1e G d"„ d#e¦  «        ¦   «         Z2g d$¢Z3dS )%é    N)ÚCallableé   )Úinitialization)ÚCacheÚDynamicCache)Úcreate_causal_mask)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringé   )ÚCLIPMLP)ÚGemma2ForCausalLM)ÚLlamaDecoderLayerÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)ÚLlama4TextL2Norm)ÚQwen3Attentioné   )ÚNanoChatConfigc                   ó   — e Zd ZdS )ÚNanoChatRMSNormN©Ú__name__Ú
__module__Ú__qualname__© ó    úk/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/nanochat/modular_nanochat.pyr   r   +   ó   € € € € € Ø€Dr$   r   c                   ó   — e Zd ZdS )ÚNanoChatRotaryEmbeddingNr   r#   r$   r%   r(   r(   /   r&   r$   r(   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        || fd¬¦  «        S )zJRotates half the hidden dims of the input with flipped signs for NanoChat..Néÿÿÿÿr   )Údim)ÚshapeÚtorchÚcat)ÚxÚx1Úx2s      r%   Úrotate_halfr2   3   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�b˜2˜#�Y BÐ'Ñ'Ô'Ð'r$   c                   óÐ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej        deej        ej        f         dz  dej        dz  de	dz  d	e
e         d
eej        ej        dz  f         fd„Zˆ xZS )ÚNanoChatAttentionÚconfigÚ	layer_idxc                 ó¼   •— t          ¦   «                              ||¦  «         | `| `t	          |j        ¬¦  «        | _        t	          |j        ¬¦  «        | _        d S ©N)Úeps)ÚsuperÚ__init__Úsliding_windowÚ
layer_typer   Úrms_norm_epsÚq_normÚk_norm©Úselfr5   r6   Ú	__class__s      €r%   r;   zNanoChatAttention.__init__;   sV   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+ØÐØˆOå%¨&Ô*=Ð>Ñ>Ô>ˆŒÝ%¨&Ô*=Ð>Ñ>Ô>ˆŒˆˆr$   NÚhidden_statesÚposition_embeddingsÚattention_maskÚpast_key_valuesÚkwargsÚreturnc                 óv  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|\  }}t          ||	||¦  «        \  }}	|                      |¦  «        }|  	                    |	¦  «        }	|�| 
                    |	|
| j        ¦  «        \  }	}
t          j        | j        j        t           ¦  «        } || ||	|
|f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr*   r   r   ç        )ÚdropoutÚscaling)r,   Úhead_dimÚq_projÚviewÚ	transposeÚk_projÚv_projr   r?   r@   Úupdater6   r   Úget_interfacer5   Ú_attn_implementationr   ÚtrainingÚattention_dropoutrM   ÚreshapeÚ
contiguousÚo_proj)rB   rD   rE   rF   rG   rH   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                   r%   ÚforwardzNanoChatAttention.forwardC   sä  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jð —{’{ <Ñ0Ô0ˆØ—[’[ Ñ,Ô,ˆ
àÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r$   )NNN)r    r!   r"   r   Úintr;   r-   ÚTensorÚtupler   r   r   rf   Ú__classcell__©rC   s   @r%   r4   r4   :   sä   ø€ € € € € ð?˜~ð ?¸#ð ?ð ?ð ?ð ?ð ?ð ?ð IMØ.2Ø(,ð*)ð *)à”|ð*)ð # 5¤<°´Ð#=Ô>ÀÑEð*)ð œ tÑ+ð	*)ð
  ™ð*)ð Ð+Ô,ð*)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð*)ð *)ð *)ð *)ð *)ð *)ð *)ð *)r$   r4   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚNanoChatMLPc                 óâ   •— t          ¦   «                              |¦  «         t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        d S )NF)Úbias)r:   r;   ÚnnÚLinearÚhidden_sizeÚintermediate_sizeÚfc1Úfc2©rB   r5   rC   s     €r%   r;   zNanoChatMLP.__init__q   s]   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”9˜VÔ/°Ô1IÐPUÐVÑVÔVˆŒÝ”9˜VÔ5°vÔ7IÐPUÐVÑVÔVˆŒˆˆr$   )r    r!   r"   r;   rj   rk   s   @r%   rm   rm   p   sA   ø€ € € € € ðWð Wð Wð Wð Wð Wð Wð Wð Wr$   rm   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚNanoChatDecoderLayerr5   r6   c                 ó°   •— t          ¦   «                              ¦   «          t          |j        ¬¦  «        | _        t          |j        ¬¦  «        | _        d S r8   )r:   r;   r   r>   Úinput_layernormÚpost_attention_layernormrA   s      €r%   r;   zNanoChatDecoderLayer.__init__x   sK   ø€ Ý‰Œ×ÒÑÔÐå.°6Ô3FÐGÑGÔGˆÔÝ(7¸FÔ<OÐ(PÑ(PÔ(PˆÔ%Ð%Ð%r$   )r    r!   r"   r   rg   r;   rj   rk   s   @r%   rx   rx   w   sW   ø€ € € € € ðQ˜~ð Q¸#ð Qð Qð Qð Qð Qð Qð Qð Qð Qð Qr$   rx   c                   ó(   — e Zd Zdej        ddfd„ZdS )ÚNanoChatPreTrainedModelÚmodulerI   Nc           	      óø   — t          j        | |¦  «         t          |t          ¦  «        rOt	          j        |j        j        d| j        j	        t          j        d| j        j        z  ¦  «        z  ¬¦  «         d S d S )NrK   r   )ÚmeanÚstd)r   Ú_init_weightsÚ
isinstancer4   ÚinitÚnormal_r[   Úweightr5   Úinitializer_rangeÚmathÚsqrtÚnum_hidden_layers)rB   r~   s     r%   r‚   z%NanoChatPreTrainedModel._init_weights�   s�   € ÝÔ% d¨FÑ3Ô3Ð3Ý�fÕ/Ñ0Ô0ð 	ÝŒLØ”Ô$ØØ”KÔ1µD´I¸aÀ$Ä+ÔB_Ñ>_Ñ4`Ô4`Ñ`ðñ ô ð ð ð ð	ð 	r$   )r    r!   r"   rp   ÚModuler‚   r#   r$   r%   r}   r}      s8   € € € € € ð B¤Ið °$ð ð ð ð ð ð r$   r}   c                   ó²   ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dedz  dej	        dz  d	e
dz  d
ee         defd„Zˆ xZS )ÚNanoChatModelr5   c                 ó~   •— t          ¦   «                              |¦  «         t          |j        ¬¦  «        | _        d S r8   )r:   r;   r   r>   Únormrv   s     €r%   r;   zNanoChatModel.__init__�   s4   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å#¨Ô(;Ð<Ñ<Ô<ˆŒ	ˆ	ˆ	r$   NÚ	input_idsrF   Úposition_idsrG   Úinputs_embedsÚ	use_cacherH   rI   c           	      óp  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          | j        ||||¬¦  «        }	|}
|                      |
|¬¦  «        }|                      |
¦  «        }
| j        d | j        j        …         D ]} ||
f|	|||dœ|¤Ž}
Œ|                      |
¦  «        }
t          |
|¬	¦  «        S )
Nz:You must specify exactly one of input_ids or inputs_embeds)r5   r   r   )Údevice)r5   r’   rF   rG   r‘   )r‘   )rF   rE   r‘   rG   )Úlast_hidden_staterG   )Ú
ValueErrorÚembed_tokensr   r5   Úget_seq_lengthr-   Úaranger,   r•   Ú	unsqueezer   Ú
rotary_embr�   ÚlayersrŠ   r	   )rB   r�   rF   r‘   rG   r’   r“   rH   Úpast_seen_tokensÚcausal_maskrD   rE   Údecoder_layers                r%   rf   zNanoChatModel.forward’   s˜  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø*.×*;Ò*;¸IÑ*FÔ*FˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå(Ø”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆð &ˆØ"Ÿošo¨mÈ,˜oÑWÔWÐàŸ	š	 -Ñ0Ô0ˆØ!œ[Ð)H¨4¬;Ô+HÐ)HÔIð 	ð 	ˆMØ)˜MØðà*Ø$7Ø)Ø /ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆÝ&Ø+Ø+ð
ñ 
ô 
ð 	
r$   )NNNNNN)r    r!   r"   r   r;   r-   Ú
LongTensorrh   r   ÚFloatTensorÚboolr   r   r	   rf   rj   rk   s   @r%   r�   r�   ‹   sì   ø€ € € € € ð=˜~ð =ð =ð =ð =ð =ð =ð .2Ø.2Ø04Ø(,Ø26Ø!%ð2
ð 2
àÔ# dÑ*ð2
ð œ tÑ+ð2
ð Ô&¨Ñ-ð	2
ð
  ™ð2
ð Ô(¨4Ñ/ð2
ð ˜$‘;ð2
ð Ð+Ô,ð2
ð 
!ð2
ð 2
ð 2
ð 2
ð 2
ð 2
ð 2
ð 2
r$   r�   c                   ó,   ‡ — e Zd ZddiZdefˆ fd„Zˆ xZS )ÚNanoChatForCausalLMÚlm_headÚcolwise_gather_outputrI   c                 ó:   •—  t          ¦   «         j        di |¤Ž dS )ak  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, AutoModelForCausalLM

        >>> model = AutoModelForCausalLM.from_pretrained("karpathy/nanochat-d32")

        >>> tokenizer = AutoTokenizer.from_pretrained("karpathy/nanochat-d32")

        >>> conversation = [
                {"role": "user", "content": "What is the capital of France?"},
            ]

        >>> inputs = tokenizer.apply_chat_template(
                conversation, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt"
            ).to(device)

        >>> with torch.no_grad():
        >>>     outputs = model.generate(**inputs, max_new_tokens=64, do_sample=False)

        >>> generated_tokens = outputs[0, inputs["input_ids"].shape[1] :]
        >>> output = tokenizer.decode(generated_tokens, skip_special_tokens=True)
        ```Nr#   )r:   rf   )rB   Úsuper_kwargsrC   s     €r%   rf   zNanoChatForCausalLM.forwardË   s'   ø€ ð2 	�‰ŒŒÐ'Ð'˜,Ð'Ð'Ð'Ð'Ð'r$   )r    r!   r"   Ú_tp_planr
   rf   rj   rk   s   @r%   r¥   r¥   Ç   sP   ø€ € € € € àÐ2Ð3€Hð(Ð)?ð (ð (ð (ð (ð (ð (ð (ð (ð (ð (r$   r¥   )r}   r�   r¥   )4rˆ   Úcollections.abcr   r-   Útorch.nnrp   Ú r   r„   Úcache_utilsr   r   Úmasking_utilsr   Úmodeling_outputsr	   r
   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úclip.modeling_clipr   Úgemma2.modeling_gemma2r   Úllama.modeling_llamar   r   r   r   r   r   Úllama4.modeling_llama4r   Úqwen3.modeling_qwen3r   Úconfiguration_nanochatr   r   r(   r2   r4   rm   rx   r}   r�   r¥   Ú__all__r#   r$   r%   ú<module>r»      s.  ðð €€€Ø $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø /Ð /Ð /Ð /Ð /Ð /Ø OÐ OÐ OÐ OÐ OÐ OÐ OÐ OØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø (Ð (Ð (Ð (Ð (Ð (Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 6Ð 5Ð 5Ð 5Ð 5Ð 5Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø 2Ð 2Ð 2Ð 2Ð 2Ð 2ð	ð 	ð 	ð 	ð 	Ð&ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð2ñ 	ô 	ð 	ð(ð (ð (ð3)ð 3)ð 3)ð 3)ð 3)˜ñ 3)ô 3)ð 3)ðlWð Wð Wð Wð W�'ñ Wô Wð WðQð Qð Qð Qð QÐ,ñ Qô Qð Qð ðð ð ð ð Ð2ñ ô ñ „ðð ð8
ð 8
ð 8
ð 8
ð 8
�Jñ 8
ô 8
ñ „ð8
ðv ð(ð (ð (ð (ð (Ð+ñ (ô (ñ „ð(ð>ð ð €€€r$   