§
    ‚ŠtjY  ã                   óÖ  — d dl Z d dlmZ ddlmZmZ ddlmZmZ ddl	m
Z
 ddlmZ ddlmZmZmZ d	d
lmZ d	dlmZmZmZmZ d	dlmZmZ  ej        e¦  «        Z ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z G d„ de¦  «        Z G d„ de¦  «        Z  G d„ de¦  «        Z! G d„ de¦  «        Z" G d„ de
¦  «        Z# G d„ de¦  «        Z$ G d„ de¦  «        Z%g d¢Z&dS ) é    N)Ústricté   )ÚCacheÚDynamicCache)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚBaseModelOutputWithPast)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚloggingé   )ÚLlamaConfig)ÚLlamaDecoderLayerÚLlamaForCausalLMÚ
LlamaModelÚLlamaPreTrainedModel)ÚQwen2AttentionÚQwen2RotaryEmbeddingzfacebook/cwm)Ú
checkpointc                   ó¬  ‡ — e Zd ZU dZdZdZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZedz  ed<   dZeee         z  dz  ed<   dZeed<   d Zeed!<   d"Zeez  ed#<   d$Zeed%<   d Zeed&<   dZedz  ed'<   d(Z eed)<   dZ!ee         dz  ed*<    e"¦   «         Z#ˆ fd+„Z$ˆ xZ%S ),Ú	CwmConfigÚcwmç    €„.Ai õ Ú
vocab_sizei   Úhidden_sizei T  Úintermediate_sizeé@   Únum_hidden_layersé0   Únum_attention_headsé   Únum_key_value_headsé€   Úhead_dimÚsiluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangegñhãˆµøä>Úrms_norm_epsTÚ	use_cacheNÚpad_token_idÚeos_token_idi ô Úbos_token_idFÚtie_word_embeddingsg        Úattention_dropouté   Úpretraining_tpÚmlp_biasÚrope_parametersé    Úsliding_windowÚlayer_typesc                 óf  •‡— | j         €dddddddœ| _         | j        €'dŠˆfd	„t          | j        ¦  «        D ¦   «         | _        | j        rt          | j        ¦  «        nd | _        t          | j        ¦  «        | _        | j        �| j        ng d
¢| _         t          ¦   «         j	        di |¤Ž d S )Nr   g      0@g      @g      ð?r5   Úllama3)Ú
rope_thetaÚfactorÚhigh_freq_factorÚlow_freq_factorÚ original_max_position_embeddingsÚ	rope_typeé   c                 ó*   •— g | ]}|‰z  d k    rdnd‘ŒS )r   Úfull_attentionÚsliding_attention© )Ú.0ÚiÚwindow_patterns     €úa/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/cwm/modular_cwm.pyú
<listcomp>z+CwmConfig.__post_init__.<locals>.<listcomp>R   s?   ø€ ð  ð  ð  àð '(¨.Ñ&8¸AÒ&=Ð&=Ð!Ð!ÐDWð ð  ð  ó    )iô iô i	ô rD   )
r4   r7   Úranger   r6   ÚintÚlistr-   ÚsuperÚ__post_init__)ÚselfÚkwargsrG   Ú	__class__s     @€rH   rO   zCwmConfig.__post_init__D   sì   øø€ ØÔÐ'à)ØØ$'Ø#&Ø48Ø%ð$ð $ˆDÔ ð ÔÐ#àˆNð ð  ð  ð  å˜tÔ5Ñ6Ô6ð ñ  ô  ˆDÔð
 ;?Ô:MÐW�c $Ô"5Ñ6Ô6Ð6ÐSWˆÔÝ Ô 0Ñ1Ô1ˆÔØ15Ô1BÐ1N˜DÔ-Ð-ÐTlÐTlÐTlˆÔØ�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'rJ   )&Ú__name__Ú
__module__Ú__qualname__Ú
model_typeÚdefault_thetar   rL   Ú__annotations__r   r   r   r!   r#   r%   r'   Ústrr(   r)   Úfloatr*   r+   Úboolr,   r-   rM   r.   r/   r0   r2   r3   r4   Údictr6   r7   ÚAttributeErrorÚattention_biasrO   Ú__classcell__©rR   s   @rH   r   r   %   sÝ  ø€ € € € € € ð €JØ€Mà€J�ÐÐÑØ€K�ÐÐÑØ"Ð�sÐ"Ð"Ñ"ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø Ð˜Ð Ð Ñ Ø€HˆcÐÐÑØ€J�ÐÐÑØ#)Ð˜SÐ)Ð)Ñ)Ø#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØ#€L�#˜‘*Ð#Ð#Ñ#Ø+/€L�#˜˜Sœ	‘/ DÑ(Ð/Ð/Ñ/Ø€L�#ÐÐÑØ %Ð˜Ð%Ð%Ñ%Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø€N�CÐÐÑØ€HˆdÐÐÑØ#'€O�T˜D‘[Ð'Ð'Ñ'Ø€N�CÐÐÑØ$(€K��c”˜TÑ!Ð(Ð(Ñ(à#�^Ñ%Ô%€Nð(ð (ð (ð (ð (ð (ð (ð (ð (rJ   r   c                   ó   — e Zd ZdS )ÚCwmRotaryEmbeddingN©rS   rT   rU   rD   rJ   rH   rb   rb   ]   ó   € € € € € Ø€DrJ   rb   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚCwmAttentionÚconfigÚ	layer_idxc                 ó¤  •— t          ¦   «                              ||¬¦  «         t          j                             |j        |j        | j        z  d¬¦  «        | _        t          j                             |j        |j	        | j        z  d¬¦  «        | _
        t          j                             |j        |j	        | j        z  d¬¦  «        | _        d S )N©rg   rh   F)Úbias)rN   Ú__init__ÚtorchÚnnÚLinearr   r!   r%   Úq_projr#   Úk_projÚv_proj©rP   rg   rh   rR   s      €rH   rl   zCwmAttention.__init__b   s¥   ø€ Ý‰Œ×Ò °)ÐÑ<Ô<Ð<Ý”h—o’o fÔ&8¸&Ô:TÐW[ÔWdÑ:dÐkp�oÑqÔqˆŒÝ”h—o’o fÔ&8¸&Ô:TÐW[ÔWdÑ:dÐkp�oÑqÔqˆŒÝ”h—o’o fÔ&8¸&Ô:TÐW[ÔWdÑ:dÐkp�oÑqÔqˆŒˆˆrJ   ©rS   rT   rU   r   rL   rl   r_   r`   s   @rH   rf   rf   a   sW   ø€ € € € € ðr˜yð r°Sð rð rð rð rð rð rð rð rð rð rrJ   rf   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚCwmDecoderLayerrg   rh   c                 óz   •— t          ¦   «                              ||¬¦  «         t          ||¬¦  «        | _        d S )Nrj   )rN   rl   rf   Ú	self_attnrs   s      €rH   rl   zCwmDecoderLayer.__init__j   s8   ø€ Ý‰Œ×Ò °)ÐÑ<Ô<Ð<Ý%¨V¸yÐIÑIÔIˆŒˆˆrJ   rt   r`   s   @rH   rv   rv   i   sW   ø€ € € € € ðJ˜yð J°Sð Jð Jð Jð Jð Jð Jð Jð Jð Jð JrJ   rv   c                   ó   — e Zd ZdS )ÚCwmPreTrainedModelNrc   rD   rJ   rH   rz   rz   o   rd   rJ   rz   c                   ó   — e Zd ZdS )ÚCwmModelOutputWithPastNrc   rD   rJ   rH   r|   r|   s   rd   rJ   r|   c                   ó¶   ‡ — e Zd ZeZdefˆ fd„Z	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  de	dz  dej
        dz  d	edz  d
ee         defd„Zˆ xZS )ÚCwmModelrg   c                 óÐ   •‡— t          ¦   «                              ‰¦  «         t          j                             ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS rD   )rv   )rE   rh   rg   s     €rH   rI   z%CwmModel.__init__.<locals>.<listcomp>}   s#   ø€ ÐaÐaÐa°I�_˜V YÑ/Ô/ÐaÐaÐarJ   )rN   rl   rm   rn   Ú
ModuleListrK   r   Úlayers)rP   rg   rR   s    `€rH   rl   zCwmModel.__init__z   s[   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”h×)Ò)ØaÐaÐaÐaÅÀvÔG_ÑA`ÔA`ÐaÑaÔañ
ô 
ˆŒˆˆrJ   NÚ	input_idsÚattention_maskÚposition_idsÚpast_key_valuesÚinputs_embedsr+   rQ   Úreturnc           	      ó   — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          |x}	t          ¦  «        s:| j        ||||dœ}
|
                     ¦   «         }t          d
i |
¤Žt          d
i |¤Ždœ}	|}|                      ||¦  «        }t!          | j        d | j        j        …         ¦  «        D ])\  }} ||f|	| j        j        |                  |||dœ|¤Ž}Œ*|                      |¦  «        }t+          ||¬	¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embeds)rg   r   r1   )Údevice)rg   r‡   r„   r†   r…   )rB   rC   )r„   r…   r†   Úposition_embeddings)Úlast_hidden_stater†   rD   )Ú
ValueErrorÚembed_tokensr   rg   Úget_seq_lengthrm   ÚarangeÚshaperŠ   Ú	unsqueezeÚ
isinstancer\   Úcopyr   r   Ú
rotary_embÚ	enumerater‚   r   r7   Únormr|   )rP   rƒ   r„   r…   r†   r‡   r+   rQ   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsÚsliding_mask_kwargsÚhidden_statesr‹   rF   Údecoder_layers                   rH   ÚforwardzCwmModel.forward€   s÷  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø*.×*;Ò*;¸IÑ*FÔ*FˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå°Ð?Ð-ÅÑFÔFð 	àœ+Ø!.Ø"0Ø#2Ø ,ðð ˆKð #.×"2Ò"2Ñ"4Ô"4Ðõ #5Ð"CÐ"C°{Ð"CÐ"CÝ%FÐ%]Ð%]ÐI\Ð%]Ð%]ð#ð #Ðð
 &ˆØ"Ÿošo¨m¸\ÑJÔJÐå )¨$¬+Ð6U¸¼Ô8UÐ6UÔ*VÑ WÔ Wð 	ð 	ÑˆAˆ}Ø)˜MØðà2°4´;Ô3JÈ1Ô3MÔNØ)Ø /Ø$7ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆÝ%Ø+Ø+ð
ñ 
ô 
ð 	
rJ   )NNNNNN)rS   rT   rU   r   Úconfig_classrl   rm   Ú
LongTensorÚTensorr   ÚFloatTensorr[   r
   r   r|   rž   r_   r`   s   @rH   r~   r~   w   sñ   ø€ € € € € Ø€Lð
˜yð 
ð 
ð 
ð 
ð 
ð 
ð .2Ø.2Ø04Ø(,Ø26Ø!%ð8
ð 8
àÔ# dÑ*ð8
ð œ tÑ+ð8
ð Ô&¨Ñ-ð	8
ð
  ™ð8
ð Ô(¨4Ñ/ð8
ð ˜$‘;ð8
ð Ð+Ô,ð8
ð 
 ð8
ð 8
ð 8
ð 8
ð 8
ð 8
ð 8
ð 8
rJ   r~   c                   ó   — e Zd ZdS )ÚCwmForCausalLMNrc   rD   rJ   rH   r¤   r¤   »   rd   rJ   r¤   )r   rz   r~   r¤   )'rm   Úhuggingface_hub.dataclassesr   Úcache_utilsr   r   Úmasking_utilsr   r   Úmodeling_outputsr	   Úprocessing_utilsr
   Úutilsr   r   r   Úllama.configuration_llamar   Úllama.modeling_llamar   r   r   r   Úqwen2.modeling_qwen2r   r   Ú
get_loggerrS   Úloggerr   rb   rf   rv   rz   r|   r~   r¤   Ú__all__rD   rJ   rH   ú<module>r±      s¹  ðð  €€€Ø .Ð .Ð .Ð .Ð .Ð .à .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø &Ð &Ð &Ð &Ð &Ð &Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3ðð ð ð ð ð ð ð ð ð ð ð ð HÐ GÐ GÐ GÐ GÐ GÐ GÐ Gð 
ˆÔ	˜HÑ	%Ô	%€ð €˜>Ð*Ñ*Ô*Øð3(ð 3(ð 3(ð 3(ð 3(�ñ 3(ô 3(ñ „ñ +Ô*ð3(ðl	ð 	ð 	ð 	ð 	Ð-ñ 	ô 	ð 	ðrð rð rð rð r�>ñ rô rð rðJð Jð Jð Jð JÐ'ñ Jô Jð Jð	ð 	ð 	ð 	ð 	Ð-ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð4ñ 	ô 	ð 	ðA
ð A
ð A
ð A
ð A
ˆzñ A
ô A
ð A
ðH	ð 	ð 	ð 	ð 	Ð%ñ 	ô 	ð 	ðð ð €€€rJ   