§
    ‚ŠtjO  ã                   óÂ  — d Z ddlmZ ddlZddlmZ ddlmZ ddlm	Z	 ddl
mZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZmZmZmZmZmZmZmZ ddlmZ  ej         e!¦  «        Z"dZ# G d„ de¦  «        Z$ G d„ de¦  «        Z% G d„ de¦  «        Z& G d„ de¦  «        Z' G d„ de¦  «        Z( G d„ de¦  «        Z) G d„ de¦  «        Z* G d „ d!e¦  «        Z+g d"¢Z,dS )#zPyTorch Qwen3 model.é    )ÚCallableNé   )ÚCache)ÚFlashAttentionKwargs)ÚCausalLMOutputWithPast)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚloggingé   )ÚGemmaMLP)ÚLlamaAttention)ÚQwen2ForCausalLMÚQwen2ForQuestionAnsweringÚQwen2ForSequenceClassificationÚQwen2ForTokenClassificationÚQwen2RMSNormÚQwen2RotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forwardé   )ÚQwen3ConfigzQwen/Qwen3-8Bc                   ó   — e Zd ZdS )ÚQwen3RMSNormN©Ú__name__Ú
__module__Ú__qualname__© ó    úe/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/qwen3/modular_qwen3.pyr   r   0   ó   € € € € € Ø€Dr    r   c                   ó   — e Zd ZdS )ÚQwen3MLPNr   r   r    r!   r$   r$   4   r"   r    r$   c                   ó   — e Zd ZdS )ÚQwen3RotaryEmbeddingNr   r   r    r!   r&   r&   8   r"   r    r&   c                   óÆ   ‡ — e Zd Zdedefˆ fd„Z	 ddej        deej        ej        f         dej        dz  de	dz  d	e
e         d
eej        ej        dz  f         fd„Zˆ xZS )ÚQwen3AttentionÚconfigÚ	layer_idxc                 óF  •— t          |d¦  «        r|j        |         nd | _        t          ¦   «                              ||¦  «         t          | j        |j        ¬¦  «        | _        t          | j        |j        ¬¦  «        | _	        | j        dk    r|j
        nd | _
        d S )NÚlayer_types)ÚepsÚsliding_attention)Úhasattrr,   Ú
layer_typeÚsuperÚ__init__r   Úhead_dimÚrms_norm_epsÚq_normÚk_normÚsliding_window)Úselfr)   r*   Ú	__class__s      €r!   r2   zQwen3Attention.__init__=   s—   ø€ Ý;BÀ6È=Ñ;YÔ;YÐc˜&Ô,¨YÔ7Ð7Ð_cˆŒÝ‰Œ×Ò˜ Ñ+Ô+Ð+Ý" 4¤=°fÔ6IÐJÑJÔJˆŒÝ" 4¤=°fÔ6IÐJÑJÔJˆŒØ7;´ÐJ]Ò7]Ð7]˜fÔ3Ð3ÐcgˆÔÐÐr    NÚhidden_statesÚposition_embeddingsÚattention_maskÚpast_key_valuesÚkwargsÚreturnc                 óz  — |j         d d…         }g |¢d‘| j        ‘R }|                      |                      |¦  «                             |¦  «        ¦  «                             dd¦  «        }|                      |                      |¦  «                             |¦  «        ¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|\  }}t          ||	||¦  «        \  }}	|�| 
                    |	|
| j        ¦  «        \  }	}
t          j        | j        j        t           ¦  «        } || ||	|
|f| j        sdn| j        | j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Néÿÿÿÿr   r   g        )ÚdropoutÚscalingr7   )Úshaper3   r5   Úq_projÚviewÚ	transposer6   Úk_projÚv_projr   Úupdater*   r   Úget_interfacer)   Ú_attn_implementationr   ÚtrainingÚattention_dropoutrC   r7   ÚreshapeÚ
contiguousÚo_proj)r8   r:   r;   r<   r=   r>   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                   r!   ÚforwardzQwen3Attention.forwardD   sà  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ 4§;¢;¨}Ñ#=Ô#=×#BÒ#BÀ<Ñ#PÔ#PÑQÔQ×[Ò[Ð\]Ð_`ÑaÔaˆØ—[’[ §¢¨]Ñ!;Ô!;×!@Ò!@ÀÑ!NÔ!NÑOÔO×YÒYÐZ[Ð]^Ñ_Ô_ˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}ÐH�C�C°$Ô2HØ”LØÔ.ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r    )N)r   r   r   r   Úintr2   ÚtorchÚTensorÚtupler   r	   r   r\   Ú__classcell__©r9   s   @r!   r(   r(   <   sß   ø€ € € € € ðh˜{ð h°sð hð hð hð hð hð hð )-ð')ð ')à”|ð')ð # 5¤<°´Ð#=Ô>ð')ð œ tÑ+ð	')ð
  ™ð')ð Ð-Ô.ð')ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð')ð ')ð ')ð ')ð ')ð ')ð ')ð ')r    r(   c                   ó4   ‡ — e Zd Zdee         defˆ fd„Zˆ xZS )ÚQwen3ForCausalLMÚsuper_kwargsr?   c                 ó6   •—  t          ¦   «         j        di |¤ŽS )a^  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, Qwen3ForCausalLM

        >>> model = Qwen3ForCausalLM.from_pretrained("Qwen/Qwen3-8B")
        >>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-8B")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```r   )r1   r\   )r8   re   r9   s     €r!   r\   zQwen3ForCausalLM.forwardo   s!   ø€ ð4 �u‰wŒwŒÐ.Ð. Ð.Ð.Ð.r    )r   r   r   r	   r
   r   r\   ra   rb   s   @r!   rd   rd   n   sU   ø€ € € € € ð/àÐ1Ô2ð/ð 
 ð/ð /ð /ð /ð /ð /ð /ð /ð /ð /r    rd   c                   ó   — e Zd ZdS )ÚQwen3ForSequenceClassificationNr   r   r    r!   rh   rh   Œ   r"   r    rh   c                   ó   — e Zd ZdS )ÚQwen3ForTokenClassificationNr   r   r    r!   rj   rj   �   r"   r    rj   c                   ó   — e Zd ZdS )ÚQwen3ForQuestionAnsweringNr   r   r    r!   rl   rl   ”   r"   r    rl   )rd   rl   ÚQwen3PreTrainedModelÚ
Qwen3Modelrh   rj   )-Ú__doc__Úcollections.abcr   r^   Úcache_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_outputsr   Úmodeling_utilsr   Úprocessing_utilsr	   Úutilsr
   r   Úgemma.modeling_gemmar   Úllama.modeling_llamar   Úqwen2.modeling_qwen2r   r   r   r   r   r   r   r   Úconfiguration_qwen3r   Ú
get_loggerr   ÚloggerÚ_CHECKPOINT_FOR_DOCr   r$   r&   r(   rd   rh   rj   rl   Ú__all__r   r    r!   ú<module>r      sÇ  ðð Ð à $Ð $Ð $Ð $Ð $Ð $à €€€à  Ð  Ð  Ð  Ð  Ð  Ø BÐ BÐ BÐ BÐ BÐ BØ 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø &Ð &Ð &Ð &Ð &Ð &Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø +Ð +Ð +Ð +Ð +Ð +ðð ð ð ð ð ð	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð -Ð ,Ð ,Ð ,Ð ,Ð ,ð 
ˆÔ	˜HÑ	%Ô	%€à%Ð ð	ð 	ð 	ð 	ð 	�<ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	ˆxñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð/ñ 	ô 	ð 	ð/)ð /)ð /)ð /)ð /)�^ñ /)ô /)ð /)ðd/ð /ð /ð /ð /Ð'ñ /ô /ð /ð<	ð 	ð 	ð 	ð 	Ð%Cñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð"=ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð 9ñ 	ô 	ð 	ðð ð €€€r    