§
    ‚Štj”  ã                   ó6  — d dl mZ d dlZd dlmZ ddlmZmZ ddlmZm	Z	 ddl
mZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& ddl'm(Z( ddl)m*Z*  ej+        e,¦  «        Z- G d„ de#¦  «        Z. G d„ de¦  «        Z/ G d„ de¦  «        Z0 G d„ de¦  «        Z1 G d„ de$¦  «        Z2 G d„ de(¦  «        Z3 G d „ d!e¦  «        Z4 G d"„ d#e!¦  «        Z5 G d$„ d%e"¦  «        Z6 G d&„ d'e ¦  «        Z7g d(¢Z8dS ))é    )ÚCallableN)Únné   )ÚCacheÚDynamicCache)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚBaseModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚGemma2RotaryEmbedding)
ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaForCausalLMÚLlamaForQuestionAnsweringÚLlamaForSequenceClassificationÚLlamaForTokenClassificationÚLlamaMLPÚLlamaPreTrainedModelÚapply_rotary_pos_embÚeager_attention_forward)ÚMistralModelé   )ÚQwen2Configc                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚQwen2MLPc                 ó.  •— t          ¦   «                              |¦  «         t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        d S )NF©Úbias)	ÚsuperÚ__init__r   ÚLinearÚhidden_sizeÚintermediate_sizeÚ	gate_projÚup_projÚ	down_proj©ÚselfÚconfigÚ	__class__s     €úe/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/qwen2/modular_qwen2.pyr(   zQwen2MLP.__init__&   s|   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ýœ 4Ô#3°TÔ5KÐRWÐXÑXÔXˆŒÝ”y Ô!1°4Ô3IÐPUÐVÑVÔVˆŒÝœ 4Ô#9¸4Ô;KÐRWÐXÑXÔXˆŒˆˆó    )Ú__name__Ú
__module__Ú__qualname__r(   Ú__classcell__©r2   s   @r3   r#   r#   %   sA   ø€ € € € € ðYð Yð Yð Yð Yð Yð Yð Yð Yr4   r#   c                   ó   — e Zd ZdS )ÚQwen2RotaryEmbeddingN©r5   r6   r7   © r4   r3   r;   r;   -   ó   € € € € € Ø€Dr4   r;   c                   óÆ   ‡ — e Zd Zdedefˆ fd„Z	 ddej        deej        ej        f         dej        dz  de	dz  d	e
e         d
eej        ej        dz  f         fd„Zˆ xZS )ÚQwen2Attentionr1   Ú	layer_idxc                 ó6  •— t          |d¦  «        r|j        |         nd | _        t          ¦   «                              ||¦  «         t          j        |j        |j        | j	        z  d¬¦  «        | _
        t          j        |j        |j        | j	        z  d¬¦  «        | _        t          j        |j        |j        | j	        z  d¬¦  «        | _        t          j        |j        | j	        z  |j        d¬¦  «        | _        | j        dk    r|j        nd | _        d S )NÚlayer_typesTr%   FÚsliding_attention)ÚhasattrrC   Ú
layer_typer'   r(   r   r)   r*   Únum_attention_headsÚhead_dimÚq_projÚnum_key_value_headsÚk_projÚv_projÚo_projÚsliding_window)r0   r1   rA   r2   s      €r3   r(   zQwen2Attention.__init__2   s  ø€ Ý;BÀ6È=Ñ;YÔ;YÐc˜&Ô,¨YÔ7Ð7Ð_cˆŒÝ‰Œ×Ò˜ Ñ+Ô+Ð+Ý”i Ô 2°FÔ4NÐQUÔQ^Ñ4^ÐeiÐjÑjÔjˆŒÝ”i Ô 2°FÔ4NÐQUÔQ^Ñ4^ÐeiÐjÑjÔjˆŒÝ”i Ô 2°FÔ4NÐQUÔQ^Ñ4^ÐeiÐjÑjÔjˆŒÝ”i Ô :¸T¼]Ñ JÈFÔL^ÐejÐkÑkÔkˆŒØ7;´ÐJ]Ò7]Ð7]˜fÔ3Ð3ÐcgˆÔÐÐr4   NÚhidden_statesÚposition_embeddingsÚattention_maskÚpast_key_valuesÚkwargsÚreturnc                 ó.  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|\  }}t          ||	||¦  «        \  }}	|�|                     |	|
| j	        ¦  «        \  }	}
t          j        | j        j        t          ¦  «        } || ||	|
|f| j        sdn| j        | j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Néÿÿÿÿr    r   g        )ÚdropoutÚscalingrN   )ÚshaperH   rI   ÚviewÚ	transposerK   rL   r   ÚupdaterA   r   Úget_interfacer1   Ú_attn_implementationr   ÚtrainingÚattention_dropoutrX   rN   ÚreshapeÚ
contiguousrM   )r0   rO   rP   rQ   rR   rS   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                   r3   ÚforwardzQwen2Attention.forward;   sÆ  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}ÐH�C�C°$Ô2HØ”LØÔ.ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r4   )N)r5   r6   r7   r!   Úintr(   ÚtorchÚTensorÚtupler   r   r
   rm   r8   r9   s   @r3   r@   r@   1   sß   ø€ € € € € ðh˜{ð h°sð hð hð hð hð hð hð )-ð')ð ')à”|ð')ð # 5¤<°´Ð#=Ô>ð')ð œ tÑ+ð	')ð
  ™ð')ð Ð-Ô.ð')ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð')ð ')ð ')ð ')ð ')ð ')ð ')ð ')r4   r@   c                   ó   — e Zd ZdS )ÚQwen2DecoderLayerNr<   r=   r4   r3   rs   rs   e   r>   r4   rs   c                   ó   — e Zd ZdS )ÚQwen2PreTrainedModelNr<   r=   r4   r3   ru   ru   i   r>   r4   ru   c                   óâ   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej	        dz  dej
        dz  dej	        dz  dedz  dej        dz  d	edz  d
ee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú
Qwen2Modelr1   c                 óp   •— t          ¦   «                              |¦  «         d| j        j        v | _        d S )NrD   )r'   r(   r1   rC   Úhas_sliding_layersr/   s     €r3   r(   zQwen2Model.__init__n   s3   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø"5¸¼Ô9PÐ"PˆÔÐÐr4   NÚ	input_idsrQ   Úposition_idsrR   Úinputs_embedsÚ	use_cacherS   rT   c           
      óø  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          |x}	t          ¦  «        s1| j        ||||dœ}
dt          di |
¤Ži}	| j        rt          di |
¤Ž|	d<   |}|                      ||¦  «        }t!          | j        d | j        j        …         ¦  «        D ]*\  }} ||f|	| j        j        |                  ||||d	œ|¤Ž}Œ+|                      |¦  «        }t+          ||r|nd ¬
¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embeds)r1   r   r    )Údevice)r1   r|   rQ   rR   r{   Úfull_attentionrD   )rQ   rP   r{   rR   r}   )Úlast_hidden_staterR   r=   )Ú
ValueErrorÚembed_tokensr   r1   Úget_seq_lengthro   ÚarangerY   r   Ú	unsqueezeÚ
isinstanceÚdictr   ry   r	   Ú
rotary_embÚ	enumerateÚlayersÚnum_hidden_layersrC   Únormr   )r0   rz   rQ   r{   rR   r|   r}   rS   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsrO   rP   ÚiÚdecoder_layers                  r3   rm   zQwen2Model.forwardr   s  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLõ °Ð?Ð-ÅÑFÔFð 	lð œ+Ø!.Ø"0Ø#2Ø ,ðð ˆKð !Õ"4Ð"CÐ"C°{Ð"CÐ"Cð#Ðð Ô&ð lÝ;\Ð;kÐ;kÐ_jÐ;kÐ;kÐ#Ð$7Ñ8à%ˆØ"Ÿošo¨m¸\ÑJÔJÐå )¨$¬+Ð6U¸¼Ô8UÐ6UÔ*VÑ WÔ Wð 		ð 		ÑˆAˆ}Ø)˜MØðà2°4´;Ô3JÈ1Ô3MÔNØ$7Ø)Ø /Ø#ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆÝ&Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
r4   )NNNNNN)r5   r6   r7   r!   r(   r   r   r   ro   Ú
LongTensorrp   r   ÚFloatTensorÚboolr   r   r   rm   r8   r9   s   @r3   rw   rw   m   s  ø€ € € € € ðQ˜{ð Qð Qð Qð Qð Qð Qð  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ð<
ð <
àÔ# dÑ*ð<
ð œ tÑ+ð<
ð Ô&¨Ñ-ð	<
ð
  ™ð<
ð Ô(¨4Ñ/ð<
ð ˜$‘;ð<
ð Ð+Ô,ð<
ð 
!ð<
ð <
ð <
ñ „^ñ „_ñ  Ôð<
ð <
ð <
ð <
ð <
r4   rw   c                   ó   — e Zd ZdS )ÚQwen2ForCausalLMNr<   r=   r4   r3   r—   r—   ´   r>   r4   r—   c                   ó   — e Zd ZdS )ÚQwen2ForSequenceClassificationNr<   r=   r4   r3   r™   r™   ¸   r>   r4   r™   c                   ó   — e Zd ZdS )ÚQwen2ForTokenClassificationNr<   r=   r4   r3   r›   r›   ¼   r>   r4   r›   c                   ó   — e Zd ZdS )ÚQwen2ForQuestionAnsweringNr<   r=   r4   r3   r�   r�   À   r>   r4   r�   )ru   rw   r—   ÚQwen2RMSNormr™   r›   r�   )9Úcollections.abcr   ro   r   Úcache_utilsr   r   Úmasking_utilsr   r	   Úmodeling_flash_attention_utilsr
   Úmodeling_outputsr   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.output_capturingr   Úgemma2.modeling_gemma2r   Úllama.modeling_llamar   r   r   r   r   r   r   r   r   r   Úmistral.modeling_mistralr   Úconfiguration_qwen2r!   Ú
get_loggerr5   Úloggerr#   r;   r@   rs   ru   rw   r—   r™   r›   r�   Ú__all__r=   r4   r3   ú<module>r°      s�  ðØ $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð à .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ BÐ BÐ BÐ BÐ BÐ Bðð ð ð ð ð ð 6Ð 5Ð 5Ð 5Ð 5Ð 5Ø &Ð &Ð &Ð &Ð &Ð &Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø :Ð :Ð :Ð :Ð :Ð :ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 4Ð 3Ð 3Ð 3Ð 3Ð 3Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,ð 
ˆÔ	˜HÑ	%Ô	%€ðYð Yð Yð Yð Yˆxñ Yô Yð Yð	ð 	ð 	ð 	ð 	Ð0ñ 	ô 	ð 	ð1)ð 1)ð 1)ð 1)ð 1)�^ñ 1)ô 1)ð 1)ðh	ð 	ð 	ð 	ð 	Ð)ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð/ñ 	ô 	ð 	ðD
ð D
ð D
ð D
ð D
�ñ D
ô D
ð D
ðN	ð 	ð 	ð 	ð 	Ð'ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð%Cñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð"=ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð 9ñ 	ô 	ð 	ðð ð €€€r4   