§
    ‚Štj×   ã                   ó  — d dl Z ddlmZmZmZ ddlmZ  ej        e¦  «        Z	 edd¬¦  «        Z
 edd¬¦  «        Z e¦   «         Z e¦   «         Zd	e j        d
ede j        fd„Zde j        dz  de j        de j        defd„Zde j        de j        dz  dede j        de j        de j        fd„Z	 	 	 	 dde j        j        de j        de j        de j        de j        dz  dededz  dedz  de j        dz  dee j        df         fd„ZdS )é    Né   )Úis_torch_npu_availableÚis_torch_xpu_availableÚlogging)Úis_torch_greater_or_equalz2.5T)Ú
accept_devz2.8Úhidden_statesÚn_repÚreturnc                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    é   N)ÚshapeÚexpandÚreshape)r	   r
   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         úf/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/integrations/sdpa_attention.pyÚ	repeat_kvr      s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTó    Úattention_maskÚkeyÚvaluec                 ó€   — t           rt          S t          o)| d u o%|j        d         |j        d         cxk    odk    nc S )Néÿÿÿÿé   )Ú_is_torch_xpu_availableÚ#_is_torch_greater_or_equal_than_2_8Ú#_is_torch_greater_or_equal_than_2_5r   )r   r   r   s      r   Úuse_gqa_in_sdpar!      sU   € õ ð 3Ý2Ð2Ý.Ðu°>ÀTÐ3IÐuÈcÌiÐXZÌmÐ_dÔ_jÐkmÔ_nÐNuÐNuÒNuÐNuÐruÒNuÐNuÐNuÐNuÐur   Úposition_biasÚ	is_causalÚqueryc                 ó   — t          j        |j        ¦  «        j        }|€—|r’|j        }|j        d         |j        d         }}t          j        ||¬¦  «        dd…df         t          j        ||¬¦  «        ddd…f         k    }	|	                     dd||¦  «        }	t          j        |	| |¦  «        }
n| }
nt          j        || |¦  «        }
|
S )a   
    Create a floating-point dtype mask to use with sdpa. The mask contains the values of `position_bias` to positions where we should
    attend to tokens, and -inf where we should not. It will be added to the QK^T result in the attention, before the softmax. Note
    that using such a mask will usually prevent sdpa from dispatching to the most efficient kernel implementations.

    Note that we cannot create this in advance when we create the mask in the model, as the position_bias is usually learned
    differently in every layer.
    Nr   )Údevicer   )	ÚtorchÚfinfoÚdtypeÚminr&   r   ÚarangeÚviewÚwhere)r"   r   r#   r$   r   Ú	min_dtyper&   Úq_lengthÚ	kv_lengthÚcausal_maskÚposition_bias_masks              r   Úcreate_position_bias_maskr3   )   sæ   € õ ”˜CœIÑ&Ô&Ô*€IàÐàð 
	/Ø”ZˆFØ"'¤+¨a¤.°#´)¸A´,�iˆHå”˜X¨fÐ5Ñ5Ô5°a°a°a¸°gÔ>Å%Ä,ÈyÐagÐBhÑBhÔBhÐimÐopÐopÐopÐipÔBqÒqð ð &×*Ò*¨1¨a°¸9ÑEÔEˆKÝ!&¤¨[¸-ÈÑ!SÔ!SÐÐð "/ÐÐõ #œ[¨¸È	ÑRÔRÐàÐr   ç        ÚmoduleÚdropoutÚscalingc	                 óJ  — |	                      dd¦  «        rt                               d¦  «         i }
t          | d¦  «        rK| j        dk    r@t          |||¦  «        s+t          || j        ¦  «        }t          || j        ¦  «        }nddi}
|j        d         }|j        d         }|�|nt          | d	d¦  «        }|dk    o|d u o|}t          j
                             ¦   «         r.t          |t          j        ¦  «        r|                     ¦   «         }t          rU|�S|j        t          j        k    r>t          j        |                     ¦   «         ¦  «                             |j        ¦  «        }|rL|€J|dk    rD||k    r>|d d …d d …d |…d d …f         }|d d …d d …d |…d d …f         }|�|d d …d d …d d …d |…f         }|�t+          |||||¦  «        }d}t          j        j        j        |||f||||d
œ|
¤Ž}|                     dd¦  «                             ¦   «         }|d fS )NÚoutput_attentionsFzƒ`sdpa` attention does not support `output_attentions=True`. Please set your attention to `eager` if you want any of these features.Únum_key_value_groupsr   Ú
enable_gqaTr   r#   )Ú	attn_maskÚ	dropout_pÚscaler#   )ÚgetÚloggerÚwarning_onceÚhasattrr:   r!   r   r   Úgetattrr'   ÚjitÚ
is_tracingÚ
isinstanceÚTensorÚitemÚ_is_torch_npu_availabler)   ÚboolÚlogical_notÚtor&   r3   ÚnnÚ
functionalÚscaled_dot_product_attentionÚ	transposeÚ
contiguous)r5   r$   r   r   r   r6   r7   r#   r"   ÚkwargsÚsdpa_kwargsr/   r0   Úattn_outputs                 r   Úsdpa_attention_forwardrU   O   s�  € ð ‡z‚zÐ% uÑ-Ô-ð 
Ý×ÒðWñ	
ô 	
ð 	
ð €KÝˆvÐ-Ñ.Ô.ð /°6Ô3NÐQRÒ3RÐ3RÝ˜~¨s°EÑ:Ô:ð 	/Ý˜C Ô!<Ñ=Ô=ˆCÝ˜e VÔ%@ÑAÔAˆEˆEà'¨Ð.ˆKàŒ{˜1Œ~€HØ”	˜!”€Ið 'Ð2�	�	½ÀÈÐUYÑ8ZÔ8Z€Ið ˜1’ÐE °4Ð!7ÐE¸I€Iõ „y×ÒÑÔð %¥*¨Y½¼Ñ"EÔ"Eð %Ø—N’NÑ$Ô$ˆ	õ
 ð WØÐ%¨.Ô*>Å%Ä*Ò*LÐ*Lå"Ô.¨~×/BÒ/BÑ/DÔ/DÑEÔE×HÒHÈÌÑVÔVˆNð ð >�^Ð+°¸1²°ÀÈXÒAUÐAUØ�!�!�!�Q�Q�Q˜	˜˜	 1 1 1Ð$Ô%ˆØ�a�a�a˜˜˜˜I˜X˜I q q qÐ(Ô)ˆàÐ$Ø)¨!¨!¨!¨Q¨Q¨Q°°°°9°H°9Ð*<Ô=ˆMð Ð Ý2°=À.ÐR[Ð]bÐdgÑhÔhˆØˆ	å”(Ô%ÔBØØØð	ð !ØØØð	ð 	ð ð	ð 	€Kð ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜ÐÐr   )r4   NNN)r'   Úutilsr   r   r   Úutils.import_utilsr   Ú
get_loggerÚ__name__r@   r    r   r   rI   rG   Úintr   rJ   r!   r3   rM   ÚModuleÚfloatÚtuplerU   © r   r   ú<module>r_      sE  ðØ €€€à KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ :Ð :Ð :Ð :Ð :Ð :ð 
ˆÔ	˜HÑ	%Ô	%€ð '@Ð&?ÀÐRVÐ&WÑ&WÔ&WÐ #Ø&?Ð&?ÀÐRVÐ&WÑ&WÔ&WÐ #Ø0Ð0Ñ2Ô2Ð Ø0Ð0Ñ2Ô2Ð ð	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð
v E¤L°4Ñ$7ð 
v¸e¼lð 
vÐSXÔS_ð 
vÐdhð 
vð 
vð 
vð 
vð#Ø”<ð#à”L 4Ñ'ð#ð ð#ð Œ<ð	#ð
 
Œð#ð „\ð#ð #ð #ð #ðX Ø Ø!Ø)-ðWð WØŒHŒOðWàŒ<ðWð 
ŒðWð Œ<ð	Wð
 ”L 4Ñ'ðWð ðWð �T‰\ðWð �d‰{ðWð ”< $Ñ&ðWð ˆ5Œ<˜ÐÔðWð Wð Wð Wð Wð Wr   