
    <h                        S SK Jr  S SKrSSKJrJr  SSKJr  \R                  " \	5      r
\" SSS9r\" S	SS9r\" 5       rS
\R                  S\S\R                  4S jrS\\R                     S\R                  S\4S jr   SS\R&                  R(                  S\R                  S\R                  S\R                  S\\R                     S\S\\   S\\   S\\R                  S4   4S jjrg)    )OptionalN   )is_torch_xpu_availablelogging)is_torch_greater_or_equalz2.5T)
accept_devz2.8hidden_statesn_repreturnc                     U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )z
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
   N)shapeexpandreshape)r	   r
   batchnum_key_value_headsslenhead_dims         `/var/www/html/shao/venv/lib/python3.13/site-packages/transformers/integrations/sdpa_attention.py	repeat_kvr      s_    
 2?1D1D.Ez!!Qa"23::5W\dlmM  e(CTTT    attention_maskkeyc                    [         (       a6  [        =(       a)    [        U[        R                  R
                  5      (       + $ [        =(       a4    U S L =(       a)    [        U[        R                  R
                  5      (       + $ )N)_is_torch_xpu_available#_is_torch_greater_or_equal_than_2_8
isinstancetorchfxProxy#_is_torch_greater_or_equal_than_2_5)r   r   s     r   use_gqa_in_sdpar"      sU     2Z:c588>>;Z7ZZ.q>T3IqR\]`bgbjbjbpbpRqNqqr   modulequeryvaluedropoutscaling	is_causalc                    UR                  SS5      (       d  UR                  S5      b  [        R                  S5        0 n	[        U S5      (       a?  [	        XB5      (       d+  [        X R                  5      n[        X0R                  5      nOSS0n	Ub.  UR                  S:X  a  US S 2S S 2S S 2S UR                  S	   24   nUR                  5       nUR                  5       nUR                  5       nUc1  UR                  S
   S:  =(       a    US L =(       a    [        U SS5      n[        R                  R                  5       (       a/  [        U[        R                  5      (       a  UR!                  5       n[        R"                  R$                  R&                  " UUU4UUUUS.U	D6n
U
R)                  SS
5      R                  5       n
U
S 4$ )Noutput_attentionsF	head_maskz`sdpa` attention does not support `output_attentions=True` or `head_mask`. Please set your attention to `eager` if you want any of these features.num_key_value_groups
enable_gqaT   r   r   r(   )	attn_mask	dropout_pscaler(   )getloggerwarning_oncehasattrr"   r   r,   ndimr   
contiguousgetattrr   jit
is_tracingr   Tensoritemnn
functionalscaled_dot_product_attention	transpose)r#   r$   r   r%   r   r&   r'   r(   kwargssdpa_kwargsattn_outputs              r   sdpa_attention_forwardrE   +   s    zz%u--K1H1TW	
 Kv-..~33C!<!<=Ce%@%@AE'.K!n&9&9Q&>'1a399R=(@A E
..
CE
  KKNQ&h>T+AhgfVacgFh	 yy*Y"E"ENN$	((%%BB	 !	 	K ''1-88:Kr   )g        NN)typingr   r   utilsr   r   utils.import_utilsr   
get_logger__name__r4   r!   r   r   r<   intr   boolr"   r>   ModulefloattuplerE    r   r   <module>rQ      s6     3 : 
		H	% '@RV&W #&?RV&W #02 	UU\\ 	U# 	U%,, 	UrHU\\$: r rRV r( # $:HHOO:<<: 
: <<	:
 U\\*: : e_: ~: 5<<:r   