§
    kŠtj`Ù  ã                   ó„   — d dl mZ d dlZd dlmZ d dlmZ d dlm	Z	m
Z
mZ d dlmZ  ee¦  «        Z G d„ de¦  «        ZdS )	é    )Ú	getLoggerN)ÚFusion)ÚNumpyHelper)Ú	NodeProtoÚTensorProtoÚhelper)Ú	OnnxModelc                   ó  ‡ — e Zd ZdZdedededededefˆ fd„Zd%d
ededefd„Z	d„ Z
	 d%d
edededeeef         fd„Zdededededededededz  fd„Zdededededededededz  fd„Zd„ Zd„ Zd„ Zd„ Zd „ Zd!efd"„Zd#„ Zd$„ Zˆ xZS )&ÚFusionAttentionUnetzB
    Fuse Attention subgraph of UNet into one Attention node.
    ÚmodelÚhidden_sizeÚ	num_headsÚis_cross_attentionÚenable_packed_qkvÚenable_packed_kvc                 ó¾   •— t          ¦   «                              ||r|rdnddg¦  «         || _        || _        || _        || _        || _        d| _        d| _        d S )NÚ	AttentionÚMultiHeadAttentionÚLayerNormalizationT)	ÚsuperÚ__init__r   r   r   r   r   Únum_heads_warningÚhidden_size_warning)Úselfr   r   r   r   r   r   Ú	__class__s          €úl/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/onnxruntime/transformers/fusion_attention_unet.pyr   zFusionAttentionUnet.__init__   s€   ø€ õ 	‰Œ×ÒØØ-Ð]Ð2CÐ]ˆKˆKÐI]Ø!Ð"ñ	
ô 	
ð 	
ð
 'ˆÔØ"ˆŒØ"4ˆÔð "3ˆÔØ 0ˆÔð "&ˆÔØ#'ˆÔ Ð Ð ó    FÚ	reshape_qÚ	is_torch2Úreturnc                 ór  — d}|r¨| j                              |d¦  «        }|rŠ|j        dk    rt          |j        ¦  «        dk    rg| j                              |j        d         ¦  «        }t          |t          j        ¦  «        r(t          |j
        ¦  «        dgk    rt          |¦  «        }nm| j                              |j        d         ¦  «        }t          |t          j        ¦  «        r.t          |j
        ¦  «        dgk    rt          |d         ¦  «        }t          |t          ¦  «        r|dk    r|S dS )zòDetect num_heads from a reshape node.

        Args:
            reshape_q (NodeProto): reshape node for Q
            is_torch2 (bool): graph pattern is from PyTorch 2.*
        Returns:
            int: num_heads, or 0 if not found
        r   é   ÚConcaté   é   )r   Ú
get_parentÚop_typeÚlenÚinputÚget_constant_valueÚ
isinstanceÚnpÚndarrayÚlistÚshapeÚint)r   r   r   r   Úreshape_parentÚq_shape_values         r   Úget_num_headsz!FusionAttentionUnet.get_num_heads2   s*  € ð ˆ	Øð 	2à!œZ×2Ò2°9¸aÑ@Ô@ˆNØð / .Ô"8¸HÒ"DÐ"DÍÈ^ÔMaÑIbÔIbÐfgÒIgÐIgØ œJ×9Ò9¸.Ô:NÈqÔ:QÑRÔR�	Ý˜i­¬Ñ4Ô4ð /½¸i¼oÑ9NÔ9NÐSTÐRUÒ9UÐ9UÝ # I¡¤�Iøð !œJ×9Ò9¸)¼/È!Ô:LÑMÔMˆMÝ˜-­¬Ñ4Ô4ð 2½¸mÔ>QÑ9RÔ9RÐWXÐVYÒ9YÐ9YÝ ¨aÔ 0Ñ1Ô1�	å�i¥Ñ%Ô%ð 	¨)°aª-¨-ØÐàˆqr   c                 ó’   — | j                              |j        d         ¦  «        }|rt          j        |¦  «        j        d         S dS )zÚDetect hidden_size from LayerNormalization node.
        Args:
            layernorm_node (NodeProto): LayerNormalization node before Q, K and V
        Returns:
            int: hidden_size, or 0 if not found
        r%   r   )r   Úget_initializerr)   r   Úto_arrayr/   )r   Úlayernorm_nodeÚlayernorm_biass      r   Úget_hidden_sizez#FusionAttentionUnet.get_hidden_sizeN   sH   € ð œ×3Ò3°NÔ4HÈÔ4KÑLÔLˆØð 	AÝÔ'¨Ñ7Ô7Ô=¸aÔ@Ð@àˆqr   r7   c                 ó¼  — |                       ||¦  «        }|dk    r| j        }| j        dk    r?|| j        k    r4| j        r-t                               d| j        › d|› d�¦  «         d| _        |                      |¦  «        }|dk    r| j        }| j        dk    r?|| j        k    r4| j        r-t                               d| j        › d|› d�¦  «         d| _        ||fS )aF  Detect num_heads and hidden_size.

        Args:
            reshape_q (NodeProto): reshape node for Q
            is_torch2 (bool): graph pattern is from PyTorch 2.*
            layernorm_node (NodeProto): LayerNormalization node before Q, K, V
        Returns:
            Tuple[int, int]: num_heads and hidden_size
        r   z--num_heads is z. Detected value is z. Using detected value.Fz--hidden_size is )r3   r   r   ÚloggerÚwarningr9   r   r   )r   r   r7   r   r   r   s         r   Úget_num_heads_and_hidden_sizez1FusionAttentionUnet.get_num_heads_and_hidden_size[   s  € ð ×&Ò& y°)Ñ<Ô<ˆ	Ø˜Š>ˆ>ØœˆIàŒ>˜AÒÐ )¨t¬~Ò"=Ð"=ØÔ%ð /Ý—’Ðw°´ÐwÐwÐU^ÐwÐwÐwÑxÔxÐxØ).�Ô&à×*Ò*¨>Ñ:Ô:ˆØ˜!ÒÐØÔ*ˆKàÔ˜aÒÐ K°4Ô3CÒ$CÐ$CØÔ'ð 1Ý—’Ør¨Ô(8ÐrÐrÈkÐrÐrÐrñô ð ð ,1�Ô(à˜+Ð%Ð%r   Úq_matmulÚk_matmulÚv_matmulr)   ÚoutputNc           
      óR  — | j          }|rt|j        d         |k    s"|j        d         |k    s|j        d         |k    r@t                               d|j        d         |j        d         |j        d         ¦  «         dS n~|j        d         |k    s-|j        d         |j        d         k    s|j        d         |k    r@t                               d|j        d         |j        d         |j        d         ¦  «         dS |dk    r+||z  dk    r"t                               d|› d|› �¦  «         dS | j                             |j        d         ¦  «        }	| j                             |j        d         ¦  «        }
| j                             |j        d         ¦  «        }|	r|
r|sdS |	j        }t          j        |	¦  «        }t          j        |
¦  «        }t          j        |¦  «        }t                               d|j	        › d	|j	        › d
|j	        › d|› �¦  «         |�r¥|j	        |j	        k    s|j	        |j	        k    rdS |j	        d         }|dk    r||k    rt          d|› d|› d�¦  «        ‚t          t          j        |j	        dd…         ¦  «        ¦  «        }| j        �rÆ| j                             d¦  «        }|}|}||z  }t          j        |                     |||¦  «        |                     |||¦  «        |                     |||¦  «        g¦  «                             ||dz  |z  ¦  «        }| j                             dd¬¦  «        }|                      |dz   ||j	        d         |j	        d         g|¬¦  «         t'          j        d|j        d         |dz   g|dz   g|¬¦  «        }| j        | j        |j        <   |                      |dz   t0          j        dgdd|d|gd¬¦  «         t'          j        d|dz   |dz   g|dz   g|dz   ¬¦  «        }| j        | j        |j        <   | j                             ||g¦  «         | j                             |||g¦  «         �nvt          j        |||fd¬¦  «        }d|z  }| j                             d ¦  «        }|                      |d!z   |||g|¬¦  «         �n| j                             d¦  «        }| j        �rü|j	        |j	        k    rdS |j	        d         }|j	        d         }||k    sJ ‚|j	        d         }|j	        d         }|j	        d         }||k    r||k    sJ ‚|}|}||z  }t          j        |                     |||¦  «        |                     |||¦  «        g¦  «                             ||d"z  |z  ¦  «        }| j                             dd#¬¦  «        }|                      |dz   ||j	        d         |j	        d         g|¬¦  «         t'          j        d|j        d         |dz   g|dz   g|¬¦  «        }| j        | j        |j        <   |                      |dz   t0          j        dgdd|d"|gd¬¦  «         t'          j        d|dz   |dz   g|d$z   g|dz   ¬¦  «        }| j        | j        |j        <   | j                             ||g¦  «         | j                             ||g¦  «         t          j        d|gt          j         ¬%¦  «        } d|z  }!|                      |d&z   ||!g| ¬¦  «         |r| j        s||d!z   |d&z   g}"nK|dz   g}"nD| j        s+|j!        d         |j!        d         |j!        d         |d&z   g}"n|j!        d         |d$z   g}"t'          j        |r	| j        sd nd|"|g|¬¦  «        }#d'|#_"        |#j#                             t'          j$        d(|¦  «        g¦  «         |r	| j        sd)n&d* %                    | j        rd+n
| j        rd,nd-¦  «        }$|  &                    |$¦  «         |#S ).á•  Create an Attention node.

        Args:
            q_matmul (NodeProto): MatMul node in fully connection for Q
            k_matmul (NodeProto): MatMul node in fully connection for K
            v_matmul (NodeProto): MatMul node in fully connection for V
            num_heads (int): number of attention heads. If a model is pruned, it is the number of heads after pruning.
            hidden_size (int): hidden dimension. If a model is pruned, it is the hidden dimension after pruning.
            input (str): input name
            output (str): output name

        Returns:
            Union[NodeProto, None]: the node created or None if failed.
        r   úRFor self attention, input hidden state for q and k/v shall be same. Got %s, %s, %sNúXFor cross attention, input hidden state for q and k/v shall be different. Got %s, %s, %súinput hidden size ú# is not a multiple of num of heads r"   úqw=ú kw=ú vw=ú hidden_size=úInput hidden size (ú,) is not same as weight dimension of q,k,v (ú:). Please provide a correct input hidden size or pass in 0r   é   ÚMatMulÚ
MatMul_QKV©Úname_prefixÚ_weight©ÚnameÚ	data_typeÚdimsÚvalsÚ_out©ÚinputsÚoutputsrV   Ú_reshape_shapeé   F©rV   rW   rX   rY   ÚrawÚReshapeÚ
_qkv_inputÚ_reshape)Úaxisr   Ú_qkv_weightr%   Ú	MatMul_KVÚ	_kv_input©ÚdtypeÚ	_qkv_biasúcom.microsoftr   úAttention (self attention)úMultiHeadAttention ({})úself attention with packed qkvúcross attention with packed kvúcross attention)'r   r)   r;   Údebugr   r5   rW   r   r6   r/   Ú
ValueErrorr0   r,   Úprodr   Úcreate_node_nameÚdstackÚreshapeÚadd_initializerr   Ú	make_nodeÚthis_graph_nameÚnode_name_to_graph_namerV   r   ÚINT64Únodes_to_addÚextendÚnodes_to_removeÚstackr   ÚzerosÚfloat32rA   ÚdomainÚ	attributeÚmake_attributeÚformatÚincrease_counter)%r   r>   r?   r@   r   r   r)   rA   Úis_self_attentionÚq_weightÚk_weightÚv_weightÚ
float_typeÚqwÚkwÚvwÚ
qw_in_sizeÚqw_out_sizeÚattention_node_nameÚcÚnÚhÚ
qkv_weightÚmatmul_node_nameÚmatmul_nodeÚreshape_nodeÚqkv_weight_dimÚ
kw_in_sizeÚ
vw_in_sizeÚkw_out_sizeÚvw_out_sizeÚ	kv_weightÚqkv_biasÚqkv_bias_dimÚattention_inputsÚattention_nodeÚcounter_names%                                        r   Úcreate_attention_nodez)FusionAttentionUnet.create_attention_node}   sê	  € ð0 !%Ô 7Ð7Ðàð 	ØŒ~˜aÔ  EÒ)Ð)¨X¬^¸AÔ->À%Ò-GÐ-GÈ8Ì>ÐZ[ÔK\Ð`eÒKeÐKeÝ—’ØhØ”N 1Ô%Ø”N 1Ô%Ø”N 1Ô%ñ	ô ð ð �tð Lfð Œ~˜aÔ  EÒ)Ð)¨h¬n¸QÔ.?À8Ä>ÐRSÔCTÒ.TÐ.TÐZbÔZhÐijÔZkÐotÒZtÐZtÝ—’ØnØ”N 1Ô%Ø”N 1Ô%Ø”N 1Ô%ñ	ô ð ð �tà˜Š?ˆ? ¨iÑ 7¸AÒ=Ð=Ý�LŠLÐi¨kÐiÐiÐ^gÐiÐiÑjÔjÐjØ�4à”:×-Ò-¨h¬n¸QÔ.?Ñ@Ô@ˆØ”:×-Ò-¨h¬n¸QÔ.?Ñ@Ô@ˆØ”:×-Ò-¨h¬n¸QÔ.?Ñ@Ô@ˆØð 	˜Xð 	¨(ð 	Ø�4ð Ô'ˆ
åÔ! (Ñ+Ô+ˆÝÔ! (Ñ+Ô+ˆÝÔ! (Ñ+Ô+ˆÝ�ŠÐ[˜2œ8Ð[Ð[¨¬Ð[Ð[°r´xÐ[Ð[ÈkÐ[Ð[Ñ\Ô\Ð\ð ñ G	BØŒx˜2œ8Ò#Ð# r¤x°2´8Ò';Ð';Ø�tàœ !œˆJà˜QŠˆ ;°*Ò#<Ð#<Ý ðN¨+ð Nð NÐcmð Nð Nð Nñô ð õ �bœg b¤h¨q¨r¨r¤lÑ3Ô3Ñ4Ô4ˆKàÔ%ñ <Ø&*¤j×&AÒ&AÐBVÑ&WÔ&WÐ#à�Ø�Ø 9Ñ,�õ  œY¨¯
ª
°1°a¸Ñ(;Ô(;¸R¿ZºZÈÈ1ÈaÑ=PÔ=PÐRT×R\ÒR\Ð]^Ð`aÐcdÑReÔReÐ'fÑgÔg×oÒoØ�q˜1‘u˜q‘yñô �
ð $(¤:×#>Ò#>¸xÐUaÐ#>Ñ#bÔ#bÐ Ø×$Ò$Ø)¨IÑ5Ø(Ø$Ô*¨1Ô-¨zÔ/?ÀÔ/BÐCØ#ð	 %ñ ô ð õ %Ô.ØØ$œN¨1Ô-Ð/?À)Ñ/KÐLØ-°Ñ6Ð7Ø)ð	ñ ô �ð BFÔAU�Ô,¨[Ô-=Ñ>à×$Ò$Ø)Ð,<Ñ<Ý)Ô/Ø˜Ø˜Q  1 a˜Øð %ñ ô ð õ  &Ô/Øà(¨6Ñ1Ø(Ð+;Ñ;ðð 1°<Ñ?Ð@Ø)¨JÑ6ð ñ  ô  �ð CGÔBV�Ô,¨\Ô->Ñ?ØÔ!×(Ò(¨+°|Ð)DÑEÔEÐEØÔ$×+Ò+¨X°xÀÐ,JÑKÔKÐKÑKõ  œX r¨2¨r l¸Ð;Ñ;Ô;�
Ø!" [¡�à&*¤j×&AÒ&AÀ+Ñ&NÔ&NÐ#à×$Ò$Ø,¨}Ñ<Ø(Ø$ nÐ5Ø#ð	 %ñ ô ð ñ ð #'¤*×"=Ò"=Ð>RÑ"SÔ"SÐØÔ$ñ 7BØ”8˜rœxÒ'Ð'Ø˜4àœX aœ[�
ØœX aœ[�
Ø! ZÒ/Ð/Ð/Ð/à œh qœk�Ø œh qœk�Ø œh qœk�Ø" kÒ1Ð1°kÀ[Ò6PÐ6PÐ6PÐPà�Ø�Ø 9Ñ,�õ œI r§z¢z°!°Q¸Ñ':Ô':¸B¿JºJÀqÈ!ÈQÑ<OÔ<OÐ&PÑQÔQ×YÒYÐZ[Ð]^ÐabÑ]bÐefÑ]fÑgÔg�	à#'¤:×#>Ò#>¸xÐU`Ð#>Ñ#aÔ#aÐ Ø×$Ò$Ø)¨IÑ5Ø(Ø#œ/¨!Ô,¨i¬o¸aÔ.@ÐAØ"ð	 %ñ ô ð õ %Ô.ØØ$œN¨1Ô-Ð/?À)Ñ/KÐLØ-°Ñ6Ð7Ø)ð	ñ ô �ð BFÔAU�Ô,¨[Ô-=Ñ>à×$Ò$Ø)Ð,<Ñ<Ý)Ô/Ø˜Ø˜Q  1 a˜Øð %ñ ô ð õ  &Ô/Øà(¨6Ñ1Ø(Ð+;Ñ;ðð 1°;Ñ>Ð?Ø)¨JÑ6ð ñ  ô  �ð CGÔBV�Ô,¨\Ô->Ñ?ØÔ!×(Ò(¨+°|Ð)DÑEÔEÐEØÔ$×+Ò+¨X°xÐ,@ÑAÔAÐAõ ”8˜Q Ð,µB´JÐ?Ñ?Ô?ˆØ˜;‘ˆà×ÒØ$ {Ñ2Ø Ø�Øð	 	ñ 	
ô 	
ð 	
ð ð 	ØÔ)ð HàØ'¨-Ñ7Ø'¨+Ñ5ð$Ð Ð ð %8¸,Ñ$FÐ#GÐ Ð àÔ(ð à”O AÔ&Ø”O AÔ&Ø”O AÔ&Ø'¨+Ñ5ð	$Ð Ð ð ”O AÔ&Ø'¨+Ñ5ð$Ð õ
  Ô)Ø-Ðg°dÔ6LÐgˆKˆKÐSgØ#Ø�HØ$ð	
ñ 
ô 
ˆð !0ˆÔØÔ ×'Ò'­Ô)>¸{ÈIÑ)VÔ)VÐ(WÑXÔXÐXð !ðØ)-Ô)?ðÐ(Ð(à*×1Ò1àÔ)ð'Ð0Ð0ð Ô(ð'Ð5Ð5à&ñô ð 	ð 	×Ò˜lÑ+Ô+Ð+ØÐr   Úq_matmul_addÚk_matmul_addÚv_matmul_addc           
      ó¶  — | j          }| j                             |dd¦  «        }	| j                             |dd¦  «        }
| j                             |dd¦  «        }|                      |¦  «        }|€dS |\  }}|                      |¦  «        }|€dS |\  }}|                      |¦  «        }|€dS |\  }}|rç|	j        d         |k    s"|
j        d         |k    s|j        d         |k    r@t
                               d|	j        d         |
j        d         |j        d         ¦  «         dS |j        d         |k    s"|j        d         |k    s|j        d         |k    r@t
                               d|j        d         |j        d         |j        d         ¦  «         dS nü|	j        d         |k    s-|
j        d         |j        d         k    s|
j        d         |k    r@t
                               d|	j        d         |
j        d         |j        d         ¦  «         dS |j        d         |k    s-|j        d         |j        d         k    s|
j        d         |k    r@t
                               d|j        d         |j        d         |j        d         ¦  «         dS |dk    r+||z  dk    r"t
                               d|› d	|› �¦  «         dS | j                             |	j        d
         ¦  «        }| j                             |
j        d
         ¦  «        }| j                             |j        d
         ¦  «        }|r|r|sdS |j        dk    rt
                               d¦  «         dS t          j
        |¦  «        }t          j
        |¦  «        }t          j
        |¦  «        }t
                               d|j        › d|j        › d|j        › d|› �¦  «         |�r:|j        |j        k    s|j        |j        k    rdS |j        d         }|dk    r||k    rt          d|› d|› d�¦  «        ‚t          t          j        |j        d
d…         ¦  «        ¦  «        }| j        �r±| j                             d¦  «        }|}|}||z  } t          j        |                     ||| ¦  «        |                     ||| ¦  «        |                     ||| ¦  «        g¦  «                             ||dz  | z  ¦  «        }!| j                             dd¬¦  «        }"|                      |"dz   t*          j        |!j        d         |!j        d
         g|!¬¦  «         t/          j        d|
j        d         |"dz   g|"dz   g|"¬¦  «        }#| j        | j        |#j        <   |j        dz   }$|                      |$t*          j        dgdd|| gd¬¦  «         | j                             d d!¬¦  «        }%t/          j        d |j        d         |$g|%dz   g|%¬¦  «        }&| j        | j        |&j        <   | j                             d d"¬¦  «        }'t/          j        d |j        d         |$g|'dz   g|'¬¦  «        }(| j        | j        |(j        <   | j                             d d#¬¦  «        })t/          j        d |j        d         |$g|)dz   g|)¬¦  «        }*| j        | j        |*j        <   | j                             d$d%¬¦  «        }+t/          j        d$|&j        d         |(j        d         |*j        d         g|+dz   g|+¬¦  «        },|,j                             t/          j         d&d¦  «        g¦  «         | j        | j        |,j        <   |,j        dz   }-|                      |-t*          j        dgdd|dz  | z  gd¬¦  «         | j                             d d'¬¦  «        }.t/          j        d |,j        d         |-g|.dz   g|.¬¦  «        }/| j        | j        |/j        <   | j                             d(d)¬¦  «        }0t/          j        d(|/j        d         |#j        d         g|0dz   g|0¬¦  «        }1| j        | j        |1j        <   |0dz   }2|                      |2t*          j        d*gdd|d| gd¬¦  «         t/          j        d |1j        d         |2g|d+z   g|0d,z   ¬¦  «        }3| j        | j        |3j        <   | j!                             |#|&|(|*|,|/|1|3g¦  «         | j"                             |	|
||||g¦  «         �n§dS | j                             d¦  «        }| j#        �r�|j        |j        k    rdS |j        d         }4|j        d         }5|4|5k    sJ ‚|j        d
         }|j        d
         }6|j        d
         }7||7k    r|6|7k    sJ ‚|4}|}|6|z  } t          j        |                     ||| ¦  «        |                     ||| ¦  «        g¦  «                             ||d-z  | z  ¦  «        }8| j                             dd.¬¦  «        }"|                      |"dz   t*          j        |8j        d         |8j        d
         g|8¬¦  «         t/          j        d|
j        d         |"dz   g|"dz   g|"¬¦  «        }#| j        | j        |#j        <   |j        dz   }9|                      |9t*          j        dgdd|| gd¬¦  «         | j                             d d"¬¦  «        }'t/          j        d |j        d         |9g|'dz   g|'¬¦  «        }(| j        | j        |(j        <   | j                             d d#¬¦  «        })t/          j        d |j        d         |9g|)dz   g|)¬¦  «        }*| j        | j        |*j        <   | j                             d$d/¬¦  «        }:t/          j        d$|(j        d         |*j        d         g|:dz   g|:¬¦  «        };|;j                             t/          j         d&d¦  «        g¦  «         | j        | j        |;j        <   |;j        dz   }<|                      |<t*          j        dgdd|d-z  | z  gd¬¦  «         | j                             d d0¬¦  «        }=t/          j        d |;j        d         |<g|=dz   g|=¬¦  «        }>| j        | j        |>j        <   | j                             d(d1¬¦  «        }?t/          j        d(|>j        d         |#j        d         g|?dz   g|?¬¦  «        }@| j        | j        |@j        <   |?dz   }2|                      |2t*          j        d*gdd|d-| gd¬¦  «         t/          j        d |@j        d         |2g|d2z   g|?d,z   ¬¦  «        }3| j        | j        |3j        <   | j!                             |#|(|*|;|>|@|3g¦  «         | j"                             |
|||g¦  «         ndS t          j$        d|gt          j%        ¬3¦  «        }Ad|z  }B|                      |d4z   t*          j        |Bg|A¬¦  «         |r| j        sdS |d+z   g}Cn| j#        sdS |j        d         |d2z   g}Ct/          j        |r	| j        sd5nd|C|g|¬¦  «        }Dd6|D_&        |Dj                             t/          j         d7|¦  «        g¦  «         |r	| j        sd8n&d9 '                    | j        rd:n
| j#        rd;nd<¦  «        }E|  (                    |E¦  «         |DS )=rC   rP   r   NrD   z_For self attention, input hidden state for LoRA q and k/v weights shall be same. Got %s, %s, %srE   zeFor cross attention, input hidden state for LoRA q and k/v weights shall be different. Got %s, %s, %srF   rG   r"   é
   zBweights are in fp16. Please run fp16 conversion after optimizationrH   rI   rJ   rK   rL   rM   rN   r   rO   rQ   rR   rT   rU   rZ   r[   r^   r$   Fr`   rb   ÚReshape_LoRA_QÚReshape_LoRA_KÚReshape_LoRA_Vr#   ÚConcat_LoRA_QKVre   ÚReshape_LoRA_QKVÚAddÚAdd_Weights_QKVr_   rc   rd   r%   rg   ÚConcat_LoRA_KVÚReshape_LoRA_KVÚAdd_Weights_KVrh   ri   rk   r   rl   r   rm   rn   ro   rp   rq   ))r   r   Úmatch_parentÚmatch_lora_pathr)   r;   rr   r5   rW   r   r6   r/   rs   r0   r,   rt   r   ru   rv   rw   rx   r   ÚFLOATr   ry   rz   r{   rV   r|   rA   r„   r~   r…   r}   r   r   r�   r‚   rƒ   r†   r‡   )Fr   r¦   r§   r¨   r   r   r)   rA   rˆ   r>   r?   r@   Úq_lora_nodesÚq_lora_last_nodeÚq_lora_matmul_1Úk_lora_nodesÚk_lora_last_nodeÚk_lora_matmul_1Úv_lora_nodesÚv_lora_last_nodeÚv_lora_matmul_1r‰   rŠ   r‹   r�   rŽ   r�   r�   r‘   r’   r“   r”   r•   r–   r—   r˜   Úlora_weight_shape_tensor_nameÚq_lora_reshape_node_nameÚq_lora_reshape_nodeÚk_lora_reshape_node_nameÚk_lora_reshape_nodeÚv_lora_reshape_node_nameÚv_lora_reshape_nodeÚqkv_lora_concat_node_nameÚqkv_lora_concat_nodeÚ'reshaped_lora_weights_shape_tensor_nameÚqkv_lora_reshaped_node_nameÚqkv_lora_reshaped_nodeÚadd_weights_node_nameÚadd_weights_nodeÚshape_tensor_namer™   r›   rœ   r�   rž   rŸ   Ú kv_lora_weight_shape_tensor_nameÚkv_lora_concat_node_nameÚkv_lora_concat_nodeÚ*reshaped_kv_lora_weights_shape_tensor_nameÚkv_lora_reshaped_node_nameÚkv_lora_reshaped_nodeÚadd_kv_weights_node_nameÚadd_kv_weights_noder    r¡   r¢   r£   r¤   sF                                                                         r   Úcreate_attention_node_loraz.FusionAttentionUnet.create_attention_node_lora  sû  € ð0 !%Ô 7Ð7Ðà”:×*Ò*¨<¸À1ÑEÔEˆØ”:×*Ò*¨<¸À1ÑEÔEˆØ”:×*Ò*¨<¸À1ÑEÔEˆà×+Ò+¨LÑ9Ô9ˆØÐØ�4Ø.:Ñ+Ð	˜?à×+Ò+¨LÑ9Ô9ˆØÐØ�4Ø.:Ñ+Ð	˜?à×+Ò+¨LÑ9Ô9ˆØÐØ�4Ø.:Ñ+Ð	˜?àð .	ØŒ~˜aÔ  EÒ)Ð)¨X¬^¸AÔ->À%Ò-GÐ-GÈ8Ì>ÐZ[ÔK\Ð`eÒKeÐKeÝ—’ØhØ”N 1Ô%Ø”N 1Ô%Ø”N 1Ô%ñ	ô ð ð �tð  Ô% aÔ(¨EÒ1Ð1Ø"Ô(¨Ô+¨uÒ4Ð4Ø"Ô(¨Ô+¨uÒ4Ð4å—’ØuØ#Ô)¨!Ô,Ø#Ô)¨!Ô,Ø#Ô)¨!Ô,ñ	ô ð ð �tð 5ð Œ~˜aÔ  EÒ)Ð)¨h¬n¸QÔ.?À8Ä>ÐRSÔCTÒ.TÐ.TÐZbÔZhÐijÔZkÐotÒZtÐZtÝ—’ØnØ”N 1Ô%Ø”N 1Ô%Ø”N 1Ô%ñ	ô ð ð �tð  Ô% aÔ(¨EÒ1Ð1Ø#Ô)¨!Ô,°Ô0EÀaÔ0HÒHÐHØ”N 1Ô%¨Ò.Ð.å—’ð)ð $Ô)¨!Ô,Ø#Ô)¨!Ô,Ø#Ô)¨!Ô,ñô ð ð �tà˜Š?ˆ? ¨iÑ 7¸AÒ=Ð=Ý�LŠLÐi¨kÐiÐiÐ^gÐiÐiÑjÔjÐjØ�4à”:×-Ò-¨h¬n¸QÔ.?Ñ@Ô@ˆØ”:×-Ò-¨h¬n¸QÔ.?Ñ@Ô@ˆØ”:×-Ò-¨h¬n¸QÔ.?Ñ@Ô@ˆØð 	˜Xð 	¨(ð 	Ø�4ð Ô Ò#Ð#Ý�LŠLÐ]Ñ^Ô^Ð^Ø�4åÔ! (Ñ+Ô+ˆÝÔ! (Ñ+Ô+ˆÝÔ! (Ñ+Ô+ˆÝ�ŠÐ[˜2œ8Ð[Ð[¨¬Ð[Ð[°r´xÐ[Ð[ÈkÐ[Ð[Ñ\Ô\Ð\ð ñ q	ØŒx˜2œ8Ò#Ð# r¤x°2´8Ò';Ð';Ø�tàœ !œˆJà˜QŠˆ ;°*Ò#<Ð#<Ý ðN¨+ð Nð NÐcmð Nð Nð Nñô ð õ �bœg b¤h¨q¨r¨r¤lÑ3Ô3Ñ4Ô4ˆKàÔ%ñ SØ&*¤j×&AÒ&AÐBVÑ&WÔ&WÐ#à�Ø�Ø 9Ñ,�õ  œY¨¯
ª
°1°a¸Ñ(;Ô(;¸R¿ZºZÈÈ1ÈaÑ=PÔ=PÐRT×R\ÒR\Ð]^Ð`aÐcdÑReÔReÐ'fÑgÔg×oÒoØ�q˜1‘u˜q‘yñô �
ð $(¤:×#>Ò#>¸xÐUaÐ#>Ñ#bÔ#bÐ Ø×$Ò$Ø)¨IÑ5Ý)Ô/Ø$Ô*¨1Ô-¨zÔ/?ÀÔ/BÐCØ#ð	 %ñ ô ð õ %Ô.ØØ$œN¨1Ô-Ð/?À)Ñ/KÐLØ-°Ñ6Ð7Ø)ð	ñ ô �ð BFÔAU�Ô,¨[Ô-=Ñ>ð 1AÔ0EÐHXÑ0XÐ-à×$Ò$Ø6Ý)Ô/Ø˜Ø˜Q  1˜Øð %ñ ô ð ð ,0¬:×+FÒ+FÀyÐ^nÐ+FÑ+oÔ+oÐ(Ý&,Ô&6ØØ,Ô3°AÔ6Ð8UÐVØ5¸Ñ>Ð?Ø1ð	'ñ 'ô 'Ð#ð JNÔI]�Ô,Ð-@Ô-EÑFð ,0¬:×+FÒ+FÀyÐ^nÐ+FÑ+oÔ+oÐ(Ý&,Ô&6ØØ,Ô3°AÔ6Ð8UÐVØ5¸Ñ>Ð?Ø1ð	'ñ 'ô 'Ð#ð JNÔI]�Ô,Ð-@Ô-EÑFð ,0¬:×+FÒ+FÀyÐ^nÐ+FÑ+oÔ+oÐ(Ý&,Ô&6ØØ,Ô3°AÔ6Ð8UÐVØ5¸Ñ>Ð?Ø1ð	'ñ 'ô 'Ð#ð JNÔI]�Ô,Ð-@Ô-EÑFð -1¬J×,GÒ,GÈÐ^oÐ,GÑ,pÔ,pÐ)Ý'-Ô'7Øà+Ô2°1Ô5Ø+Ô2°1Ô5Ø+Ô2°1Ô5ðð
 7¸Ñ?Ð@Ø2ð	(ñ 	(ô 	(Ð$ð %Ô.×5Ò5µvÔ7LÈVÐUVÑ7WÔ7WÐ6XÑYÔYÐYØJNÔJ^�Ô,Ð-AÔ-FÑGð ;OÔ:SÐVfÑ:fÐ7Ø×$Ò$Ø@Ý)Ô/Ø˜Ø˜Q  A¡¨¡	Ð*Øð %ñ ô ð ð /3¬j×.IÒ.IÈ)ÐasÐ.IÑ.tÔ.tÐ+Ý)/Ô)9ØØ0Ô7¸Ô:Ð<cÐdØ8¸6ÑAÐBØ4ð	*ñ *ô *Ð&ð MQÔL`�Ô,Ð-CÔ-HÑIð )-¬
×(CÒ(CÀEÐWhÐ(CÑ(iÔ(iÐ%Ý#)Ô#3ØØ2Ô9¸!Ô<¸kÔ>PÐQRÔ>SÐTØ2°VÑ;Ð<Ø.ð	$ñ $ô $Ð ð GKÔFZ�Ô,Ð-=Ô-BÑCð %:Ð<LÑ$LÐ!Ø×$Ò$Ø*Ý)Ô/Ø˜Ø˜Q  1 a˜Øð %ñ ô ð õ  &Ô/ØØ,Ô3°AÔ6Ð8IÐJØ0°<Ñ?Ð@Ø.°Ñ;ð	 ñ  ô  �ð CGÔBV�Ô,¨\Ô->Ñ?àÔ!×(Ò(à#Ø+Ø+Ø+Ø,Ø.Ø(Ø$ð	ñô ð ð Ô$×+Ò+¨X°xÀÈ<ÐYeÐgsÐ,tÑuÔuÐuÑuð �tà"&¤*×"=Ò"=Ð>RÑ"SÔ"SÐØÔ$ñ JØ”8˜rœxÒ'Ð'Ø˜4àœX aœ[�
ØœX aœ[�
Ø! ZÒ/Ð/Ð/Ð/à œh qœk�Ø œh qœk�Ø œh qœk�Ø" kÒ1Ð1°kÀ[Ò6PÐ6PÐ6PÐPà�Ø�Ø 9Ñ,�õ œI r§z¢z°!°Q¸Ñ':Ô':¸B¿JºJÀqÈ!ÈQÑ<OÔ<OÐ&PÑQÔQ×YÒYÐZ[Ð]^ÐabÑ]bÐefÑ]fÑgÔg�	à#'¤:×#>Ò#>¸xÐU`Ð#>Ñ#aÔ#aÐ Ø×$Ò$Ø)¨IÑ5Ý)Ô/Ø#œ/¨!Ô,¨i¬o¸aÔ.@ÐAØ"ð	 %ñ ô ð õ %Ô.ØØ$œN¨1Ô-Ð/?À)Ñ/KÐLØ-°Ñ6Ð7Ø)ð	ñ ô �ð BFÔAU�Ô,¨[Ô-=Ñ>ð 4DÔ3HÐK[Ñ3[Ð0Ø×$Ò$Ø9Ý)Ô/Ø˜Ø˜Q  1˜Øð %ñ ô ð ð ,0¬:×+FÒ+FÀyÐ^nÐ+FÑ+oÔ+oÐ(Ý&,Ô&6ØØ,Ô3°AÔ6Ð8XÐYØ5¸Ñ>Ð?Ø1ð	'ñ 'ô 'Ð#ð JNÔI]�Ô,Ð-@Ô-EÑFð ,0¬:×+FÒ+FÀyÐ^nÐ+FÑ+oÔ+oÐ(Ý&,Ô&6ØØ,Ô3°AÔ6Ð8XÐYØ5¸Ñ>Ð?Ø1ð	'ñ 'ô 'Ð#ð JNÔI]�Ô,Ð-@Ô-EÑFð ,0¬:×+FÒ+FÀxÐ]mÐ+FÑ+nÔ+nÐ(Ý&,Ô&6ØØ/Ô6°qÔ9Ð;NÔ;UÐVWÔ;XÐYØ5¸Ñ>Ð?Ø1ð	'ñ 'ô 'Ð#ð $Ô-×4Ò4µfÔ6KÈFÐTUÑ6VÔ6VÐ5WÑXÔXÐXØIMÔI]�Ô,Ð-@Ô-EÑFð >QÔ=UÐXhÑ=hÐ:Ø×$Ò$ØCÝ)Ô/Ø˜Ø˜Q  A¡¨¡	Ð*Øð %ñ ô ð ð .2¬Z×-HÒ-HÈÐ`qÐ-HÑ-rÔ-rÐ*Ý(.Ô(8ØØ/Ô6°qÔ9Ð;eÐfØ7¸&Ñ@ÐAØ3ð	)ñ )ô )Ð%ð LPÔK_�Ô,Ð-BÔ-GÑHð ,0¬:×+FÒ+FÀuÐZjÐ+FÑ+kÔ+kÐ(Ý&,Ô&6ØØ1Ô8¸Ô;¸[Ô=OÐPQÔ=RÐSØ5¸Ñ>Ð?Ø1ð	'ñ 'ô 'Ð#ð JNÔI]�Ô,Ð-@Ô-EÑFð %=Ð?OÑ$OÐ!Ø×$Ò$Ø*Ý)Ô/Ø˜Ø˜Q  1 a˜Øð %ñ ô ð õ  &Ô/ØØ/Ô6°qÔ9Ð;LÐMØ0°;Ñ>Ð?Ø1°JÑ>ð	 ñ  ô  �ð CGÔBV�Ô,¨\Ô->Ñ?ØÔ!×(Ò(à#Ø+Ø+Ø+Ø-Ø+Ø$ðñ
ô 
ð 
ð Ô$×+Ò+¨X°xÀÈ|Ð,\Ñ]Ô]Ð]Ð]ð �tõ ”8˜Q Ð,µB´JÐ?Ñ?Ô?ˆØ˜;‘ˆØ×ÒØ$ {Ñ2Ý!Ô'Ø�Øð	 	ñ 	
ô 	
ð 	
ð ð 	ØÔ)ð Hà�tà$7¸,Ñ$FÐ#GÐ Ð àÔ(ð à�tð !Ô'¨Ô*Ø'¨+Ñ5ð$Ð õ
  Ô)Ø-Ðg°dÔ6LÐgˆKˆKÐSgØ#Ø�HØ$ð	
ñ 
ô 
ˆð !0ˆÔØÔ ×'Ò'­Ô)>¸{ÈIÑ)VÔ)VÐ(WÑXÔXÐXð !ðØ)-Ô)?ðÐ(Ð(à*×1Ò1àÔ)ð'Ð0Ð0ð Ô(ð'Ð5Ð5à&ñô ð 	ð 	×Ò˜lÑ+Ô+Ð+ØÐr   c           
      óð  — |                       |||¦  «        rd S | j                             |dd¦  «        }|€#| j        s| j                             |dd¦  «        }|€d S |j        d         }||         }d }|D ]}|j        dk    r|} nŒ|€d S |                      ||¦  «        p|                      ||¦  «        }	|	�|	\  }
}}}}}}|}|                      |||
¦  «        \  }}|dk    rt           
                    d¦  «         d S |                      ||||||j        d         |j        d         ¬¦  «        }|€d S nê|                      ||¦  «        p|                      ||¦  «        }	|	€d S |	\  }
}}}}}}|}|                      |||
¦  «        \  }}|dk    rt           
                    d¦  «         d S |                      ||||||j        d         |j        d         ¬¦  «        }|€d S |                      |||
¦  «        \  }}|dk    rt           
                    d¦  «         d S | j                             |¦  «         | j        | j        |j        <   | j                             ||g¦  «         d| _        d S )Nr°   r   rb   ú*fuse_attention: failed to detect num_heads©r)   rA   T)Úfuse_a1111_fp16r   rµ   r   rA   r'   Úmatch_qkv_torch1Úmatch_qkv_torch2r=   r;   rr   r¥   Úmatch_qkv_torch1_loraÚmatch_qkv_torch2_lorarØ   r}   Úappendrz   r{   rV   r   r~   Úprune_graph)r   Únormalize_nodeÚinput_name_to_nodesÚoutput_name_to_nodeÚnode_before_layernormÚ
root_inputÚchildren_nodesÚskip_addÚnodeÚ	match_qkvr   Úreshape_qkvÚtranspose_qkvr   Úmatmul_qÚmatmul_kÚmatmul_vÚattention_last_nodeÚq_num_headsÚq_hidden_sizeÚnew_nodeÚmatmul_add_qÚmatmul_add_kÚmatmul_add_vs                           r   ÚfusezFusionAttentionUnet.fuseU  s+  € Ø×Ò Ð0CÐEXÑYÔYð 	ØˆFà $¤
× 7Ò 7¸ÈÈqÑ QÔ QÐð !Ð(°Ô1HÐ(Ø$(¤J×$;Ò$;¸NÈIÐWXÑ$YÔ$YÐ!à Ð(ØˆFà*Ô1°!Ô4ˆ
à,¨ZÔ8ˆØˆØ"ð 	ð 	ˆDØŒ|˜uÒ$Ð$Ø�Ø�ð %ð ÐØˆFà×)Ò)¨*°hÑ?Ô?ÐnÀ4×CXÒCXÐYcÐemÑCnÔCnˆ	ØÐ Ø]fÑZˆI�{ M°9¸hÈÐRZà"-Ðà)-×)KÒ)KÈIÐWeÐgpÑ)qÔ)qÑ&ˆK˜Ø˜aÒÐÝ—’ÐIÑJÔJÐJØ�ð ×1Ò1ØØØØØØ$Ô+¨AÔ.Ø*Ô1°!Ô4ð 2ñ ô ˆHð ÐØ�ð  ð ×2Ò2°:¸xÑHÔHð ÈD×LfÒLfØ˜HñMô MˆIð Ð Ø�àirÑfˆI�{ M°9¸lÈLÐZfà"-Ðà)-×)KÒ)KÈIÐWeÐgpÑ)qÔ)qÑ&ˆK˜Ø˜aÒÐÝ—’ÐIÑJÔJÐJØ�ð ×6Ò6ØØØØØØ$Ô+¨AÔ.Ø*Ô1°!Ô4ð 7ñ ô ˆHð ÐØ�à)-×)KÒ)KÈIÐWeÐgpÑ)qÔ)qÑ&ˆK˜Ø˜aÒÐÝ—’ÐIÑJÔJÐJØ�àÔ× Ò  Ñ*Ô*Ð*Ø6:Ô6JˆÔ$ X¤]Ñ3àÔ×#Ò#Ð%8¸-Ð$HÑIÔIÐIð  ˆÔÐÐr   c           
      ó  — |j         d         |k    rdnd}| j                             |g d¢|dddddg¦  «        }|€dS |\  }}}}}}| j                             |g d¢g d¢¦  «        }	|	€t                               d¦  «         dS |	\  }}}}
| j                             |g d¢g d	¢¦  «        }|�|\  }}}nF| j                             |g d
¢g d¢¦  «        }|�|\  }}}}nt                               d¦  «         dS | j                             |g d¢g d¢¦  «        }|€t                               d¦  «         dS |\  }}}}| j                             |g d¢g d¢¦  «        }|€t                               d¦  «         dS |\  }}}}}d||||||
fS )z.Match Q, K and V paths exported by PyTorch 1.*r   r"   )r°   rP   rb   Ú	Transposerb   rP   N©rb   rú   rb   rP   ©r"   r   r   r   ú&fuse_attention: failed to match v path©ÚSoftmaxÚMulrP   ©r   r   r   ©rÿ   r°   r   rP   ©r   r   r   r   ú'fuse_attention: failed to match qk pathú&fuse_attention: failed to match q path)rú   rb   rú   rb   rP   ©r"   r   r   r   r   ú&fuse_attention: failed to match k pathF©r)   r   Úmatch_parent_pathr;   rr   )r   rç   ré   Úanother_inputÚ	qkv_nodesÚ_rì   rí   Ú
matmul_qkvÚv_nodesrð   Úqk_nodesÚ_softmax_qkÚ_mul_qkÚ	matmul_qkÚ	_add_zeroÚq_nodesÚ_transpose_qr   rî   Úk_nodesrï   s                         r   rÝ   z$FusionAttentionUnet.match_qkv_torch1¯  s  € à%œ^¨AÔ.°*Ò<Ð<˜˜À!ˆØ”J×0Ò0ØØJÐJÐJØ˜D $¨¨1¨aÐ0ñ
ô 
ˆ	ð ÐØ�4à<EÑ9ˆˆAˆ{˜M¨1¨jð ”*×.Ò.¨zÐ;hÐ;hÐ;hÐjvÐjvÐjvÑwÔwˆØˆ?Ý�LŠLÐAÑBÔBÐBØ�4Ø%ÑˆˆAˆq�(à”:×/Ò/°
Ð<XÐ<XÐ<XÐZcÐZcÐZcÑdÔdˆØÐØ08Ñ-ˆ[˜' 9 9à”z×3Ò3°JÐ@cÐ@cÐ@cÐeqÐeqÐeqÑrÔrˆHØÐ#Ø?GÑ<�˜i¨°)°)å—’ÐFÑGÔGÐGØ�tà”*×.Ò.¨yÐ:gÐ:gÐ:gÐiuÐiuÐiuÑvÔvˆØˆ?Ý�LŠLÐAÑBÔBÐBØ�4Ø18Ñ.ˆˆL˜) Xà”*×.Ò.ØÐQÐQÐQÐSbÐSbÐSbñ
ô 
ˆð ˆ?Ý�LŠLÐAÑBÔBÐBØ�4à!(ÑˆˆAˆq�!�Xà�k =°)¸XÀxÐQYÐYÐYr   c           	      óT  — |j         d         |k    rdnd}| j                             |g d¢|ddddg¦  «        }|€dS |\  }}}}}| j                             |g d¢g d¢¦  «        }	|	€t                               d¦  «         dS |	\  }}}
| j                             |dd	gddg¦  «        }|�|\  }}nt                               d
¦  «         dS | j                             |g d¢g d¢¦  «        }|€t                               d¦  «         dS |\  }}}}| j                             |g d¢g d¢¦  «        }|€t                               d¦  «         dS |\  }}}}| j                             |g d¢g d¢¦  «        }|�|d         |k    rt                               d¦  «         dS d||||||
fS )z.Match Q, K and V paths exported by PyTorch 2.*r   r"   )r°   rP   rb   rú   rP   N)rú   rb   rP   ©r"   r   r   rý   rÿ   rP   r  )r   rú   rb   rP   ©r   Nr   r   r  ©r"   Nr   r   r  ©ÚSqrtÚDivr  ÚCastÚSliceÚShaperú   rb   ©Nr   r"   r   r   r   r   r   éÿÿÿÿz*fuse_attention: failed to match mul_q pathTr  )r   rç   ré   r
  r  r  rì   rí   r  r  rð   r  r  r  r  Úmul_qr  r   rî   r  Ú_mul_krï   Úmul_q_nodess                          r   rÞ   z$FusionAttentionUnet.match_qkv_torch2à  s  € à%œ^¨AÔ.°*Ò<Ð<˜˜À!ˆØ”J×0Ò0ØØ?Ð?Ð?Ø˜D $¨¨1Ð-ñ
ô 
ˆ	ð ÐØ�4à9BÑ6ˆˆAˆ{˜M¨:à”*×.Ò.¨zÐ;]Ð;]Ð;]Ð_hÐ_hÐ_hÑiÔiˆØˆ?Ý�LŠLÐAÑBÔBÐBØ�4Ø"ÑˆˆAˆxà”:×/Ò/°
¸YÈÐ<QÐTUÐWXÐSYÑZÔZˆØÐØ'/Ñ$ˆ[˜)˜)å�LŠLÐBÑCÔCÐCØ�4à”*×.Ò.¨yÐ:cÐ:cÐ:cÐetÐetÐetÑuÔuˆØˆ?Ý�LŠLÐAÑBÔBÐBØ�4Ø5<Ñ2ˆ�˜i¨à”*×.Ò.¨yÐ:cÐ:cÐ:cÐetÐetÐetÑuÔuˆØˆ?Ý�LŠLÐAÑBÔBÐBØ�4à#*Ñ ˆ��A�xð ”j×2Ò2ØØUÐUÐUØ'Ð'Ð'ñ
ô 
ˆð
 Ð +¨b¤/°YÒ">Ð">Ý�LŠLÐEÑFÔFÐFØ�4à�[ -°¸HÀhÐPXÐXÐXr   c                 ó"  — |j         d         |k    rdnd}| j                             |g d¢|ddddddg¦  «        }|€dS |\  }}}}}}}| j                             |g d¢g d¢¦  «        }	|	€t                               d¦  «         dS |	\  }}}}
| j                             |g d¢g d	¢¦  «        }|�|\  }}}nF| j                             |g d
¢g d¢¦  «        }|�|\  }}}}nt                               d¦  «         dS | j                             |g d¢g d¢¦  «        }|€t                               d¦  «         dS |\  }}}}| j                             |g d¢g d¢¦  «        }|€t                               d¦  «         dS |\  }}}}}d||||||
fS )zJMatch Q, K and V paths exported by PyTorch 1 that contains LoRA patterns.*r   r"   )r°   r°   rP   rb   rú   rb   rP   N)rb   rú   rb   r°   rü   ú+fuse_attention: failed to match LoRA v pathrþ   r  r  r  ú,fuse_attention: failed to match LoRA qk pathú+fuse_attention: failed to match LoRA q path)rú   rb   rú   rb   r°   r  ú+fuse_attention: failed to match LoRA k pathFr  )r   rç   ré   r
  r  r  rì   rí   r  r  r÷   r  r  r  r  r  r  r  r   rõ   r  rö   s                         r   rß   z)FusionAttentionUnet.match_qkv_torch1_lora  s  € à%œ^¨AÔ.°*Ò<Ð<˜˜À!ˆØ”J×0Ò0ØØQÐQÐQØ˜A˜t T¨1¨a°Ð3ñ
ô 
ˆ	ð
 ÐØ�4à?HÑ<ˆˆAˆq�+˜}¨a°ð ”*×.Ò.¨zÐ;eÐ;eÐ;eÐgsÐgsÐgsÑtÔtˆØˆ?Ý�LŠLÐFÑGÔGÐGØ�4Ø")ÑˆˆAˆq�,à”:×/Ò/°
Ð<XÐ<XÐ<XÐZcÐZcÐZcÑdÔdˆØÐØ08Ñ-ˆ[˜' 9 9à”z×3Ò3°JÐ@cÐ@cÐ@cÐeqÐeqÐeqÑrÔrˆHØÐ#Ø?GÑ<�˜i¨°)°)å—’ÐKÑLÔLÐLØ�tà”*×.Ò.¨yÐ:dÐ:dÐ:dÐfrÐfrÐfrÑsÔsˆØˆ?Ý�LŠLÐFÑGÔGÐGØ�4Ø5<Ñ2ˆˆL˜) \à”*×.Ò.ØÐNÐNÐNÐP_ÐP_ÐP_ñ
ô 
ˆð ˆ?Ý�LŠLÐFÑGÔGÐGØ�4à%,Ñ"ˆˆAˆq�!�\à�k =°)¸\È<ÐYeÐeÐer   c           
      óX  — |j         d         |k    rdnd}| j                             |g d¢|dddddg¦  «        }|€dS |\  }}}}}}| j                             |g d¢g d¢¦  «        }	|	€t                               d¦  «         dS |	\  }}}
| j                             |dd	gddg¦  «        }|�|\  }}nt                               d
¦  «         dS | j                             |g d¢g d¢¦  «        }|€t                               d¦  «         dS |\  }}}}| j                             |g d¢g d¢¦  «        }|€t                               d¦  «         dS |\  }}}}| j                             |g d¢g d¢¦  «        }|�|d         |k    rt                               d¦  «         dS d||||||
fS )zJMatch Q, K and V paths exported by PyTorch 2 that contains LoRA patterns.*r   r"   )r°   r°   rP   rb   rú   rP   N)rú   rb   r°   r  r'  rÿ   rP   r(  )r   rú   rb   r°   r  r)  r  r*  r  r!  r"  z/fuse_attention: failed to match LoRA mul_q pathTr  )r   rç   ré   r
  r  r  rì   rí   r  r  r÷   r  r  r  r  r#  r  r   rõ   r  r$  rö   r%  s                          r   rà   z)FusionAttentionUnet.match_qkv_torch2_loraD  s#  € à%œ^¨AÔ.°*Ò<Ð<˜˜À!ˆØ”J×0Ò0ØØFÐFÐFØ˜A˜t T¨1¨aÐ0ñ
ô 
ˆ	ð
 ÐØ�4à<EÑ9ˆˆAˆq�+˜}¨jà”*×.Ò.¨zÐ;ZÐ;ZÐ;ZÐ\eÐ\eÐ\eÑfÔfˆØˆ?Ý�LŠLÐFÑGÔGÐGØ�4Ø&ÑˆˆAˆ|à”:×/Ò/°
¸YÈÐ<QÐTUÐWXÐSYÑZÔZˆØÐØ'/Ñ$ˆ[˜)˜)å�LŠLÐGÑHÔHÐHØ�4à”*×.Ò.¨yÐ:`Ð:`Ð:`ÐbqÐbqÐbqÑrÔrˆØˆ?Ý�LŠLÐFÑGÔGÐGØ�4Ø9@Ñ6ˆ�˜i¨à”*×.Ò.¨yÐ:`Ð:`Ð:`ÐbqÐbqÐbqÑrÔrˆØˆ?Ý�LŠLÐFÑGÔGÐGØ�4à'.Ñ$ˆ��A�|ð ”j×2Ò2ØØUÐUÐUØ'Ð'Ð'ñ
ô 
ˆð
 Ð +¨b¤/°YÒ">Ð">Ý�LŠLÐJÑKÔKÐKØ�4à�[ -°¸LÈ,ÐXdÐdÐdr   Úadd_nodec                 ó  — | j                              |ddgddg¦  «        }|�	|\  }}||fS | j                              |g d¢g d¢¦  «        }|�
|\  }}}||fS | j                              |g d¢g d¢¦  «        }|�|\  }}}}||fS d S )NrP   r"   r   )r   rP   rP   r  )r   r   rP   rP   rü   )r   r	  )r   r,  Ú
lora_nodesÚlora_matmul_2_nodeÚlora_matmul_1_nodeÚlora_mul_noder  s          r   r¶   z#FusionAttentionUnet.match_lora_pathw  sî   € ð ”Z×1Ò1ØØ�xÐ Ø�ˆFñ
ô 
ˆ
ð Ð!Ø7AÑ4ÐÐ!3Ø&Ð(:Ð;Ð;ð ”Z×1Ò1ØØ'Ð'Ð'ØˆIˆIñ
ô 
ˆ
ð Ð!Ø5?Ñ2ˆ]˜AÐ1Ø!Ð#5Ð6Ð6ð ”Z×1Ò1ØØ.Ð.Ð.ØˆLˆLñ
ô 
ˆ
ð Ð!Ø8BÑ5ˆ]˜A˜qÐ"4Ø!Ð#5Ð6Ð6àˆtr   c           
      ó<  — | j                              |ddgddg¦  «        }|€$| j                              |ddgddg¦  «        }|€dS |\  }}|j        d         }||         }d}	|D ]}
|
j        dk    r|
}	 nŒ|	€dS |                      ||	¦  «        }|€dS |\  }}}}}}| j                              |dd¦  «        }| j                              |dd¦  «        }| j                              |dd¦  «        }|�|�| j        s||k    rn||k    r||k    sdS |j        d         |j        d         k    rdS |}|                      |d¦  «        p|                      |d¦  «        }|dk    rt           
                    d¦  «         dS |                      |¦  «        }|                      ||||||j        d         |j        d         ¬	¦  «        }|€dS | j                             |¦  «         | j        | j        |j        <   | j                             ||g¦  «         d| _        dS )
zPFuse attention of fp16 UNet exported in A1111 (stable diffusion webui) extensionr  r°   r   Nrb   FTrÚ   rÛ   )r   r	  rA   r'   Úmatch_qkv_a1111rµ   r   r)   r3   r;   rr   r9   r¥   r}   rá   rz   r{   rV   r   r~   râ   )r   rã   rä   rå   Ú
entry_pathÚ_castræ   rç   rè   ré   rê   rë   rì   rí   r   rî   rï   rð   Úcast_qÚcast_kÚcast_vrñ   rò   ró   rô   s                            r   rÜ   z#FusionAttentionUnet.fuse_a1111_fp16£  s¬  € à”Z×1Ò1°.À6È5À/ÐTUÐWXÐSYÑZÔZˆ
ØÐØœ×5Ò5°nÀvÈyÐFYÐ\]Ð_`Ð[aÑbÔbˆJØÐ!Ø�uØ'1Ñ$ˆÐ$à*Ô1°!Ô4ˆ
à,¨ZÔ8ˆØˆØ"ð 	ð 	ˆDØŒ|˜uÒ$Ð$Ø�Ø�ð %ð ÐØ�5à×(Ò(¨°XÑ>Ô>ˆ	ØÐØ�5ð ñ	
ØØØØØØð ”×(Ò(¨°6¸1Ñ=Ô=ˆØ”×(Ò(¨°6¸1Ñ=Ô=ˆØ”×(Ò(¨°6¸1Ñ=Ô=ˆàÐØÐ"Ø)-Ô)@ð #Ø˜6Ò!Ð!Ð!ÀfÐPVÒFVÐFVØ˜&Ò Ð à�5àŒ<˜Œ?˜nÔ3°AÔ6Ò6Ð6Ø�5à)Ðà×(Ò(¨°DÑ9Ô9Ða¸T×=OÒ=OÐPYÐ[`Ñ=aÔ=aˆØ˜!ÒÐÝ�LŠLÐEÑFÔFÐFØ�5à×,Ò,¨^Ñ<Ô<ˆð ×-Ò-ØØØØØØ”. Ô#Ø&Ô-¨aÔ0ð .ñ 
ô 
ˆð ÐØ�5àÔ× Ò  Ñ*Ô*Ð*Ø6:Ô6JˆÔ$ X¤]Ñ3àÔ×#Ò#Ð%8¸-Ð$HÑIÔIÐIð  ˆÔØˆtr   c           
      óÊ  — |j         d         |k    rdnd}| j                             |g d¢|dddddg¦  «        }|€dS |\  }}}}}}	| j                             |	g d¢g d¢¦  «        }
|
€t                               d¦  «         dS |
\  }}}}| j                             |	g d¢g d	¢¦  «        }|�	|\  }}}}}nt                               d
¦  «         dS | j                             |g d¢g d¢¦  «        }|€t                               d¦  «         dS |\  }}}}| j                             |g d¢g d¢¦  «        }|€t                               d¦  «         dS |\  }}}}||||||fS )zKMatch Q, K and V paths exported by A1111 (stable diffusion webui) extensionr   r"   )r°   rP   rb   rú   rb   ÚEinsumNrû   rü   rý   )r  r  rÿ   r   r:  )r   r   r   r   Nr  r  r  r  r  )r   rç   ré   r
  r  r  rì   rí   Úreshape_einsumÚ
einsum_qkvr  rð   r  r  Ú	einsum_qkr  r  r   rî   r  rï   s                        r   r3  z#FusionAttentionUnet.match_qkv_a1111ñ  sÕ  € à%œ^¨AÔ.°*Ò<Ð<˜˜À!ˆØ”J×0Ò0ØØJÐJÐJØ˜D $¨¨1¨aÐ0ñ
ô 
ˆ	ð ÐØ�4àIRÑFˆˆAˆ{˜M¨>¸:à”*×.Ò.¨zÐ;hÐ;hÐ;hÐjvÐjvÐjvÑwÔwˆØˆ?Ý�LŠLÐAÑBÔBÐBØ�4Ø%ÑˆˆAˆq�(à”:×/Ò/ØÐDÐDÐDÐFXÐFXÐFXñ
ô 
ˆð ÐØ08Ñ-ˆQ��;  9 9å�LŠLÐBÑCÔCÐCØ�4à”*×.Ò.¨yÐ:gÐ:gÐ:gÐiuÐiuÐiuÑvÔvˆØˆ?Ý�LŠLÐAÑBÔBÐBØ�4Ø18Ñ.ˆˆL˜) Xà”*×.Ò.¨yÐ:gÐ:gÐ:gÐiuÐiuÐiuÑvÔvˆØˆ?Ý�LŠLÐAÑBÔBÐBØ�4à%ÑˆˆAˆq�(à˜M¨9°hÀÈ(ÐRÐRr   )F)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r	   r0   Úboolr   r   r3   r9   Útupler=   Ústrr¥   rØ   rø   rÝ   rÞ   rß   rà   r¶   rÜ   r3  Ú__classcell__)r   s   @r   r   r      s”  ø€ € € € € ðð ð(àð(ð ð(ð ð	(ð
 !ð(ð  ð(ð ð(ð (ð (ð (ð (ð (ð:ð  yð ¸Tð Ècð ð ð ð ð8ð ð ð RWð &ð  &Ø"ð &Ø4=ð &ØJNð &à	ˆs�CˆxŒð &ð  &ð  &ð  &ðD@àð@ð ð@ð ð	@ð
 ð@ð ð@ð ð@ð ð@ð 
�TÑ	ð@ð @ð @ð @ðDTàðTð  ðTð  ð	Tð
 ðTð ðTð ðTð ðTð 
�TÑ	ðTð Tð Tð TðlX ð X ð X ðt/Zð /Zð /Zðb2Yð 2Yð 2Yðh.fð .fð .fð`1eð 1eð 1eðf*àð*ð *ð *ð *ðXLð Lð Lð\*Sð *Sð *Sð *Sð *Sð *Sð *Sr   r   )Úloggingr   Únumpyr,   Úfusion_baser   Úfusion_utilsr   Úonnxr   r   r   Ú
onnx_modelr	   r>  r;   r   © r   r   ú<module>rM     sÐ   ðð
 Ð Ð Ð Ð Ð à Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø $Ð $Ð $Ð $Ð $Ð $Ø /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ø  Ð  Ð  Ð  Ð  Ð  à	ˆ�8Ñ	Ô	€ðKSð KSð KSð KSð KS˜&ñ KSô KSð KSð KSð KSr   