§
    kŠtj&Q  ã                   ó„   — d dl mZ d dlZd dlmZ d dlmZ d dlm	Z	m
Z
mZ d dlmZ  ee¦  «        Z G d„ de¦  «        ZdS )	é    )Ú	getLoggerN)ÚFusion)ÚNumpyHelper)Ú	NodeProtoÚhelperÚnumpy_helper)Ú	OnnxModelc                   óô   ‡ — e Zd ZdZdededefˆ fd„Zdedefd„Zd	edefd
„Z	d„ Z
	 d dedededeeef         fd„Zdededededededededededz  fd„Zd„ Zd„ Zdefd„Zd!d„Zdededededef
d„Zˆ xZS )"ÚFusionMultiHeadAttentionSam2zI
    Fuse MultiHeadAttention subgraph of Segment Anything v2 (SAM2).
    ÚmodelÚhidden_sizeÚ	num_headsc                 óˆ   •— t          ¦   «                              |ddg¦  «         || _        || _        d| _        d| _        d S )NÚMultiHeadAttentionÚLayerNormalizationT)ÚsuperÚ__init__r   r   Únum_heads_warningÚhidden_size_warning)Úselfr   r   r   Ú	__class__s       €úl/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/onnxruntime/transformers/fusion_attention_sam2.pyr   z%FusionMultiHeadAttentionSam2.__init__   sM   ø€ õ 	‰Œ×Ò˜Ð 4Ð7KÐ6LÑMÔMÐMØ&ˆÔØ"ˆŒð "&ˆÔØ#'ˆÔ Ð Ð ó    Ú	reshape_qÚreturnc                 ó"  — d}| j                              |j        d         ¦  «        }|�Ht          |t          j        ¦  «        r.t          |j        ¦  «        dgk    rt          |d         ¦  «        }t          |t          ¦  «        r|dk    r|S dS )ú²Detect num_heads from a reshape node.

        Args:
            reshape_q (NodeProto): reshape node for Q
        Returns:
            int: num_heads, or 0 if not found
        r   é   Né   é   )	r   Úget_constant_valueÚinputÚ
isinstanceÚnpÚndarrayÚlistÚshapeÚint)r   r   r   Úshape_values       r   Úget_decoder_num_headsz2FusionMultiHeadAttentionSam2.get_decoder_num_heads#   s�   € ð ˆ	ð ”j×3Ò3°I´OÀAÔ4FÑGÔGˆØÐ"Ý˜+¥r¤zÑ2Ô2ð 0µt¸KÔ<MÑ7NÔ7NÐSTÐRUÒ7UÐ7UÝ ¨A¤Ñ/Ô/�	å�i¥Ñ%Ô%ð 	¨)°aª-¨-ØÐàˆqr   Ú
reshape_inc                 ón  — d}| j                              |j        d         ¦  «        }|�It          |t          j        ¦  «        r.t          |j        ¦  «        dgk    rt          |d         ¦  «        }n¥| j          	                    |dd¦  «        }|�‡t          |j        ¦  «        dk    ro| j                              |j        d         ¦  «        }|�Ht          |t          j        ¦  «        r.t          |j        ¦  «        dgk    rt          |d         ¦  «        }t          |t          ¦  «        r|dk    r|S dS )r   r   r   Né   é   ÚConcat)r   r!   r"   r#   r$   r%   r&   r'   r(   Úmatch_parentÚlen)r   r+   r   r)   Úconcat_shapes        r   Úget_encoder_num_headsz2FusionMultiHeadAttentionSam2.get_encoder_num_heads8   s%  € ð ˆ	à”j×3Ò3°JÔ4DÀQÔ4GÑHÔHˆØÐ"Ý˜+¥r¤zÑ2Ô2ð 0µt¸KÔ<MÑ7NÔ7NÐSTÐRUÒ7UÐ7UÝ ¨A¤Ñ/Ô/�	øàœ:×2Ò2°:¸xÈÑKÔKˆLØÐ'­C°Ô0BÑ,CÔ,CÀqÒ,HÐ,Hà"œj×;Ò;¸LÔ<NÈqÔ<QÑRÔR�ØÐ*Ý! +­r¬zÑ:Ô:ð 8½tÀKÔDUÑ?VÔ?VÐ[\ÐZ]Ò?]Ð?]Ý$'¨°A¬Ñ$7Ô$7˜	å�i¥Ñ%Ô%ð 	¨)°aª-¨-ØÐàˆqr   c                 ó’   — | j                              |j        d         ¦  «        }|rt          j        |¦  «        j        d         S dS )zÚDetect hidden_size from LayerNormalization node.
        Args:
            layernorm_node (NodeProto): LayerNormalization node before Q, K and V
        Returns:
            int: hidden_size, or 0 if not found
        r    r   )r   Úget_initializerr"   r   Úto_arrayr'   )r   Úlayernorm_nodeÚlayernorm_biass      r   Úget_hidden_sizez,FusionMultiHeadAttentionSam2.get_hidden_sizeT   sH   € ð œ×3Ò3°NÔ4HÈÔ4KÑLÔLˆØð 	AÝÔ'¨Ñ7Ô7Ô=¸aÔ@Ð@àˆqr   Fr7   Ú
is_encoderc                 óê  — |r|                       |¦  «        }n|                      |¦  «        }|dk    r| j        }| j        dk    r?|| j        k    r4| j        r-t                               d| j        › d|› d�¦  «         d| _        |                      |¦  «        }|dk    r| j        }| j        dk    r?|| j        k    r4| j        r-t                               d| j        › d|› d�¦  «         d| _        ||fS )a  Detect num_heads and hidden_size.

        Args:
            reshape_q (NodeProto): reshape node for Q
            layernorm_node (NodeProto): LayerNormalization node before Q, K, V
        Returns:
            Tuple[int, int]: num_heads and hidden_size
        r   z--num_heads is z. Detected value is z. Using detected value.Fz--hidden_size is )	r3   r*   r   r   ÚloggerÚwarningr9   r   r   )r   r   r7   r:   r   r   s         r   Úget_num_heads_and_hidden_sizez:FusionMultiHeadAttentionSam2.get_num_heads_and_hidden_sizea   s)  € ð ð 	>Ø×2Ò2°9Ñ=Ô=ˆIˆIà×2Ò2°9Ñ=Ô=ˆIØ˜Š>ˆ>ØœˆIàŒ>˜AÒÐ )¨t¬~Ò"=Ð"=ØÔ%ð /Ý—’Ðw°´ÐwÐwÐU^ÐwÐwÐwÑxÔxÐxØ).�Ô&à×*Ò*¨>Ñ:Ô:ˆØ˜!ÒÐØÔ*ˆKàÔ˜aÒÐ K°4Ô3CÒ$CÐ$CØÔ'ð 1Ý—’Ør¨Ô(8ÐrÐrÈkÐrÐrÐrñô ð ð ,1�Ô(à˜+Ð%Ð%r   Úq_matmulÚq_addÚk_matmulÚk_addÚv_matmulÚv_addÚoutputNc
           
      ó¨  — |dk    r+||z  dk    r"t                                d|› d|› �¦  «         dS | j                             |j        d         ¦  «        }
| j                             |j        d         ¦  «        }| j                             |j        d         ¦  «        }|
r|r|sdS t          j        |
¦  «        }t          j        |¦  «        }t          j        |¦  «        }t                                d|j        › d|j        › d|j        › d	|› �¦  «         | j                             d
¦  «        }|j	        d         |j	        d         |j	        d         g}t          j        d
||	g|¬¦  «        }d|_        |j                             t          j        d|¦  «        g¦  «         d                     d¦  «        }|                      |¦  «         |S )aF  Create an Attention node.

        Args:
            q_matmul (NodeProto): MatMul node in fully connection for Q
            q_add (NodeProto): Add bias node in fully connection for Q
            k_matmul (NodeProto): MatMul node in fully connection for K
            k_add (NodeProto): Add bias node in fully connection for K
            v_matmul (NodeProto): MatMul node in fully connection for V
            v_add (NodeProto): Add bias node in fully connection for V
            num_heads (int): number of attention heads. If a model is pruned, it is the number of heads after pruning.
            hidden_size (int): hidden dimension. If a model is pruned, it is the hidden dimension after pruning.
            output (str): output name

        Returns:
            Union[NodeProto, None]: the node created or None if failed.
        r   zinput hidden size z# is not a multiple of num of heads Nr   zqw=z kw=z vw=z hidden_size=r   ©ÚinputsÚoutputsÚnameúcom.microsoftr   úMultiHeadAttention ({})zcross attention)r<   Údebugr   r5   r"   r   r6   r'   Úcreate_node_namerE   r   Ú	make_nodeÚdomainÚ	attributeÚextendÚmake_attributeÚformatÚincrease_counter)r   r?   r@   rA   rB   rC   rD   r   r   rE   Úq_weightÚk_weightÚv_weightÚqwÚkwÚvwÚattention_node_nameÚattention_inputsÚattention_nodeÚcounter_names                       r   Úcreate_attention_nodez2FusionMultiHeadAttentionSam2.create_attention_node…   sß  € ð8 ˜Š?ˆ? ¨iÑ 7¸AÒ=Ð=Ý�LŠLÐi¨kÐiÐiÐ^gÐiÐiÑjÔjÐjØ�4à”:×-Ò-¨h¬n¸QÔ.?Ñ@Ô@ˆØ”:×-Ò-¨h¬n¸QÔ.?Ñ@Ô@ˆØ”:×-Ò-¨h¬n¸QÔ.?Ñ@Ô@ˆØð 	˜Xð 	¨(ð 	Ø�4åÔ! (Ñ+Ô+ˆÝÔ! (Ñ+Ô+ˆÝÔ! (Ñ+Ô+ˆÝ�ŠÐ[˜2œ8Ð[Ð[¨¬Ð[Ð[°r´xÐ[Ð[ÈkÐ[Ð[Ñ\Ô\Ð\à"œj×9Ò9Ð:NÑOÔOÐð ŒL˜ŒOØŒL˜ŒOØŒL˜ŒOð
Ðõ  Ô)Ø Ø#Ø�HØ$ð	
ñ 
ô 
ˆð !0ˆÔØÔ ×'Ò'­Ô)>¸{ÈIÑ)VÔ)VÐ(WÑXÔXÐXà0×7Ò7Ð8IÑJÔJˆØ×Ò˜lÑ+Ô+Ð+ØÐr   c                 óŠ  — |                       |||¦  «        rd S |                      |¦  «        }|€J|j        d         |vrd S ||j        d                  }|j        dk    rd S |                      |¦  «        }|€d S |\	  }}}}	}
}}}}|}|                      ||d¦  «        \  }}|dk    rt
                               d¦  «         d S |                      |	|
|||||||j        d         ¬¦	  «	        }|€d S | j	         
                    |¦  «         | j        | j        |j        <   | j                             ||g¦  «         d| _        d S )Nr   ÚAddFú*fuse_attention: failed to detect num_heads)rE   T)Úfuse_sam_encoder_patternÚmatch_attention_subgraphr"   Úop_typer>   r<   rM   r`   rE   Únodes_to_addÚappendÚthis_graph_nameÚnode_name_to_graph_namerJ   Únodes_to_removerR   Úprune_graph)r   Únormalize_nodeÚinput_name_to_nodesÚoutput_name_to_nodeÚ	match_qkvÚskip_addÚreshape_qkvÚtranspose_qkvr   Úmatmul_qÚadd_qÚmatmul_kÚadd_kÚmatmul_vÚadd_vÚattention_last_nodeÚq_num_headsÚq_hidden_sizeÚnew_nodes                      r   Úfusez!FusionMultiHeadAttentionSam2.fuseÅ   s›  € Ø×(Ò(¨Ð9LÐNaÑbÔbð 	ØˆFà×1Ò1°.ÑAÔAˆ	ØÐØÔ# AÔ&Ð.AÐAÐAØ�à*¨>Ô+?ÀÔ+BÔCˆHØÔ 5Ò(Ð(Ø�à×5Ò5°hÑ?Ô?ˆIàÐ Ø�àclÑ`ˆ�] I¨x¸ÀÈ%ÐQYÐ[`à)Ðà%)×%GÒ%GÈ	ÐSaÐchÑ%iÔ%iÑ"ˆ�]Ø˜!ÒÐÝ�LŠLÐEÑFÔFÐFØˆFð ×-Ò-ØØØØØØØØØ&Ô-¨aÔ0ð .ñ 

ô 

ˆð ÐØˆFàÔ× Ò  Ñ*Ô*Ð*Ø6:Ô6JˆÔ$ X¤]Ñ3àÔ×#Ò#Ð%8¸-Ð$HÑIÔIÐIð  ˆÔÐÐr   c           	      ó.  — | j                              |g d¢g d¢¦  «        }|€dS |\  }}}}}| j                              |g d¢g d¢¦  «        }|€t                               d¦  «         dS |\  }}}}	| j                              |ddgd	d	g¦  «        }
|
�|
\  }}nt                               d
¦  «         dS | j                              |g d¢g d¢¦  «        }|€t                               d¦  «         dS |\  }}}}}| j                              |g d¢g d¢¦  «        }|€t                               d¦  «         dS |\  }}}}}| j                              |g d¢g d¢¦  «        }|�|d         |k    rt                               d¦  «         dS ||||||||	|f	S )z.Match Q, K and V paths exported by PyTorch 2.*©rb   ÚMatMulÚReshapeÚ	Transposer�   )NNNr   r   N)rƒ   r‚   rb   r�   )r   r   r   Nz&fuse_attention: failed to match v pathÚSoftmaxr�   r   z'fuse_attention: failed to match qk path)ÚMulrƒ   r‚   rb   r�   )r   Nr   r   Nz&fuse_attention: failed to match q path)r   Nr   r   Nz&fuse_attention: failed to match k path)ÚSqrtÚDivr†   ÚCastÚSliceÚShaperƒ   r‚   )Nr   r   r   r   r   r   r   éÿÿÿÿz*fuse_attention: failed to match mul_q path©r   Úmatch_parent_pathr<   rM   )r   Únode_after_output_projectionÚ	qkv_nodesÚ_rr   rs   Ú
matmul_qkvÚv_nodesry   rx   Úqk_nodesÚ_softmax_qkÚ	matmul_qkÚq_nodesÚmul_qÚ_transpose_qr   ru   rt   Úk_nodesÚ_mul_krw   rv   Úmul_q_nodess                           r   re   z5FusionMultiHeadAttentionSam2.match_attention_subgraph÷   s  € à”J×0Ò0Ø(Ø?Ð?Ð?Ø$Ð$Ð$ñ
ô 
ˆ	ð ÐØ�4à9BÑ6ˆˆAˆ{˜M¨:à”*×.Ò.¨zÐ;dÐ;dÐ;dÐfuÐfuÐfuÑvÔvˆØˆ?Ý�LŠLÐAÑBÔBÐBØ�4Ø")ÑˆˆAˆu�hà”:×/Ò/°
¸YÈÐ<QÐTUÐWXÐSYÑZÔZˆØÐØ'/Ñ$ˆ[˜)˜)å�LŠLÐBÑCÔCÐCØ�4à”*×.Ò.ØÐGÐGÐGÐI^ÐI^ÐI^ñ
ô 
ˆð ˆ?Ý�LŠLÐAÑBÔBÐBØ�4Ø<CÑ9ˆ�˜i¨°à”*×.Ò.ØÐGÐGÐGÐI^ÐI^ÐI^ñ
ô 
ˆð ˆ?Ý�LŠLÐAÑBÔBÐBØ�4à*1Ñ'ˆ��A�u˜hð ”j×2Ò2ØØUÐUÐUØ'Ð'Ð'ñ
ô 
ˆð
 Ð +¨b¤/°YÒ">Ð">Ý�LŠLÐEÑFÔFÐFØ�4à˜M¨9°hÀÀxÐQVÐX`ÐbgÐgÐgr   c                 ó   — | j                              |g d¢g d¢¦  «        }|€ | j                              |g d¢g d¢¦  «        }|€| j                              |dgdg¦  «        }|€dS |d         }|                      |t          |¦  «        d	k    rd	nd ¬
¦  «        }|€dS |\  }}}	}
}}t	          j        |
d¦  «        }t          |t          ¦  «        r|g d¢k    rdS t	          j        |d¦  «        }t          |t          ¦  «        r|g d¢k    rdS t	          j        |d¦  «        }t          |t          ¦  «        r|g d¢k    rdS | j                              |	g d¢g d¢¦  «        }|€dS |\  }}}|                      ||d¦  «        \  }}|dk    rt           
                    d¦  «         dS d}| j                              |¦  «        }|€Lt          j        t          j        g d¢d¬¦  «        |¬¦  «        }| j                              || j        ¦  «         | j                              d¦  «        }t'          j        d|
j        d         |g|
j        d         dz   g|¬¦  «        }| j                             |¦  «         | j        | j        |j        <   |
}|j        d         |j        d<   |j        d         dz   |j        d<   t           
                    d|›d|›�¦  «         |                      ||||¦  «        }|€dS t          | j                              ||¦  «        ¦  «        d	k    sJ ‚|j        d         |j        d<   | j                             |¦  «         | j        | j        |j        <   | j                             |g¦  «         d| _        dS )N)rb   r‚   rƒ   r‚   ©r   Nr   r   )rb   r‰   r‰   r‚   rƒ   r‚   )r   Nr   r   r   r   rb   r   Fr‹   r   )Úinput_indexÚperm)r   r    r   r.   )r   r    r.   r   )r‚   rb   r�   )r   r   NTrc   Úbsnh_to_bsd_reshape_dims)r   r   r‹   Úint64)Údtype)rJ   r‚   Ú_BSDrG   Ú_BNSHzFound MHA: q_num_heads=z q_hidden_size=) r   r�   Ú$match_sam_encoder_attention_subgraphr1   r	   Úget_node_attributer#   r&   r>   r<   rM   r5   r   Ú
from_arrayr$   ÚarrayÚadd_initializerri   rN   r   rO   r"   rg   rh   rj   rJ   rE   Úcreate_mha_nodeÚget_childrenrk   rR   rl   )r   rm   rn   ro   ÚnodesrŽ   Úmatched_sdpaÚreshape_outÚtranspose_outÚ	split_qkvÚtranspose_qÚtranspose_kÚtranspose_vÚpermutation_qÚpermutation_kÚpermutation_vÚinput_projection_nodesr+   Úadd_inÚ	matmul_inr{   r|   Únew_dims_nameÚnew_dimsÚreshape_q_namer   Útranspose_k_bnshr}   s                               r   rd   z5FusionMultiHeadAttentionSam2.fuse_sam_encoder_pattern1  s*  € ð< ”
×,Ò,ØØ6Ð6Ð6ØˆOˆOñ
ô 
ˆð
 ˆ=Ø”J×0Ò0ØØLÐLÐLØ%Ð%Ð%ñô ˆEð
 ˆ=Ø”J×0Ò0ØØ�Ø�ñô ˆEð
 ˆ=Ø�5à',¨R¤yÐ$Ø×@Ò@Ø(½3¸u¹:¼:Èº?¸?°a°aÐPTð Añ 
ô 
ˆð ÐØ�5àWcÑTˆ�] I¨{¸KÈõ "Ô4°[À&ÑIÔIˆÝ˜=­$Ñ/Ô/ð 	°MÀ\À\À\Ò4QÐ4QØ�5õ "Ô4°[À&ÑIÔIˆÝ˜=­$Ñ/Ô/ð 	°MÀ\À\À\Ò4QÐ4QØ�5õ "Ô4°[À&ÑIÔIˆÝ˜=­$Ñ/Ô/ð 	°MÀ\À\À\Ò4QÐ4QØ�5à!%¤×!=Ò!=ØØ(Ð(Ð(ØˆLˆLñ"
ô "
Ðð
 "Ð)Ø�5Ø(>Ñ%ˆ
�F˜IØ%)×%GÒ%GÈ
ÐTbÐdhÑ%iÔ%iÑ"ˆ�]Ø˜!ÒÐÝ�LŠLÐEÑFÔFÐFØ�5ð 3ˆØ”:×-Ò-¨mÑ<Ô<ˆØÐÝ#Ô.­r¬x¸
¸
¸
È'Ð/RÑ/RÔ/RÐYfÐgÑgÔgˆHØŒJ×&Ò& x°Ô1EÑFÔFÐFØœ×4Ò4°YÑ?Ô?ˆÝÔ$ØØÔ% aÔ(¨-Ð8Ø Ô& qÔ)¨FÑ2Ð3Øð	
ñ 
ô 
ˆ	ð 	Ô× Ò  Ñ+Ô+Ð+Ø7;Ô7KˆÔ$ Y¤^Ñ4ð 'ÐØ$/Ô$5°aÔ$8ÐÔ˜qÑ!Ø%0Ô%6°qÔ%9¸GÑ%CÐÔ Ñ"å�ŠÐB ;ÐBÐB°-ÐBÐBÑCÔCÐCð ×'Ò'ØØØØñ	
ô 
ˆð ÐØ�5õ �4”:×*Ò*¨=Ð:MÑNÔNÑOÔOÐSTÒTÐTÐTÐTØ'œ¨qÔ1ˆÔ˜!ÑàÔ× Ò  Ñ*Ô*Ð*Ø6:Ô6JˆÔ$ X¤]Ñ3ØÔ×#Ò# ] OÑ4Ô4Ð4ð  ˆÔØˆtr   c           	      ó  — | j                              |g d¢|ddddg¦  «        }|€dS |\  }}}}}| j                              |g d¢g d¢¦  «        }|€t                               d¦  «         dS |\  }	}}
}| j                              |ddgddg¦  «        }|�|\  }}nt                               d	¦  «         dS | j                              |g d
¢g d¢¦  «        }|€>| j                              |g d¢g d¢¦  «        }|€t                               d¦  «         dS |d         |
k    rdS |d         }| j                              |g d
¢g d¢¦  «        }|€t                               d¦  «         dS |d         |
k    rdS |\  }}}}|||
|||	fS )z%Match SDPA pattern in SAM2 enconder.*r€   Nr   )rƒ   ÚSqueezeÚSplitr‚   )r   r   r   r   zfailed to match v pathr„   r�   zfailed to match qk path)r…   rƒ   r¿   rÀ   r�   )	r…   rƒ   r‚   rƒ   ÚMaxPoolrƒ   r‚   r¿   rÀ   )	r   Nr   r   r   r   r   r   r   zfailed to match q pathr‹   r   )r   Nr   r   zfailed to match k pathrŒ   )r   rŽ   rž   Ú	out_nodesr�   r®   r¯   Úmatmul_qk_vr’   r³   r°   rr   r“   r”   r•   r–   r±   r™   Úmul_kr²   Ú
_squeeze_ks                        r   r¥   zAFusionMultiHeadAttentionSam2.match_sam_encoder_attention_subgraphµ  s  € ð ”J×0Ò0Ø(Ø?Ð?Ð?Ø˜$  a¨Ð+ñ
ô 
ˆ	ð ÐØ�4à:CÑ7ˆˆAˆ{˜M¨;ð ”*×.Ò.¨{Ð<hÐ<hÐ<hÐjvÐjvÐjvÑwÔwˆØˆ?Ý�LŠLÐ1Ñ2Ô2Ð2Ø�4Ø3:Ñ0ˆ�a˜ Kà”:×/Ò/°¸iÈÐ=RÐUVÐXYÐTZÑ[Ô[ˆØÐØ'/Ñ$ˆ[˜)˜)å�LŠLÐ2Ñ3Ô3Ð3Ø�4à”*×.Ò.¨yÐ:bÐ:bÐ:bÐdsÐdsÐdsÑtÔtˆØˆ?Ø”j×2Ò2ØØsÐsÐsØ.Ð.Ð.ñô ˆGð
 ˆÝ—’Ð5Ñ6Ô6Ð6Ø�tà�2Œ;˜)Ò#Ð#Ø�4Ø˜a”jˆà”*×.Ò.¨yÐ:bÐ:bÐ:bÐdsÐdsÐdsÑtÔtˆØˆ?Ý�LŠLÐ1Ñ2Ô2Ð2Ø�4à�2Œ;˜)Ò#Ð#Ø�4Ø.5Ñ+ˆ�˜Z¨à˜M¨9°kÀ;ÐP[Ð[Ð[r   r²   r³   c                 ó€  — | j                              d¦  «        }|j        d         |j        d         |j        d         g}|dz   }t          j        d||g|¬¦  «        }d|_        |j                             t          j        d|¦  «        g¦  «         d 	                    d¦  «        }	|  
                    |	¦  «         |S )	a  Create a MultiHeadAttention node for SAM2 encoder.

        Args:
            reshape_q (NodeProto): Reshape node for Q, output is 3D BxSxNH format
            transpose_k (NodeProto): Transpose node for K, output is BNSH format
            transpose_v (NodeProto): Transpose node for V, output is BNSH format
            num_heads (int): number of attention heads. If a model is pruned, it is the number of heads after pruning.

        Returns:
            NodeProto: the MultiHeadAttention node created.
        r   r   Ú_outrG   rK   r   rL   zself attention)r   rN   rE   r   rO   rP   rQ   rR   rS   rT   rU   )
r   r   r²   r³   r   r\   rH   rE   r^   r_   s
             r   rª   z,FusionMultiHeadAttentionSam2.create_mha_nodeì  sÕ   € ð& #œj×9Ò9Ð:NÑOÔOÐð Ô˜QÔØÔ˜qÔ!ØÔ˜qÔ!ð
ˆð % vÑ-ˆåÔ)Ø ØØ�HØ$ð	
ñ 
ô 
ˆð !0ˆÔØÔ ×'Ò'­Ô)>¸{ÈIÑ)VÔ)VÐ(WÑXÔXÐXà0×7Ò7Ð8HÑIÔIˆØ×Ò˜lÑ+Ô+Ð+ØÐr   )F)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r	   r(   r   r   r*   r3   r9   ÚboolÚtupler>   Ústrr`   r~   re   rd   r¥   rª   Ú__classcell__)r   s   @r   r   r      s  ø€ € € € € ðð ð(àð(ð ð(ð ð	(ð (ð (ð (ð (ð (ð¨yð ¸Sð ð ð ð ð*°	ð ¸cð ð ð ð ð8ð ð ð SXð"&ð "&Ø"ð"&Ø4=ð"&ØKOð"&à	ˆs�CˆxŒð"&ð "&ð "&ð "&ðH>àð>ð ð>ð ð	>ð
 ð>ð ð>ð ð>ð ð>ð ð>ð ð>ð 
�TÑ	ð>ð >ð >ð >ð@0 ð 0 ð 0 ðd5hð 5hð 5hðtBÐdhð Bð Bð Bð BðH5\ð 5\ð 5\ð 5\ðn)àð)ð ð)ð ð	)ð
 ð)ð 
ð)ð )ð )ð )ð )ð )ð )ð )r   r   )Úloggingr   Únumpyr$   Úfusion_baser   Úfusion_utilsr   Úonnxr   r   r   Ú
onnx_modelr	   rÈ   r<   r   © r   r   ú<module>r×      sÆ   ðð
 Ð Ð Ð Ð Ð à Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø $Ð $Ð $Ð $Ð $Ð $Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø  Ð  Ð  Ð  Ð  Ð  à	ˆ�8Ñ	Ô	€ðEð Eð Eð Eð E 6ñ Eô Eð Eð Eð Er   