ktjddlmZddlmZddlmZddlmZmZm Z ddl m Z ee Z GddeZGdd eZd S) ) getLogger)Fusion) FusionUtils) NodeProto TensorProtohelper) OnnxModelc(eZdZdZd"dedeffd Zdeddeeefzfd Z d ed e ee efd e de fd Z dZdZdZdZdZdedeedezffdZ d#ded edededdezdedzf dZdZdZ d$dZdZd Zd!ZxZS)%FusionEmbedLayerNoMaskz Fuse embedding layer into one node (EmbedLayerNormalization). It supports the following model types: BERT, DistilBert, ALBert. no maskmodel descriptionct|dddg|t||_d|_d|_d|_d|_dS)NEmbedLayerNormalizationLayerNormalizationSkipLayerNormalizationF)super__init__rutils shape_infershape_infer_done attention embed_node)selfr r __class__s h/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/onnxruntime/transformers/fusion_embedlayer.pyrzFusionEmbedLayerNoMask.__init__sf   % !#; <     !''  %addreturnNc|j|dgdg}|dS|j|dgdg}|dS|d|dfS)NGatherr)r match_parent_path)rr gather_0_path gather_1_paths rmatch_two_gatherz'FusionEmbedLayerNoMask.match_two_gather%sg 44S8*qcJJ  4 44S8*qcJJ  4Qq!111r layernorminput_name_to_nodesis_distil_bertc$|j|d|d|_|jdS|jd|vrdS||jd}t d|D}|gdkrd|D]a}|jd krT|j|gd gd }|2|d jd|jdkr|d |_dSbt|dkr|djdkr|djd|vr||djd}t|dkrr|djdkra|djd|vrL||djd} | D]}|jdkr ||_dSt d| D}|r5|gdkr,|gdkr$|gdkrt ddSn,|gdkr$|gdkrt ddSdS)aCheck that LayerNormalization has a child of Attention node or subgraph like Attention. Args: layernorm (NodeProto): LayerNormalization node input_name_to_nodes (Dict[str, List[NodeProto]]): map from input name to nodes is_distil_bert (bool): whether it is DistilBert or not Returns: bool: whether there is Attention node or subgraph like Attention AttentionF) recursiveNTrcg|] }|j Sop_type.0childs r zCFusionEmbedLayerNoMask.check_attention_subgraph..Js E E E5 E E Er)MatMulr5r5rr)Addr5MultiHeadAttentionr5)NNrrr"r5r6cg|] }|j Sr.r/r1s rr4zCFusionEmbedLayerNoMask.check_attention_subgraph..gs(J(J(J5(J(J(Jr)r5r5r5Shaper)r6r5r5r5r;r;)r6r5r5r5r;z %4  A &9 9 95&y'7':; E EH E E EFF UUU U U $ $<#;;; J88III***E (U2Y_Q-?9CSTUCV-V-V/4Qx,#tt x==A  (1+"5"A"AhqkFXYZF[_rFrFr/ 0B10EFMM""a''!!$,55!!$+A.2EEE+M!,<,CA,FG!$$D|{22)-#tt3"((J(JE(J(J(J!K!K  "c"c"ccc"&]&]&]]]"&T&T&TTT [\\\u""" !%%%  [\\\utrcB|j|ddgddg}|$|j|gdgd}|dS|d|d }}|jd|krdS|j|gd gd fgd gd fg|\}}}|dS|d} |j| ddr|j| ddsdS|d} |j| ddsdS|d } | jd|krdSdS)az Match position embedding path from input_ids to Gather for DistilBert. Pattern is like the following: (input_ids) | Shape | | Gather (indices=1) | | | Cast (optional) | | | Range (start=0, end=*, delta=1) | | | Unsqueeze | / Expand | Gather Expandr;r"N)rLWhereReshaper;)r"r"r9rFrr8) UnsqueezeRangeCastr!r;)rrr"rr)rOrPr!r;)rrr"rr9T)r r#r?match_parent_pathsrcheck_node_input_value) rposition_embedding_gather input_idsoutput_name_to_noderGexpandshape_path2 range_node gather_node shape_nodes r#match_position_embedding_distilbertz:FusionEmbedLayerNoMask.match_position_embedding_distilberts* ,,-FSZH[^_ab]cdd =J00)777 E }ua%) ;q>Y & &5j33 BBBOOOT:::LLLI    5! =51X J - -j!Q ? ? DHJDeDefprsuvDwDw 5Bi  11+q!DD 52Y  A ) + +5trcdS)aYMatch position embedding path from input_ids to Gather for Roberta. Roberta Embedding Layer Pattern (* is optional since it might be removed by ORT, ? is the padding word id): (input_ids) --> Equal(B=?) -- Not -- Cast(to=6) -- CumSum(axis=1) -- Mul -- Cast(to=7) -- Add(B=1) -- Cast(to=7)* --> Gather | ^ V | +------------------------------+ Roberta new pattern from transformers v4.9: (input_ids) --> Equal(B=?) -- Not -- Cast(to=6) -- CumSum(axis=1) -- Add(B=0) -- Mul -- Cast(to=7) -- Add(B=1) --> Gather | ^ V | +-------------------------------------------+ start_node = position_embedding_gather start_index = 1 # match optional Cast node. parent = self.model.get_parent(start_node, start_index, output_name_to_node) if parent is None: return if parent.op_type == "Cast": if OnnxModel.get_node_attribute(parent, "to") != 7: return start_node = parent start_index = 0 i, path, return_indices = self.model.match_parent_paths( start_node, [ (['Add', 'Cast', 'Mul', 'CumSum', 'Cast', 'Not', 'Equal'], [start_index, 0, 0, 0, 0, 0, 0]), (['Add', 'Cast', 'Mul', 'Add', 'CumSum', 'Cast', 'Not', 'Equal'], [start_index, 0, 0, 0, 0, 0, 0, 0])], output_name_to_node) if path is not None: # constant input of Add shall be 1. i, value = self.model.get_constant_input(path[0]) if value != 1: return False _, self.padding_word_id = self.model.get_constant_input(path[-1]) return input_ids == path[-1].input[0] Fr.rrUrVrWs r match_position_embedding_robertaz7FusionEmbedLayerNoMask.match_position_embedding_robertas Zurc*|j|ddgddg|}|dS|\}}|j|jd}|t |jdkr|jddkro|j|ddgrR|j|ddgr5t |jd ks|j|d dgsdS|j}|d krtj |d dgsdSn|j|ddgsdS|j |d|} | dS| j d kr;|j| ddsdS|j | d|} n| } | | j d krdS|j| ddsdS|j | d|} | | j dkrdS|| jdkS)a  Match position embedding path from input_ids to Gather for BERT. BERT Embedding Layer Pattern: (input_ids) / / Shape / | / Gather (indices=1) / | / Add (optional, B=0) / | Gather (segment_ids) Unsqueeze (axes=0) \ | | \ Gather Slice (data[1,512], starts=0, ends=*, axes=1, steps=1) \ / | Add Gather \ / Add | LayerNormalization SlicerOr"r9NFr axesr6r!r;) r r#get_constant_valuer?rArYrrTget_opset_versionrcheck_node_attribute get_parentr0) rrUrVrWpathslice unsqueeze slice_weight opset_versionrFgatherrYs rmatch_position_embedding_bertz4FusionEmbedLayerNoMask.match_position_embedding_bertsW,z++ % k " F     <5yz44U[^DD  $L&''1,,"1%** 11%QC@@+ 11%QC@@+U[!!Q&&$**K*KESTWXVY*Z*Z&5 4466 2  3IvsKK u :44YA3GG uz$$Y3FGG <5 <5 :44T1a@@ uZ**44GHHFFF >V^x775 11&!Q?? 5 %%fa1DEE =EMW445EKN**rcj||||rdS||||rdSdS)NTF)rsr_ras rmatch_position_embeddingz/FusionEmbedLayerNoMask.match_position_embedding9sK  - -.GTg h h 4  3 34MyZm n n 4urc |jd}|r |jdnd}|jd}|js'|jd|_d|_|j|j|}|j|}|r|sJt |dkr%t |dkr|d|dks"td|d|dS|rU|j ||s:td |d |j|dS|j |jd } | t | j dkrtd dS|j |jd } | 4t | j dks| j d| j dkrtd dS|rw|j |jd } | 4t | j dks| j d| j dkrtddS| j d | j d krRt d|jd d| j d d|jd d| j d |r| j d | j d krRt d|jd d| j d d|jd d| j d | j d | j d krRt d|jd d| j d d|jd d| j d dS)zXSanity check of embedding weights, and match hidden_size of weights and shape of inputs.r"NT)updater9z^Cannot fuse EmbedLayerNormalization: input_ids and position_ids not matched in 2nd dimension: z vs FzYCannot fuse EmbedLayerNormalization: input_ids and segment_ids does not have same shape: z != rzICannot fuse EmbedLayerNormalization: word embedding table is not expectedzMCannot fuse EmbedLayerNormalization: position embedding table is not expectedzLCannot fuse EmbedLayerNormalization: segment embedding table is not expectedzword_embedding_table (z) size z <= position_embedding_table (z <= segment_embedding_table (zposition_embedding_table () r?rr infer_runtime_shaperget_edge_shaperArBinfo compare_shaperirYwarning) rword_embedding_gathersegment_embedding_gatherrUrV segment_ids position_idsinput_ids_shapeposition_ids_shapeword_embedding_tableposition_embedding_tablesegment_embedding_tables rcheck_embeddingz&FusionEmbedLayerNoMask.check_embeddingGs>)/2 ;S].4Q77Y] 06q9 $ )#z==T=JJD $(D !   '".==iHHO!%!1!@!@!N!N " 9'9 9 99O$$))*++q00#A&*)DQ)G\\PdPjklPm\\NgNmnoNp\\yQyWXYyZ\\     #)!,0G0Ma0PPP]-B-H-K]]ThTnopTq]]QiQopqQr]]{R{XYZ{[]](-a04K4QRS4TTTi1J1PQR1Sii\t\z{|\}ii]u]{|}]~iiG^GdefGgiitr input_namecd}|j|}|@|jjjt jkr|j|\}}n |}n|j|\}}||fS)aCast a graph input or node input to int32. Args: input_name (str): name of graph input or node input Returns: A tuple of casted input name and the cast node. int32_output (str): If input is int32, it is the input name, Otherwise it is output name of Cast node. input_cast_node (Union[None, NodeProto]): Cast node. It could be None if input is int32. N) r find_graph_inputtype tensor_type elem_typerINT32rcast_input_to_int32)rrinput_cast_node graph_input int32_outputs r cast_to_int32z$FusionEmbedLayerNoMask.cast_to_int32sj11*==  "+59JJJ04 0N0Nz0Z0Z- oo) ,0J,J,J:,V,V )L/_,,rFrVr}rUr~rc tg} ||\}} |jd} |jdkr|jd} |jd} n|jd} |jd} d}|N||jd\}} |||jd|jd|jd| | g}n|d|jd|jdd| | g}|B|d||\}} ||| d z| d zg}|r||n| d z}||t jd||| }d |_|j D](}|j dkr|j |g)t|j dkr.|j t j ddg| || D]}|j|j|j <|j | ||_|S)agCreate an EmbedLayerNormalization node. Note that segment embedding is optional. Args: input_ids (str): input_ids for word embeddings layernorm (NodeProto): LayerNormalization or SkipLayerNormalization node. word_embedding_gather (NodeProto): the Gather node for word embedding position_embedding_gather (NodeProto): the Gather node for position embedding segment_embedding_gather (Union[None, NodeProto]): the Gather node for segment embedding, or None. Returns: NodeProto: the EmbedLayerNormalization node created. rrr"r9reNr_output_dummy_mask_index_embedding_sum)outputsnamez com.microsoftepsilong-q=)rr create_node_namer0r?appendr make_nodedomain attributerextendrAmake_attributethis_graph_namenode_name_to_graph_name nodes_to_addr)rrVr'r}rUr~rembedding_sum_outputembedding_sum_namerrZ node_namegammabetaembed_node_inputsrembed_node_outputsrrattrFs rcreate_fused_nodez(FusionEmbedLayerNoMask.create_fused_nodes. )))44 1J//0IJJ   4 4 4OA&E?1%DDOA&E?1%D  # /!//0H0Nq0QRRNK%+A.)/2(.q1!  %+A.)/2!   #  $ $R ( ( ("00>>OL!  $ $\ 2 2 2')3YAT5TU  ,);)G%%YYiMiD  % %d + + +% % &    , & 3 3Cx9$$$++SE222 z# $ $ ) )  ' ')>y')R)R(S T T T J'''  K KD6:6JD ( 3 3   ...$rcv|j|jd|jdd|_dS)NrT)r replace_input_of_all_nodesr= prune_graph)rr'rs r finish_fusionz$FusionEmbedLayerNoMask.finish_fusion s9 --i.>q.A:CTUVCWXXXrc|jdko5t|jdkot|jddkS)Nrrer)r0rAr=)rrFs r"is_skip_layer_norm_with_sum_outputz9FusionEmbedLayerNoMask.is_skip_layer_norm_with_sum_outputsC 88nc$+>N>NQR>RnWZ[_[fgh[iWjWjmnWnnrc r||}|dS|\}}|jd} |jd} |||dsdS||d|sdS|jdkrK||} d} |} | r |jdnd}|duo|j|du}n|} |jdkrdnd} t|j| kr |j| nd}|duo|j|du}|o||vot||dk}|duo|jdkp|p|} | | ||||| | |r|nd}| r2d | j| <|s&|j ||jd | ||d S) NFr"r)rrer6r)rr_no_use__to_be_removed_r9T) r&r?rJrr0rr=r find_graph_outputrArrr)rr'add_before_layernormr(rWoptional_segment_gather two_gatherr}rUrVrneed_embedding_sum_outputsum_output_indexnode_with_sum_output sum_outputis_sum_graph_outputis_sum_used_by_multiple_nodesrs r fuse_gpt2z FusionEmbedLayerNoMask.fuse_gpt2sn(**+?@@  5;E88)/2 06q9 ,,Y8K\a,bb 5##$94AZ[[ 5   8 8 8(,(O(OPY(Z(Z % #, 0IS)!,,tJ#-T#9"u @\@\]g@h@hpt@t  #7 $8$@E$I$Iqqq +2336FFF%+,<==  $.T#9"u @\@\]g@h@hpt@t o .A AosK^_iKjGkGknoGo **44)?)$,5m9LmPm & ++   ! % # !:-@Jzzd,    % X">?SV^U_bcad"e"e " *5$;A$>!,,-F Sfgg -001I9Vijj u+D'@ $(, %##$9;SUnoo 5++   ! % $    9j111trc4|j|dgdg}|jdkr|dS|d}d}n|j|dgdg}|j|dgdg}|||dS|d}|d}n;|5|3|j|dgdg}|dS|d}|d}n|}d}||||||rdS|||||rdS|||||rdSdS)Nr6rrr!r")r r#r0rrr) rrFr(rWfirst_add_pathrrr$r%s rfusezFusionEmbedLayerNoMask.fuses55dUGaSII </ / /%#1!#4 &* # # J88zA3OOM J88zA3OOM$)B!)F'5a'8$*7*:''*}/D!%!=!=dUGaS!Q!Q!)F'5a'8$*7*:'''+$*.' >> &(;=PRi    F   &:>$ 46IK^ _ _  F  r)r )NFN)N)__name__ __module__ __qualname____doc__r strrrtupler&dictlistboolrJr_rbrsrurrrrrrrrr __classcell__rs@rr r sE  i c       2I 2$y)?S9T2T 2 2 2 2RR"#tI"67R R  RRRRh<<<|---^F+F+F+P   HHHT--c4);K6K0L----<$("```` ) ` $- ` #'"2 `Dj````D   ooorvOOOOb'''R000d"""""""rr c6eZdZddeffd ZdZfdZxZS)FusionEmbedLayerNormalizationFr cZt|d||_dS)Nz with mask)rruse_mask_index)rr rrs rrz&FusionEmbedLayerNormalization.__init__s+  ,,,,rcj|j}t|jdkr;|j|td|jnrt|jdkr8|jds+||jd<td|jn"td|jdS|D]c}td|j|jdkr|jd|jd<C|jd kr|jd|jd <ddS) Nzappend mask to %szreplace mask in %szskip mask in %szupdate mask_index in %sr+r"rer7rf) rrAr?rrBrCrr0r=)r mask_int32attention_nodesrattention_nodes r replace_maskz*FusionEmbedLayerNormalization.replace_masks4_ z A % %   # #J / / / LL,jo > > > > ! " "Q & &z/?/B &",J Q  LL-z ? ? ? ? LL*JO < < < F- ? ?N LL2N4G H H H%44*4*;A*>$Q'''+???*4*;A*>$Q'  ? ?rc*d|_d|_d|_t||||jdS|js1t d|ddS|j8|j1t d|ddS|jr|jj d}n|jj d}||}|j |r9d|D}| |||ddS||vr2t d||ddS||}|j d vrd |D}|j d krG|j d }t|t|kr|j|| |||ddSdS) NzG--use_mask_index is not set: EmbedLayerNormalization will not have maskz EmbedLayerNormalization(no mask)zLEmbedLayerNormalization will not have mask since attention node is not foundrerfc$g|] }|jdv |S)r+r7r/r2rFs rr4z6FusionEmbedLayerNormalization.fuse..%vvv$,RuBuButBuBuBurz"EmbedLayerNormalization(with mask)zHEmbedLayerNormalization will not have mask since %s is not a node output) ReduceSumrQc$g|] }|jdv |Srr/rs rr4z6FusionEmbedLayerNormalization.fuse..$rrrr)rr@rrrrrBrCincrease_counterr?r rrr0rAnodes_to_remover)rrFr(rWrchildren_nodesrrs rrz"FusionEmbedLayerNormalization.fuses1#  T.0CDDD ? " F"  LLb c c c  ! !"D E E E F > !d&:&B LLg h h h  ! !"D E E E F > 7-a0JJ-3A6J,Z8 : & &z 2 2 vvvvvO   j/ : : :  ! !"F G G G F 0 0 0 LLceo p p p  ! !"D E E E F":. <0 0 0vvvvvO|{**!Z] ~&&#o*>*>>>(//555   j/ : : :  ! !"F G G G G G 1 0r)F)rrrr rrrrrs@rrrsz--i------???*-H-H-H-H-H-H-H-H-HrrN)loggingr fusion_baser fusion_utilsronnxrrr onnx_modelr rrBr rr.rrrs $$$$$$////////// 8  P P P P P VP P P fGHGHGHGHGH$:GHGHGHGHGHr