tj UddlZddlZddlZddlZddlmZddlmZmZm Z ddl m Z ddl m Z mZmZmZddlZddlmZddlmZddlmZdd lmZd d lmZd d lmZmZd d lm Z d dl!m"Z"m#Z#d dl$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*gdZ+d Z,dZ-dZ.dZ/dZ0dZ1dZ2dZ3dZ4ej56e3e/Z7 ej56e2dZ8ede9d<GddZ:edGddZ;edGd d!Z<edGd"d#Z=edGd$d%e>Z?ede Gd&d'Z@edGd(d)eZAed d2d*ejBjCd+e e d,eeDd-eEd.eFeDe ff d/ZGGd0d1ZHdS)3N) defaultdict)CallableIterableSequence) dataclass)AnyIOLiteral NamedTuple)trace_structured) compatibility)map_arg)get_size_of_node) FxGraphDrawer)get_node_targetOperatorSupportBase) ShapeProp)!move_non_tensor_nodes_on_boundary split_by_tags)CALLABLE_NODE_OPSFxNetAccFusionsFinderis_node_output_tensorNodeListNodeSetTensors)FxNetAccNodesFinderFxNetSplitterInternalErrorSubgraph SplitResultgenerate_inputs_for_submodules NodeEventNodeEventTrackerF_fx_net_trackerz _nodes.txtz_all.txt FX_NET_ACC_SPLITTER_TRACKER_MODE%FX_NET_ACC_SPLITTER_TRACKER_DUMP_PATH)FX_NET_ACC_SPLITTER_TRACKER_TRACKED_NODES0)r(123 TRACKER_MODEc :eZdZeeeddfdedededededd f d Zd S) _SplitterSettingBaseFmin_acc_module_size skip_fusionallow_non_tensormax_acc_splitsrreturnNc(tj}|dddtd|dddtd|d d dd d |dddd d |dddd d|\}}|jr|jn||_|jr|jn||_|jr|jn||_||_|j r|j n||_ dS)Nz--min-acc-module-sizez--min_acc_module_sizeFz.Minimum size limit of an accelerator subgraph.)requiredtypehelpz--max-acc-splitsz--max_acc_splitsz,Enforce a maximum number of split subgraphs.z --skip-fusionz --skip_fusion store_truezIf true then no fusion groups. Fusion group is used to enforce no non-tensor data flow between submodules. If we don't have this constraint, setting this to false is recommended as it can reduce overhead.)defaultactionr8z--allow-non-tensorz--allow_non_tensoraFor some backends non-tensor data flow between cpu and them are not allowed. Therefore, if a node supported by accelerator but it has non-tensor inputs or outputs to a cpu node we would want to consider it as a cpu node during splitting. However, for some backends we might not care about non-tensor data flow and we can set this option to true to disable the functionality that prevent non-tensor data flow.z#--move-non-tensor-nodes-on-boundaryz#--move_non_tensor_nodes_on_boundarya=AOTI does not support non-tensor nodes on acc->acc, acc->gpu and gpu->acc boundary. For non-tensor nodes on acc->acc boundary and acc->gpu, we move the nodes from upstream to downstream. For non-tensor nodes on gpu->acc boundary, it is handled by the pre-split process. (by method reduce_acc_nodes_non_tensor_input). )r6r;r8) argparseArgumentParser add_argumentintparse_known_argsr0r1r2r3r) selfr0r1r2r3rparserargs_unknowns [/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/torch/fx/passes/splitter_base.py__init__z_SplitterSettingBase.__init__Ls(** # #A       ?       #    V    1 1>   0022h' %D $ $$  6:5E!V!1!1;%)%: PD ! !@P $25 3D 2 22 ...) __name__ __module__ __qualname__DEFAULT_MIN_ACC_MODULE_SIZEDEFAULT_SKIP_FUSIONDEFAULT_ALLOW_NON_TENSORr?boolrFrGrEr.r.Ks$?/!9 27 G G  G G  G  G ,0 G  G G G G G G rGr.)is_backward_compatiblec heZdZdZ d dejjdedejjdzddfdZdefdZ dS) r"z An event in graph split that happened on a node. source: Subject of the event desc: readable description dep: Optional dependency, usually the node that caused the event. Nsourcedescdepr4c0||_||_||_dSN)rRrSrT)rArRrSrTs rErFzNodeEvent.__init__s  rGcZ|jjd|jd|jr |jjndS)N:  #)rRnamerSrTrAs rEto_strzNodeEvent.to_strs7 +"VVdiVV482T$(--QTVVVrGrV) rHrIrJ__doc__torchfxNodestrrFr]rOrGrEr"r"sMQhm+.5:X]T5I WWWWWWWrGr"c eZdZdZdededdfdZ ddejj ded ejj dzddfd Z dd ede dede ege fdzddf dZdeeeeffdZdde ege fdzddfdZddZdS)r#z; Tracks node events during the splitter execution. tracker_mode dump_prefixr4NcV||_||_g|_i|_t|_dSrV)rdreevents node_eventsprintwriter)rArdres rErFzNodeEventTracker.__init__s,(&  rGnoderSrTc t|||}|j||j|jvrg|j|j<|j|jt |jdz dS)z1 Add a new event to the tracker. rN)r"rgappendr[rhlen)rArkrSrTevents rEaddzNodeEventTracker.adds} $c** 5!!! 9D, , ,*,D TY ' #**3t{+;+;a+?@@@@@rGF node_name recursivetabrjc |s|j}|j|gD]^}|j|}|||z|r-|j&||jjdd|z|_dS)a Print a node and its events. @param recursive: if True, print nodes that caused the events on this current node. @param tab: Indentation for dependencies. @param writer: function to write to file. If None, use print. NTz| rsrtrj)rjrhgetrgr]rT print_noder[)rArrrsrtrjidxros rErxzNodeEventTracker.print_nodes ![F#'' 266  CK$E F3' ( ( ( UY2INds 6    rGci}|jD]_}g||<|j|gD]<}|j|}|||=`|S)z1 Create dict dump on all events. )rhrwrgrmr])rAretr[ryros rEto_dictzNodeEventTracker.to_dicts%'$ 1 1DCI'++D"55 1 1 C(D   0000 1 rGc||s|j}|jD]*}|d|d||dd|+dS)zr Print all nodes in a list. @param writer: function to write to file. If None, use print. zNode: :Fz rvN)rjrhrx)rArjr[s rE print_allzNodeEventTracker.print_allsl  ![F$ L LD F#D### $ $ $ OODEtFO K K K K L LrGctddfddttdttgtffdjdkrSt jtzd 5} |d d d n #1swxYwYd ttdd ffd }jd ks jdkrsjd kr8tj tddn"djD}||d Sd S)z} Function to be invoked at the end of the finder execution to printout tracked events specified by the mode. artifactc dddS)N!fx_net_acc_splitter_finder_eventsjson)r[encodingrOrOrGrEz'NodeEventTracker.dump..s;"!!rGcPtjSrV)rdumpsr|r\srErz'NodeEventTracker.dump..stz$,,..99rG) metadata_fn payload_fnfr4c0dtdtffd }|S)Nxr4c4|dzS)N )write)rrs rEfnz6NodeEventTracker.dump..make_writer..fnswwq4x(((rG)rbr?)rrs` rE make_writerz*NodeEventTracker.dump..make_writers6 )c )c ) ) ) ) ) )IrGrwNnodesctjtzd5}|}|D]9}|d|d|dd||d|d: ddddS#1swxYwYdS)Nrz===== Tracking node z =====Tz|-rvz===== End of tracking node )openre NODES_SUFFIXrx)rrrjrrrrAs rEdump_selected_nodesz2NodeEventTracker.dump..dump_selected_nodess d&5s;; Lq$Q!&LLIFC)CCCDDDOOI4PVOWWWFJJJJKKKKL L L L L L L L L L L L L L L L L L LsAA55A9<A9rq,c>g|]\}}t|dk|S)r)rn).0r[rgs rE z)NodeEventTracker.dump..!s/)T6VWXDrG)r r rbrr?rdrre ALL_SUFFIXrlistosenvironrw-ENV_FX_NET_ACC_SPLITTER_TRACKER_TRACKED_NODESsplitrhitems)rArrrrs` @rEdumpzNodeEventTracker.dumps  :999      2c7 xs ';       ! !d&3S99 /Q{{1~~... / / / / / / / / / / / / / / / LtCy LT L L L L L L L   ! !T%6!%;%; $)) LbQQWW-1-=-C-C-E-E   & & & & &&<%;s1BB #B rV)FrqNr4N)rHrIrJr^r?rbrFr_r`rarprNrobjectrxdictrr|rrrOrGrEr#r#s{SstKO A AHM A), A388=43G A  A A A A 15   #'$.   . c49n-     L L# 7$ > L$ L L L L.'.'.'.'.'.'rGr#cjeZdZdZdejjdededdfdZ de ddfd Z d d Z d d Z defd ZdS)ra Finds a set of nodes that can be supported on ACC, excluding nodes that have non-tensor input/output to cpu nodes to prevent non-tensor data flow between backends and cpu. I.e. if we have a chain: ACC_NODE_1 -> ACC_NODE_2 -> ACC_NODE_3 -> CPU_NODE_1 where every ACC node produces non-tensor output, then they all should be treated as CPU nodes. This behavior can be turned off by passing allow_non_tensor=True. moduleoperator_supportr2r4Nc||_||_||_t|_t t tt|_ dSrV) rrr2set acc_nodesr#r?r, DUMP_PREFIXtracker)rArrr2s rErFzFxNetAccNodesFinder.__init__7sC  0 0"%%%'L(9(9;GG rG cpu_worklistcL|r|d}|jD]}||jvru|j||j|d|t |s0|j|d|||dSdS)a! Transitively excludes nodes from ACC supported set. For every node in the worklist: - removes its downstream ACC nodes from ACC supported set, - if any downstream ACC node produces non-tensor output, then it gets added into the worklist. rzacc_del|user_of_new_cpu_nodeznew_cpu_node|non_tensor_outputN)popusersrremoverrprrm)rArrkusers rE(reduce_acc_nodes_non_tensor_input_helperz))N))$///L$$T+I4PPP0662 ((/OPPP$++D111 2 2 2 2 2rGcg}|jjjD][}|jtvr||jvrt |r+|j|d| |\| |dS)z Excludes nodes from ACC supported set that have direct upstream CPU nodes that produce non-tensor outputs. z&new_cpu_node|callable_non_tensor_inputN) rgraphroprrrrrprmr)rAnon_tensor_cpu_nodesrks rE!reduce_acc_nodes_non_tensor_inputz5FxNetAccNodesFinder.reduce_acc_nodes_non_tensor_inputWs *,K%+ . .Dw///t~%%$T**  L  T#K L L L ' ' - - - - 556JKKKKKrGc: g}|jD]X}t|r|jD]>}||jvr3|||j|d|n?Y|sdS|D]}|j|||)z~ Excludes nodes from ACC supported set that produce non-tensor outputs and have downstream CPU nodes. Tz'acc_del|non_tensor_output_with_cpu_userN)rrrrmrrprr)rA new_cpu_nodesacc_noder new_cpu_nodes rE"reduce_acc_nodes_non_tensor_outputz6FxNetAccNodesFinder.reduce_acc_nodes_non_tensor_outputjs  I&(M N  (22$ND4>11%,,X666 (($&OQU 2!  - 4 4 %%l3333  9 9- H H H) IrGcft|j}t|_|jjjD]}|jtvr|j |d,|j ||s|j |dc|j |d|j ||j s(|||j |jS)Nzinit_cpu|not_callablezinit_cpu|operator_supportz(init_acc|callable_and_operator_supported)rr named_modulesrrrrrrrrpris_node_supportedr2rrr)rA submodulesns rE__call__zFxNetAccNodesFinder.__call__s$+335566 "( " "At,,,   $;<<<(:::qII    $?@@@ L  Q J K K K N  q ! ! ! !$ 6  2 2 4 4 4  3 3 5 5 5 ~rGr)rHrIrJr^r_r` GraphModulerrNrFrrrrrrrOrGrErr(s   H$ H. H H  H H H H2X2RV2222&LLLL&IIII6'rGrceZdZdS)rN)rHrIrJrOrGrErrsDrGrc8eZdZUeed<eed<dZedzed<dS)ris_accrNdevice_ordinal)rHrIrJrN__annotations__rrr?rOrGrErrs: LLL OOO!%NC$J%%%%%rGrcVeZdZUdZejjed<ee e fed<e ed<dS)r aB Stores the results of the splitter. Attributes: split_module: root module after splitting. submodule_inputs: a dict that maps submodule name to its inputs. non_acc_submodule_prefix: the prefix for non acc submodules. For acc submodule the prefix is always "_run_on_acc_". split_modulesubmodule_inputsnon_acc_submodule_prefixN) rHrIrJr^r_r`rrrrbrrOrGrEr r sO(&&&&38n$$$!!!!!!rGr modelinputstarget_submodulesdeepcopyr4c: g i d|D dtjjdtt dfddf fd }|D]P\}}||vrGt |tjjs( | |Qd fd } tj 5||dddn #1swxYwYn#t$r}||d}~wwxYw| S) a~ Generate inputs for targeting submodules in the given model. Note that if two submodules refer to the same obj, this function doesn't work. Args: model: root model. inputs: inputs to the root model. target_submodules: submodules that we want to generate inputs for. Returns: A dict that maps from submodule name to its inputs. ci|]\}}|| SrOrO)rr[mods rE z2generate_inputs_for_submodules..sKKK c#tKKKrGr module_inputs.r4NcJrtj|n||<dSrV)copyr)rrrresultssubmodule_to_namess rE pre_forwardz3generate_inputs_for_submodules..pre_forwards0,4 GDM- ( ( (- "6*+++rGc:D]}|dSrV)r)hhandless rEclean_up_handlesz8generate_inputs_for_submodules..clean_up_handless+  A HHJJJJ  rGr) rr_nnModuletupler isinstancejit ScriptModulermregister_forward_pre_hookno_grad Exception) rrrrrr[rrerrrs ` @@@rEr!r!s(G GKKU5H5H5J5JKKK EHO E#s(O PT        ((**KK c $ $ $c59#9:: Ks<<[IIJJJ ]__   E6NN                 Ns<C3C' C3'C++C3.C+/C33 D= D  DceZdZdZdZ d,dejjdee de d e d e d e d edzd dfdZd ee e ffdZd eejjeffdZd-dZd-dZdejjded ejjfdZdejjded e fdZdejjded dfdZd.de d e fdZd.de d e fdZ d/de dzd eejjeffdZ!deejjefd dfdZ"d e d efd!Z#d e d dfd"Z$d e%eeffd#Z&d e'e(fd$Z)d%e'e(d e'e(fd&Z*d%e'e(d dfd'Z+d.d(e d ejjfd)Z,d ejjfd*Z-d e.fd+Z/dS)0 _SplitterBasea Splits a GraphModule into sub-GraphModules for execution on CPU or the accelerator. Output is a GraphModule with supported and unsupported operators grouped into as few sub-GraphModules as possible. Assumes that only "call_module", "call_function" and "call_method" from FX IR can potentially be executed on the accelerator. Given the following graph: ==> b ==> // \ a d \ // ==> c ==> class SimpleModule(torch.nn.Module): def forward(self, a): b = torch.sin(a) c = torch.cos(a) d = b + c return d and providing "operator_support" that indicates that 'b' and 'c' can be executed on the accelerator, we will get the following split result: main: def forward(self, a): run_on_acc_0_0 = self._run_on_acc_0_0(a) getitem = run_on_acc_0_0[0] getitem_1 = run_on_acc_0_0[1] run_on_cpu_1_1 = self._run_on_cpu_1_1(getitem, getitem_1) return run_on_cpu_1_1 _run_on_acc_0_0: def forward(self, a): sin_1 = torch.sin(a) cos_1 = torch.cos(a) return (sin_1, cos_1) _run_on_cpu_1_1: def forward(self, sin_1, cos_1): add_1 = sin_1 + cos_1 return add_1 ld _run_on_cpu_FNr sample_inputrsettingsnon_acc_submodule_name return_tuple nodes_finderr4ct|tjjst dt |||_t|jj|||_ ||_ ||_ |%t|j|j |j j }||_|j jri|_n6t#||j|_|||_|||_i|_||_g|_dS)aP Preprocesses graph before splitting: - finds nodes supported by ACC, - finds fusion groups for ACC nodes having non-tensor IO, - builds a graph of direct dependencies, - builds a map of fused nodes to their fusions. As a result we get self.acc_nodes, self.deps and self.fusions. zExpected GraphModule, got N)rr_r`rAssertionErrorr7rr propagaterrrrr2rr1fusionsr_merge_overlapping_fusions find_depsdepsupdate_deps_for_fusionsr_node_submodule_map _return_tupletags)rArrrrrrrs rErFz_SplitterBase.__init__s2$&%("677 N !Ld6ll!L!LMM M ( $+(,77  0(  . T2DM4RL& = $ .DLLH0HHJJDL  + + - - -NN$$  $$&&&&<#35 )! rGc|jS)aReturns a map from node name to submodule name, e.g. node: main_module_impl_impl_over_arch_unary_multiple_embedding _pooling_embedding_pooling_sparse_entity_equivalence_key _proxy_embedding_bag maps to submodule name of: _run_on_acc_1 )rr\s rEget_node_submodule_mapz$_SplitterBase.get_node_submodule_mapLs ''rGctt}|jjjD]A}|jt vr|jD](}|jdkr|||)B|S)a  Builds a graph of node dependencies. Leaf nodes don't have any dependencies and the "output" node doesn't have nodes depending on it. Resulting graph has only direct dependencies, i.e. there are no transitive dependencies. output) rrrrrrrrrp)rArrkrs rErz_SplitterBase.find_depsUsz.9-=-=K%+ ) )Dw///  ) )7h&&JNN4((( ) rGc|jD]p}|j|}|D]^}|j||j||z |jD]&}||vr |j||'_qdS)z Updates graph of dependencies so that: - nodes from the same fusion depend on the same set of outer nodes, - outer nodes depending on a fusion depend on all nodes in that fusion. N)rrupdaterrp)rArkfusionfused_neighborrs rErz%_SplitterBase.update_deps_for_fusionsgs L 2 2D\$'F"( 2 2 $&&ty'@6'IJJJ*022D6)) $++D1112 2 2 2rGctjdddkrdS|jsdSi}|jD]}||t |<t |dkrdStt}| D]%\}}|D]}|| |&d|Ddtdtffd }d }|D]d}t |dkrO||d } tdt |D] } ||| } | | kr| | <d }!e|sdStt} | D])\}}| |||*| D]} | D] }| |j|< dS) a  Merge fusion groups that share nodes. FxNetAccFusionsFinder can produce overlapping fusion groups when a node is absorbed into multiple groups during expansion. When update_deps_for_fusions() later propagates deps, shared nodes act as bridges between groups, creating bidirectional dependency cycles that crash the splitter with "Subgraph can't be empty". This method detects overlapping groups via union-find and merges them into single groups before dep propagation. #_SPLITTER_MERGE_OVERLAPPING_FUSIONSr(r)Nrci|]}||SrOrO)rgids rErz<_SplitterBase._merge_overlapping_fusions..s!D!D!Ds#s!D!D!DrGrr4cj||kr%||<|}||k%|SrVrO)rparents rEfindz6_SplitterBase._merge_overlapping_fusions..findsA)q.."6!9-q 1I)q..HrGFrT)rrrwrvaluesidrnrrrrmr?rangerr)rA unique_groupsgroup node_to_gidsr rkr needs_mergegidsrootiothermerged merged_grouprs @rErz(_SplitterBase._merge_overlapping_fusionsvsU :>>? E E L L F|  F-/ \((** - -E',M"U)) $ $ }   " " F8C47H7H '--// / /JC / /T"))#.... /"E!Dm!D!D!D C C         '')) + +D4yy1}}tDG}}q#d)),,++A DaMME}}(,u &*   F&1%5%5'--// , ,JC 4499  $ $U + + + +#MMOO 2 2L$ 2 2%1 T"" 2 2 2rGrrc|S)z/ Lower the model to a backend. rOrArrs rE_lower_model_to_backendz%_SplitterBase._lower_model_to_backends  rGcdS)z When an error occurs during lowering or running the lowered mod, we use this function to find culprits in the `mod` that causes the error. zMUnable to find a culprit because _find_culprit() function is not implemented.rOrs rE _find_culpritz_SplitterBase._find_culprits _^rGsupported_nodescddddGfddt}||dd }|}|d dS) N AliceBlue chartreuse1crimson)r: supported unsupportedcReZdZdejjdeeefffd ZxZ S)E_SplitterBase._draw_graph_based_on_node_support..CustomDrawerrkr4ct|}|vr d|d<n%|jtvr d|d<n d|d<|S)Nr( fillcolorr)r:)super_get_node_stylerr)rArktemplate __class__ color_mapr#s rEr/zU_SplitterBase._draw_graph_based_on_node_support..CustomDrawer._get_node_stylesk 7722488?**,5k,BH[))W 111,5m,DH[)),5i,@H[)rG) rHrIrJr_r`rarrbr/ __classcell__)r1r2r#s@rE CustomDrawerr+sb EHM d38n rGr4 node_supportTignore_getattrznode_support.dot)rget_main_dot_graph write_raw)rArr#r4drawer dot_graphr2s ` @rE!_draw_graph_based_on_node_supportz/_SplitterBase._draw_graph_based_on_node_supports#&$  = c>$GGG--// ./////rG dump_graphc t|j}g}tt}tt}dt jjdt jdzfd|jj j D];}|j tvrt||}fd|jD}t|t!dt#t%|Dt|z } t'|d| } t'fd|jD} |j||r4||||| | f||| | f=|r||j|d} |D](\} }|D] \} } | | d| d t| d z } !)| d z } |D](\} }|D] \} } | | d| d t| d z } !)t7| | S) Nargr4cX|jd}t|ddS)N tensor_metadtype)metarwgetattr)r?rAs rE get_dtypez5_SplitterBase.node_support_preview..get_dtypes'(,,}55K;66 6rGchg|].}t|tjjr |nd/SrVrr_r`ra)rr?rEs rErz6_SplitterBase.node_support_preview..sH#-S%(-"@"@J #drGc3$K|] \}}||V dSrVrO)rrrBs rE z5_SplitterBase.node_support_preview..s8 5(((((rGc3vK|]3\}}t|tjj$||fV4dSrVrG)rkr?rEs rErIz5_SplitterBase.node_support_preview.. s\''Asc58=11'IIcNN#''''''rGz$ Supported node types in the model: z: (z, z) z& Unsupported node types in the model: )rrrrrr_r`rarBrrrrrrCrnnext enumeratereversedrkwargsrrrrmrpr<ri)rAr=rr#supported_node_typesunsupported_node_typesrktarget arg_dtypes last_indexarg_dtypes_tuplekwarg_dtypes_tuplereportstdtypesrEs @rEnode_support_previewz"_SplitterBase.node_support_previews%$+335566 $&*3//!,S!1!1 758= 7U[4-? 7 7 7 7K%+$ $ Dw///$Z66F9J Z4$-hz.B.B$C$C J ,,J %Z  %<== !&''''"k//11'''""  $66z4HH &&t,,,$V,002BDV1WXXXX&v.22%'9:  Q  2 24; P P P:-3355 V VIAv8> V V4 "4aUU$4UU=O8P8PUUUU V ==/5577 V VIAv8> V V4 "4aUU$4UU=O8P8PUUUU V grGc d|}td|D}t||z }dt|dz d|d|dz ||}td|D}t||z }d t|dz d|d|dz t|D];\}}|jrd |d n |j|d z t|jd z <|||d }| |rXt|dd }| } | D]\} } | | d|j} d} |jjD] }|jdkrd|jvrd|jdz t%||jdt&jjdt&jjdt,t.dt0t.dffd}|||j}t5j|d}ddz jjD]Q}|jdkr7t9|sd|jd z n|t=|dz }|jd!kr|}Rd"z d#t&jjddffd$ }tA|j!||jtE|z }d%|d&d'z d(|d)z || kr |} |j} |#|}n/#tH$r"d*z |%|z YwxYw ||d+z #tH$r"d,z |%|z YwxYwd-| d'z d.| d/z tMS)0Nrqc g|] }|j | SrOrrgs rErz/_SplitterBase.split_preview.., B B Bq B B B BrGz+Before removing small acc subgraphs, total z subgraphs are created:rY acc subgraphs and z cpu subgraphs. c g|] }|j | SrOr]r^s rErz/_SplitterBase.split_preview..2r`rGz*After removing small acc subgraphs, total _run_on_acc_rXz node(s) T) remove_tagpreviewr6z.dot call_moduleaccz Processing acc submodule rmain_modsubmodexample_inputsr4.cddtjjdttdfddffd }||}|||tdS)NrAr.r4c |dSrVrO)rAr sub_inputss rE get_inputszJ_SplitterBase.split_preview..get_submod_inputs..get_inputsZs&, rGz'Forward pre-hook did not capture inputs)r_rrrrrrr)rhrirjrnhandlerms @rEget_submod_inputsz6_SplitterBase.split_preview..get_submod_inputsSs :>J,#ho,7.get_bytesysS166P#vTY#v#v#vv*.>vt.L.LQ.OO***rGzTotal input size in bytes is z , total output size in bytes is rzF theoretical max qps (bounds by PCIe bandwidth) for this submodule is z. z#Run into an error during lowering! zLowering and running succeed! z$Run into an error during inference! zB Theoretical max qps (bounds by PCIe bandwidth) for this model is z bottleneck is submodule .)'put_nodes_into_subgraphsrnremove_small_acc_subgraphsrMrrrtagrevalrget_all_dot_graphsrr9PCIe_BWrrrRrDr_r`rrrrrrrrr[rrarrCmaxr  RuntimeErrorr"ri)rAr= subgraphsacc_subgraphs_numcpu_subgraphs_numrsubgraph split_modr: dot_graphsr[r;max_qpsbottleneck_modulerkrp submod_inputstotal_input_bytesr output_noderuqpslowered_submodrWrirts @@@rE split_previewz_SplitterBase.split_preview)s1133  B BI B B BCC NN->>hYhhhha(aa=Naaaa33I>>  B BI B B BCC NN->>gIgggga(aa=Naaaa$Y// : :KAx ?;$q$$$$3:Q::: G #hn--999 9GG JJ$J//   3"9iMMMF2244J#-#3#3#5#5 3 3i##tMMM2222O)J AJ ADw-''ET[,@,@HHHHH DK88&#h2&!H0&%-SM&38_ &&&&(!2 1)VTEV W W + &!!+];;$%!%&"11+((At},,4Q77P#'v'v'v'vvGG-1A&!1L1LQ1OO-tx''&' 22PEHMPdPPPPPPPP ()444lS):c$(..*=*=b*A&B&BV&K&K4L$$T***  + rGrct}|jD]\}}||vr t}|D]}||||||D]E}|||<|jD]!}||vr|||"||FdSrV)rrrrdifference_updateall_input_nodesrp)rArprocessed_noderkrnew_deprr?s rEupdate_reverse_deps_for_fusionsz-_SplitterBase.update_reverse_deps_for_fusionss L..00 & &LD&~%%eeG ( (tAw''''  % %f - - - & &!Q,11C&((S ((000""1%%%% & & &rGryct}|jjjD]S}|jt vrC|j|kr8|jD]0}|jt vr |j|kr||1T|S)z Finds parent nodes of the `tag` subgraph. Traverse the inputs of nodes in the subgraph, if input doesn't belong to the subgraph and is not a placeholder, we consider it as the parent node of the subgraph. ) rrrrrrryrrp)rAry parent_nodesrkr?s rEfind_parent_nodes_of_subgraphz+_SplitterBase.find_parent_nodes_of_subgraphsuu K%+ . .Dw+++C/..Cv!222sw#~~$((---rGcn|t|ddd}||||}t }|rd}|D]}|||kr ||jvr|}n|dS||_||| |||j vr)|j |D]}||vr| ||j D])}|j tvr||vr| |*|dSdS)z^ Extend the acc subgraph with `tag` going the reversed topological direction. rr)maxsplitr/)rN)rr?rsplitrrrrryrrprrrr) rAryrr visited_nodesrkr fusion_noder?s rEextend_acc_subgraphz!_SplitterBase.extend_acc_subgraphs %%SC!1L1LR1P-Q-Q%RR ,,T22299#>> !$  *D"  7m++T^0C0CDE|DH    % % %   d # # #t|###'<#566K"-77$((555+ * *6...3m3K3K $$S)))5 * * * * *rGct}t}|jjjD]}|jdkrLt |jdkr4||jvr||n|||jdvrc|j D]6}||jvr||!||7||fS)zK Finds nodes that consume module inputs or get_attr nodes. call_functionr>get_attrrq) rrrrrrnrrrpr)rAstarter_cpu_nodesstarter_acc_nodesrkrs rE starter_nodesz_SplitterBase.starter_nodess&)UU%(UUK%+ 0 0Dw/))c$2F.G.G1.L.L4>))%))$////%))$///w999  0 04>))%))$////%))$////  0 !"333rGc \}}t tfd|D }g}g}|s|rK|r|n|}t fd|Dd}|;|st d|t ||| }g}e|| ||||j vrP|j vr$| j | z n#| j | z |j D]E}|j tvr|j vr||0||F|H|K|r$|t |||st d|S)Nc3TK|]"}tj|dkV#dS)rN)rnr)rrrAs rErIz9_SplitterBase.put_nodes_into_subgraphs..6s6$W$WS1%6%6!%;$W$W$W$W$W$WrGc3>K|]}j|k|VdSrV)r)rrrArs rErIz9_SplitterBase.put_nodes_into_subgraphs..@s3KKqTYq\]-J-J-J-J-J-JKKrGzSubgraph can't be empty)rrzCouldn't create subgraphs)rranyrLrrmrrrprrrrrr) rAcurrent_cpu_nodescurrent_acc_nodes acc_subgraphcurrent_subgraph_nodesr current_nodesrkrrs ` @rErwz&_SplitterBase.put_nodes_into_subgraphs/se/3/A/A/C/C,,!$ "%$W$W$W$WEV$W$W$W!W!WW +-%' ( 0#4( 01=T--CTMKKKKKMKKKD |-P45NOOO  L8NOOO$0/ )+&   & & &   d # # # " ) )$ / / /t|##4>))%,,T\$-?--OPPPP%,,T\$-?--OPPP  0 07"3334>))%))$////%))$////Q ( 0#4( 0V "     4JKKK    J,-HII IrGrcg}|D]}|jrt|j|jjkr||Bt dt|jd|jj|r&|dj|jd|_|||r3|djs&|dj|j|||S)z This pass finds ACC submodules with less than specified size and merges them with adjacent CPU submodules. zBEliminating acc subgraph because it's smaller than the threshold: z < r/F)rrnrrr0rmriextend)rArrrs rErxz(_SplitterBase.remove_small_acc_subgraphsqs, "$! , ,H ,x~&&$-*KKKMM(++++Wx~..WW37=3TWW0r (//????*/ h////,&*"3,2J$++HN;;;;MM(++++ rGcNg|_|D]}|jrdt|jn|jt|j}|j||jD];}t |drtd|d||_||j |j <.s"D"D"D18"D1"D"D"DrGzGot raz non-acc subgraphs)rwrrrxrnriryr)rAracc_subgraphs_countnon_acc_subgraphs_counts rErz_SplitterBase.__call__s1133 = : 9 -i 8 8 833I>> !"D"Di"D"D"DEE"%i..3F"F f& f f;R f f f    zz||rGcH|}g}|D]\}}|||jjdkr,t ||jjkrt dt ||j|}t|||j S)NrziCannot fulfill max_acc_splits limit. This may cause split fragmentation and result in performance issues.) named_childrenrmrr3rn ValueErrorr!rr r)rArsubmodule_namesr[_modrs rEgenerate_split_resultsz$_SplitterBase.generate_split_resultsstvv %'&5577 ) )JD$  " "4 ( ( ( ( M (1 , ,O$$t}'CCC0  : $+_  <)94;VWWWrG)rFNrFrV)0rHrIrJr^r|r_r`rrrrr.rbrNrrFrrrarrrrrrrr r"rr<rZrr?rrrrrrrrrwrxryrrr rrOrGrErrs((VG'5"37/"/"$/"sm/". /" ' /" !$ /"/"*D0/" /"/"/"/"j(S#X((((4 w 67$ 2 2 2 2?2?2?2?2J8'18 _!5_w_3____08'0:B0 00004AAtAAAAAFuuuuuuux$(Dj ehmW$ %,&/0& &&&&:"(*s(*t(*(*(*(*\4uWg%5644442@$x.@@@@DDNtH~6:T(^::::: 1E %(.    X XXXXXXrGrr)Ir<rrr collectionsrcollections.abcrrr dataclassesrtypingrr r r r_torch._loggingr torch.fx._compatibilityr torch.fx.noder"torch.fx.passes.graph_manipulationr graph_drawerrrrr shape_propr split_utilsrr tools_commonrrrrrr__all__rKrLrMTRACKER_DUMP_PATHrr$ENV_FX_NET_ACC_SPLITTER_TRACKER_MODE)ENV_FX_NET_ACC_SPLITTER_TRACKER_DUMP_PATHrrrwrr,rr.r"r#rrrrr rrrbrNrr!rrOrGrErs ######8888888888!!!!!!//////////// ++++++111111!!!!!!??????''''''BBBBBBBB!!!!!!IIIIIIII     &  'I$,S)/.jnn-/@ -/JNN(#-- g() H H H H H H H H Ve,,,WWWWWWW-,W.e,,,v'v'v'v'v'v'v'-,v're,,,nnnnnnn-,nbe,,,       -, e,,, &&&&&&& -,& e,,, " " " " "* " "-, " e,,,  -- 8?- SM- }- -  #s(^ ----,-`\ X\ X\ X\ X\ X\ X\ X\ X\ X\ XrG