tjoUddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZmZddlmZddlmZmZddlmZmZddlZddlZddlZddlmZddlmcm Z!ddl"m#Z#m$Z$ddl%m&Z&dd l'm(Z(dd l)m*Z+dd l,m-Z-m.Z.dd l/m0Z0dd l1m2Z2ddl3m4Z4m5Z5ddl6m7Z7ddl8m9Z9ddl:m;Z;ddlZ>ddl?m@Z@mAZAddlBmCZCmDZDmEZEmFZFmGZGmHZHddlImJZJddlKmLZLddlMmNZNddlOmPZPdd lQm*Z*ddlRmSZSddlTmUZUmVZVmWZWmXZXddlYmZZZddl[m\Z\m]Z]m^Z^ddl_m`Z`ddlambZbdd lcmdZddd!lemfZfmgZgmhZhmiZimjZjmkZkmlZldd"lmmnZnmoZompZperddlqZrddlsZse*jtZud#evd$<ejwexZyd%evd&<ejzj{Z{ejzj|Z|eGd'd(Z}eGd)d*Z~eGd+d,Zdd0Zdd1Zdd4Zdd5Zdd7ZGd8d9Zdd:ZeZdd=Zdd>ZddAZ dddMZddNZddOZddPZddQZddRZddSZddTZddUZddYZdd^ZddaZ dddkZddqZddtZddvZddwZddxZdd{Zddd|Zdd}Z dddZ dddZ dddBdBdddZdddddZedZd dZddZd dZejd dZd dZd dZddZddZddZddZddZddZddZ dddZddZddZddÄZddĄZddńZdd̄ZddlmZddЄZddфZ dddӄZddՄZddքZdd؄ZddۄZÐd d݄Z d!ddߜd"dZ d#d$dZdS(%) annotationsN) defaultdictdeque)Callable) dataclassreplace)Any TYPE_CHECKING)countersis_node_meta_valid)(create_structured_trace_for_min_cut_info)is_with_effects)config)CustomKnapsackSolverCustomRuntimeEstimator)FakeScriptObject) is_builtin) LazyStringtrace_structured) trace_log)extract_tensor_metadata) BackwardState) is_sym_node py_sym_types) magic_methodsmethod_to_operator)find_symbol_binding_fx_nodes free_symbolsis_symbol_binding_fx_nodeoptimization_hintstatically_known_falsestatically_known_true) graph_drawer)_get_memory_budget_annotation) OrderedSet)CheckpointPolicy)GraphInfoProvider) dp_knapsackdp_knapsack_sliding_hirschberggreedy_knapsack ilp_knapsack)KnapsackEvaluator) AOTOutputSavedForBackwardsAOTOutput#SavedForBackwardsNoVcCheckAOTOutput)_is_functional_graph)is_opaque_node)get_aot_graph_name)_is_bwd_seed_offset_is_fwd_seed_offset _is_primal _is_tangentget_default_generatorget_device_rng_statesupports_graphsafe_rng) fx_graph_cseget_aten_targetraise_getitemsboolAOT_PARTITIONER_DEBUGzlogging.LoggerlogcneZdZUdZded<ded<ded<ded<ded<dd Zdd ZddZddZddZ dS)OpTypesz8Class for keeping track of different operator categorieszOrderedSet[Callable[..., Any]] fusible_opscompute_intensive_ops random_opsview_opsrecomputable_opsnodefx.Nodereturnr>c.t||jvSN)r<rCselfrHs [/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/torch/_functorch/partitioners.py is_fusiblezOpTypes.is_fusiblekst$$(888c.t||jvSrL)r<rDrMs rOis_compute_intensivezOpTypes.is_compute_intensivenst$$(BBBrQc.t||jvSrL)r<rErMs rO is_randomzOpTypes.is_randomqst$$77rQc.t||jvSrL)r<rFrMs rOis_viewzOpTypes.is_viewtst$$ 55rQc.t||jvSrL)r<rGrMs rOis_recomputablezOpTypes.is_recomputablewst$$(===rQNrHrIrJr>) __name__ __module__ __qualname____doc____annotations__rPrSrUrWrYrQrOrBrBasBB////9999....,,,,44449999CCCC88886666>>>>>>rQrBceZdZUded<ded<ded<ded<ded<ded <ded <ejdd ZddZddZddZ ddZ dS)NodeInfo list[fx.Node]inputsOrderedSet[fx.Node]_required_fw_nodesrequired_bw_nodestangents_closureunclaimed_nodesdict[fx.Node, int]fw_orderstatic_lifetime_input_nodesrJcJtdjDfdS)Nc3K|]}|VdSrLr`.0ns rO z-NodeInfo.required_fw_nodes..s" 0 01Q 0 0 0 0 0 0rQcj|SrL)rk)rqrNs rOz,NodeInfo.required_fw_nodes..s a@PrQkey)sortedrfrNs`rOrequired_fw_nodeszNodeInfo.required_fw_nodess: 0 0/ 0 0 06P6P6P6P    rQrqrIr>c||jvSrL)rfrNrqs rOis_required_fwzNodeInfo.is_required_fwsD+++rQc||jvSrL)rgr{s rOis_required_bwzNodeInfo.is_required_bwsD***rQc||jvSrL)rir{s rO is_unclaimedzNodeInfo.is_unclaimedsD(((rQintcT||jvrtd|d|j|S)NNode z not in fw nodes!)rfAssertionErrorrkr{s rO get_fw_orderzNodeInfo.get_fw_orders7 D+ + + !=!=!=!=>> >}QrQN)rJrc)rqrIrJr>)rqrIrJr) r[r\r]r_ functoolscached_propertyryr|r~rrr`rQrOrbrb{s++++****))))((((    4444    ,,,,++++))))      rQrbcBeZdZUded<ded<ded<ded<ded<dS) MinCutOptionsr>ban_if_used_far_apartban_if_long_fusible_chainsban_if_materialized_backwardban_if_not_in_allowlistban_if_reductionN)r[r\r]r_r`rQrOrrsN$$$$&&&&!!!!rQrrHrIrJch|jddtjtjfvS)N recompute)metagetr&MUST_RECOMPUTEPREFER_RECOMPUTErHs rOmust_recomputers0 9==d + +')0 rQct|jdtjo5t |jdkotd|jDS)Nvalrc3ZK|]&}|jtjjjjuV'dSrL)targettorchopsaten_assert_scalardefaultrpusers rOrrz*_is_assert_only_symbool..sD  EIDK59>8@ @      rQ) isinstancerrrSymBoollenusersallrs rO_is_assert_only_symboolrsi49==''77  OOa    MQZ     rQfx_gfx.GraphModulecH|jjD]}t|rdSdS)NTF)graphnodesrrrHs rOhas_recomputable_opsrs7   $   44  5rQc|jjD]F}t|r5t|jdr t jj|jjvrdSGdS)NtagsTF) rrrhasattrrrTagnondeterministic_seededrrs rOhas_recomputable_rng_opsrs^   4   V,,  1T[5EEE44 5rQrc t|jdtjtjfrdSt|jdtjs*t dt|jddS)Nrr'z.expected node.meta['val'] to be SymFloat, got )rrrSymIntrSymFloatrtypers rO sym_node_sizersw$)E"U\5=$ABBq di& 7 7  UT$)EBR=S=S U U    1rQceZdZddZdS)InvalidNodeBaserJstrcdS)Nz Invalid Noder`rxs rO__repr__zInvalidNodeBase.__repr__s~rQN)rJr)r[r\r]rr`rQrOrrs(rQrc6t|jdddkS)N namespace_c10d_functional)getattrrrs rOis_not_collectivers 4; T 2 26H HHrQ getitem_nodefx.Node | Nonec:|jtjkrdS|jd}|jd}t |t jr |jdkrdSd|jvrdS|jd}||vrdS||}t |t jr|SdS)zGiven a getitem node, check if it extracts from a higher-order op that has kwargs mapping the key back to an original input. Returns the original input node if found, None otherwise. Nrr' call_functionkwargs) roperatorgetitemargsrfxNodeopr)r ho_resultrvroriginal_inputs rO_get_ho_op_original_inputrs h...t!!$I  A C i ) )Y\_-L-Lty'''t  h 'F &tC[N."'** 4rQc|jtjjjjtjjjjfvrdS|jd}t|tj sdSt|S)zCheck if node is a view/reshape of a higher-order op output that aliases an input. Returns the original input node from the higher-order op's kwargs if the pattern matches, None otherwise. Nr) rrrrviewrreshaperrrrr)rHsources rO_is_copy_node_bw_onlyrsa  {59>.6 8N8VWWWt Yq\F fbg & &t $V , ,,rQenvdict[fx.Node, Any]ct|}|'||vr#t||ts||St|}|'||vr#t||ts||SdS)aTry to find a valid input replacement for an invalid forward output. This handles cases where a forward output depends on backward nodes but semantically aliases an input. For example, a view of a getitem from a triton kernel that mutates a buffer in backward, or a direct getitem from such a higher-order op. The original input may be a primal or a valid intermediate node already present in the forward graph. N)rrrr)rHrrs rO_find_input_for_invalid_outputrs+400N" c ! !3~.@@ ">"".t44N" c ! !3~.@@ ">"" 4rQF joint_graphfx.Graphrdrcoutputs outputs_descslist[AOTOutput]subgraph str | Noneignore_must_be_in_fw_bwctj}i|D]-}||j}|j|_||<.|jD]}|sHt |r|dkr||vr t|<)t|r|dkr||vr t|<M|vrR|j dkr t|<h|j dkr`tj |j i|j } fd| D} t| r t|<||fd|<|j dkr||fd|<|j d kr  g} t!||D]{\} } t#| tjrE| vrt'd | d t#| t(rd } | jt,jjjjurt | rt7| j d krmt#| j dtjrH| j dvr9t#| j dt(s| j d} | t9| } | | | 6t=d | d| | f| | }|tA| }||jd<d| D|jd<|!|"|S)a Given a graph, extracts out a subgraph that takes the specified nodes as inputs and returns the specified outputs. This includes specifying non-placeholder nodes as inputs. The general strategy is to initialize all inputs with proxies as we encounter them, and trace through the graph, only keeping values which take in valid proxies. Then, all dead code is eliminated. backwardforward placeholderrczg|]7}t|tjt|t8Sr`)rrrr)rpxrs rO z6_extract_graph_with_inputs_outputs..^sIa))3q6?33rQc|SrLr`rrs rOrtz4_extract_graph_with_inputs_outputs..g CFrQget_attrc|SrLr`rs rOrtz4_extract_graph_with_inputs_outputs..jrrQoutputrz couldn't be found in envNr'rz was invalid, but is outputdescczg|]8}t|tjr|jdnd9S) stack_traceN)rrrrr)rpvs rOrz6_extract_graph_with_inputs_outputs..sL''' &027%;%;E =!!!'''rQoutput_stack_traces)#rGraphrnamerr_must_be_in_backward InvalidNode_must_be_in_forwardrpytreearg_tree_leavesrrany node_copyziprr RuntimeErrorrrrrrcopy_rrrappendrrtupleeliminate_dead_codelint)rrdrrrr new_graphrHnew_nodeall_args output_valuesrx_desc replacementoutrs @rO"_extract_graph_with_inputs_outputsr &s$ I"$C((33  D !))& $T**  **&&'D $D))  ))&&'D  3;;  W % %#CII W ' '-tyHDKHHH!H 8}} 'D !++D2B2B2B2BCCCII W " "!++D2B2B2B2BCCCII W M-00$$ 6 a ! ! $||"#G1#G#G#GHHH#a&/22 M# H  4 <<<,Q//=AF q(("16!9bg66)q S((&s16!9~GG)#&afQi.K&"@C"H"HK*!((555$%KQ%K%K%KLLL  Q ( ( ( (   # # # #   5// 0 0C$CHV'''''CH "# !!### NN rQctjoWt|jtjjrt|j p|jtjj j kSrL) ris_non_builtin_to_includerrr_ops OpOverloadrr higher_order triton_kernel_wrapper_functionalrs rOrrsP  +  DK!6 7 7 W 4;@W@W.s$ K K$) K K K K K KrQrrr) rrr find_nodesnextiterrrr)r*r+rr fwd_outputs bwd_outputsfwd_outputs_descsbwd_outputs_descss rO_extract_fwd_bwd_outputsr9s$ K K 2 = = = J J K K KG* T,$//8/<< = =>>CGG TFS\\)  M *?*+K/**+K%&6&67%o&6&67  %68I IIrQ saved_valuesrrNonecV|D]%}|j|kr||dS&dSrL)rremove)r:r saved_values rO_remove_by_namer?sI#  t # #    , , , EE $rQfwd_module_outputs#list[fx.Node] | tuple[fx.Node, ...]ct|}tt|dz ddD]}t||s|dz}n|S)Nr')rranger)r@idxis rOfind_first_sym_noderGsk  ! !C 3)**Q.B 7 7-a011 a%C E  JrQ@-q=rtorch.fx.Graph torch.fx.Nodemaxfloatminpositionc  ||5|tjjjj|f}tjjj|jd|jd<t|jd|jd<dddn #1swxYwY||5|tjjj j|dgdf}tjjj |jddgd|jd<t|jd|jd<dddn #1swxYwY||5|tjj j j|tj f}tjj j |jdtj |jd<t|jd|jd<dddn #1swxYwY||5|tjjj j||f}tjjj |jd||jd<t|jd|jd<dddn #1swxYwY||5|tjjjj|f} tjjj|jd| jd<t| jd| jd<dddn #1swxYwY|| 5|tjjjj| |f} tjjj| jd|| jd<t| jd| jd<dddn #1swxYwY|| 5|tjj j j| tjfd|d|j} tjj j | jdtj| jd<t| jd| jd<dddn #1swxYwY| S) Nr/r tensor_metarCTfp8_scale_pos__rr)inserting_afterrrrrabsrrramaxprimsconvert_element_typefloat64 clamp_min reciprocalmulTensorfloat32r) rrHrLrNrOabs_node amax_node amax_64_nodeclamp_min_nodereciprocal_nodemul_node scale_nodes rOcalculate_quantization_scalingrgs   t $ $UU&& IN  &'   %y~199$)E:JKK e'>x}U?S'T'T m$ UUUUUUUUUUUUUUU   x ( (WW'' IN  'RD$'(  !&  3 ; ; M% 2$! ! u)@ u@U(V(V }%WWWWWWWWWWWWWWW   y ) )   ** IO 0 8U]++  $)9?#G#O#O N5 !5=$ $ % ,C  e $, , -(                   | , ,   ,, IN $ ,$-  &+Y^%=%E%E  e $c& & E".E   &. . M*                   ~ . .   -- IN % - ".  ',in&?&G&G   &' ' U#/F   '/ / ]+                    / /UU&& IN  %!3''   %y~188   '   e(?x}U?S'T'T m$UUUUUUUUUUUUUUU   x ( ( Y Y(( IO 0 8EM*8(88TY88)  "'!E!M!M M% %-" " *AQVAW)X)X  & Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y sBB22B69B6BE44E8;E8B&II  I &BLL L$BOOO BQ>>RRB2UU #U rf quant_type torch.dtyper[ clamp_maxc ||5|tjjjj|tjf}tjjj|jdtj|jd<t|jd|jd<dddn #1swxYwY||5|tjj j j ||f}tjj j |jd|jd|jd<t|jd|jd<dddn #1swxYwY||5|tjj j j||f} tjj j |jd|| jd<t| jd| jd<dddn #1swxYwY|| 5|tjj jj| |f} tjj j| jd|| jd<t| jd| jd<dddn #1swxYwY|| 5|tjjjj| |fd|d|j} tjjj| jd|| jd<t| jd| jd<dddn #1swxYwY| S)Nr/rrQfp8_quant_pos_rSrT)rUrrrrXrYrr_rrrr]r^r[rjr) rrHrfrhr[rjrOtarget_node_32scaled_target_nodeclamp_min_scaled_nodeclamp_max_scaled_nodequant_activation_nodes rOperform_quantizationrrAs   z * *   ,, IO 0 8 &-  &+Y_%I%Q%Q Ie em& & E".E   &. . M*                   ~ . .   "00 IN  % *-1  */);)B)B   & (>* * &2I  #E *2 2  .                   1 2 2   % 3 3 IN $ ,$i0!4! ! -2IN,D,L,L  #E *I- - "5)5L ! &u -5 5 "=1                   4 5 5   % 3 3 IN $ ,'3!4! ! -2IN,D,L,L ! &u -y- - "5)5L ! &u -5 5 "=1                   4 5 5   % 3 3 IO 0 8'48(88TY88!4! !  IO 0 8 8%*51:   "5) 5L ! &u -5 5 "=1                 ! s^B&CC C (BFFF1BIII/BL  LL-BOOOtensor torch.Tensorcb|}|}||zdz S)z Calculate the size of a PyTorch tensor in megabytes (MB). Args: tensor (torch.Tensor): Input tensor Returns: float: Memory size in MB i)numel element_size)rs num_elementsrws rOcalculate_tensor_sizerys2<<>>L&&((L < 'K 88rQlist[torch.dtype]ctjjjddd}d|dD}|S)N!activation_quantization_aten_passallowed_dtypesztorch.bfloat16cjg|]0}tt|dd1S).rC)rrsplit)rpdtypes rOrz&get_allowed_dtypes..s@16u{{3''+,,rQ;)r _inductorrpost_grad_fusion_optionsrr)r}s rOget_allowed_dtypesrs__+D+ c ,--:H:N:Ns:S:SN rQc<t}t|r|jdj|vrdStjjjddd}t|jd}tjjjddds||kStjjjdddr't||kpt||k St||kS)NrFr| size_in_mbdskip_dynamo_guardsquantize_dynamic_shape) rr rrrrrrrryr"r!)rHr}size_thresholdrs rOshould_quantizers'))N d # #ty'7'=^'S'Su_+D+ c,'ty'788J ? ! :+  c &&G^++ ? ! : / #& . . G)n,J+J.,HIII J )~)EFF FrQctjjjddd}t t|ddS)Nr|rhztorch.float8_e5m2rrC)rrrrrrr)rhs rOget_quant_typersO'@+ c,+,, 5***3//3 4 44rQrtuple[float, float]cFtj|}|j|jfS)z Calculate the range of values for a given torch.dtype. Args: dtype (torch.dtype): The input dtype. Returns: tuple: A tuple containing the minimum and maximum values. )rfinforNrL)rinfos rOcalculate_rangers" ;u  D 8TX rQc |dd}|jd}t}t|\}}t g}g}t |D]\} } | |kr | jddrbtj j j dddrct|| |d | } t|| | |||| } t| s|| n|| n|| 5|tjjjj| |fd | d | j } tjjj| jd || jd <t/| jd | jd<dddn #1swxYwY| | <fdt |D} t1| }||z}|r| d||z| |dz} |dt5| t6ddxxdz cc<dS)Nrr1rsaved_for_quantizationFr| use_scalingTrIrlrSrTrrQcBg|]\}}||Sr`)r)rprFrHposition_to_quants rOrz*quantize_activation_fw.. s;+21da&&rQinductor%activation_quantization_fwd_aten_passr')r2rrrdict enumeraterrrrrrrgrrrrrUrrrXrYrrrrG update_argrr )rr+rr5rhr[rjtensor_scale_nodessym_scale_nodesrOrHrf quant_nodeoutput_updated_argsrE scale_nodesrs @rOquantize_activation_fwrs     * *1 -F+a.K!!J*:66Iy(*%'O#K00-5-5$ o % %  9==15 9 9# 5%>3c-&& <4E8 24ZIx #:..7&--j9999#**:6666**400  !&!4!4 <D"J/DhDDDD"5""J  <DD Ie,jOE* 6M".66JOM2               +5 h '6? 6L6L 1 2 2C$6K  % 36I#$$6O O  a233444 Z@AAAQFAAAAAs=BG''G+ .G+ c  d|jD}d}|D]O}|jddr0|jd|jd}tjjjdddr||5d|j dd z t fd |D}dddn #1swxYwY||5| tj jjj||f }tj jj|jd ||jd <t#|jd |jd <dddn #1swxYwY||5| tj jjj||f }tj jj|jd |jd |jd <t#|jd |jd <dddn #1swxYwY||5| tj jjj||f }tj jj|jd ||jd <t#|jd |jd <dddn #1swxYwYn||5| tj jjj||fdt+|j z}tj jj|jd ||jd <t#|jd |jd <dddn #1swxYwYt-|jD]$}||kr||kr|||%Qt4ddxxdz cc<dS)Nc(g|]}|jdk |Srr1r0s rOrz*quantize_activation_bw..s$JJJ$M1I1I1I1I1IrQrF dequant_typer|r fp8_scale_ fp8_quant_c32K|]}|jk |VdSrLr)rp bwd_input scale_names rOrrz)quantize_activation_bw..(s<&&%$>Z77"7777&&rQr/rrQdequant_rTr%activation_quantization_bwd_aten_passr')rrrpoprrrrrUrrr3rrrXrYrrrdivr^rlistrkeysreplace_input_withr ) r bw_inputsactivation_noderHrrfdivided_target_node_32 dequant_noderrs @rOquantize_activation_bwrsJJ%+JJJIOH@H@ 9==15 9 9G @ IMM2 3 3 39==88L%>3c-''? **400!- 0A0A,PR0S0S!SJ!%&&&&)2&&&""J**:66  &+&9&9 <D"L1':''O  <DD Ie,l$(/ ;R',U3;;O(7               **?;;  -2-@-@ *1-z:.A..*:?9K9R9R',U3Z_U5K::*/600F0KE0RSS+/ >               **+ABB  #(#6#6 <D4lC$7$$L  <DD27> !%e, 8O$)%088L%m4               **400  #(#6#6 <D"L1'#di..8$7$$L  <DD Ie,l!%e, 8O$)%088L%m4               TZ__..// @ @<''DO,C,C++D,??? Z@AAAQFAAAAAs^0:C66C: =C: BF44F8 ;F8 BI==J J BL;;L? L? B(PP P fwd_module bwd_modulebwd_module_inputsdict[str, fx.Node]ctddfdtj|tddfdtddfdjd d jd }|D]}d |jvr|t jd d|j}j|5j |j}dddn #1swxYwY|j d}|j |j d|j d<||j d<| |j |tjjjdddrt'jd } | d} t)| D]} t+| s| } njd d jd } | D]~}d|jvrsj| 5j |j} dddn #1swxYwY| j |j | } t-jtddfddS)Nartifactc dddS)N,before_activation_quantization_fwd_aten_passstringrencodingr`r`rQrOrtz5perform_fp8_activation_quantization..qB   rQc4dddSNFT print_outputinclude_strideinclude_deviceprint_readablersrOrtz5perform_fp8_activation_quantization..u#:44tD5  rQ metadata_fn payload_fnc dddS)N+after_activation_quantization_fwd_aten_passrrr`r`rQrOrtz5perform_fp8_activation_quantization..~A   rQc4dddSrrrsrOrtz5perform_fp8_activation_quantization..rrQc dddS)N,before_activation_quantization_bwd_aten_passrrr`r`rQrOrtz5perform_fp8_activation_quantization..rrQc4dddSrrrsrOrtz5perform_fp8_activation_quantization..rrQrr1rrz^fp8_quant_pos_\d+_rrrTrr|rrrCrc dddS)N+after_activation_quantization_bwd_aten_passrrr`r`rQrOrtz5perform_fp8_activation_quantization..rrQc4dddSrrrsrOrtz5perform_fp8_activation_quantization..rrQ)rrrr2rrresubrUrrupdatereplace_all_uses_with erase_noderrrrrrreversedr7r)rrrr+quant_fwd_module_outputsfwd_noderquant_bwd_inputrquant_bwd_module_inputs bwd_input_locbw_inputscaled_fwd_module_outputsscale_bwd_inputs`` rO#perform_fp8_activation_quantizationris           :+_===                   */::h:GGJOPQR, 3 3 8= ( ()-r8=AAI!11)<< S S","2">">HM">"R"R S S S S S S S S S S S S S S S$>.9L  ' ' 6 6 6=AO !9 :3?O  0  + +O < < <   ' ' 2 2 2 6+ c-0#'z'7'B'Bm'B'T'T"U"U/3  !899  Hx((  (  %/$4$?$?8$?$L$LQ$O$TUV$W!1 0 0Hx},,%55mDDWW&0&6&B&B &B&V&VOWWWWWWWWWWWWWWW$++HM::: / :+,,,           s$!C;;C? C? !I<<J J rlOrderedSet[fx.Node] | Nonec|r d|Dng}d|D}tjjjdddr d|D}|jdd jd }d |jd D}d} |D]} | j|vrt| r| j|vr!t d | jDd | j d<| j dj | j d<d || jj d<| j dj || jj d<d } | rt||||dSdS)Ncg|] }|j Sr`rr0s rOrz2enable_activation_quantization..s;;;t;;;rQci|] }|j| Sr`rr0s rO z2enable_activation_quantization..sCCCd$)TCCCrQr|exclude_primalsFc0i|]}d|jv |j|S)primalsrr0s rOrz2enable_activation_quantization..s/   $ 8R8RDIt8R8R8RrQrr1rci|] }|j| Sr`rr0s rOrz2enable_activation_quantization..s)  4rQrz*Skipping quantization of static input %s: Trrr)rrrrrrr2rrrr@debugrrr) r:rrrlr+static_input_namessaved_values_namesr@rshould_perform_fp8_quantrHs rOenable_activation_quantizationrs ' ;;:;;;;  DClCCC 6+ c U##   (4   $)444AA!DI!L$.$4$?$?=$?$Q$Q %" , , 9* * *t/D/D *y... F RRR26DI. /(, %(8(>DIn %JN di ( -.F G@D %@P@V di ( -n ='+ $ +  $5       rQ)rlromit_aot_autograd_runtimesaved_sym_nodessaved_opaque_nodeslist[fx.Node] | Noner%tuple[fx.GraphModule, fx.GraphModule]c  #t||\}} } } |jd} gtt| } |rgngtt | }gtt | }gtt| }gtt| }|g}t|j||zz|z|z| | d|}tj }|dD]}|j s@t|jt||jt||jI|r^t!d|j Dr@t|jt||jt||jt|r&t|j|st#dt%}g}g}|D]S}t'|}|r+||||>||Tt-|j}t/j||D]c}d |jvr t5|jd |z }t7|d D]"}||vr|||#||z}d||||z|sg}g}g}D]}t=|jd t@r||E|jd d r||v||||ztC|#tED]K\}}|#kr@|jd d s%t#d|d#dtCLt|j| |z|z|z|z|z| #fdtGtCtC|ztC|ztC|zDzd|} t|j|z|z|z|z|z|z| | d|}n}t|j| |z|z|z| dtGtCtC|zDzd|} t|j|z|z|z| | d|}|j$D]}|j%dvr d|jd<tLj'(|| }!tLj'(||}"tRj*dd tW|!|"|||!|"fS)ajExtract forward and backward graph modules from a joint graph. Args: ignore_must_be_in_fw_bw: When True, disables forward/backward placement enforcement in _extract_graph_with_inputs_outputs. Needed when the joint_module is not an original fwd+bwd joint graph (e.g. a backward graph being re-partitioned for dI/dW splitting). omit_aot_autograd_runtime: When True, skips postprocessing that is only needed when the resulting modules will be wrapped in a custom autograd.Function (the AOTAutograd path). This includes: tangent input handling, version-counter check sorting of saved tensors, opaque object (FakeScriptObject) separation, and fp8 activation quantization. Set this to True when the fwd/bwd modules will be executed directly without autograd. r+rr1Nr)rc3K|]>}|jtjjjjuot |jdkV?dSrN)rrrr wait_tensorrrrros rOrrz+_extract_fwd_bwd_modules../s_) )  H 2>F F "AG !) ) ) ) ) ) rQz'backward_state_inputs must not be emptyrc|jSrLr)ss rOrtz*_extract_fwd_bwd_modules..Ws16rQrusaved_tensor_with_no_vc_checkFzi=z, no_vc_check_start_idx=z, len(saved_values)=c~g|]9}|kr"|tkrt|nt|:Sr`)rr0r/)rprFno_vc_check_start_idxr:s rOrz,_extract_fwd_bwd_modules..sa   ---!c,6G6G2G2G4A666/22   rQrc,g|]}t|Sr`)r/rprFs rOrz,_extract_fwd_bwd_modules..s0+1--rQ)rrTautograd_backwardr|),r9rr2filterr6r7r5r4rr r distributed is_availablerr?rrrr%raddrr itertoolschainrrrwclearextendrrrrrrDrrr_lazy_graph_module_make_graph_moduleinductor_configrr)$r*r:rrr+rlrrr5r6r7r8 placeholders primal_inputstangent_inputsfwd_seed_offset_inputsbwd_seed_offset_inputsbackward_state_inputs bwd_graphdistributed_enabledrH saved_symbolssaved_sym_nodes_bindingsaved_sym_nodes_derivedsymbolsymbol_bindings new_symbolsrsaved_values_with_vc_checksaved_values_no_vc_checksaved_opaque_objectsrF fwd_graphrrr s$ ` @rO_extract_fwd_bwd_modulesr+s4 !OOOCK/1B %00M0BBL7fZ667M'Q-Qvk

> > >$ 25LLMMM $b &("#% !  8 8D$)--..0@AA 8$++D1111>FF 8(//5555*11$777769QQRRR #$> ? ?!..  GAt)))y}}%DeLL(vQvv8Mvvcfgsctctvv7   2 2  " #! !         %%.//0,--./**+     $;+   .7    " #! !   %  % $  $   $;    7   2 2 , & 8 s<0033G3GGHH  $;    7    $ %$ $   $;    22 73 3 3-1DI) *&99, RRJ&99, RRJ044 /    '    '      z !!rQ)static_lifetime_input_indicesrl _joint_inputsr r,list[int] | Nonec  g}d}|jjD]1}t|st|st |r|}2|t d|jjD],}t |s||||urn-td|Dt|}t|} |rRt|jd'tj dt||||St|d}t jst%|t'|t)||g}t+|||} g} g} g} t,jd!d }d!d}d!fd }|jjD]}|jvr |jdkr&|jd|Dvr>|jt,jjjj t,jjj!j t,jjj!j"t,jj#j$j t,jj#j%j t,jj#j&j t,jj#j&j'fvrtQ|r| |$||r1|j)*dtVj,krrddl?m>}|||||t|}t|}t|d }ty| jBdkrt|d }||fS)"a Partitions the :attr:`joint_module` in a manner that closely resembles the behavior observed in the original ``.forward()`` and ``.backward()`` of the callable, i.e., the resulting forward graph contains those operators that are executed in the original ``.forward()`` callable passed to :func:`aot_function`. The default partitioner collects the operators that are between the forward inputs and the forward outputs. This helps in finding the tensors which have to be stashed for the backward pass. These stashed tensors become the output of the generated forward graph. The remaining operators are then placed in the backward graph. .. warning:: This API is experimental and likely to change. Args: joint_module(fx.GraphModule): The joint forward and backward graph. This is the result of AOT Autograd tracing. Returns: Returns the generated forward and backward Fx graph modules. Nzlast_node must not be Nonec3:K|]}|jdk |jVdSrNrrr0s rOrrz$default_partition.. s9$$tw(/B/B /B/B/B/B$$rQrzxTrying to unsafely apply AC to a non-functional graph with the default partitioner. Falling back to min-cut partitioner.)r+r,Tis_default_partitionrHrIrJr>cd|jvp6t|jdtjjS)NrQr)rrrr _subclasses FakeTensorrs rO is_tensorz$default_partition..is_tensor4s; ) Z IMM% %"3">. .  rQcntd|jDot|jdkS)Nc3@K|]}|jtjkVdSrL)rrrrs rOrrz=default_partition..is_multi_output..;s,GGD x//GGGGGGrQr)rrrrs rOis_multi_outputz*default_partition..is_multi_output9s8 GGDJGGG G G $DJ!# rQc|do-|jdvo$ p!|jtjjjjuS)NF) impure_random)rr) is_impurerrrrrrr)rHr s rOr>z$default_partition..is_impure?sW NNN / /  ('U;ei&@&L&TT rQrc3 K|] \}}|V dSrLr`rpkrs rOrrz$default_partition..Ts73 3 !QA3 3 3 3 3 3 rQrz.Trying to apply AC on a graph with impure op: z, rz Expected z to be a tensorc&g|] }|jv |Sr`r)rprqforward_node_namess rOrz%default_partition..s&UUUAFBT4T4T14T4T4TrQc34K|]}t|VdSrL)rros rOrrz$default_partition..s(77!{1~~777777rQc0g|]}t||Sr`)rr0s rOrz%default_partition..s60G0M0M rQrrr+rl)is_impure_noder'enable_activation_offloadingFrrZ)Crrrr6r5rr7rr%rrr1warningswarn#min_cut_rematerialization_partitioncleanup_recompute_tagsr(unsafe_allow_optimization_of_collectivesforce_save_collectivesforce_save_effectful_opsforce_save_bw_mutation_srcclassify_nodesrrrrr named_modulesrrrrr _assert_asyncmsgprofiler_record_function_enter_new_record_function_enter_record_function_exit_RecordFunctionrrrr& MUST_SAVEr2rrrrrrfromkeysr_sync_decision_cross_ranksrlr+rrfunctionalize_rng_opsr#reordering_to_mimic_autograd_enginerI,_activation_offloading.activation_offloadingr=thread_graphsafe_rng_from_hopsrg)r*r-r+r,rl forward_nodes last_noderHgraph_has_recomputable_opsgraph_has_recomputable_rng_ops node_infor:rrr8r;r>backward_usages fw_module bw_modulerIr rCs @@rOdefault_partitionrks.@MI"( t $ $  4(8(8 &>& 9. . .  7j TY3 3 &44663 3 3 & &   ; IN ) 1 IN ( 0 IN ( , I  9 A I  5 = I  4 < I  4 D    t     " "4 ( ( (  ?4    9== % %)9)C C Cd## *"))$////##D)))  9T?? ) $ZTZZT[ZZd## *"))$////##D)))  $     % %d + + + y D47o#=#= !BT!B!B!BCC CUUUUdjUUU 77777 7 7   " "? 3 3 3 d## &    % % % l3388::;;L4==99>>@@AAO(Odmm,>??DDFFGG (T1,2DlSS "*&/&K#3'-'$? IyO''7H'III O''7H'III!C ) #8iC4H4H$$ Iy8 BB *         %$    '    y))Iy))I.yeLLLI 9 &''!++29$OOO i rQg.Arvc||jzSrL)itemsize)rvrs rO_tensor_nbytesrns 5> !!rQcXddd|jvr6|jd}t|trdSt|ttfrt fd|DSt|t r-t fd |DSt|tj r |St|tj tfr1tj rd Std t|d |d tdt|d ||jdksFt|jtjjr$t)|jjjd krd Std|d)NrobjectrJrct|tjsdStt |d|jS)Nrfallback)rrr^rnr rvrrs rO object_nbytesz_size_of..object_nbytessC!U\** 1/ DIII17SSSrQrr'c3.K|]}|VdSrLr`rprqrvs rOrrz_size_of..s-55A}}Q''555555rQc3.K|]}|VdSrLr`rxs rOrrz_size_of..s->>A}}Q''>>>>>>rQrzCannot compute the size of z on node z. A ScriptObject may hold tensors internally and the partitioner has no general way to measure its memory footprint. Set torch._functorch.config.unsafe_treat_script_objects_as_zero_size = True to assume such objects are zero size (unsound).zUnknown metadata type rrzO didn't have `val` metadata; we should always have `val` metadata on the nodes.)rrprJr)rrrrrsumrvaluesrr^ ScriptObjectrr(unsafe_treat_script_objects_as_zero_sizerrrrrrrr&returns)rHrrvs @rO_size_ofrsTTTT   i c< ( ( 1dE] + + 555555555 5 T " " >>>>>>>>> > U\ * *  =%% % e02BC D D > qJd3iiJJ$JJJ NDIINNNNOOO w*4; 566  #+ , , 1 1q eeee  rQc .ddlm}|t}|jD]'}|jdkr||jjxxdz cc<(tdt| tj dddS)Nr)rrr'%sTrvreverse) collectionsrrrrrr[r@rrwitemsr itemgetter)rrcntrHs rO _count_opsrs''''''%+c**C ++ 7o % %  $ % % % * % % %HHT6#))++8+>q+A+A4PPPQQQQQrQ!list[torch._ops.OpOverloadPacket]cVg}ttjjD]}t tjj|}t |tjjsA|D]1}tj j |j vr| |n2|SrL) dirrrrrrrOpOverloadPacket op_overloadsr pointwiserr)r attr_nameopoverloadpacket op_overloads rO pointwise_opsr s-/C((   "59>9==*EJ,GHH  +88::  Ky"k&666 +,,,7 JrQrtuple[Any, ...] depth_maprjlist[tuple[fx.Node, int]]cfd|D}t|tjddS)Ncji|]/}t|tjjj&||0Sr`)rrrrHr)rpargrs rOrzsort_depths..sE #z#ux}?Q/R/R Ys^rQr'Tr)rwrrr)rr arg_depthss ` rO sort_depthsrsY'+J *""$$(*=a*@*@$ O O OOrQgmc4 tj i |jdD]} | fd |<dt |jjD d fd }ttt|jj}d }tj }|D]"}|j D]} ||kr |}|}#||St|jjd |D]:}|j d kr-|jtjjjjur ||;t|jj |d D] }||tj| }|S) a This pass finds the first bwd node in the graph (by looking at users of tangents) and then reorders the graph by walking from this node to all the way to the end of the graph. At each op in this traversal, we insert this op in a new graph and try to bring only the relevant subgraph from the other non-bwd edges relevant for this op. This closely mimics the behavior of autograd engine. Why is this pass required in the first place? This is an artifact of how partitioners work today. The starting point of partitioner is a joint graph, which is fwd and then bwd graph. In the case of checkpointing, we keep portions of fwd graph in their original place in the joint graph, while obtaining a bwd graph. As a result, the resulting bwd graph has copies of recomputed fwd subgraphs followed by the original bwd graph. If we run this naively, this leads to bad memory footprint, because the fwd subgraphs are live for way longer duration than necessary. This pass reorders the operations such that we prioritize the ops for the original bwd graph while only realizing those ops from the fwd graph that are necessary at any given point in the graph. rr1c|SrLr`rs rOrtz5reordering_to_mimic_autograd_engine..As ArQci|]\}}|| Sr`r`rprErHs rOrz7reordering_to_mimic_autograd_engine..Cs B B B93T3 B B BrQrHrIrJr;cX|g}t}t|dkrO|}||vs|vr0||||jz }t|dkOt |fd}|D]}|fd|<dS)Nrc|SrLr`)rqorders rOrtzSreordering_to_mimic_autograd_engine..insert_node_in_graph..Ss %(rQruc|SrLr`rs rOrtzSreordering_to_mimic_autograd_engine..insert_node_in_graph..UrrQ)r%rrrall_input_nodesrwr)rH cur_nodesinsertable_nodesrrrs rOinsert_node_in_graphzAreordering_to_mimic_autograd_engine..insert_node_in_graphEsF 0: )nnq  ==??D'''43;;   & & & - -I)nnq  ""28J8J8J8JKKK$ D DD!++D2B2B2B2BCCCII D DrQNrrHrIrJr;)rrrr2rrrrrr7mathinfrrrrrrrr GraphModule) rrHrrfirst_node_in_bwd minimum_ordertangentrnew_gmrrrs @@@rOr`r`%s. I"$C##}#55@@''.>.>.>.>??D B B "(.(A(A B B BEDDDDDDDD&&bhn==>>NHM!))M ) )DT{]** %d $(! )   RX^$$%?u->'?%?@'' 7o % %$+9M9U*U*U  & & &RX^$$U+<%=%?%?@##T""""X ! !"i 0 0F MrQritorch.fx.GraphModulerjfw_nodebw_nodedevice torch.device rng_countlast_fwd_inputlast_bwd_input#tuple[torch.fx.Node, torch.fx.Node]c|j}|td|j} |j} tjjj} |j|5|jd|} t| | j d<| }dddn #1swxYwY|j|5|jd|} t| | j d<| }dddn #1swxYwYt|j }| |d<|j|5| d| |jg|jR|}dddn #1swxYwY||| |t|j }| |d<| |5| d| |jg|jR|}||| |dddn #1swxYwY||fS) a% Note [CUDA Graph Safe RNG Functionalization] CUDA Graph capture doesn't work with get_rng_state and set_rng_state because these functions operate on CPU values, while CUDA Graph RNG capture uses on-device CUDA tensors. To solve this, we use graphsafe_set_state with a CUDA Generator registered to the CUDA Graph before capture begins. graphsafe_set_state updates the generator's pointer to reference a different GeneratorImpl, ensuring subsequent calls are correctly forwarded to the desired generator (and its cuda-tensor RNG state during graph capture). For each RNG operation's forward/backward pair: - We create two generators initialized with identical values - Each forward and backward call advances its respective generator equally - This keeps generators synchronized so forward and backward operations use matching RNG values When forward is called multiple times before backward (causing desynchronization): - We save the forward RNG state - We update the backward Generator's state before executing backward Before each CUDA Graph replay, replay_prologue updates captured RNG pointers with current states, ensuring backward Generator changes are reflected during replay. This function modifies both forward and backward computation graphs by: Creating RNG state placeholders for both passes Updating the forward node to use graph-safe RNG state Updating the backward node to use graph-safe RNG state For more details: https://github.com/pytorch/pytorch/issues/113541 Nzdevice_idx must not be Nonefwd_rng_state_rbwd_rng_state_ rng_staterrr)indexrrr_prims rng_primsgraphsafe_run_with_rng_staterUrr8 clone_staterrr create_noderrrrinserting_before)rirjrrrrrr device_idxfw_graphbw_graphr fwd_rng_state bwd_rng_state fw_kwargsfunctional_fw_node bwd_kwargs rng_outputs rO%apply_graphsafe_rng_functionalizationrssvRJ:;;;HH#(<#9#V   ( ( 8 8''!334PY4P4PQQ $9&$A$A$M$M$O$O 5!&'''''''''''''''  ( ( 8 8''!334PY4P4PQQ $9&$A$A$M$M$O$O 5!& '''''''''''''''W^$$I*Ik  ( ( 1 1  %11  (.07<00 2                  !!"4555    gn%%J+J{  " "7 + +%%))  (.07<00 *  %%j111G$$$%%%%%%%%%%%%%%% > ))sKA B,,B03B0A D&&D*-D*$(FFF;AII I num_sym_nodesc > 'tj}dd}dd 'd d }||}||}||} i} |jjD]} t | rtt | jd r_tjj | jj vrB| j |vs | j | vrV|| j } || j } | | j }| |d| | <tj j j}tj j j}d}|jdD]} d| j vr| }n|t#dg}t%t'|jd}t%t'|jd}t)'fd| D}|tjdt1|dk}tjj}t4jo| o|j p |jj}t?| D]Z\}}|d} |d}'| }|j}|j}|r+|)tA|rtC||| |||||\}}\|"| 5|#d|| jg| j$R| j%}|#dtLj'|dfi}|||j(d<|#dtLj'|dfi} tSj)| j(| _(| *| |+| |,|dddn #1swxYwY|"|5dt%|}!|-|!}"|||"j(d<dddn #1swxYwY|"|5|#d||"|jg|j$R|j%} |*| |+|dddn #1swxYwY\|rt%t]|jd}#|#j$d}$t1|$|z }%|$d|%t_|z|$|%dz}&|j0|&|j+|#|1|1||fS)!NgmodrrJrci}|jjD]I}|jdkr.get_rng_opssd+- J$ / /D?**DK00+I59III*. TY'rQrHrItorch.device | Nonecd|jvrdS|jd}t|ts|f}|D]9}t|tjrt |jr |jcS:tjdS)zV Check the example value of the node outputs to find the device type. rNcpu)rrrrr^r:r)rH candidates candidates rO get_devicez)functionalize_rng_ops..get_devices  ! !4Yu% *e,, '$J# , ,I)U\22 ,))*:;;,$++++|E"""rQrrtcFddlm}|}|td|5|=t|r.|t |cdddS|t jcdddS#1swxYwYdS)Nr)detect_fake_modezfake_mode must not be None) torch._guardsrrr: from_tensorr9r get_rng_state)rr fake_modes rOget_sample_rng_statez3functionalize_rng_ops..get_sample_rng_state s/222222$$&&   !=>> >  @ @!&)>?? @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @s3B$%BBBr)fwdbwdrr1rzaCouldn't find tangent node in graph inputs. This is unexpected, please file a bug if you see thisc3:K|]}|dVdS)rNr`)rp node_pairrs rOrrz(functionalize_rng_ops..;sC)2 9U#$$rQrr'rrrrrrrng_state_output_r)rrrJr)rHrIrJr)rrrJrt)2rcountrrrrrrrrrrrrrun_and_save_rng_staterun_with_rng_stater2rr3rr%r{discardrrrrgraphsafe_rng_functionalizationfallback_random test_configs*graphsafe_rng_func_ignores_fallback_randomrr:rrrrrrrrcopyrrrrr4rr recompile)(r*rirjruidrrjoint_graph_rng_opsfw_graph_rng_opsbw_graph_rng_opsrecomputable_rng_ops_maprH base_noderrrun_and_save_rngrbw_tangent_start_nodefw_rng_state_outputsrrdevicesmulti_graphsafe_devices ind_config'use_rng_graphsafe_rng_functionalizationrrrrrrstater state_namebw_rng_state_nodefw_output_node fw_outputssym_node_start_idxrrs( @rOr_r_sl2 /  C    ####$ @ @ @ @&+l33"{9--"{9--!"(SS 4  S V,, S 1T[5EEE y 000DIEU4U4U+DI6I&ty1G&ty1G:A'2R2R $Y /|-D/B **m*<<  ! !$( ! E "$ o   +-(9?#=#=#=#O#OPPQQN(9?#=#=#=#O#OPPQQN6N6U6U6W6WG OOEL''((("'llQ.'J. ' ' * * R&Q ,!**B*I*I*K*K L LP-P- 9E"E"G$$?? 4G -"&v..#.S . . *NNN**733! 3! 3%-%9%9#$  #>&: & &"!,,#$,a0 - %9$8$@$@ 5!%11#$*2 #')GL"9"9 --j999##G,,,$++E222C! 3! 3! 3! 3! 3! 3! 3! 3! 3! 3! 3! 3! 3! 3! 3H**+@AA M M<c<< $,$8$8$D$D!0D0DV0L0L!&u- M M M M M M M M M M M M M M M **733 - -%11#&*! #>2   --j999##G,,, - - - - - - - - - - - - - - -& 3d9?#=#=#=#J#JKKLL#(+  __}< *** +()) *+,,- .  w'''"">222   i s8C'O  O O ,;P33P7 :P7 AR77R; >R; c|jjD]Y}t|jtjjr3|jjdkr#t|stj |j d<ZdS)z By default, the partitioner is not allowed to recompute collectives unless they come from a user-annotated AC region. See Note [Recomputing collectives in the partitioner] rrN) rrrrrrrrrr&r\r)r*rHs rOrPrPst "(@@ t{EJ$9 : : @ %);;;"4((<&6%?DIk " @@rQcdfd |jjD]:}t|r)t|st |s |;dS)a\ Force save outputs from with_effects nodes wrapping effectful ops. Effectful ops (registered via _register_effectful_op) should not be recomputed because they may have arbitrary global side effects (I/O, RNG state, collectives, etc.). We mark the tensor outputs of with_effects as MUST_SAVE to prevent recomputation of the effectful op. The with_effects node returns a tuple (token, result). We recursively find all leaf outputs extracted via getitem and mark them as MUST_SAVE. Since these are saved, the with_effects op doesn't need to be recomputed in backward. rHrIrJr;c|jD]h}|jtjurS|t |jdttfstj |jd<idS)Nrr) rrrrrrrrrr&r\)rHrmark_getitem_outputss rOrz6force_save_effectful_ops..mark_getitem_outputssxJ H HD{h...$$T***!$)--"6"6 FFH-=-GDIk*  H HrQNr)rrrrr)r*rHrs @rOrQrQsHHHHHH"('' D ! ! '"4(( ').. ' !  & & & ''rQct}t|jjD]}|jdkr|jt jjj j u}|rmt|r | |j dt|r.|j d|vrtj|j djd<dSdS)Nrrr'r)r%rrrrrrrrrrr#rrr r&r\r)r*has_mutation_in_bwrHis_copy_s rOrRrRs5?LL+122 7h   ;%)."6">>  +D11 5"&&ty|444*400 LTYq\EW5W5W1A1K ! !+. EE!rQc|jtjkrdS|jd}t |t jurtdt |d|jvo |j dkS)NFrz#expected parent to be fx.Node, got rQr) rrrrrrrrrr)rHparents rOis_getitem_of_multi_outputr sk {h&&&u Yq\F F||27""Q4<<QQRRR  + J?0JJrQr4cX|jjD]}t|r|jD]S}t|rBd|jvr9d|jvr0|jd|jdkrt j|jd<T|jddr2td|jDst j|jd<d|jvrWtd|jDr9t|rd|j djvs|rt j|jd<|S)a If there are two consecutive checkpointed blocks with no operator in between, we would still want to stash the tensor at the boundary of checkpointed blocks. The following pass makes the last output node non-recomputable to allow for that. ac_graph_idrhas_backward_hookFc34K|]}t|VdSrLrrs rOrrz)cleanup_recompute_tags..sCEE)-t$$EEEEEErQc34K|]}t|VdSrLrrs rOrrz)cleanup_recompute_tags..+s*@@TN4((@@@@@@rQr) rrrrrr&r\rrr r)r*r4rHrs rOrNrNso"(+@+@ $  * @  H H"4((H%22%22 -049]3KKK-=-GDIk*y}}0%88 DEE15EEEBB D&*:)C +&  * *@@TZ@@@@@ ++400 +6CdiPQlFW5W5W$6X&6%?DIk " rQrgmin_cut_optionsdont_ban)tuple[list[fx.Node], OrderedSet[fx.Node]]c <=>?@ABCDEFGHIJKttItrZtd|jD}|tdIjDz }t d|dbd =dbd >db=>Ifd A d dlGn"#t$r}td |d}~wwxYwdcAIfd CddCIfd }dcAfd BdeBIfd }Gj HtfstA| rd'} d}n#t j} d(}n|| j!\} }|rH| t jks | tDkr-H| jd z| jdz| d)|n(H| jd z| jdz| *| j#D]5}H| jdz|jd zt jd+6dhAfd0 }j$rj%D]}fd1|j#D}fd2|j#D}tM|d kr||tO|}tQ|j#D]}|rz)||kraA||rU|d?}ts|d@5}!|!:@dddn #1swxYwYn#tv$r}dA|dB}Yd}~nd}~wwxYwdC<Gj=j>?H?tod:dD?fdE=tH}"|"r i}#d6g}$didG}%|"D]!\}&}'}(|$A|'|&d6kr5|%|'})|#B|)gA|(W|'dkr5|%|&})|#B|)gA|(|%|&|%|'kr5|%|&})|#B|)gA|(|%|&}*|%|'}+|#B|+gAdH|*#g},|#CD];\}-}.|,AdI|-dJ|.D]}/|,AdK|/dS)r_overloadpacketN)rrrrrr0s rOrrz solve_min_cut..EsZ& & w/))gdkCT.U.U)  + , ,))))& & rQc34K|]}t|VdSrL)rr s rOrrz solve_min_cut..Js94 4 CFF4 4 4 4 4 4 rQz&Ops banned from re-materialization: %sarIbrJr>c|jtjjjkrdS|jd}tjj|\}}|D]2}|j |}||urdSt|tr||vrdS3dSNFrT) rrrrauto_functionalizedr_higher_order_opsauto_functionalizeget_mutable_argsrrr)rr mutable_opmutable_arg_namesrSrrs rO!can_fuse_into_auto_functionalizedz8solve_min_cut..can_fuse_into_auto_functionalizedOs 8uy-A A A5VAY   # 6 G G     &  D(4.CCxxtt#t$$ 8844urQc|jtjjjkrdS|jd}|D]/}|jd}|t d||}||urdS0dS)NFtensors_to_clonerzkwargs must not be NoneT)rrrrrrr)rrr$rrrs rO.can_fuse_into_triton_kernel_wrapper_functionalzEsolve_min_cut..can_fuse_into_triton_kernel_wrapper_functionalcs| 8uy-N N N5H%78%  D(8,F~$%>???,CCxxtturQcHt|tjkrdS||rdS||rdS|jtjur*|jdjtjj j urdS |o |S)NTrF) r<rcatrrrrrrrrrP)rrr%r(op_typess rOrPz!solve_min_cut..is_fusibleps 1   ) )4 , ,Q 2 2 4 9 9!Q ? ? 4 H( ( (q  y%FGG 5""1%%@(*=*=a*@*@@rQrzANeed networkx installed to perform smart recomputation heuristicsrHcz|rdSt|g}t|dkr|}|jD]P}|s||sdS|r||Qt|dkdSr)rWr%rrrr|r)rHrcurrrPrgr+s rOis_materialized_backwardsz0solve_min_cut..is_materialized_backwardss   D ! ! 5v&& )nnq  --//C  ( ( //55 jjd>S>S 44##D))(MM$''' )nnq  urQrc@|jdkrdS|jtjurdS|jddt jkrdStj r |rdS|jtj j tjj fvrdSjr|sdSn?|rdS|rdSt'|rdSjr;|r0t*d |t/|jd S|jd kr|jtjkrd Sjr8t9d |jD}t=|}|dz|krdSdS)zRReturns reason string if node should be banned from recomputation, None otherwise.rNrzmarked MUST_SAVEznot in recomputable allowlistz random opzcompute intensive opznon-builtin opzmaterialized backwards: %s %szmaterialized in backwardiztoo far from backwardc3hK|]-}t|tjt|V.dSrL)rrrrr s rOrrzBsolve_min_cut..should_ban_recomputation..sM%% !*Q2H2H% %%%%%%rQrz reduction op)rrrrrrr&r\rrecompute_viewsrWrlift_fresh_copyr lift_freshrrYrUrSrrr@rrr dist_from_bwmax_dist_from_bwrrzrr)rHinput_tensors_size output_sizer.rr+s rOshould_ban_recomputationz/solve_min_cut..should_ban_recomputations 7o % %4 ;(* * *4 9==d + +/?/I I I%%  ! h&6&6t&<&< 4 ;4/79PQ Q Q4  2 (++D11 766 7!!$'' #"{,,T22 .--(.. (''  7 ..is_materialized..s/EE$zz$--EEEEEErQ)rrr)rHrPs`rOis_materializedz&solve_min_cut..is_materializedsA 7m # #4EEEEE$*EEEEEEErQrlretuple[float, str | None]c tjr||vrdSt|}tjr#|rt jdfSt|jdtr.t|jdtj s tdfSt|dtt|jddzz}|r|dfS|d zdfS) zReturns (weight, cannot_save_reason). cannot_save_reason is None for finite weights, or a string explaining why the node cannot be saved for infinite weights. rzview op (recompute_views=True)rz$SymFloat (non-SymInt symbolic value)g?rr'N)r treat_parameters_as_free_to_saverr1rWrrrrrrrINT_INFrrLrNr4)rHrlmem_szr;r+s rOget_node_weightz&solve_min_cut..get_node_weights  3 3337$  ! >h&6&6t&<&< >8== = di& 5 5 Gdi. == G FFF cST%6!.ban_recomputation_if_allowed s   D ! ! 5 8  4; (=>>@K)-?? > m u $   5 DI  *TYu-=u~"N"N 5   I X4:R0000@R    trQr_outsinkz;must be available for backward: input required for gradientrFrEz3must be computed in backward: required for gradientz+must recompute: marked by checkpoint policyz primal inputzforward RNG seedrrQznon-tensor outputz cannot save: )rGzdata dependency start_nodesrc max_rangercg}|D]-}tj| ||df.t|dkrtj|\}}}|s |S|jD]l} |rU ||kr1 || ||f}||vrtj||mt|dk|S)z Finds the first unfusible node in the chain of nodes starting from `start_nodes` and returns its position. Tr)heapqheappushrrheappoprr|) rPrQ sorted_nodesrqrSrHnode_is_fusiblerrrPrgs rOfind_first_unfusiblez+solve_min_cut..find_first_unfusible s4 9;  O OA N<)*@*@*C*CQ)M N N N N,!##',}\'B'B $At_" 4 --d333 : :++D11 : --d33i?? !..t44" 4..6C ,..|S999,!##rQcdg|],}||-Sr`)r|rrprrgs rOrz!solve_min_cut.. sK++D11&&t,,rQc>g|]}||Sr`)r|rZs rOrz!solve_min_cut.. s<I4L4LT4R4RrQz1used above/below fusible %s:(%s) -> %s -> %s:(%s)rztoo long %s %s %s %srFTrrc dddS)Nmin_cut_failed_fx_graphrrr`r`rQrOrtzsolve_min_cut.. s5 (%%rQcSrLr`) fx_graph_strsrOrtzsolve_min_cut.. s<rQrmin_cut_failed_graphz.txtwz(failed to write: ) c dddS)Nmin_cut_failed_edge_listrrr`r`rQrOrtzsolve_min_cut.. s2$!!rQcSrLr`) edge_list_strsrOrtzsolve_min_cut.." s}rQ node_namecndD]1}||r|dt| cS2|S)N)rErM)endswithr)rhsuffixs rO get_base_namez$solve_min_cut..get_base_name6 sO-99F ))&119(CKK<88889  rQz depends on z :z - z -> c dddS)Nmin_cut_failed_svgrrr`r`rQrOrtzsolve_min_cut..a s 4$,))rQcSrLr`) svg_contentsrOrtzsolve_min_cut..e s{rQzFX graph dump: zMin-cut graph visualization: z[Production debugging: Use tlparse to extract debug artifacts (min_cut_failed_fx_graph, min_cut_failed_edge_list, min_cut_failed_svg)] aAOT Autograd failed to partition the joint forward-backward graph. The partitioner determines which intermediate values to save from the forward pass vs recompute in the backward pass. This error means a value is required for backward, but cannot be saved AND cannot be recomputed. This is a bug in PyTorch. Please file an issue at https://github.com/pytorch/pytorch/issues Nodes involved in the conflict: z [For PyTorch developers: one of the above constraints is wrong. Either the node should be recomputable, saveable, or not required for backward.] [Debug: min-cut path] z-Failed to compute min-cut on following graph:rchdjjSNrcjoin readwriteedgelistgenerate_edgelistnxrKsrOrtzsolve_min_cut.. & ","7"I"I("S"STTrQchdjjSrsrtrysrOrtzsolve_min_cut.. r{rQc3,K|]}||fVdSrLr`)rprqrKs rOrrz solve_min_cut.. s,88Q $888888rQc3(K|] }|v|fV dSrLr`)rpr non_reachableus rOrrz solve_min_cut.. s1AAa=.@.@q!f.@.@.@.@AArQz node_in[:-3]=z != node_out[:-4]=ci|]\}}|| Sr`r`rs rOrz!solve_min_cut.. sHHHic4cHHHrQc3(K|] }|V dSrLr`rprH name_to_nodes rOrrz solve_min_cut.. s(22d 222222rQc|SrLr`)rnode_idxs rOrtzsolve_min_cut.. s (1+rQru)rrIrrIrJr>rZ)rHrIrJr)rHrIrlrerJr<)r)rHrIrCrrJr>)rPrcrQrrJr)rhrrJr)Kr%get_default_op_listr?rrGr@rnetworkx ImportErrorrDiGraphrrgrhrHrrrrr6r5r|rrrr^rrMrrrrr2rlr@rrryrrLrrrrSrUrrT minimum_cutNetworkXUnboundedr>rhandlers owning_modulerrr_get_unique_pathopenwrite Exceptionrurvrwrx_find_infinite_capacity_pathr setdefaultrvisualize_min_cut_graphrrrget_name_to_noderrw)Lrrgrrjoint_module_ops ops_ignoreder8rBrLrH ban_reasonis_non_tensor_nodeweightcannot_save_reasonrrX used_nodeordersfw_usersfirst_unfusible_usevisited start_nodefusible start_orderrSr- cut_value partition unbounded_excstructured_tracing_enabled fx_graph_filer*finf_pathnode_constraintsraw_path_nodesrl from_nodeto_noderCbase from_baseto_baseconstraint_linesrh constraintscconstraints_str raw_path_strsvg_pathlocal_files_msg tlparse_msg reachablecutsetnbrs cut_nodesnode_innode_outr:rJr%r(rgr_rPr;r.rrrrzrKr+rqrsL ``` @@@@@@@@@@@@@@@@rO solve_min_cutr:s\ <<"$$H H%& & #)& & &    '4 4 $54 4 4 * *   9;GGG(    AAAAAAAA&  O           99999999vFFFFFF &$&$&$&$&$&$&$Prz||H(2 L@!YY 7h    9. . .9555!!I&!XX "!!I%!XP "  $       E!D      d   C ( (~ > > > >  & & C ( (/A B B B .-d33  # #D ) ) ;j ; ( (z : : :  " E}DI'EUty SDIe4Del)S)S%S  t   =..//F!%     e$$}6F&G 9%% 9%)""%8"")8i;** &F&  V6TX#5#579J9J    E! F";'9;;        di%/V1Cf  U U UJ  D    F" E!(      24,;"4 ; ;I%OF !*H6{{Q&:&:8S[[&Q&Q#!)/22;;D!0066;%22488;NNN&Jy$77O <//$O%%229==/ %22488 54T:::1#V'1||%+! V! VJ++J77 '' 33Z@2G$00<> C   **3//+2CCCG ))HH."!..s33!..z:: 10555IVVD!0066V&JsD11V! 44w1G1G1M1Mt0TUUU5g,,""8b-r~h&II 99 VVV%))*<%=%="%) #' "0  6  +++!&tD,%%  0///     --CVLLMmS)) &Q %%% & & & & & & & & & & & & & & & 6 6 65555MMMMMM 6 ","7"I"I("S"STT  -,,,     099 ^ !68 &ZN ! ! ! ! /7  * 7F%%g...(((=11D$//b99@@HHHH&&(=33D$//b99@@HHHH"]9--w1G1GGG(=33D$//b99@@HHHH!. i 8 8I+mG44G$//<<CC1i11 +- *:*@*@*B*B : :& ; ''(9Y(9(9(9:::$::A$++LQLL9999:#ii(899O!;;~66L%>$   G '..I<<<<< = =1  '..I<<<<< = = . .' .9 . ..rQtuple[str | None, str | None]c ddl} ddl}n-#t$r tddYdSwxYw|j|}||d}| D]}|| | d}| t||tdkr|d |d }t%d d }t'|d 5} | |dddn #1swxYwY||fS)zVisualize the min-cut graph to an SVG file. Returns (path_to_svg, svg_content) tuple. Both are None if pydot is unavailable. rNzMInstall pydot to visualize the min-cut graph for debugging: pip install pydotT)exc_info)NNrGrredutf-8min_cut_failed.svgra)rpydotrr@rnx_pydotto_pydot to_stringgraph_from_dot_data get_edges get_sourceget_destination set_labelrrM set_color create_svgdecoderrr) rKrzr dot_format dot_graphedgerrqrrs rOrr s   [    zz %%h//99;;J))*55a8I##%%""$//++,T-A-A-C-CDZP s6{{### U5\\ ! ! NN5 ! ! !&&((//88K 0&99H h     [  s &55FFFc Fgtjtjtjtjtjtjtjtjtj tj tj tj tj tjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtj tj!tj"tj#tj$tj%tj&tj'tj(tj)tj*tj+tj,tj-tj.tj/tj0tj1tj2tj3tj4tj5tj6tj7tj8tj9tj:tj;tj<tj=tj>tj?tj@tjAtjBtjCtjDtjEtjFtjHtjItjJtjKtjL}tjItjJtjMg}|tjNtjOtjPtjRtjStjTtjUtjVtjWg z }|}|gtjtjXtjYtjZtjLtj[tj\tj@tj\tj]tjRtjVtj^tjNtjStjOtj_tj`tjatjbtjctjdtjetjftjgtjhtjitjjtjTtjktjltjmtjntjotjptjqz }|tjrtjsgz }||z }|tz }|tjugz }|dtDz }t|}ttdtftjztj{tj|g}tj}tj~tjtjtjtjtjtjtjtjtjg }||z}t|t||t||S)Nc,g|]}t|Sr`)r)rpms rOrz'get_default_op_list.. s! N N N1!3A!6!6 N N NrQ.)rrrratan2r]rLrNpow remainderfmod__and____or____xor__ __lshift__ __rshift__eqnegegtleltrV bitwise_notceilfloorfracnegreluroundsilutruncr@log10log1plog2lgammaexpexpm1erferfccosacoscoshsinasinsinhtanatantanhatanhsqrtrsqrtr\sigmoidsoftplus thresholdthreshold_backwardclampwherelerpaddcmulgelu gelu_backwardrzmean_grad_sum_to_size sum_to_sizerWtotype_asrrsqueeze unsqueezersub_to_copyaliasrslicetrXbroadcast_in_dimexpand as_stridedpermuteselectrrYprepare_softmax_onlineclone full_likevarstd _unsafe_viewrbroadcast_tensors scalar_tensorones new_zerosr2arangetriuvar_meanisinfrfullzerosempty empty_likeargmaxmaximumiota'_low_memory_max_pool_offsets_to_indicesrgatherr zeros_likerr%rr native_dropout rand_like randn_likemm convolutionconvolution_backwardbmmaddmm#_scaled_dot_product_flash_attention'_scaled_dot_product_efficient_attention_flash_attention_forward_efficient_attention_forwardupsample_bilinear2d _scaled_mmrB)default_recomputable_opsrecomputable_view_opsrFrGrErDrCs rOrr sL: L: L: L: L:  L:  L: L: L: L: L: L: L: L: L: L: !L:" #L:$ %L:& 'L:( )L:* +L:, -L:. /L:0 1L:2 3L:4 5L:6 7L:8 9L:: ;L:< =L:> ?L:@ AL:B CL:D EL:F GL:H IL:J KL:L ML:N OL:P QL:R SL:T UL:V WL:X YL:Z [L:\ ]L:^ _L:` aL:b cL:d eL:f gL:h iL:j kL:l mL:n oL:p qL:r sL:t uL:v wL:x yL:z {L:| }L:~ L:@ AL:B CL:D EL:F GL:H IL:J KL:L ML:N OL:P QL:R SL:T UL:V WL:Z"\4>4:F           %H%! %! "%! $%! %! %!  %!  %!  %! %! %! %! %! %! %! %! !%!" #%!$ %%!& '%!( )%!* +%!, -%!. /%!0 1%!2 3%!4 5%!6 7%!8 9%!: ;%!< =%!> ?%!@ A%!B C%!D E%!F  G%!H 5I%!%NT[ 99( / ! N N N N NN!":;;HS#X./ dndo>J   !   0 4 % )    #Z/K ())8   rQc2i}|jD] }|||j< |SrL)rr)rrrHs rOrr s-')L ''"& TY rQmemory list[float]runtimes max_memoryall_recomputable_banned_nodes"tuple[float, list[int], list[int]]c0tj}|dkrt|||S|dkrt|||S|dkrt |||S|dkrt |||S|dkrkt dtj ||||}t ||t| t| St|tr||||||\}} d || fStd |) Ngreedyilpdpr*dynamic_memory_budget_dpzdynamic_memory_budget_dp is an experimental solver. It does not guarantee performance improvements. Additionally, it is not guaranteed to be stable.)rre recorded_knapsack_input_memories recorded_knapsack_input_runtimes)graph_info_provider) knapsack_algomax_mem_budgetrOz,Not aware of memory budget knapsack solver: )ractivation_memory_budget_solverr+r,r)r*r@warningr(inialize_from_graphr-get_knee_point_memory_budgetrrr) rrarcrdrgreSOLVERrnsaved_node_idxrecomp_node_idxs rO#_optimize_runtime_with_given_memoryrx sr 3F vx<<< 5FHj999 468Z888 3 3 3-fh KKK - - -  ?   0C#*G-3-5        $7   **))+    F0 1 1T*0& KY8U+ + '^_55R&RRSSSrQ no_dispatchrrtct|j}d fd fd|D}fd|D}|||S) Ndtorch.SymInt | intrJrc&t|S)Nrs)r )r|rts rOrealize_symbolz8_remove_symbols_without_guarding..realize_symbol s X6666rQc&g|] }|Sr`r`rprrs rOrz4_remove_symbols_without_guarding.. s# . . .1^^A   . . .rQc&g|] }|Sr`r`rs rOrz4_remove_symbols_without_guarding.. s# 4 4 4AnnQ 4 4 4rQ)stride)r|r}rJr)rshapernew_empty_strided)rrtrrrs ` @rO _remove_symbols_without_guardingr s MME777777 / . . . . . .E 4 4 4 4 4 4 4F  uV  4 44rQc tj}dd}|dkrdS|dkrnt5dd lm}t j|jjf\ |  fd }|cdddS#1swxYwYdS|d krdd l m }t j|jjf\ |d 5}j i dddn #1swxYwY| }t|dSt|t r |St#d|)Nrr rJcvt|tjrAt|jdtjrt |jddSt|tjrAt|jdtjrt|jddSt|tjr't|jdtj rdSt|tjr't|jdtj rdS|S)Nrrrrs?T) rrrrrr^rrr rrrus rOmaterialize_argz)estimate_runtime..materialize_arg s a ! ! j &M&M 3AF5MDQQQ Q 27 # #  16%=%,(O(O $QVE]TBBB B 27 # #  16%=%.(Q(Q 3 27 # #  16%=%-(P(P 4HrQtestingr'profiler) benchmarkercjiSrL)r)rrrHsrOrtz"estimate_runtime.. s;4;3O3O3OrQflops)FlopCounterModeF)displayz Not aware of runtime estimator: )rr rJr )r*activation_memory_budget_runtime_estimatorrz$torch._inductor.runtime.benchmarkingrrtree_maprr benchmark_gputorch.utils.flop_counterrrget_total_flopsrLrrr) rH RUNTIME_MODErrmsrmode counted_flopsrrs ` @@rOestimate_runtimer sDL    y  q  " " ]]   H H H H H H!??TY t_d?}|0|tb2d@|"|AdS)HNr'rzJThe valid ranges for memory budget are 0 <= m <= 1. The provided value is )rrrrrF)rrrrr:rcrJrMcLttt|dz SNeA)rzmapr)r:s rOestimate_activations_sizez:choose_saved_values_set..estimate_activations_sizeK s3x..//#55rQszc|dz z z Srr`)r max_act_size min_act_sizes rOget_normalized_sizez4choose_saved_values_set..get_normalized_sizeT sS\L899rQ activationsc,|z z z SrLr`)rrrrs rO get_mem_ratioz.choose_saved_values_set..get_mem_ratioW s())+66E < '  rQ)rrr)r)get_node_storagec3.K|]}|VdSrLr`)rprHrs rOrrz*choose_saved_values_set..y s/TT4 0 0 6 6TTTTTTrQrJrec"fd|DS)Nc|g|]8}|jtdkr|vst|6|9S)r)r4rr)rprFrinput_storagess rOrzRchoose_saved_values_set..get_recomputable_banned_nodes..~ s`    S))$$Q''~==033>  >==rQr`)rJrrs rOget_recomputable_banned_nodesz>choose_saved_values_set..get_recomputable_banned_nodes{ s3      !    rQcdg|]-}|jddtjk+|.S)rF)rrr&r\r s rOrz+choose_saved_values_set.. sA 6::k5 ) )-=-G G G G G GrQcg|]}|v| Sr`r`)rprFmust_save_nodess rOrz+choose_saved_values_set.. s*!!!0H0H0H0H0HrQTrc@g|]}t|Sr`r)rprFrs rOrz+choose_saved_values_set.. s8-.HQKK((rQc,g|]}t|Sr`)rr0s rOrz+choose_saved_values_set.. s.#'rQryrrgrbrrtuple[list[fx.Node], float]c 5t|t|d| \}}}dddn #1swxYwYt}|D].} | |#t$rY+wxYw| st dt|| |\}} tr1t| |||d D|| | ||fS)Nrz:dont_ban must be a subset of all_recomputable_banned_nodesc,g|]}t|Sr`rr s rOrzNchoose_saved_values_set..get_saved_values_knapsack.. s+'''$%HQKK'''rQ)rresaved_node_idxsrecomputable_node_idxsexpected_runtimememories_banned_nodes normalized_memories_banned_nodesruntimes_banned_nodesmin_cut_saved_valuesrrrsaved_values_act_size&more_aggressive_saved_values_mem_ratio/aggressive_recomputation_saved_values_mem_ratio) rxrLr%r BaseExceptionissubsetrrr?r )rrgrrrrrrEr:rSaggressive_optionsrrerrrrrrzrs rOget_saved_values_knapsackz:choose_saved_values_set..get_saved_values_knapsack s[]]   4%%M1%%-    &               )3 )  C  :3?@@@@       !>??  L (        a !  4'.K /'=!1'')F'''2G&;%1+))&?&? &M&M7]@o#    &---s!'?AAA99 BBrtuple[float, float, float]cb|\}}|t|z |fS)N)rgr)rz)rr:rrrrrgrs rOestimate_for_budgetz4choose_saved_values_set..estimate_for_budget sU-F-FYK... *L*)**-== l++ rQrOrgMbP?r>cg|] }|d S)r>r`rpitems rOrz+choose_saved_values_set.. 000DG000rQcg|] }|d Sr'r`rs rOrz+choose_saved_values_set.. rrQ) )figsizeo)markerz.4fz offset points)rrcenter) textcoordsxytexthaz Memory Budgetz Runtime of Recomputed Componentsz:Pareto Frontier of Memory Budget vs. Recomputation Runtime)exist_okr_rank_memory_budget_paretorSrz%Generated Pareto frontier curve at %s)rrgr)r:rcrJrM)rrMrJrM)rrcrJrM)rJrerJrc)rrMrgrbrrrJr)rrMrJr)3rrrban_recompute_used_far_apart!ban_recompute_long_fusible_chains#ban_recompute_materialized_backwardban_recompute_not_in_allowlistban_recompute_reductionsaggressive_recomputationrrdrtorch._inductor.fx_utilsrr%rwrrtorch.utils._mode_utilsrzvisualize_memory_budget_paretorrsortmatplotlib.pyplotpyplotfigureplotrannotatexlabelylabeltitlegridgcfshowrgetcwdmemory_budget_pareto_dirmakedirsrrris_initializedget_rankrrur3savefigr@rr)+rrgrrruntime_optimized_saved_valuesrSmore_aggressive_optionsmore_aggressive_saved_values%aggressive_recomputation_saved_valuesrJrrecomputable_banned_nodesroptionsbisectslhsrhsmidpltx_valuesy_valuesrFtxtfigfig_dir rank_suffixfig_namerrrerrrrrrrrrrrrzrs+`` @@@@@@@@@@@@@@@@rOchoose_saved_values_setr & s qMA-- hYf h h   $$A#)#K%+%O & E8 O& ! "'',).$)    (5))%"A --6666-,Y-=>>L,,-KLLL|##--:::::::        &##(%*  '4Y 7''# !.;];W-X-X*- ==++  %;HY 2;;7)<7Dm-7737FF44999999TTTT9CSTTTTTN       !> =l K K*O !!!!,!!!%+!x%%%! ())Q../112O+H4333337.7.7.7.7.7.7.7.7.7.7.7.7.7.7.r,AG          '&s++-@-@-E-EF 1:abb>WQZ^ + + GAJ/0G /";;==Sq6CF?T))NN3'''NN3'''))3q6CF?a*?@@qrr7c!""g%%NNC:...qrr7c!""g%%NNC:... /  ''''''0000000000 7 ### 8C000 ))  FAs LL hqk"*      ?### 5666 NOOO ggii  )++  * 65G K$ / / / /   ) ) + + B0A0P0P0R0R BA5#4#=#=#?#?AAK7<< TKTT:L:N:NTTT   H ;XFFF % $#yk     rQdict[torch.fx.Node, str]citt|jD] }|jdkrq|jd}t |tjrdt|j |j f}nIt |tj rd}n,d}n)|jdkrd}n|jt|j f}ttfd|jD}t#jt||fd|<"S) aCompute a forward-looking structural hash for each node. Each node's hash captures its "role" in the graph toward the output: hash(self_key, sorted(cone_hash(user) for user in users)). For placeholders, self_key uses tensor metadata instead of the node name (which varies across ranks). This makes the hash invariant to the original node naming and ordering. rr)rsymintrr)rc3(K|] }|V dSrLr`)rprhashess rOrrz_cone_hashes..L s'"A"A6!9"A"A"A"A"A"ArQr)rrrrrrrrr^rr requires_gradrrrrwrhashlibsha256encode hexdigest)rrHrself_key user_hashesrs @rO _cone_hashesr- s@(*Fek**++ 7m # #)--&&C#u|,, ," NN%- C.. ,4+ W "HHT[!1!12HF"A"A"A"Adj"A"A"AAABB ~ ;' ( ( / / 8 8  )++ t  MrQc t| t|jd}|jD]}|jD]}||xxdz cc<i d fd }d}g}|jD]4}||dkr&t j|||||f|dz }5g}|rt j|\}}} || t  | <| jD]D}||xxdzcc<||dkr&t j|||||f|dz }E|dt|DS) aBuild a canonical name mapping for graph nodes using Kahn's algorithm. Returns a dict mapping each node to a deterministic name like "node_0", "node_1", etc. The mapping is invariant to the original node ordering and naming, so structurally equivalent graphs (e.g., traced with different dict iteration orders across distributed ranks) produce identical mappings. Does NOT modify the graph. rr'rHrKrJrc|jdkrz|jd}t|tjrt |j|jf}nt|tj rd}nd}d||fS|jdkrdt |j fS|jdkrd Stfd |j D}d t |j |fS) Nrr)rr`rrr'r)c3(K|] }|V dSrLr`)rprq canonical_idxs rOrrz@_canonical_node_names.._canonical_key..{ s(!Q!Qq-"2!Q!Q!Q!Q!Q!QrQr>) rrrrrr^rrrrrrr)rHrmeta_key input_indicesrcones rO_canonical_keyz-_canonical_node_names.._canonical_keyg s 7m # #)--&&C#u|,,  NN%-C.. &xd, , W " "s4;''( ( W 4!!Q!Q!Q!QD. s$ H H H'!TD+!++ H H HrQ)rHrKrJr) rrr]rrrSrTrUrrr) rindegrHrr!rreadycanonical_orderrSr-rr s @@rO_canonical_node_namesr'T s   D&*mmEK&C&CE J  D $KKK1 KKKK /1M88888882G8:E  ;!   N5>>$#7#7$"G H H H qLG%'O  M%(( 1cs### // cI  D $KKK1 KKKT{au~~d';';Wd&KLLL1    I HY-G-G H H HHrQlist[torch.fx.Node]cddlmdd}tjrKtjr-tjdkr ||s|St|d Ddfd }||rt55fd |Dg}d ttjD}tj ||dg}i}t|D]t\}} fd | D} d} | D]B} t| } | | z } |tjkr | || j<C| |d<|| utj|tjj}tj|tjjjjt/tj|}d|d|t5ddfdfd||D}dddn #1swxYwYdddn #1swxYwY|S)Nr)unset_fake_temporarilyrrJrJr>c|jD]7}t|jtjjr|jjdvrdS8dS)N>c10d_functionalrTF)rrrrrrr)rrHs rOhas_collectivesz3_sync_decision_cross_ranks..has_collectives sP%  D UZ2 +'+RRRtturQr'ci|]\}}|| Sr`r`r@s rOrz._sync_decision_cross_ranks.. s<<<$!QA<<.has_same_nodes.._node_hash_str sPt}$$#A,/////l66QT66AH66 6rQ/c3.K|]}|VdSrLr`)rprqr4s rOrrzE_sync_decision_cross_ranks..has_same_nodes.. sA   N1        rQc|SrLr`r2s rOrtzD_sync_decision_cross_ranks..has_same_nodes.. s Yq\rQrurcg|]}dSrLr`rprSs rOrzF_sync_decision_cross_ranks..has_same_nodes.. sNNNqdNNNrQc30K|]}d|kVdSrr`)rpr all_inputss rOrrzE_sync_decision_cross_ranks..has_same_nodes.. s,::!:a=A%::::::rQ)rqrKrJr)rurwrrrrrrDrrget_world_sizerzall_gather_objectr)rnode_strrdr4r;r3r*s @@rOhas_same_nodesz2_sync_decision_cross_ranks..has_same_nodes s 7 7 7 7 7 788    K-3I3I3I3IJJJ      8 899CCEENNE%*;*J*J*L*L$M$MNNN ]] D D2244 D D   / / F C C C D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D::::z::::::s6> D  !C6* D 6C: :D =C: >D  DDc g|] }| Sr`r`)rprr3s rOrz._sync_decision_cross_ranks.. s;;; ! ;;;rQcg|]}gSr`r`r9s rOrz._sync_decision_cross_ranks.. s%::::::rQc g|] }| Sr`r`)rpop_namereverse_canonicals rOrz._sync_decision_cross_ranks.. s,3:%g.rQz total size)rr1zpicked_rank_idx=z, saved_nodes of current rank=rc dddS)N)aot_joint_graph_sync_decision_cross_ranksrrr`r`rQrOrtz,_sync_decision_cross_ranks.. sG (%%rQcSrLr`)sync_decision_cross_ranks_strsrOrtz,_sync_decision_cross_ranks.. s#@rQrc g|] }| Sr`r`)rprqrDs rOrz._sync_decision_cross_ranks.. s,)*!!$rQ)rrJrJr>)torch._subclasses.fake_tensorr*rrrrr<r'rrzrDr=rrrrrrsdistributed_c10d_get_object_coll_device all_reduceReduceOpMAXrargminrr)rr:r-r?objectssaved_ops_names_all_ranks saved_sizessaved_ops_with_sizesrEsaved_ops_names saved_nodes saved_sizerH size_of_nodesaved_sizes_tensorpicked_rank_idxr3rDrHr*s @@@@rOr^r^ sEDDDDD &&((   , , . .   , , . . 2 2 OK ( ( 3%k22I<<)//*;*;<<<;;;;;;;.~k"". ]]- - 2244- - <;;;l;;;M  'GGD#+D>>L,.Je/88:::::F,TY75?$\2"":....!&(9QQSS"""    ( ("u'8'I'R'V )   "%,/A"B"B"G"G"I"IJJO-E-E-EoC-E-E ) A@@@     .G.XLW- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ^ s7 J?G J( J?(J, ,J?/J, 0J??KKmodulercrd}|rdnd}tt|jd}|jdtjjjD]}t||j dj }t|tj rg}|jdD]}||jvr|j|5|j|d|} |d z }|jd | jd <| }|| d d d n #1swxYwY|r|j|5|jdtjjjg|j |Ri} || d d d d n #1swxYwY|jd} | r"| \} } g| d|DR}|| f| jd<|j||S)u Graph-safe RNG lets torch.compile use CUDA Graphs for graphs with RNG ops. For graphs without HOPs, the partitioner adds placeholder nodes fwd_rng_state_* and bw_rng_state_* to the forward and backward graphs. At runtime, the AOTDispatcher retrieves these RNG states and passes them to the compiled graphs. This works well for no-HOP graphs. With HOPs, the partitioner runs recursively: it first partitions the HOP (producing forward/backward HOP subgraphs) and then stitches them back into the outer joint graph. For HOPs that contain RNG ops, the outer joint graph now includes HOP subgraph modules with extra RNG placeholders. We must thread these placeholders through the outer module partitioned forward and backward graphs—this function does exactly that. It collects the RNG placeholder nodes from the HOPs and creates corresponding placeholders in the outer forward and backward graphs. There is a catch: for a short period, the joint graph is in a “bad” state. The HOP subgraphs expect additional inputs (because of the new placeholders), but the outer graph call sites don't yet provide them. We can't fix this in the joint graph because the joint graph's input signature is fixed (primals, tangents). As a compromise, we keep the joint graph in somewhat of a bad state for some time and, once the outer forward and backward graphs are partitioned, insert the corresponding RNG placeholders and wire up the calls. rrrrr1rr1rSr'rNT)propagate_metaeager_input_valsc(g|]}|jdS)r)r)rpinps rOrz2thread_graphsafe_rng_from_hops..AsDDDc#(5/DDDrQ)r3rrr2rrrinvoke_subgraphrrrrrrrrUrrrrrrr)r[rr rng_string last_inputhop_nodernew_rng_inputsplaceholder_nodernew_hop_node_with_fixed_args eager_vals eager_args eager_kwargsnew_eager_argss rOrbrb s<I$/D_Jhv|66-6HHIIJJJL++ 59#9#I,,2,268=#3#:;; h / /( 2,.N$,N$=$=$=$O$O 9 9 !1!666 55jAA99$*L$<$<)77I77%% "Q 0@0Ee0L u-%. &--i888999999999999999 2\11(;;  39<3K3K' .>9(-9.99 440 224T3               &]../ABB  /9,J &#&DD^DDD&&N '$M056HI ''111 Ms&.AE  E E 3AGG G list[int]c t|jt|jjD]n}|jdkrd|jvr|n$t|r||vr|j ottt|jj}ttt|jj}||z}t||\}}} } } d|Dt!|j||| d} tfd| jDtfd|jjD} tfdt#|D}d }i}|jjD]}|vr |||<|d z }t%|| | ||S) Nrtangentsrc34K|]}||jdk|VdS)Nrr1)rprs rOrrz!classify_nodes..is;!-ADH4D4D4D4D4D4DrQrc3HK|]}|jdk |jVdSr1r2rs rOrrz!classify_nodes..osC88  7h   TY    88rQc3,K|]}|v|v |VdSrLr`)rprHrgrys rOrrz!classify_nodes..tsF66  ( ( (T9J-J-J -J-J-J-J66rQc3*K|] \}}|v |VdSrLr`)rprFpr,s rOrrz!classify_nodes..ys;--a!7T2T2T2T2T2T2T--rQrr')rrr%rrrrrrrrrr6r5r9rr rrb)r*r,r+rHrrrdr5r6r7r8rhforward_only_graphrirlfw_cntrkrrgrys ` @@@rOrSrSLs $L$677L-7\\"(11 7m # # dk(A(A  ! !$ ' ' ' ' !$ ' ' (  ! !$ ' ' ' $ $ $  $ $TZ 0 0 0 L,>,DEEFFM!&)P>V"W"WXX 3 3F OOOCK/1B)--//<FK1BI.88888&,888.. ,666666 &,666,,O #----- ..---##FH"( $ $ $#HTN aKF #  rQr)r,compilerc( |j||j}tjrt |}||_|j}t |}t|} |rt|d}tj st|t|t||g}t|||} t| jdkrt!||||| jSt%|jjD]} | jdkrt+d| _"| | sd| _?t+d| _| jD]$} t3| j| jdz| _%tj} |jD]?} t7| jd dt<r| jd } n@t?}g}|jD]^} tA| }||!|)| jd kr*| | r|"| _t?|}|#D]~\}}t7|tHj%r_|j|urV|jjD]I} tA| }||!|)| jd kr|"| Jt|dkr tMd tO|d |rK|r-tMd t|dd|DdtQtS|} tU|| | }tj+rtW||}tYt[d|}tYt[t\|}tYt[d|}t_|||||| j\}}|r$| r"ta|||t|\}}tc|}tj2rddl3m2}||||| jti|}ti|}tk|d}tk|d}tlrtOd|D}tod|Ddz }tp9d|tp9d|t?d|jjD}t?d|jjD}||z}tut*} |jjD]G} | j;|vr.s$H$H$HQV$H$H$HrQr)rc@t|ot| SrL)rrrqs rOrtz5min_cut_rematerialization_partition..*sk!nnG-DQ-G-G)GrQcBt| ot| SrL)rr2rzs rOrtz5min_cut_rematerialization_partition../s[^^+EN14E4E0ErQrFrHrJTcJg|] }t|t|f!Sr`)rrr s rOrz7min_cut_rematerialization_partition..Zs)KKK SVV4KKKrQc34K|]}t|VdSrLrr s rOrrz6min_cut_rematerialization_partition..]s('J'J 'J'J'J'J'J'JrQz'Theoretical Activations Stored: %.2f GBz,Theoretical Per Activation Storage Sizes: %sc3:K|]}|jdk |jVdSrNr2r0s rOrrz6min_cut_rematerialization_partition..b9% % 47o;U;UDI;U;U;U;U% % rQc3:K|]}|jdk |jVdSrr2r0s rOrrz6min_cut_rematerialization_partition..errQrz# remat/fw/bw: %d/%d/%drzCount of Ops Rematerialized: %s)Crrrrcser;rrrNrOrPrQrRrSrrgrkrlrrrrr4r|rrNactivation_memory_budgetrrrrMr%r$rrrTrrrrwr3r4r r^rrr2r+r_r`rIrar=rbr?rzr@rrrrrrrrrr)"r*r-rvr+r,r cse_graphrrerfrgrHrrregion_budgetsunannotated_fw_opsbudget all_budgetsrSrrr:rrrirjrI sorted_sizestotal_activations_size_gbfw_module_nodesbw_module_nodes remat_nodescountsrematerialized_opss" rOrMrMsD**,,,  Dz' && & $K!5l!C!C%=l%K%K"!X-lQVWWW  :-|,,,\***|,,,$,(*%3_I 9 &''1,,  +*G(1(M     +122RR 7h   #CD  ))$// R !D   #CD   R R$'(94;Lq;P$Q$Q!! R3M! dimmOT::E B B  Io6M E )3 N(*!,,.t44     v & & & & W ' 'I,D,DT,J,J '  % %d + + +&0%?%?K,,..443 c2> * * 4sy /K/K  4 41$77=OOA&&&&W//&--d333 ;! 4!+.. 4 4 4    0  L)**LL %I$H5G$H$H$H LLL T+..// *#L (M1+|LL  G G  O f^\BBCCEE|TTL 4'-'$-$IIy" ) #8iC4H4H$$ Iy4I>>I*         %$     1    y))Iy))I.yeLLLI.ydKKKIHKKlKKKLL %('J'J\'J'J'J$J$JS$P! : >DyK''GDKAR,S,S's4;677888A=888  %         $ LLNN 3A 6 6    24FGGG i rQfx_graphTtracedfnamefigname clear_metaprogstr | list[str] | Noneparse_stack_tracedot_graph_shapecx|rDtj|j}tj||}|jjD] }i|_ tj |\} } | sdtj z} t d| | tj||||} | } | | }| d} | dvr| |dS| t'| d| z}||dSt'| d| z}|||dS)NrzWriting FX graph to file: %s%s)rr>dotrawwrite_)r)rdeepcopyrrrrrrrsplitextrtorch_compile_graph_formatr@rr# FxGraphDrawerget_main_dot_graphlstriprr)rrrrrrrrrHrextgr graph_format write_methods rO draw_graphr{s`M&,//  22L&  DDII  ''ID# 6F55HH -tS999"+'    A ANSNNE::c??L~%%  q(\"9::  Uq(\"9::  U&&&&&&rQrZ)rrrJr>)rHrIrJr)rrIrJr)rHrIrJr)rHrIrrrJr)NF)rrrdrcrrcrrrrrr>rJr)r*rr+rrJr,)r:rcrrrJr;)r@rArJr)rHrIr) rrJrHrKrLrMrNrMrOrrJrK)rrJrHrKrfrKrhrir[rMrjrMrOrrJrK)rsrtrJrM)rJrz)rHrKrJr>)rJri)rrirJr)r)rrJr+rrJr;)rrJrJr;) rrrrrrr+rrJr;)Nr) r:rcrrrrrlrr+rrJr;rL)r*rr:rcrrcrrr+rrlrrr>rr>rJr) r*rr-r r+rr,r.rlrrJr)rvrrrirJr)rrrJr;)rJr)rrrrjrJr)rrrJr)rirrjrrrKrrKrrrrrrKrrKrJr) r*rrirrjrrrrJr)r*rrJr;)r*rr4r>rJr) rrrgrbrrrrrJr)rKrrJr)rrrrrJr)rKrrJr)rJrB)rrrJr)rrrarbrcrbrdrMrgrbrercrJrf)rrtrtrrJrt)rHrIrJrMr)rrrgrbrrMrJrc)rrJrJr )rrJr:r(rJr()r[rrr>rJr)r*rr,rlr+rrJrb)r) r*rr-r rvrr+rr,r.rJr)rTNFN)rrrrrrrr>rrrr>rrrJr;) __future__rrrrrSrloggingrrros.pathrrKrrrcollections.abcr dataclassesrrtypingr r rtorch._inductor.inductor_primstorch.distributedtorch.fxrtorch.utils._pytreeutils_pytreertorch._dynamo.utilsr r ;torch._functorch._activation_checkpointing.ac_logging_utilsr $torch._functorch._aot_autograd.utilsrtorch._inductorrr!torch._inductor.custom_graph_passrr"torch._library.fake_class_registryrtorch._library.utilsrtorch._loggingrrtorch._logging._internalrrJr%torch.fx.experimental._backward_stater"torch.fx.experimental.proxy_tensorrrtorch.fx.experimental.sym_noderr%torch.fx.experimental.symbolic_shapesrrrr r!r"torch.fx.passesr#torch.fx.tracebackr$torch.utils._ordered_setr%torch.utils.checkpointr&r-_activation_checkpointing.graph_info_providerr("_activation_checkpointing.knapsackr)r*r+r,,_activation_checkpointing.knapsack_evaluatorr-_aot_autograd.descriptorsr.r/r0_aot_autograd.functional_utilsr1_aot_autograd.graph_compiler2_aot_autograd.logging_utilsr3_aot_autograd.utilsr4r5r6r7r8r9r: compile_utilsr;r<r=rrzsympydebug_partitionerr?r_ getLoggerr[r@rrrXrBrbrrrrrrrrrrrrr rrrrr r#rrr9r?rGrgrrryrrrrrrrrr+rkrr@rnrrcacherrr`rr_rPrQrRr rNrrrrrrrxrrzrrr rr'r^rbrSrMrr`rQrOrs. """""""     ********$$$$$$********%%%%%%%% %%%%$$$$$$$$$<<<<<<<<A@@@@@555555@?????++++++77777777......AAAAAA??????HHHHHHHHLLLLLLLL)(((((<<<<<<//////333333LLLLLL LKKKKK A@@@@@777777;;;;;;IHHHHHHHHHLLL%66666'g'111111 y~  >>>>>>> >2          B       IIIIo 2 - - - -H $) rrrrrjXXXXCCCCBBBBJJJJKKKK    5555JJJJ$ KKKKK\C!C!C!C!L9999"GGGG45555    FGFGFGFGFGRMGMGMGMGh XXXXX~?C ' ' ' ' ' \04 o"?C$)&+o"o"o"o"o"o"n7;>B j j j j j j Z #c((""""))))ZRRRR      PPPPKKKK\[*[*[*[*|a a a a H @ @ @ @''''<.KKKK5555x,0 v &v &v &v &v &rB / / / /$!$!$!$!NffffR-T-T-T-T`0/////5555*N*N*N*N`D D D D D N$$$$N@I@I@I@IF````FOOOOd>>>>Hk 7; k k k k k k b#'#"&%'%'%'%'%'%'%'rQ