tj 5UddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlmZddlmZddl m Z ddl!m"Z"m#Z#m$Z$ddl%m&Z&ddlm'Z'm(Z(dd l)m*Z*dd lm+Z+dd l,m,Z,m-Z-dd l.m/Z/m0Z0m1Z1m2Z2m3Z3dd l4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;ddlZ>m?Z?m@Z@ddlAZAddlBmCcmDZDddlEmFZGddlAmHZHmIZIddlJmKZKddlLmMZMddlNmOZOmPZPmQZQmRZRddlSmTZTmUZUmVZVmWZWddlXmYZYmZZZm[Z[ddl\m]Z^ddl_m`Z`maZaddlbmcZcddldmeZemfZfmgZgmhZhmiZimjZjmkZkmlZlmmZmmnZnmoZompZpmqZqmrZrddlsmtZtmuZuddlvmwZwmxZxmyZymzZzm{Z{m|Z|m}Z}m~Z~ddlmZmZddlmZddlmZmZddlmZmZmZmZmZmZmZddlmZdd lBmZdd!lmZdd"lmZmZmZdd#lmZdd$lmTZdd%lmZmZmZdd&lmZmZdd'lmZdd(lmZmZmZdd)lmZdd*lmZd+d,lmZmZmZmZd+d-lmZd+d.lmZmZd+d/lmZd+d0lmZd+d1lmZd+d2lmZe;d3ZGd4d5e@ZGd6d7e@d89ZGd:d;e@ZGd<d=e@d89Ze:rNdd>lmZmZmZmZdd?lmZd+d@lmZd+dAlmZd+dBlmZd+dClmZd+dDlmZmZd+dElmZmZd+dFlmZmZd+dGlmZd+dHlmZejdIkZeTjZeAjedJZeAjedKZe jeZdLZddQZe'dddSZe'dddTZd dUZd dWZd dZZdd[lmZed d]ZGd^d_ZGd`daeZGdbdceZd ddZd dgZdddlZ dddqZdddtZGdudvZ dddyZdd{Z dddZejGddZddZddZddZe1e/e0e2efZddZGddejZddZeddZ d dZ ejGddZ Gdde Z ejdGddZeZded<ddZddZd dZejGddZGddZd!dZd"dZGdde7eZejGddeZGddeeZejd#dZeGdd¦ZeGdÄdĦZGdńdƦZ da!dǐed<d$d̄Z"e j#$edͦZ%e j#$e%dΦZ&dϐZ'ejd%dӄZ(d&dԄZ)d'd(dքZ*d)dۄZ+eGd܄dݦZ,d*dZ-eGdde,Z.eGdde.Z/eGdde.Z0d+dZ1d,dZ2eGddZ3d-dZ4eddZ5GddZ6e'd.dZ7GddZ8eGdde8Z9ddl:m]Z;eGdde8ZGdde>Z?Gdde>Z@dS(/) annotationsN) bisect_right)Set)copy)c_void_pCDLLcdll) timedelta) lru_cachepartial)Path)_TemporaryFileWrapper)timetime_ns)BuiltinFunctionTypeBuiltinMethodType FunctionType MethodType ModuleType)AnycastGenericLiteralNoReturn TYPE_CHECKINGTypeVar) NotRequiredoverrideSelf TypedDict)SymIntTensor)get_interface_for_device) SkipFrame)CompileEventLoggercounters dynamo_timedget_metrics_context)config config_commsexcmetrics)custom_backend_codegen_configscustom_backend_passesinit_backend_registration) compile_utils)rocm_compile_command rocm_compiler)in_toplevel_process)_LINKER_SCRIPT_set_gpu_runtime_env _TORCH_PATHbatch_convert_cubins_to_objconvert_cubin_to_obj CppBuilder CppOptionsCppTorchDeviceOptionsget_compiler_version_infoget_cpp_compilerget_ld_and_objcopy&get_name_and_dir_from_output_file_pathnormalize_path_separatorrun_asm_build_object)invalid_vec_isa pick_vec_isa)CustomGraphModulePassCustomGraphPassCustomGraphPassCallableCustomPartitionerFnCustomPartitionerFnTypeCustomPassBaseCustomSchedulerPassget_custom_graph_passes)has_frozen_paramsis_frozen_param)_reload_python_module) cache_dirdefault_cache_dir) ALIGN_BYTESclear_on_fresh_cachedetermine_aoti_mmap_flagsis_linux is_windowsparallel_num_threadsXPU_KERNEL_FORMAT)FakeScriptObject)is_opaque_reference_type)trace_structured)extract_tensor_metadata FakeTensorTensorMetadata)log_cache_bypass)r)) CacheArtifactCacheArtifactFactoryCacheArtifactRecorder)TensorPropertiesWeights)CUSTOM_OBJ_FILENAME_PREFIX)guarding_hint_or_throwhas_guarding_hintShapeEnv)_device_constructors) OrderedSet)CODE_CACHE_KEY_STRATEGYCOMPACT_CACHE_KEY_STRATEGYFX_GRAPH_CACHE_KEY_STRATEGYSYSTEM_CACHE_KEY_STRATEGYCompiledFxGraph) cache_stats create_cache)autotune_cache)AutotuneCacheBundler) TritonBundler)VTceZdZUded<dS)SystemDeviceInfo str | NonenameN__name__ __module__ __qualname____annotations__W/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/torch/_inductor/codecache.pyryrysrryc.eZdZUded<ded<ded<dS)SystemVersionInforztritonstrcudahipNr|rrrrrs. IIIOOOOOrrF)totalc.eZdZUded<ded<ded<dS) SystemInforhashzNotRequired[SystemDeviceInfo]devicezNotRequired[SystemVersionInfo]versionNr|rrrrrs3 III))))++++++rrceZdZUded<ded<ded<ded<ded<ded <ded <ded <ded <ded <ded<ded<ded<ded<dS) CacheInfoz Literal['bypass', 'hit', 'miss'] cache_statercache_status_detailedcache_status_guard_exprtriton_bundler_metaint time_saved_ns time_taken_nsephemeral_timeout_increasecache_bypass_reasoncache_event_timekey list[str] componentscache_bypass_exception_typecache_bypass_tracebackboolcache_bypass_hard_exceptionNr|rrrrrs1111    #### HHH$$$$%%%%%%%%%%rr)Callable GeneratorKeysViewSequence)Future)_CompileFxKwargs)BuildOptionsBase GraphLowering) ChoiceCaller)CompiledFxGraphConstants OutputCode) JsonDataTy RemoteCache)HalideInputSpec HalideMeta)CachingAutotuner) InputTypewin32 output_code autotuningatargetrreturnrc ddt|ddt|ddt|ddt|fDS)N.c3K|]}||V dSNr.0xs r z/_device_constructor_sort_key..s>           rr~rr})joingetattrreprrs r_device_constructor_sort_keyrsn 88   FL$ / / FND 1 1 FJ - - LL        rtuple[Any, ...]c`ttttS)Nr)tuplesortedrhrrrr#_default_device_constructor_targetsrs% ,..4PQQQ R RRrcg}tD]M}t|ddx}ttjj|dx}||Nt |S)Nr})rrtorchopsatenappendr)packetsrr{packets r,_default_device_constructor_overload_packetsrspG577##FJ55 5D > eindD99 9F F NN6 " " " >>rc,tjjdndS)N cubin_path hsaco_path)rrrrrrget_cpp_wrapper_cubin_path_namers =,4<<,FrrcV|dkrtjjdndS|dkrtSdS)Nrcubinhsacoxpu)rrrrWrs rget_kernel_bin_formatrs5 -+3ww@ 5  rr device_typedict[str, str]c"tjtjttjjtt| d}|S)zJ Gets all the current device information used to compile the .so. ) AOTI_PLATFORM AOTI_MACHINE AOTI_CPU_ISAAOTI_COMPUTE_CAPABILITY) sysplatformmachinerr _inductor cpu_vec_isarCupperr#get_compute_capability)rmetadatas rget_device_informationrst  (**EO7DDFFGGMMOO#& $[ 1 1 H H J J$ $   H Or)prefetchable_cacherzc.ddlm}m}|r |SdS)Nr) HAS_TRITON triton_key)%torch._inductor.runtime.triton_compatrr)r_triton_key_impls rrrsH"!!! 4rceZdZeejd dZeeejddZddZ dd Z dd Z d S) CacheBaserrc$td5t}dddn #1swxYwY ddi}d|i}tjtj}tjj|j|d<tjj|d<n|j|d<tjj |d<||d}||tj |dS#ttf$rdtj iicYSwxYw) NzCacheBase.get_system.triton_keyr{rrr)rr)rrrr)r'rrrget_device_propertiescurrent_devicerr{ gcnArchNamerrn key_from_jsonAssertionError RuntimeError)triton_version device_info version_infodevice_properties hash_inputs r get_systemzCacheBase.get_systemsu; < < * *'\\N * * * * * * * * * * * * * * * I-3TNK/7.HL % @ @ ))++!! }!-&7&< F#',}'9 V$$&7&C F#&+m&7 U#%'**J &'1? KK   - I I I5CBGGH H H H Is+//B-C%%'DDr cttjt dt dS)Ncacher)r ospathrrOrrrrrget_local_cache_pathzCacheBase.get_local_cache_path<s9BGLLgy7K7K7M7Mf7UVVWWWrNonecBt|_dSr)rrsystemselfs r__init__zCacheBase.__init__Bs**,, rdict[str, Any]c|}|siSt|5}tj|}dddn #1swxYwY|dS)Nr)r is_fileopenjsonload)rlocal_cache_pathlocal_cache_fp local_caches rget_local_cachezCacheBase.get_local_cacheEs4466'')) I " # # 4~)N33K 4 4 4 4 4 4 4 4 4 4 4 4 4 4 47##sAA"Arc|}tt|tj|j|ddddS)N)r rindentT make_dirs)r  write_atomicrrdumpsr )rrrs rupdate_local_cachezCacheBase.update_local_cacheMs`4466  ! ! J$+ DDQ O O O      rN)rr)rr rr )rr)rrrr ) r}r~r staticmethod functoolsrrrRr rrr#rrrrrs_III_\I:_XXX_\X----$$$$      rrceZdZd dZd d Zd S) LocalCachekeysrrdict[str, Any] | NonecZ|}|}|D]}||vr ||}dS|Sr)r)rr)r sub_cachers rlookupzLocalCache.lookupWsH$$&&   Ce||!#J ttrvaluerr c|}|}|ddD] }||i||}!|||d<||dS)Nr)r setdefaultr#)rr.r)rr,rs r set_valuezLocalCache.set_valuecsx$$&& ": ' 'C  b ) ) )!#II# $r( &&&&&rN)r)rrr*)r)rr.rrr )r}r~rr-r2rrrr(r(Vs<     ' ' ' ' ' 'rr(ceZdZ ddd ZdS)PersistentCacheNchoiceslist[ChoiceCaller]oprinputs benchmark1Callable[[Any], dict[ChoiceCaller, float]] | None hint_override int | Nonerdict[ChoiceCaller, float]cN tj ||d|n| i d  fd }tjr|ni}||s|| t fdDsJ|i| i i D].\}} | |  |</| | S) a Check to see if we have benchmarked the given choice callers. For each choice caller: 1. Check local_cache[op][inputs][choice][precision], return benchmark if cached. 2. If benchmark is not None: a. `max_autotune_gemm=True`: benchmark the choice, update local_cache[op][inputs][choice], and return the benchmark. b. `max_autotune_gemm=False`: don't benchmark the choice, return nothing. N_rrrrcd}D]v}|}||iiivr|||<td}|S)z2Check if `cache` contains data for all the choicesTF)hash_keyget) rhitchoice choice_hash cache_keyr5r7 precisiontimingss r check_cachez+PersistentCache.lookup..check_cachesC!  $oo// %))B"3"3"7"7 2"F"F"J"Jr##',Bi &:9&Ek&RGFOO CJrc3 K|]}|vV dSrr)rrDrHs rrz)PersistentCache.lookup..s(??Vv(??????r)rrrr) rget_float32_matmul_precisionr)autotune_local_cacherallr1itemsrAr#) rr5r7r8r9r;rIrrDtimingrFrGrHs `` @@@rr-zPersistentCache.lookuppsw$688 3@3Lv// ///RX            170KSd**,,,QS  K(( 1y/Di((G????w????? ? ??  " "2r * * * O & &y" 5 5 @ @B O O O")--// R RKQ B *95foo6G6GHH  # #K 0 0 0rr) r5r6r7rr8rr9r:r;r<rr=)r}r~rr-rrrr4r4os2%) 3333333rr4ctjtd}tj|stj|d|S)NlocksTexist_ok)rr rrOexistsmakedirs)lock_dirs r get_lock_dirrWsKw||IKK11H 7>>( # #- Ht,,,, Ordatabytesc*tj|Sr)rlr)rXs r sha256_hashr[s % )$ / //rrcode str | bytesextracX|rtj||Stj|Sr)rkr)r\r^s r code_hashr`s. 8&*4777 " &t , ,,rbasename extension specified_dirtuple[str, str, str]c`|rOtj|r|}natjt |}n4tjt |dd}tj||d|}|||fS)Nrjr)rr isabsrrO)rarbrcsubdirr s rget_pathris: 7== ' ' >"FFW\\)++}==FFikk8AaC=99 7<<8 9 9i 9 9 : :D VT !!rcontent hash_typec|dvrt||S|ddthvrtt|Std|)N>ptxspvr\amdgcnrrzUnknown hash type )r`rWrr)rjr^rks rget_hashrps\444%(((Wg'8999g''' 9i99 : ::rc6eZdZdZ dddddd ZddZddZdS)WritableTempFileao Avoid "Permission denied error" on Windows: with tempfile.NamedTemporaryFile("w", suffix=".gv") as temp_file: # Not writable on Windows: # https://docs.python.org/3/library/tempfile.html#tempfile.NamedTemporaryFile Example: with WritableTempFile("w", suffix=".gv") as temp_file: tree.to_dotfile(temp_file.name) wN)encodingsuffixmoderrtrrurr c0||_||_||_dSr)rvrtru)rrvrtrus rrzWritableTempFile.__init__s    r_TemporaryFileWrapper[Any]chtj|j|j|jd|_|jS)NF)rtrudelete)tempfileNamedTemporaryFilervrtru temp_filers r __enter__zWritableTempFile.__enter__s4!4 I dk%   ~rexc_typeexc_valexc_tbc|j tj|jjdS#t $r}t rn|Yd}~dSd}~wwxYwr)r}closerunlinkr{OSError _IS_WINDOWS)rrrres r__exit__zWritableTempFile.__exit__st   Idn) * * * * *     s; A AA)rs)rvrrtrrurrr )rrx)rrrrrrrr )r}r~r__doc__rr~rrrrrrrrss  26d      rrrrtuple[str, str]c|#t|||}t|||\}}}tj|st ||d||fS)NTr)rpstriprirr rTr!) rjrbr^rkrcrra_subdirr s rwritersq {w}}y99&sI}EEHgt 7>>$  4T7d3333 T>rtextc.t|ddS)zT Write the `text` to a file and return the path computed based on the hash. txtrjr)rs r write_textrs u  a  rpath_r r encode_utf_8r ct|ttfs Jdt|}|r|jdd|jdt jdtj dz }t|trdnd}| ||rdnd 5}| |dddn #1swxYwY | | dS#t$r6tstj|| t j|YdSwxYw) Nz6Only strings and byte arrays can be saved in the cacheT)parentsrSrz.tmprswbutf-8rtr)srcdst) isinstancerrYr parentmkdirrgetpid threading get_identrrrenameFileExistsErrorrshutilcopy2remove)rrjr rr tmp_path write_modefs rr!r! s gU| , ,@ , ;;D7 $666{JJJy/B/D/DJJJJH"7C00:dJ z|,MGG N NRS  t$$$$$     t,,,, (s$;CC!$C!)D} |t ||/#t $rY;wxYw|r|dt |SdS)aProduce a deterministic string key for an otherwise-unpicklable object. Used by FxGraphCachePickler.reducer_override as a fallback for objects whose types don't support default pickling (e.g. pybind11 enums). The key is derived from the object's fully-qualified type name plus values obtained via known accessor patterns (pybind11 enum, Python enum, etc.). Returns ``None`` if no accessor succeeds, letting the caller decide how to handle the failure. rc|jjSr)typer{os rz%_get_stable_obj_key..js !&+rc|jSrr{rs rrz%_get_stable_obj_key..ks!&rc|jSrr.rs rrz%_get_stable_obj_key..ls!'r:N)rr~rrr Exceptionr)rrtype_idpartsaccessors r_get_stable_obj_keyr[s S A0000G E   LLXXc]]++ , , , ,    H  *))DKK))) 4s+A A'&A'ceZdZUdZiZded< d-d.fd Zd/dZed/dZ d0dZ d1dZ d2dZ d3dZ d4dZd5d Zd6d#Zd7d%Zd8d'Zd8d(Zd9d,ZxZS):FxGraphCachePicklera: Custom pickler to customize the pickling of some objects (Tensors), only for the purpose of computing a hash for keying into the FxGraphCache. Tensors contain objects that don't pickle and/or vary between runs, and we want to capture the data that allow us to compute a stable, but safe hash. zdict[type, bool]_pickleable_type_cacheFgmtorch.fx.GraphModulehas_user_defined_triton_kernelsrdevice_id_agnosticrr crtj|_||_t |jt j|_|j ttj |j tjtj |jtjjjtj |jtjtj |jtjtj |jtjjjjtj |jt:tj |ji|r&tj |j|j|j <d|_!dS)a Create an FX graph pickler. If include_non_inlined=True, then pickling will include the _values_ for all Tensors. (Note that any tensors are constants attached as attributes to the GraphModule). Otherwise, pickling will include only the metadata for these tensors. If device_id_agnostic=True, device indices in TensorMetadata are normalized to 0, so that the same graph on different GPUs produces identical bytes. TN)"ioBytesIO_stream_device_id_agnosticsuperrcopyregdispatch_tablerupdater\r&r _reduce_fake_tensorrr"_reduce_tensornn parameter Parameterr!_reduce_symintSymBool_reduce_symboolfx experimental_backward_state BackwardState_reduce_unsupportedrX_reduce_fake_script_object_reduce_graph_module __class__fast)rrrrrs rrzFxGraphCachePickler.__init__sKz|| #5  &&&%499;; ""I-d.FGG i/0CDD",i.?@S.T.T i/0CDD y01EFF%5CYEV,FF!)"3D4S"T"T   + 090A)11D  -  rrrc@t|}||jvst|trtS|j|x}|s||StS |tj }n#tttj f$r"d|j|<||cYSt$r;}dt|vr$d|j|<||cYd}~Sd}~wwxYwd|j|<|S)a_Fallback reducer for objects not registered in dispatch_table. This handles extension types (e.g. pybind11 enums) that don't support default pickling. Instead of bypassing the FX graph cache entirely, we serialize a deterministic string representation of the object which is sufficient for cache-key hashing. NFis not pickleableT)rrr_PICKLE_NATIVE_TYPES_TUPLENotImplementedrrB_reduce_unpicklable __reduce_ex__pickleDEFAULT_PROTOCOL TypeErrorAttributeError PicklingErrorrr)rrr pickleableresultrs rreducer_overridez$FxGraphCachePickler.reducer_overridesJ II # # #z#7Q'R'R #! !599!<< ??FF>6+?@ 1 1 1-2D ' *++C00 0 0 0   "c!ff,,16+A.//44444444    *.#A& s*0B;D D/D D D  Dct|}|$tdt|jt|ffS)Nz5Cannot produce stable cache key for unpicklable type )rBypassFxGraphCacherrr)rrs rrz'FxGraphCachePickler._reduce_unpicklablesP!#&& ;$,99),, "C6))rrr".tuple[Callable[[T], T], tuple[TensorMetadata]]ct|}|jr3tj|t j|jjd}t|ffS)z7 Custom reducer to pickle FakeTensors. rr)rrrrrrrr)rrrs rrz'FxGraphCachePickler._reduce_fake_tensorsX 9;;  # "*ho.BA!F!FH $$rHtuple[Callable[[T], T], tuple[TensorMetadata | TensorMetadataAndValues]]cddlm}|jrt|t |}|jr3tj|tj |j j d}t|r|j |s t|ffSt}|}t|z }|dkrt#jd|ddtt'||ffS) z Custom reducer to pickle Tensors. If we see tensors, we know they're constants stored as attributes on the GraphModule. rjrrrg?z0FX graph cache copying of a large constant took z.1zs. Please file an issue.)graphr is_mkldnnCacheabilityValidator check_tensorrrrrrrrrMcan_inline_constantrrtolistwarningswarnr)rrrrstartrelapseds rrz"FxGraphCachePickler._reduce_tensors% )((((( ; 2 " . .q 1 1 18;;  # "*ho.BA!F!FH 1   )&Gm&G&J&J )XK( (&&5. S== M(7R(((    06BBDEErsr!#tuple[Callable[[T], T], tuple[str]]c0tt|ffS)z3 Custom reducer to pickle SymInts. rrrrs rrz"FxGraphCachePickler._reduce_symintsQ ""r torch.SymBoolc0tt|ffS)z4 Custom reducer to pickle SymBools. rrs rrz#FxGraphCachePickler._reduce_symbools Q ""rrc:tddS)z{ Custom reducer to handle any objects that we don't support and therefore raise to bypass caching. Reduce unsupportedN)r bypassrs rrz'FxGraphCachePickler._reduce_unsupported!s $$%9:::::r&tuple[Any, tuple[dict[str, Any], str]]c|\}\}}|d}tjdd|}tjdd|}||d<|||ffS)a Custom reducer for graph module to handle irrelevant data for user defined triton kernels Essentially what we are doing here is a huge hack where user defined triton kernel contain a dynamo time side table and the arguments to the call_function are indices into this side table. These arguments are not for hashing purposes since we included the source code into the cache key and the numbers are prone to give false negatives due to ordering. _codezkernel_idx = \d+rzconstant_args_idx = \d+) __reduce__resub)rrfnrXimportsr\s rrz(FxGraphCachePickler._reduce_graph_module(sc!mmooOT7G}v)2t44v0"d;;W D'?""rrX*tuple[Callable[..., Any], tuple[Any, ...]]c:|jzt|j}tj|r7tj|r#tj|st |jffStj|rd|_t |j |j|jffSNF) real_objr opaque_objectis_opaque_type should_hoist has_membersrscript_class_namer wrapped_obj)rrclss rrz.FxGraphCachePickler._reduce_fake_script_object;s : !qz""C ,S11 8!.s33 8&1#66 8 !4 677+C00 "" (;QZHIIrrYc |||j|jd|jdS#t t tjtf$r$}t |Yd}~nRd}~wt$rB}dt|vr+dt|vrt |d}~wwxYw |jd|jddS#|jd|jdwxYw)z< Pickle an object and return a byte string. rNpybind11r)dumprgetvalueseektruncaterrrr ValueErrorr bypass_for_pickle_errorrr)rrrs rr"zFxGraphCachePickler.dumpsPsg % IIcNNN<((** L  a L ! !! $ $ $ $>6+?L = = = ! 9 9! < < < < < < < <   SVV##(;s1vv(E(E%==a@@@    = L  a L ! !! $ $ $ $ $ L  a L ! !! $ $ $ $s5-A$$"C4B% D.% C42=C//C44D..6E$rcT||}tj|S)zE Serialize an object and return a hash of the bytes. )r"rlrrrserialized_datas rrpzFxGraphCachePickler.get_hashds$**S//)-o>>>rcT||}tj|S)zG Serialize an object and return an FX graph cache key. )r"rmrr8s rget_keyzFxGraphCachePickler.get_keyks$**S//*.???rinpFxGraphHashDetailsrcd fd }g}t|D]1\}}t|trkt t |D]M}||}|d|d|d|d|||Nt|trZ|D]D\}} | }|d|d|d|d|| E|}|d|d|d ||3|S) z Get a printable string describing in more detail all the attributes comprising an object. Useful for debugging when one graph hashes to a different value than another. rrrrct|tjrtt |St|t r9|dd}t|dkr|n |dddzSt|j vr6tj t||dSt|S)Nrr)errorsz...rj) rrr"rrrYdecodelenrr)rvalrs rget_strz0FxGraphCachePickler.debug_lines..get_strys#u|,, @EEFFFC'' jjj;;!#hh$..ssCJ4FFcd11194.tCyy9#>>qABBB3xxr[z] z]: z: rrrr) varsrNrlistrangerCrprdict) rr<rElinesattrriihkvs ` r debug_lineszFxGraphCachePickler.debug_linesrs c** > >ID##t$$ >C//LLB c"g..ALL!JQ!J!J$!J!J!J!JB8H8H!J!JKKKKLC&& >IIKKEEDAq a((ALL!CQ!C!C$!C!C!C!Cwwqzz!C!CDDDDEMM#&& <<.srrrbrr)rpkgutil iter_modules module_finder find_specr{originrrencodereadispkgbuild_code_hashsubmodule_search_locations)rUrWrXlibspecmodulers rreresAg*5&99?O?OPPP V V **38T::!!! &$   $1 MM$)**733 4 4 4 MM!&&(( # # # $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ 9 V D; ___f U U U V Vs'ACC C c>tdd5tjs dd}|tcd d d Sd d lm}|d d cd d d S#1swxYwYd S)zS Compute a key that contains relevant information about torch source files inductor_codecache_torch_keyFlog_pt2_compile_eventrootrrrYc(d}tjtfd|D}t j}|tj dt|gd||D]p}tj |rOt|d5}|| dddn #1swxYwYq|S)N)z"codegen/aoti_runtime/interface.cppz script.ldcPg|]"}tj|#Sr)rr r)rr inductor_roots r z4torch_key..get_code_hash..s)SSS!rw||M1==SSSrrrr\)rr dirname__file__hashlibsha256rr __version__rbrerTrrcdigest)rn extra_filesrXr rrqs @r get_code_hashz torch_key..get_code_hashs, !# 9 9 SSSS{SSS  )) e/66w??@@@F333'44Dw~~d++4!$--4"MM!&&((333444444444444444}}&s(C44C8 ;C8 Nr)parutilztorch/src_hash.txtascii)rnrrrY) r'r) is_fbcoder6libfb.pyr{get_file_contentsrstriprb)rzr{s r torch_keyrs4 4E R R RXX!! . ' ' ' '&!=---XXXXXXXX0 %$$$$$(()=>>EEGGNNwWW5XXXXXXXXXXXXXXXXXXs'B?BBBcJtjtSr)rr rsrtrrrget_inductor_rootrs 7??8 $ $$rceZdZUdZded<dS)OrderedSetHolderzb See FxGraphHashDetails. Holds a sorted list to support stable hashing of set kwargs. rrNNrrrrrrs* rrceZdZdZdS)rzI Exception to indicate that the FxGraphCache should be bypassed. N)r}r~rrrrrrrsrrT)frozenceZdZUdZded<dZded<dZded <d Zd ed <dZd ed<d+dZ d,dZ e d-dZ e d.dZ e d/dZe d0d!Zd+d"Zd+d#Zd+d$Zd+d%Zd+d&Z d1d2d*ZdS)3r z Centralized validation for deciding whether an FX graph can use the cache. The cache key pickler still defends itself when serializing arbitrary objects, but non-cacheable inputs should be rejected here before hashing starts. rrrSequence[InputType]example_inputsN_CompileFxKwargs | None fx_kwargsTrrequire_shape_envShapeEnv | None shape_envrr ct||||||d||j|j||jdSdS)NTinclude_constants) _check_custom_passes_check_frozen_params_check_runtime_constant_folding_check_compiler_bisector_check_shape_envvalidate_graph_check_cache_key_objectrrrs rvalidatezCacheabilityValidator.validates !!### !!### ,,... %%'''  d333 $$T%8999 > %  ( ( 8 8 8 8 8 & %rrc|jD]F}t|tjjs#|jjD]}t|jtj j rH|j s/| d|j |jdkrLtt|j|jtjjr| d|rM|jdkrB |||j}n#t&$rYwxYw||HdS)Nz!Can't cache HigherOrderOperator: rzCan't cache torchbind objectsget_attr)rmodulesrrr GraphModulernodesr_opsHigherOrderOperator cacheablerr{r7r_C ScriptObject _get_attrrr)rrrinoderMs rrz$CacheabilityValidator.validate_graphsjgoo'' 7 7Ffeh&:;;  * 7 7t{EJ,JKK K1133KKPDK)>!#~~fdkBB)!!! !00666- 7 7 7sD99 EErr"cL|jrtddSdS)Nzmkldnn tensors unpickleable)r r rrs rr z"CacheabilityValidator.check_tensors4 ; H ! ( ()F G G G G G H Hrreasonrrc t|r)r)rs rrzCacheabilityValidator.bypasss (((rrrctdt|||jft d|)NzFailed to pickle cache keyexc_info)logwarningr __traceback__r)rs rr6z-CacheabilityValidator.bypass_for_pickle_errorsE (1ggq!/2    !!=>>AErrirrc&ddlm}|||S)Nr)r)torch.fx.graph_moduler)rirrs rrzCacheabilityValidator._get_attr"s&333333y(((rc~tdkrZttjD]@}t |t r|s|dAtj ttj ttj D]@}t |t r|s|dAtj ttj ttj D]@}t |t r|s|dAtj4t tjts|dtj4t tjts|dtjD];}t%|r*t |t s|dQRR C ABBB  * 6f=?RSS D BCCC6 H HA{{ H:a#A#A H FGGG H Hrct|jr6tjds|ddSdSdS)Nz,pytorch/inductor:allow_freezing_with_cachingz$Skipping graph with frozen constants)rLrr_utils_internaljustknobs_checkrrs rrz*CacheabilityValidator._check_frozen_paramsJsf TW % % @e.C.S.S :/ /  @ KK> ? ? ? ? ? @ @ @ @rcVtjjr|ddSdS)NzORuntime constant folding can introduce constants that aren't static across runs)r) aot_inductoruse_runtime_constant_foldingrrs rrz5CacheabilityValidator._check_runtime_constant_foldingQs?   ;  KK%       rcddlm}|jr1td|ddSdS)Nr)CompilerBisectorz%don't cache graph when bisect enabledzcompiler bisector enabled)!torch._inductor.compiler_bisectorrbisection_enabledrdebugr)rrs rrz.CacheabilityValidator._check_compiler_bisectorXsUFFFFFF  - 5 II= > > > KK3 4 4 4 4 4 5 5rc|jr8|j3td|ddSdSdS)Nzfx graph cache no shape envz No shape env)rrrrrrs rrz&CacheabilityValidator._check_shape_env_sQ  ! (dn&< II3 4 4 4 KK ' ' ' ' ' ( (&<&> F  c5< ( ( 9   c " " " F UX2BP Q Q 9 KK, - - - - - T " " 9!iikk : : U,,S$777,,UD9999 : :dE;7 8 8 9 9 9,,T48888 9 9 9 9rr$)rrrr )rr"rr )rrrr)rrrr)rirrrrrr)rrrrrr )r}r~rrrrrrrrrr%r rr6rrrrrrrrrrr r s*,N,,,,)-I----"""""!%I%%%% 9 9 9 977778HHH\H)))\)FFF\F)))\) H H H HD@@@@5555((((!%9999999rr OrderedSet[str]"_warned_pre_grad_pass_missing_uuid custom_passcXt|to|duSr)rrErrs r_custom_pass_has_uuidrs) k? 3 3 V 8H8H8J8JRV8VVrcLt|ddpt|jS)Nr)rrrrs r_custom_pass_namers$ ; 5 5 Wk9J9J9WWrLiteral['early', 'late']ctj}ttj}d|D}|rdd|Dnd}|dkrf| }|rdnd}|dkrW|rU|t vrLt |td|tj d d | |dkr|rtd |d |S)aResolve the effective pre-grad pass timing from the config. "default" is resolved based on whether the custom pass provides a UUID: passes with a UUID (or no custom pass) run "late" (after cache lookup), passes without a UUID run "early" (before cache lookup). Raises RuntimeError if a custom pass without a UUID is explicitly set to run "late", since the cache key cannot account for it. c0g|]}t||Sr)rrrs rrrz0resolve_pre_grad_pass_timing..s6 $[11r, c34K|]}t|VdSr)rrs rrz/resolve_pre_grad_pass_timing..s+XX[#K00XXXXXXrzdefaultlaterzpre_grad_custom_pass %s does not implement uuid(); falling back to early timing (pre-grad pass cache will be bypassed). Implement uuid() on your CustomGraphPass to enable caching.backend_compileT)pre_grad_pass_missing_uuidpre_grad_pass_namezpre_grad_custom_pass zx must implement uuid() to run late (after cache lookup). Either implement uuid() or set pre_grad_pass_timing to 'early'.) r)pre_grad_pass_timingrKrrrrrrr%try_add_pt2_compiler)rO custom_passespasses_missing_uuidmissing_uuid_pass_names supports_lates rrrsK392MF+F,GHHM(   XXDWXXXXXX  // (5g W   &.PPP2667NOOO R,  #6%/3'> / /$; / / /   MrceZdZUded<dS)HashableOpaqueValuerordinalNr|rrrrrsLLLLLrrc"eZdZdZddgZdZejejej ej ej ej ej fZejjjejjjejjj ejjj ejjj ejjj ejjj fZed*d Zed*d Zed+d Zed,dZed-dZed.dZed/dZd0d Zd1d#Zd2d%Zd3d(Zd)S)4r=zz Object to capture all the details for a compiled FX graph relevant to computing a safe and stable cache key. graph_idcompile_region_name)rD example_valuer.rrrct|tjrdSt|ttt t frtfd|DSt|trRtfdtj | | DSdS)NTc3BK|]}|VdSr_contains_tensorrrr.s rrz6FxGraphHashDetails._contains_tensor..s1>>1s++A..>>>>>>rc3BK|]}|VdSrrrs rrz6FxGraphHashDetails._contains_tensor..sE$$Q''rF) rrr"rIrri frozensetanyrKrrr)rr.r.s` rrz#FxGraphHashDetails._contains_tensors eU\ * * 4 edE:yA B B ?>>>>>>>>> > eT " " "u||~~FF urct|tjr|jjdkSt|t t ttfrtfd|DSt|trRtfdtj | |DSdS)Ncpuc3BK|]}|VdSr_contains_cpu_tensorrs rrz:FxGraphHashDetails._contains_cpu_tensor..s1BBqs//22BBBBBBrc3BK|]}|VdSrrrs rrz:FxGraphHashDetails._contains_cpu_tensor..sE((++rF)rrr"rrrIrrirrrKrrr)rrs` rrz'FxGraphHashDetails._contains_cpu_tensors eU\ * * .<$- - edE:yA B B CBBBBEBBBBB B eT " " "u||~~FF urrzct|tjr|jSt|tr+ tj|jS#t $rYdSwxYwdSr)rrrrrrrs r _device_typezFxGraphHashDetails._device_typeso eU\ * * :  eS ! !  |E**//   tt tsA AArtargetsrrc@||vs||vrdSt|dd}||vS)NToverloadpacket)r)r.rr roverload_packets r_is_factory_targetz%FxGraphHashDetails._is_factory_targets; W  ' 1 14!&*:DAA'))rr torch.fx.NodecDtfd|jDS)Nc3DK|]}|jv j|VdSr)r)rrrs rrz6FxGraphHashDetails._tensor_metadata..s>  "#BRBRDIcNBRBRBRBR  r)rTENSOR_METADATA_KEYS)r.rs `r_tensor_metadataz#FxGraphHashDetails._tensor_metadatas>    &)&>      rcV|jdkrdS||jtt }||j|j|j}|s|sdS||j d}||St|jdd}|Ft|j |j D]+\}}|jdkr||}||cS,|rdStjtjjS)N call_functionr_schema)r7r rrrLIKE_FACTORY_TARGETSLIKE_FACTORY_PACKETSrkwargsrBrzip argumentsargsr{rrget_default_devicer)r.ris_default_device_factoryis_like_factoryrschemaargr.s r_factory_device_typez'FxGraphHashDetails._factory_device_typesC 7o % %4$'$:$: K / 1 1 8 : :% % ! 00 K  $  $   * _ 4&&t{x'@'@AA  " i66  !&"2DI>> + + U8x''"%"2"25"9"9K".****  4|E46677<|D](}t3|t4jjs#t;j|j d| |j d| D]}ddl!m"}| #|j$d}d}t3||r9|j%r+tMtd|j%D}|j'}| |}| (|j$d }|j|||f*tSd |D }|rAt4j*+r#t4j*,|_-t5j.t5j/t4j0j1j2f|_3thj5j6d kr:thj5j7s)|8||rts|_:thj;j<|_<t5j=|_>t4j?j@jAjBt4j?j@jAjCt4j?j@jAjDf|_E||jFGd }|kthjHjI}|jJ|jKkr|jJ|jJ|kr |jJ|_Ln+|jJ |jJ|ks|jK|jK|kr||_Mt|_OtQ|_RtijSd d|_TtjSd d|_Vtdkr$|XthjY|_Y|XthjZ|_Zt4j[j4j\|_]|Xthj^|_^|Xthj_|__|Xthj`|_`|athjb|_b|athjc|_c|athjd|_dttt|jXtji|_hdtj D|_j|kthjl|_mi|_ntp}|A|jnr..s(*T*Tr3r77*T*T*T*T*T*TrN)r all_kwargsrN)rcs rrz.FxGraphHashDetails.__init__..sa'"'"()%+*T*TQ\\^^=Q=Q=S=S*T*T*T$T$T'"'"'"'"'"'"rconstant_args_idxc3JK|]}t|tjVdSr)rrr"rs rrz.FxGraphHashDetails.__init__..s."W"W1:a#>#>"W"W"W"W"W"Wrrjcudagraph_annotationFTignore_private_configsreadonly_valuesrcHi|]\}}|||dd S)NFTr7)save_config_portable)rr custom_configs r z/FxGraphHashDetails.__init__.. sL/ / / & ( M66',d7)((rc4i|]\}}t||Srr0)rsymrDs rr=z/FxGraphHashDetails.__init__..s4)))CC#)))rc,t|dSNrr0rs rrz-FxGraphHashDetails.__init__..!s#aPQd))rr)qrrYrrrrCrrcconfig cache_key_tagrrrNEXCLUDED_KWARGSrrir*torch._higher_order_ops.triton_kernel_wrapr(r)r*torch._inductor.codegen.wrapperr+user_defined_triton_sourcerrrrrrrr find_nodestriton.runtime.autotunerr, get_kernelrconfigsrr"get_constant_argsr accelerator is_availablecurrent_device_indexdefault_cuda_device_index$are_deterministic_algorithms_enabled-is_deterministic_algorithms_warn_only_enabledutils deterministicfill_uninitialized_memory!deterministic_algorithms_settingsr)cppthreadsdynamic_threadsr$rVcpp_runtime_thread_counttraceprovenance_tracking_levelget_default_dtype default_dtypebackendsrmatmulfp32_precision&allow_fp16_reduced_precision_reduction&allow_bf16_reduced_precision_reductioncuda_matmul_settingsrrBr cudagraphsfwdbwdcudagraph_overrider6r torch_versionrr system_infor;inductor_configr*inductor_config_commsr_get_custom_pass_detailrr _functorchbundled_autograd_cacheprecompile_enabledrrr_get_custom_pass_detail_unsaferrrr/rmapr.rr-!_get_custom_partitioner_fn_detailcustom_partitioner_fn_custom_partitioner_fnvar_to_hint_override FxGraphCache_get_shape_env)rrrrr%processed_inputs seen_opaquesr<rPrQr(r)r*r+rirr,kernelrK kernel_source constant_argsno_tensor_inputs annotationrrs rrzFxGraphHashDetails.__init__MslCE79 ! - -C'S 22 -c77,..,?L@Q@Q,R,RLC) '' RWW(=>>>> '',,,,.$2 -/9??,,-- * *DAq,,,77sJ///)9(C(CDN1%%()DN1%                68' >**,,# # !&%(*>??%OL++*3S,L++*3Q,   DCBBBBB.99$+l:STTF"G!&)44+!>&) &'"'"-3^'"'"'"!"!"''G "(RQ"" %6$G$G $78%%M3::& w?= D #"W"W"W"W"WWWW  V 1 > > @ @ V-2->-S-S-U-UD *  6 8 8  ? A A K % ?2 . J  " "J. #//NCC #-A,B,BD ) *0)O&#466 N  & 5 N  & M N  & M% ! >%;<Z^33 8R!~002<./ n0Z^w5N5NN.:>W3L3L0:D-'[[$//11%:#($   &2%F#($& & & " ( ) )W 4 4(,(D(D+))D %*.)E)E  ,* * &#("2"9"P*.*F*F  -+ + '&*%A%A  (& & "'+&B&B  )' ' #(,'J'J  *( ( $)-(K(K  +) ) %/3.Q.Q  1/ / + "###%* ,.C.J.L.L M M& & " / / *H)M)O)O / / / +'+&L&L  (' ' # 57! //11  Y%C )) &288::@S@S!!!)))D % % % ! rr Any | Nonec|sdSt|trfd|DSt|trtfd|DSt|tr|St|tr|St |rdStdtt|)Nc:g|]}|Srrqrrrs rrrzEFxGraphHashDetails._get_custom_pass_detail_unsafe..3s'PPPqD77::PPPrc3BK|]}|VdSrrrs rrzDFxGraphHashDetails._get_custom_pass_detail_unsafe..5s1UUA< 2 2 &##%% % K  4MSk9J9J5K5KMMNNNr|CustomGraphPassCallable | list[CustomGraphPassCallable] | tuple[CustomGraphPassCallable, ...] | CustomGraphModulePass | Nonec2|sdSt|ttfrtfd|DSt|ttfs,t dt t||S)Nc3BK|]}|VdSr)rmrs rrz=FxGraphHashDetails._get_custom_pass_detail..Ns1NNQ55a88NNNNNNrzunknown custom pass type: ) rrIrrErDrrrrrs` rrmz*FxGraphHashDetails._get_custom_pass_detail@s 4 kD%= 1 1 ONNNN+NNNNN N+9N'OPP X !Vc${BSBS>T>T!V!VWW W!!!rrtrHc`|sdSt|tsJ|Sr)rrGr)rrts rrsz4FxGraphHashDetails._get_custom_partitioner_fn_detailSs;% 4/1DEEEEE$))+++rN)r.rrr)r.rrrz)rrr rrrrr)rrrr)rrrrz)rr"rrrr) rr"rrrrr%r&rr )rrrr)rrrr)rtrHrr)r}r~rrrDrr empty_like full_like ones_like rand_like randint_like randn_like zeros_likerrrr classmethodrrr%rr rr!r$rrqrmrsrrrr=r=s"#89O3         !       # ! !   [    [ \***[*   [ ===[=B[(VVVVDOOOO"""""&,,,,,,rr=rrrrrrr%r&tuple[str, list[str]]c<t||||}t|jdk}t||}||}||}d|} td|d| ||fS)z= Generate a unique hash of the FX graph for caching. r z$FX graph cache hash details for key z: ) r=rCrGrr;rRrrr) rrrr%detailsrpicklerrrR debug_strs rcompiled_fx_graph_hashr\s!^YPPG&)'*L&M&MQR&R#!"&EFFG //' " "C%%g..K +&&IIIHSHHYHHIII  rrrctjrtjsdSt |dz}t jrStjd}t d|||t ||zdz z }t d|tj t||S)z} Ephemerally increases the NCCL timeout when compiling for a distributed job Returns amount of seconds increased rgeAz>pytorch/remote_cache:ephemeral_timeout_fudge_factor_percentagezNEphemeral NCCL timeout increase fudge factor %d and original increase value %ddzIncreasing NCCL timeout by %d)seconds)r distributedrNis_initializedrr)r}rjustknobs_getval_intrinfodistdistributed_c10d"_add_ephemeral_timeout_for_all_pgsr )rincreased_timeout_sec fudge_factors r.add_ephemeral_timeout_increase_for_distributedrrs   ) ) + +53D3S3S3U3Uq  455  Q,AA L    \  !   %:\%IC%O!P!PPHH ,.CDDD<</000 ! rceZdZdZed"dZed#dZed$dZed%dZed&dZ ed'd Z d!S)( GuardedCachezJ Mixin for caches that have guards associated with their entries. r.type[GuardedCache[T]]_keyrrc td)Nz.Implement _get_tmp_dir_for_key on parent classNotImplementedError)r.rs r_get_tmp_dir_for_keyz!GuardedCache._get_tmp_dir_for_keys!"RSSSrr local_hitr local_miss remote_hit remote_missr c td)Nz(Implement _record_result on parent classrr.rrrrrs r_record_resultzGuardedCache._record_results""LMMMrlocal remote_cacheRemoteCache[JsonDataTy] | None,Generator[tuple[T, bytes, bool], None, None]c#K|r||}tj|rt tj|D]}|dr ttj||d5}| }tj ||dfVdddn #1swxYwY#t$rtddYwxYw|r ||x}lt!|t"sJ|d} t!| t$t&fsJt)j| }tj ||dfVdSdS#t$r&td|jdYdSwxYwdS) Nrr\Tz,fx graph cache unable to load compiled graphrrXFz %s unable to load compiled graph)rrr rTrlistdir startswithrrrcrloadsrrrrBrrKrrYbase64 b64decoder}) r.rrrrhr rrj cache_datarXs riterate_over_candidatesz$GuardedCache.iterate_over_candidatessc  --c22Fw~~f%% "2:f#5#566  Ds++! !"',,vt"<".sJ   Au|!dZed?dZed@dZ e dAdBd Z edCd#Z edDd$Z edEd'Z edFd*Ze dGdHd0ZedId6ZedJd7Ze dAdKd;ZedLd<ZdS)Mrwa6 Supports caching and reusing compiled Fx graphs. The overall strategy is as follows: - This cache stores entries on disk. When saving an entry, we can't serialize callables (that could be C++, Triton, etc.), so we serialize their own disk cache location. We then recreate the compiled artifact after fetching from disk. - For indexing the cache, we gather the fields relevant to identifying an FxGraph (the graph module, graph inputs, system settings etc.) into an FxGraphCacheDetails object, pickle it, and compute a hash for the key. See FxGraphCachePickler. - Among the metadata we store, we also include a guards expression that's appropriate for validating any symbols for Tensor arguments that have symbolic bounds. On cache lookup then, we evaluate those guards in the current context to validate that a cached entry can be served. - A given graph could have multiple compiled versions, corresponding to different sets of guards. Therefore, we store cache entries in the form: // - On lookup, we compute the key from the graph details, iterate over all leaf files in the corresponding subdirectory, deserialize the entry, and evaluate its guards expression. If the evaluation succeeds, we have a cache hit. If it fails, we compile the graph and store a new entry. - Finally, on a cache hit, we need to make sure any guards that would have been created during compilation are added to the current context. rrcZtjtdS)zS Get the toplevel temporary directory for storing compiled graphs. fxgraph)rr rrOrrr _get_tmp_dirzFxGraphCache._get_tmp_dirWs w||IKK333rr.type[FxGraphCache]rctjt|dd|S)zA Return the disk location for a given cache key. rjrf)rr rrwrr.rs rrz!FxGraphCache._get_tmp_dir_for_key^s0 w||L5577QqS3GGGrrrrrrr c|r3tjdtjtjd|r?tjtjdtjtjd||r3tjdtjtjd|rAtjtjdtjtjd|dSdS) zG Called by GuardedCache to record hit/miss statistics. LocalFxGraphCache!inductor_fx_local_cache_hit_count"inductor_fx_remote_cache_hit_count!inductor_fx_remote_cache_hit_keys"inductor_fx_local_cache_miss_count#inductor_fx_remote_cache_miss_count"inductor_fx_remote_cache_miss_keysN)rqrCr%try_increment_topleveladd_to_set_toplevelmissrs rrzFxGraphCache._record_resultes   O/ 0 0 0  #"53       #"54     #"63       0 1 1 1  #"54       #"55     #"64       rrrp cache_infor constantsr(tuple[CompiledFxGraph | None, CacheInfo]c jx}rtj|}|x}t||d<t jd|jt jd|jt|jdkrt jtj d |dd l m }|j |j jn#t$rd|fcYSwxYwt!j}j t%j| t*jjt2d xxjz cc<t6d t6d t;ddfdt;dfdt;dfd fdt;ddfdt;ddfdt=r.j r't=!dj |fS)ah Cache specific post compile steps that need to run if we find a graph in the cache This includes putting bundled triton artifacts in the right place, reloading the PyCodeCache artifact, etc. These don't always happen (i.e. on a cache miss, so they are in a separate function from CompiledFxGraph.post_compile) Nrinductor_compile)cached_kernel_nameszAOTAutogradCache.inductor_loadrnum_triton_bundlesrjrr\rzOutput code: %szOutput code written to: %sartifactc dddS)Nfx_graph_runnablestringr{rtrrrrrz5FxGraphCache.cache_hit_post_compile..s+$!!rcjSr)runnable_graph_strrsrrz5FxGraphCache.cache_hit_post_compile..s u7r) metadata_fn payload_fninductor_post_grad_graphcjSr)inductor_post_grad_graph_strrsrrz5FxGraphCache.cache_hit_post_compile..s uArr inductor_output_codecHtjdS)N)filename file_path)rr abspath) artifact_pathsrrz5FxGraphCache.cache_hit_post_compile..s#)W__];;rcSrrrsrrz5FxGraphCache.cache_hit_post_compile..strc dddS)N*inductor_provenance_tracking_node_mappingsrrrrrrrz5FxGraphCache.cache_hit_post_compile..sD"!!rcjSr)inductor_provenance_mapping_strrsrrz5FxGraphCache.cache_hit_post_compile..s uDrc dddS)N0inductor_provenance_tracking_kernel_stack_tracesrrrrrrrz5FxGraphCache.cache_hit_post_compile..sJ"!!rcjSr)$inductor_provenance_stack_traces_strrsrrz5FxGraphCache.cache_hit_post_compile..s uIrinductor_provenance)"_triton_bundleru read_and_emitrr%rrrCrrafter_deserializationrrsave_output_code source_coderrsinductor_meta_from_configrt begin_compile'_set_compile_context_for_autotune_cacher,CachedMetricsHelper apply_deltasmetrics_deltasr&counter_deltasoutput_code_logrrZr( in_progressr add_to_set) rrrbundlerrr inductor_metarr\s ` @@rcache_hit_post_compilez#FxGraphCache.cache_hit_post_compiles?) )6 "/"=f"E"E ++847II 01"6&D.s!<<.s4rrrTz*fx graph cache key %s post-load guards: %s)rwrxrr)&unsafe_skip_cache_dynamic_shape_guardsevaluate_guards_expressionrrextern_libs_keyrrS_tritontriton_backend_extern_libs_keyrrarrrecord_if_presentrrrrguardsr/)rrrrrrrsymintsrrrr guard_infobackendcurrentchecks r _lookup_graphzFxGraphCache._lookup_graphs&!//11 $$$55nEE<>+->>wGG    %///6B 23Z''388::C@@RR      ):GDDEEED==== IIsz22377w~~f%% / K . . . . w||FK$8$899T7d333333rctjsdStj|}|jx}rtj|dSdSr)ru is_enabledrrrr)rjrr-s rrz FxGraphCache._emit_triton_bundleJsX'))  F W%%) )6 0  ' / / / / / 0 0rcompiled_graphrcddlm}t||sJdt|dt}|Jt|}||}||||_ tj j } tj j | |_n#t $rYnwxYwt#|} |  t'j| } nH#t $r;t*dd t.d d xxdz cc<YdSwxYw t1t2|| |r/t|| t9jd |r[t=| jpd dz} tAj!| "d| d} ||| dSdS#t $r;t*dd t.d dxxdz cc<YdSwxYw)z= Store a serialized CompiledFxGraph on disk. rjrozserialization for z NYIN) placeholdersr=z1fx graph cache unable to serialize compiled graphTrrfxgraph_cache_pickle_errorrrg.Ar|)rX time_taken_msz!fx graph unable to write to cachefxgraph_cache_write_error)# compile_fxrprrrwrxrget_pruned_guardsproduce_guards_expressionrrrSr9r:r;r8rrprepare_for_serializationrr"rrr&rarrecordrrqputr_time_taken_nsr b64encoderB)rrGrrrrprr>r=r@disk_compiled_graphrjrKrs r _save_graphzFxGraphCache._save_graphSs 0/////./::  ;n!5!5 ; ; ;  :!//11 $$$55nEE,,W55%.%H%H &I& & " k)88::G-2[-@-Q-Q..N * *    D ">2255777 l#677GG    KKCd     Z != > > >! C > > > FF   C !"7"<"<">"> D D K KG T T T 5223@@@ 3444 2 #%8%G%L1QT$T U U ",W55<> K.; ?+"'&96!T)FmTTU&@RJ;< HH5s ; ; ; Z !5 6 6 6! ; 6 6 6(.J} %z))rc tjtdS#t$rYdSwxYw)z. Clear out the on-disk cache. N)rrmtreerwrFileNotFoundErrorrrrclearzFxGraphCache.clear sK   M,3355 6 6 6 6 6     DD s +/ ==r)r.rrrrr)r.rrrrrrrrrrrrr )rrprrrrrrr)rrrrrrrrrrrr0rr)rrrjrYrr )rjrYrr ) rrrGrrrrrrrrr )rrrr )rNT) rrrrrrrrrr ) rrrrrrr%r&r[rrr\)rr)rrrRrrrrrrrrfrrrrr0rrr$)r}r~rrr%rrrrr/rCrrrVrXrZr`rerlrprrrrwrw9s:444\4 HHH[H ***[*Xd!d!d!\d!LBQBQBQBQ\BQH 4 4 4\ 4000\0>C>C>C\>C@   \ /1-1"& \&"&"&"&\"&H    \  /*/*/*/*\/*b\rrwr cdd} ||rtj|S|drtj|S|dfS)NrrctrdSdS)Nz.pyd.sorrrrget_module_ext_typez;split_aot_inductor_output_path..get_module_ext_type s  65rz.pt2rr)endswithrr split)r rus rsplit_aot_inductor_output_pathrx s O }}((**++w}}T""" v  w}}T"""RxrceZdZUiZded<eejZe dddZ eddZ eddZ dS)CudaKernelParamCachedict[str, dict[str, Any]]rNrrparamsdict[str, str | None]rbin_typeasmrzasm_typerr cPd}tjjr5tjjs Jd|ds Jd|d}t |||t tjjd|\}} | } t| \}}tjj rHddtdd d i} || vs Jd tj | \} }| | |z} d } tjj stjjrntjj|r[|rY|s Jd |s Jd|dvr|nd}t |||t tjjd|\}} | |t#<| |d<| |d<||j|<dS)Nz:package_cpp_only requires triton kernel names to be unique mangled_namezMissing kernel namer)rkrcrrz.fatbinz.spvrz.hsacoz8multi_arch_kernel_binary only supported in CUDA/XPU/ROCmrzMissing kernel assembly codezMissing kernel assembly type>rmrnror\runtime_bin_pathr)r)rpackage_cpp_onlyrunique_kernel_namesrrx output_pathr?emit_multi_arch_kernelrWrr splitextrrrrr)r.rr|rr~rrrar?bin_pathrbin_type_to_ext base_pathasm_path hash_kinds rrzCudaKernelParamCache.set3 s   / .=4  L  4.) @ @+@ @ @)n-H  8#/   8$$ DF'    - 0*"    #rr*c8|j|dSrrrs rrBzCpuTritonKernelCache.get rrrc4|jSrrrs rrzCpuTritonKernelCache.get_keys rrN) rrrrYrrYrrrrrr rr) r}r~rrrrr%rprrrrBrrrrrr sJJ')E)))),u{++K   [ ,((([(   [   rrc*eZdZdZeddZdS)AotCodeCompilerz. Compile AOT Inductor generated code. rr wrapper_coder kernel_codeserialized_extern_kernel_nodesrzradditional_filesrrlist[str | Weights] | strc >1nopqrstuvwxyz{|}tt}tddt|j} t | } tjo dkojzttj j \} } tj j rd fdtd| | tj j \} {d {dztd | | tj j \}udsdttj jst#t$j t$jt$jt*d d dd5}tj j}d|}|ssdd|dd|dd|dstsd| |\}tdddn #1swxYwYt1d5} |ddf|t6j|jddddn #1swxYwYtj jra| {tj j s| utj js| ttB"d{tB"dutGd {fd!fd"#tGd ufd$fd%#tj js3tB"d&ttGd tfd'sfd(#tI{}tIu}|j%| z yy&sy'd)*tQtIyd+z }dyzfd1 }d2d3l)m*}tW}|t$j || d4ztX5}|5|rtQ|-d6}t#|d75}||dddn #1swxYwYtj jr| |tj j.}|d8<|/tatQ|1|j2d9}tj j.3D]9\}}ti|tPrti|tPs Jd::t#|d75}|tkj6tj j.dddn #1swxYwYtQ|1|j2d9}toj8||tj jr;| |tj j s| || rtj j n!tQ|-d;} tsfd<j:D}!d2dl;pdd@}"tj j<stj j=dAkret}dBd)C5fdDj:Door#d2dEl?m@}#gwgxd2}$oD]p}%A|%}|"|}&w |$x |&|$|!r|&n|&tzdFz tztzz }$qt|$}'pjD|$zE|'}(pF|(ndnopwxfdH })|#5}*|*G|)ttoD]} dddn #1swxYwY~(ndI}'dddn #1swxYwYndI}'tj j=dJkr5tfdKj:D}+| |+t|'},t|,\}-}.|-r|.rtdLd}/|-r,|j2dM}0tQ|1|0}/j|.|-z|dN}1tdd)tj j dO|1}2tddPd)i|1}3tj jMrtst}dQd)C5tjR}4t|4| fdStj j i|1|2_Qtx}5rt|5| fi|1|3_Qdddn #1swxYwYttQ|j2{tQ|j%|2T}6|6}7|6S}8ttQ|j2utQ|j%|3T}9|9}:|9S};tdU|7tdV|:tj j rtQ|1|j2dW}<|2U|<| |<|6V||6W|{| |nk |6XnA#tjZtf$r(}=dXtQ|=vrtdY|=|=d}=~=wwxYw|9X|.sr|'}>d2}?|-rktj]dZ|,}>|/Jt#|/d[5}@|@|'dddn #1swxYwY| |/nutttjad2tjbtjcjdd\e}?tj]d]|,d^z|?}>||>tjg}Ad2}Bi}Ctji3D]\}B\}%}Dti|Dtjjjkjlr|Djm}Dti|DtjnjosJt|B}Etd_|%|E|E|C|%<tjnq|D}Ft$j |j%|E}Gt|G|Fd)| |G|Crt$j |j%d`}Ht#|Hd75}|tkj6|Cdddn #1swxYwY| |Htttdavtjwjxrdbnt}I|I}J|Jjy8}K|Jjyz|Krtj j{r Jdcg}Lg}Mg}NtsZg}Otz\}P}Qtt6j~jddi}Rtj3D]1\}S}T|S|Rvr |Tdex}Ur|M |U|Tt}Vtj j{rdfkrtjwjx,|N |U|V|Tvdgfnvd2dhlm}Wt$j&|Ustdi|Udj|W|U|Vdk}X|Xstdl|Sdmt"dn|Vtj jr|O |V|Sf3|Nrd2dEl?m@}#t jqt jvtojdorvbt$jvx}YrAt$j |Ydo}Zt$j&|Zr|Zrdqrvfdr }[t}dsd)C5|#5}*t|*G|[|Ndddn #1swxYwYdddn #1swxYwY|Or t|Ot$j| tt}\|L |\n^#tj$rKtdu|OD]+\}V}S|L t!|V|S|P|Q,YnwxYwt#| \}]}^t|jzv}_|Kr"dwkrtdx|jzgdyz}_|8|;|Ag|K|L}`t|]|`|^|_T}a|a}b|aS} td{|bt#{d|5}|d|d}|7d|d~|bddddn #1swxYwYt#ud|5}|d|d}|:d|d~|bddddn #1swxYwYtj j rtQ|1|j2d}c|_U|c| |c| t$|.rtQ|1|j2d}dt#|dd[5}@|@|'|@tj]dZ|?dddn #1swxYwY| |dn+| |A|aW||Atj j{r=tjwjx,|a||M||Mn6g|K|L}`||`|`D]}e|aW||e|a|n3|aX|`D]}f|f|Kvrt%j|f|.rtj jdkrtddd}g|g}htd|h}it#| d5}j|j}k|jd|i|k|izz z|j|'|jtj]dZ|?dddn #1swxYwYtj jr| | dddn #1swxYwYtjjd2krtjj}lt$j |j%d}mt#|md75}|tkj6|lddddn #1swxYwY| |mtj jr|S| S)z Returns the .so path, or returns a list of files that were generated if config.aot_inductor.package=True. ri)vec_isaraot_moder{sources BuildOptionrrrz wrapper.cpp)r^rcrz.// Triton kernels are embedded as comments in z kernel.cppcsrcr aoti_runtimezmodel.hAOTInductorModelz<>zAOTInductorModel((zAOTInductorModel :z :rOrcrNzw+rW)rbzWrapper code written to: %szKernel code written to: %s graph_dumpcdddS)Ninductor_aot_wrapper_coderWr{rrr) wrapper_pathsrrz)AotCodeCompiler.compile..: s3(rcSrr)rsrrz)AotCodeCompiler.compile..? s|rr cdddS)Ninductor_aot_kernel_coderWrr) kernel_pathsrrz)AotCodeCompiler.compile..C s2'rcSrr)rsrrz)AotCodeCompiler.compile..H s{rzHeader code written to: %scdddS)Ninductor_aot_header_coderWrr) header_pathsrrz)AotCodeCompiler.compile..N s6! +rcSrr) header_codesrrz)AotCodeCompiler.compile..S s;rTrRzCMakeLists.txtconstsbytes | bytearrayrrrcLtjj}|dkrXjt jzr%t|dkrtddndd}n(|dkrdd }n|d krd}d }ntd |d krd }t|dk}t|dk}d&fd }d'd}d(fd }|r||t||\} } n,|r|t|\} } n||t|\} } t| | ttjj \} } t| } tjd} t#t| jt| t| j| }|}|d ur3|r1t+t| |t| jn||r|rt/|d!5}|d|d}t4jd"kr|d#n|d$}|d%ksJ||d}|t|kr5| ||d}||z }|t|k5dddn #1swxYwYt;j| |S))Nlinuxi5wzPModels with buffer mutation included doesn't support constants greater than 2GB!z .ldata, "aw"z .lrodata, "a"rdarwinz __DATA,__datar?rFzUnsupported platform: rrArrr align_bytesr symbol_prefixris_large_constsrrrcdd}|d|dz }|d|dz }||dz }|s|D] }|d|dz } |s|dz }n|d z }|d t|d z dz }|d |d z }||dz }|dfS)N .section r .balign  .globl _binary_constants_bin_start _binary_constants_bin_start: z .byte z .space 1 z .quad 0x1234567899abcdef z .space .globl _binary_constants_bin_end _binary_constants_bin_end: z weights.SrC)rrrr consts_asmr3 section_attrs rformat_consts_to_gnu_asmzRAotCodeCompiler.compile.._compile_consts..format_consts_to_gnu_asm s =L<<< :;:::: W=WWWW NNNN & B#77"&6&6&6&66 "5"n4 "@@J"Ac&kkAo"A"A"AAJSSSSS LLLL !;..rct|}d}|}|d|dz }|d|d|dz }d}|D]}||dz }|d z}|d zdkr|d z }|d z }|d|d |dz }|dfS)Nz#if defined(__clang__) || defined (__GNUC__) #define ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize("address"))) #else #define ATTRIBUTE_NO_SANITIZE_ADDRESS #endif ATTRIBUTE_NO_SANITIZE_ADDRESS zalignas(z ) extern zunsigned char z_binary_constants_bin_start[z] = { rrrjz z}; z) extern unsigned char * z_binary_constants_bin_end; z weights.cppr)rrr consts_size asan_attr const_cpp count_bytesr3s rformat_consts_to_cppzNAotCodeCompiler.compile.._compile_consts..format_consts_to_cpp s"&kk % & > >>>> pmppYdpppp  ,,AA)I"-/K"R'1,,!V+ X% { {{m{{{{  -//rctrd}d}n3dd}|d|dz }|d|dz }||dz }|d |d z }||d z }d }||fS) a This function handles zero-sized constants because the C++ standard prohibits zero-length arrays: https://stackoverflow.com/questions/9722632/what-happens-if-i-define-a-0-size-array-in-c-c On Windows (MSVC): The compiler reports error C2466 for zero-sized arrays: https://learn.microsoft.com/en-us/cpp/error-messages/compiler-errors-1/compiler-error-c2466 Solution: Use assembly compilation to handle this case. Why not use Win32 assembly for all paths? ml64 only supports alignment up to 16 bytes, which isn't optimal for performance. Cross-platform implementation: Linux: Added '-pedantic' to disable zero-sized arrays in C++ compiler Windows: MSVC naturally rejects zero-sized arrays by default z option casemap:none .data ?_binary_constants_bin_start@@3PAEA: align 16 ?_binary_constants_bin_end@@3PAEA: align 16 public ?_binary_constants_bin_start@@3PAEA public ?_binary_constants_bin_end@@3PAEA end rrrrrrrrrrSrt)rrasm_codeasm_extrs rget_zero_consts_asm_codezRAotCodeCompiler.compile.._compile_consts..get_zero_consts_asm_code s(" H$GG>l>>>H <[ < < <._compile_consts` s"("5"JM7""(:eo6J6J6L6L+M+MM 36{{]22(n$2LL#2L " X%%. # W$$ " % "#FH#F#FGGGe## % !&kkD0O"%f++"2  / / / / / /2 0 0 0 00, ), ), ), ), ), )\ (@(@K))% XX',D,D#]--)K-A,@ ]--)K !"344'F KAx H~~H#8'!"3 $$$ (''H x//0 N &::<R>RSSSS$$&&& "= "(E**"aFF1III&&,,C=H44!DEEE XX&IJJ %????FF9%%%CF ++WWVCDD\22r F ++"""""""""""""""" Ih   Os-C LL  L rFileLock.locktimeoutz.jsonrsAOTI_DEVICE_KEYz_metadata.jsonz"Metadata must only contain stringsrsc3^K|]'}|jv |jV(dSr)folded_constantsget_original_value_of_constantis_cudarr{rs rrz*AotCodeCompiler.compile..U sNu55544T::B5555rr torch.Tensorrc|dkrdS|jr$tjj|S|SrA)numelr rrmkldnn_nbytesuntyped_storagenbytesrs r_constant_nbytesz1AotCodeCompiler.compile.._constant_nbytes] sY7799>>1;7 9+33A666((**11333r binary_blobaoti_serialize_constantsrlc&g|] }|jv |Sr)rrs rrrz+AotCodeCompiler.compile..k s1&&& u'======r)ThreadPoolExecutorrjr c |}|dkrdS|}|jrFtjj|} |z||dS|} |z||dSrA) rr rrrdata_ptrmemmoverr) rnrr!t_cpu base_addrconstant_namesctypesroffsetssizess r_workerz(AotCodeCompiler.compile.._worker s %aA Avv & % D D^TUEV W WA { "+09+;+D+DQ+G+G &y71:/ExQR S S S S S)*(9(9(;(;(?(?(A(A &$- $:ENN. se !u'=== 5d;!@@FF,U_T-BCC> >==rz?use_external_weights and use_mmap_weights cannot both be True.z _weights.blob)rruse_mmap_weightsuse_mmap_weights_externalrrr min_optimizeraoti_precompile_header)rr1rz#aot wrapper compilation command: %sz"aot kernel compilation command: %sz_compile_flags.jsonz is too big to optimizezUPlease use torch._inductor.config.aot_inductor.compile_wrapper_opt_level = 'O0' flag.qr)rjqqrzsaving script object %s as %szcustom_objs_config.json)rocmrrr5z. Check that ROCm toolchain is available and LLVM IR is valid.zCreated multi-arch bundle: %s fatbinaryasm_cubin_and_rawtuple[str, str, str | None]c|\}}}8|6tj|rd|ddd|dd|g}nCtdgt jd|d|d d d d d d } t j|d d d dS#t j$rE}tt j |d|j d|j tj d}~wwxYw)Nz --create=z-64z--image3=kind=elf,sm=z,file=z--image3=kind=ptx,sm=z nvcc is required to build fatbinz-fatbin-oz-gencodez arch=compute_z,code=compute_z ,code=sm_T)capture_outputrrBz failed with: stdout: z stderr: )file)rr rTrshlexrw subprocessrunCalledProcessErrorprintrstdoutstderrr) r<asm_fcubin_f raw_cubin_fcmdr current_archr;nvccs r_compile_fatbinz0AotCodeCompiler.compile.._compile_fatbin s7H3w %1 + 7 "{ ; ;!8 !* 5G 5 5 % Y Y YK Y Y S S SE S S #CC $|&23U&V&V V #!&T!2!2 # ) #!& #!% # !( # !+ #![ Z ZL Z Z #!+ #!V U U| U U #C "&N #Dt4 *<"""!#(:c?? k kAH k kabai k k%(Z" "sB C4/AC//C4aoti_compile_fatbin) cpp_compilerzABatched cubin embedding failed, falling back to per-cubin objcopy)rrrrricpx)z-fsyclz-fsycl-targets=intel_gpu_pvcz-Xspirv-translatorzi-spirv-ext=+SPV_INTEL_split_barrier,+SPV_INTEL_2d_block_io,+SPV_INTEL_subgroup_matrix_multiply_accumulate)compilerrrrr extra_flagszaot linkage command: %srz// Compile cmd // z// Link cmd // z_linker_flags.jsonz_serialized_weights.binwindowszKwhen cross_target_platform is windows, use_mmap_weights should not be true.c tr[ddlm}m}m}ddlmmm G fdd|}|}|j |||j }nddl }| }|S)Nr)byref StructurewindllDWORDLPVOIDWORDc VeZdZdfdfdfdfdfdfdfdfd fd fd fg Zd S) CAotCodeCompiler.compile..get_page_size..SYSTEM_INFOwProcessorArchitecture wReserved dwPageSizelpMinimumApplicationAddresslpMaximumApplicationAddressdwActiveProcessorMaskdwNumberOfProcessorsdwProcessorTypedwAllocationGranularitywProcessorLevelwProcessorRevisionN)r}r~r_fields_rZsr SYSTEM_INFOr_g sn%=t$D%0$$7%15$9%BF$K%BF$K%%>$F%6$=%94$@ ,"rrl)rr'rWrXrYctypes.wintypesr[r\r]kernel32 GetSystemInforbresource getpagesize) rWrXrYrlsi sys_page_sizerpr[r\r]s @@@r get_page_sizez.AotCodeCompiler.compile..get_page_size\ s'C LKKKKKKKKK " " " " " " " " "i " " ""-B"O99%%))DDD,.MMM+OOO,4,@,@,B,BM,,ri@za+b zkernel_information.jsonrr)rrrrrr)rrrr)rrrr r)r<r=rr )rr)r5rCr9r;rrget_command_liner)r}rxrrrrrrdynamic_linkagerrr rsrtrcrrr writelinesflushrvrrr{packagerr*rrZr rrTrrtorch.utils._filelockr rW LOCK_TIMEOUT with_suffixrrr with_namerrNrrr"rrrMrr'package_constants_in_so package_constants_on_disk_formatr'concurrent.futuresrrrQ bytearrayc_ubyte from_buffer addressofrrrJrCrcrSrprecompile_headersr_get_cpp_wrapper_header_precompile_headerprecompiled_header_get_cpp_prefix_headerrrsave_flags_to_jsonsave_compile_cmd_to_cmakesave_src_to_cmakerr+CppCompileErrorr$structpackrrrrandintiinfoint64maxrrr enumeratetorchbind_constants_libraryfake_class_registryrXr'rrrd _pickle_saver! ROCmCodeCache CUDACodeCache XPUCodeCacherBrr aot_kernels_orprr>rrrrzrr$torch._inductor.rocm_multiarch_utilsr7embed_kernel_binarycuda_compile_utils_nvcc_arch_as_compile_option_cuda_compilerwhichrIr7r=rCrErr8r?r4save_kernel_asm_to_cmakeextendsave_link_cmd_to_cmakercross_target_platformtellr[r\rcreate_kernel_information_json)|r.rrrrrrgenerated_filespicked_vec_isavec_isa_cmd_gen cpp_commandspecified_output_pathspecified_artifact_name wrapper_keyr?rmodel_class_name class_namerwrapper_path_operatorkernel_path_operator cmake_pathrr rVlockextern_kernel_nodes_jsonr meta_jsonrPrQkernel_meta_json output_soall_cudarr total_sizer{r#serialized_weightsbuf_viewr*pool weights_dictruse_external_weightsr.external_weights_pathexternal_weights_filenamecompile_commandwrapper_build_optionskernel_build_options header_file cpp_prefixwrapper_builderwrapper_compile_cmd wrapper_okernel_builderkernel_compile_cmdkernel_o compile_flagsr aot_constants magic_number f_weightsrcustom_obj_idxqual_name_to_idconstantcustom_obj_namecustom_obj_bytescustom_obj_pathconstants_config_json cache_cls gpu_codecache gpu_kernels_ocubins_o asm_files fatbin_cmdscubins_to_embedldobjcopykernels kernel_namer.asm_file cubin_filer7successnvcc_dirrrO combined_obj output_namerso_build_optionsobj_srcs so_builderlink_cmd linker_flags weight_filero_filert page_size_ page_sizef_soso_size kernel_infokernel_info_jsonr%r&r'rMr;rrrrNr(r)rrrs| ``` ` @@@@@@@@@@@@@@rcompilezAotCodeCompiler.compile sn 0@%$-&'   ?;;==>>      J;%#7 JEN  +6+>+J K K ! #   / 99lK%@AALK$)  /#B % % % ! \ N\ M M M    /#B    ;  "2  GOOBGOOH$=$=>>"  #)#6#U B0@BB ffhh  ''(<>O*>O>O>OPPW0Z2B2B2BCCW1j3D3D3DEE "'"7( """;+               :d # # 9q  LL,k4@ A A A GGIII G  %  8 8 8 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9   & 4  " "< 0 0 0&7 4&&{333&6 4&&{333:LIII9;GGG      ,+++           +***    "2   !={ K K K  /...    !%\ 2 2#K0018;F '')) 3  # #T # 2 2 2/003CCDD } } } } } } } } } ~ 322222>>x GLL;#8 9 9<   f 6f 6- E+.)55g>>,,(2C88$> """"""""""" 0/11%T%)XXguS=P=P7Q7Q%R%R%% $%%%%%%%%%%%%%%%% %H-0*u:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1x&)""CGWWW& %*O     &&|444011K5N66 2 "2$ (8 "U%) !# /D/I,X,X,X)(+)334MNN))% "N*$4-A%6) //O%:%!!'!4!EE%%"%% ! $9$$!$!$$ "5 k !":RVWWW"9#en###K@R##@@*0)<)M%M@* @@)< &>I'-233#4;<<0 N "0!@!@!B!B %::<EEE&&7777 >(* )02K))%/559GGDJ77888999999999999999&&'<===&%#cEM->&&; NN  &IKKM)7<<>>M  ' - - / / / !.EREHI=?KK 9;01BCC G!!'"68NPRSS*>*D*J*J*L*L3J3J&K"'11!#(</x3!((222!&'F'H'H!IJ+B&R'611 =,4'..!):uyyAS7T7T U  $&7>>(#;#;"&2%`+3%`%`%`'"'"!"'L&K-53=-1 '''G$+"&2%dVa%d%d%d'"'"!"  HH%DjQQQ*>J'.. K/HIII:EEEEEEE#5#R#T#TL-<>>D & [ 9 9I'9N9N-NX'$&GLL;$G$G 7>>)442(1I)")")")")")")")"X%%:RVWWWEE**,,E04TXXo{CCDDD EEEEEEEEEEEEEEEEEEEEEEEEEEEEEE #'B+GOOI66)9););((( ! 5555%8    @8G3J $OO 4$. R!"!" 'MY&W&W #K4&'"3      !5!5#8#* +"^&7 ! ! ! $$$ &"8XQ QQH#  %, J "2244H"7799I II/ : : :lC(( 9A E.AEEEFFF78777888 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 k3'' 91 D.@DDDEEE78777888 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 "3k 6")3305III   !33LAAA&&|444&&~666$G"%-7749RRR##K k400HI!(:;;;! C(F(FGGGHHHHHHHHHHHHHHH$**;7777$**844400XFFF '> F )177 INNN#**95555 ;::H#**8444'FF"44ZEEEE11*====  """&&&F.. If%%%%#2C*@IMM*i"-"-"-"-H"/J #E: 6 6Ii//C4"&))++ 49w7J+J#KLLL #5666 6;sL#A#ABBB CCCCCCCCCCCCCCC&.6#**9555Mf 6f 6f 6f 6f 6f 6f 6f 6f 6f 6f 6f 6f 6f 6f 6P < 1Q 6 6//NNPPK!w||%,.G   &,, ; ;q999::: ; ; ; ; ; ; ; ; ; ; ; ; ; ; ;  " "#3 4 4 4   & ## "s>B III-AKKK85A^<-T A^<T A^<T C8A^<7Y A^<Y A^<Y D#A^<=C,c)4b* c*b. .c1b. 2c9 A^<c A^< c DA^<%A i A^<i A^<i FA^<8p  A^< q ##qq  AA^<r5) A^<5r9 9A^<<r9 =G0A^<-({! A^<!{% %A^<({% )LA^<G1 AIG<%AH-H! AIH-AH1 H1AIH4AH1 H5AIH8 A^<IAI IA^<I AI I A^<IAAJ%J$A^<J%AALK=A^<K?ALLCA^<OAAPP A^<PAP PA^<PAP PA^<P0AARQ8 A^<RAR RA^<R AR R CA^<U >AVV A^<VAV VA^<VAV VE%A^<\A0A]?]3 A^<]?A^ ^A^<^A^ ^)A^<^<A__A_`4*Aa*a*Aa.a1Aa.N)rrrrrrrrzrrrrrr)r}r~rrrrrrrrr sDddd[dddrrz CDLL | None_libgompr7r list[c_void_p] | c_void_p | Nonec d fd fd|D}|ds J|dzd}t|dD]/\}}|d krtj|}t ||}0t |s J|d zt}t|j j |D]\}}|j r |||j <|r|t| d=||i|} | dSt| tt fr\d | D} | D])} t| t"js J|d z*t"jj| St| t"js J|d zt"jj| S)Nr rrctt|dkr$tjj|St |ttfr$t|fd|DS|S)Nzc3.K|]}|VdSrr)rr convert_args rrz9custom_op_wrapper..convert_arg.. s+99[[^^999999r) rrrr_aoti&alloc_tensor_by_stealing_from_void_ptrrrIr)r rs rrz&custom_op_wrapper..convert_arg sy tCyy>>2 2 28>HHMM M dE] + + 4999999S99999 9Jrc&g|] }|Srr)rr rs rrrz%custom_op_wrapper.. s#7773kk#&&777rz torch.ops.z, can not be called through custom_op_wrapperrrz, can not be loaded through custom_op_wrapperc>g|]}|tjgn|Sr)rtensor)rrs rrrz%custom_op_wrapper.. s+GGG1ai%,r"""QGGGrz returns a list of non-tensorsz returns a non-tensor)r rrr)rrrw importlib import_modulerrrKrrr kwarg_onlyr{rCrrIrrr"rr#unsafe_alloc_void_ptrs_from_tensors!unsafe_alloc_void_ptr_from_tensor) r7rconverted_argsfuncrrrfunc_argconv_argrrrs @rcustom_op_wrapperr s8777$777N == & & ;; & D"((3--((  1 66*1--DtQ D>>NN2 NNNN>VVF!$,"8.II--(   -$,F8= ! + CKK<>> * T> ,V , ,F ~t&4-((JGGGGG V VAa.. U U5U0U U U. Ux~AA&III fel + +IIR2I-III + 8> ; ;F C CCrprecompiled_headersrQ)z at::vec::zprod_masked_reduce(headerhashable_cmd_linerc ^tr Jd|ddtj5}t |dz }|d|dt t|ddt|tdi|ddi }| dd }|| }dddn #1swxYwYtdi|ddi}td|dd||zt| zt\} } t | | | } tjt"dt%tjt"| d| f| S)Nz>CppBuilder does not currently support precompiling on Windows!rTz header.hppz #include preprocessingTrrrrcttjdd|fdd}|jdS)zReading the whole preprocessed header in for hashing is very expensive, but calling a fast hashing utility in a subprocess is cheap.opensslsha512T)r@rr0)rCrDrGrw)r cmd_outputs r_get_file_checksumz._precompile_header.._get_file_checksum sD$Hh/4J$**,,R0 0r precompilingrO)rjrbr^rcrRr r)rrrr)rpopr{TemporaryDirectoryr rr9rr;rrrr< get_compiler _HEADER_DIRrrU_HEADER_LOCK_DIR_worker_compile_cppr r) rrrpreprocessing_dirpreprocessing_header preprocessorrpreprocessor_hashheader_build_option header_hashheader_full_path cpp_builders rrr sf H? t,,,  $ & &T*;#$566E''(@V(@(@(@AAA!)**3B3/,---TTTTtTTT    1 1 1 1/.|/P/P/R/RSS'TTTTTTTTTTTTTTT*0UU/UUPTUUU$)(V(((  '(;(H(H(J(JKK L" % % %!K!  'KK 40000  %+'<'<'<==  sB#C**C.1C.c4|drdSdS)Nrz torch/csrc/inductor/cpp_prefix.h)rrs rrr!s# 211 4rrc|ddd}tjjo|dk}d|rdndd |rd n|d S) zGiven a device type (and optionally whether we're in AOT Inductor mode), returns the path to the cpp_wrapper header file to be precompiled.rrjmaxsplitrrztorch/csrc/inductor/ aoti_include cpp_wrapper/ array_refz.h)rwr)rallow_stack_allocation)rr base_device is_array_refs rrr'so,,sQ,//2K&=V+QVBVL <% 8>>= < <& 7;;K < < <rbase_device_typesourceexplicit bool | Nonecd||SdS|dkrdStfdtDS)NFrTc3 K|]}|vV dSrr)rmarkerr5s rrz)_resolve_needs_vec_isa..<s(JJFvJJJJJJr)r_VEC_ISA_CPP_SOURCE_MARKERS)r4r5r6s ` r_resolve_needs_vec_isar<3sN ~u5  t JJJJ.IJJJ J JJrceZdZUdZiZded<eejZiZ ded<ed!d Z e d!d Z e d"dZ e d#d$dZe d%d ZdS)& CppCodeCachezCompiles and caches C++ libraries. Users of this class supply the source code to be compiled, while compilation flags are set by CppBuilder.*dict[str, Callable[[], CDLL | ModuleType]]rrcpp_compile_command_flagsr rrrCDLL | ModuleTypec*tj|Sr)r LoadLibrary)r rs r_load_library_innerz CppCodeCache._load_library_innerHs%%%rc  |||}||_|S#ttf$r}dt |vrWt jdr8tj da |||}||_|cYd}~Sdt |vr;t|dtj dtj d|d}~wwxYw)Ngompz/usr/lib64/libgomp.so.1z(failed to map segment from shared objectz3. The most common reason this may occur is if the zl folder is mounted with noexec (e.g., by default Docker mounts tmp file systems as noexec). Please remount zi with exec enabled, or set another temporary directory with TORCHINDUCTOR_CACHE_DIR environment variable.) rDr ImportErrorrrrr rTr rCrr{ gettempdir)r.r rrrs r _load_libraryzCppCodeCache._load_libraryLs  ,,T377FFJMW%   QBGNN3L$M$M +,EFF00s;;   9SVVCC]]XM`MbMb]]3;3F3H3H]]]    s"!C,A"C'C,A C''C,rrzcdS)z_ Given a device type, returns the path to a CPP header file to be precompiled. Nrr.rs r_get_uncompiled_headerz#CppCodeCache._get_uncompiled_headercs trrNr main_coder submit_fnrrT Sequence[str]optimized_code needs_vec_isar7kernel_needs_vec_isac !"#$%|ddd}t|||} t|||}| s|rtnt} ij||t jd} i| d| r| nti} i| d|r| nti} t|dup|dk}tdt||d | }tdd d i| }dd}||}||}t|d|d|\#}|rt|d|\}}n tj }#j vrddlm}tjt%#dz}d"d$t jrYt(sR|x}rt-||fd|i| |_|r$t1|x}rt-||fi| |_t3|\}}t5||||}|rt3|\}}t5||||}t5|||gtdi| |}t9jt<||||f%t?|!n=t9jt<||f%t?|!d!"#$%fd } |T||t@5tj!!s |%"dddn #1swxYwY| j #<j #S)z\Compile and load a C++ library. Returns a callable that returns the loaded library.rrjr+r)rrTrrNrr0rT build_optionrrrcJtdd|S)zWriting the code to file will calculate a hash, which we need to vary if the command line flags change. This implements a mostly-generic way of validating that.rrr)r9rv)rTs rget_hashable_command_linez:CppCodeCache.load_async..get_hashable_command_lines.#<   !rzmain.cpp r^z optimized.cppr r r1)r{rrrrc>}|JJSr)rrI)r binary_pathr.futurerrg worker_fns rload_fnz(CppCodeCache.load_async..load_fnsR;) &Y[[F!>>>++K==C??? rr r)rTrrrrr)"rwr<rCrBr@r)r}r5r;rrrdevnullrr{r r rrWcpp_cache_precompile_headersrrLrrrr?r9rr&r r r@r|rT)&r.rMrrNrTrPrQrRr4main_needs_vec_isarshared_compile_commandmain_compile_commandoptimized_compile_commandr1main_build_optionoptimized_build_optionrV main_cmd_lineoptimized_cmd_line main_pathr?optimized_pathr  lock_pathr main_namer main_builderoptimized_nameoptimized_builderlinkerr]rZr[rrgr\s&` @@@@@r load_asynczCppCodeCache.load_asyncjs>',,S1,==a@3 i   6 n.B   " !%9 !LNNN   " +" &&!'!1!3!3 " " "   $ );P~~  % $% )=R~~?% % ! &T1I[E5I 1 n--%  #    "7" " " ( " "  ! ! ! !212CDD 667MNN zN)L)L])L)L   Y  ( %7I!!! A~~  ZN ci   6 6 6 6 6 6 \^^S7]CCI)-FC2 ;  77 DDD6;M%<<&2</ <<%8"1G 1T1T'Tv@R*AA4 AA*=%K9$U$U !Iz%!-% L" $J"%%!%/'* 6) %%%!$"$99;;)>>@@ !6 O O8N O O)   &-'!#4f= 7v7R7R7T7TUU %-'\O 7 5577            $Xi>>>667>>+666!*9!5!5666666666666666%CIcNy~s+MM  M rrc,|j|i|Sr)rqr.rrs rrzCppCodeCache.loads!.~s~t.v..000r)r rrrrrArrrrz)rNrNNN)rMrrrrNrrTrOrPrzrQr7rRr7rrrrrrrr)r}r~rrrrr%rprr@rDrrIrLrqrrrrr>r>?sCC9;E::::,u{++K022222&&&\&[,[ !%'%)%),0oooo[ob111[111rr>rk cpp_buildersSequence[CppBuilder]cddlm}||t5|D]G}tj|s|H ddddS#1swxYwYdS)Nrr r )r{r r|rr rTrr)rkrvr builders rr r !s/..... )\ 2 2 2  #  G7>>'">">"@"@AA                     sA A22A69A6ceZdZUiZded<eZded<ed,dZdd d Z d Z d Z d Z e jdZed-dZed.dZe d/d0d(Zed1d+ZdS)2CppPythonBindingsCodeCacher?rr_loaded_module_namesrr ctjtjD]"}tj|d#tjdSr)r{rrpr|rrrrs rrz&CppPythonBindingsCodeCache.cache_clear3s]"(..000.C ( (D KOOD$ ' ' ' '"7==?????rFTinclude_pytorchsharedr{zkernel({}); Py_RETURN_NONE;ra? // Python bindings to call {entry_func}(): #define PY_SSIZE_T_CLEAN #include #include #include #include // This is defined in guards.cpp so we don't need to import PyTorch headers that are slooow. // We manually link it below to workaround issues with fbcode build. static void* (*_torchinductor_pyobject_tensor_data_ptr)(PyObject* obj); template static inline T parse_arg(PyObject* args, size_t n) {{ static_assert(std::is_pointer_v, "arg type must be pointer or long"); return static_cast(_torchinductor_pyobject_tensor_data_ptr(PyTuple_GET_ITEM(args, n))); }} template <> inline int64_t parse_arg(PyObject* args, size_t n) {{ auto result = PyLong_AsSsize_t(PyTuple_GET_ITEM(args, n)); if(result == -1 && PyErr_Occurred()) [[unlikely]] throw std::runtime_error("expected int arg"); return result; }} template <> inline uintptr_t parse_arg(PyObject* args, size_t n) {{ auto result = PyLong_AsVoidPtr(PyTuple_GET_ITEM(args, n)); if(result == reinterpret_cast(-1) && PyErr_Occurred()) [[unlikely]] throw std::runtime_error("expected int arg"); return reinterpret_cast(result); }} template <> inline float parse_arg(PyObject* args, size_t n) {{ auto result = PyFloat_AsDouble(PyTuple_GET_ITEM(args, n)); if(result == -1.0 && PyErr_Occurred()) [[unlikely]] throw std::runtime_error("expected float arg"); return static_cast(result); }} {extra_parse_arg} static PyObject* {entry_func}_py(PyObject* self, PyObject* args) {{ try {{ if(!PyTuple_CheckExact(args)) [[unlikely]] throw std::runtime_error("tuple args required"); if(PyTuple_GET_SIZE(args) != {arg_len}) [[unlikely]] throw std::runtime_error("requires {arg_len} args"); {call_entry_func} }} catch(std::exception const& e) {{ PyErr_SetString(PyExc_RuntimeError, e.what()); return nullptr; }} catch(...) {{ PyErr_SetString(PyExc_RuntimeError, "unhandled error"); return nullptr; }} }} static PyMethodDef py_methods[] = {{ {{"{entry_func}", {entry_func}_py, METH_VARARGS, ""}}, {{NULL, NULL, 0, NULL}}}}; static struct PyModuleDef py_module = {{PyModuleDef_HEAD_INIT, "{entry_func}", NULL, -1, py_methods}}; PyMODINIT_FUNC PyInit_{entry_func}(void) {{ const char* str_addr = std::getenv("_TORCHINDUCTOR_PYOBJECT_TENSOR_DATA_PTR"); if(!str_addr) {{ PyErr_SetString(PyExc_RuntimeError, "_TORCHINDUCTOR_PYOBJECT_TENSOR_DATA_PTR must be set"); return nullptr; }} char* endptr = nullptr; errno = 0; uintptr_t addr = std::strtoull(str_addr, &endptr, 10); if(errno != 0 || endptr == str_addr || addr == 0) {{ PyErr_SetString(PyExc_RuntimeError, "Failed to parse _TORCHINDUCTOR_PYOBJECT_TENSOR_DATA_PTR"); return nullptr; }} _torchinductor_pyobject_tensor_data_ptr = reinterpret_cast(addr); PyObject* module = PyModule_Create(&py_module); if (module == NULL) {{ return NULL; }} #ifdef Py_GIL_DISABLED PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); #endif return module; }} r rrrcttjjjjt jd<|d|j} tj |S#t$rYnwxYwtj ||}|Jtj |}|tj |<t j||jJ|j||S)N'_TORCHINDUCTOR_PYOBJECT_TENSOR_DATA_PTRr)rrr_dynamor='_torchinductor_pyobject_tensor_data_ptrrenvironentry_functionrrKeyErrorrutilspec_from_file_locationmodule_from_specr{r|rloader exec_module)r.r r module_namerhris rrDz.CppPythonBindingsCodeCache._load_library_innersAD H  # KA A  <=33s133  ;{+ +    D ~55k4HH0066#) K "7;;KHHH{&&& ''' sA A"!A"rrzc t|Sr)rrKs rrLz1CppPythonBindingsCodeCache._get_uncompiled_headers%f---rrNr0rargtypesrOrMrrQr7rR num_outputsrrNrrTrc x ddt|D} jt |j| jj|} || z|||| || dd fd } | S) a Wrap a C++ function in fast Python bindings. Args: argtypes: The types of args to ENTRY_FUNCTION(), e.g. ["float*", "long"] main_code: C++ source code containing ENTRY_FUNCTION(). Will be built at -O3 if kernel_code is None (to maximize performance in any kernels that are present), or -O1 otherwise (to minimize compile time). needs_vec_isa: Whether the generated wrapper requires CPU vectorized host helpers. If omitted, this is inferred from the generated source as a conservative fallback. kernel_needs_vec_isa: Whether the separately compiled kernel source requires CPU vectorized host helpers. Only relevant when kernel_code is provided. kernel_code: If present, C++ source code that will be built at -O3 and linked to main_code. Returns: A python version of ENTRY_FUNCTION() rc3VK|]$\}}d|ddd|dV%dS)z parse_arg(args, rN)r)rr#argtypes rrzBCppPythonBindingsCodeCache.load_pybinding_async..s\  7 E266 D D D D D      r) array_len)arg_lencall_entry_func entry_funcextra_parse_arg)rNrTrPrQrRNrrct!ttsJtjSr)rrrr)r. get_resultrsrr[z?CppPythonBindingsCodeCache.load_pybinding_async..futures<~#!&*5555563#566 6rr^) rrsuffix_templateformatrCcall_entry_functionrrrq)r.rrMrrQrRrrNrTr parseargsrur[rrs` @@rload_pybinding_asyncz/CppPythonBindingsCodeCache.load_pybinding_asyncsBII  '11     $++MM3::9EE)/666MM ,   ^^   #&'!5$   7 7 7 7 7 7 7 7 rrrc,|j|i|Sr)rrss rload_pybindingz)CppPythonBindingsCodeCache.load_pybindings#8's'888:::rr$)r rrrrrrt)rNNr0NrN)rrOrMrrrrQr7rRr7rrrNrrTrOrrzrrru)r}r~rrrrir|r%rr@rrrtextwrapdedentrrrDrLrrrrrr{r{.sA8:E::::,6JLL8888@@@\@!!! N7O%hoU WWOr[$...[. !%),0%'"&<<<<[<|;;;[;;;rr{ceZdZUiZded<eddZdddZdZd Z e j d Z e ddZdS)CppWrapperCodeCacher?rrr cBtjdSr)rrrprrrrzCppWrapperCodeCache.cache_clears!'')))))rTr~inductor_entry_cppzreturn inductor_entry_cpp({});a #include static inline std::vector unpack_tensor_handle_list(PyObject* pyvec) {{ std::vector result; size_t result_len = PyList_GET_SIZE(pyvec); result.reserve(result_len); for (size_t i = 0; i < result_len; i++) {{ // AtenTensorHandle is essentially a pointer void* elem = PyCapsule_GetPointer(PyList_GET_ITEM(pyvec, i), NULL); result.push_back(reinterpret_cast(elem)); }} return result; }} static inline PyObject* pack_tensor_handle_list(const std::array& arr) {{ PyObject* result = PyList_New({array_len}); for (size_t i = 0; i < {array_len}; i++) {{ PyObject *elem = arr[i] == nullptr ? Py_NewRef(Py_None) // Store AtenTensorHandle as PyCapsulate : PyCapsule_New(reinterpret_cast(arr[i]), NULL, NULL); PyList_SET_ITEM(result, i, elem); }} return result; }} template <> inline std::vector parse_arg>(PyObject* args, size_t n) {{ return unpack_tensor_handle_list(PyTuple_GET_ITEM(args, n)); }} PyObject* inductor_entry_cpp(std::vector&& input_handles) {{ // For outputs, we only allocate an array to hold returned tensor handles, // not the actual output tensor storage. std::array output_handles{{}}; try {{ inductor_entry_impl(input_handles.data(), output_handles.data()); if (PyErr_Occurred()) {{ return nullptr; }} return pack_tensor_handle_list(output_handles); }} catch(std::exception const& e) {{ PyErr_SetString(PyExc_RuntimeError, e.what()); return nullptr; }} catch(...) {{ PyErr_SetString(PyExc_RuntimeError, "unhandled error"); return nullptr; }} }} rrrzc t|Sr)rrKs rrLz*CppWrapperCodeCache._get_uncompiled_header;s&v...rNr$rt)r}r~rrrr%rr@rrrrrrrLrrrrrs8:E::::***\* !!*N:%ho2 44Ol///[///rrceZdZUiZded<eejZdZded<e j dZ e e j dzZ e e j dzZ e j d Zed)dZed*dZeejd+dZed,dZeejd-dZeejd-dZe d.d/d"Zed0d%Zed+d&Zed1d(ZdS)2HalideCodeCachez*dict[str, Callable[[], ModuleType | CDLL]]rNrz_standalone_runtime_patha #include "{halideruntime_h}" #include "{headerfile}" #include #include namespace c10 {{ inline long div_floor_integer(long a, long b) {{ if ((a<0) != (b<0)) {{ const auto quot = a / b; const auto rem = a % b; return rem ? quot - 1 : quot; }} return a / b; }} }} z void kernel({argdefs}) {{ {buffers} int err = halide_kernel({buffer_names}); if(err != 0) throw std::runtime_error("halide_kernel failed"); }} a{ #include static const halide_device_interface_t* cuda_interface = halide_cuda_device_interface(); void kernel({argdefs}, uintptr_t stream) {{ {buffers} int err = halide_kernel(reinterpret_cast(stream), {buffer_names}); if(err != 0) throw std::runtime_error("halide_kernel failed"); }} a #include "{}" #include static int acquire_context(void* user_context, void** cuda_context_out, bool create) {{ return cuCtxGetCurrent(reinterpret_cast(cuda_context_out)); }} static int release_context(void* user_context) {{ return 0; }} static int get_stream(void* user_context, void* cuda_context, void** stream_out) {{ *stream_out = user_context; return 0; }} static int register_halide_hooks() {{ halide_set_cuda_acquire_context(&acquire_context); halide_set_cuda_release_context(&release_context); halide_set_cuda_get_stream(&get_stream); return 0; }} int inductor_register_halide_hooks_result = register_halide_hooks(); r{rr rrrrrc |jJ|j*t|jt|jksJ|jJ|jp|jd|j}|r d|d}d}d}d}n d}d}d|d}d }g} t |j|jD]!\} } | d | d | d"d |d t| dkrd|dd | dnd|d|d|d |d|d |d|d |d|d |d| d |dt| d |d|d|dg S)Nz + zreinterpret_cast(rcuda_interfacenullptrhalide_buffer_flag_device_dirty0zreinterpret_cast(halide_buffer_flag_host_dirtyzhalide_dimension_t(0, rzhalide_buffer_t ;rzhalide_dimension_t z _dims[] = {z};zhalide_dimension_t * z_dims = nullptr;z .device = z.device_interface = z.host = z .flags = z.type = z.dimensions = z.dim = z_dims;z.padding = nullptr;) shapestriderCoffsetalias_ofr{rrr halide_type) r.r{r rr!rdevice_interfacehostflagsdimssizers r_codegen_bufferzHalideCodeCache._codegen_buffersy$$$z%#ci..C OO*K*K*KKz%%%l.ch??3:??  4>8>>>F/ D5EEF( <<<.s???t]D]]???rHalideRuntimeCuda.hzHalideRuntime.hrc3`K|])}|j |d|jV*dS)NrW)r bindings_typer{)rrs rrz0HalideCodeCache._codegen_glue..sP:%??$$//qv//%%%%r)halideruntime_hrargdefsbuffers buffer_names)rrrr is_bufferrrrctyper{rlstripglue_template_cudaglue_template_cppr find_header) r.rrrrrrr  glue_template glue_codes r _codegen_gluezHalideCodeCache._codegen_glues,,..>T[89999t{****  .. . .FAs}} .##NqNN333s22=Q==#wOOPPPP#)++++##CH----))??w???@@GGII29T..s?T !((OO)0G%%6G"II <00)   rctddt}|}td|j|j|j|gdS)NOIrrr) r9r:rvr[rrrstandalone_runtime_cuda_initrb)r. command_gen command_lines r config_hashzHalideCodeCache.config_hashs!"    #3355  II)*4   fWoo   rruerrmsgcbtjjd}||jst d |jd}t j|D]}|dr tj dt j ||g}n#tj $rY`wxYwtjd|d}|rt j t j |d|}t j |r!t j |cS n"#t($r}t ||d}~wwxYwt |) Nhalidez$halide python bindings not installedrrslddz(/.*)/libHalide.sorrj)r machinery PathFinderr`rfrrrrvrC check_outputr rSubprocessErrorr searchrBrgrouprTr) rurrhrrAoutmr rs r_search_for_filez HalideCodeCache._search_for_files"-77AA <EFF F .4Q7F 6** 9 9=='' 9!(5"BGLL$>$>?&5!!! ! "7G9L9LMMA9!w||BGOOAGGAJJ,G,GPP7>>$//9#%7??4#8#8888 9 . . .v&&A - .6"""sB:F94B.-F.C=F?CB.Hs2   $'CL.load]s( #  """"?$$ $r)rr)r rir`rrrrUrr rTr!r executable schedulerrrrrr&r rC check_callrrrrbuild_standalone_runtimetouch_worker_task_halider)r.rr"rNdirpathgenfilelibfilerdonefilelockfile need_compilejobsrL binding_typestaskrrrs @@rgenerate_halide_asyncz%HalideCodeCache.generate_halide_asyncs  1 14899         Gd++++g 4455g 1122#4455 w'((w'((7>>(333   G + . . . + C~ M D#":":4>"J"JKLLL JJtyy{{ # # # KK )**?EE F F F  +/=    <<>> .   - - -22    dJ / / #">">"@"@A%1;dkkt"&,,..;e 3     KK )%:: ; ; ;$%8(DIID #,9T??#9   % % % % % % %  rrrc,|j|i|Sr)r rss rgenerate_halidezHalideCodeCache.generate_halideds#9(s($9&99;;;rc |jr+tj|jr|jStjrdnd}d}|dkrdnd}|jr5tj|jrJt}nt}t|d|d| z }tj |d t|d z }t|d z }t|d z }t|d z } t||z } tj|sfddl } ddlm} | |t 5tj|st#|d5} |dkr@| |j|ddddn #1swxYwY| | | |t1| \}}t3||| g|t5|}t7jt;j|tA|dddn #1swxYwYtj| sJ| |_| S)NrrzlibStandaloneHalideRuntime.soz host-cudarzhalide-runtime--TrRrrz hooks.cppzstandalone_halide_runtime.arr rsr)rr)!rrr rTrrrNrPrOr rrUrrr{r r|rrrrrcompile_standalone_runtimeTargetr?r9r;rCrrBrwrvr)r.rlibnamerbaser done_file lock_file hook_filea_fileso_filehlr rr{rhalide_cmd_gens rrz(HalideCodeCache.build_standalone_runtimehst  ' 0BGNN  (- -  0/ / % 7 7 9 9Dffu 1 +v 5 56  ' w~~c&BCC C CC %&&DD;;Dt**MMM#//:K:KMMM Gd++++&()) &()) +-.. W<<==g'((w~~i(( %     6 6 6 6 6 6)\22 % %w~~i00%i--&&00GG # @ G G$'OO4I$J$J!"!"11&"))F:K:KLLL'Mg'V'V$D*%/!!*F 3#-$9(3%%% &&&N) N$C$C$E$EFF)$$$1 % % % % % % % % % % % % % % %2w~~g&&&&&'.$s81K1AH8 KH K H B(KKKrcdS)z5Header precompiling is currently disabled for halide.NrrKs rrLz&HalideCodeCache._get_uncompiled_headers tr)r{rr rrrrr)rrrrrrr)rurrrrr)r{rrrr)rrr"rrNrrr)rrrrrrrt)r}r~rrrr%rprrrrrWrrrrrrr&rrrrr rrrLrrrrr@s?8:E::::,u{++K+///// X_ F& "" /(/  # # $38? $$ B! ! ! [! F[B_   _[ $###\#._ @ @ @_\ @_NNN_\N BFBBBB[BH<<<[<888[8t[rrrr list[partial[Any]]cddlm} ||t5|D] }| ddddS#1swxYwYdS#tj$r}t jddkrtt|dd^}}}t j | dr-t| }d} || d ksJGd d } |d } t!| t"sJ| || d z<t%jt%jd dg|dd} || | }t-dd5} | |dddn #1swxYwYt3d||d}~wwxYw)Nrr  HALIDE_REPRO1rL)rrrpythonz hl.main()rjceZdZddZdS) _worker_task_halide..OutrrcdS)Nrrrs r__repr__z)_worker_task_halide..Out.__repr__s$urNr)r}r~rr&rrrOutr$s(%%%%%%rr'r?z import sys, tempfile with tempfile.TemporaryDirectory() as out: sys.argv = zrepro.pyz? hl.main() rrszwrote repro.py: )r{r r|rCrrrrBrr rarr read_textcountindexrrrrrrrrrr)rr r jobrr"scriptrLr\mainr'cireplfds rrrs......$ Xh - -                         %    :>>. ) )S 0 0#*1e\#B#B FFSw''228<< BF||--//&zz$''1,,,,%%%%%%%%YYt__!"c*****!ceeBF O*4(:c(:  ||D$//*c**,bHHT[[]]+++,,,,,,,,,,,,,,,"#9a#9#9::A A sWA7 A;A;AG2EG-(G G-G G-G G--G2rcZt|d5 ddddS#1swxYwYdS)Nr)r)rs rrrsy h                       s  $$ceZdZUdZgZded<iZded<iZded<ed%d&dZ e d%ddd'dZ e d(ddd)dZ ed*d+d Z ee jd,d$ZdS)- PyCodeCachez:Caches generated Python modules and their source mappings.zlist[ModuleType]rzdict[str, ModuleType]modules_no_attrz dict[str, list[tuple[Any, ...]]]linemapsrr"rr^rrc&t|d|S)NpyrXr)r.r"r^s rrzPyCodeCache.writes[$e4444rNset_sys_modulesr9r7rc\t|d|\}}||||S)Nr7rXr8)rload_by_key_path)r.r"r^r9rr s rrzPyCodeCache.loads7+t5999 T##C#OOOrrr linemaplist[tuple[int, str]] | Noneattrsr*c|g}t}||n|}|?||jvr6|j|}|r%tj|j||St |||}|rtt||j |<|+| D]\}} t||| |r&| ||j|<|j ||S)Nr8) r3r4rrr1r}rNrIrr5rNsetattrr) r.rr r<r>r9 in_toplevelmodrPrQs rr;zPyCodeCache.load_by_key_paths ?G)++ )8)@++o =TS%888%d+C : &&s|S999J#COOO  5!%c7m!4!4CL     # #1Q""""  $},/#D) K  s # # # rFpurgerr c|r=|jD]5} |jsJtj|j&#t$rY2wxYw|j|jdS)z Clear the in-memory module cache. If purge=True, also delete all the corresponding on-disk source files. N)rrtrrrorpr4)r.rCrBs rrzPyCodeCache.cache_clear s  {  <''<Icl++++(D  !!#####s "0 ==linenorlist[dict[str, Any]] | Nonec||jvrdSt|j|dkrdS|j|\}}t||}|dkrdS||dz }|sdSdd}||S) Nrrj stack_tracerrlist[dict[str, Any]]cbd}tj||}dt|DS)Nz"File "(.+)", line (\d+), in (.+)\nc<g|]\}}}|t||dS))rrr{)r)rrlr#s rrrzPPyCodeCache.stack_frames_for_code..parse_stack_trace..Gs>Aq!A::r)r findallreversed)rHregexmatchess rparse_stack_tracez.parse_stack_traceBsC:Ej 44G'00 r)rHrrrI)r5rCr)r.r rErLrrentryrQs rstack_frames_for_codez!PyCodeCache.stack_frames_for_code0s s| # #4 s|D! " "a ' '4|D) u  ' ' 664a!e  4    ! '''rr)r"rr^rrr)r"rr^rr9r7rrr) rrr rr<r=r>r*r9r7rrF)rCrrr )r rrErrrF)r}r~rrrrr4r5rrrr;rr&rrSrrrr3r3sDDD !#G"""".0O////13H33335555[5P (, PPPPP[P 15'+ &(,&&&&&[&P $ $ $ $[ $_(((_[(((rr3rr"rcRtt||Sr)rr3r)rr"s r_load_triton_kernel_from_sourcerWOs" ;##K00+ > >>rctjrtjdd5}t |5}|cdddcdddS#1swxYwYdddn #1swxYwYtj }ttj j gd|| S)zz Compute a key representing the state of the CUTLASS library. Note: OSS and fbcode will have different keys. cutlass_libraryz src_hash.txtNr)r)r}r resourcesr rrcrbrurvrecutlass cutlass_dirrx) resource_path resource_file combined_hashs r cutlass_keyr`Usi1   $ $!>   1     1$1 %%''..00  1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1N$$MV^/0"mDDD    ! !!s5B&B* BB B B BB!BcReZdZdZddZddZddZdd Zdd ZddZ ddZ ddZ dS) DLLWrapperz A wrapper for a dynamic library.lib_pathrrr cJ||_d|_|dSr&)rcis_openr)rrcs rrzDLLWrapper.__init__ms#!   rcb|js'tj|j|_d|_dSdSr4)rer rCrcDLLrs rrzDLLWrapper.openus5| ' 66DHDLLL  rcN|jr|d|_dSdSr&)re_dlclosers rrzDLLWrapper.closezs. < ! MMOOO DLLL ! !rcd} ddl}|jrdSn#t$rYdSwxYwtrFt d}t |dst d}t |dr|j}n@tr#ddl}|dd}|j }ntd|ptr$tg|_ ||j jdStr.ddl}ddlm}|jg|_ ||j jdSdSt"d dS) Nrdlclosezlibc.sornT)use_last_errorz&Unsupported env, failed to do dlclose!)wintypeszKdll unloading function was not found, library may not be unloaded properly!)r is_finalizingrrTrrrkrUr' FreeLibraryrrrrg_handlermHMODULErr)r f_dlclosersymsr'rnrms rrizDLLWrapper._dlcloses   JJJ s ""      FF  :: P::D4++ 'ItY'' ) L \\ P MMM{{:d{CCH ,II%&NOO O  zz ,&.Z " $(*+++++ , ++++++&.&6%7 " $(*+++++  , , KK]     s  ((r{Callable[..., None]cz|jstd|jt|j|dfd }|S)NzCannot use closed DLL library: rrrr cD|}|rtdjdS)NzError in function: )rr})rerrmethods r _wrapped_funcz-DLLWrapper.__getattr__.._wrapped_funcs<&$-C L"#J#J#JKKK L Lrrrrr )rerrcrrg)rr{ryrxs @r __getattr__zDLLWrapper.__getattr__sd| RPPPQQ Q4(( L L L L L L rrc|Srrrs rr~zDLLWrapper.__enter__s rrrc.|dSrr)rrs rrzDLLWrapper.__exit__ rc.|dSrr~rs r__del__zDLLWrapper.__del__rrN)rcrrr r$)r{rrrt)rrrz) r}r~rrrrrrir{r~rrrrrrbrbjs**    !!!! ****X    rrbrc |dzS)z, standard format for the error path z.errorr)rs rbinary_error_pathrs  !!rceZdZUdZejGddZiZded<gZ ded<dZ d ed <dZ d ed <e d/dZ eedd0dZe d1dZe d2d3dZe d4dZe edd5d!Ze d6d"Ze d2d7d$Ze d8d&Ze d2d9d.ZdS):CUTLASSCodeCacheaB A cache for managing the compilation and loading source code specifically for CUTLASS. This class handles writing source code to files, compiling them into shared objects, and caching the results to avoid redundant compilations. It also manages error handling and logging for the compilation process. c2eZdZUded<ded<dZded<dS)CUTLASSCodeCache.CacheEntryr input_pathrNrz error_json)r}r~rrrrrr CacheEntryrs8!% %%%%%%rrdict[str, CacheEntry]rrrrr_SOURCE_CODE_SUFFIX_BACKENDrr c|j|j|jdSr)rrprrrrs rrzCUTLASSCodeCache.cache_clearsC  !!! rr)maxsizecaching_enabledrcaching_availablerc|stddS|sdS ddlm}|S#t$rtdYdSwxYw)ad Get or create the class instance of the CUTLASSKernelBinaryRemoteCache. Args: caching_enabled: Whether binary remote caching is enabled caching_available: Whether we're in fbcode environment Returns: CUTLASSKernelBinaryRemoteCache: The class instance of the kernel binary remote cache z6CUTLASSKernelBinaryRemoteCache not requested, skippingNr)CUTLASSKernelBinaryRemoteCachezECUTLASSKernelBinaryRemoteCache not available, remote caching disabled)rr-torch._inductor.fb.kernel_binary_remote_cacherrG)rrrs rget_kernel_binary_remote_cachez/CUTLASSCodeCache.get_kernel_binary_remote_caches  IIN O O O4  4       2133 3    IIW   44  s4$AActrrrs r _use_re_buildzCUTLASSCodeCache._use_re_build!!rN src_filesdst_file dst_file_ext extra_argsrVctrrr.rrrrs r_compile_commandz!CUTLASSCodeCache._compile_command s "!rctrrrs r_source_code_extraz#CUTLASSCodeCache._source_code_extrarrr"rctjjr(t|dgd|}|}n|}t ||j|\}}||fSz Writes source code into a file with dst_file_ext as the file extension. Returns the hash key of source code, and the path to the file. dummy_input dummy_outputrX)r)r[cutlass_hash_with_compile_cmdrrrrr)r.r"rrr^rrs rrzCUTLASSCodeCache.writesv > 7 -"$$m_nlSSO$EE**,,E S-DERRRZJrct|||\}}|dt|j |zS)z Returns the deterministic output path for `compile(source_code, dst_file_ext)` without performing the compile. Useful when a caller needs to reference the compiled artifact path before an async compile has finished. N)rrCr)r.r"rr?rs rget_output_pathz CUTLASSCodeCache.get_output_path*s? +|<< :9S!89999:\IIrrdc . |dkr:||d|\}}}|||\}}|gd} }n|||\}}|gd} }||z} | |jvrddlm} t } | t j| |dzt} | 5|d t|j  |z}t|}| tjjo tj tj }||||t j|rt+|d 5}|}d d d n #1swxYwYt/j|\}}|6tjjr%||tjj|||||j| <t;j||t j|sX|||||}t+|d 5}|d|d|j d| d|dd d d n #1swxYwYtC}tD#d|j | |tIj%|} |&rNddl'm(}||t j)|t j*|n+tWj,|tVj-t j.n#tVj/$rR}|0|j12d | ||||t;j||j1|d }~wtf$rb}dti|vrJ|0ti|| ||||t;j|ti|||d }~wwxYwtC}|j d| d||z d|}tD5|n"tD#d|j | ||6tjjr%||tjj|||d |j| <d d d n #1swxYwY|j| }|j6Dt/j|j6\}}t;j||7d |j| j8||fS)z Compiles CUDA source_code into a file with dst_file_ext extension. If dst_file_ext is "so", first compiles to ".o" and then links to ".so". Returns a tuple of dst_file_path, hash_key, source_code_path rrLinking Compilationrr r r N)rrrrrrz// rWz cmd // z %s %s: %s)run_build_command)rHenvzCOMPILE FAILED WITHz took z seconds. Command: z-%s %s skipped: %s since output already exists)9rrrr{r rWrr rr|rCrrrr)r[use_binary_remote_cacheforce_disable_cachesr}rBrTrrcrrupload_to_binary_remote_cacherRbinary_remote_cache_force_writerr+CUDACompileErrorrrrrrrBrwrtriton.fb.re_build_helperrrsrarCrSTDOUTrrE_record_compile_erroroutputrBrrrrrbr)r.r"rrobj_pathr?rrroperation_name key_with_extr rVrr error_pathbinary_remote_cachefhr cmd_parts error_outputrLr start_timererrorend_timelog_duration_msg cache_entrys rrzCUTLASSCodeCache.compile4s 4   [[c:FFNHa!ii \BBOC)1 I~II"ii \BBOC)3 m~I\) sy ( ( 6 6 6 6 6 6#~~H8BGLL3=AA<XXXD` X` X()HC0G,H,H+H)HILX .{;; &)&H&H$*N$J%8"77&,&6&8&8'I''# '2(++KDDD7>>*--Hj7;;;/r%'WWYY ///////////////.2j.D.D+I|+7"NH8 ,//&(V/2nn"K//CIl+.y,GGGw~~k227..!; jCj#..W!  Ucl U U^ U Uc U U UVVVWWWWWWWWWWWWWWW"&JIIk3<MMM % C 0 0I"$,,.. SSSSSS-- ) " ; ; " 0 0 = = '3 )*2C&8 W W W11!L//88(%&'/ "29elKKQVV$ $ $ $0CJJ>>55 #E , ) * + 3 #&"6y#e**"M"MSXX#  $ $vvH*-,'~'~'~'~xZdOd'~'~y|'~'~$HH-....IIG &# (3D4(++#V^%S+...[RV*W*W ,'A` X` X` X` X` X` X` X` X` X` X` X` X` X` X` XD479\3J  ! -&*j1G&H&H #I|&y,2E2Eg2N2NOO O ,'3S*EEs7B"""[" (, """"[""""["Yt___["JJJ[JQUFFFF[FBGGG[G $([rrcbeZdZdZdZeddZe dddZeddZdS)rcuCUDArrc(tjSr)r use_re_buildrs rrzCUDACodeCache._use_re_builds!.000rNrrrrrrrVc2tj||||SN)r)rcuda_compile_commandrs rrzCUDACodeCache._compile_commands'"6 x*    rcttjtjtjt g}|Sr)rrr_nvcc_compiler_options_nvcc_host_compiler_optionsr`r.r^s rrz CUDACodeCache._source_code_extrasJ#133"9;;">@@      rrrrr) r}r~rrrrrrrrrrrrsH111[1 (,     [  [rrceZdZUdZdZiZded<eddZe dddZ eddZ eddZ dS)rrWXPUzdict[str, DLLWrapper] dll_cacherrcdSr&rrs rrzXPUCodeCache._use_re_buildsurNrrrrrrrVc2tj||||Sr)xpu_compile_utilsxpu_compile_commandrs rrzXPUCodeCache._compile_commands'!4 x*    rcttjtjt g}|Sr)rr_sycl_compiler_sycl_compiler_optionsr`rs rrzXPUCodeCache._source_code_extra#s<!022!8::     rr"rc|dkrtd|d||||\}}}||jvrt||j|<|j|||fSr)rrrrbrs rrzXPUCodeCache.load.s 4  X-9XXJUXX 58KK 5 5 1 x!1  - -+5m+D+DCM- ( m,h8HIIrrrrrr) r}r~rrrrrrrrrrrrrrr sH')I))))[ (,     [  [JJJ[JJJrrceZdZUejGddZiZded<gZded<dZ dZ e dd Z e ddZe dddZe ddZdS)rc$eZdZUded<ded<dS)ROCmCodeCache.CacheEntryrrrNr|rrrrrEs'rrrrrrrWFrr c~tjtjdSr)rrrprrrrrzROCmCodeCache.cache_clearOs2!!####))+++++rr"rrrc|ttdgd|}t||j|\}}||fSr)rr1rr)r.r"r cuda_commandrrs rrzROCmCodeCache.writeTsT -., O O   0    ZJrNrrVrdc|jsGd|_ttt t |||\}}||jvrddlm }t}|tj ||dzt}|5|dt|j |z} tj | st%|g| ||} t'} | d} t+j| t*jdtj} td | n2#t*j$r }t5j| |j|d}~wwxYwt'}d || z d | }t|ntd || t<|| |j|<dddn #1swxYwY|j|j ||fS) z Compiles source_code into a file with dst_file_ext extension, using the compile command specific for the ROCm platform. Returns a tuple of dst_file_path, hash_key, source_code_path Trr r r NrW)rHrrzCompilation output: %szCompilation took z seconds. Compile command: z+Skip compiling %s: output %s already exists)!_logged_compiler_versionrrr<rr2rrr{r rWrr rr|rCrrTr1rrwrCrrrrEr+rrrrrr)r.r"rrrrr rVrrrLrrrrrrs rrzROCmCodeCache.compilecs+ G+/C ( II/MOO0D0DEE F F F))K>>Z ci   6 6 6 6 6 6#~~H8BGLL3=AA<XXXD S S()HC0G,H,H+H)HILX w~~k22.# k<C"&J # #I W!+!8%#-#4!% " """  ":FCCCC%8WWW!29elKKQVVW#vvH'r8j;P'r'rmp'r'r$HH-....IIE"# "/!9!9*k!R!R #7 S S S S S S S S S S S S S S S: #*C<r?r@rAtorch._inductor.cpu_vec_isarBrC!torch._inductor.custom_graph_passrDrErFrGrHrIrJrKtorch._inductor.freezing_utilsrLrM%torch._inductor.runtime.compile_tasksrN%torch._inductor.runtime.runtime_utilsrOrPtorch._inductor.utilsrQrRrSrTrUrVrW"torch._library.fake_class_registryrXrYtorch._loggingrZtorch._subclasses.fake_tensorr[r\r]torch._utils_internalr^torch.compilerrBtorch.compiler._cacher_r`ra)torch.export.pt2_archive._package_weightsrbrc"torch.export.pt2_archive.constantsrd%torch.fx.experimental.symbolic_shapesrerfrgtorch.utils._devicerhtorch.utils._ordered_setrirFrkrlrmrnrrprrqrrruntimersruntime.autotune_cacherttriton_bundlerru virtualizedrvrwryrrrrrrrrrrMrr(rrrirrrrrr runtime.hintsrrruntime.triton_heuristicsrrSrrfile_lock_timeoutr|_logginggetArtifactLoggerr}r*autotuning_log getLoggerrAOTAUTOGRAD_CACHE_PREFIXrrrrrrtorch.utils._functoolsrtorch_key_cacherrr(r4rWr[r`rirprrrrr!rrrrrrrrPicklerrrerrrrrr rrrrrrr=rrrregisterrrwrrxrzrrrrr rrrr;rrrr<r>r r{rrrrr3rWr`rbrrrtorch._inductor.codegen.xpurrrrrrrrrrDs"""""""      ......''''''''''((((((((******QPPPPPPPPPPPPPPPPPDDDDDDDDDDDD 444444444 CCCCCC'''''' ?>>>>>>>>>>> MLLLLLEDDDDD FEEEEEEE                    NMMMMMMMGGGGGGNNNNNNNN@?????AAAAAA++++++ 322222,,,,,, POOOOOOOIIIIII 544444////// )(((((33333333######888888)))))) GCLLy  ,,,,,,,, &&&&& &&&&" !GGGGGGGGGGGG)))))),,,,,,------$$$$$$      AAAAAAAA55555555::::::::;;;;;;      lg% ' .228]KK11(LIIg!!     4SSSS 4GGGG    IHHHHH    7 7 7 7 7 7 7 7 t''''''''244444i444n0000-----9; " " " " ";;;;;""""""""P &!!!! >         $ 8[[[[[&.[[[| V V V V XXXXB%%%%  d###b9b9b9b9b9b9b9$#b9J7Ajll"BBBBWWWWXXXX0000f U,U,U,U,U,U,U,U,p ,!!!!8['['['['['71:['['['|     M   aaaaa<0aaaH      Y Y Y Y Y Y Y Y x# # # # # # # # LjjjjjjjjZ.D.D.D.Dhgll,,..0EFF 7<< W55B BBBBJ      K K K K^1^1^1^1^1^1^1^1B     E;E;E;E;E;E;E;E;PE/E/E/E/E/4E/E/E/Peeeee0eeeP ''''T    s(s(s(s(s(s(s(s(l???? """"(UUUUUUUUp """ "VVVVVVVVr#####$###LKJJJJJ2J2J2J2J2J#2J2J2Jj`G`G`G`G`G`G`G`GF""""""""      ?   ")))))O)))))r