tjPd ddlZddlZddlZddlZddlZddlmcmZddlm Z m Z m Z m Z m Z ddlmZddlmZmZmZGddZGddeZGd d eZGd d eZGd deZGddeZGddeZGddeZGddeZGddeZde dzdefdZde fdZ de fdZ! d$d e"d!e"d"edzddfd#Z#dS)%N) _EventType_ExtraFields_PyCall_ExtraFields_PyCCall_ExtraFields_TorchOp_ProfilerEvent)profile)index_of_first_match traverse_bfs traverse_dfsceZdZdZddededdfdZedefdZd e fd Z d Z d e e fd Z d e e defdZd e fdZdZd e fdZd e fdZd e fdZd e fdZd e fdZdS)Patternz Base class for all patterns, subclass this class and implement match() to define custom patterns. In subclass, define description and skip property. Fprofshould_benchmarkreturnNc^||_||_d|_d|_d|_|j |jjtd|jj|_ i|_ |j D]5}|j |j g |6dS)Nz!Please specify a name for patternz(Please specify a description for patternz,profiler and kineto_results must not be None)rrname descriptionurlprofilerkineto_resultsAssertionErrorexperimental_event_tree event_treetid_root setdefault start_tidappend)selfrrevents ]/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/torch/profiler/_pattern_matcher.py__init__zPattern.__init__s 07 E = DM$@$H !OPP P-6NNPP9; _ H HE M $ $U_b 9 9 @ @ G G G G H HcdSNFrs r!skipz Pattern.skip)sur#r c8|jdt|}|S)Nz [Source Code Location] )rsource_code_location)rr msgs r!reportzPattern.report-s+ W W:Nu:U:U W W  r#c#>Kt|jEd{VdS)z Traverse the event tree and yield all events. Override this method in subclass to customize the traversal. N)r rr's r!eventTreeTraversalzPattern.eventTreeTraversal3s0  00000000000r#eventsc|jdt|d}|jr't|dr||n|S|S)N: z events matched. benchmark)rlenrhasattrbenchmark_summary)rr/default_summarys r!summaryzPattern.summary:sb!YGG#f++GGG   4--%&&v...$  r#c rdtdtfd}t|dstd||t d|D}t fd|D}|jdt|d |||z d t||z d d S) Ntime_nsrcJgd}|D]}|dkr |dd|cS|dz}|ddS)N)nsusmsiz.2f z sr&)r9unit_lstunits r! format_timez.Pattern.benchmark_summary..format_timeFs`)))H  ! !T>>%222D22222D %%%% %r#r2zPlease implement benchmark()c3$K|] }|jV dSN)duration_time_ns.0r s r! z,Pattern.benchmark_summary..Qs%GGuE2GGGGGGr#c3RK|]!}t||jzV"dSrC) input_shapesrD)rFr shapes_factor_maps r!rGz,Pattern.benchmark_summary..RsL   l511 2U5K K      r#r1z* events matched. Total Estimated Speedup: z (zX)) intstrr4rr2sumrr3round)rr/rA original_timenew_timerJs @r!r5zPattern.benchmark_summaryEs & & & & & &t[)) A !?@@ @ NN622GGGGGGG           y x xCKK x x(3 MH4L(M(M x xQVWdgoWoqrQsQs x x x r#ct)zu Return True if the event matches the pattern. This method should be overridden in subclass. )NotImplementedErrorrr s r!matchz Pattern.match[s "!r#cZjrgSfdD}|S)Nc>g|]}||Sr&)rU)rFr rs r! z*Pattern.matched_events..es:   DJJu2     ^(<     ,">""""^ 777777^77778^8888r#r c @eZdZ d dedededdffd Zdefd ZxZ S) NamePatternFrrrrNcnt||d||_||_dS)NzMatched Name Event: )superr"rr)rrrr __class__s r!r"zNamePattern.__init__s; /0008$88 r#r cDtj|j|jduSrC)researchrrTs r!rUzNamePattern.matchsyEJ//t;;r#ro) rprqrrrrMrtr"rrU __classcell__r{s@r!rxrxsAF#&:> <><<<<<<< aten::fill_ aten::normal_aten::uniform_ aten::zero_)rzr"rrrinit_opsrrrr{s r!r"zExtraCUDACopyPattern.__init__sN /000- nB    r#c6|jj p |jj SrCr with_stack record_shapesr's r!r(zExtraCUDACopyPattern.skip9''Fty/F+FFr#c|jdkrdS|}|jsdS|jd}|jdkrdS|jsdS|jd}|jdkrdSt|}t|dkrdS|d|d|dkrdS|}|j}|dS||}|dS|jr$|jd}|j|jvrd S|j$|j|jvS) Nzaten::toFrizaten::_to_copyz aten::copy_rKrr_T)rr` input_dtypesr3r\rkr)rr to_eventdtypess r!rUzExtraCUDACopyPattern.matchs. : # #5~ 5r" :) ) )5~ 5r" : & &5e$$ v;;??5 !9 q VAY 6 65  =5 U## =5n N2&EzT]**t n  zT]**r#r/cd|D}|D]v}|d}tjdd|i}tjdd|i}|dj}|dj}||z ||<w|S)Nc.i|]}t|dSgrIrEs r! z2ExtraCUDACopyPattern.benchmark.."JJJ%\%00#JJJr#rztorch.ones(size).to("cuda")sizestmtglobalsztorch.ones(size, device="cuda") )r2Timertimeitmean) rr/rJshaperto_timerde_timerto_timede_times r!r2zExtraCUDACopyPattern.benchmarksJJ6JJJ& 9 9E8D 2VTNH!6Hoob)).Goob)).G'.'8 e $ $  r#ro)rprqrrrsrrtr"rur(rUrvrr2rrs@r!rrs   W           GdGGGXG#+#+#+L !^ 4 ! ! ! ! ! ! ! !r#rcDeZdZdZd dededdffd ZdZd efd Z xZ S) ForLoopIndexingPatterna This pattern identifies if we use a for loop to index a tensor that can be vectorized. example: tensor = torch.empty((100, 100)) for i in range(100): tensor[i] = i Pattern: aten::select | ... | aten::select | ... (Repeat) Algorithm: We start at node aten::select, and we check if we can find this alternating patterns. We also keep a dictionary to avoid duplicate match in the for loop. FrrrNct||d|_d|_t |_dS)NzFor Loop Indexing Patternz6For loop indexing detected. Vectorization recommended.)rzr"rrsetvisitedrs r!r"zForLoopIndexingPattern.__init__s< /000/ S!$ r#c#>Kt|jEd{VdS)zN We need to use BFS traversal order to avoid duplicate match. N)r rr's r!r.z)ForLoopIndexingPattern.eventTreeTraversals0 00000000000r#r c 0|jdkrdS|j|jvrdSd}||\}}t |dkrdSdt fd}t |d}|dS|g|d|z}|t |dz d}tdt |t |D]Q}|||||t |zr+|dz }|j||jQ|dkS) N aten::selectFr_rct|t|krdSt||dD]\}}|j|jkrdSdS)NFT)strict)r3zipr)list1list2op1op2s r!same_opsz.ForLoopIndexingPattern.match..same_opss`5zzSZZ''uuT::: ! !S8sx'' 55(4r#c|jdkS)Nr)res r!z.ForLoopIndexingPattern.match..s qv?Wr#rr) ridrrbr3rtr rangeadd) rr repeat_countrenextrnext_select_idx indexing_opsis r!rUzForLoopIndexingPattern.match sR : ' '5 8t| # #5 ""5))4 t99>>5 d    /t5W5WXX  "5w&6&6!77 C %%)++,q#d))S%6%677  Ax d1q3|3D3D/D+D&EFF !    a,,,,r!!r#ro) rprqrrrsrrtr"r.rrUrrs@r!rrs ''W'''''''' 111 ">""""""""r#rc|eZdZd dededdffd ZefdZdedefd Z defd Z d e efd Z xZ S)FP32MatMulPatternFrrrNcvt||d|_d|_d|_dS)NzFP32 MatMul Patternz|You are currently using GPU that supports TF32. Please enable TF32 by setting 'torch.backends.cuda.matmul.allow_tf32 = True'zUhttps://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devicesrzr"rrrrs r!r"zFP32MatMulPattern.__init__-sA /000)  [ kr#ctjjd}n5tdtjD}|dupt jp |jj S)NFc3dK|]+}ttjdd|dkV,dS)z sm_|compute_rPN)rLr}sub)rFarchs r!rGz)FP32MatMulPattern.skip..<sQBF>2t4455;r#) torchversionhipallcuda get_arch_listrzr(rr)rhas_tf32r{s r!r(zFP32MatMulPattern.skip6su =  (HH!J4466H5 OEGGLO 8O4OOr#r c|jtjkrdSt|jt s)t dt|jj|j dkr|jj durdSdS)NF#expected _ExtraFields_TorchOp, got aten::mmT) tagrTorchOp isinstance extra_fieldsrrtyperprallow_tf32_cublasrTs r!rUzFP32MatMulPattern.matchBs 9 * * *5%,.BCC  Yd5;M6N6N6WYY  : # #!3u<<tur#c|jSrC)rrTs r!r,zFP32MatMulPattern.reportOs r#r/cd|D}|D]}tj|ddtj}tj|ddtj}tjd||d}tjdd ||d }d tjjj_| d j }| d j } | |z ||<|S) Nc.i|]}t|dSrrrEs r!rz/FP32MatMulPattern.benchmark..Srr#rrdevicedtyper_torch.mm(matrixA, matrixB)matrixAmatrixBrz,torch.backends.cuda.matmul.allow_tf32 = True)rsetuprFr) rrandnfloat32r2rbackendsrmatmul allow_tf32rr) rr/rJrrr fp32_timer tf32_timer fp32_time tf32_times r!r2zFP32MatMulPattern.benchmarkRsJJ6JJJ& = =Ek%(6OOOGk%(6OOOG"1$+@@J#1D$+@@J 5:EN  & 1"))"--2I"))"--2I'09'< e $ $  r#ro)rprqrrrrtr"rur(rrUr,rvr2rrs@r!rr,skkWkkkkkkkk P P P PX P > d     N    !^ 4!!!!!!!!r#rcBeZdZdZd dededdffd Zdedefd ZxZ S) OptimizerSingleTensorPatterna{ This pattern identifies if we are using the single-tensor version of an optimizer. example: optimizer = torch.optim.SGD(model.parameters(), lr=0.1) By adding foreach=True to enable multi-tensor optimizer, we can gain speedup when the kernels are relatively small. Pattern: XXXXX: _single_tensor_ Algorithm: String match FrrrNct||d|_gd|_d|_d|_dS)NzOptimizer Single Tensor Pattern)adamsgdadamwzDetected optimizer running with single tensor implementation. Please enable multi tensor implementation by passing 'foreach=True' into optimizer.r)rzr"roptimizers_with_foreachrrrs r!r"z%OptimizerSingleTensorPattern.__init__vsN /0005 '?'?'?$ b r#r cZ|jD]"}|jd|rdS#dS)N_single_tensor_TF)rrendswith)rr optimizers r!rUz"OptimizerSingleTensorPattern.matchsF5  Iz""#@Y#@#@AA tt ur#ro rprqrrrsrrtr"rrUrrs@r!rrgs  W>dr#rcBeZdZdZd dededdffd Zdedefd ZxZ S) SynchronizedDataLoaderPatterna This pattern identifies if we are using num_workers=0 in DataLoader. example: torch.utils.data.DataLoader(dataset, batch_size=batch_size) Add num_workers=N to the arguments. N depends on system configuration. Pattern: dataloader.py(...): __iter__ dataloader.py(...): _get_iterator NOT dataloader.py(...): check_worker_number_rationality Algorithm: If we don't see check_worker_number_rationality call in the dataloader __iter__, It is not an asynchronous dataloader. FrrrNcvt||d|_d|_d|_dS)NzSynchronized DataLoader PatternzDetected DataLoader running with synchronized implementation. Please enable asynchronous dataloading by setting num_workers > 0 when initializing DataLoader.zjhttps://pytorch.org/tutorials/recipes/recipes/tuning_guide.html#enable-async-data-loading-and-augmentationrrs r!r"z&SynchronizedDataLoaderPattern.__init__sC /0005  n   : r#r c"dtdtfd} |j}n#t$rYdSwxYw||jdsdS|jsdS|jd}||jdsdS|jsdS|jd}||jd S) Nr function_namec|tjddddo||S)Nrutilsdataz dataloader.py) startswithospathjoinr)rrs r!is_dataloader_functionzCSynchronizedDataLoaderPattern.match..is_dataloader_functionsC?? WgvGG/-- .. /r#F__iter__r _get_iteratorcheck_worker_number_rationality)rMrUnicodeDecodeErrorr`)rr rres r!rUz#SynchronizedDataLoaderPattern.matchs / /S / / / /  AA!   55 &%ej*== 5~ 5q!%%ej/BB 5~ 5q!))%*6WXXXXs  **rorrs@r!rrs"   W           Y>YdYYYYYYYYr#rcBeZdZdZd dededdffd Zdedefd ZxZ S) GradNotSetToNonePatterna This pattern identifies if we are not setting grad to None in zero_grad. example: optimizer.zero_grad() By setting set_to_none=True, we can gain speedup Pattern: XXXXX: _zero_grad NOT aten::zeros aten::zero_ aten::zero_ is called on each parameter in the model. We also want to make sure it is not called by aten::zeros. Algorithm: String match FrrrNcvt||d|_d|_d|_dS)Nz,Gradient Set To Zero Instead of None PatternzfDetected gradient set to zero instead of None. Please add 'set_to_none=True' when calling zero_grad().zxhttps://pytorch.org/tutorials/recipes/recipes/tuning_guide.html#disable-gradient-calculation-for-validation-or-inferencerrs r!r"z GradNotSetToNonePattern.__init__sD /000B  F   H r#r c|jdsdS|jsdSt|jD] }|jdkr|jjdkrdS!dS)Nz : zero_gradFrz aten::zerosT)rrr`r r\)rr sub_events r!rUzGradNotSetToNonePattern.matchsrz""=11 5~ 5%en55  I-//$)]::ttur#rorrs@r!r r s$   W            > d        r#r cXeZdZdZd dededdffd ZefdZd e fd Z xZ S) &Conv2dBiasFollowedByBatchNorm2dPatternau This pattern identifies if we are enabling bias in Conv2d which is followed by BatchNorm2d. Bias doesn't do anything when followed by batchnorm. Pattern: nn.Module: Conv2d | nn.Module: BatchNorm2d ... aten::conv2d AND dtype of third argument is not null The third argument is the bias Algorithm: String match FrrrNcvt||d|_d|_d|_dS)Nz5Enabling Bias in Conv2d Followed By BatchNorm PatternzcDetected bias enabled in Conv2d that is followed by BatchNorm2d. Please set 'bias=False' in Conv2d.zhttps://pytorch.org/tutorials/recipes/recipes/tuning_guide.html#disable-bias-for-convolutions-directly-followed-by-a-batch-normrrs r!r"z/Conv2dBiasFollowedByBatchNorm2dPattern.__init__sA /000K A O r#cF|jjduptjSr%)rrrzr()rr{s r!r(z+Conv2dBiasFollowedByBatchNorm2dPattern.skipsy&%/?577<?r#r c&|jdkrdStt|dkst|ddS||d}|sdS||}|sdS|jdS)Nz aten::conv2dFrKc6|jdS)Nznn.Module: Conv2d)rrrs r!rz>Conv2dBiasFollowedByBatchNorm2dPattern.match..sQV../BCCr#znn.Module: BatchNorm2d)rr3rrnrgrrTs r!rUz,Conv2dBiasFollowedByBatchNorm2dPattern.match s : ' '5 |E"" # #a ' '<+>+>q+A+I5  CC   5 U## 5z$$%=>>>r#ro) rprqrrrsrrtr"rur(rrUrrs@r!rrs    W        @@@@X@?>????????r#rcreZdZd dededdffd ZedefdZdedefd Z d e efd Z xZ S) MatMulDimInFP16PatternFrrrNcvt||d|_d|_d|_dS)Nz3Matrix Multiplication Dimension Not Aligned PatternzUDetected matmul with dimension not aligned. Please use matmul with aligned dimension.z[https://pytorch.org/tutorials/recipes/recipes/tuning_guide.html#use-mixed-precision-and-amprrs r!r"zMatMulDimInFP16Pattern.__init__s8 /000I rpr#c6|jj p |jj SrCrr's r!r(zMatMulDimInFP16Pattern.skip"rr#r cd}|jdvrdSt|sdSt|d}|tjtjfvr|t |dsdSdS)Nc:tfd|DS)Nc3DK|]}|ddD] }|zdkVdS)Nrr&)rFrdimmultiples r!rGzCMatMulDimInFP16Pattern.match..mutiple_of..(sDUUu%PRPSPS*UU3sX~*UUUUUUUr#)rshapesr!s `r! mutiple_ofz0MatMulDimInFP16Pattern.match..mutiple_of's&UUUUUUUUU Ur#)rz aten::bmmz aten::addmmFrT)rrrbfloat16halfrI)rr r$ arg_dtypes r!rUzMatMulDimInFP16Pattern.match&s V V V :E E E5E"" 5 ''* 4 4 4ZZ   > > 44ur#r/cnd}d|D}|D]!}tj|ddtj}tj|ddtj}tjd||d }tj||dd dtj}tj||dd dtj}tjd||d }|d j} |d j} | | z ||<#|S) Nc fd|DS)NcDg|]}tj|z zSr&)mathceil)rFrr!s r!rXzNMatMulDimInFP16Pattern.benchmark..closest_multiple..7s-OOOuHty)9:::OOOr#r&r"s `r!closest_multiplez:MatMulDimInFP16Pattern.benchmark..closest_multiple6sOOOOOOO Or#c.i|]}t|dSrrrEs r!rz4MatMulDimInFP16Pattern.benchmark..9rr#rrrr_rrrr%r)rrfloat16r2rrr) rr/r.rJrrrnot_aligned_dim_timeraligned_dim_timernot_aligned_dim_timealigned_dim_times r!r2z MatMulDimInFP16Pattern.benchmark5sn P P PKJ6JJJ& O OEk%(6OOOGk%(6OOOG$-O1$+@@%%% !k  q1--fEMGk  q1--fEMG!*1$+@@!!! $9#?#?#C#C#H 077;;@ '7:N'N e $ $  r#ro) rprqrrrrtr"rur(rrUrvr2rrs@r!rrsqqWqqqqqqqq GdGGGXG > d    !^ 4!!!!!!!!r#rr rc|r|jtjks|jtjkrt |jt tfs)tdt|jj |jj j dtjzs%|jj j d|jj jS|j}|dS)Nz:expected _ExtraFields_PyCall or _ExtraFields_PyCCall, got r:zNo source code location found)rrPyCallPyCCallrrrrrrrpcaller file_namerrsep line_numberr\r s r!r*r*Qs   9 ) ) )UY*:L-L-L"%8:N$O %? 233<??%,6AA'BFBRSS h,3=gg@R@Y@eggg    + *r#ct|jts)tdt |jjt d|jjDS)Nrc3RK|]"}tt|ddV#dS)sizesr&N)tuplegetattrrFrs r!rGzinput_shapes..fs6SSAwq'2..//SSSSSSr#rrrrrrprAinputsr=s r!rIrIash e(*> ? ?  U$u7I2J2J2S U U    SS9K9RSSS S SSr#ct|jts)tdt |jjt d|jjDS)Nrc38K|]}t|ddVdS)rN)rBrCs r!rGzinput_dtypes..ns.NNqGT**NNNNNNr#rDr=s r!rrish e(*> ? ?  U$u7I2J2J2S U U    NNE4F4MNNN N NNr#FTr print_enablejson_report_dirc i}t||t||t||t||t ||t ||t ||g}t}g}dddg}|d|D]} | } | s|| | | D]} | | } | |vr|| | | t| d\} }|| gt!|| j| j| jd|t(j|d}t(j|rRt1|5}t3j|}|||}dddn #1swxYwYt1|d5}t3j||d dddn #1swxYwY|d ||z }|ddd|r$t;d |dSdS) Nz(----------------------------------------zTorchTidy ReportzMatched Events:r6)r<rrmessageztorchtidy_report.jsonw)indentzSummary: )rrrrr rrrrrYr7r,rr*splitrrLrrrrrrexistsopenjsonloadupdatedumpprint)rrrHrI report_dict anti_patternsreported summaries message_list anti_patternrYr report_msg src_locationline_nojson_report_pathfexisiting_reports r!report_all_anti_patternsrdqsn KT#344$ 011$T+;<<%d,<==&677.t5EFFt%566 MuuHI;;;;>>#  E%,,U33J))##J/// Z((((>* + + /&'' /1#'9Q<<  '' 444.  / / / / / / / / / / / / / / /"C ( ( 0A Ik1Q / / / / 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ###IL8??X??@@@' dii %%&&&&&''s$&,HH"%H"9II!$I!)FTN)$rSr,rr}rtorch.utils.benchmarkrr2torch._C._profilerrrrrrtorch.profilerrtorch.profiler._utilsr r r r rxrrrrrr rrrMr*rIrrtrdr&r#r!risG )))))))))#"""""RRRRRRRRRRppppppppl < < < < <' < < <T!T!T!T!T!7T!T!T!n<"<"<"<"<"W<"<"<"~8!8!8!8!8!8!8!8!v7@6Y6Y6Y6Y6YG6Y6Y6Yt,,,,,g,,,^(?(?(?(?(?W(?(?(?V3!3!3!3!3!W3!3!3!l + 5 +# + + + + TTTTTOOOOO#"& 8'8'8'8'4Z 8'  8'8'8'8'8'8'r#