§
    kŠtjD  ã                   ó¶  — d Z ddlZddlZddlZddlZddlZddlmZ ddlZddlZddl	m
Z
mZmZmZmZmZmZmZmZmZmZ ddlmZ ddlmZmZ ddlmZmZmZmZ ddlm Z  dd	l!m"Z"  ej#        d
¦  «        Z$ ej%        d¬¦  «        Z%dej&        vr e'e%¦  «        ej&        d<   ddl(Z(ddl)m*Z*m+Z+m,Z, d„ Z-d„ Z.de/de/fd„Z0d„ Z1d„ Z2d„ Z3e4dk    r e3¦   «          dS dS )a>  Benchmarking the inference of pretrained transformer models.
PyTorch/TorchScript benchmark is based on https://github.com/huggingface/transformers/blob/master/examples/benchmarks.py.
One difference is that random input_ids is generated in this benchmark.

For onnxruntime, this script will convert a pretrained model to ONNX, and optimize it when -o parameter is used.

Example commands:
    Export all models to ONNX, optimize and validate them:
        python benchmark.py -b 0 -o -v -i 1 2 3
    Run OnnxRuntime on GPU for all models:
        python benchmark.py -g
    Run OnnxRuntime on GPU for all models with fp32 optimization:
        python benchmark.py -g -o
    Run OnnxRuntime on GPU with fp16 optimization:
        python benchmark.py -g -o -p "fp16"
    Run TorchScript on GPU for all models:
        python benchmark.py -e torchscript -g
    Run TorchScript on GPU for all models with fp16:
        python benchmark.py -e torchscript -g -p "fp16"
    Run ONNXRuntime and TorchScript on CPU for all models with quantization:
        python benchmark.py -e torchscript onnxruntime -p "int8" -o
    Run OnnxRuntime with bfloat16 fastmath mode kernels on aarch64 platforms with bfloat16 support:
        python benchmark.py --enable_arm64_bfloat16_fastmath_mlas_gemm

It is recommended to use run_benchmark.sh to launch benchmark.
é    N)Údatetime)ÚConfigModifierÚOptimizerInfoÚ	PrecisionÚcreate_onnxruntime_sessionÚget_latency_resultÚinference_ortÚinference_ort_with_io_bindingÚoutput_detailsÚoutput_fusion_statisticsÚoutput_summaryÚsetup_logger)ÚFusionOptions)ÚMODEL_CLASSESÚMODELS)Úcreate_onnxruntime_inputÚexport_onnx_model_from_ptÚexport_onnx_model_from_tfÚload_pretrained_model)Úversion)ÚQuantizeHelperÚ F)ÚlogicalÚOMP_NUM_THREADS)Ú
AutoConfigÚAutoTokenizerÚLxmertConfigc                 ó¼	  — dd l }g }| r^d|                     ¦   «         vrHd|                     ¦   «         vr2d|                     ¦   «         vrt                               d¦  «         |S d}|dk    r@t          j        }d}d|                     ¦   «         vrt                               d	¦  «         |S |t          j        k    rt                               d
|› d�¦  «         |D �]û}t          |         d         }|
D �]á}|t          |¦  «        k    r �nÊ|d |…         }t          |         d         |_	        t          j        |¦  «        }d|v r‚t          j        ¦   «         5  t          |t          |         d         t          |         d         t          |         d         |||||| |||||||¦  «        \  }} }!}"d d d ¦  «         n# 1 swxY w Y   d|v rWt          |t          |         d         t          |         d         t          |         d         |||||| |||||||¦  «        \  }} }!}"| s�Œ4t!          || |d|||¬¦  «        }#|#€�ŒNd„ |#                     ¦   «         D ¦   «         }$g }%| rdnd}&t%          j        ||¬¦  «        }'t)          j        t-          |¦  «        t-          |¦  «        t-          |!|'j        ¦  «        g¦  «        }(t)          j        t-          |¦  «        |'j        g¦  «        })|D �]ç}*|*dk    rŒ
|D �]Ù}+|"�|+|"k    rŒd|v rt(          j        nt(          j        },t5          |!|*|+||'|,¦  «        }-d|j        ||&||| ||||*|+|                     ¦   «         t;          t=          j        ¦   «         ¦  «        dœ}.|'j	        dv r/t                                d|› d|*d|'j!        |'j!        g› �¦  «         n"t                                d|› d|*|+g› �¦  «         |rtE          |#|-|.|	|*|¦  «        }/n¶|# #                    |$|-¦  «        }0|(g}1tI          t          |0¦  «        ¦  «        D ]J}2|2dk    r-t          |         d         dk    r|1 %                    |)¦  «         Œ5|1 %                    |(¦  «         ŒKd|v rt(          j&        nt(          j'        }3tQ          |#|-|.|	|$|0|%|1|*|&|3|¦  «        }/t                                |/¦  «         | %                    |/¦  «         �ŒÛ�Œé�Œã�Œý|S )Nr   ÚCUDAExecutionProviderÚMIGraphXExecutionProviderÚDmlExecutionProviderzŽPlease install onnxruntime-gpu or onnxruntime-directml package instead of onnxruntime, and use a machine with GPU for testing gpu performance.Útensorrté   ÚTensorrtExecutionProviderzhPlease install onnxruntime-gpu-tensorrt package, and use a machine with GPU for testing gpu performance.zOptimizerInfo is set to zA, graph optimizations specified in FusionOptions are not applied.é   Úpté   é   ÚtfT)Úenable_all_optimizationÚnum_threadsÚverboseÚ(enable_mlas_gemm_fastmath_arm64_bfloat16c                 ó   — g | ]	}|j         ‘Œ
S © )Úname)Ú.0Únode_args     ú`/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/onnxruntime/transformers/benchmark.pyú
<listcomp>z#run_onnxruntime.<locals>.<listcomp>Û   s   € ÐXÐXÐX°( ¤ÐXÐXÐXó    ÚcudaÚcpu©Ú	cache_dirÚonnxruntime©Úenginer   Ú	providersÚdeviceÚ	optimizerÚ	precisionÚ
io_bindingÚ
model_nameÚinputsÚthreadsÚ
batch_sizeÚsequence_lengthÚcustom_layer_numr   ©ÚvitÚswinzRun onnxruntime on ú with input shape Úgpt))r:   Úget_available_providersÚloggerÚerrorr   ÚNOOPTÚwarningr   ÚlenÚ
model_typer   ÚparseÚtorchÚno_gradr   r   r   Úget_outputsr   Úfrom_pretrainedÚnumpyÚprodÚmaxÚhidden_sizeÚint64Úint32r   Ú__version__Úget_layer_numÚstrr   ÚnowÚinfoÚ
image_sizer	   ÚrunÚrangeÚappendÚlonglongÚintcr
   )4Úuse_gpuÚproviderÚmodel_namesÚmodel_classÚconfig_modifierr@   r+   Úbatch_sizesÚsequence_lengthsÚrepeat_timesÚinput_countsÚoptimizer_infoÚvalidate_onnxr9   Úonnx_dirr,   Ú	overwriteÚdisable_ort_io_bindingÚuse_raw_attention_maskÚmodel_fusion_statisticsÚmodel_sourceÚ(enable_arm64_bfloat16_fastmath_mlas_gemmÚargsr:   ÚresultsÚwarm_up_repeatrB   Úall_input_namesÚ
num_inputsÚinput_namesÚfusion_optionsÚonnx_model_fileÚis_valid_onnx_modelÚ
vocab_sizeÚmax_sequence_lengthÚort_sessionÚort_output_namesÚoutput_buffersr>   ÚconfigÚmax_last_state_sizeÚmax_pooler_sizerE   rF   Úinput_value_typeÚ
ort_inputsÚresult_templateÚresultÚort_outputsÚoutput_buffer_max_sizesÚiÚ	data_types4                                                       r3   Úrun_onnxruntimer•   X   ss  € ð2 ÐÐÐà€Gàð	à$¨K×,OÒ,OÑ,QÔ,QÐQÐQØ(°×0SÒ0SÑ0UÔ0UÐUÐUØ#¨;×+NÒ+NÑ+PÔ+PÐPÐPå�Šð ]ñ	
ô 	
ð 	
ð ˆà€NØ�:ÒÐÝ&Ô,ˆØˆØ&¨k×.QÒ.QÑ.SÔ.SÐSÐSÝ�LŠLØzñô ð ð ˆNà�Ô,Ò,Ð,Ý�ŠØx ~ÐxÐxÐxñ	
ô 	
ð 	
ð "ð e+ñ e+ˆ
Ý  Ô,¨QÔ/ˆØ&ð c	+ñ c	+ˆJØ�C Ñ0Ô0Ò0Ð0Ø‘à)¨+¨:¨+Ô6ˆKÝ$ ZÔ0°Ô3ˆDŒOÝ*Ô0°Ñ6Ô6ˆNà�|Ð#Ð#Ý”]‘_”_ð ð õ 2Ø"Ý˜zÔ*¨1Ô-Ý˜zÔ*¨1Ô-Ý˜zÔ*¨1Ô-Ø#Ø'Ø!Ø Ø#ØØ!Ø&Ø%Ø.Ø!Ø/Ø&ñ#ô ñØ'Ø+Ø"Ø+ðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð2 �|Ð#Ð#õ .ØÝ˜:Ô& qÔ)Ý˜:Ô& qÔ)Ý˜:Ô& qÔ)ØØ#ØØØØØØ"Ø!Ø*ØØ+Ø"ñ#ô ñØ#Ø'ØØ'ð* 'ð Ùå4ØØØØ(,Ø'ØØ9aðñ ô ˆKð Ð"ÙàXÐX¸k×>UÒ>UÑ>WÔ>WÐXÑXÔXÐØˆNØ&Ð1�V�V¨EˆFÝÔ/°
ÀiÐPÑPÔPˆFÝ"'¤*å˜Ñ$Ô$ÝÐ(Ñ)Ô)Ý˜
 FÔ$6Ñ7Ô7ðñ#ô #Ðõ $œj­#¨kÑ*:Ô*:¸FÔ<NÐ)OÑPÔPˆOØ)ð L+ñ L+�
Ø ’?�?ØØ'7ð I+ñ I+�OØ*Ð6¸?ÐM`Ò;`Ð;`Ø à6:¸lÐ6JÐ6J¥u¤{ {ÕPUÔP[Ð$Ý!9Ø"Ø"Ø'Ø#ØØ(ñ"ô "�Jð #0Ø#.Ô#:Ø%-Ø"(Ø%3Ø%.Ø*@Ð&@Ø&0Ø",Ø#.Ø&0Ø+:Ø,;×,IÒ,IÑ,KÔ,KÝ$'­¬©¬Ñ$7Ô$7ð'ð '�Oð" Ô(¨OÐ;Ð;ÝŸšð H°*ð  Hð  HÐQ[Ð]^Ð`fÔ`qÐsyô  tEð  QFð  Hð  Hñô ð ð õ ŸšÐ$w¸*Ð$wÐ$wÐYcÐetÐXuÐ$wÐ$wÑxÔxÐxà-ð "Ý!.Ø'Ø&Ø+Ø(Ø&Ø*ñ"ô "˜˜ð '2§o¢oÐ6FÈ
Ñ&SÔ&S˜Ø3FÐ2GÐ/Ý!&¥s¨;Ñ'7Ô'7Ñ!8Ô!8ð Tð T˜AØ  Ašv˜v­&°Ô*<¸QÔ*?À5Ò*HÐ*Hà 7× >Ò >¸Ñ OÔ OÐ OÐ Oà 7× >Ò >Ð?RÑ SÔ SÐ SÐ Sà6:¸lÐ6JÐ6J¥E¤N NÕPUÔPZ˜	Ý!>Ø'Ø&Ø+Ø(Ø,Ø'Ø*Ø3Ø&Ø"Ø%Ø*ñ"ô "˜õ —K’K Ñ'Ô'Ð'Ø—N’N 6Ñ*Ô*Ð*Ñ*ñSI+ñL+ùð\ €Ns   ÅAGÇGÇGc                 ó°  ‡‡— g }| r:t           j                             ¦   «         st                               d¦  «         |S t          j        d¦  «         |D �]}}t          j        ||	|¬¦  «        }|                     |¦  «         t          ||||¬¦  «        }|j
        dv r
|d         g}nt          j        ||¬¦  «        }|j        }t                               d|› �¦  «         t                               d	|                     ¦   «         › �¦  «         |t          j        k    r|                     ¦   «          t          j        | rd
nd¦  «        }|                     |¦  «         |t          j        k    rt+          j        |¦  «        }|D �]F}|dk    rŒ
|D �]8}|j
        dv rzt                               d|› d|d|j        |j        g› �¦  «         t          j        |d|j        |j        f|t          j        k    rt           j        nt           j        |¬¦  «        ŠnX|�||k    rŒ�t                               d|› d||g› �¦  «         t          j        d|j        dz
  ||ft           j        |¬¦  «        Š	 |	r t           j                              |‰¦  «        n|
rt          j!        |¦  «        n|Š ‰‰¦  «         tE          j#        ˆˆfd„|d¬¦  «        }|	rdn|
rdndt           j$        d| rdndd|d|d|||| %                    ¦   «         tM          tO          j(        ¦   «         ¦  «        dœ}| )                    tU          ||¦  «        ¦  «         t                               |¦  «         | +                    |¦  «         �Œê# tX          $ rC}t           -                    |¦  «         t           j         .                    ¦   «          Y d }~�Œ2d }~ww xY w�ŒH�Œ|S )NzYPlease install PyTorch with Cuda, and use a machine with GPU for testing gpu performance.F)Útorchscriptr9   )rŠ   r9   Úcustom_model_classrH   r   r8   zModel zNumber of parameters zcuda:0r7   zRun PyTorch on rK   r%   )ÚsizeÚdtyper>   r'   )ÚlowÚhighr™   rš   r>   c                  ó   •—  ‰ ‰¦  «        S ©Nr/   ©Ú	inferenceÚ	input_idss   €€r3   ú<lambda>zrun_pytorch.<locals>.<lambda>Œ  ó   ø€ °Y°Y¸yÑ5IÔ5I€ r5   ©ÚrepeatÚnumberr—   Útorch2rU   ÚNAr6   r   r;   )/rU   r6   Úis_availablerN   rO   Úset_grad_enabledr   rX   Úmodifyr   rS   r   Úmodel_max_lengthÚdebugÚnum_parametersr   ÚFLOAT16Úhalfr>   ÚtoÚINT8r   Úquantize_torch_modelrc   rd   ÚrandnÚfloat16Úfloat32Úrandintr…   ÚlongÚjitÚtraceÚcompileÚtimeitr¥   r_   r`   ra   r   rb   Úupdater   rg   ÚRuntimeErrorÚ	exceptionÚempty_cache)rj   rl   rm   rn   r@   r+   ro   rp   rq   r—   r§   r9   r,   r}   rB   rŠ   ÚmodelÚ	tokenizerÚmax_input_sizer>   rE   rF   Úruntimesr�   Úer    r¡   s                            @@r3   Úrun_pytorchrÆ   8  sU  øø€ ð €GØð •u”z×.Ò.Ñ0Ô0ð Ý�ŠÐpÑqÔqÐqØˆå	Ô˜5Ñ!Ô!Ð!à!ð U-ñ U-ˆ
ÝÔ+¨JÀKÐ[dÐeÑeÔeˆØ×Ò˜vÑ&Ô&Ð&Ý%ØØØØ*ð	
ñ 
ô 
ˆð Ô Ð/Ð/à 0°Ô 3Ð4ÐÐå%Ô5°jÈIÐVÑVÔVˆIà&Ô7ˆNå�ŠÐ%˜eÐ%Ð%Ñ&Ô&Ð&Ý�ŠÐE¨U×-AÒ-AÑ-CÔ-CÐEÐEÑFÔFÐFà�	Ô)Ò)Ð)Ø�JŠJ‰LŒLˆLå”¨'Ð<˜h˜h°uÑ=Ô=ˆØ�Š�ÑÔÐà�	œÒ&Ð&Ý"Ô7¸Ñ>Ô>ˆEà%ð 7	-ñ 7	-ˆJØ˜QŠˆØà#3ð 3-ñ 3-�ØÔ$¨Ð7Ð7Ý—K’KØ¨*ÐÐÈÐUVÐX^ÔXiÐkqÔk|ÐH}ÐÐñô ð õ !&¤Ø(¨!¨VÔ->ÀÔ@QÐRØ/8½IÔ<MÒ/MÐ/M�eœm˜mÕSXÔS`Ø%ð!ñ !ô !�I�Ið &Ð1°oÈÒ6VÐ6VØ å—K’KÐ o°*Ð oÐ oÐQ[Ð]lÐPmÐ oÐ oÑpÔpÐpÝ %¤ØØ#Ô.°Ñ2Ø(¨/Ð:Ý#œjØ%ð!ñ !ô !�Ið-à=HÐw�œ	Ÿš¨¨yÑ9Ô9Ð9ÐflÐNwÍeÌmÐ\aÑNbÔNbÐNbÐrwð ð �I˜iÑ(Ô(Ð(å%œ}Ð-IÐ-IÐ-IÐ-IÐ-IÐR^ÐghÐiÑiÔi�Hð 4?Ð"c - -ÐPVÐDcÀHÀHÐ\cÝ#(Ô#4Ø%)Ø,3Ð"> & &¸Ø%'Ø%.Ø&(Ø&0Ø"#Ø#.Ø&0Ø+:Ø,;×,IÒ,IÑ,KÔ,KÝ$'­¬©¬Ñ$7Ô$7ðð �Fð  —M’MÕ"4°X¸zÑ"JÔ"JÑKÔKÐKÝ—K’K Ñ'Ô'Ð'Ø—N’N 6Ñ*Ô*Ð*Ñ*øÝ#ð -ð -ð -Ý×$Ò$ QÑ'Ô'Ð'Ý”J×*Ò*Ñ,Ô,Ð,Ð,Ð,Ð,Ð,Ñ,øøøøð-øøøñc3-ñ	7	-ðr €Ns   É7D	NÎ
O	Î8O
	Ï
O	Údo_eager_modeÚuse_xlac                 ó2   ‡ ‡‡‡— ddl mŠ dd lŠˆ ˆˆˆfd„}|S )Nr   )Úwrapsc                 óÂ   •‡ —  ‰‰ ¦  «        ˆ fd„¦   «         } ‰‰ ¦  «        ‰                      ‰¬¦  «        ˆ fd„¦   «         ¦   «         }‰du r‰du s
J d¦   «         ‚|S |S )Nc                  ó   •—  ‰| i |¤ŽS rž   r/   ©r|   ÚkwargsÚfuncs     €r3   Úrun_in_eager_modezFrun_with_tf_optimizations.<locals>.run_func.<locals>.run_in_eager_mode®  s   ø€ à�4˜Ð( Ð(Ð(Ð(r5   )Újit_compilec                  ó   •—  ‰| i |¤ŽS rž   r/   rÍ   s     €r3   Úrun_in_graph_modezFrun_with_tf_optimizations.<locals>.run_func.<locals>.run_in_graph_mode²  s   ø€ ð �4˜Ð( Ð(Ð(Ð(r5   TFzcCannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`.)Úfunction)rÏ   rÐ   rÓ   rÇ   r)   rÈ   rÊ   s   `  €€€€r3   Úrun_funcz+run_with_tf_optimizations.<locals>.run_func­  s¬   øø€ Ø	ˆˆt‰Œð	)ð 	)ð 	)ð 	)ñ 
Œð	)ð 
ˆˆt‰ŒØ	�Š ˆÑ	)Ô	)ð	)ð 	)ð 	)ð 	)ñ 
*Ô	)ñ 
Œð	)ð ˜DÐ Ð Ø˜eÐ#Ð#Ð#Øuñ $Ô#Ð#ð %Ð$à$Ð$r5   )Ú	functoolsrÊ   Ú
tensorflow)rÇ   rÈ   rÕ   r)   rÊ   s   `` @@r3   Úrun_with_tf_optimizationsrØ   ¨  sS   øøøø€ ØÐÐÐÐÐàÐÐÐð%ð %ð %ð %ð %ð %ð %ð %ð$ €Or5   c                 óæ  ‡‡‡‡‡‡ — g }dd l Š ‰ j        j                             |¦  «         | s‰ j                             g d¦  «         | r5‰ j                             ¦   «         st                               d¦  «         |S | r¯‰ j         	                    d¦  «        }	 ‰ j                             |d         d¦  «         ‰ j        j
                             |d         d¦  «         ‰ j                             d¬¦  «         n1# t          $ r$}t                               |¦  «         Y d }~nd }~ww xY w|t           j        k    s|t           j        k    rt'          d¦  «        ‚|D �]�}t)          j        ||	¬¦  «        Š|                     ‰¦  «         t/          |‰|	|d¬	¦  «        Št1          j        ||	¬¦  «        }|j        }t5          d
d
¬¦  «        ˆfd„¦   «         }t5          d
d
¬¦  «        ˆfd„¦   «         }t5          d
d
¬¦  «        ˆˆˆ fd„¦   «         }‰j        r|Šnt9          ‰t:          ¦  «        r|Šn|Š|D �]°}|dk    rŒ
|D �]¢}|�||k    rŒt                               d|› d||g› �¦  «         t?          j         ¦   «         Šˆˆfd„tC          ||z  ¦  «        D ¦   «         }‰  "                    |||f‰ j#        ¬¦  «        Š	  ‰‰¦  «         tI          j%        ˆˆfd„|d¬¦  «        }d‰ j&        d| rdndd|d|d|||| '                    ¦   «         tQ          tS          j*        ¦   «         ¦  «        dœ}| +                    tY          ||¦  «        ¦  «         t                               |¦  «         | -                    |¦  «         �ŒD# t          $ rS}t                               |¦  «         ddl.m/} | 0                    ¦   «         }| 1                    ¦   «          Y d }~�Œœd }~ww xY w�Œ²�Œ�|S )Nr   ÚGPUzVPlease install Tensorflow-gpu, and use a machine with GPU for testing gpu performance.Tz/gpu:0)r>   z+Mixed precision is currently not supported.r8   )rŠ   r9   r˜   Úis_tf_modelF)rÇ   rÈ   c                 ó   •—  ‰| d¬¦  «        S )NF)Útrainingr/   ©r¡   rÁ   s    €r3   Úencoder_forwardz'run_tensorflow.<locals>.encoder_forwardü  s   ø€ à�5˜¨UÐ3Ñ3Ô3Ð3r5   c                 ó    •—  ‰| | d¬¦  «        S )NF)Údecoder_input_idsrÝ   r/   rÞ   s    €r3   Úencoder_decoder_forwardz/run_tensorflow.<locals>.encoder_decoder_forward   s   ø€ à�5˜°iÈ%ÐPÑPÔPÐPr5   c                 óª   •— ‰j                              dd‰j        g¦  «        }‰j                              dd‰j        g¦  «        } ‰| ||d¬¦  «        S )Nr'   F)Úvisual_featsÚ
visual_posrÝ   )ÚrandomÚnormalÚvisual_feat_dimÚvisual_pos_dim)r¡   ÚfeatsÚposrŠ   rÁ   r)   s      €€€r3   Úlxmert_forwardz&run_tensorflow.<locals>.lxmert_forward  sf   ø€ à”I×$Ò$ a¨¨FÔ,BÐ%CÑDÔDˆEØ”)×"Ò" A q¨&Ô*?Ð#@ÑAÔAˆCØ�5ØØ"ØØð	ñ ô ð r5   zRun Tensorflow on rK   c                 óL   •— g | ] }‰                      d ‰j        dz
  ¦  «        ‘Œ!S )r   r'   )r·   r…   )r1   r“   rŠ   Úrngs     €€r3   r4   z"run_tensorflow.<locals>.<listcomp>!  s/   ø€ ÐmÐmÐmÀA˜#Ÿ+š+ a¨Ô):¸QÑ)>Ñ?Ô?ÐmÐmÐmr5   )Úshaperš   c                  ó   •—  ‰ ‰¦  «        S rž   r/   rŸ   s   €€r3   r¢   z run_tensorflow.<locals>.<lambda>'  r£   r5   r'   r¤   r×   r¨   r6   r7   r   r;   )r6   )2r×   rŠ   Ú	threadingÚ set_intra_op_parallelism_threadsÚset_visible_devicesÚtestÚis_built_with_cudarN   rO   Úlist_physical_devicesÚexperimentalÚset_memory_growthÚ
distributeÚOneDeviceStrategyr¾   r¿   r   r¯   r²   ÚNotImplementedErrorr   rX   r«   r   r   r¬   rØ   Úis_encoder_decoderÚ
isinstancer   rc   ræ   ÚRandomrf   Úconstantr^   r¼   r¥   r_   r`   ra   r   rb   r½   r   rg   Únumbar6   Úget_current_deviceÚreset)!rj   rl   rm   rn   r@   r+   ro   rp   rq   r9   r,   r}   Úphysical_devicesrÅ   rB   rÂ   rÃ   rß   râ   rì   rE   rF   ÚvaluesrÄ   r�   r6   r>   rŠ   r    r¡   rÁ   rî   r)   s!                              @@@@@@r3   Úrun_tensorflowr  Â  sæ  øøøøøø€ ð €GàÐÐÐà„IÔ×8Ò8¸ÑEÔEÐEàð 1Ø
Œ	×%Ò% b¨%Ñ0Ô0Ð0àð �r”w×1Ò1Ñ3Ô3ð Ý�ŠÐmÑnÔnÐnØˆàð  Øœ9×:Ò:¸5ÑAÔAÐð	 ØŒI×)Ò)Ð*:¸1Ô*=¸uÑEÔEÐEØŒIÔ"×4Ò4Ð5EÀaÔ5HÈ$ÑOÔOÐOØŒM×+Ò+°8Ð+Ñ<Ô<Ð<Ð<øÝð 	 ð 	 ð 	 Ý×Ò˜QÑÔÐÐÐÐÐÐøøøøð	 øøøð •IÔ%Ò%Ð%¨µi´nÒ)DÐ)DÝ!Ð"OÑPÔPÐPà!ð Y#ñ Y#ˆ
ÝÔ+¨JÀ)ÐLÑLÔLˆØ×Ò˜vÑ&Ô&Ð&å%ØØØØ*Øð
ñ 
ô 
ˆõ "Ô1°*È	ÐRÑRÔRˆ	à"Ô3ˆõ 
#°ÀÐ	FÑ	FÔ	Fð	4ð 	4ð 	4ð 	4ñ 
GÔ	Fð	4õ 
#°ÀÐ	FÑ	FÔ	Fð	Qð 	Qð 	Qð 	Qñ 
GÔ	Fð	Qõ 
#°ÀÐ	FÑ	FÔ	Fð	ð 	ð 	ð 	ð 	ð 	ñ 
GÔ	Fð	ð Ô$ð 	(Ø/ˆIˆIÝ˜¥Ñ-Ô-ð 	(Ø&ˆIˆIà'ˆIà%ð +	#ñ +	#ˆJØ˜QŠˆØà#3ð '#ñ '#�Ø!Ð-°/ÀNÒ2RÐ2RØå—’Ðn°ÐnÐnÐPZÐ\kÐOlÐnÐnÑoÔoÐoå”m‘o”o�ØmÐmÐmÐmÐmÍÈzÐ\kÑOkÑIlÔIlÐmÑmÔm�ØŸKšK¨°zÀ?Ð6SÐ[]Ô[c˜KÑdÔd�	ð#Ø�I˜iÑ(Ô(Ð(å%œ}Ð-IÐ-IÐ-IÐ-IÐ-IÐR^ÐghÐiÑiÔi�Hð #/Ø#%¤>Ø%)Ø,3Ð"> & &¸Ø%'Ø%.Ø&(Ø&0Ø"#Ø#.Ø&0Ø+:Ø,;×,IÒ,IÑ,KÔ,KÝ$'­¬©¬Ñ$7Ô$7ðð �Fð  —M’MÕ"4°X¸zÑ"JÔ"JÑKÔKÐKÝ—K’K Ñ'Ô'Ð'Ø—N’N 6Ñ*Ô*Ð*Ñ*øÝ#ð #ð #ð #Ý×$Ò$ QÑ'Ô'Ð'Ø*Ð*Ð*Ð*Ð*Ð*à!×4Ò4Ñ6Ô6�FØ—L’L‘N”N�N�N�N�N�N‘Nøøøøð#øøøñE'#ñ	+	#ðZ €Ns3   ÂA"D  Ä 
D.Ä
D)Ä)D.Ë	CNÎ
O*	ÎAO%	Ï%O*	c                  óN  — t          j        ¦   «         } |                      ddddt          g d¢t	          t          j        ¦   «         ¦  «        dd                     t          j        ¦   «         ¦  «        z   ¬¦  «         |                      d	dd
t          dddgd¬¦  «         |                      ddt          d t	          t          ¦  «        dd                     t          ¦  «        z   ¬¦  «         |                      ddddt          dgg d¢d¬¦  «         |                      dddt          t          j
                             dd¦  «        d¬¦  «         |                      ddt          t          j
                             dd¦  «        d¬¦  «         |                      dd dd!d"¬#¦  «         |                      d$dt          d d%¬¦  «         |                      d&d't          t          j        t	          t          ¦  «        d(¬)¦  «         |                      d*dd!d+¬#¦  «         |                      d,dd!d-¬#¦  «         |                      d.d/t          t          j        t	          t          ¦  «        d0¬)¦  «         |                      d1d2dd!d3¬#¦  «         |                      d4d5dd d6¬7¦  «         |                      d8d9dd d:¬7¦  «         |                      d;d<dd d=¬7¦  «         |                      d>d?ddd
gt          g d@¢dA¬B¦  «         |                      dCdDddEt          dF¬G¦  «         |                      dHdIdt          d
g¬J¦  «         |                      dKdLdt          g dM¢¬J¦  «         |                      dNdd!dO¬#¦  «         |                      d¬P¦  «         |                      dQdRddt          dSgdT¬U¦  «         |                      dVdt          d dW¬¦  «         |                      dXdd!dY¬#¦  «         |                      d¬Z¦  «         t#          j        | ¦  «         |                      ¦   «         }|S )[Nz-mz--modelsFú+)zbert-base-casedzroberta-baseÚgpt2z Pre-trained models in the list: z, )ÚrequiredÚnargsÚtypeÚdefaultÚchoicesÚhelpz--model_sourcer'   r&   r)   zExport onnx from pt or tfz--model_classz!Model type selected in the list: )r	  r  r  r  r  z-ez	--enginesr:   )r:   rU   r§   r—   r×   zEngines to benchmarkz-cz--cache_dirú.Úcache_modelsz%Directory to cache pre-trained models)r	  r  r  r  z
--onnx_dirÚonnx_modelszDirectory to store onnx modelsz-gz	--use_gpuÚ
store_truezRun on gpu device)r	  Úactionr  z
--providerzExecution provider to usez-pz--precisionzfPrecision of model to run. fp32 for full precision, fp16 for half precision, and int8 for quantization)r  r  r  r  z	--verbosezPrint more informationz--overwritezOverwrite existing modelsz-oz--optimizer_infozjOptimizer info: Use optimizer.py to optimize onnx model as default. Can also choose from by_ort and no_optz-vz--validate_onnxzValidate ONNX modelz-fz--fusion_csvz:CSV file for saving summary results of graph optimization.)r	  r  r  z-dz--detail_csvz#CSV file for saving detail results.z-rz--result_csvz$CSV file for saving summary results.z-iz--input_counts)r'   r(   r%   zXNumber of ONNX model inputs. Please use 1 for fair comparison with Torch or TorchScript.)r	  r
  r  r  r  r  z-tz--test_timeséd   z8Number of repeat times to get average inference latency.)r	  r  r  r  z-bz--batch_sizes)r
  r  r  z-sz--sequence_lengths)é   é   é   é    é@   é€   é   z--disable_ort_io_bindingz=Disable running ONNX Runtime with binded inputs and outputs. )rw   z-nz--num_threadsr   zThreads to use)r	  r
  r  r  r  z--force_num_layersz%Manually set the model's layer numberz*--enable_arm64_bfloat16_fastmath_mlas_gemmzHEnable bfloat16 mlas gemm kernels on aarch64. Supported only for CPU EP )r{   )ÚargparseÚArgumentParserÚadd_argumentra   Úlistr   ÚkeysÚjoinr   ÚosÚpathr   ÚFLOAT32r   ÚBYSCRIPTÚintÚset_defaultsr   Úadd_argumentsÚ
parse_args)Úparserr|   s     r3   Úparse_argumentsr+  F  sI  € ÝÔ$Ñ&Ô&€Fà
×ÒØØØØÝØ;Ð;Ð;Ý•V”[‘]”]Ñ#Ô#Ø/°$·)²)½F¼K¹M¼MÑ2JÔ2JÑJð ñ 	ô 	ð 	ð ×ÒØØØÝØØ�t�Ø(ð ñ ô ð ð ×ÒØØÝØÝ•]Ñ#Ô#Ø0°4·9²9½]Ñ3KÔ3KÑKð ñ ô ð ð ×ÒØØØØÝØ�ØOÐOÐOØ#ð ñ 	ô 	ð 	ð ×ÒØØØÝÝ”—’˜S .Ñ1Ô1Ø4ð ñ ô ð ð ×ÒØØÝÝ”—’˜S -Ñ0Ô0Ø-ð ñ ô ð ð ×Ò˜˜k°EÀ,ÐUhÐÑiÔiÐià
×ÒØØÝØØ(ð ñ ô ð ð ×ÒØØÝÝÔ!Ý•Y‘”Øuð ñ ô ð ð ×Ò˜¨e¸LÐOgÐÑhÔhÐhà
×ÒØØØØ(ð	 ñ ô ð ð ×ÒØØÝÝÔ&Ý•]Ñ#Ô#Øyð ñ ô ð ð ×ÒØØØØØ"ð ñ ô ð ð ×ÒØØØØØIð ñ ô ð ð ×ÒØØØØØ2ð ñ ô ð ð ×ÒØØØØØ3ð ñ ô ð ð ×ÒØØØØØ�ÝØ�	�	Øgð ñ 	ô 	ð 	ð ×ÒØØØØÝØGð ñ ô ð ð ×Ò˜˜o°S½sÈQÈCÐÑPÔPÐPà
×ÒØØØÝØ,Ð,Ð,ð ñ ô ð ð ×ÒØ"ØØØLð	 ñ ô ð ð ×Ò¨uÐÑ5Ô5Ð5à
×ÒØØØØÝØ�Øð ñ ô ð ð ×ÒØØÝØØ4ð ñ ô ð ð ×ÒØ4ØØØWð	 ñ ô ð ð ×ÒÀÐÑGÔGÐGåÔ Ñ'Ô'Ð'à×ÒÑÔ€DØ€Kr5   c                  ó¶
  — t          ¦   «         } t          | j        ¦  «         | j        t          j        k    r#| j        st                               d¦  «         d S | j        t          j	        k    r,| j        r%| j
        dvrt                               d¦  «         d S t          | j        ¦  «        dk    r(t          | j        d                  d         dv rdg| _        t          d	„ | j        D ¦   «         ¦  «        | _        t                               d
| › �¦  «         t$          j                             | j        ¦  «        sK	 t%          j        | j        ¦  «         n0# t.          $ r# t                               d| j        ¦  «         Y nw xY wd| j        v }d| j        v }d| j        v }d| j        v }d| j        v }|r]t3          j        t6          j        ¦  «        t3          j        d¦  «        k     r)t                               dt6          j        › �¦  «         d S t;          | j        ¦  «        }g }| j        D �]h}t7          j        |¦  «         t                                t6          j!         "                    ¦   «         ¦  «         |s|s|�r| j#        dgk    rt           $                    d¦  «         |rK|tK          | j        | j        | j&        || j        || j'        | j        | j(        dd| j        | j        ¦  «        z  }|rK|tK          | j        | j        | j&        || j        || j'        | j        | j(        dd| j        | j        ¦  «        z  }|rK|tK          | j        | j        | j&        || j        || j'        | j        | j(        dd| j        | j        ¦  «        z  }|rI|tS          | j        | j        | j&        || j        || j'        | j        | j(        | j        | j        ¦  «        z  }i }	|r¸	 | j*         }
|tW          | j        | j
        | j        | j&        || j        || j'        | j        | j(        | j#        | j,        | j-        | j        | j.        | j        | j/        | j0        |
|	| j1        | j2        | ¦  «        z  }�Œ=# tf          $ r t           4                    d¦  «         Y �Œdw xY w�Œjtk          j6        ¦   «          7                    d¦  «        }|	r| j8        pd|› d�}ts          |	|¦  «         t          |¦  «        dk    r(| j'        dgk    rt           $                    d¦  «         d S | j:        pd|› d�}tw          ||¦  «         | j<        pd|› d�}t{          ||| ¦  «         d S )Nzfp16 is for GPU only)Úmigraphxzint8 is for CPU onlyr'   r   r%   )rI   Úswimr   c                 ó,   — h | ]}|d k    rt           n|’ŒS )r   )Ú	cpu_count)r1   Úxs     r3   ú	<setcomp>zmain.<locals>.<setcomp>  s$   € ÐTÐTÐT¸a¨A°ªF¨F�y˜y¸ÐTÐTÐTr5   zArguments: z#Creation of the directory %s failedrU   r§   r—   r:   r×   z2.0.0z2PyTorch version must be >=2.0.0 and you are using zB--input_counts is not implemented for torch or torchscript engine.TFÚ	Exceptionz%Y%m%d-%H%M%SÚbenchmark_fusion_z.csvzNo any result available.Úbenchmark_detail_Úbenchmark_summary_)>r+  r   r,   r@   r   r¯   rj   rN   rO   r²   rk   rR   Úmodelsr   rp   Úsortedr+   rc   r"  r#  Úexistsr9   ÚmkdirÚOSErrorÚenginesr   rT   rU   r_   r   Úforce_num_layersÚset_num_threadsr­   Ú
__config__Úparallel_inforr   rQ   rÆ   rm   ro   Ú
test_timesr  Úuse_mask_indexr•   rs   rt   ru   rv   rw   rz   r{   r3  r¿   r   rb   ÚstrftimeÚ
fusion_csvr   Ú
detail_csvr   Ú
result_csvr   )r|   Úenable_torchÚenable_torch2Úenable_torchscriptÚenable_onnxruntimeÚenable_tensorflowrn   r}   r+   ry   rx   Ú
time_stampÚcsv_filenames                r3   ÚmainrN    s¼  € ÝÑÔ€Då�”ÑÔÐà„~�Ô*Ò*Ð*°4´<Ð*Ý�ŠÐ+Ñ,Ô,Ð,Øˆà„~�œÒ'Ð'¨D¬LÐ'¸T¼]ÐR^Ð=^Ð=^Ý�ŠÐ+Ñ,Ô,Ð,Øˆå
ˆ4Œ;ÑÔ˜1ÒÐ¥¨¬°A¬Ô!7¸Ô!:¸oÐ!MÐ!MØ!# ˆÔåÐTÐTÀ4ÔCSÐTÑTÔTÑUÔU€DÔå
‡K‚KÐ$˜dÐ$Ð$Ñ%Ô%Ð%åŒ7�>Š>˜$œ.Ñ)Ô)ð Pð	PÝŒH�T”^Ñ$Ô$Ð$Ð$øÝð 	Pð 	Pð 	PÝ�LŠLÐ>ÀÄÑOÔOÐOÐOÐOð	Pøøøð ˜dœlÐ*€LØ ¤Ð,€MØ&¨$¬,Ð6ÐØ&¨$¬,Ð6ÐØ$¨¬Ð4Ðàð �œ¥uÔ'8Ñ9Ô9½G¼MÈ'Ñ<RÔ<RÒRÐRÝ�ŠÐ]Í%ÔJ[Ð]Ð]Ñ^Ô^Ð^Øˆå$ TÔ%:Ñ;Ô;€Oà€GàÔ'ð g.ñ g.ˆÝÔ˜kÑ*Ô*Ð*Ý�Š•UÔ%×3Ò3Ñ5Ô5Ñ6Ô6Ð6Øð 5	˜=ð 5	Ð,>ñ 5	ØÔ  Q CÒ'Ð'Ý—’ÐcÑdÔdÐdà!ð Ø�;Ø”LØ”KØÔ$Ø#Ø”NØØÔ$ØÔ)Ø”OØØØ”NØ”Lñô ñ �ð  ð Ø�;Ø”LØ”KØÔ$Ø#Ø”NØØÔ$ØÔ)Ø”OØØØ”NØ”Lñô ñ �ð  ð Ø�;Ø”LØ”KØÔ$Ø#Ø”NØØÔ$ØÔ)Ø”OØØØ”NØ”Lñô ñ �ð  ð 	Ø•~Ø”Ø”ØÔ ØØ”ØØÔ ØÔ%Ø”Ø”Ø”ñô ñ ˆGð #%ÐØð 	.ð.Ø-1Ô-@Ð)@Ð&Ø�?Ø”LØ”MØ”KØÔ$Ø#Ø”NØØÔ$ØÔ)Ø”OØÔ%ØÔ'ØÔ&Ø”NØ”MØ”LØ”NØÔ/Ø*Ø+ØÔ%ØÔAØñ/ô ñ �‘øõ2 ð .ð .ð .Ý× Ò  Ñ-Ô-Ð-Ð-Ñ-ð.øøøñ9	.õ> ”‘”×(Ò(¨Ñ9Ô9€JØð HØ”ÐNÐ*N¸jÐ*NÐ*NÐ*NˆÝ Ð!8¸,ÑGÔGÐGå
ˆ7�|„|�qÒÐØÔ ˜sÒ"Ð"Ý�NŠNÐ5Ñ6Ô6Ð6Øˆà”?ÐJÐ&J¸*Ð&JÐ&JÐ&J€LÝ�7˜LÑ)Ô)Ð)à”?ÐKÐ&K¸:Ð&KÐ&KÐ&K€LÝ�7˜L¨$Ñ/Ô/Ð/Ð/Ð/s%   ÅE Å*FÆFÏ%B
Q1Ñ1$RÒRÚ__main__)5Ú__doc__r  Úloggingr"  ræ   r¼   r   rY   ÚpsutilÚbenchmark_helperr   r   r   r   r   r	   r
   r   r   r   r   r‚   r   Úhuggingface_modelsr   r   Úonnx_exporterr   r   r   r   Ú	packagingr   Úquantize_helperr   Ú	getLoggerrN   r0  Úenvironra   rU   Útransformersr   r   r   r•   rÆ   ÚboolrØ   r  r+  rN  Ú__name__r/   r5   r3   ú<module>r]     s‰  ðð ð ð6 €€€Ø €€€Ø 	€	€	€	Ø €€€Ø €€€Ø Ð Ð Ð Ð Ð à €€€Ø €€€ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð )Ð (Ð (Ð (Ð (Ð (Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4ðð ð ð ð ð ð ð ð ð ð ð ð Ð Ð Ð Ð Ð Ø *Ð *Ð *Ð *Ð *Ð *à	ˆÔ	˜2Ñ	Ô	€àˆFÔ UÐ+Ñ+Ô+€	ð ˜BœJÐ&Ð&Ø$' C¨	¡N¤N€B„JÐ Ñ!à €€€Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @ð]ð ]ð ]ð@mð mð mð`¨Tð ¸Dð ð ð ð ð4Að Að AðHEð Eð EðP_0ð _0ð _0ðD ˆzÒÐØ€D�F„F€F€F€Fð Ðr5   