§
    kŠtjÃN  ã                   ód  — d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
m
Z
 d dlmZ d dlZd dlZd dlZd dlmZmZ e	 G d„ d¦  «        ¦   «         Ze	 G d„ d	¦  «        ¦   «         Z	 	 	 dd„Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Z d„ Z!d„ Z"e#dk    rdZ$ e"¦   «          dS dS )é    N)Ú	dataclass)Údatetime)ÚPath)Úgenerate_test_dataÚget_bert_inputsc                   ó’   — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   dS )ÚTestSettingÚ
batch_sizeÚsequence_lengthÚ
test_casesÚ
test_timesÚuse_gpuÚuse_io_bindingÚproviderÚintra_op_num_threadsÚseedÚverboseÚlog_severityÚaverage_sequence_lengthÚrandom_sequence_lengthN)Ú__name__Ú
__module__Ú__qualname__ÚintÚ__annotations__ÚboolÚstr© ó    úe/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/onnxruntime/transformers/bert_perf_test.pyr	   r	   !   s™   € € € € € € à€O€O�OØÐÐÑØ€O€O�OØ€O€O�OØ€M€M�MØÐÐÑØ€M€M�MØÐÐÑØ
€I€I�IØ€M€M�MØÐÐÑØ Ð Ð Ñ Ø Ð Ð Ñ Ð Ð r   r	   c                   ól   — e Zd ZU eed<   eed<   eed<   eed<   eed<   edz  ed<   edz  ed<   eed	<   dS )
ÚModelSettingÚ
model_pathÚinput_ids_nameÚsegment_ids_nameÚinput_mask_nameÚ	opt_levelNÚinput_tuning_resultsÚoutput_tuning_resultsÚ	mask_type)r   r   r   r   r   r   r   r   r    r"   r"   2   so   € € € € € € à€O€O�OØÐÐÑØÐÐÑØÐÐÑØ€N€N�NØ ™*Ð$Ð$Ñ$Ø ™:Ð%Ð%Ñ%Ø€N€N�N€N€Nr   r"   é   c                 óà  — dd l }|                     |¦  «         |r%d|                     ¦   «         vrt          d¦  «         |r1|dk    rddg}n)|dk    rddg}n|d	k    rddg}n|d
k    rg d¢}nddg}ndg}|                     ¦   «         }	||	_        |j        j        |	_        |€|j	        j
        |	_        n|dk    r|j	        j        |	_        ng|dk    r|j	        j        |	_        nO|dk    r|j	        j        |	_        n7|dk    r|j	        j        |	_        n|dk    r|j	        j
        |	_        n||	_        |�||	_        |                     | |	|¬¦  «        }
|r­|dk    rd|
                     ¦   «         v sJ ‚n¦|dk    rd|
                     ¦   «         v sJ ‚n‡|d	k    rd|
                     ¦   «         v sJ ‚nh|d
k    r1d|
                     ¦   «         v sJ ‚d|
                     ¦   «         v sJ ‚n1d|
                     ¦   «         v sJ ‚nd|
                     ¦   «         v sJ ‚|�Nt'          |¦  «        5 }|
                     t+          j        |¦  «        ¦  «         d d d ¦  «         n# 1 swxY w Y   |
S )Nr   ÚCUDAExecutionProviderzWarning: Please install onnxruntime-gpu package instead of onnxruntime, and use a machine with GPU for testing gpu performance.ÚdmlÚDmlExecutionProviderÚCPUExecutionProviderÚmigraphxÚMIGraphXExecutionProviderÚcudaÚtensorrt)ÚTensorrtExecutionProviderr-   r0   é   r+   é   éc   )Ú	providersr5   )ÚonnxruntimeÚset_default_logger_severityÚget_available_providersÚprintÚSessionOptionsÚlog_severity_levelÚExecutionModeÚORT_SEQUENTIALÚexecution_modeÚGraphOptimizationLevelÚORT_ENABLE_ALLÚgraph_optimization_levelÚORT_DISABLE_ALLÚORT_ENABLE_BASICÚORT_ENABLE_EXTENDEDÚORT_ENABLE_LAYOUTr   ÚInferenceSessionÚget_providersÚopenÚset_tuning_resultsÚjsonÚload)r#   r   r   r   rE   r   Útuning_results_pathr:   Úexecution_providersÚsess_optionsÚsessionÚfs               r    Úcreate_sessionrU   >   s…  € ð ÐÐÐà×+Ò+¨LÑ9Ô9Ð9àð 
Ð+°;×3VÒ3VÑ3XÔ3XÐXÐXÝð Nñ	
ô 	
ð 	
ð ð 7Ø�uÒÐØ#9Ð;QÐ"RÐÐØ˜Ò#Ð#à+Ø&ð#ÐÐð ˜ÒÐØ#:Ð<RÐ"SÐÐØ˜Ò#Ð#ð#ð #ð #ÐÐð $;Ð<RÐ"SÐÐà5Ð6Ðà×-Ò-Ñ/Ô/€LØ&2€LÔ#Ø"-Ô";Ô"J€LÔàÐ'Ø0;Ô0RÔ0aˆÔ-Ð-Ø	! QÒ	&Ð	&Ø0;Ô0RÔ0bˆÔ-Ð-Ø	! QÒ	&Ð	&Ø0;Ô0RÔ0cˆÔ-Ð-Ø	! QÒ	&Ð	&Ø0;Ô0RÔ0fˆÔ-Ð-Ø	! QÒ	&Ð	&Ø0;Ô0RÔ0dˆÔ-Ð-Ø	! RÒ	'Ð	'Ø0;Ô0RÔ0aˆÔ-Ð-à0HˆÔ-àÐ'Ø,@ˆÔ)à×*Ò*¨:°|ÐObÐ*ÑcÔc€Gàð AØ�uÒÐØ)¨W×-BÒ-BÑ-DÔ-DÐDÐDÐDÐDÐDØ˜Ò#Ð#Ø.°'×2GÒ2GÑ2IÔ2IÐIÐIÐIÐIÐIØ˜ÒÐØ*¨g×.CÒ.CÑ.EÔ.EÐEÐEÐEÐEÐEØ˜Ò#Ð#Ø.°'×2GÒ2GÑ2IÔ2IÐIÐIÐIÐIØ*¨g×.CÒ.CÑ.EÔ.EÐEÐEÐEÐEÐEà*¨g×.CÒ.CÑ.EÔ.EÐEÐEÐEÐEÐEà%¨×)>Ò)>Ñ)@Ô)@Ð@Ð@Ð@Ð@àÐ&ÝÐ%Ñ&Ô&ð 	5¨!Ø×&Ò&¥t¤y°¡|¤|Ñ4Ô4Ð4ð	5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð €Ns   È/(I#É#I'É*I'c                 óÆ   — t           j        t          j        t           j        t          j        t           j        t          j        t           j        t          j        i}||          S )N)ÚtorchÚfloat32ÚnpÚfloat16Úint32Úint64Úlonglong)Ú
torch_typeÚtype_maps     r    Ú
numpy_typer`   “   s:   € åŒ•r”zÝŒ•r”zÝŒ•R”XÝŒ•R”[ð	€Hð �JÔÐr   c                 óŒ   ‡— ˆfd„|                       ¦   «         D ¦   «         }ˆfd„|                      ¦   «         D ¦   «         }||fS )Nc                 óf   •— i | ]-\  }}|t          j        |¦  «                             ‰¦  «        “Œ.S r   ©rW   Ú
from_numpyÚto©Ú.0ÚnameÚarrayÚdevices      €r    ú
<dictcomp>z/create_input_output_tensors.<locals>.<dictcomp>ž   s8   ø€ Ð`Ð`Ð`Á+À$È�T�5Ô+¨EÑ2Ô2×5Ò5°fÑ=Ô=Ð`Ð`Ð`r   c                 óf   •— i | ]-\  }}|t          j        |¦  «                             ‰¦  «        “Œ.S r   rc   rf   s      €r    rk   z/create_input_output_tensors.<locals>.<dictcomp>Ÿ   s8   ø€ ÐbÐbÐbÁ;À4È�d�EÔ,¨UÑ3Ô3×6Ò6°vÑ>Ô>ÐbÐbÐbr   )Úitems)ÚinputsÚoutputsrj   Úinput_tensorsÚoutput_tensorss     `  r    Úcreate_input_output_tensorsrr   �   sZ   ø€ Ø`Ð`Ð`Ð`ÐQW×Q]ÒQ]ÑQ_ÔQ_Ð`Ñ`Ô`€MØbÐbÐbÐbÐRY×R_ÒR_ÑRaÔRaÐbÑbÔb€NØ˜.Ð(Ð(r   c           
      óÊ  — |                       ¦   «         }|                     ¦   «         D ]R\  }}|                     ||j        j        dt          |j        ¦  «        |j        |                     ¦   «         ¦  «         ŒS|                     ¦   «         D ]R\  }}| 	                    ||j        j        dt          |j        ¦  «        |j        |                     ¦   «         ¦  «         ŒS|S ©Nr   )
Ú
io_bindingrm   Ú
bind_inputrj   Útyper`   ÚdtypeÚshapeÚdata_ptrÚbind_output)Úsessrp   rq   ru   rh   Útensors         r    Úcreate_io_bindingr~   £   sí   € Ø—’Ñ"Ô"€JØ%×+Ò+Ñ-Ô-ð 
ð 
‰ˆˆfØ×ÒØØŒMÔØÝ�v”|Ñ$Ô$ØŒLØ�OŠOÑÔñ	
ô 	
ð 	
ð 	
ð '×,Ò,Ñ.Ô.ð 
ð 
‰ˆˆfØ×ÒØØŒMÔØÝ�v”|Ñ$Ô$ØŒLØ�OŠOÑÔñ	
ô 	
ð 	
ð 	
ð Ðr   c                 ó&  — g }g }|j         rdnd}t          |¦  «        D ]ï\  }}|                      ||¦  «        }	|                     |	¦  «         i }
t	          t          |¦  «        ¦  «        D ]}|	|         |
||         <   Œt          ||
|¦  «        \  }}t          | ||¦  «        }|                      |¦  «         t          j
        ¦   «         }|                      |¦  «         t          j
        ¦   «         |z
  }|                     |¦  «         Œð||fS )Nr3   Úcpu)r   Ú	enumerateÚrunÚappendÚrangeÚlenrr   r~   Úrun_with_iobindingÚtimeitÚdefault_timer)rS   Ú
all_inputsÚoutput_namesÚtest_settingÚresultsÚlatency_listrj   Ú_test_case_idrn   Úresultro   Úirp   rq   ru   Ú
start_timeÚlatencys                    r    Ú%onnxruntime_inference_with_io_bindingr“   º   s,  € Ø€GØ€LØ#Ô+Ð6ˆVˆV°€FÝ!*¨:Ñ!6Ô!6ð %ð %Ñˆ�vØ—’˜\¨6Ñ2Ô2ˆØ�Š�vÑÔÐØˆÝ•s˜<Ñ(Ô(Ñ)Ô)ð 	1ð 	1ˆAØ'-¨a¤yˆG�L ”OÑ$Ð$å(CÀFÈGÐU[Ñ(\Ô(\Ñ%ˆ�~Ý& w°¸~ÑNÔNˆ
ð 	×"Ò" :Ñ.Ô.Ð.åÔ)Ñ+Ô+ˆ
Ø×"Ò" :Ñ.Ô.Ð.ÝÔ&Ñ(Ô(¨:Ñ5ˆØ×Ò˜GÑ$Ô$Ð$Ð$à�LÐ Ð r   c                 ó„  — t          |¦  «        dk    r(|                      |t          j        |¦  «        ¦  «         g }g }t	          |¦  «        D ]n\  }}t          j        ¦   «         }|                      ||¦  «        }t          j        ¦   «         |z
  }	|                     |¦  «         |                     |	¦  «         Œo||fS rt   )r…   r‚   ÚrandomÚchoicer�   r‡   rˆ   rƒ   )
rS   r‰   rŠ   rŒ   r�   rŽ   rn   r‘   r�   r’   s
             r    Úonnxruntime_inferencer—   Ó   s¿   € Ý
ˆ:�„˜ÒÐà�Š�L¥&¤-°
Ñ";Ô";Ñ<Ô<Ð<à€GØ€LÝ!*¨:Ñ!6Ô!6ð %ð %Ñˆ�vÝÔ)Ñ+Ô+ˆ
Ø—’˜\¨6Ñ2Ô2ˆÝÔ&Ñ(Ô(¨:Ñ5ˆØ�Š�vÑÔÐØ×Ò˜GÑ$Ô$Ð$Ð$Ø�LÐ Ð r   c                 ó‚  — |                      ¦   «         }dt          j                             | ¦  «        › d�}|d|j        › d|j        › d�                     dd¦  «        z  }|d|j        › d|j        › d�z  }|d	|j	        › d
|j
        › d�z  }|d|j        › d|j        › d�z  }|d|j        › d�z  }|d|j        › �z  }|S )Nzmodel=ú,zgraph_optimization_level=z,intra_op_num_threads=zGraphOptimizationLevel.ORT_Ú zbatch_size=z,sequence_length=ztest_cases=z,test_times=zuse_gpu=z,use_io_binding=zaverage_sequence_length=zrandom_sequence_length=)Úget_session_optionsÚosÚpathÚbasenamerE   r   Úreplacer
   r   r   r   r   r   r   r   )r#   rS   r‹   rR   Úoptions        r    Ú	to_stringr¡   ã   s-  € Ø×.Ò.Ñ0Ô0€LØ5•b”g×&Ò& zÑ2Ô2Ð5Ð5Ð5€FØ
ð  L¨,Ô*Oð  Lð  LÐgsô  hIð  Lð  Lð  L÷  Tò  TØ% rñô ñ €Fð Ðe˜LÔ3ÐeÐeÀlÔFbÐeÐeÐeÑe€FØ
Ð[˜LÔ3Ð[Ð[ÀÔAXÐ[Ð[Ð[Ñ[€FØ
Ð]˜Ô-Ð]Ð]¸|Ô?ZÐ]Ð]Ð]Ñ]€FØ
ÐP¨Ô)MÐPÐPÐPÑP€FØ
ÐM¨Ô(KÐMÐMÑM€FØ€Mr   c           	      ó   — t          | j        |j        |j        || j        |j        | j        ¬¦  «        }d„ |                     ¦   «         D ¦   «         }t          | j        ||¦  «        }||v rt          d|¦  «         d S t          d|¦  «         g }|j
        rBt          |j        ¦  «        D ],}	t          ||||¦  «        \  }
}|                     |¦  «         Œ-n@t          |j        ¦  «        D ]+}	t          |||¦  «        \  }
}|                     |¦  «         Œ,t!          j        |¦  «        dz  }t%          j        |¦  «        }t!          j        |d¦  «        }t!          j        |d¦  «        }t!          j        |d¦  «        }t!          j        |d	¦  «        }t!          j        |d
¦  «        }|j        d|z  z  }|||||||f||<   t          d                     t-          |d¦  «        t-          |d¦  «        ¦  «        ¦  «         | j        �r t0          j                             | j        ¦  «        }t0          j                             |¦  «        rZ|}|                     dd¦  «        d         › dt;          j        ¦   «                              ¦   «         › d�}t          d|d|d¦  «         |                      ¦   «         }tC          |d¦  «        5 }tE          j#        ||¦  «         d d d ¦  «         n# 1 swxY w Y   t          d|¦  «         d S d S )N)r   rP   c                 ó   — g | ]	}|j         ‘Œ
S r   )rh   )rg   Úoutputs     r    ú
<listcomp>z run_one_test.<locals>.<listcomp>ü   s   € ÐDÐDÐD F�F”KÐDÐDÐDr   zskip duplicated test:zRunning test:éè  é2   éK   éZ   é_   r8   g     @�@z,Average latency = {} ms, Throughput = {} QPSú.2fz.jsonr6   r   ú.zWARNING:zexists, will write tozinstead.ÚwzTuning results is saved to)$rU   r#   r   r   r'   r   r(   Úget_outputsr¡   r=   r   r„   r   r“   Úextendr—   rY   ri   Ú
statisticsÚmeanÚ
percentiler
   Úformatr)   rœ   r�   ÚabspathÚexistsÚrsplitr   ÚnowÚ	timestampÚget_tuning_resultsrL   rN   Údump)Úmodel_settingr‹   Úperf_resultsr‰   r   rS   rŠ   ÚkeyÚall_latency_listÚ_irŒ   r�   Ú
latency_msÚaverage_latencyÚ
latency_50Ú
latency_75Ú
latency_90Ú
latency_95Ú
latency_99Ú
throughputÚoutput_pathÚold_output_pathÚtrsrT   s                           r    Úrun_one_testrË   ò   s|  € ÝØÔ ØÔØÔØØÔØ!Ô.Ø)Ô>ðñ ô €Gð EÐD¨g×.AÒ.AÑ.CÔ.CÐDÑDÔD€Lå
�MÔ,¨g°|Ñ
DÔ
D€CØ
ˆlÐÐÝÐ% sÑ+Ô+Ð+Øˆå	ˆ/˜3ÑÔÐàÐØÔ"ð 	2Ý˜Ô/Ñ0Ô0ð 	2ð 	2ˆBÝ$IØ˜ \°<ñ%ô %Ñ!ˆG�\ð ×#Ò# LÑ1Ô1Ð1Ð1ð		2õ ˜Ô/Ñ0Ô0ð 	2ð 	2ˆBÝ$9¸'À:È|Ñ$\Ô$\Ñ!ˆG�\Ø×#Ò# LÑ1Ô1Ð1Ð1õ ”Ð*Ñ+Ô+¨dÑ2€Jå ”o jÑ1Ô1€OÝ”˜z¨2Ñ.Ô.€JÝ”˜z¨2Ñ.Ô.€JÝ”˜z¨2Ñ.Ô.€JÝ”˜z¨2Ñ.Ô.€JÝ”˜z¨2Ñ.Ô.€JØÔ(¨F°_Ñ,DÑE€Jð 	ØØØØØØð€L�Ñõ 
Ø6×=Ò=½fÀ_ÐV[Ñ>\Ô>\Õ^dÐeoÐqvÑ^wÔ^wÑxÔxñô ð ð Ô*ñ 
9Ý”g—o’o mÔ&IÑJÔJˆÝŒ7�>Š>˜+Ñ&Ô&ð 	aØ)ˆOØ*×1Ò1°'¸1Ñ=Ô=¸aÔ@ÐfÐfÅ8Ä<Á>Ä>×C[ÒC[ÑC]ÔC]ÐfÐfÐfˆKÝ�*˜oÐ/FÈÐU_Ñ`Ô`Ð`à×(Ò(Ñ*Ô*ˆÝ�+˜sÑ#Ô#ð 	 qÝŒI�c˜1ÑÔÐð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	åÐ*¨KÑ8Ô8Ð8Ð8Ð8ð
9ð 
9s   ËK1Ë1K5Ë8K5c                 ó–   — t          j        t          | ||||f¬¦  «        }|                     ¦   «          |                     ¦   «          d S )N)ÚtargetÚargs)ÚmultiprocessingÚProcessrË   ÚstartÚjoin)r»   r‹   r¼   r‰   r   Úprocesss         r    Úlaunch_testrÔ   7  sP   € ÝÔ%ÝàØØØØ ð
ð	ñ 	ô 	€Gð ‡M‚M�O„O€OØ‡L‚L�N„N€N€N€Nr   c                 óŽ  — |j         �t          | ||||j         ¦  «         d S t          j        d¬¦  «        }t          j        d¬¦  «        }t	          ||h¦  «        }t          dt          d|¦  «        ¦  «        D ]}||vr|                     |¦  «         Œ|                     d¬¦  «         |D ]}t          | ||||¦  «         Œd S )NF)ÚlogicalTr6   é   )Úreverse)	r   rÔ   ÚpsutilÚ	cpu_countÚlistr„   Úminrƒ   Úsort)	r»   r‹   r¼   r‰   rÚ   Úlogical_coresÚcandidate_threadsr�   r   s	            r    Úrun_perf_testsrà   F  s  € ØÔ(Ð4ÝØØØØØÔ-ñ	
ô 	
ð 	
ð 	ˆåÔ ¨Ð/Ñ/Ô/€IÝÔ$¨TÐ2Ñ2Ô2€Må˜m¨YÐ7Ñ8Ô8ÐÝ�1•c˜"˜mÑ,Ô,Ñ-Ô-ð (ð (ˆØÐ%Ð%Ð%Ø×$Ò$ QÑ'Ô'Ð'øØ×Ò 4ÐÑ(Ô(Ð(à 1ð að aÐÝ�M <°¸zÐK_Ñ`Ô`Ð`Ð`ðað ar   c                 óP  — t          | j        | j        | j        | j        ¦  «        \  }}}t          d|j        › d|j        › d|j        › �¦  «         t          |j        |j        |j        |j
        |j        ||||j        |j        | j        ¬¦  «        }t          | |||¦  «         d S )NzGenerating z samples for batch_size=z sequence_length=)r*   )r   r#   r$   r%   r&   r=   r   r
   r   r   r   r   r   r   r*   rà   )r»   r‹   r¼   Ú	input_idsÚsegment_idsÚ
input_maskr‰   s          r    Úrun_performancerå   ^  sí   € Ý)8ØÔ ØÔ$ØÔ&ØÔ%ñ	*ô *Ñ&€Iˆ{˜Jõ 
ð 	P�lÔ-ð  	Pð  	PÀ|ÔG^ð  	Pð  	PÐq}ô  rNð  	Pð  	Pñô ð õ $ØÔØÔ$ØÔØÔØÔØØØØÔ,ØÔ+ØÔ)ðñ ô €Jõ �= ,°¸jÑIÔIÐIÐIÐIr   c                  óø  — t          j        ¦   «         } |                      ddt          d¬¦  «         |                      dddt          dd¬	¦  «         |                      d
ddt          d¬¦  «         |                      ddt          dd¬¦  «         |                      dddt          dd¬¦  «         |                      ddt          g d¢dd¬¦  «         |                      ddt          dd¬¦  «         |                      dddd ¬!¦  «         |                      d¬"¦  «         |                      d#dt          d$g d%¢d&¬'¦  «         |                      d(ddd)¬!¦  «         |                      d¬*¦  «         |                      d+ddd,¬!¦  «         |                      d¬-¦  «         |                      d.dt          d d/¬¦  «         |                      d0d1dt          d d2¬¦  «         |                      d3dt          d d4¬¦  «         |                      d5dt          d d6¬¦  «         |                      d7dt          d d8¬¦  «         |                      d9d t          d:¬;¦  «         |                      d<d t          d=¬;¦  «         |                      d>d?d@t          dA¬;¦  «         |                      dBdCdddD¬!¦  «         |                      d¬E¦  «         |                      dFdt          d$dG¬¦  «         |                      ¦   «         }|S )HNz--modelTzbert onnx model path)Úrequiredrw   Úhelpz-bz--batch_sizeú+zKbatch size of input. Allow one or multiple values in the range of [1, 128].)rç   rw   Únargsrè   z-sz--sequence_lengthz maximum sequence length of inputz	--samplesFé
   z!number of samples to be generated)rç   rw   Údefaultrè   z-tz--test_timesr   zJnumber of times to run per sample. By default, the value is 1000 / samplesz--opt_level)r   r6   r+   r7   r8   r8   zfonnxruntime optimization level: 0 - disable all, 1 - basic, 2 - extended, 3 - layout, 99 - enable all.)rç   rw   Úchoicesrì   rè   z--seedr7   zPrandom seed. Use the same seed to make sure test data is same in multiple tests.z	--verboseÚ
store_truezprint verbose information)rç   Úactionrè   )r   z--log_severityr+   )r   r6   r+   r7   é   z.0:Verbose, 1:Info, 2:Warning, 3:Error, 4:Fatal)rç   rw   rì   rí   rè   z	--use_gpuzuse GPU)r   z--use_io_bindingzuse io_binding)r   z
--providerzExecution provider to usez-nz--intra_op_num_threadsz>=0, set intra_op_num_threadsz--input_ids_namezinput name for input idsz--segment_ids_namezinput name for segment idsz--input_mask_namezinput name for attention maskz--input_tuning_resultsz3tuning results (json) to be loaded before benchmark)rì   rw   rè   z--output_tuning_resultsz1tuning results (json) to be saved after benchmarkz-az--average_sequence_lengthéÿÿÿÿz)average sequence length excluding paddingz-rz--random_sequence_lengthz3use uniform random instead of fixed sequence length)r   z--mask_typezmmask type: (1: mask index or sequence length, 2: raw 2D mask, 3: key len, cumulated lengths of query and key))ÚargparseÚArgumentParserÚadd_argumentr   r   Úset_defaultsÚ
parse_args)ÚparserrÎ   s     r    Úparse_argumentsrø   {  s(  € ÝÔ$Ñ&Ô&€FØ
×Ò˜	¨DµsÐAWÐÑXÔXÐXà
×ÒØØØÝØØZð ñ ô ð ð ×ÒØØØÝØ/ð ñ ô ð ð ×ÒØØÝØØ0ð ñ ô ð ð ×ÒØØØÝØØYð ñ ô ð ð ×ÒØØÝØ Ð Ð ØØuð ñ ô ð ð ×ÒØØÝØØ_ð ñ ô ð ð ×ÒØØØØ(ð	 ñ ô ð ð ×Ò ÐÑ&Ô&Ð&à
×ÒØØÝØØ��Ø=ð ñ ô ð ð ×Ò˜¨e¸LÈyÐÑYÔYÐYØ
×Ò ÐÑ&Ô&Ð&à
×ÒÐ*°UÀ<ÐVfÐÑgÔgÐgØ
×Ò uÐÑ-Ô-Ð-à
×ÒØØÝØØ(ð ñ ô ð ð ×ÒØØ ØÝØØ,ð ñ ô ð ð ×ÒØØÝØØ'ð ñ ô ð ð ×ÒØØÝØØ)ð ñ ô ð ð ×ÒØØÝØØ,ð ñ ô ð ð ×ÒØ ØÝØBð	 ñ ô ð ð ×ÒØ!ØÝØ@ð	 ñ ô ð ð ×ÒØØ#ØÝØ8ð ñ ô ð ð ×ÒØØ"ØØØBð ñ ô ð ð ×Ò¨uÐÑ5Ô5Ð5à
×ÒØØÝØØ|ð ñ ô ð ð ×ÒÑÔ€DØ€Kr   c                  ó‚  — t          ¦   «         } | j        dk    r*t          dt          d| j        z  ¦  «        ¦  «        | _        | j        dk    r| j        | _        t          j        ¦   «         }| 	                    ¦   «         }t          | j        ¦  «        }t          |¦  «        dk    rt          |¦  «        dk    st          d¦  «        ‚t          | j        | j        | j        | j        | j        | j        | j        | j        ¦  «        }|D ]z}t/          || j        | j        | j        | j        | j        | j        | j        | j        | j        | j        | j        | j        ¦  «        }tA          d|¦  «         tC          |||¦  «         Œ{tE          | #                    ¦   «         dd„ ¬	¦  «        }tH          j%         &                    tO          | j        ¦  «        j(        d
 )                    | j        rdndd &                    d„ tE          |¦  «        D ¦   «         ¦  «        | j        tU          j+        ¦   «          ,                    d¦  «        ¦  «        ¦  «        }t[          |dd¬¦  «        5 }	t]          j/        |	dd¬¦  «        }
d }|D ]”\  }}| 0                    d¦  «        }|€8g d¢}| 1                    d„ |D ¦   «         ¦  «         |
 2                    |¦  «         d„ |D ¦   «         }| 1                    d„ |D ¦   «         ¦  «         |
 2                    |¦  «         Œ•	 d d d ¦  «         n# 1 swxY w Y   tA          d|¦  «         d S )Nr   r6   r¦   é€   z batch_size not in range [1, 128]ztest settingFc                 ó   — | d         S )Nr6   r   )Úxs    r    ú<lambda>zmain.<locals>.<lambda>M  s   € ÈqÐQRÌt€ r   )rØ   r½   zperf_results_{}_B{}_S{}_{}.txtÚGPUÚCPUú-c                 ó,   — g | ]}t          |¦  «        ‘ŒS r   )r   ©rg   rü   s     r    r¥   zmain.<locals>.<listcomp>S  s   € Ð=Ð=Ð= •c˜!‘f”fÐ=Ð=Ð=r   z%Y%m%d-%H%M%Szw+rš   )Únewlineú	ú
)Ú	delimiterÚlineterminatorr™   )zLatency(ms)ÚLatency_P50ÚLatency_P75ÚLatency_P90ÚLatency_P95ÚLatency_P99zThroughput(QPS)c                 óD   — g | ]}|                      d ¦  «        d         ‘ŒS )ú=r   ©Úsplitr  s     r    r¥   zmain.<locals>.<listcomp>g  s&   € Ð@Ð@Ð@°A §¢¨¡¤¨Q¤Ð@Ð@Ð@r   c                 ó.   — g | ]}t          |d ¦  «        ‘ŒS )r«   )r³   r  s     r    r¥   zmain.<locals>.<listcomp>j  s"   € Ð<Ð<Ð<¨1•f˜Q Ñ&Ô&Ð<Ð<Ð<r   c                 óD   — g | ]}|                      d ¦  «        d         ‘ŒS )r  r6   r  r  s     r    r¥   zmain.<locals>.<listcomp>k  s&   € Ð;Ð;Ð;¨q˜1Ÿ7š7 3™<œ<¨œ?Ð;Ð;Ð;r   zTest summary is saved to)3rø   r   Úmaxr   Úsamplesr   r   rÏ   ÚManagerÚdictÚsetr
   rÜ   Ú	Exceptionr"   Úmodelr$   r%   r&   r'   r(   r)   r*   r	   r   r   r   r   r   r   r   r   r=   rå   Úsortedrm   rœ   r�   rÒ   r   Úparentr³   r   r·   ÚstrftimerL   ÚcsvÚwriterr  r¯   Úwriterow)rÎ   Úmanagerr¼   Úbatch_size_setr»   r
   r‹   Úsorted_resultsÚsummary_fileÚtsv_fileÚ
tsv_writerÚheadersr½   Úperf_resultÚparamsÚvaluess                   r    Úmainr*    s…  € ÝÑÔ€Dà„˜!ÒÐÝ˜a¥ T¨D¬LÑ%8Ñ!9Ô!9Ñ:Ô:ˆŒàÔ# qÒ(Ð(Ø'+Ô';ˆÔ$åÔ%Ñ'Ô'€GØ—<’<‘>”>€Lå˜œÑ)Ô)€NÝ�ÑÔ 1Ò$Ð$­¨^Ñ)<Ô)<ÀÒ)CÐ)CÝÐ:Ñ;Ô;Ð;å ØŒ
ØÔØÔØÔØŒØÔ!ØÔ"ØŒñ	ô 	€Mð %ð Cð Cˆ
Ý"ØØÔ ØŒLØŒOØŒLØÔØŒMØÔ%ØŒIØŒLØÔØÔ(ØÔ'ñ
ô 
ˆõ  	ˆn˜lÑ+Ô+Ð+Ý˜ |°\ÑBÔBÐBÐBõ ˜L×.Ò.Ñ0Ô0¸%À^À^ÐTÑTÔT€Nå”7—<’<ÝˆTŒZÑÔÔØ(×/Ò/Ø”\Ð,ˆEˆE uØ�HŠHÐ=Ð=¥f¨^Ñ&<Ô&<Ð=Ñ=Ô=Ñ>Ô>ØÔ ÝŒL‰NŒN×#Ò# OÑ4Ô4ñ		
ô 	
ñô €Lõ 
ˆl˜D¨"Ð	-Ñ	-Ô	-ð (°Ý”Z °DÈÐNÑNÔNˆ
ØˆØ .ð 	(ð 	(ÑˆC�Ø—Y’Y˜s‘^”^ˆFØˆðð ð �ð —’Ð@Ð@¸Ð@Ñ@Ô@ÑAÔAÐAØ×#Ò# GÑ,Ô,Ð,à<Ð<°Ð<Ñ<Ô<ˆFØ�MŠMÐ;Ð;°FÐ;Ñ;Ô;Ñ<Ô<Ð<Ø×Ò Ñ'Ô'Ð'Ð'ð#	(ð(ð (ð (ñ (ô (ð (ð (ð (ð (ð (ð (øøøð (ð (ð (ð (õ, 
Ð
$ lÑ3Ô3Ð3Ð3Ð3s   É&B1L$Ì$L(Ì+L(Ú__main__)Nr+   N)%rò   r  rN   rÏ   rœ   r•   r°   r‡   Údataclassesr   r   Úpathlibr   ÚnumpyrY   rÙ   rW   Úbert_test_datar   r   r	   r"   rU   r`   rr   r~   r“   r—   r¡   rË   rÔ   rà   rå   rø   r*  r   Ú__spec__r   r   r    ú<module>r1     s<  ðð €€€Ø 
€
€
€
Ø €€€Ø Ð Ð Ð Ø 	€	€	€	Ø €€€Ø Ð Ð Ð Ø €€€Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à Ð Ð Ð Ø €€€Ø €€€Ø >Ð >Ð >Ð >Ð >Ð >Ð >Ð >ð ð!ð !ð !ð !ð !ñ !ô !ñ „ð!ð  ðð ð ð ð ñ ô ñ „ðð  "ØØðRð Rð Rð Rðj ð  ð  ð)ð )ð )ðð ð ð.!ð !ð !ð2!ð !ð !ð ð ð ðB9ð B9ð B9ðJð ð ðað að að0Jð Jð Jð:_ð _ð _ðDQ4ð Q4ð Q4ðh ˆzÒÐà€Hà€D�F„F€F€F€Fð	 Ðr   