§
    kŠtjfU  ã                   ój  — d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZ	d dl
Z
d dlZd dlZd dlmZmZ d dlmZ d dlmZ d dlmZmZmZ d dlmZ d dlmZmZmZ d dlZ ej         e!¦  «        Z"de j#        fd	„Z$de j#        fd
„Z%d„ Z&d„ Z'd„ Z(d„ Z)d„ Z*d„ Z+d„ Z,d„ Z-e!dk    r e-¦   «          dS dS )é    N)Úmeasure_memoryÚsetup_logger)Úget_library_path)ÚORTModelForSpeechSeq2Seq)ÚProfilerActivityÚprofileÚrecord_function)Útrange)ÚAutoModelForSpeechSeq2SeqÚWhisperConfigÚWhisperProcessorÚargsc                 ó¦  ‡ ‡‡	‡
— ‰ j         dvrt          d¦  «        ‚ˆ fd„Šˆ fd„Š	‰ j        ‰ j        ‰ j        ‰ j        ‰ j        ‰ j        dœ}‰ j         dk    rÞ|                     ¦   «         D ]9\  }}t          j
        |gd|v rt          j        nt          j        ¬¦  «        ||<   Œ:‰ j        r)t          j
        ‰ j        gt          j        ¬¦  «        |d	<   ‰ j        r)t          j
        ‰ j        gt          j        ¬¦  «        |d
<   ‰ j        r)t          j
        ‰ j        gt          j        ¬¦  «        |d<   t&                               d‰ j        › �¦  «         ˆˆ	fd„}t-          ‰ |‰ j        ¦  «          |‰ j        ¦  «        }‰ j        r||d<   |S t&                               d¦  «         ‰ j         dk    rdndŠ
ˆ ˆ
fd„}t-          ‰ ||¦  «          ||¦  «        }‰ j         dk    r||d<   |S |                     ‰ j        rt4          j        nt4          j        ‰ j        ¬¦  «        |d<   ‰ j        |d<   d|d<   d|d<   ‰ j        r
‰ j        |d<   |S )N>   Úortúhf-ortúhf-pt-eagerúhf-pt-compilez/Unable to auto-detect inputs for provided modelc                  ób   •— t          j        ‰j        ¦  «        } t          j        | ¦  «        } | S ©N)ÚwhisperÚ
load_audioÚ
audio_pathÚpad_or_trim)Úaudior   s    €úo/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/onnxruntime/transformers/models/whisper/benchmark.pyÚload_via_ffmpegz#get_inputs.<locals>.load_via_ffmpeg$   s*   ø€ ÝÔ" 4¤?Ñ3Ô3ˆÝÔ# EÑ*Ô*ˆØˆó    c                  ó
  •— t          ‰j        d¦  «        5 } t          j        t	          |                      ¦   «         ¦  «        t          j        ¬¦  «        }t          j        |g¦  «        }d d d ¦  «         n# 1 swxY w Y   |S )NÚrb©Údtype)Úopenr   ÚnpÚasarrayÚlistÚreadÚuint8Úarray)Úfr   r   s     €r   Úload_via_numpyz"get_inputs.<locals>.load_via_numpy)   s£   ø€ Ý�$”/ 4Ñ(Ô(ð 	&¨AÝ”J�t A§F¢F¡H¤H™~œ~µR´XÐ>Ñ>Ô>ˆEÝ”H˜e˜WÑ%Ô%ˆEð	&ð 	&ð 	&ñ 	&ô 	&ð 	&ð 	&ð 	&ð 	&ð 	&ð 	&øøøð 	&ð 	&ð 	&ð 	&ð ˆs   —AA8Á8A<Á?A<)Ú
max_lengthÚ
min_lengthÚ	num_beamsÚnum_return_sequencesÚlength_penaltyÚrepetition_penaltyr   Úpenaltyr    Údecoder_input_idsÚlogits_processorÚtemperaturezLoad audio: c                 ó0   •— | r
 ‰¦   «         n	 ‰¦   «         S r   © )Úonnx_e2er   r*   s    €€r   ú<lambda>zget_inputs.<locals>.<lambda>D   s!   ø€ ¸Ð%X ^ ^Ñ%5Ô%5Ð%5ÀÀÑGXÔGX€ r   Úaudio_streamzFeature extraction: r#   Úptc                 óT   •— ‰j                              | g‰‰j        ¬¦  «        j        S )N)Úreturn_tensorsÚsampling_rate)Ú	processorÚfeature_extractorr=   Úinput_features)r   r   Úreturn_types    €€r   r8   zget_inputs.<locals>.<lambda>P   s1   ø€  ¤×!AÒ!AØ	ˆ ¸4Ô;Mð "Bñ "ô "äð r   r@   )r!   ÚdeviceÚinputsÚno_repeat_ngram_sizeTÚearly_stoppingÚ	use_cacheÚforced_decoder_ids)Úbenchmark_typeÚ	Exceptionr+   r,   r-   r.   r/   r0   Úitemsr#   r(   Úfloat32Úint32Úhas_decoder_input_idsr2   Úhas_logits_processorr3   Úhas_temperaturer4   ÚloggerÚinfor   Útime_fnÚhas_audio_streamÚtoÚuse_fp16ÚtorchÚfloat16Útarget_devicerD   )r   rC   ÚkÚvÚload_audio_fnÚ
audio_dataÚprocessor_fnr@   r   r*   rA   s   `       @@@r   Ú
get_inputsr^       sÜ  øøøø€ ØÔÐ"SÐSÐSÝÐIÑJÔJÐJðð ð ð ð ð
ð ð ð ð ð ”oØ”oØ”^Ø $Ô 9ØÔ-Ø"Ô5ðð €Fð Ô˜eÒ#Ð#à—L’L‘N”Nð 	Xð 	X‰DˆAˆqÝœ ! ¸)Àq¸.¸.­B¬J¨JÍbÌhÐWÑWÔWˆF�1‰IˆIØÔ%ð 	]Ý*,¬(°DÔ4JÐ3KÕSUÔS[Ð*\Ñ*\Ô*\ˆFÐ&Ñ'ØÔ$ð 	[Ý)+¬°4Ô3HÐ2IÕQSÔQYÐ)ZÑ)ZÔ)ZˆFÐ%Ñ&ØÔð 	SÝ$&¤H¨dÔ.>Ð-?ÅrÄzÐ$RÑ$RÔ$RˆF�=Ñ!õ ‡K‚KÐ0˜tœÐ0Ð0Ñ1Ô1Ð1ØXÐXÐXÐXÐX€MÝˆD�- Ô!6Ñ7Ô7Ð7Ø�˜tÔ4Ñ5Ô5€JàÔð à!+ˆˆ~ÑØˆõ ‡K‚KÐ&Ñ'Ô'Ð'ØÔ-°Ò6Ð6�$�$¸D€Kðð ð ð ð €Lõ ˆD�, 
Ñ+Ô+Ð+Ø!�\ *Ñ-Ô-€NàÔ˜eÒ#Ð#à#1ˆÐÑ Øˆà%×(Ò(Ø#œ}Ð?�eŒmˆmµ%´-ÈÔHZð )ñ ô €Fˆ8Ñð &*Ô%>€FÐ!Ñ"Ø#€FÐÑØ€Fˆ;ÑàÔð >Ø'+Ô'=ˆÐ#Ñ$à€Mr   c                 óâ  — d\  }}d\  }}| j         dv r§| j        r| j        n| j        }t          j        ¦   «         }t	          j        || j        rt          j        nt          j	        d¬¦  «         
                    | j        ¦  «        }t          j        ¦   «         }| j         dk    rt          j        |¦  «        }nv| j         dv rVt          j        ¦   «         }| j        |_        |                     t%          ¦   «         ¦  «         | j        rd|_        d|_        nt-          d| j         › �¦  «        ‚| j         d	k    r�t/          | j        ¦  «        t2          u r| j        d
         n| j        }t/          | j        ¦  «        t2          u r| j        d         nd }t          j        ¦   «         }t5          j        | j        |||d¬¦  «        }t          j        ¦   «         }| j         dk    rjt8                               d| j        › �¦  «         t          j        ¦   «         }t          j        | j        || j        g¬¦  «        }t          j        ¦   «         }t8                               d||z
  › d�¦  «         |S )N)NN¾   r   r   T)Útorch_dtyperF   r   >   r   r   é   úCannot recognize r   r   )ÚproviderÚprovider_optionsÚsession_optionsÚuse_io_bindingr   zLoading model from )Ú	providerszLoaded model in ú s) rH   Úhf_pt_model_pathÚ
model_nameÚtimer   Úfrom_pretrainedrU   rV   rW   rK   rT   rX   Úcompiler   ÚSessionOptionsr   Úenable_profilingÚregister_custom_ops_libraryr   ÚverboseÚlog_verbosity_levelÚlog_severity_levelrI   ÚtypeÚexecution_providerÚtupler   Úhf_ort_dir_pathrP   rQ   Úort_model_pathÚInferenceSession)r   ÚmodelÚsess_optionsÚ
start_timeÚend_timeÚsourcerd   re   s           r   Ú	get_modelr€   h   sk  € Ø$Ñ€Eˆ<Ø%Ñ€J�ð ÔÐ>Ð>Ð>Ø*.Ô*?ÐT�Ô&Ð&ÀTÄ_ˆÝ”Y‘[”[ˆ
Ý)Ô9ØØ)-¬ÐI�œ˜½E¼MØð
ñ 
ô 
÷ Š"ˆTÔÑ
 Ô
 ð	 	õ
 ”9‘;”;ˆàÔ /Ò1Ð1Ý”M %Ñ(Ô(ˆEøà	Ô	Ð 1Ð	1Ð	1ÝÔ)Ñ+Ô+ˆØ(,¬ˆÔ%Ø×0Ò0Õ1AÑ1CÔ1CÑDÔDÐDØŒ<ð 	0Ø/0ˆLÔ,Ø./ˆLÔ+øõ ÐA¨DÔ,?ÐAÐAÑBÔBÐBàÔ˜hÒ&Ð&å15°dÔ6MÑ1NÔ1NÕRWÐ1WÐ1W�4Ô*¨1Ô-Ð-Ð]aÔ]tˆÝ9=¸dÔ>UÑ9VÔ9VÕZ_Ð9_Ð9_˜4Ô2°1Ô5Ð5ÐeiÐå”Y‘[”[ˆ
Ý(Ô8ØÔ ØØ-Ø(Øð
ñ 
ô 
ˆõ ”9‘;”;ˆàÔ˜eÒ#Ð#å�ŠÐ?¨$Ô*=Ð?Ð?Ñ@Ô@Ð@Ý”Y‘[”[ˆ
ÝÔ$ØÔØØÔ.Ð/ð
ñ 
ô 
ˆõ
 ”9‘;”;ˆå
‡K‚KÐ< 8¨jÑ#8Ð<Ð<Ð<Ñ=Ô=Ð=à€Lr   c                 ó.  — t          |¦  «        t          u r|d         n|}t          |¦  «        t          u r|d         n|}t          j        | j        ¦  «        }| j        dk    rt          | j        ¦  «        n t          | j        t          j
        d¬¦  «        }| j        r% ||¦  «        }t                               |¦  «         |D ]} ||¦  «         Œ| j        dk    rt          j                             |¦  «         t!          j        ¦   «         }	| j        dk    rt          | j        ¦  «        n t          | j        t          j
        d¬¦  «        }
|
D ]} ||¦  «         Œ| j        dk    rt          j                             |¦  «         t!          j        ¦   «         }| j        dk    rt                               d¦  «         d}||	z
  | j        z  }||z  }t                               d	|› d
�¦  «         t                               d|› d�¦  «         d S )Nr   rb   r   zWarm up)ÚfileÚdescÚcpuÚ	BenchmarkÚ z	Latency: ri   zThroughput: z qps)ru   rw   rV   rB   rX   rH   ÚrangeÚwarmup_runsr
   ÚsysÚstdoutrr   rP   rQ   ÚcudaÚsynchronizerl   Únum_runs)r   ÚfnrC   Úwarmup_inputsÚbenchmark_inputsÚtorch_deviceÚwarmup_rangeÚoutputsÚ_r}   Úbench_ranger~   Ú
batch_sizeÚlatencyÚ
throughputs                  r   rR   rR   ¨   s  € Ý!% f¡¤µÐ!6Ð!6�F˜1”I�I¸F€MÝ$(¨¡L¤LµEÐ$9Ð$9�v˜a”y�y¸vÐÝ”< Ô 2Ñ3Ô3€Lð
 Ô %Ò'Ð'õ 	ˆdÔÑÔÐå�DÔ$­3¬:¸IÐFÑFÔFð ð „|ð Ø�"�]Ñ#Ô#ˆÝ�Š�GÑÔÐàð ð ˆØ
ˆˆ=ÑÔÐÐð „{�eÒÐÝŒ
×Ò˜|Ñ,Ô,Ð,Ý”‘”€Jð Ô %Ò'Ð'õ 	ˆdŒmÑÔÐå�D”M­¬
¸ÐEÑEÔEð ð
 ð ð ˆØ
ˆÐÑÔÐÐà„{�eÒÐÝŒ
×Ò˜|Ñ,Ô,Ð,ÝŒy‰{Œ{€Hð Ô˜eÒ#Ð#Ý�Š�B‰Œˆà€JØ˜*Ñ$¨¬Ñ5€GØ˜gÑ%€Jå
‡K‚KÐ'˜GÐ'Ð'Ð'Ñ(Ô(Ð(Ý
‡K‚KÐ/˜zÐ/Ð/Ð/Ñ0Ô0Ð0Ø
€Fr   c                 ó,  — | j                              ¦   «         › d| j        › d| j        › d|j                             dd¦  «        › d|› dt          j                             ¦   «         d›�}d }| j         dv �rt          t          j
        t          j        gdd¬¦  «        5 }t          d¦  «        5   ||¦  «         d d d ¦  «         n# 1 swxY w Y   d d d ¦  «         n# 1 swxY w Y   |                     d¬	¦  «                             | j        | j        ¬
¦  «        }t"          j                             | j        |› d�¦  «        }t+          |d¦  «        5 }|                     |¦  «         d d d ¦  «         n# 1 swxY w Y   n ||¦  «         |› d�}|S )Nú-r”   z%Y-%m-%d_%H:%M:%Sr`   T)Ú
activitiesÚrecord_shapesÚprofile_memoryÚmodel_inferenceé   )Úgroup_by_stack_n)Úsort_byÚ	row_limitz.logÚwú.json)rH   ÚlowerÚ	precisionrB   Ú__name__ÚreplaceÚdatetimeÚnowr   r   ÚCPUÚCUDAr	   Úkey_averagesÚtableÚpt_filter_byÚpt_num_rowsÚosÚpathÚjoinÚ
log_folderr"   Úwrite)	r   rŽ   rC   Úinputs_typeÚprefixÚfilenameÚprofÚ	prof_datar)   s	            r   Ú
profile_fnr»   Ù   s›  € ð Ô#×)Ò)Ñ+Ô+ð  gð  g¨d¬nð  gð  g¸t¼{ð  gð  gÈRÌ[×M`ÒM`ÐadÐfiÑMjÔMjð  gð  gÐmxð  gð  gõ  |Dô  |M÷  |Qò  |Qñ  |Sô  |Sð  gð  gð  g€FØ€HàÔÐ>Ð>Ñ>åÝ(Ô,Õ.>Ô.CÐDÐTXÐimð
ñ 
ô 
ð 	àÝ Ð!2Ñ3Ô3ð ð Ø��6‘
”
�
ðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	ð
 ×%Ò%°qÐ%Ñ9Ô9×?Ò?ÈÔHYÐeiÔeuÐ?ÑvÔvˆ	å”7—<’< ¤°F°°°ÑAÔAˆÝ�(˜CÑ Ô ð 	 AØ�GŠG�IÑÔÐð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	øð
 	ˆˆ6‰
Œ
ˆ
ð Ð#Ð#Ð#ˆà€OsH   ÂCÂ/CÂ;CÃC	ÃCÃC	ÃCÃC"Ã%C"ÅE8Å8E<Å?E<c                 óä  ‡‡— t          j        ¦   «         }t          j        |¦  «        }|                     d¬¦  «          ‰‰¦  «         t
                               d|                     d ¬¦  «        › d�¦  «         t          j        ¦   «          t          j
                             ¦   «          t          | j        dk    ˆˆfd„| j        ¬¦  «         t          j                             ¦   «          d S )Ngš™™™™™¹?)ÚintervalzCPU usage: ú%r„   c                  ó   •—  ‰ ‰¦  «        S r   r6   )rŽ   rC   s   €€r   r8   zmeasure_fn.<locals>.<lambda>  s   ø€ ¸r¸rÀ&¹z¼z€ r   )Úis_gpuÚfuncÚmonitor_type)r±   ÚgetpidÚpsutilÚProcessÚcpu_percentrP   rQ   ÚgcÚcollectrV   r‹   Úempty_cacher   rB   rÂ   r‰   rŠ   Úflush)r   rŽ   rC   ÚpidÚprocesss    ``  r   Ú
measure_fnrÍ   ö   sÛ   øø€ å
Œ)‰+Œ+€CÝŒn˜SÑ!Ô!€GØ×Ò ÐÑ%Ô%Ð%à€B€v�J„J€JÝ
‡K‚KÐC˜g×1Ò1¸4Ð1Ñ@Ô@ÐCÐCÐCÑDÔDÐDõ „J�L„L€LÝ	„J×ÒÑÔÐÝ˜4œ;¨%Ò/Ð7IÐ7IÐ7IÐ7IÐ7IÐX\ÔXiÐjÑjÔjÐjõ „J×ÒÑÔÐÐÐr   c                 óª  ‡ ‡‡
— ˆfd„Š
ˆ ˆ
fd„}|}‰ j         dk    r ||¦  «         ‰ j        �rt          ‰ ||d¦  «        }‰ j         dk    �ræ|d t          d¦  «         …         }‰j        j                             ¦   «         }|dz   }t          j         	                    |¦  «        rXt                               d|› d	|› �¦  «         t          j        |t          j                             ‰ j        |¦  «        ¦  «         ‰j        j                             ¦   «         }|d
z   }t          j         	                    |¦  «        rXt                               d|› d	|› �¦  «         t          j        |t          j                             ‰ j        |¦  «        ¦  «         ‰j        j                             ¦   «         }|dz   }t          j         	                    |¦  «        rXt                               d|› d	|› �¦  «         t          j        |t          j                             ‰ j        |¦  «        ¦  «         d S t                               d¦  «         t%          ‰ ||¦  «          ||¦  «        \  }}	t                               dt          |d         ¦  «        › d�¦  «         t                               d|	d         › �¦  «         t'          ‰ ||¦  «         d S )Nc                 ó"   •—  ‰j         di | ¤Ž}|S )Nr6   )Úgenerate)rC   Úpredicted_idsr{   s     €r   Úget_pred_idsz&run_hf_inference.<locals>.get_pred_ids
  s    ø€ à&˜œÐ0Ð0¨Ð0Ð0ˆØÐr   c                 ó¾   •—  ‰| ¦  «        }g }t          ‰j        ¦  «        D ]7}|                     ‰j                             |d¬¦  «        d         ¦  «         Œ8||fS )NT©Úskip_special_tokensr   )r‡   r.   Úappendr>   Úbatch_decode)rC   rÑ   Útranscriptionr”   r   rÒ   s       €€r   Úgen_and_decz%run_hf_inference.<locals>.gen_and_dec  sq   ø€ à$˜ VÑ,Ô,ˆØˆÝ�tÔ0Ñ1Ô1ð 	jð 	jˆAØ× Ò  ¤×!<Ò!<¸]Ð`dÐ!<Ñ!eÔ!eÐfgÔ!hÑiÔiÐiÐiØ˜mÐ+Ð+r   r   zgen-and-decr   r¤   z-encoder.jsonú	Renaming ú to z-decoder.jsonz-decoder-with-past.jsonz
Evaluating PyTorch...úGenerated token length: r   ú tokensúTranscription: )rH   r   r»   ÚlenÚencoderÚsessionÚend_profilingr±   r²   ÚisfilerP   ÚwarningÚrenamer³   r´   ÚdecoderÚdecoder_with_pastrQ   rR   rÍ   )r   rC   r{   rÙ   Úgenerate_fnÚnew_lognameÚ
new_prefixÚold_lognamerÑ   rØ   rÒ   s   ` `       @r   Úrun_hf_inferencerì     sÁ  øøø€ ðð ð ð ð ð
,ð ,ð ,ð ,ð ,ð ,ð  €KàÔ˜oÒ-Ð-àˆ�FÑÔÐà„|ñ Ý   {°F¸MÑJÔJˆØÔ (Ò*Ñ*à$ _­¨G©¬ } _Ô5ˆJàœ-Ô/×=Ò=Ñ?Ô?ˆKØ$ Ñ6ˆKÝŒw�~Š~˜kÑ*Ô*ð SÝ—’ÐI¨;ÐIÐI¸KÐIÐIÑJÔJÐJÝ”	˜+¥r¤w§|¢|°D´OÀ[Ñ'QÔ'QÑRÔRÐRàœ-Ô/×=Ò=Ñ?Ô?ˆKØ$ Ñ6ˆKÝŒw�~Š~˜kÑ*Ô*ð SÝ—’ÐI¨;ÐIÐI¸KÐIÐIÑJÔJÐJÝ”	˜+¥r¤w§|¢|°D´OÀ[Ñ'QÔ'QÑRÔRÐRàÔ1Ô9×GÒGÑIÔIˆKØ$Ð'@Ñ@ˆKÝŒw�~Š~˜kÑ*Ô*ð SÝ—’ÐI¨;ÐIÐI¸KÐIÐIÑJÔJÐJÝ”	˜+¥r¤w§|¢|°D´OÀ[Ñ'QÔ'QÑRÔRÐRàˆõ ‡K‚KÐ)Ñ*Ô*Ð*ÝˆD�+˜vÑ&Ô&Ð&Ø#. ;¨vÑ#6Ô#6Ñ €M�=Ý
‡K‚KÐI­3¨}¸QÔ/?Ñ+@Ô+@ÐIÐIÐIÑJÔJÐJÝ
‡K‚KÐ4 -°Ô"2Ð4Ð4Ñ5Ô5Ð5Ýˆt�[ &Ñ)Ô)Ð)Ð)Ð)r   c                 ó®  ‡ ‡— dˆ ˆfd„	}ˆfd„}ˆfd„}ˆ fd„}‰ j         dk    r|n|} ||¦  «        }‰ j        r€t          ‰ ||d¦  «        }	‰                     ¦   «         }
t                               d|
› d	|	› �¦  «         t          j        |
t          j         	                    ‰ j
        |	¦  «        ¦  «         d S t                               d
¦  «         |}t          ‰ ||¦  «          ||¦  «        }‰ j         dk    r|                     ¦   «         }|d         }‰ j        r*t                               d|d         d         › �¦  «         nz ||d         d         ¦  «        }t                               dt          |¦  «        › d�¦  «         ‰ j                             |d         d¬¦  «        d         }t%          |› ¦  «         t'          ‰ ||¦  «         d S )NFc                 ó˜  •— d„ ‰                      ¦   «         D ¦   «         }t          |                      ¦   «         ¦  «        }||z
  }t          |¦  «        r,t                               d|› �¦  «         t          d¦  «        ‚||z
  }t          |¦  «        r&|D ]#}t                               d|› d�¦  «         | |= Œ$‰j        dk    r„‰ 	                    ¦   «         }|  
                    ¦   «         D ]\  }}	|                     ||	¦  «         Œ‰                     ¦   «         D ])}
|                     |
j        ‰j        ‰j        ¬¦  «         Œ*|S | S )Nc                 ó   — h | ]	}|j         ’Œ
S r6   ©Úname©Ú.0Úmodel_inputs     r   ú	<setcomp>z@run_ort_inference.<locals>.prepare_ort_inputs.<locals>.<setcomp>K  s   € ÐOÐOÐO¨[˜Ô(ÐOÐOÐOr   z(The following model inputs are missing: zEThere are missing inputs to the model. Please add them and try again.zRemoving unnecessary input 'z' from user provided inputsr„   )Údevice_typeÚ	device_id)r^   ÚsetÚkeysrß   rP   ÚerrorrI   rQ   rB   Ú
io_bindingrJ   Úbind_cpu_inputÚget_outputsÚbind_outputrñ   r÷   )rC   ÚwarmupÚmodel_inputsÚuser_inputsÚmissing_inputsÚunnecessary_inputsÚunnecessary_inputrû   rY   rZ   Úoutputr   r{   s              €€r   Úprepare_ort_inputsz-run_ort_inference.<locals>.prepare_ort_inputsI  sw  ø€ àOÐO¸E×<LÒ<LÑ<NÔ<NÐOÑOÔOˆÝ˜&Ÿ+š+™-œ-Ñ(Ô(ˆØ%¨Ñ3ˆÝˆ~ÑÔð 	eÝ�LŠLÐTÀNÐTÐTÑUÔUÐUÝÐcÑdÔdÐdð )¨<Ñ7ÐÝÐ!Ñ"Ô"ð 	.Ø%7ð .ð .Ð!Ý—’ÐiÐ;LÐiÐiÐiÑjÔjÐjØÐ,Ð-Ð-ð Œ;˜%ÒÐØ×)Ò)Ñ+Ô+ˆJØŸš™œð 0ð 0‘��1Ø×)Ò)¨!¨QÑ/Ô/Ð/Ð/Ø×+Ò+Ñ-Ô-ð gð g�Ø×&Ò& v¤{ÀÄÐW[ÔWeÐ&ÑfÔfÐfÐfØÐàˆr   c                 ó2   •— ‰                      | ¦  «         | S r   )Úrun_with_iobinding)rû   r{   s    €r   Úwith_io_bindingz*run_ort_inference.<locals>.with_io_bindingd  s   ø€ à× Ò  Ñ,Ô,Ð,ØÐr   c                 ó4   •— ‰                      d | ¦  «        }|S r   )Úrun)rC   r“   r{   s     €r   Úwithout_io_bindingz-run_ort_inference.<locals>.without_io_bindingi  s   ø€ à—)’)˜D &Ñ)Ô)ˆØˆr   c                 ó†   •— ‰j         | v r6t          j        | ‰j         k    ¦  «        d         d         }| d |dz   …         S | S )Nr   rb   )Úeos_token_idr#   Úwhere)r  Ú	first_endr   s     €r   Úhandle_outputz(run_ort_inference.<locals>.handle_outputn  sJ   ø€ ØÔ Ð&Ð&Ýœ ¨4Ô+<Ò!<Ñ=Ô=¸aÔ@ÀÔCˆIØ˜/˜I¨™M˜/Ô*Ð*àˆr   r„   Úe2erÚ   rÛ   z
Evaluating ONNX Runtime...r   rÞ   rÜ   rÝ   TrÔ   )F)rB   r   r»   râ   rP   rä   r±   rå   r²   r³   r´   rQ   rR   Úcopy_outputs_to_cpurS   rß   r>   r×   ÚprintrÍ   )r   rC   r{   r  r	  r  r  rè   Ú
ort_inputsré   rë   Úort_evaluate_inputsÚort_outputsÚactual_outputrØ   s   ` `            r   Úrun_ort_inferencer  H  s:  øø€ ðð ð ð ð ð ð ð6ð ð ð ð ð
ð ð ð ð ð
ð ð ð ð ð &*¤[°EÒ%9Ð%9�/�/Ð?Q€KØ#Ð# FÑ+Ô+€Jà„|ð Ý   {°JÀÑFÔFˆð ×)Ò)Ñ+Ô+ˆÝ�ŠÐA ;ÐAÐA°KÐAÐAÑBÔBÐBÝ
Œ	�+�rœwŸ|š|¨D¬O¸[ÑIÔIÑJÔJÐJàˆõ ‡K‚KÐ.Ñ/Ô/Ð/Ø$ÐåˆD�+Ð2Ñ3Ô3Ð3Ø�+˜jÑ)Ô)€KØ„{�eÒÐØ!×5Ò5Ñ7Ô7ˆØ˜a”.€KàÔð 	"å�ŠÐ9 k°!¤n°QÔ&7Ð9Ð9Ñ:Ô:Ð:Ð:ð &˜ k°!¤n°QÔ&7Ñ8Ô8ˆÝ�ŠÐJ­s°=Ñ/AÔ/AÐJÐJÐJÑKÔKÐKØœ×3Ò3°KÀ´NÐX\Ð3Ñ]Ô]Ð^_Ô`ˆå�Ð Ñ!Ô!Ð!åˆt�[ *Ñ-Ô-Ð-Ð-Ð-r   c                 ó¤   — | j         dv rt          | ||¦  «         d S | j         dk    rt          | ||¦  «         d S t          d| j         › �¦  «        ‚)N>   r   r   r   r   rc   )rH   rì   r  rI   )r   rC   r{   s      r   Úrun_inferencer  š  si   € ØÔÐHÐHÐHÝ˜˜v uÑ-Ô-Ð-Ð-Ð-Ø	Ô	 Ò	%Ð	%Ý˜$ ¨Ñ.Ô.Ð.Ð.Ð.åÐA¨DÔ,?ÐAÐAÑBÔBÐBr   c            	      ó"	  — t          j        ¦   «         } |                      ddt          dg d¢¬¦  «         |                      ddt          dd¬	¦  «         |                      d
dt          ddg d¢d¬¦  «         |                      dt          dd¬¦  «         |                      dt          dd¬¦  «         |                      dt          dd¬¦  «         |                      ddt          dd¬	¦  «         |                      ddt          t          j                             ¦   «         rdndddg¬¦  «         |                      d d!t          d"¬#¦  «         |                      d$d%t          d&¬#¦  «         |                      d'd(t          d)¬#¦  «         |                      d*t          d+¬#¦  «         |                      d,t          d-d.¬¦  «         |                      d/t          d0¬#¦  «         |                      d1t          d"¬#¦  «         |                      d2t          d3¬#¦  «         |                      d4t          d3¬#¦  «         |                      d5t          d6¬#¦  «         |                      d7t          d6¬#¦  «         |                      d8t          d9¬#¦  «         |                      d:t          d;d<¬¦  «         |                      d=t          d3d>¬¦  «         |                      d?t          d6d@¬¦  «         |                      dAdBdC¬D¦  «         |                      dEt          dFdG¬¦  «         |                      dHt          dIdJ¬¦  «         |                      dKdBdC¬D¦  «         |                      dLt          t          j
                             dM¦  «        dN¬¦  «         |                      ¦   «         }t          j                             |j        ¦  «         t	          j        |j        ¦  «         |j        |_        dO|j        v rA|j                             ¦   «         › dP�|_        |j        dQk    r|j        dR|j        if|_        |j        dSk    r|j        s
J dT¦   «         ‚|j        dOk    r|j        s
J dU¦   «         ‚t3          j        |j        ¦  «        |_        |S )VNz-btz--benchmark-typeT)r   r   r   r   )ru   ÚrequiredÚchoicesz-mz--model-namez;Hugging Face name of model (e.g. 'openai/whisper-large-v2'))ru   r  Úhelpz-pz--precisionÚfp32)Úint4Úint8Úfp16r   zePrecision for model. For ONNX models, the model's precision should be set before running this script.)ru   r  Údefaultr  r  z--hf-pt-model-pathr†   zNPath to directory containing all PyTorch files (e.g. tokenizer, PyTorch model))ru   r$  r  z--hf-ort-dir-pathzaPath to directory containing all ONNX files (e.g. tokenizer, encoder, decoder, decoder_with_past)z--ort-model-pathzPath to ONNX modelz-az--audio-pathz%Path to audio file for E2E evaluationz-dz--devicer‹   r„   )ru   r$  r  z-idz--device-idr   )ru   r$  z-wz--warmup-runsrŸ   z-nz
--num-runsé
   z--seedé   z--sampling-ratei€>  zSampling rate for audio (in Hz)z--max-lengthiÀ  z--min-lengthz--num-beamsrb   z--num-return-sequencesz--length-penaltyg      ð?z--repetition-penaltyz--no-repeat-ngram-sizeé   z--decoder-input-idsz[]zìThe forced decoder ids for generation. Format is [start token, timestamp token, language token, task token]. Default is [start token]. See `decoder_input_ids` in https://github.com/microsoft/Olive/tree/main/examples/whisper for details.z--logits-processorzLWhether to use timestamps logits processor or not (0 for false, 1 for true).z--temperaturez!Temperature value for generation.z	--profileFÚ
store_true)r$  Úactionz--pt-filter-byÚself_cpu_time_totalz"What to filter PyTorch profiler byz--pt-num-rowsiè  z.Number of rows for PyTorch profiler to displayz	--verbosez--log-folderú.zFolder to cache log filesr   ÚExecutionProviderÚCUDAExecutionProviderr÷   r   z,Please specify a path to `--hf-ort-dir-path`z+Please specify a path to `--ort-model-path`)ÚargparseÚArgumentParserÚadd_argumentÚstrrV   r‹   Úis_availableÚintÚfloatr±   r²   r³   Ú
parse_argsr#   ÚrandomÚseedÚmanual_seedrB   rÂ   rH   Úupperrv   r÷   rx   ry   ÚastÚliteral_evalr2   )Úparserr   s     r   r5  r5  £  s   € ÝÔ$Ñ&Ô&€Fà
×ÒØØÝØØAÐAÐAð ñ ô ð ð ×ÒØØÝØØJð ñ ô ð ð ×ÒØØÝØØØ0Ð0Ð0Øtð ñ ô ð ð ×ÒØÝØØ]ð	 ñ ô ð ð ×ÒØÝØØpð	 ñ ô ð ð ×ÒØÝØØ!ð	 ñ ô ð ð ×Ò˜˜nµ3ÀÐLsÐÑtÔtÐtØ
×ÒØØÝÝœ*×1Ò1Ñ3Ô3Ð>��¸Ø˜�ð ñ ô ð ð ×Ò˜˜}µ3ÀÐÑBÔBÐBØ
×Ò˜˜oµCÀÐÑCÔCÐCØ
×Ò˜˜lµ¸bÐÑAÔAÐAØ
×Ò˜¥s°AÐÑ6Ô6Ð6ð ×ÒÐ)µ¸UÐIjÐÑkÔkÐkð ×Ò˜­S¸#ÐÑ>Ô>Ð>Ø
×Ò˜­S¸!ÐÑ<Ô<Ð<Ø
×Ò˜­C¸ÐÑ;Ô;Ð;Ø
×ÒÐ0µsÀAÐÑFÔFÐFØ
×ÒÐ*µÀÐÑDÔDÐDØ
×ÒÐ.µUÀCÐÑHÔHÐHØ
×ÒÐ0µsÀAÐÑFÔFÐFð ×ÒØÝØð |ð	 ñ ô ð ð ×ÒØÝØØ[ð	 ñ ô ð ð ×ÒØÝØØ0ð	 ñ ô ð ð ×Ò˜¨U¸<ÐÑHÔHÐHØ
×ÒØ�sÐ,AÐHlð ñ ô ð ð ×Ò˜­c¸4ÐFvÐÑwÔwÐwØ
×Ò˜¨U¸<ÐÑHÔHÐHØ
×Ò˜­S½"¼'¿,º,ÀsÑ:KÔ:KÐRmÐÑnÔnÐnà×ÒÑÔ€Dõ „I‡N‚N�4”9ÑÔÐÝ	Ô�d”iÑ Ô Ð àœ€DÔà�Ô#Ð#Ð#Ø%)¤[×%6Ò%6Ñ%8Ô%8Ð"KÐ"KÐ"KˆÔØÔ"Ð&=Ò=Ð=Ø'+Ô'>ÀÈdÌnÐ@]Ð&^ˆDÔ#ð Ô˜hÒ&Ð&ØÔ#ÐSÐSÐ%SÑSÔSÐ#ØÔ˜eÒ#Ð#ØÔ"ÐQÐQÐ$QÑQÔQÐ"õ !Ô-¨dÔ.DÑEÔE€DÔà€Kr   c                  óê  — t          ¦   «         } t          | j        ¦  «         t                               | j        ¦  «         dt          j        j        _	        t          j        | j        ¦  «        }t          j        | j        ¦  «        }| j        dk    r
d| j        › �n| j        }| j        dk    p| j        dv o
| j        dk    }t#          | d|¦  «         t#          | d|¦  «         t#          | d|¦  «         t#          | d	d
¦  «         t#          | d|j        ¦  «         t                               d| j        › �¦  «         t)          | ¦  «        }| j        dk    rxd„ |                     ¦   «         D ¦   «         }d|v | _        t#          | dd|v ¦  «         t#          | dd|v ¦  «         t#          | dd|v ¦  «         | j        g k    r|j        g| _        t-          | ¦  «        }t3          | ||¦  «         d S )NTr„   zcuda:r#  >   r!  r"  r>   rX   rU   rS   Fr  zForced decoder prompt ids: r   c                 ó   — h | ]	}|j         ’Œ
S r6   rð   rò   s     r   rõ   zmain.<locals>.<setcomp>;  s   € ÐSÐSÐS°˜KÔ,ÐSÐSÐSr   r9   rM   r2   rN   r3   rO   r4   )r5  r   rr   rP   rQ   Ú__dict__rV   ÚbackendsÚcudnnÚ	benchmarkr   rm   rk   r   rB   r÷   r¦   Úsetattrr  r2   r€   rH   r^   rS   Údecoder_start_token_idr  )r   Úconfigr>   rX   rU   r{   Úort_model_inputsrC   s           r   ÚmainrG  $  sþ  € Ý‰<Œ<€DÝ�”ÑÔÐÝ
‡K‚K�”ÑÔÐØ%)…E„NÔÔ"åÔ*¨4¬?Ñ;Ô;€FÝ Ô0°´ÑAÔA€IØ04´¸uÒ0DÐ0DÐ,˜DœNÐ,Ð,Ð,È$Ì+€MØŒ~ Ò'Ðh¨D¬NÐ>NÐ,NÐ,gÐSWÔS^ÐbgÒSg€HåˆD�+˜yÑ)Ô)Ð)ÝˆD�/ =Ñ1Ô1Ð1ÝˆD�*˜hÑ'Ô'Ð'ÝˆDÐ$ eÑ,Ô,Ð,ÝˆD�. &Ô"5Ñ6Ô6Ð6å
‡K‚KÐF¨dÔ.DÐFÐFÑGÔGÐGõ �d‰OŒO€EØÔ˜eÒ#Ð#àSÐSÀ×@PÒ@PÑ@RÔ@RÐSÑSÔSÐØ .Ð2BÐ BˆÔÝ�Ð-Ð/BÐFVÐ/VÑWÔWÐWÝ�Ð,Ð.@ÐDTÐ.TÑUÔUÐUÝ�Ð'¨Ð:JÐ)JÑKÔKÐKàÔ! RÒ'Ð'Ø&,Ô&CÐ%DˆDÔ"å˜ÑÔ€FÝ�$˜ Ñ&Ô&Ð&Ð&Ð&r   Ú__main__).r.  r:  r©   rÇ   Úloggingr±   r‰   rl   Únumpyr#   rÄ   rV   r   Úbenchmark_helperr   r   Úonnxruntime_extensionsr   Úoptimum.onnxruntimer   Útorch.profilerr   r   r	   Útqdmr
   Útransformersr   r   r   Úonnxruntimer   Ú	getLoggerr§   rP   Ú	Namespacer^   r€   rR   r»   rÍ   rì   r  r  r5  rG  r6   r   r   ú<module>rT     s  ðð €€€Ø 
€
€
€
Ø €€€Ø 	€	€	€	Ø €€€Ø 	€	€	€	Ø 
€
€
€
Ø €€€à Ð Ð Ð Ø €€€Ø €€€Ø €€€Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 8Ð 8Ð 8Ð 8Ð 8Ð 8Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ Ð Ð Ð Ð Ð Ø SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ Sà Ð Ð Ð à	ˆÔ	˜8Ñ	$Ô	$€ðE�XÔ'ð Eð Eð Eð EðP=�HÔ&ð =ð =ð =ð =ð@.ð .ð .ðbð ð ð:ð ð ð$=*ð =*ð =*ð@O.ð O.ð O.ðdCð Cð Cð~ð ~ð ~ðB!'ð !'ð !'ðH ˆzÒÐØ€D�F„F€F€F€Fð Ðr   