§
    kŠtj™( ã                   ó  — d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
mZ ddlZddlZddlZddlmZmZ ddlmZ ddlmZmZmZ dd	lmZ dd
lmZmZmZmZmZm Z m!Z!m"Z" ddl#m$Z$m%Z%m&Z&m'Z' ddl(m)Z* ddl+m,Z, ddl-m.Z/ ddl0m1Z1m2Z2  ej3        d¦  «        Z4 G d„ de¦  «        Z5dede6e7         dz  dej8        fd„Z9dej8        fd„Z:dej8        fd„Z;dfde7de<fd„Z=dfde7de<de<fd„Z>de7de<d e<de%fd!„Z?d"ej        d#efd$„Z@d"ej        d#efd%„ZAd"ej        d#efd&„ZB	 	 	 	 dgd)ed*ed+e7d,eCd-eDdz  d.eDdz  fd/„ZEd0ed1efd2„ZF	 dhd"ed,eCde6e         fd3„ZGd4„ ZHd5„ ZId6„ ZJd7efd8„ZKd7ed9e<d:e<de<fd;„ZLd7efd<„ZMd=ed>e7fd?„ZNdg fd=ed@eCdAe6eC         fdB„ZOd=efdC„ZPd=ed>e7fdD„ZQ	 	 	 did=edGe7dHeCdIeCdJeCf
dK„ZRd7efdL„ZSd7efdM„ZTdNefdO„ZUdfdPe7de<fdQ„ZV	 dfdPe7dRe7de<de<fdS„ZWdT„ ZXe5jY        fdej8        dUe5fdV„ZZdej8        d=ee!z  dWej[        dXej[        dYeCdZeCd[e6e6eC                  deDe7ef         fd\„Z\d]„ Z]	 	 djdej8        d_e6e7         dz  d`e<fda„Z^dedej8        d_e6e7         dz  fdb„Z_dkde6e7         dz  d_e6e7         dz  fdc„Z)e`ddk    r e)¦   «          dS dS )laë  
This converts GPT2 or T5 model to onnx with beam search operator.

Example 1: convert gpt2 model with beam search:
    python convert_generation.py -m gpt2 --output gpt2_beam_search.onnx

Example 2: convert gpt2 model with beam search containing specific cuda optimizations:
    python convert_generation.py -m gpt2 --output gpt2_beam_search.onnx --use_gpu                       --past_present_share_buffer --use_decoder_masked_attention

Example 3: convert gpt2 model with beam search with mixed precision and enable SkipLayerNorm strict mode:
    python convert_generation.py -m gpt2 --output gpt2_beam_search.onnx --use_gpu -p fp16 --use_sln_strict_mode

Example 4: convert T5 model with beam search in two steps:
    python -m models.t5.convert_to_onnx -m t5-small
    python convert_generation.py -m t5-small --model_type t5                     --decoder_onnx ./onnx_models/t5-small_decoder.onnx                       --encoder_decoder_init_onnx ./onnx_models/t5-small_encoder.onnx          --output ./onnx_models/t5_small_beam_search.onnx

Example 5: convert T5 model with beam search. All in one step:
    python convert_generation.py -m t5-small --model_type t5 --output t5_small_beam_search.onnx

Example 6: convert T5 model with beam search containing specific cuda optimizations. All in one step:
    python convert_generation.py -m t5-small --model_type t5 --output t5_small_beam_search.onnx           --use_gpu --past_present_share_buffer --use_decoder_masked_attention

Example 7: convert MT5 model with external data file like mt5-base-beamsearch.onnx.data in below example.
    python convert_generation.py -m google/mt5-base --model_type mt5 --output mt5-base-beamsearch.onnx -e

Example 8: convert gpt2 model with greedy search:
    python convert_generation.py -m gpt2 --output gpt2_greedy_search.onnx --num_beams 1 --num_return_sequences 1

Example 9: convert gpt2 model with sampling:
    python convert_generation.py -m gpt2 --output gpt2_sampling.onnx --num_beams 1 --num_return_sequences 1 --top_p 0.6
é    N)ÚEnum)ÚPath)ÚAny)Ú	PrecisionÚsetup_logger)ÚNumpyHelper)Ú
GraphProtoÚ
ModelProtoÚTensorProto)Ú	OnnxModel)Ú
GPT2ConfigÚGPT2LMHeadModelÚGPT2TokenizerÚ	MT5ConfigÚMT5ForConditionalGenerationÚT5ConfigÚT5ForConditionalGenerationÚT5Tokenizer)ÚGraphOptimizationLevelÚInferenceSessionÚSessionOptionsÚget_available_providers)Úmain)ÚPRETRAINED_GPT2_MODELS)Úexport_onnx_models)ÚPRETRAINED_MT5_MODELSÚPRETRAINED_T5_MODELSÚ c                   ó    — e Zd ZdZdZdZd„ ZdS )ÚGenerationTypeÚbeam_searchÚgreedy_searchÚsamplingc                 ó   — | j         S ©N)Úvalue)Úselfs    úi/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/onnxruntime/transformers/convert_generation.pyÚ__str__zGenerationType.__str___   s
   € ØŒzÐó    N)Ú__name__Ú
__module__Ú__qualname__Ú
BEAMSEARCHÚGREEDYSEARCHÚSAMPLINGr)   © r*   r(   r    r    Z   s2   € € € € € Ø€JØ"€LØ€Hðð ð ð ð r*   r    ÚargvÚreturnc                 óî  — t          j        ¦   «         }|                     d¦  «        }|                     dddt          dd                     t          t          z   t          z   ¦  «        z   ¬¦  «         |                     dd	t          d
g d¢dd                     g d¢¦  «        z   ¬¦  «         |                     dd	t          t          j
                             dd¦  «        d¬¦  «         |                     dd	t          dd¬¦  «         |                     dd	t          dd¬¦  «         |                     dd	dd¬¦  «         |                     d	¬¦  «         |                     d¦  «        }|                     ddt          d¬¦  «         |                     d d!d	t          t          j        j        t          j        j        t          j        j        gd"¬¦  «         |                     d#d$d	d%d&gd'¬(¦  «         |                     d)d*d	dd+¬¦  «         |                     d	¬,¦  «         |                     d-d.d	dd/¬¦  «         |                     d	¬0¦  «         |                     d1d2d	dd3¬¦  «         |                     d	¬4¦  «         |                     d5d6d	dd7¬¦  «         |                     d	¬8¦  «         |                     d9d:d	dd;¬¦  «         |                     d	¬<¦  «         |                     d=d	dd>¬¦  «         |                     d	¬?¦  «         |                     d@¦  «        }|                     dAd	ddB¬¦  «         |                     d	¬C¦  «         |                     dDd	ddE¬¦  «         |                     d	¬F¦  «         |                     dGd	d¬H¦  «         |                     d	¬I¦  «         |                     dJt           d	dKdL¬M¦  «         |                     dNd	ddO¬¦  «         |                     d	¬P¦  «         |                     dQd	ddR¬¦  «         |                     d	¬S¦  «         |                     dTd	ddU¬¦  «         |                     d	¬V¦  «         |                     dWd	ddX¬¦  «         |                     d	¬Y¦  «         |                     dZd	dd[¬¦  «         |                     d	¬\¦  «         |                     d]d	dd^¬¦  «         |                     d	¬_¦  «         |                     d`d	dda¬¦  «         |                     d	¬b¦  «         |                     dc¦  «        }|                     ddt           d	dedf¬M¦  «         |                     dgt           d	dhdi¬M¦  «         |                     djt           d	dkdl¬M¦  «         |                     dmt           d	dedn¬M¦  «         |                     dot"          d	dedp¬M¦  «         |                     dqt"          d	dedr¬M¦  «         |                     dst"          d	dtdu¬M¦  «         |                     dvt"          d	dtdw¬M¦  «         |                     dxt"          d	t#          dy¦  «         dz¬M¦  «         |                     d{t           d	ded|¬M¦  «         |                     d}t"          d	d~d¬M¦  «         |                     d€t           d	dKd�¬M¦  «         |                     d‚t           d	dƒd„¬M¦  «         |                     d…t           d	dƒd†¬M¦  «         |                     d‡t           d	dƒdˆ¬M¦  «         |                     d‰¦  «        }|                     dŠd	dd‹¬¦  «         |                     d	¬Œ¦  «         |                     d�d	ddŽ¬¦  «         |                     d	¬�¦  «         |                     d�d	dd‘¬¦  «         |                     d	¬’¦  «         |                     d“d	dd”¬¦  «         |                     d	¬•¦  «         |                     d–d	dd—¬¦  «         |                     d	¬˜¦  «         |                     d™d	t           dedš¬¦  «         |                     d›d	ddœ¬¦  «         |                     d	¬�¦  «         |                     | ¦  «        }|S )žzªParse arguments

    Args:
        argv (Optional[List[str]], optional): _description_. Defaults to None.

    Returns:
        argparse.Namespace: Parsed arguments.
    zInput optionsz-mú--model_name_or_pathTzEPytorch model checkpoint path, or pretrained model name in the list: ú, )ÚrequiredÚtypeÚhelpz--model_typeFÚgpt2)r:   Út5Úmt5z*Model type (default is gpt2) in the list: )r7   r8   ÚdefaultÚchoicesr9   ú--cache_dirú.Úcache_modelsz%Directory to cache pre-trained models)r7   r8   r=   r9   z--decoder_onnxr   zLPath of onnx model for decoder. Specify it when you have exported the model.z--encoder_decoder_init_onnxzgPath of ONNX model for encoder and decoder initialization. Specify it when you have exported the model.z	--verboseÚ
store_truezPrint more information)r7   Úactionr9   )ÚverbosezOutput optionsú--outputz,Output path for onnx model with beam search.z-pú--precisionzTPrecision of model to run. fp32 for full precision, fp16 for half or mixed precisionz-bú--op_block_listÚ*ÚautozÿDisable certain onnx operators when exporting model to onnx format. When using defaultvalue for gpt2 type of model fp16 precision, it will be set to ["Add", "LayerNormalization", "SkipLayerNormalization", "FastGelu"]. Other situation, it will be set to [])r7   Únargsr=   r9   z-eú--use_external_data_formatz!save external data for model > 2G)Úuse_external_data_formatz-sz--run_shape_inferencezrun shape inference)Úrun_shape_inferencez-dpvsz--disable_pad_vocab_sizez³Do not pad logits MatMul weight to be a multiple of 8 along the dimension where dim value is the vocab size. The logits MatMul may hence be of poor performance for fp16 precision.)Údisable_pad_vocab_sizez-dsgdz,--disable_separate_gpt2_decoder_for_init_runz™Do not create separate decoder subgraphs for initial and remaining runs. This does not allow for optimizations based on sequence lengths in each subgraph)Ú*disable_separate_gpt2_decoder_for_init_runz-iz--disable_shared_initializersz™do not share initializers in encoder and decoder for T5 or in the init decoder and decoder for GPT2. It will increase memory usage of t5/mt5/gpt2 models.)Údisable_shared_initializersz--encoder_decoder_initzbAdd decoder initialization to encoder for T5 model. This is legacy format that will be deprecated.)Úencoder_decoder_initz6Beam search parameters that stored in the output modelz--output_sequences_scoreszoutput sequences scores)Úoutput_sequences_scoresz--output_token_scoreszoutput token scores)Úoutput_token_scoresz--early_stopping)r7   rC   )Úearly_stoppingz--no_repeat_ngram_sizer   zNo repeat ngram size)r8   r7   r=   r9   z--vocab_maskz\Enable vocab_mask. This mask applies only to every generated token to filter some bad words.)Ú
vocab_maskz--past_present_share_bufferzWUse shared buffer for past and present, currently work for gpt2 greedy/sampling search.)Úpast_present_share_bufferz--use_decoder_masked_attentionzëUses `DecoderMaskedSelfAttention` or `DecoderMaskedMultiHeadAttention` to optimize the decoding Attention computation. Must be used with `past_present_share_buffer`. Currently, only Attention head sizes of 32, 64 and 128 are supported.)Úuse_decoder_masked_attentionz--prefix_vocab_maskzeEnable prefix_vocab_mask. This mask can be used to filter bad words in the first generated token only)Úprefix_vocab_maskz--custom_attention_maskz]Enable custom_attention_mask. This mask can be used to replace default encoder attention mask)Úcustom_attention_maskz--presence_maskz!Presence mask for custom sampling)Úpresence_maskz--seedzRandom seed for sampling op)ÚseedzYBeam search parameters not stored in the output model, for testing parity and performancez--min_lengthé   zMin sequence lengthz--max_lengthé2   zMax sequence lengthz--num_beamsé   z	Beam sizez--num_return_sequencesz&Number of return sequence <= num_beamsz--length_penaltyz<Positive. >1 to penalize and <1 to encourage short sentence.z--repetition_penaltyz-Positive. >1 to penalize and <1 to encourage.z--temperatureç      ð?z6The value used to module the next token probabilities.z--top_pzTop P for samplingz--filter_valueÚInfzFilter value for Top P samplingz--min_tokens_to_keepzAMinimum number of tokens we keep per batch example in the output.z--presence_penaltyç        z%presence penalty for custom sampling.z--customz&If 1 customized top P logic is appliedz--vocab_sizeéÿÿÿÿzIVocab_size of the underlying model used to decide the shape of vocab maskz--eos_token_idzKcustom eos_token_id for generating model with existing onnx encoder/decoderz--pad_token_idzKcustom pad_token_id for generating model with existing onnx encoder/decoderz0Other options for testing parity and performancez--use_sln_strict_modez_Enable strict mode for SLN in CUDA provider. This ensures a better accuracy but will be slower.)Úuse_sln_strict_modeú	--use_gpuz)use GPU for inference. Required for fp16.)Úuse_gpuz--disable_parityzdo not run parity test)Údisable_parityz--disable_perf_testzdo not run perf test)Údisable_perf_testz--torch_performanceztest PyTorch performance)Útorch_performancez--total_runsz4Number of times of inference for latency measurementz--save_test_dataz-save test data for onnxruntime_perf_test tool)Úsave_test_data)ÚargparseÚArgumentParserÚadd_argument_groupÚadd_argumentÚstrÚjoinr   r   r   ÚosÚpathÚset_defaultsr   ÚFLOAT32r&   ÚFLOAT16ÚintÚfloatÚ
parse_args)r2   ÚparserÚinput_groupÚoutput_groupÚmodel_groupÚbeam_parameters_groupÚ
test_groupÚargss           r(   Úparse_argumentsr   c   sb  € õ Ô$Ñ&Ô&€Fà×+Ò+¨OÑ<Ô<€Kà×ÒØØØÝØTØ
�)Š)Õ*Õ-AÑAÕDYÑYÑ
ZÔ
Zñ[ð ñ ô ð ð ×ÒØØÝØØ%Ð%Ð%Ø9¸D¿IºIÐF[ÐF[ÐF[Ñ<\Ô<\Ñ\ð ñ ô ð ð ×ÒØØÝÝ”—’˜S .Ñ1Ô1Ø4ð ñ ô ð ð ×ÒØØÝØØ[ð ñ ô ð ð ×ÒØ%ØÝØØvð ñ ô ð ð ×ÒØØØØ%ð	 ñ ô ð ð ×Ò ÐÑ&Ô&Ð&à×,Ò,Ð-=Ñ>Ô>€Là×ÒØØÝØ;ð	 ñ ô ð ð ×ÒØØØÝÝÔ!Ô'ÝÔ"Ô(­)Ô*;Ô*AÐBØcð ñ ô ð ð ×ÒØØØØØ�ðXð ñ 	ô 	ð 	ð ×ÒØØ$ØØØ0ð ñ ô ð ð ×Ò°uÐÑ=Ô=Ð=à×ÒØØØØØ"ð ñ ô ð ð ×Ò°%ÐÑ8Ô8Ð8à×ÒØØ"ØØðbð ñ ô ð ð ×Ò°UÐÑ;Ô;Ð;à×ÒØØ6ØØðGð ñ ô ð ð ×ÒÈÐÑOÔOÐOà×ÒØØ'ØØðEð ñ ô ð ð ×Ò¸%ÐÑ@Ô@Ð@à×ÒØ ØØØqð	 ñ ô ð ð ×Ò°5ÐÑ9Ô9Ð9à×+Ò+Ð,dÑeÔe€Kà×ÒØ#ØØØ&ð	 ñ ô ð ð ×Ò°UÐÑ;Ô;Ð;à×ÒØØØØ"ð	 ñ ô ð ð ×Ò°ÐÑ7Ô7Ð7à×ÒÐ/¸%ÈÐÑUÔUÐUØ×Ò¨EÐÑ2Ô2Ð2à×ÒØ ÝØØØ#ð ñ ô ð ð ×ÒØØØØkð	 ñ ô ð ð ×Ò¨ÐÑ.Ô.Ð.à×ÒØ%ØØØfð	 ñ ô ð ð ×Ò°uÐÑ=Ô=Ð=à×ÒØ(ØØðð	 ñ ô ð ð ×Ò¸%ÐÑ@Ô@Ð@à×ÒØØØØtð	 ñ ô ð ð ×Ò¨uÐÑ5Ô5Ð5à×ÒØ!ØØØlð	 ñ ô ð ð ×Ò°5ÐÑ9Ô9Ð9à×ÒØØØØ0ð	 ñ ô ð ð ×Ò¨5ÐÑ1Ô1Ð1à×ÒØØØØ*ð	 ñ ô ð ð ×Ò %ÐÑ(Ô(Ð(à"×5Ò5Øcñô Ðð ×&Ò& ~½CÈ%ÐYZÐavÐ&ÑwÔwÐwà×&Ò& ~½CÈ%ÐY[ÐbwÐ&ÑxÔxÐxà×&Ò& }½3ÈÐXYÐ`kÐ&ÑlÔlÐlà×&Ò&Ø ÝØØØ5ð 'ñ ô ð ð ×&Ò&ØÝØØØKð 'ñ ô ð ð ×&Ò&ØÝØØØ<ð 'ñ ô ð ð ×&Ò&ØÝØØØEð 'ñ ô ð ð ×&Ò&ØÝØØØ!ð 'ñ ô ð ð ×&Ò&ØÝØÝ�u‘”�Ø.ð 'ñ ô ð ð ×&Ò&ØÝØØØPð 'ñ ô ð ð ×&Ò&ØÝØØØ4ð 'ñ ô ð ð ×&Ò&ØÝØØØ5ð 'ñ ô ð ð ×&Ò&ØÝØØØXð 'ñ ô ð ð ×&Ò&ØÝØØØZð 'ñ ô ð ð ×&Ò&ØÝØØØZð 'ñ ô ð ð ×*Ò*Ð+]Ñ^Ô^€Jà×ÒØØØØnð	 ñ ô ð ð ×Ò°ÐÑ6Ô6Ð6à×ÒØØØØ8ð	 ñ ô ð ð ×Ò EÐÑ*Ô*Ð*à×ÒØØØØ%ð	 ñ ô ð ð ×Ò¨5ÐÑ1Ô1Ð1à×ÒØØØØ#ð	 ñ ô ð ð ×Ò¨eÐÑ4Ô4Ð4à×ÒØØØØ'ð	 ñ ô ð ð ×Ò¨eÐÑ4Ô4Ð4à×ÒØØÝØØCð ñ ô ð ð ×ÒØØØØ<ð	 ñ ô ð ð ×Ò¨5ÐÑ1Ô1Ð1à×Ò˜TÑ"Ô"€Dà€Kr*   r~   c                 ó@  — | j         }d|d| j        dd| j        ddddd	g}| j        r|                     d
| j        g¦  «         | j        r|                     d¦  «         | j        r|                     d¦  «         t          | j	        ¦  «        r0|                     dg¦  «         |                     | j	        ¦  «         | j        t          j        j        k    r| j        s
J d¦   «         ‚| j        rt                               d|› �¦  «         t!          |¬¦  «         dS )zqConvert GPT-2 model to onnx

    Args:
        args (argparse.Namespace): arguments parsed from command line
    r5   rE   z--optimize_onnxrF   z--test_runsÚ1z--test_casesÚ10z--overwriter?   rd   rK   rG   zEfp16 or mixed precision model cannot run in CPU. Please add --use_gpuzarguments for convert_to_onnx:)r2   N)Úmodel_name_or_pathÚdecoder_onnxÚ	precisionÚ	cache_dirÚextendre   ÚappendrL   ÚlenÚop_block_listr   rt   r&   rD   ÚloggerÚinfoÚconvert_gpt2_to_onnx)r~   Ú
model_nameÚ	argumentss      r(   Úgpt2_to_onnxr�   ÷  sM  € ð Ô(€Jð 	ØØØÔØØØŒØØØØØð€Ið „~ð :Ø×Ò˜-¨¬Ð8Ñ9Ô9Ð9Ø„|ð &Ø×Ò˜Ñ%Ô%Ð%ØÔ$ð 7Ø×ÒÐ5Ñ6Ô6Ð6å
ˆ4ÔÑÔð -Ø×ÒÐ+Ð,Ñ-Ô-Ð-Ø×Ò˜Ô+Ñ,Ô,Ð,à„~�Ô*Ô0Ò0Ð0ØŒ|ÐdÐdÐdÑdÔdˆ|ð
 „|ð BÝ�ŠÐ@°YÐ@Ð@ÑAÔAÐAå˜iÐ(Ñ(Ô(Ð(Ð(Ð(r*   c                 óÖ  — t          | j        | j        t          | j        ¦  «        j        | j        | j        | j        t          j
        j        k    | j        ddddd| j        | j        | j        t          j
        j        k    ¬¦  «        }t                               d|d         › �¦  «         t                               d|d         › �¦  «         |d         | _        |d         | _        dS )	znConvert T5 model to onnx

    Args:
        args (argparse.Namespace): arguments parsed from command line
    FT)rƒ   r†   Ú
output_dirre   rL   Úoptimize_onnxr…   rD   Úuse_decoder_start_tokenÚ	overwriteÚdisable_auto_mixed_precisionÚuse_int32_inputsÚ
model_typerQ   Úforce_fp16_iozonnx model for encoder: r   zonnx model for decoder: r\   N)Úexport_t5_onnx_modelsrƒ   r†   r   ÚoutputÚparentre   rL   r…   r   rt   r&   r˜   rQ   r‹   ÚdebugÚencoder_decoder_init_onnxr„   )r~   Úpathss     r(   Ú
t5_to_onnxr    $  sÛ   € õ "ØÔ2Ø”.Ý˜œÑ$Ô$Ô+Ø”Ø!%Ô!>Ø”~­Ô):Ô)@Ò@Ø”.ØØ %ØØ%*ØØ”?Ø!Ô6Ø”~­Ô):Ô)@Ò@ðñ ô €Eõ$ ‡L‚LÐ6¨E°!¬HÐ6Ð6Ñ7Ô7Ð7Ý
‡L‚LÐ6¨E°!¬HÐ6Ð6Ñ7Ô7Ð7Ø%*¨1¤X€DÔ"Ø˜aœ€DÔÐÐr*   TÚ	onnx_pathrL   c                 óØ   — ddl m} t          j        | d¬¦  «        }|                     |dd¬¦  «        }|rt          j        || |¬¦  «         d	S t                               d¦  «         d	S )
zÇShape inference on an onnx file, which will be overwritten.

    Args:
        onnx_path (str): Path of onnx model
        use_external_data_format(bool): output tensors to external data or not.
    r   )ÚSymbolicShapeInferenceT©Úload_external_dataF)Ú
auto_mergeÚguess_output_rank©Úsave_as_external_dataz4Failed to run symbolic shape inference on the model.N)	Ú&onnxruntime.tools.symbolic_shape_inferr£   ÚonnxÚ
load_modelÚinfer_shapesr   Úsaver‹   Úwarning)r¡   rL   r£   ÚmodelÚouts        r(   Úshape_inferencer²   B  s†   € ð NÐMÐMÐMÐMÐMåŒO˜I¸$Ð?Ñ?Ô?€EØ
 ×
-Ò
-¨eÀÐX]Ð
-Ñ
^Ô
^€CØ
ð OÝŒ�s˜IÐ=UÐVÑVÔVÐVÐVÐVå�ŠÐMÑNÔNÐNÐNÐNr*   c                 ó~  — t          j        | d¬¦  «        }|j        j        d         j        }t          |¦  «        }|                     ¦   «         }||v sJ ‚||         }|j        dk    rdS d}|                     |j	        d         ¦  «        }|€A| 
                    |dd¦  «        }	|	€dS |                     |	j	        d         ¦  «        }|€dS d}|j        t          j        j        k    rdS t          |j        ¦  «        d	k    rdS |j        d         }
|
d
z  dk    rdS t#          j        |
d
z  ¦  «        d
z  }||
z
  }|j        rß|rbt)          j        |j        d         |ft(          j        ¬¦  «        }t)          j        t1          j        |¦  «        |fd¬¦  «        }||j        d<   nat)          j        ||j        d         ft(          j        ¬¦  «        }t)          j        t1          j        |¦  «        |fd¬¦  «        }||j        d<   |                     ¦   «         |_        ndS t          j        || |¬¦  «         dS )zâPad the logits MatMul weight in the provided decoder model, which will be overwritten.

    Args:
        onnx_path (str): Path of onnx model
        use_external_data_format(bool): output tensors to external data or not.
    Tr¤   r   ÚMatMulFr\   NÚ	Transposeé   é   ©Údtype©Úaxisr¨   )r«   r¬   Úgraphr›   Únamer   Úoutput_name_to_nodeÚop_typeÚget_initializerÚinputÚmatch_parentÚ	data_typer   ÚDataTypert   r‰   ÚdimsÚmathÚceilÚraw_dataÚnpÚzerosÚfloat16Úconcatenater   Úto_arrayÚtobytesr®   )r¡   rL   Údecoder_model_protoÚlogits_output_nameÚdecoder_modelr¾   Úmatmul_nodeÚpad_along_axis_1Úlogits_weightÚtranspose_before_matmulÚactual_vocab_sizeÚpadded_vocab_sizeÚpaddingÚpadding_dataÚweight_with_paddings                  r(   Úpad_weights_of_logits_matmulrÛ   T  s}  € õ œ/¨)ÈÐMÑMÔMÐà,Ô2Ô9¸!Ô<ÔAÐåÐ1Ñ2Ô2€Mà'×;Ò;Ñ=Ô=ÐØÐ!4Ð4Ð4Ð4Ð4à%Ð&8Ô9€KàÔ˜hÒ&Ð&Øˆuð
 ÐØ!×1Ò1°+Ô2CÀAÔ2FÑGÔG€MØÐØ"/×"<Ò"<¸[È+ÐWXÑ"YÔ"YÐà"Ð*Ø�5à%×5Ò5Ð6MÔ6SÐTUÔ6VÑWÔWˆàÐ Ø�5à Ðð Ô¥+Ô"6Ô">Ò>Ð>Øˆuõ ˆ=ÔÑÔ !Ò#Ð#Øˆuð &Ô*¨1Ô-Ðà˜AÑ !Ò#Ð#àˆtåœ	Ð"3°aÑ"7Ñ8Ô8¸1Ñ<ÐØÐ"3Ñ3€Gð Ôð Øð 	6Ýœ8 ]Ô%7¸Ô%:¸GÐ$DÍBÌJÐWÑWÔWˆLÝ"$¤.µ+Ô2FÀ}Ñ2UÔ2UÐWcÐ1dÐklÐ"mÑ"mÔ"mÐØ$5ˆMÔ˜qÑ!Ð!åœ8 W¨mÔ.@ÀÔ.CÐ$DÍBÌJÐWÑWÔWˆLÝ"$¤.µ+Ô2FÀ}Ñ2UÔ2UÐWcÐ1dÐklÐ"mÑ"mÔ"mÐØ$5ˆMÔ˜qÑ!à!4×!<Ò!<Ñ!>Ô!>ˆÔÐàˆuõ „NÐ&¨	ÐIaÐbÑbÔbÐbØˆ4r*   Ú
model_pathre   rc   c                 ó"  ‡— t          ¦   «         }t          j        |_        |rddgndg}|rQdt	          ¦   «         vrt          d¦  «        ‚t                               d¦  «         |rddi}d|iŠˆfd„|D ¦   «         }t          | ||¬¦  «        }|S )	a„  Create OnnxRuntime session.

    Args:
        model_path (str): onnx model path
        use_gpu (bool): use GPU or not
        use_sln_strict_mode (bool): use strict mode for skip layer normalization or not

    Raises:
        RuntimeError: CUDAExecutionProvider is not available when --use_gpu is specified.

    Returns:
        onnxruntime.InferenceSession: The created session.
    ÚCUDAExecutionProviderÚCPUExecutionProviderz5CUDAExecutionProvider is not available for --use_gpu!zuse CUDAExecutionProviderÚ"enable_skip_layer_norm_strict_modeTc                 ó0   •— g | ]}|‰v r
|‰|         fn|‘ŒS r1   r1   )Ú.0r½   Úprovider_optionss     €r(   ú
<listcomp>z&create_ort_session.<locals>.<listcomp>»  sB   ø€ ð #ð #ð #ØY]°$Ð:JÐ2JÐ2J�Ð'¨Ô-Ð.Ð.ÐPTð#ð #ð #r*   )Ú	providers)	r   r   ÚORT_DISABLE_ALLÚgraph_optimization_levelr   ÚRuntimeErrorr‹   rŒ   r   )rÜ   re   rc   Úsess_optionsÚexecution_providersÚcuda_provider_optionsÚort_sessionrã   s          @r(   Úcreate_ort_sessionrí   ¢  s×   ø€ õ "Ñ#Ô#€LÝ,BÔ,R€LÔ)ØOVÐtÐ2Ð4JÐKÐKÐ]sÐ\tÐØð 
Ø"Õ*AÑ*CÔ*CÐCÐCÝÐVÑWÔWÐWå�KŠKÐ3Ñ4Ô4Ð4Øð 	Ø%IÈ4Ð$PÐ!Ø 7Ð9NÐOÐð#ð #ð #ð #Øatð#ñ #ô #Ðõ # :¨|ÐGZÐ[Ñ[Ô[€KØÐr*   r¼   r…   c           
      ót  — |t           j        j        k    }t          | j        ¦  «        }|dz
  }|dk    sJ ‚g d¢d„ t          |¦  «        D ¦   «         z   }t          | j        ¦  «        t          |¦  «        k    r4t          dt          |¦  «        › dt          | j        ¦  «        › �¦  «        ‚t          |¦  «        D ]©\  }}| j        |         j        |k    r(t          d|› d|› d| j        |         j        › �¦  «        ‚t          j
        }|dk    r|rt          j        nt          j        }| j        |         j        j        j        }	|	|k    rt          d|› d	|› d|	› �¦  «        ‚Œªt                               d
¦  «         dgd„ t          |¦  «        D ¦   «         z   }
t          | j        ¦  «        t          |
¦  «        k    r4t          dt          |
¦  «        › dt          | j        ¦  «        › �¦  «        ‚t          |
¦  «        D ]—\  }}| j        |         j        |k    r(t          d|› d|› d| j        |         j        › �¦  «        ‚|rt          j        nt          j        }| j        |         j        j        j        }||k    rt          d|› d	|› d|› �¦  «        ‚Œ˜t                               d¦  «         dS )aæ  Verify GPT-2 subgraph

    Args:
        graph (onnx.GraphProto): onnx graph of GPT-2
        precision (Precision): Precision (FLOAT16 or FLOAT32) of the model.

    Raises:
        ValueError: Number of inputs not expected.
        ValueError: Input name is not expected.
        ValueError: Input data type is not expected.
        ValueError: Number of outputs not expected.
        ValueError: Output name is not expected.
        ValueError: Output data type is not expected.
    é   r\   )Ú	input_idsÚposition_idsÚattention_maskc                 ó   — g | ]}d |› �‘ŒS )Úpast_r1   ©râ   Úis     r(   rä   z(verify_gpt2_subgraph.<locals>.<listcomp>Ø  s    € ÐHqÐHqÐHqÐYZÈÐQRÈÈÐHqÐHqÐHqr*   ú Number of inputs expected to be ú. Got úInput ú is expected to be ú$ is expected to have onnx data type z:Verifying GPT-2 graph inputs: name and data type are good.Úlogitsc                 ó   — g | ]}d |› �‘ŒS )Úpresent_r1   rõ   s     r(   rä   z(verify_gpt2_subgraph.<locals>.<listcomp>é  s   € Ð$PÐ$PÐ$P¸ ^° ^ ^Ð$PÐ$PÐ$Pr*   ú!Number of outputs expected to be úOutput z;Verifying GPT-2 graph outputs: name and data type are good.N)r   rt   r&   r‰   rÁ   ÚrangeÚ
ValueErrorÚ	enumerater½   r   ÚINT32ÚFLOATr8   Útensor_typeÚ	elem_typer‹   rŒ   r›   )r¼   r…   Ú
is_float16Úinput_countÚlayer_countÚexpected_inputsrö   Úexpected_inputÚexpected_typeÚ
input_typeÚexpected_outputsÚexpected_outputÚoutput_types                r(   Úverify_gpt2_subgraphr  Ã  sû  € ð �iÔ/Ô5Ò5€Jå�e”kÑ"Ô"€KØ ‘/€KØ˜!ÒÐÐÐàEÐEÐEÐHqÐHqÕ^cÐdoÑ^pÔ^pÐHqÑHqÔHqÑq€OÝ
ˆ5Œ;ÑÔ�3˜Ñ/Ô/Ò/Ð/ÝÐj½CÀÑ<PÔ<PÐjÐjÕX[Ð\aÔ\gÑXhÔXhÐjÐjÑkÔkÐkå& Ñ7Ô7ð 
pð 
pÑˆˆ>ØŒ;�qŒ>Ô .Ò0Ð0ÝÐg aÐgÐg¸NÐgÐgÐRWÔR]Ð^_ÔR`ÔReÐgÐgÑhÔhÐhå#Ô)ˆØ�Š6ˆ6Ø3=ÐT�KÔ/Ð/Å;ÔCTˆMà”[ ”^Ô(Ô4Ô>ˆ
Ø˜Ò&Ð&ÝÐn aÐnÐnÈ]ÐnÐnÐblÐnÐnÑoÔoÐoð 'å
‡K‚KÐLÑMÔMÐMà �zÐ$PÐ$P½UÀ;Ñ=OÔ=OÐ$PÑ$PÔ$PÑPÐÝ
ˆ5Œ<ÑÔ�CÐ 0Ñ1Ô1Ò1Ð1ÝÐm½SÐAQÑ=RÔ=RÐmÐmÕZ]Ð^cÔ^jÑZkÔZkÐmÐmÑnÔnÐnå'Ð(8Ñ9Ô9ð qð qÑˆˆ?ØŒ<˜Œ?Ô ?Ò2Ð2ÝÐj qÐjÐj¸_ÐjÐjÐTYÔT`ÐabÔTcÔThÐjÐjÑkÔkÐkà/9ÐP�Ô+Ð+½{Ô?PˆØ”l 1”oÔ*Ô6Ô@ˆØ˜-Ò'Ð'ÝÐo aÐoÐoÈ]ÐoÐoÐbmÐoÐoÑpÔpÐpð (å
‡K‚KÐMÑNÔNÐNð €Fr*   c           
      ó  — |t           j        j        k    }|rt          j        nt          j        }t          | j        ¦  «        }|dz
  dz  }|dk    sJ ‚ddg}t          |¦  «        D ]2}|                     d|› �¦  «         |                     d|› �¦  «         Œ3t          |¦  «        D ]2}|                     d|› �¦  «         |                     d	|› �¦  «         Œ3t          | j        ¦  «        t          |¦  «        k    r4t          d
t          |¦  «        › dt          | j        ¦  «        › �¦  «        ‚t          |¦  «        D ]‘\  }}| j        |         j        |k    r(t          d|› d|› d| j        |         j        › �¦  «        ‚|dk     rt          j        n|}	| j        |         j        j        j        }
|
|	k    rt          d|› d|	› d|
› �¦  «        ‚Œ’dg}t          |¦  «        D ]2}|                     d|› �¦  «         |                     d|› �¦  «         Œ3t          | j        ¦  «        t          |¦  «        k    r4t          dt          |¦  «        › dt          | j        ¦  «        › �¦  «        ‚t          |¦  «        D ]}\  }}| j        |         j        |k    r(t          d|› d|› d| j        |         j        › �¦  «        ‚| j        |         j        j        j        }||k    rt          d|› d|› d|› �¦  «        ‚Œ~dS )áð  Verify T5 decoder subgraph

    Args:
        graph (onnx.GraphProto): onnx graph of T5 decoder
        precision (Precision): Precision (FLOAT16 or FLOAT32) of the model.

    Raises:
        ValueError: Number of inputs not expected.
        ValueError: Input name is not expected.
        ValueError: Input data type is not expected.
        ValueError: Number of outputs not expected.
        ValueError: Output name is not expected.
        ValueError: Output data type is not expected.
    r¶   r^   r\   rð   Úencoder_attention_maskÚpast_key_self_Úpast_value_self_Úpast_key_cross_Úpast_value_cross_r÷   rø   rù   rú   rû   rü   Úpresent_key_self_Úpresent_value_self_rÿ   r   N)r   rt   r&   r   r  r‰   rÁ   r  rˆ   r  r  r½   r  r8   r  r  r›   )r¼   r…   r  Ú
float_typer	  r
  r  rö   r  r  r  r  r  r  s                 r(   Úverify_t5_decoder_subgraphr  û  s”  € ð �iÔ/Ô5Ò5€JØ(2ÐI•Ô$Ð$½Ô8I€Jå�e”kÑ"Ô"€KØ ‘? qÑ(€KØ˜!ÒÐÐÐð #Ð$<Ð=€OÝ�;ÑÔð 7ð 7ˆØ×ÒÐ3°Ð3Ð3Ñ4Ô4Ð4Ø×ÒÐ5°!Ð5Ð5Ñ6Ô6Ð6Ð6Ý�;ÑÔð 8ð 8ˆØ×ÒÐ4°Ð4Ð4Ñ5Ô5Ð5Ø×ÒÐ6°1Ð6Ð6Ñ7Ô7Ð7Ð7å
ˆ5Œ;ÑÔ�3˜Ñ/Ô/Ò/Ð/ÝÐj½CÀÑ<PÔ<PÐjÐjÕX[Ð\aÔ\gÑXhÔXhÐjÐjÑkÔkÐkå& Ñ7Ô7ð pð pÑˆˆ>ØŒ;�qŒ>Ô .Ò0Ð0ÝÐg aÐgÐg¸NÐgÐgÐRWÔR]Ð^_ÔR`ÔReÐgÐgÑhÔhÐhà-.°ªU¨U�Ô)Ð)¸
ˆØ”[ ”^Ô(Ô4Ô>ˆ
Ø˜Ò&Ð&ÝÐn aÐnÐnÈ]ÐnÐnÐblÐnÐnÑoÔoÐoð 'ð !�zÐÝ�;ÑÔð ;ð ;ˆØ×ÒÐ 7°AÐ 7Ð 7Ñ8Ô8Ð8Ø×ÒÐ 9°aÐ 9Ð 9Ñ:Ô:Ð:Ð:å
ˆ5Œ<ÑÔ�CÐ 0Ñ1Ô1Ò1Ð1ÝÐm½SÐAQÑ=RÔ=RÐmÐmÕZ]Ð^cÔ^jÑZkÔZkÐmÐmÑnÔnÐnå'Ð(8Ñ9Ô9ð oð oÑˆˆ?ØŒ<˜Œ?Ô ?Ò2Ð2ÝÐj qÐjÐj¸_ÐjÐjÐTYÔT`ÐabÔTcÔThÐjÐjÑkÔkÐkØ”l 1”oÔ*Ô6Ô@ˆØ˜*Ò$Ð$ÝÐm qÐmÐmÈjÐmÐmÐ`kÐmÐmÑnÔnÐnð %ð	oð or*   c           
      ó^  — |t           j        j        k    }d| j        d         j        v }g d¢}|r
|dd…         }t          | j        ¦  «        t          |¦  «        k    r4t          dt          |¦  «        › dt          | j        ¦  «        › �¦  «        ‚t          |¦  «        D ]‰\  }}| j        |         j        |k    r(t          d|› d	|› d| j        |         j        › �¦  «        ‚t          j
        }| j        |         j        j        j        }||k    rt          d|› d
|› d|› �¦  «        ‚ŒŠ|r�t          | j        ¦  «        dz  dk    sJ ‚t          | j        ¦  «        dz  }	|	dk    sJ ‚g }
t          |	¦  «        D ]2}|
                     d|› �¦  «         |
                     d|› �¦  «         Œ3nät                                d¦  «         t          | j        ¦  «        dz
  dz  dk    sJ ‚t          | j        ¦  «        dz
  dz  }	|	dk    sJ ‚ddg}
t          |	¦  «        D ]2}|
                     d|› �¦  «         |
                     d|› �¦  «         Œ3t          |	¦  «        D ]2}|
                     d|› �¦  «         |
                     d|› �¦  «         Œ3t          | j        ¦  «        t          |
¦  «        k    r4t          dt          |
¦  «        › dt          | j        ¦  «        › �¦  «        ‚t          |
¦  «        D ]—\  }}| j        |         j        |k    r(t          d|› d	|› d| j        |         j        › �¦  «        ‚|rt          j        nt          j        }| j        |         j        j        j        }||k    rt          d|› d
|› d|› �¦  «        ‚Œ˜t                                d¦  «         dS )r  Úcrossr   )Úencoder_input_idsr  Údecoder_input_idsNr¶   r÷   rø   rù   rú   rû   r\   Úpresent_key_cross_Úpresent_value_cross_zZThis format is deprecated. Please export T5 encoder in new format with only cross outputs.r^   rü   Úencoder_hidden_statesr  r  rÿ   r   zMT5 encoder graph verified: name and data type of inputs and outputs are good.)r   rt   r&   r›   r½   r‰   rÁ   r  r  r   r  r8   r  r  r  rˆ   r‹   r¯   r  rŒ   )r¼   r…   r  Ú
new_formatr  rö   r  r  r  r
  r  r  r  s                r(   Ú'verify_t5_encoder_decoder_init_subgraphr&  G  sa  € ð �iÔ/Ô5Ò5€JØ˜EœL¨œOÔ0Ð0€Jðð ð €Oð
 ð .Ø)¨"¨1¨"Ô-ˆÝ
ˆ5Œ;ÑÔ�3˜Ñ/Ô/Ò/Ð/ÝÐj½CÀÑ<PÔ<PÐjÐjÕX[Ð\aÔ\gÑXhÔXhÐjÐjÑkÔkÐkå& Ñ7Ô7ð pð pÑˆˆ>ØŒ;�qŒ>Ô .Ò0Ð0ÝÐg aÐgÐg¸NÐgÐgÐRWÔR]Ð^_ÔR`ÔReÐgÐgÑhÔhÐhå#Ô)ˆØ”[ ”^Ô(Ô4Ô>ˆ
Ø˜Ò&Ð&ÝÐn aÐnÐnÈ]ÐnÐnÐblÐnÐnÑoÔoÐoð 'ð ð "@Ý�5”<Ñ Ô  1Ñ$¨Ò)Ð)Ð)Ð)Ý˜%œ,Ñ'Ô'¨1Ñ,ˆØ˜aÒÐÐÐð ÐÝ�{Ñ#Ô#ð 	@ð 	@ˆAØ×#Ò#Ð$<¸Ð$<Ð$<Ñ=Ô=Ð=Ø×#Ò#Ð$>¸1Ð$>Ð$>Ñ?Ô?Ð?Ð?ð	@õ 	�ŠÐsÑtÔtÐtÝ�E”LÑ!Ô! AÑ%¨Ñ*¨aÒ/Ð/Ð/Ð/Ý˜5œ<Ñ(Ô(¨1Ñ,°Ñ2ˆØ˜aÒÐÐÐð %Ð&=Ð>ÐÝ�{Ñ#Ô#ð 	?ð 	?ˆAØ×#Ò#Ð$;¸Ð$;Ð$;Ñ<Ô<Ð<Ø×#Ò#Ð$=¸!Ð$=Ð$=Ñ>Ô>Ð>Ð>Ý�{Ñ#Ô#ð 	@ð 	@ˆAØ×#Ò#Ð$<¸Ð$<Ð$<Ñ=Ô=Ð=Ø×#Ò#Ð$>¸1Ð$>Ð$>Ñ?Ô?Ð?Ð?å
ˆ5Œ<ÑÔ�CÐ 0Ñ1Ô1Ò1Ð1ÝÐm½SÐAQÑ=RÔ=RÐmÐmÕZ]Ð^cÔ^jÑZkÔZkÐmÐmÑnÔnÐnå'Ð(8Ñ9Ô9ð rð rÑˆˆ?ØŒ<˜Œ?Ô ?Ò2Ð2ÝÐj qÐjÐj¸_ÐjÐjÐTYÔT`ÐabÔTcÔThÐjÐjÑkÔkÐkà/9ÐP�Ô+Ð+½{Ô?PˆØ”l 1”oÔ*Ô6Ô@ˆØ˜-Ò'Ð'ÝÐp qÐpÐpÈmÐpÐpÐcnÐpÐpÑqÔqÐqð (õ ‡K‚KÐ_Ñ`Ô`Ð`Ð`Ð`r*   Úshared_é   Úgraph1Úgraph2Úshared_prefixÚmin_elementsÚsignature_cache1Úsignature_cache2c                 ó   — i }i }g }g }	g }
| j         D ]Ó}|j        rt          |j        ¦  «        |k    sŒ"|j         D ]©}|j        rt          |j        ¦  «        |k    sŒ"t          j        ||||¦  «        rp||j        z   ||j        <   |                     |¦  «         |j        |vr>||j        z   }|||j        <   |	                     |¦  «         |
                     |¦  «          nŒªŒÔt                               d|
› �¦  «         | j	        D ]R}t          t          |j        ¦  «        ¦  «        D ].}|j        |         |
v rt          d|j        |         › �¦  «        ‚Œ/ŒS|j	        D ]R}t          t          |j        ¦  «        ¦  «        D ].}|j        |         |
v rt          d|j        |         › �¦  «        ‚Œ/ŒS|	D ]}|j                              |¦  «         Œ|j        D ]}|j        |v r||j                 |_        Œ|j	        D ]ˆ}t          t          |j        ¦  «        ¦  «        D ]d}|j        |         |v rS||j        |                  }t                               d|j        › d|› d|j        |         › d|› �¦  «         ||j        |<   ŒeŒ‰|D ]}| j                              |¦  «         Œ| j        D ]}|j        |v r||j                 |_        Œ| j	        D ]ˆ}t          t          |j        ¦  «        ¦  «        D ]d}|j        |         |v rS||j        |                  }t                               d|j        › d|› d|j        |         › d|› �¦  «         ||j        |<   ŒeŒ‰|	D ]}||j                 |_        Œ|	D ]…}t           j                             |¦  «        j        }t           j                             |j        |j        |¦  «        }| j                             |¦  «         |j                             |¦  «         Œ†|	S )	a…  Remove initializers with same value from two graphs.

    Args:
        graph1 (GraphProto): the first graph to process
        graph2 (GraphProto): the second graph to process
        shared_prefix (str): add prefix to the shared initializers among two graphs
        min_elements (int, optional): minimal number of elements for initializers to be considered. Defaults to 1024.
        signature_cache1 (dict): Optional dictionary to store data signatures of tensors in graph1 in order to speed up comparison
        signature_cache2 (dict): Optional dictionary to store data signatures of tensors in graph2 in order to speed up comparison
    zshared initializers:zname is found in graph 1: zname is found in graph 2: zgraph 2 rename node z input z from z to zgraph 1 rename node )ÚinitializerrÅ   Úsumr   Úhas_same_valuer½   rˆ   r‹   r�   Únoder  r‰   rÁ   rè   ÚremoveÚ
value_infor«   Únumpy_helperrÍ   ÚshapeÚhelperÚmake_tensor_value_inforÃ   )r)  r*  r+  r,  r-  r.  Úmapping_initializers_1Úmapping_initializers_2Úshared_initializers_1Úshared_initializers_2Úshared_initializers_namesÚinitializer1Úinitializer2Úshared_namer3  Újr0  r5  Únew_namer7  s                       r(   Úremove_shared_initializersrD  £  sÑ  € ð&  ÐØÐØÐØÐØ "ÐàÔ*ð ð ˆØÔ!ð 	¥c¨,Ô*;Ñ&<Ô&<ÀÒ&LÐ&LØà"Ô.ð 	ð 	ˆLØ Ô%ð ­#¨lÔ.?Ñ*@Ô*@ÀLÒ*PÐ*PØåÔ'¨°lÐDTÐVfÑgÔgð 	Ø<IÈLÔL]Ñ<]Ð& |Ô'8Ñ9Ø%×,Ò,¨\Ñ:Ô:Ð:àÔ$Ð,BÐBÐBØ"/°,Ô2CÑ"C�KØ@KÐ*¨<Ô+<Ñ=Ø)×0Ò0°Ñ>Ô>Ð>Ø-×4Ò4°[ÑAÔAÐAØ�ð	øõ ‡L‚LÐCÐ(AÐCÐCÑDÔDÐDð ”ð Qð QˆÝ•s˜4œ:‘”Ñ'Ô'ð 	Qð 	QˆAØŒz˜!Œ}Ð 9Ð9Ð9Ý"Ð#OÀÄ
È1ÄÐ#OÐ#OÑPÔPÐPð :ð	Qð
 ”ð Qð QˆÝ•s˜4œ:‘”Ñ'Ô'ð 	Qð 	QˆAØŒz˜!Œ}Ð 9Ð9Ð9Ý"Ð#OÀÄ
È1ÄÐ#OÐ#OÑPÔPÐPð :ð	Qð
 -ð /ð /ˆØÔ×!Ò! +Ñ.Ô.Ð.Ð.ð Ô'ð Fð Fˆ
ØŒ?Ð4Ð4Ð4Ø4°Z´_ÔEˆJŒOøð ”ð )ð )ˆÝ•s˜4œ:‘”Ñ'Ô'ð 	)ð 	)ˆAØŒz˜!Œ}Ð 6Ð6Ð6Ø1°$´*¸Q´-Ô@�Ý—’Ðl°D´IÐlÐlÀaÐlÐlÈtÌzÐZ[Ì}ÐlÐlÐbjÐlÐlÑmÔmÐmØ (�”
˜1‘øð		)ð -ð /ð /ˆØÔ×!Ò! +Ñ.Ô.Ð.Ð.ð Ô'ð Fð Fˆ
ØŒ?Ð4Ð4Ð4Ø4°Z´_ÔEˆJŒOøð ”ð )ð )ˆÝ•s˜4œ:‘”Ñ'Ô'ð 	)ð 	)ˆAØŒz˜!Œ}Ð 6Ð6Ð6Ø1°$´*¸Q´-Ô@�Ý—’Ðl°D´IÐlÐlÀaÐlÐlÈtÌzÐZ[Ì}ÐlÐlÐbjÐlÐlÑmÔmÐmØ (�”
˜1‘øð		)ð -ð Dð DˆØ1°+Ô2BÔCˆÔÐà,ð -ð -ˆÝÔ!×*Ò*¨;Ñ7Ô7Ô=ˆÝ”[×7Ò7¸Ô8HÈ+ÔJ_ÐafÑgÔgˆ
àÔ× Ò  Ñ,Ô,Ð,ØÔ× Ò  Ñ,Ô,Ð,Ð,à Ð r*   Úencoder_modelrÑ   c                 óB  — t          | ¦  «        }t          |¦  «        }|                     d¦  «         |                     d¦  «         i i }}|                     |¦  «         |                     |¦  «         t          |j        j        |j        j        d||¬¦  «        }|S )NÚe_Úd_Ús_)r+  r-  r.  )r   Úadd_prefix_to_namesÚremove_duplicated_initializerrD  r°   r¼   )rE  rÑ   ÚencoderÚdecoderr-  r.  Úinitializerss          r(   Úget_shared_initializersrO    s®   € Ý˜Ñ&Ô&€GÝ˜Ñ&Ô&€GØ×Ò Ñ%Ô%Ð%Ø×Ò Ñ%Ô%Ð%Ø)+¨RÐ&ÐØ×)Ò)Ð*:Ñ;Ô;Ð;Ø×)Ò)Ð*:Ñ;Ô;Ð;Ý-ØŒÔØŒÔØØ)Ø)ðñ ô €Lð Ðr*   c                 ó¢  — g }| j         D ]7}|j        rt          |j        ¦  «        |k    sŒ"|                     |¦  «         Œ8|D ]}| j                              |¦  «         Œ|D ]k}t
          j                             |¦  «        j        }t
          j	         
                    |j        |j        |¦  «        }| j                             |¦  «         Œl|S )a^  Remove initializers of a graph, when they have number of elements larger than a threshold.

    Args:
        graph (GraphProto): the graph.
        min_elements (int, optional): minimal number of elements for initializers to be considered. Defaults to 1024.

    Returns:
        List[TensorProto]: initializers that are removed from the graph.
    )r0  rÅ   r1  rˆ   r4  r«   r6  rÍ   r7  r8  r9  r½   rÃ   r5  )r¼   r,  Úmoved_initializersÚtensorr0  r7  r5  s          r(   Úmove_initializersrS    sç   € ð ÐØÔ#ð *ð *ˆØ”ð 	¥ F¤KÑ 0Ô 0°LÒ @Ð @ØØ×!Ò! &Ñ)Ô)Ð)Ð)à)ð .ð .ˆØÔ× Ò  Ñ-Ô-Ð-Ð-ð *ð ,ð ,ˆÝÔ!×*Ò*¨;Ñ7Ô7Ô=ˆÝ”[×7Ò7¸Ô8HÈ+ÔJ_ÐafÑgÔgˆ
ØÔ×Ò 
Ñ+Ô+Ð+Ð+àÐr*   c                 ó  — | j         dk    rt          d| j        › d�¦  «        ‚| j         dk    r| j        }nË| j         dk    r| j        }n¸| j         dk    r| j        }n¥| j         dk    r| j        }n’| j         dk    r| j        }n| j         d	k    r| j        }nl| j         d
k    r| j	        }nY| j         dk    r| j
        }nF| j         dk    r| j        }n3| j         dk    r| j        }n t          d| j        › d| j         › d�¦  «        ‚| j        |fS )zÄ
    Convert attribute to kwarg format for use with onnx.helper.make_node.
        :parameter attribute: attribute in AttributeProto format.
        :return: attribute in {key: value} format.
    r   z
attribute z does not have type specified.r\   r¶   rï   r^   é   é   é   r·   é	   é
   z has unsupported type r@   )r8   r  r½   Úfrö   ÚsÚtÚgÚfloatsÚintsÚstringsÚtensorsÚgraphs)Ú	attributer&   s     r(   Ú_attribute_to_pairrd  >  sB  € ð „~˜ÒÐÝÐT i¤nÐTÐTÐTÑUÔUÐUð „~˜ÒÐØ”ˆˆØ	Œ˜1Ò	Ð	Ø”ˆˆØ	Œ˜1Ò	Ð	Ø”ˆˆØ	Œ˜1Ò	Ð	Ø”ˆˆØ	Œ˜1Ò	Ð	Ø”ˆˆØ	Œ˜1Ò	Ð	ØÔ ˆˆØ	Œ˜1Ò	Ð	Ø”ˆˆØ	Œ˜1Ò	Ð	ØÔ!ˆˆØ	Œ˜1Ò	Ð	ØÔ!ˆˆØ	Œ˜2Ò	Ð	ØÔ ˆˆåÐ] i¤nÐ]Ð]ÈIÌNÐ]Ð]Ð]Ñ^Ô^Ð^àŒN˜EÐ"Ð"r*   c                 ó¶   — i }| j         D ]+}t          |¦  «        \  }}|                     ||i¦  «         Œ,| j        r|                     d| j        i¦  «         |S )NÚdomain)rc  rd  Úupdaterf  )r3  ÚkwargsÚattrÚkeyr&   s        r(   Ú	kwargs_ofrk  c  sj   € Ø€FØ”ð $ð $ˆÝ)¨$Ñ/Ô/‰ˆˆeØ�Š�s˜E�lÑ#Ô#Ð#Ð#Ø„{ð /Ø�Š�x ¤Ð-Ñ.Ô.Ð.Ø€Mr*   c                 ó\   — t          d„ | j        j        j        j        D ¦   «         ¦  «        S )Nc                 ó8   — g | ]}|j         r|j         n|j        ‘ŒS r1   )Ú	dim_paramÚ	dim_value)râ   Úds     r(   rä   zshape_of.<locals>.<listcomp>n  s'   € ÐgÐgÐgÀA !¤+Ð?�!”+�+°A´KÐgÐgÐgr*   )Útupler8   r  r7  Údim)Úvis    r(   Úshape_ofrt  m  s*   € ÝÐgÐgÈÌÔI\ÔIbÔIfÐgÑgÔgÑhÔhÐhr*   Úsubgc                 ó  — d}d}g }t          | j        ¦  «        D ]ƒ\  }}||k    rbt          |¦  «        }t          j                             |j        |j        j        j	        |d         |d         |d         d|d         g¬¦  «        }| 
                    |g¦  «         Œ„| 
                    t          j                             dt          j        j        dg¬	¦  «        g¦  «         |                      d
¦  «         | j         
                    |¦  «         g }t          | j        ¦  «        D ]ƒ\  }}||k    rbt          |¦  «        }t          j                             |j        |j        j        j	        |d         |d         |d         d|d         g¬¦  «        }| 
                    |g¦  «         Œ„|                      d¦  «         | j         
                    |¦  «         g }| j        D ]ñ}	|	}
|	j        dk    rÌt#          |	¦  «        }|                     ddi¦  «         g }| 
                    |	j        ¦  «         t'          |¦  «        dk     r)| 
                    dg¦  «         t'          |¦  «        dk     °)t'          |¦  «        dk     r| 
                    dg¦  «         t          j        j        d||	j        fd|	j        i|¤Ž}
| 
                    |
g¦  «         Œò|                      d¦  «         | j         
                    |¦  «         | S )Nrï   r\   r   r¶   Úmax_seq_lenr^   ©r  r7  Úpast_sequence_length©r7  rÁ   r›   Ú	AttentionrV   rV  r   rW  r½   r3  )r  rÁ   rt  r«   r8  r9  r½   r8   r  r  r‡   r   r  Ú
ClearFieldr›   r3  r¿   rk  rg  r‰   Ú	make_node)ru  Úinput_past_0Úoutput_past_0Ú
new_inputsrö   rs  r7  Únew_outputsÚ	new_nodesr3  Únew_noderh  Úniss                r(   Ú1update_decoder_subgraph_past_present_share_bufferr…  q  sý  € Ø€LØ€MØ€JÝ˜4œ:Ñ&Ô&ð  ð  ‰ˆˆ2Ø�ÒÐÝ˜R‘L”LˆEÝ”×3Ò3Ø”Øœ'Ô-Ô7Ø˜Q”x  q¤¨5°¬8°]ÀEÈ!ÄHÐMð 4ñ ô ˆBð
 	×Ò˜2˜$ÑÔÐÐØ×Ò•t”{×9Ò9Ð:PÕRVÔRbÔRhÐqrÐpsÐ9ÑtÔtÐuÑvÔvÐvØ‡O‚O�GÑÔÐØ„J×Ò�jÑ!Ô!Ð!à€KÝ˜4œ;Ñ'Ô'ð !ð !‰ˆˆ2Ø�ÒÐÝ˜R‘L”LˆEÝ”×3Ò3Ø”Øœ'Ô-Ô7Ø˜Q”x  q¤¨5°¬8°]ÀEÈ!ÄHÐMð 4ñ ô ˆBð
 	×Ò˜B˜4Ñ Ô Ð Ð Ø‡O‚O�HÑÔÐØ„K×Ò�{Ñ#Ô#Ð#à€IØ”	ð %ð %ˆØˆØŒ<˜;Ò&Ð&Ý˜t‘_”_ˆFØ�MŠMÐ6¸Ð:Ñ;Ô;Ð;ØˆCØ�JŠJ�t”zÑ"Ô"Ð"Ý�c‘(”(˜Q’,�,Ø—
’
˜B˜4Ñ Ô Ð õ �c‘(”(˜Q’,�,å�3‰xŒx˜!Š|ˆ|Ø—
’
Ð2Ð3Ñ4Ô4Ð4Ý”{Ô,¨[¸#¸t¼{ÐeÐeÐQUÔQZÐeÐ^dÐeÐeˆHØ×Ò˜(˜Ñ$Ô$Ð$Ð$Ø‡O‚O�FÑÔÐØ„I×Ò�YÑÔÐØ€Kr*   Úis_beam_searchÚswitch_attentionc                 ó
  — |rîg }t          | j        ¦  «        D ]\  }}|                     |g¦  «         Œ|                     t          j                             dt          j        j        dg¬¦  «        g¦  «         |                     t          j                             dt          j        j        g d¢¬¦  «        g¦  «         |                      d¦  «         | j                             |¦  «         |�r�g d¢}g }| j	        D �]Q}|j
        dk    �r,t          |¦  «        }	|	                     ¦   «         D ]7}
|
d	k    r  d
S |
|vr'|
dk    rt                               d|
› d�¦  «         |	|
= Œ8g }|                     |j        ¦  «         |rŽt          |¦  «        dk     r)|                     dg¦  «         t          |¦  «        dk     °)t          |¦  «        dk     r|                     dg¦  «         t          |¦  «        dk     r|                     dg¦  «         t          j        j        d||j        fd|j        i|	¤Ž}|                     |g¦  «         �ŒS|                      d¦  «         | j	                             |¦  «         dS )aS  Update the Attention nodes to DecoderMaskedSelfAttention.

    Args:
        subg (GraphProto): GraphProto of the decoder subgraph
        is_beam_search (bool): Boolean specifying if the sampling algo is BeamSearch
        switch_attention (bool): Boolean specifying if `Attention` is to be switched with `DecoderMaskedSelfAttention`
    Ú
beam_widthr\   rz  Úcache_indirection©Ú
batch_sizer‰  rw  rÁ   ©rV   Ú	num_headsÚscaleÚmask_filter_valuerf  r{  Úqkv_hidden_sizesFÚunidirectionalzRemoving attribute: zB from Attention node while switching to DecoderMaskedSelfAttentionrW  r   r·   rX  ÚDecoderMaskedSelfAttentionr½   r3  T)r  rÁ   r‡   r«   r8  r9  r   r  r|  r3  r¿   rk  Úcopyr‹   r¯   r‰   r}  r›   r½   )ru  r†  r‡  r€  Ú_irs  Ú'decoder_masked_attention_supported_attrr‚  r3  rh  Úkr„  s               r(   Ú4update_decoder_subgraph_use_decoder_masked_attentionr˜  ¢  sØ  € ð ð &Øˆ
Ý ¤
Ñ+Ô+ð 	$ð 	$‰FˆB�Ø×Ò˜r˜dÑ#Ô#Ð#Ð#ð 	×Ò�4œ;×=Ò=¸lÍDÔL\ÔLbÐklÐjmÐ=ÑnÔnÐoÑpÔpÐpØ×Òå”×2Ò2Ø'ÝÔ$Ô*ØEÐEÐEð 3ñ ô ðñ	
ô 	
ð 	
ð 	�Š˜Ñ Ô Ð ØŒ
×Ò˜*Ñ%Ô%Ð%àñ 4$ð3
ð 3
ð 3
Ð/ð ˆ	Ø”Ið (	%ñ (	%ˆDØŒ|˜{Ò*Ñ*Ý" 4™œ�ØŸš™œð &ð &�Að Ð.Ò.Ð.Ø$˜u˜u˜uàÐ GÐGÐGð Ð 0Ò0Ð0Ý"ŸNšNØ |°qÐ |Ð |Ð |ñô ð ð # 1˜Iøà�Ø—
’
˜4œ:Ñ&Ô&Ð&ð "ð :Ý˜c™(œ( Qš,˜,ØŸ
š
 B 4Ñ(Ô(Ð(õ ˜c™(œ( Qš,˜,å˜3‘x”x !’|�|ØŸ
š
 L >Ñ2Ô2Ð2Ý˜3‘x”x !’|�|ØŸ
š
Ð$7Ð#8Ñ9Ô9Ð9å”{Ô,Ø0ØØ”Kðð ð œð	ð
 ðð �ð ×Ò˜d˜VÑ$Ô$Ð$Ñ$Ø�Š˜ÑÔÐØŒ	×Ò˜Ñ#Ô#Ð#àˆ4r*   c                 ó|  — t          ¦   «         }g }d„ t          | j        ¦  «        D ¦   «         }i }i }| j        D ]E}|j        D ]*}|r&||vr|g||<   Œ||                              |¦  «         Œ+|j        D ]	}|r|||<   Œ
ŒF| j        D �]1}|j        dk    �r"|j        d         r|j        d         sŒ*|j        d         |j        d         }
}	d}d|
v r;| j        D ]2}|j        dk    r%|j        d         |
k    r|j        d         j        } nŒ3n| j	        D ]}|j
        |
k    r|} nŒ|€Œ¡t          j                             |¦  «        }|j        dk    �re|                     ¦   «         dv �rN|j        d         |v �r>||	         }|j        d	k    r|j        d         s�Œ|j        d         |v rÈ|j        d                              d
¦  «        s |j        d                              d¦  «        rˆ|                     ¦   «         dk    rp|                     |j        d         ¦  «         |                     |¦  «         t%          ||j        d                  ¦  «        dk    r|                     |¦  «         �Œì|j        d         |vr�Œý||j        d                  }|j        dk    r|j        d         s�Œ*||j        d                  }|j        dk    r|j        d         s�ŒW|j        d         |v rË|j        d                              d
¦  «        s |j        d                              d¦  «        r‹|                     ¦   «         dk    rs|                     |j        d         ¦  «         |                     |||g¦  «         t%          ||j        d                  ¦  «        dk    r|                     |¦  «         �Œ1�Œ3||fS )az  Correct graph which originally use dim of past_seq_len from input_ids's shape which is fixed to max_seq_len after
       shared past/present buffer

    Args:
        subg (GraphProto): GraphProto of the decoder subgraph
    return:
        tensor_names_to_rename : set of tensor names which is equal to past_sequence_length
        nodes_to_remove : list of node to remove
    c                 ó$   — i | ]\  }}|j         |“ŒS r1   ©r½   )râ   ÚindexÚinps      r(   ú
<dictcomp>z+find_past_seq_len_usage.<locals>.<dictcomp>  s    € ÐQÐQÐQ©Z¨U°C˜œ 5ÐQÐQÐQr*   ÚGatherr\   r   NÚ	Constant_ÚConstant>   r\   r¶   ÚShaper  r  r¶   ÚReshaperµ   )Úsetr  rÁ   r3  rˆ   r›   r¿   rc  r\  r0  r½   r«   r6  rÍ   ÚsizeÚitemÚ
startswithÚaddr‰   r‡   )ru  Útensor_names_to_renameÚnodes_to_removeÚgraph_input_namesÚinput_name_to_nodesr¾   r3  Ú
input_nameÚoutput_nameÚshape_tensor_nameÚshape_index_nameÚini_gather_indicesÚ
const_noderR  Úgather_indices_arrÚ
shape_nodeÚreshape_nodeÚtranspose_nodes                     r(   Úfind_past_seq_len_usager·  ø  st  € õ !™UœUÐØ€OàQÐQ½9ÀTÄZÑ;PÔ;PÐQÑQÔQÐàÐØÐØ”	ð 	8ð 	8ˆØœ*ð 	Að 	AˆJØð AØÐ%8Ð8Ð8Ø7;°fÐ'¨
Ñ3Ð3à'¨
Ô3×:Ò:¸4Ñ@Ô@Ð@øØœ;ð 	8ð 	8ˆKØð 8Ø37Ð# KÑ0øð	8ð ”	ð H!ñ H!ˆð Œ<˜8Ò#Ñ#Ø”:˜a”=ð ¨¬
°1¬ð Øð 48´:¸a´=À$Ä*ÈQÄ-Ð/ÐØ!%ÐØÐ.Ð.Ð.à"&¤)ð ð �JØ!Ô)¨ZÒ7Ð7¸JÔ<MÈaÔ<PÐTdÒ<dÐ<dØ-7Ô-AÀ!Ô-DÔ-FÐ*Ø˜øøð #Ô.ð ð �FØ”{Ð&6Ò6Ð6Ø-3Ð*Ø˜ð 7ð "Ð)ØÝ!%Ô!2×!;Ò!;Ð<NÑ!OÔ!OÐð #Ô'¨1Ò,Ñ,Ø&×+Ò+Ñ-Ô-°Ð7Ñ7Ø”J˜q”MÐ%8Ð8Ñ8à0Ð1BÔC�
Ø"Ô*¨gÒ5Ð5¸*Ô:JÈ1Ô:MÐ5Ùð Ô$ QÔ'Ð+<Ð<Ð<à"Ô(¨Ô+×6Ò6Ð7GÑHÔHð =ð &Ô+¨AÔ.×9Ò9Ð:LÑMÔMð =ð
 +×/Ò/Ñ1Ô1°QÒ6Ð6ð +×.Ò.¨t¬{¸1¬~Ñ>Ô>Ð>Ø#×*Ò*¨4Ñ0Ô0Ð0ÝÐ.¨zÔ/@ÀÔ/CÔDÑEÔEÈÒJÐJØ'×.Ò.¨zÑ:Ô:Ð:Ù àÔ# AÔ&Ð.AÐAÐAÙØ2°:Ô3CÀAÔ3FÔG�Ø$Ô,°	Ò9Ð9¸lÔ>PÐQRÔ>SÐ9ÙØ!4°\Ô5GÈÔ5JÔ!K�Ø&Ô.°+Ò=Ð=À.ÔBVÐWXÔBYÐ=Ùð #Ô(¨Ô+Ð/@Ð@Ð@à&Ô,¨QÔ/×:Ò:Ð;KÑLÔLð Að *Ô/°Ô2×=Ò=Ð>PÑQÔQð Að
 +×/Ò/Ñ1Ô1°QÒ6Ð6ð +×.Ò.¨t¬{¸1¬~Ñ>Ô>Ð>Ø#×*Ò*¨D°*¸lÐ+KÑLÔLÐLÝÐ.¨~Ô/DÀQÔ/GÔHÑIÔIÈQÒNÐNØ'×.Ò.¨~Ñ>Ô>Ð>Ù ùà! ?Ð2Ð2r*   r°   Úpast_seq_len_namec                 ó2  — d}t          t          d„ | j        j        j        ¦  «        ¦  «        }|D ]€}t          |j        ¦  «        dk     r2|j                             d¦  «         t          |j        ¦  «        dk     °2|j                             |¦  «         |j                             |¦  «         Œ�| j        j        j                             t          j	         
                    |t          j        g d¢¬¦  «        ¦  «         |                      ¦   «          | S )NrŠ  c                 ó   — | j         dk    S ©NÚMultiHeadAttention©r¿   ©r3  s    r(   ú<lambda>z.add_cache_indirection_to_mha.<locals>.<lambda>d  ó   € ¨¬Ð9MÒ)M€ r*   r·   r   ©rŒ  r‰  Úmax_sequence_lengthrz  )ÚlistÚfilterr°   r¼   r3  r‰   rÁ   rˆ   r«   r8  r9  r   r  Útopological_sort)r°   r¸  Úcache_indirection_nameÚ	mha_nodesr3  s        r(   Úadd_cache_indirection_to_mharÈ  a  s  € à0ÐÝ•VÐMÐMÈuÌ{ÔO`ÔOeÑfÔfÑgÔg€IØð 2ð 2ˆõ �$”*‰oŒo Ò!Ð!ØŒJ×Ò˜bÑ!Ô!Ð!õ �$”*‰oŒo Ò!Ð!àŒ
×ÒÐ+Ñ,Ô,Ð,ØŒ
×ÒÐ0Ñ1Ô1Ð1Ð1à	„KÔÔ×"Ò"ÝŒ×*Ò*Ø"¥KÔ$5Ð=pÐ=pÐ=pð 	+ñ 	
ô 	
ñô ð ð
 
×ÒÑÔÐØ€Lr*   r¹   Úskip_node_idxsc                 ó²  — d}g }t          t          d„ | j        j        j        ¦  «        ¦  «        }t          |¦  «        D �]\\  }}||v rŒd}|j        D ]}	|	j        dk    r	|	j        } nŒ|}
|
dk    r3| j        j        j	        D ]!}|j        |j
        d         k    r	|j        }
 nŒ"d}| j        j        j
        D ];}|j        |j
        d         k    r#|j        j        j        j        d         j        } nŒ<t#          |j        ¦  «        dk     r2|j                             d	¦  «         t#          |j        ¦  «        dk     °2|› d
|dz  › �}|j                             |¦  «         |                     t(          j                             ||
d|d|g¬¦  «        ¦  «         �Œ^| j        j        j                             |¦  «         |                      ¦   «          | S )NÚoutput_cross_qkc                 ó   — | j         dk    S r»  r½  r¾  s    r(   r¿  z&add_output_qk_to_mha.<locals>.<lambda>z  rÀ  r*   r   rŽ  rï   Útarget_sequence_lengthr\   r¶   r   Ú_rŒ  Úsequence_lengthrz  )rÃ  rÄ  r°   r¼   r3  r  rc  r½   rö   r0  rÁ   rÃ   r8   r  r7  rr  ro  r‰   r›   rˆ   r«   r8  r9  r‡   rÅ  )r°   r¹   rÉ  Úoutput_qk_basenameÚ
output_qksrÇ  Úidxr3  rŽ  ÚattÚoutput_qk_dtyperö   rÍ  Úoutput_qk_names                 r(   Úadd_output_qk_to_mharÖ  v  s$  € à*ÐØ€JÝ•VÐMÐMÈuÌ{ÔO`ÔOeÑfÔfÑgÔg€IÝ˜yÑ)Ô)ð (
ñ (
‰	ˆˆTà�.Ð Ð Øð ˆ	Ø”>ð 	ð 	ˆCØŒx˜;Ò&Ð&ØœE�	Ø�ð 'ð
  ˆØ˜aÒÐØ”[Ô&Ô2ð ð �Ø”6˜TœZ¨œ]Ò*Ð*Ø&'¤k�OØ�Eð +ð
 ":ÐØ”Ô"Ô(ð 	ð 	ˆAØŒv˜œ AœÒ&Ð&Ø)*¬Ô);Ô)AÔ)EÀaÔ)HÔ)RÐ&Ø�ð 'õ �$”+ÑÔ Ò"Ð"ØŒK×Ò˜rÑ"Ô"Ð"õ �$”+ÑÔ Ò"Ð"ð /Ð;Ð;°¸±Ð;Ð;ˆØŒ×Ò˜>Ñ*Ô*Ð*Ø×ÒÝŒK×.Ò.ØØØ# YÐ0AÐCYÐZð /ñ ô ñ	
ô 	
ð 	
ñ 	
ð 
„KÔÔ×#Ò# JÑ/Ô/Ð/Ø	×ÒÑÔÐØ€Lr*   c                 ó†  ‡‡— d}d}d}t          t          d„ | j        j        j        ¦  «        ¦  «        d         }|                      |g d¢g d¢¦  «        }|                      |dd	gdd
g¦  «        }|�|}n!|�|}nt                               d¦  «         d S |d         }|j        dk    �ro|d         }	|                      |	ddgddg¦  «        Š‰€t                               d¦  «         d S |                      |	g d¢g d¢¦  «        }
|
€t                               d¦  «         d S |
d         }‰|
d
d …         k    rt                               d¦  «         d S t          t          ˆfd„| j        j        j        ¦  «        ¦  «        d         }| j        j        j         	                    |¦  «         | j        j        j         	                    ‰d         ¦  «         | j        j        j         	                    ‰d
         ¦  «         ||	j
        d<   ||j
        d<   �n"|                      |g d¢g d¢¦  «        }|€t                               d¦  «         d S |d
         }|                      |g d¢g d¢¦  «        Š‰€t                               d¦  «         d S ‰d         }|dd …         ‰d
d …         k    rt                               d¦  «         d S t          t          ˆfd„| j        j        j        ¦  «        ¦  «        d         }| j        j        j         	                    |¦  «         t          t          ˆfd„| j        j        j        ¦  «        ¦  «        d         }| j        j        j         	                    |¦  «         | j        j        j         	                    ‰d
         ¦  «         | j        j        j         	                    ‰d         ¦  «         | j        j        j         	                    ‰d          ¦  «         | j        j        j         	                    ‰d!         ¦  «         ||j
        d<   ||j
        d<   | j        j        j
                             t          j                             |t          j        d
g¬"¦  «        ¦  «         t          j                             d#|g|g|                      d#¦  «        ¬$¦  «        }t          j                             |t          j        g ¬"¦  «        }t          j                             d%|g|g|                      d%¦  «        t          j        ¬&¦  «        }t          j                             |t          j        g ¬"¦  «        }| j        j        j                             ||g¦  «         | j        j        j                             ||g¦  «         |                      ¦   «          | |fS )'Nry  Úpast_seq_len_int32Úpast_seq_len_int64c                 ó   — | j         dk    S )NÚLayerNormalizationr½  )Úns    r(   r¿  z*fix_past_sequence_length.<locals>.<lambda>Ð  s   €  ¤Ð.BÒ!B€ r*   r   )ÚAddrŸ  ÚTileÚExpandÚ	UnsqueezeÚRange)r   r\   r\   r   r   r   rÝ  ÚSlicer\   zBCannot identify base path for fixing past_sequence_length subgraphrb   rá  rŸ  r¢  zDCannot identify gather path for fixing past_sequence_length subgraph)rÝ  rŸ  r¢  ©r\   r   r   zACannot identify add path for fixing past_sequence_length subgraphz]Gather path and add path do not share the same nodes for calculating the past_sequence_lengthc                 óH   •— | j         d         ‰d         j        d         k    S ©Nr   r\   ©r›   rÁ   )rÜ  Úgather_paths    €r(   r¿  z*fix_past_sequence_length.<locals>.<lambda>  s   ø€ °1´8¸A´;À+ÈaÄ.ÔBVÐWXÔBYÒ3Y€ r*   )rà  rÝ  rŸ  r¢  r£  rµ   )r¶   r   r   r   r   r   zGCannot identify input_ids path for fixing past_sequence_length subgraph)rà  rŸ  r¢  r£  rµ   )r\   r   r   r   r   zFCannot identify past_key path for fixing past_sequence_length subgraphr¶   ziThe input_ids path and past_key path do not share the same nodes for calculating the past_sequence_lengthc                 óH   •— | j         d         ‰d         j        d         k    S rå  ræ  ©rÜ  Úpast_key_paths    €r(   r¿  z*fix_past_sequence_length.<locals>.<lambda>'  s!   ø€ °1´8¸A´;À-ÐPQÔBRÔBXÐYZÔB[Ò3[€ r*   c                 óH   •— | j         d         ‰d         j        d         k    S )Nr   éþÿÿÿr\   ræ  ré  s    €r(   r¿  z*fix_past_sequence_length.<locals>.<lambda>)  s!   ø€ °A´H¸Q´KÀ=ÐQSÔCTÔCZÐ[\ÔC]Ò4]€ r*   rï   r^   rz  ÚSqueeze©ÚinputsÚoutputsr½   ÚCast©rï  rð  r½   Úto)rÃ  rÄ  r°   r¼   r3  Úmatch_parent_pathr‹   rŒ   r¿   r4  rÁ   rˆ   r«   r8  r9  r   r  r}  Úcreate_node_nameÚINT64r‡   r5  rÅ  )r°   r¸  rØ  rÙ  r3  Úbase_path_hfÚbase_path_oaiÚ	base_pathÚ	base_nodeÚ
range_nodeÚadd_pathÚadd_nodeÚconstant_in_gatherÚinput_ids_pathÚunsqueeze_nodeÚconstant_in_reshapeÚsqueeze_nodeÚsqueeze_outputÚ	cast_nodeÚcast_outputrç  rê  s                       @@r(   Úfix_past_sequence_lengthr  ª  sï  øø€ ðD /ÐØ-ÐØ-Ðå•ÐBÐBÀEÄKÔDUÔDZÑ[Ô[Ñ\Ô\Ð]^Ô_€Dà×*Ò*ØØAÐAÐAØÐÐñô €Lð
 ×+Ò+ØØ	�ÐØ	
ˆAˆñô €Mð
 ÐØ ˆ	ˆ	Ø	Ð	"Ø!ˆ	ˆ	å�ŠÐXÑYÔYÐYØˆØ˜"”€IàÔ˜GÒ#Ñ#à˜r”]ˆ
à×-Ò-ØØ�wÐØ�ˆFñ
ô 
ˆð
 ÐÝ�KŠKÐ^Ñ_Ô_Ð_ØˆFà×*Ò*ØØ&Ð&Ð&ØˆIˆIñ
ô 
ˆð
 ÐÝ�KŠKÐ[Ñ\Ô\Ð\ØˆFØ˜A”;ˆà˜( 1 2 2œ,Ò&Ð&Ý�KŠKÐwÑxÔxÐxØˆFõ "¥&Ð)YÐ)YÐ)YÐ)YÐ[`Ô[fÔ[lÔ[qÑ"rÔ"rÑsÔsÐtuÔvÐØŒÔÔ×%Ò%Ð&8Ñ9Ô9Ð9ØŒÔÔ×%Ò% k°!¤nÑ5Ô5Ð5ØŒÔÔ×%Ò% k°!¤nÑ5Ô5Ð5ð 1ˆ
Ô˜ÑØ.ˆŒ�qÑÑð ×0Ò0ØØKÐKÐKØÐÐñ
ô 
ˆð
 Ð!Ý�KŠKÐaÑbÔbÐbØˆFØ! !Ô$ˆà×/Ò/ØØDÐDÐDØˆOˆOñ
ô 
ˆð
 Ð Ý�KŠKÐ`ÑaÔaÐaØˆFØ& qÔ)ˆà˜!˜"˜"Ô ¨q¨r¨rÔ!2Ò2Ð2Ý�KŠKØ{ñô ð ð ˆFõ "¥&Ð)[Ð)[Ð)[Ð)[Ð]bÔ]hÔ]nÔ]sÑ"tÔ"tÑuÔuÐvwÔxÐØŒÔÔ×%Ò%Ð&8Ñ9Ô9Ð9Ý"¥6Ð*]Ð*]Ð*]Ð*]Ð_dÔ_jÔ_pÔ_uÑ#vÔ#vÑwÔwØô
Ðð 	ŒÔÔ×%Ò%Ð&9Ñ:Ô:Ð:ØŒÔÔ×%Ò% m°AÔ&6Ñ7Ô7Ð7ØŒÔÔ×%Ò% m°AÔ&6Ñ7Ô7Ð7ØŒÔÔ×%Ò% m°AÔ&6Ñ7Ô7Ð7ØŒÔÔ×%Ò% m°AÔ&6Ñ7Ô7Ð7ð #5ˆÔ˜QÑØ.ˆŒ�qÑð 
„KÔÔ×"Ò"ÝŒ×*Ò*Ð+<½kÔ>OÐXYÐWZÐ*Ñ[Ô[ñô ð õ
 ”;×(Ò(ØØ!Ð"Ø#Ð$Ø×#Ò# IÑ.Ô.ð	 )ñ ô €Lõ ”[×7Ò7Ð8JÍKÔL]ÐegÐ7ÑhÔh€NÝ”×%Ò%ØØ"Ð#Ø#Ð$Ø×#Ò# FÑ+Ô+ÝÔð &ñ ô €Iõ ”+×4Ò4Ð5GÍÔIZÐbdÐ4ÑeÔe€Kð 
„KÔÔ×!Ò! <°Ð";Ñ<Ô<Ð<Ø	„KÔÔ ×'Ò'¨¸Ð(EÑFÔFÐFØ	×ÒÑÔÐØÐ#Ð#Ð#r*   c                 óê  — d}d}| j         j        j                             t          j                             |t          j        dg¬¦  «        t          j                             |t          j        g d¢¬¦  «        g¦  «         t          t          d„ | j         j        j        ¦  «        ¦  «        }t          |¦  «        D �]\  }}d}|j        D ]}|j        dk    r	|j        } nŒd	|d
z  › �}	t          j                             |	t          j        d|ddg¬¦  «        }
|d
z  dk    r$| j         j        j                             |
¦  «         t          j                             d|j        d         |j        d         |j        d
         ddt)          |j        ¦  «        dk    r|j        d         ndt)          |j        ¦  «        dk    r|j        d         nd||||j        d         g|j        d         t)          |j        ¦  «        dk    r|j        d         ndt)          |j        ¦  «        dk    r|j        d
         nd|d
z  dk    r|	ndg|j                             dd¦  «        d||d
z  d¬¦  «        }|d
z  dk    r|j                             d¦  «         | j         j        j                             |¦  «         | j         j        j                             |g¦  «         �Œ!|                      ¦   «          | S )Nr‰  rŠ  r\   rz  rÁ  c                 ó   — | j         dk    S r»  r½  r¾  s    r(   r¿  z(replace_mha_with_dmmha.<locals>.<lambda>b  rÀ  r*   r   rŽ  Úoutput_cross_qk_r¶   rŒ  zencode_sequence_length / 2ÚDecoderMaskedMultiHeadAttentionr   r^   rV  rW  rï   r¼  úcom.microsoft)rï  rð  r½   rf  rŽ  Ú	output_qkrV   )r°   r¼   rÁ   r‡   r«   r8  r9  r   r  rÃ  rÄ  r3  r  rc  r½   rö   r  r›   rˆ   r}  r‰   Úreplacer4  rÅ  )r°   r¸  r‰  rŠ  rÇ  rÒ  r3  rŽ  rÓ  Úqk_output_nameÚ	qk_outputÚ
dmmha_nodes               r(   Úreplace_mha_with_dmmhar  S  sû  € à€JØ+Ðà	„KÔÔ×"Ò"åŒK×.Ò.¨z½;Ô;LÐUVÐTWÐ.ÑXÔXÝŒK×.Ò.Ø!¥;Ô#4Ð<oÐ<oÐ<oð /ñ ô ð	
ñô ð õ •VÐMÐMÈuÌ{ÔO`ÔOeÑfÔfÑgÔg€IÝ˜yÑ)Ô)ð 14ñ 14‰	ˆˆTàˆ	Ø”>ð 	ð 	ˆCØŒx˜;Ò&Ð&ØœE�	Ø�ð 'ð
 7¨C°1©HÐ6Ð6ˆÝ”K×6Ò6Ø�KÔ-°lÀIÈqÐRnÐ5oð 7ñ 
ô 
ˆ	ð �‰7�aŠ<ˆ<ØŒKÔÔ$×+Ò+¨IÑ6Ô6Ð6õ ”[×*Ò*Ø-à”
˜1”Ø”
˜1”Ø”
˜1”ØØÝ!$ T¤Z¡¤°1Ò!4Ð!4�”
˜1”�¸"Ý!$ T¤Z¡¤°1Ò!4Ð!4�”
˜1”�¸"Ø!ØØ!Ø”
˜1”ðð ”˜A”Ý"% d¤j¡/¤/°AÒ"5Ð"5�”˜A”�¸2Ý"% d¤j¡/¤/°AÒ"5Ð"5�”˜A”�¸2Ø"%¨¡'¨Q¢, ,��°Bð	ð ”×"Ò"Ð#7Ð9ZÑ[Ô[Ø"ØØ˜Q‘wØ&'ð3 +ñ 
ô 
ˆ
ð6 �‰7�aŠ<ˆ<àÔ×$Ò$ RÑ(Ô(Ð(àŒÔÔ×%Ò% dÑ+Ô+Ð+ØŒÔÔ×%Ò% z lÑ3Ô3Ð3Ñ3à	×ÒÑÔÐØ€Lr*   r\   rb   Ú	attn_maskÚkv_num_headsÚ
world_sizeÚwindow_sizec                 ó  — |                       t          j                             dt          j        dgdg¬¦  «        ¦  «         t          j                             d|dg|dz   g|                      d¦  «        ¬¦  «        }t          j                             d|dz   dgdg|                      d¦  «        ¬¦  «        }t          j                             d	dgd
g|                      d	¦  «        t          j        ¬¦  «        }t          j                             d|g|dz   g|                      d¦  «        ¬¦  «        }t          j                             d|dz   dgdg|                      d¦  «        d¬¦  «        }	t          j                             d	dgdg|                      d	¦  «        t          j        ¬¦  «        }
| j	        j
        j                             |||||	|
g¦  «         t          t          d„ | j	        j
        j        ¦  «        ¦  «        }t          |¦  «        D �]æ\  }}|                      |g d¢g d¢¦  «        }|                      |ddgddg¦  «        }d\  }}}|�|\  }}}n|�|\  }}|                      |g d¢g d¢¦  «        }|                      |ddgddg¦  «        }d\  }}}|�|\  }}}n|�|\  }}|                      |ddgddg¦  «        }|                      |dgdg¦  «        }d\  }}|�|\  }}n
|�|d         }d}|�|�|j        D ]}|j        dk    r|j        }Œd}|j        D ]}|j        dk    r|j        }Œ|j        d         |j        d         k    o|j        d         |j        d         k    }|d uo|d uo|d u} |d u o|d u o|d u }!d\  }"}#}$|�rê| s|!�råt+          j        |                      |j        d         ¦  «        ¦  «        }%t+          j        |                      |j        d         ¦  «        ¦  «        }&t+          j        |                      |j        d         ¦  «        ¦  «        }'|%j        d          }(t3          j        |%|&|'fd¬!¦  «                             |(d"|(z  ¦  «        })t          j                             |)d#|› �¬$¦  «        })|                       |)¦  «         t          j                             d|j        d         |)j        g|)j        › d%�g|                      d¦  «        ¬¦  «        }*| j	        j
        j                             |*g¦  «         | j	        j
        j                             |¦  «         | j	        j
        j                             |¦  «         | j	        j
        j                             |¦  «         |*j        d         }"| �ræt+          j        |                      |j        d         ¦  «        ¦  «        }+t+          j        |                      |j        d         ¦  «        ¦  «        },t+          j        |                      |j        d         ¦  «        ¦  «        }-|+j        d          }(t3          j        |+|,|-fd¬!¦  «                             d"|(z  ¦  «        }.t          j                             |.d&|› �¬$¦  «        }.|                       |.¦  «         t          j                             d|*j        d         |.j        g|.j        › d%�g¬'¦  «        }/| j	        j
        j                             |/g¦  «         | j	        j
        j                             |¦  «         | j	        j
        j                             |¦  «         | j	        j
        j                             |¦  «         |/j        d         }"n'|j        d         }"|j        d         }#|j        d         }$t          j                             d(|"|#|$|j        d)         |j        d*         |j        d         |
j        d         |�|j        d         nd+|�|j        d"         nd+g	|j        |j                              d,d(¦  «        d-||z  |dk    r||z  n||z  |tC          |d uo|d u¦  «        |¬.¦
  «
        }0| j	        j
        j                             |¦  «         | j	        j
        j                             |0g¦  «         |�$| j	        j
        j                             |¦  «         |�$| j	        j
        j                             |¦  «         �Œè| S )/NÚoner\   ©r½   rÃ   rÅ   ÚvalsÚ	ReduceSumÚ	_row_sumsrî  ÚSubÚseqlens_k_int64rñ  Ú	seqlens_krò  r¢  Ú_shaperŸ  Útotal_seq_len_int64r   )rï  rð  r½   r»   Útotal_seq_lenc                 ó   — | j         dk    S r»  r½  r¾  s    r(   r¿  z&replace_mha_with_gqa.<locals>.<lambda>   rÀ  r*   )ÚRotaryEmbeddingrÝ  r´   )r   r   r   r#  r´   )NNNrã  rÝ  r¶   ©NNÚinterleavedrŽ  )r   r   r   rb   rº   rï   ÚQKV_Weight_r›  Ú_outputÚ	QKV_Bias_)rï  rð  ÚGroupQueryAttentionrV  rW  r   r¼  r  )	rï  rð  r½   rf  rŽ  r  Úlocal_window_sizeÚ	do_rotaryÚrotary_interleaved)"Úadd_initializerr«   r8  Úmake_tensorr   rö  r}  rõ  r  r°   r¼   r3  r‡   rÃ  rÄ  r  rô  rc  r½   rö   rÁ   r   rÍ   rÀ   r7  rÉ   ÚstackÚreshaper6  Ú
from_arrayr4  r›   r  ru   )1r°   r  r  r  r  Úreduce_sum_nodeÚsub_nodeÚseqlen_k_cast_noder´  Úgather_nodeÚtotal_seqlen_cast_noderÇ  rÒ  r3  Úq_path_1Úq_path_2Úq_rotaryÚq_addÚq_matmulÚk_path_1Úk_path_2Úk_rotaryÚk_addÚk_matmulÚv_path_1Úv_path_2Úv_addÚv_matmulr%  rÓ  rŽ  Úroot_input_is_sameÚall_paths_have_biasÚall_paths_have_no_biasÚq_input_to_attentionÚk_input_to_attentionÚv_input_to_attentionÚqwÚkwÚvwrr  Ú
qkv_weightÚpacked_matmul_nodeÚqbÚkbÚvbÚqkv_biasÚpacked_add_nodeÚgqa_nodes1                                                    r(   Úreplace_mha_with_gqarV  š  s»	  € ð& 
×ÒÝŒ×ÒØÝ!Ô'Ø�Ø�ð	 	 ñ 	
ô 	
ñô ð õ ”k×+Ò+ØØ˜5Ð!Ø˜[Ñ(Ð)Ø×#Ò# KÑ0Ô0ð	 ,ñ ô €Oõ Œ{×$Ò$ØØ˜KÑ'¨Ð/Ø"Ð#Ø×#Ò# EÑ*Ô*ð	 %ñ ô €Hõ œ×.Ò.ØØ!Ð"Ø�Ø×#Ò# FÑ+Ô+ÝÔð /ñ ô Ðõ ”×&Ò&ØØˆ{Ø˜XÑ%Ð&Ø×#Ò# GÑ,Ô,ð	 'ñ ô €Jõ ”+×'Ò'ØØ˜HÑ$ eÐ,Ø&Ð'Ø×#Ò# HÑ-Ô-Øð (ñ ô €Kõ "œ[×2Ò2ØØ%Ð&Ø Ð!Ø×#Ò# FÑ+Ô+ÝÔð 3ñ ô Ðð 
„KÔÔ×!Ò!àØØØØØ"ð	
ñ	ô 	ð 	õH •VÐMÐMÈuÌ{ÔO`ÔOeÑfÔfÑgÔg€IÝ˜yÑ)Ô)ð B4ñ B4‰	ˆˆTà×*Ò*¨4Ð1UÐ1UÐ1UÐW`ÐW`ÐW`ÑaÔaˆØ×*Ò*¨4Ð2CÀXÐ1NÐQRÐTUÐPVÑWÔWˆà$4Ñ!ˆ�%˜ØÐØ(0Ñ%ˆH�e˜X˜XØÐ!Ø!)ÑˆH�hð ×*Ò*¨4Ð1UÐ1UÐ1UÐW`ÐW`ÐW`ÑaÔaˆØ×*Ò*¨4Ð2CÀXÐ1NÐQRÐTUÐPVÑWÔWˆà$4Ñ!ˆ�%˜ØÐØ(0Ñ%ˆH�e˜X˜XØÐ!Ø!)ÑˆH�hð ×*Ò*¨4°%¸Ð1BÀQÈÀFÑKÔKˆØ×*Ò*¨4°(°¸a¸SÑAÔAˆà$‰ˆˆxØÐØ&‰OˆE�8�8ØÐ!Ø ”{ˆHð ˆØÐ HÐ$8ØÔ)ð (ð (�Ø”8˜}Ò,Ð,Ø"%¤%�Køð ˆ	Ø”>ð 	"ð 	"ˆCØŒx˜;Ò&Ð&ØœE�	øð &œ^¨AÔ.°(´.ÀÔ2CÒCÐnÈÌÐWXÔHYÐ]eÔ]kÐlmÔ]nÒHnÐð $¨4Ð/Ð[°EÀÐ4EÐ[È%ÐW[ÐJ[ÐØ!&¨$ Ð!R°5¸D°=Ð!RÀUÈdÀ]Ðð LVÑHÐÐ2Ð4HØñ .	6Ð#6ð .	6Ð:Pñ .	6ÝÔ% e×&;Ò&;¸H¼NÈ1Ô<MÑ&NÔ&NÑOÔOˆBÝÔ% e×&;Ò&;¸H¼NÈ1Ô<MÑ&NÔ&NÑOÔOˆBÝÔ% e×&;Ò&;¸H¼NÈ1Ô<MÑ&NÔ&NÑOÔOˆBà”(˜2”,ˆCÝœ 2 r¨2 ,°QÐ7Ñ7Ô7×?Ò?ÀÀQÈÁWÑMÔMˆJÝÔ*×5Ò5°jÐGZÐUXÐGZÐGZÐ5Ñ[Ô[ˆJØ×!Ò! *Ñ-Ô-Ð-å!%¤×!6Ò!6ØØ œ qÔ)¨:¬?Ð;Ø&œOÐ4Ð4Ð4Ð5Ø×+Ò+¨HÑ5Ô5ð	 "7ñ "ô "Ðð ŒKÔÔ"×)Ò)Ð+=Ð*>Ñ?Ô?Ð?ØŒKÔÔ"×)Ò)¨(Ñ3Ô3Ð3ØŒKÔÔ"×)Ò)¨(Ñ3Ô3Ð3ØŒKÔÔ"×)Ò)¨(Ñ3Ô3Ð3Ø#5Ô#<¸QÔ#?Ð ð #ñ AÝ Ô)¨%×*?Ò*?ÀÄÈAÄÑ*OÔ*OÑPÔP�Ý Ô)¨%×*?Ò*?ÀÄÈAÄÑ*OÔ*OÑPÔP�Ý Ô)¨%×*?Ò*?ÀÄÈAÄÑ*OÔ*OÑPÔP�à”h˜r”l�Ýœ8 R¨¨R L°qÐ9Ñ9Ô9×AÒAÀ!ÀcÁ'ÑJÔJ�ÝÔ,×7Ò7¸ÐGXÐSVÐGXÐGXÐ7ÑYÔY�Ø×%Ò% hÑ/Ô/Ð/Ý"&¤+×"7Ò"7ØØ.Ô5°aÔ8¸(¼-ÐHØ (¤Ð6Ð6Ð6Ð7ð #8ñ #ô #�ð
 ”Ô!Ô&×-Ò-¨Ð.?Ñ@Ô@Ð@Ø”Ô!Ô&×-Ò-¨eÑ4Ô4Ð4Ø”Ô!Ô&×-Ò-¨eÑ4Ô4Ð4Ø”Ô!Ô&×-Ò-¨eÑ4Ô4Ð4Ø'6Ô'=¸aÔ'@Ð$øð $,¤?°1Ô#5Ð Ø#+¤?°1Ô#5Ð Ø#+¤?°1Ô#5Ð õ ”;×(Ò(Ø!à$Ø$Ø$Ø”
˜1”Ø”
˜1”Ø"Ô)¨!Ô,Ø&Ô-¨aÔ0Ø&.Ð&:�” Ô"Ð"ÀØ&.Ð&:�” Ô"Ð"Àð
ð ”KØ”×"Ò"Ð#7Ð9NÑOÔOØ"Ø :Ñ-Ø5AÀQÒ5FÐ5F˜) zÑ1Ð1ÈLÐ\fÑLfØ)Ý˜(¨$Ð.ÐG°8À4Ð3GÑHÔHØ*ð) )ñ 
ô 
ˆð, 	ŒÔÔ×%Ò% dÑ+Ô+Ð+ØŒÔÔ×%Ò% x jÑ1Ô1Ð1àÐØŒKÔÔ"×)Ò)¨(Ñ3Ô3Ð3ØÐØŒKÔÔ"×)Ò)¨(Ñ3Ô3Ð3ùà€Lr*   c           	      ó†  ‡ ‡— d}d„ ‰ j         D ¦   «         }|dk     rA||                              d¦  «        s&|dz  }|dk     r||                              d¦  «        ¯&d}t          ‰ j        ¦  «        |z
  dz  }d|z  |z   Šˆˆ fd„t	          |¦  «        D ¦   «         }t          d|› �¦  «         t          ‰ j         ‰         ¦  «        }t          d|› �¦  «         |d	         }|d         }|d         }	d	}
‰ j        D �](}|j        d
k    �r|j         d         |v �r	t          d|j	        › d|j        › �¦  «         |
dz  }
||j         d                  }d|› �}dgdt          |j        ¦  «        z
  z  }| 
                    |¦  «         |j                             |¦  «         |j                             t          j                             dd¦  «        g¦  «         t          j                             |t"          j        ||d|	g¦  «        }‰ j                             |g¦  «         �Œ*|
|k    rt'          d|› d|
› �¦  «        ‚d S )Nr\   c                 ó   — g | ]	}|j         ‘Œ
S r1   r›  ©râ   Úgis     r(   rä   zBupdate_decoder_subgraph_output_cross_attention.<locals>.<listcomp>‹  ó   € Ð6Ð6Ð6 R˜œÐ6Ð6Ð6r*   rï   Úpastr¶   c                 óB   •— i | ]}‰j         |d z  ‰z            j        |“ŒS )r¶   )rÁ   r½   )râ   ÚlayerÚinput_cross_past_0ru  s     €€r(   rž  zBupdate_decoder_subgraph_output_cross_attention.<locals>.<dictcomp>’  s0   ø€ ÐsÐsÐsÐX]˜TœZ¨°©	Ð4FÑ(FÔGÔLÈeÐsÐsÐsr*   z    -- past_key_cross_inputs = zpast_key_cross_0_shape is r   r
  z'    -- add cross QK output from: node: z with output: r	  r   r  z#Did not add cross QK for all layersz vs )rÁ   r§  r‰   r›   r  Úprintrt  r3  r¿   r½   rˆ   r‡   rc  r«   r8  Úmake_attributer9  r   r  r  )ru  Úinput_self_past_0r«  Úoutput_self_present_0Ú
num_layersÚpast_key_cross_inputsÚinput_past_key_cross_0_shapeÚbatch_size_dimÚnum_heads_dimÚcross_seq_len_dimÚnum_layer_output_qkr3  r^  Úcross_attention_out_nameÚappended_namesÚcross_attentionr_  s   `               @r(   Ú.update_decoder_subgraph_output_cross_attentionrn  ˆ  s¦  øø€ ØÐà6Ð6¨4¬:Ð6Ñ6Ô6ÐØ
˜aÒ
Ð
Ð(9Ð:KÔ(L×(WÒ(WÐX^Ñ(_Ô(_Ð
Ø˜QÑÐð ˜aÒ
Ð
Ð(9Ð:KÔ(L×(WÒ(WÐX^Ñ(_Ô(_Ð
àÐå�d”kÑ"Ô"Ð%:Ñ:¸qÑ@€JØ˜Z™Ð*;Ñ;ÐØsÐsÐsÐsÐsÕafÐgqÑarÔarÐsÑsÔsÐÝ	Ð
CÐ,AÐ
CÐ
CÑDÔDÐDå#+¨D¬JÐ7IÔ,JÑ#KÔ#KÐ Ý	Ð
EÐ'CÐ
EÐ
EÑFÔFÐFØ1°!Ô4€NØ0°Ô3€MØ4°QÔ7ÐàÐØ”	ð 2ñ 2ˆØŒLÐ=Ò=Ñ=ÀDÄJÈqÄMÐUjÐDjÑDjÝÐb¸D¼IÐbÐbÐUYÔU`ÐbÐbÑcÔcÐcØ 1Ñ$ÐØ)¨$¬*°Q¬-Ô8ˆEØ'A¸%Ð'AÐ'AÐ$Ø ˜T Q­¨T¬[Ñ)9Ô)9Ñ%9Ñ:ˆNØ×!Ò!Ð":Ñ;Ô;Ð;ØŒK×Ò˜~Ñ.Ô.Ð.ØŒN×!Ò!¥4¤;×#=Ò#=¸kÈ1Ñ#MÔ#MÐ"NÑOÔOÐOå"œk×@Ò@Ø(ÝÔ!Ø °Ð3DÐEñô ˆOð
 ŒK×Ò Ð0Ñ1Ô1Ð1ùØ˜jÒ(Ð(ÝÐd¸zÐdÐdÐObÐdÐdÑeÔeÐeð )Ð(r*   c           
      óz  — d}d„ | j         D ¦   «         }|dk     rA||                              d¦  «        s&|dz  }|dk     r||                              d¦  «        ¯&d}t          t          | j         ¦  «        |z
  dz  ¦  «        }d|z  |z   }g }g }| j        D ]#}|j        dk    r|                     |g¦  «         Œ$t          |¦  «        |k     rdS d }	| j        D ]}|j        d	k    r|}	 nŒg d
¢}
d}t          | ¦  «        \  }}t          |¦  «        dk    r«|D ]}t          d|› d|› d�¦  «         Œ|D ]!}t          d|j        › d|j	        › �¦  «         Œ"t          j                             ddgdgd¬¦  «        }t          j                             ddg|gdt          j        ¬¦  «        }|                     ||g¦  «         | j        D �]È}t          |j        ¦  «        dk    ry|	�w|j        d         |	j         d         k    r[t          j                             ddgdgdt          j        ¬¦  «        }|j        d         |j         d<   |                     |g¦  «         |j        dk    �ræt!          |¦  «        }|                     ¦   «         D ]	}||
vr||= Œ
|j         d         |j         d         |j         d         g}|                     t          |j         ¦  «        dk    r|j         d         ndg¦  «         |                     t          |j         ¦  «        dk    r|j         d         ndg¦  «         |                     t          |j         ¦  «        dk    r|j         d         ndg¦  «         |                     t          |j         ¦  «        dk    r|j         d         ndg¦  «         |                     dg¦  «         |                     d g¦  «         |                     d!g¦  «         |                     t          |j         ¦  «        dk    r|j         d         ndg¦  «         d|d"<   t          j        j        d#||j        fd$|j	        i|¤Ž}||vr>t%          |j         ¦  «        D ]\  }}||v r
||j         |<   Œ|                     |g¦  «         �ŒÊ|                      d%¦  «         | j                             |¦  «         d&„ | j         D ¦   «         }g }t%          | j         ¦  «        D ]‚\  }}||k    ra||k     r[t)          |¦  «        }t          j                             |j	        |j        j        j        |d         |d         d'|d         g¬(¦  «        }|                     |g¦  «         Œƒd|vrF|                     t          j                             dt          j        j        dg¬)¦  «        g¦  «         d |vrF|                     t          j                             d t          j        j        dg¬)¦  «        g¦  «         d!|vrG|                     t          j                             d!t          j        j        g d*¢¬)¦  «        g¦  «         |                      d+¦  «         | j                              |¦  «         g }t%          | j        ¦  «        D ]|\  }}||k    r[t)          |¦  «        }t          j                             |j	        |j        j        j        |d         |d         d'|d         g¬(¦  «        }|                     |g¦  «         Œ}|                      d,¦  «         | j                             |¦  «         d-S ).Nr\   c                 ó   — g | ]	}|j         ‘Œ
S r1   r›  rY  s     r(   rä   zSupdate_decoder_subgraph_share_buffer_and_use_decoder_masked_mha.<locals>.<listcomp>´  r[  r*   rï   r\  r^   r¶   r¼  FÚRelativePositionBiasr�  Ú#past_sequence_length_squeezed_int64r   zFound tensor name `z` to be renamed to `ú`zFound node to remove: type = z	, name = rí  ry  Úpast_sequence_length_squeezedÚ!node_past_sequence_length_squeezer›  rñ  Ú&node_past_sequence_length_squeeze_cast)r½   ró  Úpast_sequence_length_int64Úpast_sequence_length_castr   rU  rV  rW  r‰  rŠ  rV   r
  r½   r3  c                 ó   — g | ]	}|j         ‘Œ
S r1   r›  )râ   r�  s     r(   rä   zSupdate_decoder_subgraph_share_buffer_and_use_decoder_masked_mha.<locals>.<listcomp>!  s   € Ð7Ð7Ð7 S˜œÐ7Ð7Ð7r*   rw  rx  rz  r‹  rÁ   r›   T)rÁ   r§  ru   r‰   r3  r¿   r‡   r·  r`  r½   r«   r8  r}  r   rö  r›   rk  r”  r  r|  rt  r9  r8   r  r  r  )ru  rb  r«  Úoutput_self_past_0rd  r_  r‚  Ú	old_nodesr3  Úrel_pos_bias_noder–  Útarget_squeezed_past_seq_namer©  rª  Úname_to_renameÚnrr  r  rh  r—  r„  rœ  r½   Úorig_input_namesr€  rö   rs  r7  r�  s                                r(   Ú?update_decoder_subgraph_share_buffer_and_use_decoder_masked_mhar�  ±  s  € ØÐà6Ð6¨4¬:Ð6Ñ6Ô6ÐØ
˜aÒ
Ð
Ð(9Ð:KÔ(L×(WÒ(WÐX^Ñ(_Ô(_Ð
Ø˜QÑÐð ˜aÒ
Ð
Ð(9Ð:KÔ(L×(WÒ(WÐX^Ñ(_Ô(_Ð
àÐå•c˜$œ*‘o”oÐ(9Ñ9¸QÑ>Ñ?Ô?€JØ˜Z™Ð*;Ñ;Ðà€IØ€IØ”	ð %ð %ˆØŒ<Ð/Ò/Ð/Ø×Ò˜d˜VÑ$Ô$Ð$øõ ˆ9�~„~˜
Ò"Ð"Øˆuð ÐØ”	ð ð ˆØŒ<Ð1Ò1Ð1Ø $ÐØˆEð 2ð/ð /ð /Ð+ð %JÐ!Ý.EÀdÑ.KÔ.KÑ+Ð˜OÝ
Ð!Ñ"Ô" QÒ&Ð&Ø4ð 	nð 	nˆNÝÐl¨ÐlÐlÐLiÐlÐlÐlÑmÔmÐmÐmØ!ð 	Rð 	RˆBÝÐP°"´*ÐPÐPÀrÄwÐPÐPÑQÔQÐQÐQå”{×,Ò,ØØ#Ð$Ø,Ð-Ø4ð	 -ñ 
ô 
ˆõ ”K×)Ò)ØØ,Ð-Ø*Ð+Ø9ÝÔ ð *ñ 
ô 
ˆ	ð 	×Ò˜,¨	Ð2Ñ3Ô3Ð3à”	ð 0%ñ 0%ˆÝˆtŒ{ÑÔ˜aÒÐÐ$5Ð$AÀdÄkÐRSÄnÐXiÔXoÐpqÔXrÒFrÐFrÝœ×-Ò-ØØ'Ð(Ø-Ð.Ø0ÝÔ$ð .ñ ô ˆIð &Ô,¨QÔ/ˆDŒJ�q‰MØ×Ò˜i˜[Ñ)Ô)Ð)àŒ<Ð/Ò/Ñ/Ý˜t‘_”_ˆFØ—[’[‘]”]ð "ð "�ØÐCÐCÐCØ˜q˜	øð ”
˜1”Ø”
˜1”Ø”
˜1”ðˆCð �JŠJ­¨T¬Z©¬¸1Ò)<Ð)<˜œ
 1œ˜À"ÐEÑFÔFÐFØ�JŠJ­¨T¬Z©¬¸1Ò)<Ð)<˜œ
 1œ˜À"ÐEÑFÔFÐFØ�JŠJ­¨T¬Z©¬¸1Ò)<Ð)<˜œ
 1œ˜À"ÐEÑFÔFÐFØ�JŠJ­¨T¬Z©¬¸1Ò)<Ð)<˜œ
 1œ˜À"ÐEÑFÔFÐFØ�JŠJÐ.Ð/Ñ0Ô0Ð0Ø�JŠJ˜�~Ñ&Ô&Ð&Ø�JŠJÐ+Ð,Ñ-Ô-Ð-Ø�JŠJ­¨T¬Z©¬¸1Ò)<Ð)<˜œ
 1œ˜À"ÐEÑFÔFÐFà23ˆFÐ.Ñ/å”;Ô(Ø1ØØ”ðð ð ”Yð	ð
 ðð ˆDð �Ð&Ð&Ý(¨¬Ñ4Ô4ð Fð F‘��tØÐ1Ð1Ð1Ø(E�D”J˜uÑ%øØ×Ò˜d˜VÑ$Ô$Ð$ùà‡O‚O�FÑÔÐØ„I×Ò�YÑÔÐØ7Ð7¨D¬JÐ7Ñ7Ô7Ðà€JÝ˜4œ:Ñ&Ô&ð  ð  ‰ˆˆ2ØÐ!Ò!Ð! aÐ*<Ò&<Ð&<Ý˜R‘L”LˆEÝ”×3Ò3Ø”Øœ'Ô-Ô7Ø˜Q”x  q¤¨=¸%À¼(ÐCð 4ñ ô ˆBð
 	×Ò˜2˜$ÑÔÐÐØÐ%5Ð5Ð5Ø×ÒÝŒ[×/Ò/Ð0FÍÔHXÔH^ÐghÐfiÐ/ÑjÔjÐkñ	
ô 	
ð 	
ð Ð+Ð+Ð+Ø×Ò�4œ;×=Ò=¸lÍDÔL\ÔLbÐklÐjmÐ=ÑnÔnÐoÑpÔpÐpØÐ"2Ð2Ð2Ø×Òå”×2Ò2Ø'ÝÔ$Ô*ØEÐEÐEð 3ñ ô ðñ	
ô 	
ð 	
ð 	‡O‚O�GÑÔÐØ„J×Ò�jÑ!Ô!Ð!à€KÝ˜4œ;Ñ'Ô'ð !ð !‰ˆˆ2ØÐ"Ò"Ð"Ý˜R‘L”LˆEÝ”×3Ò3Ø”Øœ'Ô-Ô7Ø˜Q”x  q¤¨=¸%À¼(ÐCð 4ñ ô ˆBð
 	×Ò˜B˜4Ñ Ô Ð Ð Ø‡O‚O�HÑÔÐØ„K×Ò�{Ñ#Ô#Ð#àˆ4r*   Úmodel_protoc                 óþ  — t          | ¦  «        }|                     ¦   «         }g }g }|                     ¦   «         D �]m}|j        dk    �r^d|j        d         v rd|j        d         v rŒ.||j        d                  }||j        d                  }||j        d                  }|                     |j        d         ¦  «        }	|                     |j        d         ¦  «        }
|                     |j        d         ¦  «        }|	r|
r|s dS t          j        |	¦  «        }t          j        |
¦  «        }t          j        |¦  «        }t          j	        |||gd¬¦  «        }| 
                    d	d
¬¦  «        }t          j                             |dz   |	j        dk    rt          j        nt          j        |j        d         |j        d         g|                     ¦   «                              ¦   «         ¬¦  «        }| j        j                             |g¦  «         t          j                             d	|j        d         |dz   g|dz   g|¬¦  «        }|j        d         |j        d<   d|j        d<   d|j        d<   |                     |g¦  «         |                     |||g¦  «         �Œo|                     |¦  «         |                     |¦  «         |                     ¦   «          |                     ¦   «          dS )Nr
  Úpast_key_crossr\   Úpast_value_crossr¶   r   Frº   r´   Ú
MatMul_QKV)Úname_prefixÚ_weightr  Ú_outrî  r   T)r   r¾   Únodesr¿   rÁ   rÀ   r   rÍ   rÉ   rÌ   rõ  r«   r8  r.  rÃ   r   r  rt   r7  ÚflattenÚtolistr¼   r0  r‡   r}  r›   Ú	add_nodesÚremove_nodesÚupdate_graphrÅ  )r‚  Ú
onnx_modelr¾   Únodes_to_addrª  r3  r;  r@  rD  Úq_weightÚk_weightÚv_weightrK  rL  rM  rN  Úmatmul_node_nameÚweightrÒ   s                      r(   Úpack_qkv_for_decoder_masked_mhar—  P  sû  € Ý˜;Ñ'Ô'€JØ$×8Ò8Ñ:Ô:Ðà€LØ€OØ× Ò Ñ"Ô"ð *Cñ *CˆØŒ<Ð<Ò<Ñ<Ø 4¤:¨a¤=Ð0Ð0Ð5GÈ4Ì:ÐVWÌ=Ð5XÐ5XØØ*¨4¬:°a¬=Ô9ˆHØ*¨4¬:°a¬=Ô9ˆHØ*¨4¬:°a¬=Ô9ˆHà!×1Ò1°(´.ÀÔ2CÑDÔDˆHØ!×1Ò1°(´.ÀÔ2CÑDÔDˆHØ!×1Ò1°(´.ÀÔ2CÑDÔDˆHØð  ð ¨hð Ø�u�uåÔ% hÑ/Ô/ˆBÝÔ% hÑ/Ô/ˆBÝÔ% hÑ/Ô/ˆBåœ¨¨R°¨¸1Ð=Ñ=Ô=ˆJà)×:Ò:¸8ÐQ]Ð:Ñ^Ô^ÐÝ”[×,Ò,Ø%¨	Ñ1Ø08Ô0BÀaÒ0GÐ0G�;Ô,Ð,Í[ÔM`Ø Ô& qÔ)¨:Ô+;¸AÔ+>Ð?Ø×'Ò'Ñ)Ô)×0Ò0Ñ2Ô2ð	 -ñ ô ˆFð ÔÔ)×0Ò0°&°Ñ:Ô:Ð:åœ+×/Ò/ØØ œ qÔ)Ð+;¸iÑ+GÐHØ)¨FÑ2Ð3Ø%ð	 0ñ ô ˆKð (Ô.¨qÔ1ˆDŒJ�q‰MØˆDŒJ�q‰MØˆDŒJ�q‰Mà×Ò  Ñ.Ô.Ð.Ø×"Ò" H¨h¸Ð#AÑBÔBÐBùà×Ò˜Ñ&Ô&Ð&Ø×Ò˜OÑ,Ô,Ð,Ø×ÒÑÔÐà×ÒÑ!Ô!Ð!àˆ4r*   Údecoder_onnx_pathc                 óÖ  — t          j        | d¬¦  «        }t          t          |j        j        ¦  «        ¦  «        D ]”}|j        j        |         j        dk    s|j        j        |         j        dk    r\|j        j        |         j        j        j	        j
        d         }|                     d¦  «        r|                     ¦   «          d|_        Œ•t          j        || |¬¦  «         dS )aQ  Update the input shapes for the inputs "input_ids" and "position_ids" and make the sequence length dim value 1 for each of them.
       The decoder model will be over-written.

    Args:
        decoder_onnx_path (str): Path of GPT-2 decoder onnx model
        use_external_data_format(bool): output tensors to external data or not.
    Tr¤   rð   rñ   r\   rn  r¨   )r«   r¬   r  r‰   r¼   rÁ   r½   r8   r  r7  rr  ÚHasFieldÚClearro  r   r®   )r˜  rL   rÏ   rö   Úshape_dim_protos        r(   Ú*update_input_shapes_for_gpt2_decoder_modelr�  ‹  só   € õ œ/Ð*;ÐPTÐUÑUÔUÐÝ•3Ð*Ô0Ô6Ñ7Ô7Ñ8Ô8ð *ð *ˆàÔ%Ô+¨AÔ.Ô3°{ÒBÐBØ"Ô(Ô.¨qÔ1Ô6¸.ÒHÐHà1Ô7Ô=¸aÔ@ÔEÔQÔWÔ[Ð\]Ô^ˆOð ×'Ò'¨Ñ4Ô4ð (Ø×%Ò%Ñ'Ô'Ð'ð )*ˆOÔ%øå„NØØØ6ðñ ô ð ð
 ˆ4r*   Úinit_decoder_onnx_pathc           	      óP	  — t          j        | d¬¦  «        }|j        j        d         j        }t          |¦  «        }|                     ¦   «         }||v sJ ‚||         }|j        dk    rdS |                     |g d¢g d¢¦  «        }|€|                     |g d	¢g d
¢¦  «        }|€8|                     |g d¢g d¢¦  «        }|€|                     |g d¢g d¢¦  «        }|€dS |d         }	|	j        dk    }
|
s�d}|                     |	g d¢|dddg¦  «        }|€d}|                     |	g d¢|dddg¦  «        }|€d}|                     |	g d¢|ddg¦  «        }|€d}|                     |	g d¢|ddg¦  «        }n|d}|                     |	g d¢|ddg¦  «        }|€d}|                     |	g d¢|ddg¦  «        }|€d}|                     |	ddg|dg¦  «        }|€d}|                     |	ddg|dg¦  «        }|€dS |dk    rdnd}|
s| 	                    |	d|¦  «        }n| 	                    |	d|¦  «        }|€dS |d         }|d         }t           j
                             dt          j        dgdg¬¦  «        }t           j
                             dt          j        dgdg¬¦  «        }t           j
                             dt          j        dgdg¬¦  «        }t           j
                             dt          j        dgdg¬¦  «        }|                     |¦  «         |                     |¦  «         |                     |¦  «         |                     |¦  «         d|j        d         z   }t           j
                             d|j        d         ddddg|g|                     dd¦  «        ¬ ¦  «        }|
s|j        d         n|j        d!         }d|j        d         z   }t           j
                             d|ddddg|g|                     dd"¦  «        ¬ ¦  «        }|                     |¦  «         |                     |¦  «         |                     ||j        d         |¦  «         |                     |	||¦  «         |                     ¦   «          t          j        |||¬#¦  «         dS )$a„  Generates the initial decoder GPT2 subgraph and saves it for downstream use.
       The initial decoder model will be saved to init_decoder_onnx_path.

    Args:
        decoder_onnx_path (str): Path of GPT-2 decoder onnx model
        init_decoder_onnx_path (str): Path of GPT-2 init decoder onnx model
        use_external_data_format(bool): output tensors to external data or not.
    Tr¤   r   r´   F)rñ  rÛ  rÝ  rÝ  rñ  r´   rñ  ÚFastGelurñ  r´   rñ  rÛ  rÝ  )r   r   r   r\   r   r   r   r   r   r   r   r   r   N)
rñ  ÚSkipLayerNormalizationrñ  r´   rñ  r   rñ  r´   rñ  r¡  )
r   r   r\   r   r   r   r   r   r   r   )rÛ  rÝ  rÝ  r´   r   r´   rÛ  rÝ  )r   r   r\   r   r   r   r   r   )r¡  r´   r   r´   r¡  )r   r\   r   r   r   rb   r¡  )rÝ  rñ  r´   r{  r\   )rÝ  r´   r{  )rñ  r´   r{  r{  rÝ  rì  ÚSliceLastTokenStartsr  ÚSliceLastTokenEndsÚSliceLastTokenAxesÚSliceLastTokenStepsÚedge_modified_râ  ÚGatherLastToken_0_rî  rï   ÚGatherLastToken_1_r¨   )r«   r¬   r¼   r›   r½   r   r¾   r¿   rô  rÂ   r8  r.  r   r  r-  r}  rõ  rý  Úreplace_node_inputrÅ  r®   )r˜  rž  rL   Úinit_decoder_model_protorÐ   Úgpt2_init_decoder_modelr¾   Úlogits_matmul_nodeÚ"logits_matmul_to_residual_add_pathÚresidual_add_nodeÚis_skiplayernorm_pathÚ&residual_add_to_attention_parent_indexÚresidual_add_to_attention_pathÚ residual_add_to_add_parent_indexÚadd_before_residual_addÚ	attentionÚmatmul_after_attentionÚslice_startsÚ
slice_endsÚ
slice_axesÚslice_stepsÚslice_0_output_nameÚslice_node_0Úadd_before_residual_add_outputÚslice_1_output_nameÚslice_node_1s                             r(   Úgenerate_gpt2_init_decoderr¿  «  s¤  € õ  $œÐ/@ÐUYÐZÑZÔZÐà1Ô7Ô>¸qÔAÔFÐå'Ð(@ÑAÔAÐà1×EÒEÑGÔGÐØÐ!4Ð4Ð4Ð4Ð4à,Ð-?Ô@Ðð Ô! XÒ-Ð-Øˆuð *A×)RÒ)RØð	
ð 	
ð 	
ð 	0Ð/Ð/ñ#*ô *Ð&ð* *Ð1Ø-D×-VÒ-VØðð ð ð +Ð*Ð*ñ.
ô .
Ð*ð$ *Ð1à-D×-VÒ-VØð	ð 	ð 	ð %Ð$Ð$ñ.
ô .
Ð*ð  .Ð5Ø1H×1ZÒ1ZØ"ðð ð ð  ��ñ
2ô 
2Ð.ð *Ð1Øˆuà:¸2Ô>Ðð .Ô5Ð9QÒQÐð !ð EØ12Ð.Ø)@×)RÒ)RØØ2Ð2Ð2Ø3°Q¸¸1Ð=ñ*
ô *
Ð&ð *Ð1Ø56Ð2Ø-D×-VÒ-VØ!Ø6Ð6Ð6Ø7¸¸A¸qÐAñ.ô .Ð*ð *Ð1Ø56Ð2Ø-D×-VÒ-VØ!Ø.Ð.Ð.Ø7¸¸AÐ>ñ.ô .Ð*ð *Ð1Ø56Ð2Ø-D×-VÒ-VØ!Ø.Ð.Ð.Ø7¸¸AÐ>ñ.ô .Ð*øð 23Ð.Ø)@×)RÒ)RØØ+Ð+Ð+Ø3°Q¸Ð:ñ*
ô *
Ð&ð *Ð1Ø56Ð2Ø-D×-VÒ-VØ!Ø/Ð/Ð/Ø7¸¸AÐ>ñ.ô .Ð*ð *Ð1Ø56Ð2Ø-D×-VÒ-VØ!Ø˜;Ð'Ø7¸Ð;ñ.ô .Ð*ð *Ð1Ø56Ð2Ø-D×-VÒ-VØ!Ø˜;Ð'Ø7¸Ð;ñ.ô .Ð*ð &Ð-Øˆuà,RÐVWÒ,WÐ,W q qÐ]^Ð$ð !ð 
Ø"9×"FÒ"FØ˜uÐ&Fñ#
ô #
ÐÐð #:×"FÒ"FØØ$Ø,ñ#
ô #
Ðð Ð&Øˆuà.¨rÔ2€IØ;¸BÔ?Ðå”;×*Ò*Ø#ÝÔ#ØˆSØˆTð	 +ñ ô €Lõ ”×(Ò(Ø!ÝÔ#ØˆSØˆTð	 )ñ ô €Jõ ”×(Ò(Ø!ÝÔ#ØˆSØˆSð	 )ñ ô €Jõ ”+×)Ò)Ø"ÝÔ#ØˆSØˆTð	 *ñ ô €Kð ×+Ò+¨LÑ9Ô9Ð9Ø×+Ò+¨JÑ7Ô7Ð7Ø×+Ò+¨JÑ7Ô7Ð7Ø×+Ò+¨KÑ8Ô8Ð8ð +¨YÔ-=¸aÔ-@Ñ@ÐÝ”;×(Ò(ØàÔ˜QÔØ"Ø Ø Ø!ð
ð %Ð%Ø$×5Ò5°gÐ?SÑTÔTð )ñ ô €Lð" 2GÐmÐÔ& qÔ)Ð)ÐLcÔLjÐklÔLmð #ð +Ð-DÔ-KÈAÔ-NÑNÐÝ”;×(Ò(Øà*Ø"Ø Ø Ø!ð
ð %Ð%Ø$×5Ò5°gÐ?SÑTÔTð )ñ ô €Lð ×$Ò$ \Ñ2Ô2Ð2Ø×$Ò$ \Ñ2Ô2Ð2ð ×.Ò.Ð/EÀyÔGWÐXYÔGZÐ\oÑpÔpÐpØ×.Ò.Ð/@ÐB`ÐbuÑvÔvÐvð ×,Ò,Ñ.Ô.Ð.õ „NØ ØØ6ðñ ô ð ð
 ˆ4r*   c                 ó€  — t          d¦  «        }t          |j        ¦  «        }t          |j        ¦  «        }t          |j        ¦  «        }| j        j        D ]l}|j        j        j        j	        D ]S}| 
                    d¦  «        r<|j        ||||fv r/t          |j        ¦  «        }|                     ¦   «          ||_        ŒTŒm| j        j        D ]l}|j        j        j        j	        D ]S}| 
                    d¦  «        r<|j        ||||fv r/t          |j        ¦  «        }|                     ¦   «          ||_        ŒTŒmdS )zoMake dim_proto numeric.

    Args:
        model: T5 encoder and decoder model.
        config: T5 config.
    r\   rn  N)rn   rŽ  Úd_modelÚd_kvr¼   r›   r8   r  r7  rr  rš  rn  ru   r›  ro  rÁ   )	r°   ÚconfigrÏ  rŽ  Úhidden_sizeÚ	head_sizerR  Ú	dim_protoro  s	            r(   Úmake_dim_proto_numeric_t5rÇ  Ö	  ss  € õ ˜!‘f”f€OÝ�FÔ$Ñ%Ô%€IÝ�f”nÑ%Ô%€KÝ�F”KÑ Ô €Ià”+Ô$ð 
0ð 
0ˆØœÔ0Ô6Ô:ð 		0ð 		0ˆIØ×!Ò! +Ñ.Ô.ð 0°9Ô3FØØØØð	Kð 4ð 4õ   	Ô 3Ñ4Ô4�	Ø—’Ñ!Ô!Ð!Ø&/�	Ô#øð		0ð ”+Ô#ð 
0ð 
0ˆØœÔ0Ô6Ô:ð 		0ð 		0ˆIØ×!Ò! +Ñ.Ô.ð 0°9Ô3FØØØØð	Kð 4ð 4õ   	Ô 3Ñ4Ô4�	Ø—’Ñ!Ô!Ð!Ø&/�	Ô#øð		0ð
0ð 
0r*   Úgeneration_typec                 óš(  — | j         dk    }|t          j        k    }|t          j        k    }|t          j        k    }| j        }t                               d|› �¦  «         t          | j	        ¦  «        dk    rz| j	        d         dk    ri|r`| j
        t          j        j        k    rFg d¢| _	        t                               d| j	        › �¦  «         t                               d¦  «         ng | _	        |s|r=|st          d	¦  «        ‚| j        rt          d
¦  «        ‚| j        rt          d¦  «        ‚|r|r| j        st%          d¦  «        ‚| j        r|st%          d¦  «        ‚| j        r| j        st%          d¦  «        ‚|rà| j        rGt*          j                             | j        ¦  «        r#t                               d| j        › �¦  «         ný| j        sP| j        › d| j
        › d�}t3          t3          | j        ¦  «        j        |¦  «                             ¦   «         | _        t                               d| j        › d| j        › d�¦  «         t;          | ¦  «         nk| j        r2| j        r+t                               d| j        › d| j        › �¦  «         n2t                               d| j        › d�¦  «         t?          | ¦  «         d}| j         s{| j
        t          j        j        k    ra|r_|s|s|rYt                               d| j        › d�¦  «         tC          | j        | j"        ¦  «        }|st           #                    d¦  «         d}	d}
| j$        sÑ|rÏ|s|s|rÉt                               d| j        › d�¦  «         d | j
        › d�}t3          t3          | j        ¦  «        j        |¦  «                             ¦   «         }
tK          | j        |
| j"        ¦  «        }	|	st           #                    d!¦  «         |	r)tM          | j        | j"        ¦  «        st%          d"¦  «        ‚|s	| j'        s|	rrt                               d#| j        › d�¦  «         tQ          | j        | j"        ¦  «         |	r3t                               d#|
› d�¦  «         tQ          |
| j"        ¦  «         |r!tS          j*        | j        | j+        ¬$¦  «        }nL| j         d%k    r!tY          j*        | j        | j+        ¬$¦  «        }n t[          j*        | j        | j+        ¬$¦  «        }| j.        rt                               d&|› �¦  «         |j/        }|r|j/        n|j0        }|j1        }| j1        d'k    r| j1        }| j/        d'k    r| j/        }| j0        d'k    r| j0        }te          j3        | j        d(¬)¦  «        }| j         › d*�|j4        _5        d}| j         dk    ratm          |j4        | j
        ¦  «         |	rDte          j3        |
d(¬)¦  «        }| j         › d+�|j4        _5        tm          |j4        | j
        ¦  «         nto          |j4        | j
        ¦  «         d}|rg d,¢}n|s|rg d-¢}| j8        r| 9                    d.¦  «         n| 9                    d/¦  «         | j:        r| 9                    d0¦  «         n| 9                    d/¦  «         | j;        r| 9                    d1¦  «         n| 9                    d/¦  «         |rU| j<        r| j=        r| 9                    d2¦  «         n| 9                    d/¦  «         | j>        r| 9                    d3¦  «         d4g}| j        r| 9                    d5¦  «         | j        r&| j        s
J d6¦   «         ‚| 9                    d7¦  «         d}|r,td          j?         @                    d8||d9| j         › �¬:¦  «        }n[|r,td          j?         @                    d;||d<| j         › �¬:¦  «        }n-|r+td          j?         @                    d=||d>| j         › �¬:¦  «        }d?|_A        d}|rºtd          j?         B                    d@|¦  «        td          j?         B                    dA|¦  «        td          j?         B                    dB| jC        ¦  «        td          j?         B                    dC| jD        rdnd¦  «        td          j?         B                    dD| j         dk    rdnd¦  «        g}�nÿ|r’td          j?         B                    d@|¦  «        td          j?         B                    dA|¦  «        td          j?         B                    dD| j         dk    rdnd¦  «        td          j?         B                    dB| jC        ¦  «        g}�nk|�rhtd          j?         B                    d@|¦  «        td          j?         B                    dA|¦  «        td          j?         B                    dD| j         dk    rdnd¦  «        td          j?         B                    dB| jC        ¦  «        td          j?         B                    dE| jE        ¦  «        td          j?         B                    dF| jF        ¦  «        td          j?         B                    dG| jG        ¦  «        td          j?         B                    dH| jH        ¦  «        td          j?         B                    dI| j<        ¦  «        td          j?         B                    dJ| jI        ¦  «        g
}|r4| J                    td          j?         B                    dK|¦  «        g¦  «         |jK         J                    |¦  «         g }| j         dLv �rz| j'        r=t                               dM| j        › d�¦  «         tQ          | j        | j"        ¦  «         te          j3        | j        d(¬)¦  «        }t          |j4        jL        ¦  «        dNk    rdOndP}| j         › dQ|› �|j4        _5        t›          |j4        | j
        ¦  «         t�          ||¦  «         t�          ||¦  «         |r½| j        st%          dR¦  «        ‚t                               dS¦  «         tŸ          |j4        ¦  «        rt                               dT¦  «         nt                               dU¦  «         t¡          |¦  «        rt                               dV¦  «         nt                               dW¦  «         | jQ        sGt¥          ||¦  «        }t                               t          |¦  «        › dXdY„ |D ¦   «         › dZ�¦  «         |jS        dk    s
J d[¦   «         ‚|jK         J                    td          j?         B                    dO|j4        ¦  «        td          j?         B                    d\|j4        ¦  «        td          j?         B                    d]|jS        ¦  «        g¦  «         �nÁ|	rè| jQ        sGt¥          ||¦  «        }t                               t          |¦  «        › dXd^„ |D ¦   «         › d_�¦  «         |r.t                               d`¦  «         t©          |j4        ¦  «         | j        r%t«          |j4        |d¦  «        st%          da¦  «        ‚|jK         9                    td          j?         B                    db|j4        ¦  «        ¦  «         n>t­          |j4        ¦  «        }t                               t          |¦  «        › dc�¦  «         |r.t                               dd¦  «         t©          |j4        ¦  «         | j        r%t«          |j4        |d(¦  «        st%          de¦  «        ‚|jK         9                    td          j?         B                    d\|j4        ¦  «        ¦  «         td          j?         W                    dft°          jY        dgdhg¦  «        }td          j?         W                    dit°          jY        dg¦  «        }td          j?         W                    djt°          jY        dg¦  «        }td          j?         W                    dkt°          jY        dg¦  «        }td          j?         W                    dlt°          jY        dg¦  «        }td          j?         W                    dmt°          jZ        dg¦  «        }td          j?         W                    dnt°          jZ        dg¦  «        }d} |r
|||||||g} n
|s|r||||g} | j8        rAtd          j?         W                    d.t°          jY        |g¦  «        }!|  9                    |!¦  «         | j:        rBtd          j?         W                    d0t°          jY        dg|g¦  «        }"|  9                    |"¦  «         | j;        rBtd          j?         W                    d1t°          jY        dgdhg¦  «        }#|  9                    |#¦  «         | j<        rI| j=        rBtd          j?         W                    d2t°          jY        dg|g¦  «        }$|  9                    |$¦  «         |rH| j>        rAtd          j?         W                    d3t°          jY        dg¦  «        }%|  9                    |%¦  «         d}&|r.td          j?         W                    d4t°          jY        g do¢¦  «        }&n1|s|r-td          j?         W                    d4t°          jY        dgdig¦  «        }&|&g}'| j        rBtd          j?         W                    d5t°          jZ        dgdlg¦  «        }(|' 9                    |(¦  «         | j        rDtd          j?         W                    d7t°          jZ        dpdgdk|g¦  «        })|' 9                    |)¦  «         td          j?         [                    |g|s
| j         › dq�n	| j         › dr�| |'|¦  «        }*td          j?         \                    |*ds|j]        ¬t¦  «        }+| j"        rtddul^m_}, |, `                    td          ja        ¦  «        |, `                    dv¦  «        k     rt           #                    dw¦  «         tÅ          jc        |+| j        d(d(¬x¦  «         nte          jc        |+| j        ¦  «         t                               dy| j        › �¦  «         dS )zzˆConvert model according to command line arguments.

    Args:
        args (argparse.Namespace): arguments parsed from command line
    r:   z**** past_present_share_buffer=r\   r   rI   )rÝ  rÛ  r¡  r   z**** Setting op_block_list to zI**** use --op_block_list if you want to override the block operator list.z<Currently only gpt2 with greedy search/sampling is supportedzLoutput_sequences_scores currently is not supported in greedy search/samplingzHoutput_token_scores currently is not supported in greedy search/samplingzi`use_decoder_masked_attention` MUST be turned on to use `past_present_share_buffer` in case of BeamSearchzS`past_present_share_buffer` MUST be turned on to use `use_decoder_masked_attention`z?`use_decoder_masked_attention` option is only supported on GPUsz)skip convert_to_onnx since path existed: Ú_past_z.onnxzConvert GPT model z	 to onnx z ...z,skip convert_to_onnx since paths specified: z and zConvert model z to onnx ...Fz=Pad logits MatMul weights for optimal MatMul perf in fp16 on z. The file will be overwritten.z]Tried and failed to pad logits MatMul weights. Performance may be sub-optimal for this MatMulNz*Creating an initial run GPT2 decoder from z. Úgpt2_init_past_zuTried and failed to generate the init decoder GPT2 model. Performance may be sub-optimal for the initial decoding runzGCould not update the input shapes for the non-initial decoder subgraph.z Run symbolic shape inference on ©r†   r;   zConfig=rb   Tr¤   z decoderz init decoder©rð   Ú
max_lengthÚ
min_lengthÚ	num_beamsÚnum_return_sequencesÚlength_penaltyÚrepetition_penalty©rð   rÎ  rÏ  rÓ  rU   r   rX   rò   rZ   r[   Ú	sequencesÚsequences_scoresz8--output_token_scores requires --output_sequences_scoresÚscoresÚ
BeamSearchÚBeamSearch_rî  ÚGreedySearchÚGreedySearch_ÚSamplingÚ	Sampling_r  Úeos_token_idÚpad_token_idÚno_repeat_ngram_sizerT   r˜   ÚtemperatureÚtop_pÚfilter_valueÚmin_tokens_to_keepÚcustomÚpresence_penaltyÚ
vocab_size©r;   r<   zSymbolic shape inference on r¶   rL  zencoder and decoder initú zMpast_present_share_buffer is only supported with use_decoder_masked_attentionzl*****update t5 decoder subgraph to share past/present buffer and use decoder_masked_multihead_attention*****z4*****update t5 decoder subgraph successfully!!!*****zF*****DecoderMaskedMultiHeadAttention is not applied to T5 decoder*****z9*****pack qkv for decoder masked mha successfully!!!*****z3*****pack qkv for decoder masked mha failed!!!*****z shared initializers (c                 ó   — g | ]	}|j         ‘Œ
S r1   r›  rõ   s     r(   rä   z,convert_generation_model.<locals>.<listcomp>?  s   € Ð<ZÐ<ZÐ<ZÈ¸Q¼VÐ<ZÐ<ZÐ<Zr*   z>) in encoder and decoder subgraphs are moved to the main graphz%decoder_start_token_id should be >= 0rM  Údecoder_start_token_idc                 ó   — g | ]	}|j         ‘Œ
S r1   r›  rõ   s     r(   rä   z,convert_generation_model.<locals>.<listcomp>[  s   € Ð@^Ð@^Ð@^ÈAÀÄÐ@^Ð@^Ð@^r*   zC) in decoder and init decoder subgraphs are moved to the main graphzY*****update init decoder subgraph to make past and present share buffer******************zLCould not update the init decoder subgraph to use DecoderMaskedSelfAttentionÚinit_decoderz: initializers from the decoder are moved to the main graphzT*****update decoder subgraph to make past and present share buffer******************zGCould not update the decoder subgraph to use DecoderMaskedSelfAttentionrð   rŒ  rÏ  rÎ  rÏ  rÐ  rÑ  rÒ  rÓ  )rŒ  rÑ  rÎ  zmax_length - sequence_lengthz beam searchz greedy searchzonnxruntime.transformers)Úproducer_nameÚopset_imports)Úversionz1.12.0z0Require onnx >= 1.12 to save large (>2GB) model!)r©   Úall_tensors_to_one_filezmodel save to )dr˜   r    r.   r/   r0   rV   r‹   rŒ   r‰   rŠ   r…   r   rt   r&   ÚNotImplementedErrorrR   rS   rW   r  re   r„   rp   rq   Úexistsrƒ   r   r›   rœ   Úas_posixr�   rž   r    rN   rÛ   rL   r¯   rO   r¿  r�  rM   r²   r   Úfrom_pretrainedr†   r   r   rD   rÞ  rß  rç  r«   r¬   r¼   r½   r  r  rU   rˆ   rX   rY   rå  rZ   r[   r8  r}  rf  ra  rà  rT   rá  râ  rã  rä  ræ  r‡   rc  rÁ   r&  rÇ  r�  r—  rP   rO  rë  r…  r˜  rS  r9  r   r  r  Ú
make_graphÚ
make_modelÚopset_importÚ	packagingrð  ÚparseÚ__version__r   r®   )-r~   rÈ  Úis_gpt2Úis_beamsearchÚis_greedysearchÚis_samplingrV   Úonnx_filenameÚlogits_matmul_weight_paddedÚgpt2_init_decoder_generatedÚgpt2_init_decoder_onnx_pathÚgpt2_init_decoder_onnx_filenamerÃ  rÞ  rß  rç  rÑ   r«  rï  rð  r3  Úattr_to_extendrN  rE  Úsuffixrð   rÎ  rÏ  rÐ  rÑ  rÒ  rÓ  Úgraph_inputsrU   rX   rò   rZ   r[   rÕ  Úgraph_outputsrÖ  r×  Ú	new_graphÚ	new_modelrð  s-                                                r(   Úconvert_generation_modelr  û	  sR  € ð ”O vÒ-€GØ)­^Ô-FÒF€MØ+­~Ô/JÒJ€OØ'­>Ô+BÒB€KØ&*Ô&DÐå
‡K‚KÐMÐ2KÐMÐMÑNÔNÐNÝ
ˆ4ÔÑÔ !Ò#Ð#¨Ô(:¸1Ô(=ÀÒ(GÐ(GØð 
	$�t”~­Ô):Ô)@Ò@Ð@ð"ð "ð "ˆDÔõ �KŠKÐM¸Ô9KÐMÐMÑNÔNÐNÝ�KŠKÐcÑdÔdÐdÐdà!#ˆDÔàð r˜+ð rØð 	fÝ%Ð&dÑeÔeÐeØÔ'ð 	vÝ%Ð&tÑuÔuÐuØÔ#ð 	rÝ%Ð&pÑqÔqÐqð !ð 
 ]ð 
¸4Ô;\ð 
ÝØwñ
ô 
ð 	
ð Ô(ð pÐ1Jð pÝÐnÑoÔoÐoð Ô(ð \°´ð \ÝÐZÑ[Ô[Ð[àð ØÔð 	¥¤§¢°Ô0AÑ!BÔ!Bð 	Ý�KŠKÐWÀDÔDUÐWÐWÑXÔXÐXÐXàÔ$ð ]Ø#'Ô#:Ð WÐ WÀ$Ä.Ð WÐ WÐ W�Ý$(­¨d¬kÑ):Ô):Ô)AÀ=Ñ$QÔ$Q×$ZÒ$ZÑ$\Ô$\�Ô!å�KŠKÐf¨TÔ-DÐfÐfÈtÔO`ÐfÐfÐfÑgÔgÐgÝ˜ÑÔÐÐàÔð 	 Ô!?ð 	Ý�KŠKØw¸tÔ?PÐwÐwÐW[ÔWuÐwÐwñô ð ð õ �KŠKÐN¨Ô)@ÐNÐNÐNÑOÔOÐOÝ�tÑÔÐð #(ÐàÔ'ðàŒN�iÔ/Ô5Ò5Ð5Øð 6àð 6à-ð 6à1<ð 6õ 	�Šð,ÈDÔL]ð ,ð ,ð ,ñ	
ô 	
ð 	
õ 'CÀ4ÔCTÐVZÔVsÑ&tÔ&tÐ#Ø*ð 	Ý�NŠNØoñô ð ð #(ÐØ"&ÐàÔ;ðhàðhð ðhð .ðhð 2=ðhõ
 	�ŠÐVÀÔARÐVÐVÐVÑWÔWÐWà*Q¸D¼NÐ*QÐ*QÐ*QÐ'å&*­4°´Ñ+<Ô+<Ô+CÐEdÑ&eÔ&e×&nÒ&nÑ&pÔ&pÐ#å&@ØÔØ'ØÔ)ñ'
ô '
Ð#ð +ð 	Ý�NŠNðNñô ð ð 'ð 	hÕ/YØÔ˜tÔ<ñ0
ô 0
ð 	hõ ÐfÑgÔgÐgð
 #ð X dÔ&>ð XÐB]ð XÝ�ŠÐi°tÔ7HÐiÐiÐiÑjÔjÐjÝ˜Ô)¨4Ô+HÑIÔIÐIØ&ð 	XÝ�KŠKÐwÐ;VÐwÐwÐwÑxÔxÐxÝÐ7¸Ô9VÑWÔWÐWàð ^ÝÔ+¨DÔ,CÈtÌ~Ð^Ñ^Ô^ˆˆØ	Œ˜DÒ	 Ð	 ÝÔ)¨$Ô*AÈTÌ^Ð\Ñ\Ô\ˆˆåÔ*¨4Ô+BÈdÌnÐ]Ñ]Ô]ˆà„|ð (Ý�ŠÐ&˜fÐ&Ð&Ñ'Ô'Ð'àÔ&€LØ*1ÐJ�6Ô&Ð&°vÔ7J€LØÔ"€Jð „˜"ÒÐØ”_ˆ
àÔ˜BÒÐØÔ(ˆØÔ˜BÒÐØÔ(ˆå”O DÔ$5È$ÐOÑOÔO€MØ"&¤/Ð;Ð;Ð;€MÔÔà"ÐØ„˜&Ò Ð Ý˜]Ô0°$´.ÑAÔAÐAð 'ð 	PÝ&*¤oÐ6QÐfjÐ&kÑ&kÔ&kÐ#Ø48´OÐ1RÐ1RÐ1RÐ#Ô)Ô.Ý Ð!8Ô!>ÀÄÑOÔOÐOøå" =Ô#6¸¼ÑGÔGÐGà€FØð 
ð
ð 
ð 
ˆˆð 
ð 
˜Kð 
ð
ð 
ð 
ˆð „ð Ø�Š�lÑ#Ô#Ð#Ð#à�Š�bÑÔÐàÔð Ø�ŠÐ)Ñ*Ô*Ð*Ð*à�Š�bÑÔÐàÔ!ð Ø�ŠÐ&Ñ'Ô'Ð'Ð'à�Š�bÑÔÐàð "ØŒ;ð 	˜4Ô-ð 	Ø�MŠM˜/Ñ*Ô*Ð*Ð*à�MŠM˜"ÑÔÐàŒ9ð 	"Ø�MŠM˜&Ñ!Ô!Ð!àˆm€GØÔ#ð +Ø�ŠÐ)Ñ*Ô*Ð*àÔð !ØÔ+ÐgÐgÐ-gÑgÔgÐ+Ø�Š�xÑ Ô Ð à€DØð 
ÝŒ{×$Ò$ØØØØ0˜tœÐ0Ð0ð	 %ñ 
ô 
ˆˆð 
ð 
ÝŒ{×$Ò$ØØØØ2 ¤Ð2Ð2ð	 %ñ 
ô 
ˆˆð 
ð 
ÝŒ{×$Ò$ØØØØ.˜Tœ_Ð.Ð.ð	 %ñ 
ô 
ˆð "€D„Kà€NØð 
åŒK×&Ò& ~°|ÑDÔDÝŒK×&Ò& ~°|ÑDÔDÝŒK×&Ò&Ð'=¸tÔ?XÑYÔYÝŒK×&Ò&Ð'7¸dÔ>QÐ9X¸¸ÐWXÑYÔYÝŒK×&Ò& |¸$¼/ÈVÒ:SÐ:S°Q°QÐYZÑ[Ô[ð
ˆ‰ð 
ð 
åŒK×&Ò& ~°|ÑDÔDÝŒK×&Ò& ~°|ÑDÔDÝŒK×&Ò& |¸$¼/ÈVÒ:SÐ:S°Q°QÐYZÑ[Ô[ÝŒK×&Ò&Ð'=¸tÔ?XÑYÔYð	
ˆ‰ð 
ñ 
åŒK×&Ò& ~°|ÑDÔDÝŒK×&Ò& ~°|ÑDÔDÝŒK×&Ò& |¸$¼/ÈVÒ:SÐ:S°Q°QÐYZÑ[Ô[ÝŒK×&Ò&Ð'=¸tÔ?XÑYÔYÝŒK×&Ò& }°dÔ6FÑGÔGÝŒK×&Ò& w°´
Ñ;Ô;ÝŒK×&Ò& ~°tÔ7HÑIÔIÝŒK×&Ò&Ð';¸TÔ=TÑUÔUÝŒK×&Ò& x°´Ñ=Ô=ÝŒK×&Ò&Ð'9¸4Ô;PÑQÔQð
ˆð #ð VØ×Ò�tœ{×9Ò9¸,È
ÑSÔSÐTÑUÔUÐUà„N×Ò˜.Ñ)Ô)Ð)à€Là„˜-Ð'Ñ'ØÔ#ð 	[Ý�KŠKÐv°tÔ7UÐvÐvÐvÑwÔwÐwÝ˜DÔ:¸DÔ<YÑZÔZÐZÝœ¨Ô(FÐ[_Ð`Ñ`Ô`ˆÝ! -Ô"5Ô";Ñ<Ô<ÀÒAÐA��ÐGaˆØ&*¤oÐ#@Ð#@¸Ð#@Ð#@ˆÔÔ Ý/°Ô0CÀTÄ^ÑTÔTÐTå! -°Ñ8Ô8Ð8Ý! -°Ñ8Ô8Ð8ð %ð 	SØÔ4ð rÝ Ð!pÑqÔqÐqå�KŠKØ~ñô ð õ OÈ}ÔObÑcÔcð fÝ—’ÐRÑSÔSÐSÐSå—’ÐdÑeÔeÐeå.¨}Ñ=Ô=ð SÝ—’ÐWÑXÔXÐXÐXå—’ÐQÑRÔRÐRàÔ/ð 	å2°=À-ÑPÔPˆLÝ�KŠKÝ�|Ñ$Ô$ð  [ð  [Ð<ZÐ<ZÈ\Ð<ZÑ<ZÔ<Zð  [ð  [ð  [ñô ð ð Ô,°Ò1Ð1Ð1Ð3ZÑ1Ô1Ð1àŒ×Òå”×*Ò*¨9°mÔ6IÑJÔJÝ”×*Ò*¨9°mÔ6IÑJÔJÝ”×*Ò*Ð+CÀVÔEbÑcÔcðñ	
ô 	
ð 	
ñ 	
ð 'ð 	jð Ô3ð å6Ð7NÐP]Ñ^Ô^�Ý—’Ý˜<Ñ(Ô(ð  dð  dÐ@^Ð@^ÐQ]Ð@^Ñ@^Ô@^ð  dð  dð  dñô ð ð
 )ð aÝ—’ÐwÑxÔxÐxÝAÐBYÔB_Ñ`Ô`Ð`ð Ô0ð qÕ9mØ'Ô-¨}¸eñ:ô :ð qõ !Ð!oÑpÔpÐpàŒN×!Ò!¥$¤+×"<Ò"<¸^ÐMdÔMjÑ"kÔ"kÑlÔlÐlÐlõ -¨]Ô-@ÑAÔAˆLÝ�KŠK�3˜|Ñ,Ô,ÐhÐhÐhÑiÔiÐið %ð 	SÝ�KŠKÐnÑoÔoÐoÝ=¸mÔ>QÑRÔRÐRð Ô,ð 	hÕ5iØÔ °ñ6
ô 6
ð 	hõ ÐfÑgÔgÐgàŒ×Ò�dœk×8Ò8¸ÀMÔDWÑXÔXÑYÔYÐYõ ”×2Ò2°;ÅÔ@QÐT`ÐbsÐStÑuÔu€IÝ”×3Ò3°LÅ+ÔBSÐVWÐUXÑYÔY€JÝ”×3Ò3°LÅ+ÔBSÐVWÐUXÑYÔY€JÝ”×2Ò2°;ÅÔ@QÐTUÐSVÑWÔW€IÝœ;×=Ò=Ð>TÕVaÔVgÐjkÐilÑmÔmÐÝ”[×7Ò7Ð8HÍ+ÔJ[Ð^_Ð]`ÑaÔa€NÝœ×;Ò;Ð<PÕR]ÔRcÐfgÐehÑiÔiÐà€LØð 
àØØØØ ØØð
ˆˆð 
ð 
˜Kð 
àØØØð	
ˆð „ð (Ý”[×7Ò7¸ÅkÔFWÐZdÐYeÑfÔfˆ
Ø×Ò˜JÑ'Ô'Ð'àÔð /Ý œK×>Ò>Ø¥Ô!2°\À:Ð4Nñ
ô 
Ðð 	×ÒÐ-Ñ.Ô.Ð.àÔ!ð ,Ýœ×;Ò;Ø�kÔ/°,Ð@QÐ1Rñ
ô 
ˆð 	×Ò˜NÑ+Ô+Ð+à„{ð +�tÔ)ð +Ýœ×:Ò:Ø�[Ô.°¸zÐ0Jñ
ô 
ˆð 	×Ò˜MÑ*Ô*Ð*àð "�t”yð "ÝŒ{×1Ò1°&½+Ô:KÈaÈSÑQÔQˆØ×Ò˜DÑ!Ô!Ð!ð €IØð 
Ý”K×6Ò6ØÝÔØ@Ð@Ð@ñ
ô 
ˆ	ˆ	ð
 
ð 
˜Kð 
Ý”K×6Ò6ØÝÔØ˜<Ð(ñ
ô 
ˆ	ð �K€MàÔ#ð /Ýœ;×=Ò=ØÝÔØÐ1Ð2ñ
ô 
Ðð
 	×ÒÐ-Ñ.Ô.Ð.àÔð %Ý”×3Ò3ØÝÔØ+¨\¸;È
ÐSñ
ô 
ˆð
 	×Ò˜VÑ$Ô$Ð$å”×&Ò&Ø	ˆØ1@Ð	hˆDŒOÐ	)Ð	)Ð	)Ð	)ÈÌÐFhÐFhÐFhØØØñô €Iõ ”×&Ò&ØØ0Ø#Ô0ð 'ñ ô €Ið Ô$ð *Ø%Ð%Ð%Ð%Ð%Ð%à�=Š=�Ô)Ñ*Ô*¨W¯]ª]¸8Ñ-DÔ-DÒDÐDÝ�NŠNÐMÑNÔNÐNåŒØØŒKØ"&Ø$(ð		
ñ 	
ô 	
ð 	
ð 	
õ 	Œ	�)˜Tœ[Ñ)Ô)Ð)Ý
‡K‚KÐ. ¤Ð.Ð.Ñ/Ô/Ð/Ð/Ð/r*   rð   rò   rÞ  rß  Úbad_words_idsc                 óF  — | j         r-t          j                             ¦   «         st	          d¦  «        ‚| j        t          j        j        k    r| 	                    ¦   «          t          j
        | j         rdnd¦  «        }|                     |¦  «         t          j        d¦  «         |                     |¦  «        }|                     |¦  «        }g }t          | j        ¦  «        D ]š}	t          j        ¦   «         }
|                     ||| j        | j        | j        | j        | j        ||| j        | j        | j        |r|ndd| j        p| j        ¬¦  «        }	|                     t          j        ¦   «         |
z
  ¦  «         Œ›|j        d         }dd	lm}  |||¦  «        S )
aœ  Test PyTorch performance of text generation.

    Args:
        args (argparse.Namespace): arguments parsed from command line
        model (Union[GPT2LMHeadModel, T5ForConditionalGeneration]): PyTorch model
        input_ids (torch.Tensor): input_ids
        attention_mask (torch.Tensor): Attention mask
        eos_token_id (int): EOS token ID
        pad_token_id (int): Padding token ID
        bad_words_ids (List[List[int]]): Words shall not be generated.

    Raises:
        RuntimeError: PyTorch with CUDA is not available for --use_gpu

    Returns:
        Dict[str, Any]: A dictionary with string with metric name, and value can be integer or string.
    z=Please install PyTorch with Cuda for testing gpu performance.zcuda:0ÚcpuFNT©rð   rò   rÎ  rÏ  rÐ  rT   rà  rÞ  rß  rÑ  rÒ  rÓ  r  Úreturn_dict_in_generateÚoutput_scoresr   ©Úget_latency_result)re   ÚtorchÚcudaÚis_availablerè   r…   r   rt   r&   ÚhalfÚdeviceró  Úset_grad_enabledr  Ú
total_runsÚtimeÚgeneraterÎ  rÏ  rÐ  rT   rà  rÑ  rÒ  rÓ  rR   rS   rˆ   r7  Úbenchmark_helperr  )r~   r°   rð   rò   rÞ  rß  r  r  Útorch_latencyrÎ  ÚstartrŒ  r  s                r(   Útest_torch_performancer   ÷  s¡  € ð4 „|ð \�EœJ×3Ò3Ñ5Ô5ð \ÝÐZÑ[Ô[Ð[à„~�Ô*Ô0Ò0Ð0Ø�
Š
‰ŒˆåŒ\ d¤lÐ=˜(˜(¸Ñ>Ô>€FØ	‡H‚HˆVÑÔÐå	Ô˜5Ñ!Ô!Ð!Ø—’˜VÑ$Ô$€IØ#×&Ò& vÑ.Ô.€Nà€MÝ�4”?Ñ#Ô#ð 2ð 2ˆÝ”	‘”ˆØ�NŠNØØ)Ø”Ø”Ø”nØÔ.Ø!%Ô!:Ø%Ø%Ø!%Ô!:ØÔ.Ø#Ô6Ø+8ÐB˜-˜-¸dØ$(ØÔ6ÐR¸$Ô:Rð ñ 
ô 
ˆð" 	×Ò�TœY™[œ[¨5Ñ0Ñ1Ô1Ð1Ð1Ø” Ô#€JØ3Ð3Ð3Ð3Ð3Ð3àÐ˜m¨ZÑ8Ô8Ð8r*   c                 ó  — t          j        | j        t           j        ¬¦  «        }t	          | j        d         ¦  «        D ]J}d}t	          | j        d         ¦  «        D ]+}| |         |         |k    r|dk    rd||         |<   Œ&|dz  }Œ,ŒK|S )Nr¸   r   r\   )rÉ   Úonesr7  Úint32r  )rð   rß  rò   rö   Úabs_posrB  s         r(   Úcreate_attention_maskr%  9  s    € Ý”W˜Yœ_µB´HÐ=Ñ=Ô=€NÝ�9”? 1Ô%Ñ&Ô&ð ð ˆØˆÝ�y” qÔ)Ñ*Ô*ð 	ð 	ˆAØ˜Œ|˜AŒ ,Ò.Ð.°7¸a²<°<Ø'(�˜qÔ! !Ñ$Ð$à˜1‘��ð		ð
 Ðr*   FÚ	sentencesÚ	is_greedyc                 óH  — | j         dk    sJ ‚t          j        | j        | j        ¬¦  «        }d|_        |j        |_        t          j        | j        | j        |j	        ¬¦  «        }|€g d¢} ||dd¬	¦  «        }|d
         }|d         }d}| 
                    |d¬¦  «        }	d„ |	D ¦   «         }	| j        rt                               d|	¦  «         ng }	|j        }
|
j	        }|
j	        }|
j        }g }d}| j        �s@t#          d¦  «         t#          d¦  «         |                     ||| j        | j        | j        | j        | j        ||| j        | j        | j        |	r|	ndd| j        p| j        ¬¦  «        }t#          d
|¦  «         t#          d¦  «         t#          d|j        ¦  «         | j        rt#          d|j        ¦  «         | j        rt#          d|j        ¦  «         tA          |j        ¦  «        D ]E\  }}| !                    |d¬¦  «        }| "                    |¦  «         t#          |› d|› �¦  «         ŒFt#          d¦  «         t#          d¦  «         |r¶| #                    ¦   «          $                    ¦   «          %                    tL          j'        ¦  «        tM          j(        | j        gtL          j'        ¬¦  «        tM          j(        | j        gtL          j'        ¬¦  «        tM          j(        | j        gtL          j)        ¬¦  «        dœ}�n#| #                    ¦   «          $                    ¦   «          %                    tL          j'        ¦  «        tM          j(        | j        gtL          j'        ¬¦  «        tM          j(        | j        gtL          j'        ¬¦  «        tM          j(        | j        gtL          j'        ¬¦  «        tM          j(        | j        gtL          j'        ¬¦  «        tM          j(        | j        gtL          j)        ¬¦  «        tM          j(        | j        gtL          j)        ¬¦  «        dœ}| j        r6tM          j*        |tL          j'        ¬¦  «        }| j        r
|	D ]}d||<   Œ||d<   | j+        rtY          ||¦  «        |d<   |j-        d         }| j.        rAt           /                    d¦  «         tM          j*        ||ftL          j'        ¬¦  «        }||d <   | j0        r¾tc          | j2        ¦  «        j3         4                    ¦   «         }t                               d!|¦  «         dd"l5m6} t           /                    d#|› d$�¦  «         |g}tA          |¦  «        D ]A\  }}tn          j8         9                    |d%tu          |¦  «        z   ¦  «        } |||¦  «         ŒBt                               d&|¦  «         | j;        rdS t                               d'¦  «         ty          | j2        | j=        | j>        ¦  «        }t                               d(¦  «         | ?                    d|¦  «        }g }t�          | jA        ¦  «        D ]T}t…          jB        ¦   «         }| ?                    d|¦  «        }| "                    t…          jB        ¦   «         |z
  ¦  «         ŒUdd)lCmD}  |j-        d         } | ||¦  «        }!t#          d*¦  «         |d         }"t#          d|"¦  «         | j        rt#          d|d+         ¦  «         | j        rt#          d|d,         ¦  «         |rf|"j-        \  }}#g }$t�          |¦  «        D ]I}| !                    |"|         d¬¦  «        }|$ "                    |¦  «         t#          d-|› d.|› �¦  «         ŒJn�|"j-        \  }}%}#g }$t�          |¦  «        D ]d}t�          |%¦  «        D ]R}&| !                    |"|         |&         d¬¦  «        }|$ "                    |¦  «         t#          d-|› d/|&› d|› �¦  «         ŒSŒe|rÛ|j         E                    || j        d0¦  «        }'t�          jG        |"¦  «        }(t#          d¦  «         t#          d1¦  «         t#          |'¦  «         t#          |¦  «         t#          d¦  «         t#          d2¦  «         t#          |(¦  «         t#          |$¦  «         t#          d¦  «         ||$k    })t#          d3|)rd4nd5¦  «         |)|!d6<   | jH        r%t“          | ||||||	¦  «        }*t#          d7|*¦  «         t#          d8|!¦  «         |!S )9a9  Test GPT-2 model

    Args:
        args (argparse.Namespace): arguments parsed from command line
        sentences (Optional[List[str]], optional): input text. Defaults to None.

    Returns:
        Union[Dict[str, Any], None]: A dictionary with string with metric name, and value can be integer or string.
    r:   rÌ  Úleft)r†   rß  N)zThe product is releasedzI enjoy walking in the parkzTest best way to investÚptT©Úreturn_tensorsrØ   rð   rò   úwalk in park)Úadd_prefix_spacec                 ó   — g | ]}|g‘ŒS r1   r1   ©râ   Úword_ids     r(   rä   z"test_gpt_model.<locals>.<listcomp>m  ó   € Ð<Ð<Ð< 7�g�YÐ<Ð<Ð<r*   r  ú2--------------------------------------------------úCTest PyTorch model and beam search with huggingface transformers...r  ú!huggingface transformers outputs:rÕ  rÖ  r×  ©Úskip_special_tokensú: ú'Testing beam search with onnxruntime...r¸   rÔ  rÍ  r   rU   zYUse prefix vocab mask with all ones in ORT, but no corresponding setting for Torch model.rX   Útest_data_dir©Úoutput_test_datazSaving test_data to z/test_data_set_* ...Útest_data_set_ú
ORT inputszCreating ort session......zRun ort session......r  úORT outputs:r\   r¶   úbatch z sequence: ú
 sequence rb   úTorch Sequences:úORT Sequences:zTorch and ORT result isÚsameÚ	differentÚparityúTorch LatencyÚORT)Jr˜   r   rõ  rƒ   r†   Úpadding_sideÚ	eos_tokenÚ	pad_tokenr   rÞ  ÚencoderU   r‹   r�   rÃ  rç  rf   r`  r  rÎ  rÏ  rÐ  rT   rà  rÑ  rÒ  rÓ  rR   rS   rÕ  rÖ  r×  r  Údecoderˆ   r  ÚnumpyÚastyperÉ   r#  ÚarrayÚfloat32r"  rY   r%  r7  rX   rŒ   ri   r   r›   rœ   rô  Úbert_test_datar<  rp   rq   ro   rn   rg   rí   re   rc   Úrunr  r  r  r  r  r0  r  Ú
LongTensorrh   r   )+r~   r&  r'  Ú	tokenizerr°   rï  rð   rò   Ú	bad_wordsr  rÃ  rÞ  rß  rç  Útorch_decoded_sequencesÚbeam_outputsrö   ÚsequenceÚdecoded_sequencerU   Úbad_word_idrŒ  rX   r:  r<  Ú
all_inputsÚdirrì   ÚresultÚlatencyrÎ  r  r  r›   rÕ  rÎ  Úort_decoded_sequencesÚnum_sequencesrB  Útorch_sequencesÚort_sequencesÚis_sameÚtorch_latency_outputs+                                              r(   Útest_gpt_modelrf  E  s€	  € ð Œ?˜fÒ$Ð$Ð$Ð$åÔ-¨dÔ.EÐQUÔQ_Ð`Ñ`Ô`€IØ#€IÔØ#Ô-€IÔåÔ+ØÔØ”.ØÔ+ðñ ô €Eð Ðð
ð 
ð 
ˆ	ð ˆY�y°¸tÐDÑDÔD€FØ�{Ô#€IØÐ,Ô-€Nà€IØ×$Ò$ YÀÐ$ÑFÔF€MØ<Ð<¨mÐ<Ñ<Ô<€MØ„ð Ý�Š�_ mÑ4Ô4Ð4Ð4àˆàŒ\€FØÔ&€LØÔ&€LØÔ"€Jà ÐØ€LØÔñ .Ýˆh‰ŒˆÝÐSÑTÔTÐTØ—~’~ØØ)Ø”Ø”Ø”nØÔ.Ø!%Ô!:Ø%Ø%Ø!%Ô!:ØÔ.Ø#Ô6Ø+8ÐB˜-˜-¸dØ$(ØÔ6ÐR¸$Ô:Rð &ñ 
ô 
ˆõ" 	ˆk˜9Ñ%Ô%Ð%ÝÐ1Ñ2Ô2Ð2Ýˆk˜<Ô1Ñ2Ô2Ð2ØÔ'ð 	EÝÐ$ lÔ&CÑDÔDÐDØÔ#ð 	1Ý�(˜LÔ/Ñ0Ô0Ð0Ý$ \Ô%;Ñ<Ô<ð 	.ð 	.‰KˆAˆxØ(×/Ò/°ÈdÐ/ÑSÔSÐØ#×*Ò*Ð+;Ñ<Ô<Ð<Ý�QÐ,Ð,Ð*Ð,Ð,Ñ-Ô-Ð-Ð-å	ˆ(�O„O€OÝ	Ð
3Ñ4Ô4Ð4àð 
à"Ÿš™œ×.Ò.Ñ0Ô0×7Ò7½¼ÑAÔAÝœ( D¤OÐ#4½B¼HÐEÑEÔEÝœ( D¤OÐ#4½B¼HÐEÑEÔEÝ"$¤(¨DÔ,CÐ+DÍBÌJÐ"WÑ"WÔ"Wð	
ð 
ˆ‰ð #Ÿš™œ×.Ò.Ñ0Ô0×7Ò7½¼ÑAÔAÝœ( D¤OÐ#4½B¼HÐEÑEÔEÝœ( D¤OÐ#4½B¼HÐEÑEÔEÝœ 4¤>Ð"2½"¼(ÐCÑCÔCÝ$&¤H¨dÔ.GÐ-HÕPRÔPXÐ$YÑ$YÔ$YÝ œh¨Ô(;Ð'<ÅBÄJÐOÑOÔOÝ"$¤(¨DÔ,CÐ+DÍBÌJÐ"WÑ"WÔ"Wð
ð 
ˆð „ð *Ý”W˜jµ´Ð:Ñ:Ô:ˆ
ØŒ?ð 	,Ø,ð ,ð ,�Ø*+�
˜;Ñ'Ð'Ø)ˆˆ|ÑàÔ!ð RÝ#8¸ÀLÑ#QÔ#QˆÐÑ à” Ô#€JØÔð 8Ý�ŠÐoÑpÔpÐpÝœG Z°Ð$<ÅBÄHÐMÑMÔMÐØ&7ˆÐ"Ñ#àÔð 
*Ý˜Tœ[Ñ)Ô)Ô0×9Ò9Ñ;Ô;ˆÝ�Š�_ mÑ4Ô4Ð4Ø3Ð3Ð3Ð3Ð3Ð3å�ŠÐN¨=ÐNÐNÐNÑOÔOÐOà�Xˆ
Ý" :Ñ.Ô.ð 	*ð 	*‰IˆAˆvÝ”'—,’,˜}Ð.>ÅÀQÁÄÑ.GÑHÔHˆCØÐ˜S &Ñ)Ô)Ð)Ð)å
‡L‚L�˜vÑ&Ô&Ð&àÔð Øˆå
‡L‚LÐ-Ñ.Ô.Ð.Ý$ T¤[°$´,ÀÔ@XÑYÔY€Kå
‡L‚LÐ(Ñ)Ô)Ð)Ø�_Š_˜T 6Ñ*Ô*€Fð €GÝ�4”?Ñ#Ô#ð ,ð ,ˆÝ”	‘”ˆØ�OŠO˜D &Ñ)Ô)ˆØ�Š•t”y‘{”{ UÑ*Ñ+Ô+Ð+Ð+à3Ð3Ð3Ð3Ð3Ð3à” Ô#€JØÐ ¨Ñ4Ô4€Få	ˆ.ÑÔÐØ�q”	€IÝ	ˆ+�yÑ!Ô!Ð!ØÔ#ð -ÝÐ  &¨¤)Ñ,Ô,Ð,ØÔð #Ýˆh˜˜qœ	Ñ"Ô"Ð"àð EØ#,¤?Ñ ˆ�ZØ "ÐÝ�zÑ"Ô"ð 	=ð 	=ˆAØ(×/Ò/°	¸!´ÐRVÐ/ÑWÔWÐØ!×(Ò(Ð)9Ñ:Ô:Ð:ÝÐ;˜1Ð;Ð;Ð)9Ð;Ð;Ñ<Ô<Ð<Ð<ð	=ð
 3<´/Ñ/ˆ�] JØ "ÐÝ�zÑ"Ô"ð 	Eð 	EˆAÝ˜=Ñ)Ô)ð Eð E�Ø#,×#3Ò#3°I¸a´LÀ´OÐY]Ð#3Ñ#^Ô#^Ð Ø%×,Ò,Ð-=Ñ>Ô>Ð>ÝÐC˜qÐCÐC¨AÐCÐCÐ1AÐCÐCÑDÔDÐDÐDðEð
 ð #Ø&Ô0×8Ò8¸ÀTÔE^Ð`bÑcÔcˆÝÔ(¨Ñ3Ô3ˆÝˆh‰ŒˆÝÐ Ñ!Ô!Ð!ÝˆoÑÔÐÝÐ%Ñ&Ô&Ð&Ýˆh‰ŒˆÝÐÑÔÐÝˆmÑÔÐÝÐ#Ñ$Ô$Ð$Ýˆh‰Œˆà)Ð-BÒBˆÝÐ'°7Ð)K¨¨ÀÑLÔLÐLØ"ˆˆxÑàÔð 
5Ý5ØØØØØØØñ 
ô  
Ðõ 	ˆoÐ3Ñ4Ô4Ð4å	ˆ%�ÑÔÐà€Mr*   c                 óJ  — | j         dv sJ ‚| j        rt                               d¦  «         dS t	          j        | j        | j        ¬¦  «        }d|_        | j         dk    r!t          j        | j        | j        ¬¦  «        }n t          j        | j        | j        ¬¦  «        }|€ddg} ||d	d
¬¦  «        }|d         }|d         }d}|                     |¦  «        dd…         }d„ |D ¦   «         }| j        rt                               d|¦  «         ng }|j        }	|	j        }
|	j        }|	j        }t                               d|
› d|› d|› �¦  «         g }| j        �s@t%          d¦  «         t%          d¦  «         |                     ||| j        | j        | j        | j        | j        |
|| j        | j        | j        |r|ndd
| j        p| j        ¬¦  «        }t%          d|¦  «         t%          d¦  «         t%          d|j        ¦  «         | j        rt%          d|j        ¦  «         | j        rt%          d|j         ¦  «         tC          |j        ¦  «        D ]E\  }}| "                    |d
¬¦  «        }| #                    |¦  «         t%          |› d|› �¦  «         ŒFt%          d¦  «         t%          d¦  «         tI          j%        |tH          j&        ¬¦  «        }| j        r
|D ]}d ||<   Œ| '                    ¦   «          (                    ¦   «          )                    tH          j&        ¦  «        tI          j*        | j        gtH          j&        ¬¦  «        tI          j*        | j        gtH          j&        ¬¦  «        tI          j*        | j        gtH          j&        ¬¦  «        tI          j*        | j        gtH          j&        ¬¦  «        tI          j*        | j        gtH          j+        ¬¦  «        tI          j*        | j        gtH          j+        ¬¦  «        d!œ}| j        r||d"<   | j,        rt[          ||¦  «        |d<   | j.        r t_          | j0        ¦  «        j1         2                    ¦   «         }t                               d#|¦  «         d d$l3m4} |g}tC          |¦  «        D ]A\  }}tj          j6         7                    |d%tq          |¦  «        z   ¦  «        } |||¦  «         ŒBt                               d&|¦  «         ts          | j0        | j:        | j;        ¦  «        }g }ty          | j=        ¦  «        D ]T}t}          j>        ¦   «         }| ?                    d|¦  «        }| #                    t}          j>        ¦   «         |z
  ¦  «         ŒU|j@        d          }d d'lAmB}  |||¦  «        }t%          d(¦  «         |d          } t%          d| ¦  «         | j        rt%          d|d)         ¦  «         | j        rt%          d|d*         ¦  «         | j@        \  }}!}"g }#ty          |¦  «        D ]d}ty          |!¦  «        D ]R}$| "                    | |         |$         d
¬¦  «        }|# #                    |¦  «         t%          d+|› d,|$› d|› �¦  «         ŒSŒe| j        sÛ|j         C                    || j        d¦  «        }%t‰          jE        | ¦  «        }&t%          d¦  «         t%          d-¦  «         t%          |%¦  «         t%          |¦  «         t%          d¦  «         t%          d.¦  «         t%          |&¦  «         t%          |#¦  «         t%          d¦  «         ||#k    }'t%          d/|'rd0nd1¦  «         |'|d2<   | jF        r%t�          | ||||
||¦  «        }(t%          d3|(¦  «         t%          d4|¦  «         |S )5a=  Test T5 or MT5 model

    Args:
        args (argparse.Namespace): arguments parsed from command line
        sentences (Optional[List[str]], optional): input text. Defaults to None.

    Returns:
        Union[Dict[str, Any], None]: A dictionary with string with metric name, and value can be integer or string.
    rè  zLSkipping parity test as prefix vocab mask is not implemented by Hugging FaceNrÌ  r)  r;   z4translate English to French: The product is releasedz¬summarize: research continues to show that pets bring real health benefits to their owners. Having a dog around can lead to lower levels of stress for both adults and kids.r*  Tr+  rð   rò   r-  rb   c                 ó   — g | ]}|g‘ŒS r1   r1   r0  s     r(   rä   z!test_t5_model.<locals>.<listcomp>I  r2  r*   r  zeos_token_id:z, pad_token_id:z, vocab_size:r3  r4  r  r5  rÕ  rÖ  r×  r6  r8  r9  r¸   r   rÍ  rU   r:  r;  r=  r>  r  r?  r\   r¶   r@  rA  rB  rC  zTorch and ORT result is rD  rE  rF  rG  rH  )Hr˜   rX   r‹   r�   r   rõ  rƒ   r†   rI  r   r   rL  rU   rÃ  rÞ  rß  rç  rf   r`  r  rÎ  rÏ  rÐ  rT   rà  rÑ  rÒ  rÓ  rR   rS   rÕ  rÖ  r×  r  rM  rˆ   rÉ   r"  r#  r  rN  rO  rP  rQ  rY   r%  ri   r   r›   rœ   rô  rR  r<  rp   rq   ro   rn   rí   re   rc   r  r  r  rS  r7  r  r  r0  r  rT  rh   r   ))r~   r&  rU  r°   rï  rð   rò   rV  r  rÃ  rÞ  rß  rç  rW  rX  rö   rY  rZ  rU   r[  r:  r<  r\  r]  rì   r_  rÎ  r  r^  rŒ  r  r›   rÕ  ra  rÎ  r`  rB  rb  rc  rd  re  s)                                            r(   Útest_t5_modelri    s  € ð Œ?˜mÐ+Ð+Ð+Ð+àÔð Ý�ŠÐcÑdÔdÐdØˆtåÔ+¨DÔ,CÈtÌ~Ð^Ñ^Ô^€IØ#€IÔà„˜$ÒÐÝ*Ô:ØÔ#Ø”nð
ñ 
ô 
ˆˆõ
 ,Ô;ØÔ#Ø”nð
ñ 
ô 
ˆð ÐàBð {ð
ˆ	ð ˆY�y°¸tÐDÑDÔD€FØ�{Ô#€IØÐ,Ô-€Nà€IØ×$Ò$ YÑ/Ô/°°°Ô4€MØ<Ð<¨mÐ<Ñ<Ô<€MØ„ð Ý�Š�_ mÑ4Ô4Ð4Ð4àˆàŒ\€FØÔ&€LØÔ&€LØÔ"€JÝ
‡L‚LÐe ÐeÐe¸lÐeÐeÐYcÐeÐeÑfÔfÐfà ÐØÔñ .Ýˆh‰ŒˆÝÐSÑTÔTÐTØ—~’~ØØ)Ø”Ø”Ø”nØÔ.Ø!%Ô!:Ø%Ø%Ø!%Ô!:ØÔ.Ø#Ô6Ø+8ÐB˜-˜-¸dØ$(ØÔ6ÐR¸$Ô:Rð &ñ 
ô 
ˆõ$ 	ˆk˜9Ñ%Ô%Ð%ÝÐ1Ñ2Ô2Ð2Ýˆk˜<Ô1Ñ2Ô2Ð2ØÔ'ð 	EÝÐ$ lÔ&CÑDÔDÐDØÔ#ð 	1Ý�(˜LÔ/Ñ0Ô0Ð0Ý$ \Ô%;Ñ<Ô<ð 	.ð 	.‰KˆAˆxØ(×/Ò/°ÈdÐ/ÑSÔSÐØ#×*Ò*Ð+;Ñ<Ô<Ð<Ý�QÐ,Ð,Ð*Ð,Ð,Ñ-Ô-Ð-Ð-å	ˆ(�O„O€OÝ	Ð
3Ñ4Ô4Ð4å”˜*­R¬XÐ6Ñ6Ô6€JØ„ð (Ø(ð 	(ð 	(ˆKØ&'ˆJ�{Ñ#Ð#ð —]’]‘_”_×*Ò*Ñ,Ô,×3Ò3µB´HÑ=Ô=Ý”h ¤Ð0½¼ÐAÑAÔAÝ”h ¤Ð0½¼ÐAÑAÔAÝ”X˜tœ~Ð.µb´hÐ?Ñ?Ô?Ý "¤¨$Ô*CÐ)DÍBÌHÐ UÑ UÔ UÝœ( DÔ$7Ð#8ÅÄ
ÐKÑKÔKÝ œh¨Ô(?Ð'@ÍÌ
ÐSÑSÔSðð €Fð „ð *Ø)ˆˆ|ÑàÔ!ð RÝ#8¸ÀLÑ#QÔ#QˆÐÑ àÔð *Ý˜Tœ[Ñ)Ô)Ô0×9Ò9Ñ;Ô;ˆÝ�Š�_ mÑ4Ô4Ð4Ø3Ð3Ð3Ð3Ð3Ð3à�Xˆ
Ý" :Ñ.Ô.ð 	*ð 	*‰IˆAˆvÝ”'—,’,˜}Ð.>ÅÀQÁÄÑ.GÑHÔHˆCØÐ˜S &Ñ)Ô)Ð)Ð)å
‡L‚L�˜vÑ&Ô&Ð&å$ T¤[°$´,ÀÔ@XÑYÔY€Kð €GÝ�4”?Ñ#Ô#ð ,ð ,ˆÝ”	‘”ˆØ—’  vÑ.Ô.ˆØ�Š•t”y‘{”{ UÑ*Ñ+Ô+Ð+Ð+Ø” Ô#€JØ3Ð3Ð3Ð3Ð3Ð3àÐ ¨Ñ4Ô4€Få	ˆ.ÑÔÐØ�q”	€IÝ	ˆ+�yÑ!Ô!Ð!ØÔ#ð -ÝÐ  &¨¤)Ñ,Ô,Ð,ØÔð #Ýˆh˜˜qœ	Ñ"Ô"Ð"à.7¬oÑ+€Z� 
ØÐÝ�:ÑÔð Að AˆÝ�}Ñ%Ô%ð 	Að 	AˆAØ(×/Ò/°	¸!´¸Q´ÐUYÐ/ÑZÔZÐØ!×(Ò(Ð)9Ñ:Ô:Ð:ÝÐ?˜1Ð?Ð?¨Ð?Ð?Ð-=Ð?Ð?Ñ@Ô@Ð@Ð@ð	Að
 Ôð #Ø&Ô0×8Ò8¸ÀTÔE^Ð`bÑcÔcˆÝÔ(¨Ñ3Ô3ˆÝˆh‰ŒˆÝÐ Ñ!Ô!Ð!ÝˆoÑÔÐÝÐ%Ñ&Ô&Ð&Ýˆh‰ŒˆÝÐÑÔÐÝˆmÑÔÐÝÐ#Ñ$Ô$Ð$Ýˆh‰Œˆà)Ð-BÒBˆÝÐ(°GÐ*L¨&¨&ÀÑMÔMÐMØ"ˆˆxÑàÔð 
5Ý5ØØØØØØØñ 
ô  
Ðõ 	ˆoÐ3Ñ4Ô4Ð4å	ˆ%�ÑÔÐØ€Mr*   c                 ó^  — t          | ¦  «        }t          |j        ¦  «         |j        dv r¯|j        r;t
          j                             |j        ¦  «        st          d|j        › �¦  «        ‚|j	        r;t
          j                             |j	        ¦  «        st          d|j	        › �¦  «        ‚|j        r|j	        r|j	        r|j        st          d¦  «        ‚|j
        dk    o
|j        dk    }|j        dk    rƒ|r�|j        dk    r[|j        dk     rPt          |t          j        ¦  «         t                                d	¦  «         |j        d
k    s|j        s|j        rdS n*t          |t          j        ¦  «         nt          |¦  «         t                                d¦  «         |j        dv rt+          ||¬¦  «        }nt-          |||¬¦  «        }|rU|j        r,t                                d|j        › d|j        › d�¦  «         n"t                                d|j        › �¦  «         |S )a/  Main entry function

    Args:
        argv (Optional[List[str]], optional): _description_. Defaults to None.
        sentences (Optional[List[str]], optional): input text. Defaults to None.

    Raises:
        ValueError: Path does not exist: --encoder_decoder_init_onnx
        ValueError: Path does not exist: --decoder_onnx
        ValueError: --decoder_onnx and --encoder_decoder_init_onnx are not used together for T5

    Returns:
        Union[Dict[str, Any], None]: A dictionary with string with metric name, and value can be integer or string.
    rè  z1Path does not exist: --encoder_decoder_init_onnx z$Path does not exist: --decoder_onnx zB--decoder_onnx shall use together with --encoder_decoder_init_onnxr\   r:   ra   r_   z�The test for gpt2_sampling onnx model is limited to non-custom model with small top_p(e.g <=0.01) value. The result should be the same as gpt2 greedy search.g{®Gáz„?Nzstart testing model...)r&  )r&  r'  zOutput files: r6   z.datazOutput file: )r   r   rD   r˜   rž   rp   rq   ró  r  r„   rÐ  rÑ  râ  r  r    r0   r‹   rŒ   rå  r[   r/   ri  rf  rL   r›   )r2   r&  r~   r'  r^  s        r(   r   r   Ù  sW  € õ  ˜4Ñ Ô €DÝ�”ÑÔÐà„˜-Ð'Ð'ØÔ)ð 	sµ"´'·.².ÀÔA_Ñ2`Ô2`ð 	sÝÐqÐQUÔQoÐqÐqÑrÔrÐrØÔð 	Y¥R¤W§^¢^°DÔ4EÑ%FÔ%Fð 	YÝÐWÀDÔDUÐWÐWÑXÔXÐXØÔ*ð 	c°4Ô3Dð 	cØÔð	cØ&*Ô&Dð	cõ ÐaÑbÔbÐbà” !Ò#ÐF¨Ô(AÀQÒ(F€Ià„˜&Ò Ð  YÐ ØŒ:˜ÒÐ ¤
¨SÒ 0Ð 0Ý$ T­>Ô+BÑCÔCÐCÝ�KŠKð pñô ð ð Œz˜DÒ Ð  D¤KÐ °4´9Ð Ø�ð !õ % T­>Ô+FÑGÔGÐGÐGå  Ñ&Ô&Ð&å
‡K‚KÐ(Ñ)Ô)Ð)Ø„˜-Ð'Ð'Ý˜t¨yÐ9Ñ9Ô9ˆˆå °	ÀYÐOÑOÔOˆàð 7ØÔ(ð 	7Ý�KŠKÐJ¨¬ÐJÐJ¸¼ÐJÐJÐJÑKÔKÐKÐKå�KŠKÐ5¨¬Ð5Ð5Ñ6Ô6Ð6à€Mr*   Ú__main__r%   )T)r'  r(  NN)r(  )r   r\   rb   )NFr$  )aÚ__doc__rj   ÚloggingrÆ   rp   r  Úenumr   Úpathlibr   Útypingr   rN  rÉ   r«   r  r  r   r   Úfusion_utilsr   r	   r
   r   r�  r   Útransformersr   r   r   r   r   r   r   r   Úonnxruntimer   r   r   r   Ú4onnxruntime.transformers.models.gpt2.convert_to_onnxr   r�   Ú0onnxruntime.transformers.models.gpt2.gpt2_helperr   Ú2onnxruntime.transformers.models.t5.convert_to_onnxr   rš   Ú,onnxruntime.transformers.models.t5.t5_helperr   r   Ú	getLoggerr‹   r    rÃ  rn   Ú	Namespacer   r�   r    Úboolr²   rÛ   rí   r  r  r&  ru   ÚdictrD  rO  rS  rd  rk  rt  r…  r˜  r·  rÈ  rÖ  r  r  rV  rn  r�  r—  r�  r¿  rÇ  r.   r  ÚTensorr   r%  rf  ri  r+   r1   r*   r(   ú<module>r}     sò  ðð
#ð #ðJ €€€Ø €€€Ø €€€Ø 	€	€	€	Ø €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à Ð Ð Ð Ø €€€Ø €€€Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ø $Ð $Ð $Ð $Ð $Ð $Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ø  Ð  Ð  Ð  Ð  Ð  ð	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ðð ð ð ð ð ð ð ð ð ð ð ðð ð ð ð ð ð TÐ SÐ SÐ SÐ SÐ Sðð ð ð ð ð ðð ð ð ð ð ð ð ð
 
ˆÔ	˜2Ñ	Ô	€ðð ð ð ð �Tñ ô ð ðQð Q˜$˜sœ) dÑ*ð Q°hÔ6Hð Qð Qð Qð Qðh*)�xÔ)ð *)ð *)ð *)ð *)ðZ!�XÔ'ð !ð !ð !ð !ð<Oð O˜sð O¸dð Oð Oð Oð Oð$Kð K¨Cð KÈ4ð KÐ[_ð Kð Kð Kð Kð\ 3ð °ð ÈDð ÐUeð ð ð ð ðB5 ¤ð 5¸Ið 5ð 5ð 5ð 5ðpIo d¤oð IoÀ)ð Ioð Ioð Ioð IoðXYa°4´?ð YaÈyð Yað Yað Yað Yað~ #ØØ$(Ø$(ðg!ð g!Øðg!àðg!ð ðg!ð ð	g!ð
 ˜T‘kðg!ð ˜T‘kðg!ð g!ð g!ð g!ðT¨:ð Àjð ð ð ð ð( ðð Øðàðð 
ˆ+Ôðð ð ð ð>"#ð "#ð "#ðJð ð ðið ið ið.¸Jð .ð .ð .ð .ðbSØ
ðSØ&*ðSØ>BðSà	ðSð Sð Sð Sðlf3 *ð f3ð f3ð f3ð f3ðR¨	ð Àcð ð ð ð ð* 9:ÐWYð 1ð 1 	ð 1°#ð 1È4ÐPSÌ9ð 1ð 1ð 1ð 1ðhf$ Ið f$ð f$ð f$ð f$ðRD )ð DÀð Dð Dð Dð DðT ØØðkð kØðkàðkð ðkð ð	kð
 ðkð kð kð kð\&f¸ð &fð &fð &fð &fðR\È*ð \ð \ð \ð \ð~8°ð 8ð 8ð 8ð 8ðvð À#ð Ðaeð ð ð ð ðF &*ðhð hØðhàðhð #ðhð 
ð	hð hð hð hðV	"0ð "0ð "0ðN '5Ô&?ðy0ð y0Ø
Ô
ðy0à#ðy0ð y0ð y0ð y0ðx?9Ø
Ô
ð?9àÐ7Ñ7ð?9ð Œ|ð?9ð ”Lð	?9ð
 ð?9ð ð?9ð ˜˜Sœ	”?ð?9ð 
ˆ#ˆsˆ(„^ð?9ð ?9ð ?9ð ?9ðD	ð 	ð 	ð #'ØðTð TØ
Ô
ðTà�CŒy˜4ÑðTð ðTð Tð Tð Tðnzð z˜Ô*ð z°t¸C´yÀ4Ñ7Gð zð zð zð zðz8ð 8ˆt�CŒy˜4Ñð 8°4¸´9¸tÑ3Cð 8ð 8ð 8ð 8ðv ˆzÒÐØ€D�F„F€F€F€Fð Ðr*   