§
    kŠtj;S  ã                   ó0  — d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlZd dl	Z	d dl
mZ d dlmZmZ eeeee         z  f         Z ej        e¦  «        Z G d„ d¦  «        Z G d	„ d
¦  «        Z G d„ d¦  «        Z G d„ de¦  «        Z G d„ d¦  «        ZdS )é    N)ÚOrderedDict)ÚMapping)ÚAny)ÚTensorProto)ÚInferenceSessionÚ
RunOptionsc                   ó–  — e Zd Zedededefd„¦   «         Zededefd„¦   «         Zedefd„¦   «         Zedefd„¦   «         Z	edede
eef         fd	„¦   «         Zedefd
„¦   «         Zedej        fd„¦   «         Zedej        fd„¦   «         Zedede
eej        f         fd„¦   «         Zedede
eej        f         fd„¦   «         ZdS )Ú
TypeHelperÚort_sessionÚnameÚreturnc                 óž   — t          |                      ¦   «         ¦  «        D ]\  }}|j        |k    r	|j        c S Œt	          d|› d�¦  «        ‚)Nzinput name ú
 not found)Ú	enumerateÚ
get_inputsr   ÚtypeÚ
ValueError)r   r   Ú_iÚinputs       úh/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/onnxruntime/transformers/io_binding_helper.pyÚget_input_typezTypeHelper.get_input_type   sc   € å" ;×#9Ò#9Ñ#;Ô#;Ñ<Ô<ð 	"ð 	"‰IˆB�ØŒz˜TÒ!Ð!Ø”zÐ!Ð!Ð!ð "åÐ7 tÐ7Ð7Ð7Ñ8Ô8Ð8ó    c                 óž   — t          |                      ¦   «         ¦  «        D ]\  }}|j        |k    r	|j        c S Œt	          d|› d�¦  «        ‚)Nzoutput name r   )r   Úget_outputsr   r   r   )r   r   r   Úoutputs       r   Úget_output_typezTypeHelper.get_output_type   se   € å# K×$;Ò$;Ñ$=Ô$=Ñ>Ô>ð 	#ð 	#‰JˆB�ØŒ{˜dÒ"Ð"Ø”{Ð"Ð"Ð"ð #õ Ð8¨Ð8Ð8Ð8Ñ9Ô9Ð9r   Úort_typec                 ón  — t           j        t           j        t           j        t           j        t
          t           j        t           j        t           j        t           j	        t           j
        t           j        t           j        t           j        t           j        dœ}| |vrt          | › d�¦  «        ‚||          S )N)útensor(int64)útensor(int32)útensor(float)útensor(float16)útensor(bool)útensor(uint8)útensor(int8)útensor(double)útensor(int16)útensor(uint16)útensor(uint32)útensor(uint64)útensor(complex64)útensor(complex128)ú not found in map)ÚnumpyÚint64Úint32Úfloat32Úfloat16ÚboolÚuint8Úint8Úfloat64Úint16Úuint16Úuint32Úuint64Ú	complex64Ú
complex128r   )r   Úort_type_to_numpy_type_maps     r   Úort_type_to_numpy_typez!TypeHelper.ort_type_to_numpy_type#   s†   € õ #œ[Ý"œ[Ý"œ]Ý$œ}Ý Ý"œ[Ý!œJÝ#œmÝ"œ[Ý#œlÝ#œlÝ#œlÝ!&¤Ý"'Ô"2ð&
ð &
Ð"ð  Ð5Ð5Ð5Ý Ð;Ð;Ð;Ñ<Ô<Ð<à)¨(Ô3Ð3r   c                 óZ  — i dt           j        “dt           j        “dt           j        “dt           j        “dt           j        “dt           j        “dt           j        “dt           j        “d	t           j	        “d
t           j
        “dt           j        “dt           j        “dt           j        “dt           j        “dt           j        “dt           j        “dt           j        “t           j        t           j        t           j        t           j        dœ¥}| |vrt-          | › d�¦  «        ‚||          S )Nr   r    r!   r"   útensor(bfloat16)r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   útensor(float8e4m3fn)útensor(float8e4m3fnuz))útensor(float8e5m2)útensor(float8e5m2fnuz)útensor(int4)útensor(uint4)r-   )Útorchr/   r0   r1   r2   Úbfloat16r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   Úfloat8_e4m3fnÚfloat8_e4m3fnuzÚfloat8_e5m2Úfloat8_e5m2fnuzÚint4Úuint4r   )r   Úort_type_to_torch_type_maps     r   Úort_type_to_torch_typez!TypeHelper.ort_type_to_torch_type:   sN  € ð&
Ø�Uœ[ð&
à�Uœ[ð&
ð �Uœ]ð&
ð �uœ}ð	&
ð
 ¥¤ð&
ð �EœJð&
ð �Uœ[ð&
ð �EœJð&
ð �eœmð&
ð �Uœ[ð&
ð �eœlð&
ð �eœlð&
ð �eœlð&
ð  ¥¤ð&
ð !¥%Ô"2ð&
ð  #¥EÔ$7ð!&
ð" %¥eÔ&;ð#&
õ$ #(Ô"3Ý&+Ô&;Ý!œJÝ"œ[ð+&
ð &
ð &
Ð"ð. Ð5Ð5Ð5Ý Ð;Ð;Ð;Ñ<Ô<Ð<à)¨(Ô3Ð3r   c                 ó  — i }|                       ¦   «         D ])}t                               |j        ¦  «        ||j        <   Œ*|                      ¦   «         D ])}t                               |j        ¦  «        ||j        <   Œ*|S )z9Create a mapping from input/output name to onnx data type)r   r
   Úort_type_to_onnx_typer   r   r   )r   Úname_to_onnx_typer   r   s       r   Úget_io_onnx_type_mapzTypeHelper.get_io_onnx_type_mapX   sŠ   € ð ÐØ ×+Ò+Ñ-Ô-ð 	Yð 	YˆEÝ,6×,LÒ,LÈUÌZÑ,XÔ,XÐ˜eœjÑ)Ð)à!×-Ò-Ñ/Ô/ð 	[ð 	[ˆFÝ-7×-MÒ-MÈfÌkÑ-ZÔ-ZÐ˜fœkÑ*Ð*Ø Ð r   c                 ó†  — i dt           j        “dt           j        “dt           j        “dt           j        “dt           j        “dt           j        “dt           j        “dt           j        “d	t           j	        “d
t           j
        “dt           j        “dt           j        “dt           j        “dt           j        “dt           j        “dt           j        “dt           j        “t           j        t           j        t           j        t           j        t           j        t           j        dœ¥}| |vrt1          | › d�¦  «        ‚||          S )Nr   r    r!   r"   r@   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   rA   rB   )rC   rD   ztensor(float4e2m1)rE   rF   ztensor(string)r-   )r   ÚINT64ÚINT32ÚFLOATÚFLOAT16ÚBFLOAT16ÚBOOLÚUINT8ÚINT8ÚDOUBLEÚINT16ÚUINT16ÚUINT32ÚUINT64Ú	COMPLEX64Ú
COMPLEX128ÚFLOAT8E4M3FNÚFLOAT8E4M3FNUZÚ
FLOAT8E5M2ÚFLOAT8E5M2FNUZÚ
FLOAT4E2M1ÚINT4ÚUINT4ÚSTRINGr   )r   Úort_type_to_onnx_type_maps     r   rR   z TypeHelper.ort_type_to_onnx_typec   sj  € ð%
Ø�[Ô.ð%
à�[Ô.ð%
ð �[Ô.ð%
ð �{Ô2ð	%
ð
 ¥Ô 4ð%
ð �KÔ,ð%
ð �[Ô.ð%
ð �KÔ,ð%
ð �kÔ0ð%
ð �[Ô.ð%
ð �kÔ0ð%
ð �kÔ0ð%
ð �kÔ0ð%
ð  ¥Ô!6ð%
ð !¥+Ô"8ð%
ð  #¥KÔ$<ð!%
ð" %¥kÔ&@ð#%
õ$ #.Ô"8Ý&1Ô&@Ý"-Ô"8Ý'Ô,Ý(Ô.Ý)Ô0ð/%
ð %
ð %
Ð!ð2 Ð4Ð4Ð4Ý Ð;Ð;Ð;Ñ<Ô<Ð<à(¨Ô2Ð2r   Ú
numpy_typec                 ó   — t           j        t          j        t           j        t          j        t           j        t          j        t           j        t          j        t          t          j        t           j        t          j        t           j        t          j        t           j	        t          j	        t           j
        t          j
        t           j        t          j        t           j        t          j        t           j        t          j        t           j        t          j        t           j        t          j        i}| |vrt!          | › d�¦  «        ‚||          S ©Nr-   )r.   r/   rG   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r   )rn   Únumpy_type_to_torch_type_maps     r   Únumpy_type_to_torch_typez#TypeHelper.numpy_type_to_torch_typeƒ   sº   € õ ŒK�œÝŒK�œÝŒM�5œ=ÝŒM�5œ=Ý•%”*ÝŒK�œÝŒJ�œ
ÝŒM�5œ=ÝŒK�œÝŒL�%œ,ÝŒL�%œ,ÝŒL�%œ,ÝŒO�Uœ_ÝÔ�eÔ.ð(
Ð$ð" Ð9Ð9Ð9Ý 
Ð=Ð=Ð=Ñ>Ô>Ð>à+¨JÔ7Ð7r   Ú
torch_typec                 ó   — t           j        t          j        t           j        t          j        t           j        t          j        t           j        t          j        t           j        t          t           j        t          j        t           j        t          j        t           j	        t          j	        t           j
        t          j
        t           j        t          j        t           j        t          j        t           j        t          j        t           j        t          j        t           j        t          j        i}| |vrt!          | › d�¦  «        ‚||          S rp   )rG   r/   r.   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r   )rs   Útorch_type_to_numpy_type_maps     r   Útorch_type_to_numpy_typez#TypeHelper.torch_type_to_numpy_type›   sº   € õ ŒK�œÝŒK�œÝŒM�5œ=ÝŒM�5œ=ÝŒJ�ÝŒK�œÝŒJ�œ
ÝŒM�5œ=ÝŒK�œÝŒL�%œ,ÝŒL�%œ,ÝŒL�%œ,ÝŒO�Uœ_ÝÔ�eÔ.ð(
Ð$ð" Ð9Ð9Ð9Ý 
Ð=Ð=Ð=Ñ>Ô>Ð>à+¨JÔ7Ð7r   c                 ó  — i }|                       ¦   «         D ])}t                               |j        ¦  «        ||j        <   Œ*|                      ¦   «         D ])}t                               |j        ¦  «        ||j        <   Œ*|S )z:Create a mapping from input/output name to numpy data type)r   r
   r>   r   r   r   )r   Úname_to_numpy_typer   r   s       r   Úget_io_numpy_type_mapz TypeHelper.get_io_numpy_type_map³   óŒ   € ð  ÐØ ×+Ò+Ñ-Ô-ð 	[ð 	[ˆEÝ-7×-NÒ-NÈuÌzÑ-ZÔ-ZÐ˜uœzÑ*Ð*à!×-Ò-Ñ/Ô/ð 	]ð 	]ˆFÝ.8×.OÒ.OÐPVÔP[Ñ.\Ô.\Ð˜vœ{Ñ+Ð+Ø!Ð!r   c                 ó  — i }|                       ¦   «         D ])}t                               |j        ¦  «        ||j        <   Œ*|                      ¦   «         D ])}t                               |j        ¦  «        ||j        <   Œ*|S )z:Create a mapping from input/output name to torch data type)r   r
   rP   r   r   r   )r   Úname_to_torch_typer   r   s       r   Úget_io_torch_type_mapz TypeHelper.get_io_torch_type_map¾   rz   r   N)Ú__name__Ú
__module__Ú__qualname__Ústaticmethodr   Ústrr   r   r>   rP   ÚdictÚintrT   rR   r.   Údtyperr   rG   rv   ry   r}   © r   r   r
   r
      sê  € € € € € Øð9Ð$4ð 9¸Cð 9ÀCð 9ð 9ð 9ñ „\ð9ð ð:¨3ð :°3ð :ð :ð :ñ „\ð:ð ð4¨ð 4ð 4ð 4ñ „\ð4ð, ð4¨ð 4ð 4ð 4ñ „\ð4ð: ð!Ð*:ð !¸tÀCÈÀH¼~ð !ð !ð !ñ „\ð!ð ð3¨ð 3ð 3ð 3ñ „\ð3ð> ð8¨U¬[ð 8ð 8ð 8ñ „\ð8ð. ð8¨U¬[ð 8ð 8ð 8ñ „\ð8ð. ð"Ð+;ð "ÀÀSÈ%Ì+ÐEUÔ@Vð "ð "ð "ñ „\ð"ð ð"Ð+;ð "ÀÀSÈ%Ì+ÐEUÔ@Vð "ð "ð "ñ „\ð"ð "ð "r   r
   c            
       óž   — e Zd Zedefd„¦   «         Zedej        dej        dej        deej                 fd„¦   «         Z	edd	„¦   «         Z
d
S )ÚIOBindingHelperr   c                 ó   — i }|                      ¦   «         D ]f\  }}t                               | |¦  «        }t                               |¦  «        }t	          j        t          j        |¦  «        ||¬¦  «        ||<   Œg|S )zpReturns a dictionary of output name as key, and 1D tensor as value. The tensor has enough space for given shape.)r…   Údevice)Úitemsr
   r   rP   rG   Úemptyr.   Úprod)r   Úoutput_shapesrŠ   Úoutput_buffersr   Úshaper   rs   s           r   Úget_output_buffersz"IOBindingHelper.get_output_buffersË   s�   € ð ˆØ(×.Ò.Ñ0Ô0ð 	cð 	c‰KˆD�%Ý!×1Ò1°+¸tÑDÔDˆHÝ#×:Ò:¸8ÑDÔDˆJÝ#(¤;­u¬z¸%Ñ/@Ô/@È
Ð[aÐ#bÑ#bÔ#bˆN˜4Ñ Ð ØÐr   Ú	input_idsÚposition_idsÚattention_maskÚpastc                 óÊ  — t                                | ¦  «        }|                      ¦   «         }|                     ¦   «         sJ ‚|                     d|j        j        d|d         t          |                     ¦   «         ¦  «        | 	                    ¦   «         ¦  «         |�¨t          |¦  «        D ]˜\  }	}
|
                     ¦   «         sJ ‚|
 	                    ¦   «         }|dk    r| 	                    ¦   «         }|                     d|	› �|
j        j        d|d|	› �         t          |
                     ¦   «         ¦  «        |¦  «         Œ™|�q|                     ¦   «         sJ ‚|                     d|j        j        d|d         t          |                     ¦   «         ¦  «        | 	                    ¦   «         ¦  «         |�q|                     ¦   «         sJ ‚|                     d|j        j        d|d         t          |                     ¦   «         ¦  «        | 	                    ¦   «         ¦  «         |                      ¦   «         D ]ž}|j        }||         }t                               |› d|j        j        › dt          |                     ¦   «         ¦  «        › �¦  «         |                     ||j        j        d||         ||         | 	                    ¦   «         ¦  «         ŒŸ|S )	zdIO binding for a session: bind inputs (input_ids, position_ids, attention_mask, past_*) and outputs.r’   r   NÚpast_r”   r“   z device type=z shape=)r
   rT   Ú
io_bindingÚis_contiguousÚ
bind_inputrŠ   r   ÚlistÚsizeÚdata_ptrr   r   r   ÚloggerÚdebugÚbind_output)r   r’   r“   r”   r•   r�   rŽ   rS   r˜   ÚiÚpast_ir�   r   Úoutput_nameÚoutput_buffers                  r   Úprepare_io_bindingz"IOBindingHelper.prepare_io_bindingÕ   s  € õ '×;Ò;¸KÑHÔHÐð !×+Ò+Ñ-Ô-ˆ
ð ×&Ò&Ñ(Ô(Ð(Ð(Ð(Ø×ÒØØÔÔ!ØØ˜kÔ*Ý�—’Ñ!Ô!Ñ"Ô"Ø×ÒÑ Ô ñ	
ô 	
ð 	
ð ÐÝ& t™_œ_ð ð ‘	��6Ø×+Ò+Ñ-Ô-Ð-Ð-Ð-à!Ÿ?š?Ñ,Ô,�Ø˜q’=�=ð  )×1Ò1Ñ3Ô3�Hà×%Ò%Ø˜A�K�KØ”MÔ&ØØ% k¨a k kÔ2Ý˜Ÿš™œÑ'Ô'Øñô ð ð ð Ð%Ø!×/Ò/Ñ1Ô1Ð1Ð1Ð1Ø×!Ò!Ø ØÔ%Ô*ØØ!Ð"2Ô3Ý�^×(Ò(Ñ*Ô*Ñ+Ô+Ø×'Ò'Ñ)Ô)ñô ð ð Ð#Ø×-Ò-Ñ/Ô/Ð/Ð/Ð/Ø×!Ò!ØØÔ#Ô(ØØ! .Ô1Ý�\×&Ò&Ñ(Ô(Ñ)Ô)Ø×%Ò%Ñ'Ô'ñô ð ð "×-Ò-Ñ/Ô/ð 	ð 	ˆFØ œ+ˆKØ*¨;Ô7ˆMÝ�LŠL˜KÐtÐt°mÔ6JÔ6OÐtÐtÕX\Ð]j×]oÒ]oÑ]qÔ]qÑXrÔXrÐtÐtÑuÔuÐuØ×"Ò"ØØÔ$Ô)ØØ! +Ô.Ø˜kÔ*Ø×&Ò&Ñ(Ô(ñô ð ð ð Ðr   Tc                 ó®  — g }|                       ¦   «         D ]½}|j        }||         }||         }|dt          j        |¦  «        …                              |¦  «                             ¦   «                              ¦   «         }	|r:|                     |	                     ¦   «                              ¦   «         ¦  «         Œ¨|                     |	¦  «         Œ¾|S )z3Copy results to cpu. Returns a list of numpy array.r   )	r   r   r.   r�   ÚreshapeÚcloneÚdetachÚappendÚcpu)
r   r�   rŽ   Úreturn_numpyÚort_outputsr   r£   Úbufferr�   Úcopy_tensors
             r   Ú"get_outputs_from_io_binding_bufferz2IOBindingHelper.get_outputs_from_io_binding_buffer*  sÐ   € ð ˆØ!×-Ò-Ñ/Ô/ð 	0ð 	0ˆFØ œ+ˆKØ# KÔ0ˆFØ! +Ô.ˆEØ  ¥U¤Z°Ñ%6Ô%6Ð!6Ô7×?Ò?ÀÑFÔF×LÒLÑNÔN×UÒUÑWÔWˆKØð 0Ø×"Ò" ;§?¢?Ñ#4Ô#4×#:Ò#:Ñ#<Ô#<Ñ=Ô=Ð=Ð=à×"Ò" ;Ñ/Ô/Ð/Ð/ØÐr   N)T)r~   r   r€   r�   r   r‘   rG   ÚTensorr›   r¥   r°   r†   r   r   rˆ   rˆ   Ê   s¶   € € € € € ØðÐ(8ð ð ð ñ „\ðð ðRà”<ðRð ”lðRð œð	Rð
 �5”<Ô ðRð Rð Rñ „\ðRðh ðð ð ñ „\ðð ð r   rˆ   c                   óÔ   — e Zd ZdZddedej        fd„Zdedefd„Z	d	„ Z
d
edej        fd„Zdefd„Zddeeej        f         dedefd„Zeddedededeeef         fd„¦   «         ZdS )ÚCudaSessionzLInference Session with IO Binding for ONNX Runtime CUDA or TensorRT providerFr   rŠ   c                 óö  — || _         d„ | j                              ¦   «         D ¦   «         | _        d„ | j                              ¦   «         D ¦   «         | _        t
                               | j         ¦  «        | _        t
                               | j         ¦  «        | _	        | j          
                    ¦   «         | _
        || _        t          ¦   «         | _        t          ¦   «         | _        || _        i | _        d S )Nc                 ó   — g | ]	}|j         ‘Œ
S r†   ©r   )Ú.0r   s     r   ú
<listcomp>z(CudaSession.__init__.<locals>.<listcomp>?  s   € ÐRÐRÐR¨5˜EœJÐRÐRÐRr   c                 ó   — g | ]	}|j         ‘Œ
S r†   r¶   )r·   r   s     r   r¸   z(CudaSession.__init__.<locals>.<listcomp>@  s   € ÐVÐVÐV¨V˜Vœ[ÐVÐVÐVr   )r   r   Úinput_namesr   Úoutput_namesr
   rT   Úio_name_to_onnx_typer}   Úio_name_to_torch_typer˜   Úenable_cuda_graphr   Úinput_tensorsÚoutput_tensorsrŠ   Úbuffer_sharing)Úselfr   rŠ   r¾   s       r   Ú__init__zCudaSession.__init__=  sÓ   € Ø&ˆÔØRÐR°DÔ4D×4OÒ4OÑ4QÔ4QÐRÑRÔRˆÔØVÐV°tÔ7G×7SÒ7SÑ7UÔ7UÐVÑVÔVˆÔÝ$.×$CÒ$CÀDÔDTÑ$UÔ$UˆÔ!Ý%/×%EÒ%EÀdÔFVÑ%WÔ%WˆÔ"ØÔ*×5Ò5Ñ7Ô7ˆŒØ!2ˆÔå(™]œ]ˆÔÝ)™mœmˆÔØˆŒð /1ˆÔÐÐr   Ú
input_namer£   c                 óZ   — || j         v sJ ‚|| j        v sJ ‚|| j        |<   || j        |<   d S ©N)rº   r»   rÁ   )rÂ   rÄ   r£   s      r   Úset_buffer_sharingzCudaSession.set_buffer_sharingM  sJ   € Ø˜TÔ-Ð-Ð-Ð-Ð-Ø˜dÔ/Ð/Ð/Ð/Ð/Ø*5ˆÔ˜JÑ'Ø+5ˆÔ˜KÑ(Ð(Ð(r   c                 ó   — | ` | `| `d S rÆ   )r¿   rÀ   r˜   )rÂ   s    r   Ú__del__zCudaSession.__del__S  s   € ØÐØÐØˆOˆOˆOr   r   Útensorc           	      ó  — |j         j        �|j         j        nd}t          |j        ¦  «        dk    rdgnt	          |j        ¦  «        }| j                             ||j         j        || j        |         || 	                    ¦   «         ¦  «         || j
        v rh| j                             | j
        |         |j         j        || j        |         || 	                    ¦   «         ¦  «         || j        | j
        |         <   d S d S )Nr   é   )rŠ   ÚindexÚlenr�   r›   r˜   rš   r   r¼   r�   rÁ   r    rÀ   )rÂ   r   rÊ   Ú	device_idÚtensor_shapes        r   Úbind_input_and_buffer_sharingz)CudaSession.bind_input_and_buffer_sharingX  s	  € Ø+1¬=Ô+>Ð+J�F”MÔ'Ð'ÐPQˆ	Ý! &¤,Ñ/Ô/°1Ò4Ð4˜�s�s½$¸v¼|Ñ:LÔ:LˆàŒ×"Ò"ØØŒMÔØØÔ% dÔ+ØØ�OŠOÑÔñ	
ô 	
ð 	
ð �4Ô&Ð&Ð&ØŒO×'Ò'ØÔ# DÔ)Ø”Ô"ØØÔ)¨$Ô/ØØ—’Ñ!Ô!ñô ð ð >DˆDÔ Ô 3°DÔ 9Ñ:Ð:Ð:ð 'Ð&r   Ú
shape_dictc                 ó2  — | j         rÕ|                     ¦   «         D ]À\  }}|| j        v r²|| j        v r@t	          | j        |         j        ¦  «        t	          |¦  «        k    rŒHt          d¦  «        ‚| j        |         }t          j	        t	          |¦  «        |¬¦  «         
                    | j        ¬¦  «        }|| j        |<   |                      ||¦  «         ŒÁ|                     ¦   «         D �]$\  }}|| j        v �r|| j        v r1t	          | j        |         j        ¦  «        t	          |¦  «        k    rŒJ|| j        v rŒT| j        |         }t          j	        t	          |¦  «        |¬¦  «         
                    | j        ¬¦  «        }|| j        |<   | j                             ||j        j        |j        j        �|j        j        nd| j        |         t+          |                     ¦   «         ¦  «        |                     ¦   «         ¦  «         �Œ&dS )z Allocate tensors for I/O Bindingz(Expect static input shape for cuda graph)r…   )rŠ   Nr   )r¾   r‹   rº   r¿   Útupler�   ÚRuntimeErrorr½   rG   rŒ   ÚtorŠ   rÑ   r»   rÀ   rÁ   r˜   r    r   rÍ   r¼   r›   rœ   r�   )rÂ   rÒ   r   r�   Útorch_dtyperÊ   s         r   Úallocate_bufferszCudaSession.allocate_buffersp  s  € àÔ!ð 	EØ)×/Ò/Ñ1Ô1ð Eð E‘��eØ˜4Ô+Ð+Ð+à˜tÔ1Ð1Ð1Ý  Ô!3°DÔ!9Ô!?Ñ@Ô@ÅEÈ%ÁLÄLÒPÐPØ$Ý*Ð+UÑVÔVÐVà"&Ô"<¸TÔ"B�KÝ"œ[­¨u©¬¸[ÐIÑIÔI×LÒLÐTXÔT_ÐLÑ`Ô`�FØ/5�DÔ& tÑ,Ø×6Ò6°t¸VÑDÔDÐDøà%×+Ò+Ñ-Ô-ð 	ñ 	‰KˆD�%Ø�tÔ(Ð(Ñ(à˜4Ô.Ð.Ð.µ5¸Ô9LÈTÔ9RÔ9XÑ3YÔ3YÕ]bÐchÑ]iÔ]iÒ3iÐ3iØà˜4Ô.Ð.Ð.Øà"Ô8¸Ô>�Ýœ¥U¨5¡\¤\¸ÐEÑEÔE×HÒHÐPTÔP[ÐHÑ\Ô\�Ø,2�Ô# DÑ)à”×+Ò+ØØ”MÔ&Ø+1¬=Ô+>Ð+J�F”MÔ'Ð'ÐPQØÔ-¨dÔ3Ý˜Ÿš™œÑ'Ô'Ø—O’OÑ%Ô%ñô ð ùð	ð 	r   NTÚ	feed_dictÚrun_optionsÚsynchronizec                 óè  — |                      ¦   «         D ]â\  }}t          |t          j        ¦  «        r|                     ¦   «         sJ ‚|| j        v r¤| j        r‡| j        |                              ¦   «         |                     ¦   «         k    sJ ‚| j        |         j	        |j	        k    sJ ‚|j
        j        dk    sJ ‚| j        |                              |¦  «         ŒÌ|                      ||¦  «         Œã|rS| j                             ¦   «          | j                             | j        |¦  «         | j                             ¦   «          n | j                             | j        |¦  «         | j        S )z$Bind input tensors and run inferenceÚcuda)r‹   Ú
isinstancerG   r±   r™   rº   r¾   r¿   Únelementr…   rŠ   r   Úcopy_rÑ   r˜   Úsynchronize_inputsr   Úrun_with_iobindingÚsynchronize_outputsrÀ   )rÂ   rÙ   rÚ   rÛ   r   rÊ   s         r   ÚinferzCudaSession.infer–  sp  € à%ŸOšOÑ-Ô-ð 		Eð 		E‰LˆD�&Ý˜f¥e¤lÑ3Ô3ÐN¸×8LÒ8LÑ8NÔ8NÐNÐNÐNØ�tÔ'Ð'Ð'ØÔ)ð EØÔ-¨dÔ3×<Ò<Ñ>Ô>À&Ç/Â/ÑBSÔBSÒSÐSÐSÐSØÔ-¨dÔ3Ô9¸V¼\ÒIÐIÐIÐIØ!œ=Ô-°Ò7Ð7Ð7Ð7ØÔ& tÔ,×2Ò2°6Ñ:Ô:Ð:Ð:à×6Ò6°t¸VÑDÔDÐDøàð 	NØŒO×.Ò.Ñ0Ô0Ð0ØÔ×/Ò/°´ÀÑMÔMÐMØŒO×/Ò/Ñ1Ô1Ð1Ð1àÔ×/Ò/°´ÀÑMÔMÐMàÔ"Ð"r   r   rÏ   r¾   Ústreamr   c                 óB   — | d|dœ}|dk    rt          |¦  «        |d<   |S )NÚkSameAsRequested)rÏ   Úarena_extend_strategyr¾   r   Úuser_compute_stream)r‚   )rÏ   r¾   rå   Úoptionss       r   Úget_cuda_provider_optionsz%CudaSession.get_cuda_provider_options¬  s:   € ð #Ø%7Ø!2ð
ð 
ˆð �QŠ;ˆ;Ý-0°©[¬[ˆGÐ)Ñ*àˆr   ©F)NT)r   )r~   r   r€   Ú__doc__r   rG   rŠ   rÃ   r‚   rÇ   rÉ   r±   rÑ   Ú	ShapeDictrØ   rƒ   r   r3   rä   r�   r„   r   rë   r†   r   r   r³   r³   :  sE  € € € € € ØVÐVð1ð 1Ð$4ð 1¸e¼lð 1ð 1ð 1ð 1ð 6¨Sð 6¸sð 6ð 6ð 6ð 6ðð ð ð
D°#ð D¸u¼|ð Dð Dð Dð Dð0$¨9ð $ð $ð $ð $ðL#ð #˜t C¨¬Ð$5Ô6ð #ÀZð #Ðeið #ð #ð #ð #ð, ðð ¨Sð ÀTð ÐSVð Ð_cÐdgÐilÐdlÔ_mð ð ð ñ „\ðð ð r   r³   c                   ó¨   ‡ — e Zd Z	 	 	 	 ddedej        deded	ed
ede	e
e
f         dz  fˆ fd„Zddedefd„Zdde	e
ej        f         defˆ fd„Zˆ xZS )Ú
GpuBindingFéÿÿÿÿr   Nr   rŠ   rÒ   Úenable_gpu_graphÚgpu_graph_idrå   rÁ   c                 ó@  •— t          ¦   «                              |||¦  «         |r0|                     ¦   «         D ]\  }}	|                      ||	¦  «         Œ|                      |¦  «         || _        |rt          j        |¦  «        nd | _        || _	        d | _
        d S rÆ   )ÚsuperrÃ   r‹   rÇ   rØ   ró   ÚcopyÚdeepcopyrÒ   rå   Úlast_run_gpu_graph_id)rÂ   r   rŠ   rÒ   rò   ró   rå   rÁ   rÄ   r£   Ú	__class__s             €r   rÃ   zGpuBinding.__init__¼  s²   ø€ õ 	‰Œ×Ò˜ fÐ.>Ñ?Ô?Ð?Øð 	AØ+9×+?Ò+?Ñ+AÔ+Að Að AÑ'�
˜KØ×'Ò'¨
°KÑ@Ô@Ð@Ð@à×Ò˜jÑ)Ô)Ð)Ø(ˆÔà7GÐQ�$œ-¨
Ñ3Ô3Ð3ÈTˆŒØˆŒà%)ˆÔ"Ð"Ð"r   Údisable_cuda_graph_in_runr   c                 óŒ   — t          ¦   «         }|rdn| j        }|                     dt          |¦  «        ¦  «         || _        |S )Nrñ   ró   )r   ró   Úadd_run_config_entryr‚   rø   )rÂ   rú   rê   ró   s       r   Úget_run_optionszGpuBinding.get_run_optionsÓ  sG   € Ý‘,”,ˆà6ÐM�r�r¸DÔ<Mˆà×$Ò$ ^µS¸Ñ5FÔ5FÑGÔGÐGà%1ˆÔ"àˆr   rÙ   c                 ó¬   •— |                       |¦  «        }| j        r|                     dd¦  «         t          ¦   «                              ||¦  «        S )NÚ'disable_synchronize_execution_providersÚ1)rý   rå   rü   rõ   rä   )rÂ   rÙ   rú   rÚ   rù   s       €r   rä   zGpuBinding.inferÞ  sR   ø€ Ø×*Ò*Ð+DÑEÔEˆàŒ;ð 	]Ø×,Ò,Ð-VÐX[Ñ\Ô\Ð\å‰wŒw�}Š}˜Y¨Ñ4Ô4Ð4r   )Frñ   r   Nrì   )r~   r   r€   r   rG   rŠ   rî   r3   r„   rƒ   r‚   rÃ   r   rý   r±   rä   Ú__classcell__)rù   s   @r   rð   rð   »  s  ø€ € € € € ð "'ØØØ04ð*ð *à%ð*ð ”ð*ð ð	*ð
 ð*ð ð*ð ð*ð ˜S #˜Xœ¨Ñ-ð*ð *ð *ð *ð *ð *ð.	ð 	¸ð 	È*ð 	ð 	ð 	ð 	ð5ð 5˜t C¨¬Ð$5Ô6ð 5ÐSWð 5ð 5ð 5ð 5ð 5ð 5ð 5ð 5ð 5ð 5r   rð   c            	       ój   — e Zd ZdZddedej        dedefd„Z	 	 dde	de
deeef         d
z  defd„Zd
S )ÚGpuBindingManagerz­A manager for I/O bindings that support multiple CUDA Graphs.
    One cuda graph is reused for same input shape. Automatically add a new cuda graph for new input shape.
    r   rÌ   r   rŠ   rå   Úmax_cuda_graphsc                 óZ   — || _         || _        g | _        d | _        || _        || _        d S rÆ   )r   rŠ   Úgraph_bindingsÚno_graph_bindingrå   r  )rÂ   r   rŠ   rå   r  s        r   rÃ   zGpuBindingManager.__init__ì  s:   € Ø&ˆÔØˆŒð !ˆÔð !%ˆÔàˆŒà.ˆÔÐÐr   FNrÒ   Úuse_cuda_graphrÁ   r   c           	      óº  — | j         D ]}|j        |k    r|c S Œt          | j         ¦  «        | j        k    s|sQ| j        €)t          | j        | j        || j        |¬¦  «        | _        n| j         	                    |¦  «         | j        S t          | j        | j        |dt          | j         ¦  «        | j        |¬¦  «        }| j          
                    |¦  «         |S )N)rå   rÁ   T)rò   ró   rå   rÁ   )r  rÒ   rÎ   r  r  rð   r   rŠ   rå   rØ   rª   )rÂ   rÒ   r  rÁ   Úgpu_graph_bindings        r   Úget_bindingzGpuBindingManager.get_bindingú  s  € ð "&Ô!4ð 	)ð 	)Ðà Ô+¨zÒ9Ð9Ø(Ð(Ð(Ð(ð :õ ˆtÔ"Ñ#Ô# tÔ';Ò;Ð;ÀNÐ;ØÔ$Ð,Ý(2ØÔ$ d¤k°:ÀdÄkÐbpð)ñ )ô )�Ô%Ð%ð Ô%×6Ò6°zÑBÔBÐBØÔ(Ð(õ 'ØÔØŒKØØ!Ý˜TÔ0Ñ1Ô1Ø”;Ø)ð
ñ 
ô 
Ðð 	Ô×"Ò"Ð#4Ñ5Ô5Ð5Ø Ð r   )r   rÌ   )FN)r~   r   r€   rí   r   rG   rŠ   r„   rÃ   rî   r3   rƒ   r‚   rð   r  r†   r   r   r  r  ç  s¬   € € € € € ðð ð/ð /Ð$4ð /¸e¼lð /ÐTWð /Ðnqð /ð /ð /ð /ð"  %Ø04ð	 !ð  !àð !ð ð !ð ˜S #˜Xœ¨Ñ-ð	 !ð
 
ð !ð  !ð  !ð  !ð  !ð  !r   r  )rö   ÚloggingÚcollectionsr   Úcollections.abcr   Útypingr   r.   rG   Úonnxr   Úonnxruntimer   r   r‚   rÔ   r›   r„   rî   Ú	getLoggerr~   rž   r
   rˆ   r³   rð   r  r†   r   r   ú<module>r     s¥  ðØ €€€Ø €€€Ø #Ð #Ð #Ð #Ð #Ð #Ø #Ð #Ð #Ð #Ð #Ð #Ø Ð Ð Ð Ð Ð à €€€Ø €€€Ø Ð Ð Ð Ð Ð à 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4ð �C˜  c¤Ñ*Ð*Ô+€	à	ˆÔ	˜8Ñ	$Ô	$€ðt"ð t"ð t"ð t"ð t"ñ t"ô t"ð t"ðnmð mð mð mð mñ mô mð mð`~ð ~ð ~ð ~ð ~ñ ~ô ~ð ~ðB)5ð )5ð )5ð )5ð )5�ñ )5ô )5ð )5ðX3!ð 3!ð 3!ð 3!ð 3!ñ 3!ô 3!ð 3!ð 3!ð 3!r   