ktjddlZddlZddlZddlZddlmZddlmZm Z ddl m Z ddl m Z ddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z ddl!m"Z"Gd d eZ#dS) N)onnx_pb) BaseQuantizerQuantizationParams) TensorData) ONNXModel)TENSOR_NAME_QUANT_SUFFIXQuantizationModeQuantizedValueQuantizedValueType __producer__ __version__add_infer_metadataattribute_to_kwargcompute_scale_zpcompute_scale_zp_float8 find_by_nameget_qmin_qmax_for_qTypeget_qrange_for_qType ms_domainquantize_onnx_initializer&save_and_reload_model_with_shape_infersnap_zero_point_to_uint8tensor_proto_to_array)CreateOpQuantizerc^eZdZ d*dZdZdZdZdZdZdZ d Z d+d Z d Z d Z dZdZd,dZ d-dZdZ d*dZdejdejdedejdedeeejdzff dZdedejddfdZd.dZdZd+d Z d/d"Z d0d$Z!d1d%Z" d2d&Z#d'Z$d(Z%d)Z&dS)3 ONNXQuantizerNc tj|||||||| | | | |s|jt |jj}d|jjD|_|jd|jj D|jd|jj Dt||_||_ ||_ |jdk|_d|jvo |jd|_g|_d|_i|_|jd|jj D|jd|jj D|jjjjD]:} |jt,| j d ;|j t0vrt3d |j ||_d |_d |_d |_d|_i|_ |j!|_"dS)Nci|] }|j| Sname).0vis e/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/onnxruntime/quantization/onnx_quantizer.py z*ONNXQuantizer.__init__..IsMMMMMMci|] }|j| Sr r!r#ots r%r&z*ONNXQuantizer.__init__..Js$N$N$NRRWb$N$N$Nr'ci|] }|j| Sr r!r#its r%r&z*ONNXQuantizer.__init__..Ks$M$M$MRRWb$M$M$Mr' MatMulConstBOnly/ci|] }|jd Srr!r)s r%r&z*ONNXQuantizer.__init__..Ws!J!J!J"'1!J!J!Jr'ci|] }|jd Sr2r!r,s r%r&z*ONNXQuantizer.__init__..Xs!I!I!I"'1!I!I!Ir'rzunsupported quantization mode fixed_quantization_range_uint8fixed_quantization_range_int8 fixed_zero fixed_zero_zp)#r__init__modelreplace_gemm_with_matmulrgraph value_info value_infosupdateoutputinputrmodestatic opset_versionfuse_dynamic_quant extra_optionsq_matmul_const_b_only new_nodes graph_scope tensor_namesnodedictfromkeysr ValueErrorcalculate_quantization_paramsquantization_paramsfixed_qrange_uint8_namefixed_qrange_int8_namefixed_zero_namefixed_zero_zp_namequantized_value_mapget_non_initializer_inputsgenerated_value_names)selfr9 per_channel reduce_rangerArB weight_qTypeactivation_qType tensors_rangenodes_to_quantizenodes_to_excludeop_types_to_quantizerErJs r%r8zONNXQuantizer.__init__(s`             * J / / 1 1 1:4:;KLLEMMek6LMMMD    # #$N$N5;;M$N$N$N O O O   # #$M$M5;;L$M$M$M N N N"5))DJ  "&"4r"9%74;M%M%xRVRdewRx"   !J!Ju{7I!J!J!JKKK   !I!Iu{7H!I!I!IJJJJ$*/ D DD   $ $T]]4;%B%B C C C C 9, , ,IdiIIJJ J#'#E#E#G#G (H$&E#+"1$& &*Z%J%J%L%L"""r'ctj|d|jjj}t |t ||j|j|j |j |j |j |j |j|j|j|j }||_|j|d|_||jjjS)z generate submodel for the subgraph, so that we re-utilize current quantization implementation. quantize the submodel update subgraph and set it back to node onnx-quantizer) producer_name opset_importsr0)onnxhelper make_modelr9 opset_importrrrXrYrArBrZr[r\r]r^r_rEparentrHquantize_modelr;)rWsubgraph graph_key warped_model sub_quantizers r%quantize_subgraphzONNXQuantizer.quantize_subgraphqs {-- ***7.  <(((%      I K    !    "  !  %      $ '+'7$E$E$E$E !$$&&&"(..r'cd|jD}t|dkr|S|jr|jn|jdt|j}i}|jD]}|jt jjkr-|j| |j |d|ji}n|jt jj krZg}|j D]F}| | ||d|jdt|gG|j|i}nt|}||t jj|j|j|jfd|ji|S)z| Check subgraph, if any, quantize it and replace it. return new_nodes added for quantizing subgraph czg|]8}|jtjjks|jtjjk6|9Sr )typerdAttributeProtoGRAPHGRAPHS)r#attrs r% z>ONNXQuantizer.quantize_node_with_sub_graph..sG   yD/555dFYF`9`9` 9`9`9`r'r _node_count_:r") attributelenr"op_typerGrqrdrrrsrngrtgraphsextendrr>re make_noder@r?) rWrJ graph_attrs node_namekwargsrukvvaluerjs r%quantize_node_with_sub_graphz*ONNXQuantizer.quantize_node_with_sub_graphs      {  q K!%bDII4<0b0bSQUQ_M`M`0b0b N  DyD/555i!7!79@Z@Zty@Z@Z![![\d1888 $ HLL 22 (#, G Gty G G3u:: G Gi''-- MM"    {$T\4:t{eeQUQZe^deeer'cbtd|jDS)zQ Detect if model already has QuantizeLinear or DequantizeLinear. c3BK|]}|jdkp |jdkVdS)QuantizeLinearDequantizeLinearN)r{r#rJs r% z.ONNXQuantizer.has_QDQ_nodes..sH  W[DL, , R @R0R      r')anyr9nodes)rWs r% has_QDQ_nodeszONNXQuantizer.has_QDQ_nodessA  _c_i_o_o_q_q      r'ct||jdS|j|j|SdS)NTF)rr9 initializerrhfind_initializer_in_path)rWinitializer_names r%rz&ONNXQuantizer.find_initializer_in_pathsJ ($**@*@*B*B C C O4 ; ";778HII Iur'c|j||D]&}|jD]}|j|'dSN)rGr~r?rVadd)rWrrJ output_names r% add_new_nodeszONNXQuantizer.add_new_nodessa e$$$ < .s%bbbe XaHaHaEHaHaHar'c(g|]}|jdk |S)z com.microsoft)rrs r%rvz0ONNXQuantizer.quantize_model..s$ZZZ4;/;Y;Y;Y;Y;Yr'r) rloggingwarningr9renable_subgraph_quantizationrrzrGrquantizeranger?rVr_dequantize_outputsr; ClearFieldrJr~rhclean_initializers RuntimeErrorstrr rbrproducer_versionrgversionrr) rWrJnumber_of_existing_new_nodes op_quantizerir_initializers_not_foundms_opsetms_nodesrs r%rizONNXQuantizer.quantize_modelsG       On    J$$&& @ @D0 ?88>>+.t~+>+> (,T488L  ! ! # # #7T^9L9LMM @ @#'>!#4#;@@K.22;????@ @   """ %%f--- &&t~666 ; (, (E(E(G(G %A%)**Q.."#UX[\rXsXs#sttt)5 &,7 )bbtz'7'Dbbb )ZZZZZH ) (599;; ! ( zr'cd|jvr.tjd||jd|jdStd|d)NDefaultTensorTypezDget_tensor_type returns DefaultTensorType for tensor name %r, use %dz)Unable to find data type for weight_name=a7. shape_inference failed to return a type probably this node is from a different domain or using an input produced by such an operator. This may happen if you quantize a model already quantized. You may use extra_options `DefaultTensorType` to indicate the default weight type, usually `onnx.TensorProto.FLOAT`.)rErinforrW tensor_names r%_get_default_tensor_typez&ONNXQuantizer._get_default_tensor_typess $"4 4 4 LV"#67    %&9: : J  J J J   r'FcPt||j}||jS||jvrd|j|}|jdr=|r*|jjjdkr| |S|jjjS|j r|j |r| |SdS|j |}||S|j r%|j r|j |}||S|r| |SdS)N tensor_typer)rr9r data_typer=rqHasFieldr elem_typerrrhis_valid_quantize_weightget_tensor_type)rWr mandatoryweightr$otyperess r%rzONNXQuantizer.get_tensor_typesFk4:+A+A+C+CDD  # # $* * *!+.Bw .. 5F!4!>!!C!C88EEEw*441 t{7J B44[AAA4 44[AA  L  ,  +--k::C  >00== =tr'c||r||S||jvr~|j|}|jdr5|jjjtjj tjj fvrdStj d|d|jddS|j r!|jr|j|Stj d|ddS) NrTz00== = $* * *!+.Bw .. 273F3P&,&.U44t Or{rrhjhorrr   5  , < <;..{;; ; 7K 7 7 7   ur'c|tjjkr||||S|tjjkr||||St d|d)a Create nodes for dynamic quantization of input and add them to nodes_list. parameter input_name: Name of the input. parameter nodes_list: new nodes are appended to this list. parameter qType: type to quantize to. parameter initial_type: type to quantize from return: scale_name, zero_point_name, scale_shape, zero_point_shape. zUnexpected value for qType=r)rrINT8+_get_dynamic_input_quantization_params_int8UINT8,_get_dynamic_input_quantization_params_uint8rM)rW input_name nodes_listqType initial_types r%&_get_dynamic_input_quantization_paramsz4ONNXQuantizer._get_dynamic_input_quantization_params7st J*/ / /CCJPZ\hii i J*0 0 0DDZQ[]ijj j?u???@@@r'ctjj}|dz}|dz}tjd|g|dzg|d}|||dz}tjd|g|dzg|d} || |d z} tjd |jdg| dzg| } || |d z} tjd | jdg| dzg| } || |d z}tjd | jd| jdg|dzg|}||tj|j |gt|d z g}|j ||dz}tjd|jd|j g|g|}||tj|j |gdg}|j |||j ggfS)az Create nodes for dynamic quantization of input to int8 and add them to nodes_list parameter input_name: Name of the input. parameter nodes_list: new nodes are appended to this list. parameter initial_type: initial weight type (FLOAT or FLOAT16) return: scale_name, zero_point_name, scale_shape, zero_point_shape. _scale _ReduceMin ReduceMin:0rkeepdims _ReduceMax ReduceMax_AbsAbs_Abs_MaxMax@ scale_DivDiv)rrrrdrerappendr? make_tensorrQrr9add_initializerrS)rWrrrrinput_scale_namereduce_min_namereduce_min_nodereduce_max_namereduce_max_nodereduce_min_abs_namereduce_min_abs_nodereduce_max_abs_namereduce_max_abs_node abs_max_name abs_max_nodeinitializer_divscale_div_namescale_div_nodeinitializer_zps r%rz9ONNXQuantizer._get_dynamic_input_quantization_params_int8Fs&+&0$|3+//  L t # $  0   /***$|3+//  L t # $  0   /***.6"k33   #A & ' 4 ' (     -...-6"k33   #A & ' 4 ' (     -...!J. {,,  ' *,?,Fq,I J D !    ,'''+11  '   !% ( (3 . /    ""?333#k1..    #T%@ A       .)))001H%QSVWUXYY "">222!8"b@@r'ctjj}|dz}|dz}|dz}tjd|g|dzg|d}|||dz} tjd |g| dzg| d} || tj|j|gt|g} |j | tj|j |gd g} |j | |d z} tjd | j d|j dg| dzg| }|||d z}tjd|j d|jg|g|}|||dz}tjd |j |j dg|dzg|}|||dz}tjd|j d|g|dzg|}|||dz}tjd|j |dzg|}|||dz}tjd|j |g||}||||ggfS)a{ Create nodes for dynamic quantization of input to uint8 and add them to nodes_list parameter input_name: Name of the input. parameter nodes_list: new nodes are appended to this list. parameter initial_type: initial weight type (FLAOT or FLOAT16) return: scale_name, zero_point_name, scale_shape, zero_point_shape. r _zero_pointrrrrrrr _scale_SubSub _scale_Divr_zero_point_Sub_zero_point_Div_zero_point_FloorFloor_zero_point_CastCast)to)rrrrdrerrrrPrr9rrRr?)rWrrrrr input_zp_namerrrrinitializer_qrangeinitializer_qvaluescale_sub_namescale_sub_noderr zp_sub_name zp_sub_node zp_div_name zp_div_node zp_floor_name zp_floor_node zp_cast_name zp_cast_nodes r%rz:ONNXQuantizer._get_dynamic_input_quantization_params_uint8sb&,%0"]2 $|3+//  L t # $  0   /***$|3+//  L t # $  0   /***"[44  (   !% ( ( )    ""#5666![44T5Iq(A B d " #     .)))#l2..   "1 %t'C D       .)))!#44 k++   !?#9!#< = 4     +&&& #44 k++    "$4 5 4     +&&&"%88  --g{7IM\`L`Kacpqq -(((!$66 {,,V]5IM?\hmr,ss ,'''B66r'c"|j}||R|j ||jvrtjd|ddS|j|}t |t s#t dt|d|d|t|dkrtd |d |tj |d g}t|d d r%|d j tjtjfvr(tdt|d d|tj |d g}|j tjksJ|d}nttj |g}tj |g}|j|}d |vr"|d j }||}|j tjksJg} |dz} g} |dz} t$j| || |} |j| |j tjkrt2jj}nA|j tjkrt2jj}ntd|j d|t$j| || |d}|j|d| | | | fS)a\ Create initializers and inputs in the graph for zero point and scale of output. Zero point and scale values are obtained from self.quantization_params if specified. parameter param_name: Name of the quantization parameter. return: result, scale_name, zero_point_name, scale_shape, zero_point_shape. Nz$Quantization parameters for tensor:"z" not specified)FrrrUnexpected type  for rzbQuantization parameters should contain zero point, scale, quant type. Specified values for output z: zero_pointscaledtypez and param_name= quant_typerrzUnexpected dtype=z for param_name=)T)r[rOrr isinstancer TypeErrorrqrzrMnparrayhasattrrfloat32float16float64astyperdrerraveltolistr9rrrrrreshape)rW param_name use_scale use_zeropointzero_point_typeparamszero_point_values scale_valuesrzero_point_shapezero_point_name scale_shape scale_nameinit_zp scale_type init_scales r%_get_quantization_paramsz&ONNXQuantizer._get_quantization_paramss5/   5'/:TE]3]3] _J___```,,-j9Ff&899 W U4<< U Uj U U UVVV~V!1!1 J3=JJAGJJ !#&*>)? @ @ 6'?G44 kw8MVXV`bdblUm8m8m !iD4I4I!i!i[e!i!ijjj8VG_$566L%3333$\2OO "- 9 9 8YK00L-j9F&  w-+22599 %3333$}4 (* +)) _.>@Q@W@W@Y@Y@`@`@b@b   ""7+++   + +#/5JJ  2: - -#/7JJc1CccU_ccdd d[,,Z[R^RfRfglRmRmRtRtRvRvww  "":...Z+?OOOr'c |j|}|dks Jd|tz}|dz} | |d||} } } n||\} } } } } g}| r'tjd|| | g|g| }n|jrdS|jrF|tj j kr1|dz} |dz} tjd |g|| | g| }n\|Jd |d |d |d || ||||\} } }}tjd|| | g|g| }t||| | ||j |<g||S)a Given an input for a node (which is not a initializer), this function - add nodes to compute zero point and scale for this input if they don't exist. - add new QuantizeLinear node to quantize the input. :param node: node being quantized in NodeProto format. :param input_index: index of input in node.input. :param qType: type to quantize to. :param given_scale_name: if those inputs need to be quanitzed using this scale tensor. :param given_zp_name: if those inputs to be quantized using this zeropoint tensor. :param initial_type: type of the weight to quantize :return: List of newly created nodes in NodeProto format. rz*Cannot access undefined variable in graph._QuantizeLinearNTrrrDynamicQuantizeLinearzCCannot quantize input without knowing the initial type, input_name=z, input_index=z, qType=z, node=r)r@r r3rdrerrBrDrrrrr rT)rWrJ input_indexrgiven_scale_name given_zp_namerrr ql_node_name data_foundr/zp_namerr qlinear_noder.zp_shapes r%_get_quantize_input_nodesz'ONNXQuantizer._get_quantize_input_nodes3s "Z , R!M #;; !$55  (}/H/35E}G JJ484Q4QR\4]4] 1J GQ % ;00 Z1  LL{ t& 5J4J4P+P+P'(2 $}4#{44+L *g6    $//h",hh>IhhSXhhaehh0//?? ESXgs?tt #{44$W5 M    0>j+Wacjlq/r/r ,%% %%r'ct||jvr |j|S|j|j|SdSr)rTrhfind_quantized_value)rWrs r%rBz"ONNXQuantizer.find_quantized_valueysA 1 1 1+J7 7 ; ";33J?? ?tr'c &tj|} |d| zz|z } tj|tj} tj|tj} | | z}|| kr|dkrz| |z }| |z}| 6t jd|d|d|dd tj||fSt jd | d |d |d|d d ||fSd |fS)zHAdjust a single weight scale to ensure the int32 bias does not overflow.rrrNzIncreasing scale for weight `z` by the ratio z to ensure bias `z` has a valid scale.TzIncreased scale[z] for weight `z ` by ratio F)rabsritemr rrr!)rWbias_val input_scale weight_scaleweight_scale_dtype weight_name bias_nameqrangemultiplicative_epsilonidxabsmaxbias_smallest_valid_scaleinput_scale_fp64weight_scale_fp64bias_candidate_scaleratio new_scales r%$adjust_single_weight_scale_if_neededz2ONNXQuantizer.adjust_single_weight_scale_if_neededs!!$:cFl$Kf$T!8K$4$4$6$6bjIIIH\%6%6%8%8 KKK/2CC #< < | |fS) zOChecks if the bias scale is too small and increases the weight scale if needed.)FNgqh?rDrFrT)rO)sizerriinfoint32rmaxr minrminimummaximumrErWr"shaperzr)rWrHrIrKrXrYbias_float_data int32_inforNrMrJupdatedrminrmaxrPchangedrVrs r%#_adjust_weight_scale_for_int32_biasz1ONNXQuantizer._adjust_weight_scale_for_int32_biass  ;/88Xbh'' !'*. ;;;bhz~XYGYacak>l>l>ll)/ #:o1133RXarz5R5R5RSSD:o1133RXarz5R5R5RSSDZt bfTll;;F!%!J!J" & " " GY (    #C (:$;$;q$@$@<-a011 # #%)%N%N#A& O&L*&O & &"#&/LO"G $$r'rVc||jvrdS|j|}t||j}t|j|j}t|j|j}t|j|j}||||dS|j||j|tj |}|j } tj |tj|j} tj | |j|j} |j| t+||j|| | |j} |j| dS)zCRe-quantizes the given weight initializer using the provided scale.NrD)quant_weight_name)rTrr9rr/r=q_nameremove_initializerrd numpy_helperto_arrayaxisrasarrayretensor_dtype_to_np_dtyper from_arrayr$dimsrrrZ) rWrKrVqv weight_tp scale_initzp_init q_weight_initweight_zero_pointrqscale_npnew_scale_init new_q_weights r%_requantize_weightz ONNXQuantizer._requantize_weights d6 6 6 F  %k 2 dj.D.D.F.FGG !"-1G1G1I1IJJ rz4:+A+A+C+CDD$RY 0F0F0H0HII   2goI^ F %%j111 %%m444 -66w??w:it{/S/ST]Tg/h/hiii*55h6F6Fz6W6WY[Yfgg "">2221       i     ""<00000r'?c ||jvr|j|jS|j|j}t||j}t |}||jvr|j|j}n8||jvr||\} }} } } ntd|dt||j} t | } |j|j } t| |j} | tj | nd}|j}||jr|sv|jt$jjfvr]t||j}|| ||||\}}|r||||}||| ||\}}}}}}||jvsJt1||||t2j|jdkrdnd||}||j|<|S)z] Quantized the bias. Zero Point == 0 and Scale == Input_Scale * Weight_Scale z Expected z5 to be in quantized value map for static quantizationNrr) node_type node_qtype)rTrmr/rr9rrrOr3rMr=rdrorprXr\rrZrrrrjrquantize_bias_static_implr r Initializer)rWrLrrKbetaweight_scale_nameweight_initializerrIrrinputscale_initializerrHweight_zp_nameweight_zp_initr{rYbias_initializer did_updatenew_weight_scalequantized_bias_namequantized_bias_scale_namequantized_bias_zp_namebias_scale_datarrquantized_values r%quantize_bias_staticz"ONNXQuantizer.quantize_bias_statics 0 0 0+I6= =!4[AL)*;TZ=S=S=U=UVV,-?@@  1 1 1#7 CN   43 3 3+/+H+H+T+T (AAqqjjjjkk k!-.> @V@V@X@X!Y!Y+,BCC 1+>F%ndj6L6L6N6NOOJXJdD-66~FFFjn)  )!& *%))++ *!j&<&A%CCC+Itz7M7M7O7OPP +/+S+S  ,, (J( 0'' 5EFFF/   * *9k +""r'c8||jvp||jvp||jvS)zq only check for value info and newly generated tensor names, initializers are checked separately )r=rIrVrs r%contains_tensorzONNXQuantizer.contains_tensorKs4 D, , ;t00 ;t99 r'c :|||dddd|S)NFrrJindicesinitializer_use_weight_qTyperYop_level_per_channelrq from_subgraph_ONNXQuantizer__quantize_inputs)rWrJrrs r%quantize_activationz!ONNXQuantizer.quantize_activationUs4%%).!&'&   r'rc :|||d||||S)NTrr)rWrJrrYrrqrs r%quantize_weightzONNXQuantizer.quantize_weightbs6%%)-%!5'&   r'Tc g}g} g} g} |D]4} |j| } | |jvr\|j| }||j| |j| |ju| s@| d|d| dt | |j}||j r2|r0| |j |r|j n|j ||\}}}n)|||r|j n|j |\}}}| || ||||| r|j| dz|j|j}||j| }||jvrj|j|}|dsJd|d|jdsJd|d|jjj}n$||jvsJd |d |j|}||| |j | }|d S|r||n| ||d }|jdkr\| |j||jd| |jd9| |jd||jd| |jd|jz|j|| g||||d\}}}}| |d||d| |dtAd| d|j!| | || fS)a Given a node, this function quantizes the inputs as follows: - If input is an initializer, quantize the initializer data, replace old initializer with new initializer - Else, add QuantizeLinear nodes to perform quantization parameter node: node being quantized in NodeProto format. parameter indices: input indices to quantize. return: (List of quantized input names, List of zero point names used for input quantization, List of scale names used for input quantization, List of new QuantizeLinear nodes created) rNr5rqz value_info=z has no type.rz is not a tensor.zshape inference failed for zF and attribute 'tensor_names' does not have any value for this tensor.r7)NNNNrrrrT)rrYrrqrz!Invalid tensor name to quantize: z @graph scope)"r@rTrr/r=rmrr9rrXquantize_weight_per_channelr"rZr[quantize_initializerrfind_node_by_namerGr;r=rrqrrrIr@rr~r{r?rhrrMrH)rWrJrrrYrrqr scale_nameszero_point_namesquantized_input_namesrr8 node_inputrr q_weight_namer=r/r>rr<rquantize_input_nodesparent_quantized_input_namesparent_zero_point_namesparent_scale_namesrs r%__quantize_inputszONNXQuantizer.__quantize_inputsus.  ""c rc rKK0JT555"&"::"F""?#=>>> ''(?@@@%,,_-CDDD %,,R000""2&&& ''+++&z4:3I3I3K3KLLK&#(< 88#(-Id))tOd$  %" :>9R9R#-Id))tOd$::6M7J &,,];;; ''000"":....%%j11; r#z;;!22DNDJDTDTDVDV    '!%K!8J!T%555%)%5j%A )226::cc>>>,*,,, ?>> (,'8'D +/+I+Ik4+@|,J,,(,3777$;**+?@@@@ %9:::#7#;L'+;;;)001DEEE&&|'9!'<===$++L,>q,ABBBB)001DQ1GHHH&&|':1'=>>>$++L,?,BCCCC( K11 M1M!-)="&2 0+&&,,-I!-LMMM""#5a#8999 ''(?(BCCCC!!pZ!p!p^b^n!p!pqqq$&6 UJJr'c|j|jvr&|j|j}|j|j|jfS|||||\}}}t |j|||tjd}||j|j<|||fS)a :param weight: TensorProto initializer :param qType: type to quantize to :param keep_float_weight: Whether to quantize the weight. In some cases, we only want to qunatize scale and zero point. If keep_float_weight is False, quantize the weight, or don't quantize the weight. :return: quantized weight name, zero point name, scale name N) r"rTrmr=r/quantize_initializer_implr r r) rWrrrYkeep_float_weightrrr=r/s r%rz"ONNXQuantizer.quantize_initializers ;$2 2 2"6v{CO&'*  .2-K-K E<):. . * w ) K     *    1@ -gz11r'c||jvr!|j|}|j|j|jfS||||||\}}} t ||| |t j|}||j|<||| fSr)rTrmr=r/ quantize_weight_per_channel_implr r r) rWrKrZ channel_axisrYrrrr=r/s r%rz)ONNXQuantizer.quantize_weight_per_channels $2 2 2"6{CO&'*  .2-R-R |\CT. . * w )      *    1@ -gz11r'c||jvr4||jvr*|j|}t|j|j}|jjjdks|jjjdkr5|3|j,|*tj |j dksJ|dz}|j ||j |j}|@|j|j|jg}tjd||g||j}|S||jdksJdS)a Given a value (input/output) which is quantized, add a DequantizeLinear node to dequantize it back to float32 or float16 parameter value_name: value to dequantize parameter new_nodes_list: List of new nodes created before processing current node return: None if there is already a DequantizeLinear node that dequantizes it A DequantizeLinear node otherwise raNr_DequantizeLinearr)rqr)rTrVrr/r9rrbrqrdrorpr\rrGr;rmr=rerr?)rW value_namerrx dqlinear_name dqlinear_nodedqlinear_inputsdequantize_nodes r%_dequantize_valuezONNXQuantizer._dequantize_value9sb $2 2 24Ke9e9e"6zBO&o&@$*BXBXBZBZ[[Jz-1AAA  .2BBBzG]#'/%-1B1K1KJ1W1W1\`a1a1a1aa&)<!3DEE&**62>!3DEE+// T=YZZ .22>5II 4Zlfoppp d.tT4yRVReff e; dN^Nd@d@dmv@d":dDI\###KD%0BTY^ku/v/v/v  , ,""r'r)F)NN)NNN)r)FFrF)TFFrF)FF)TF)'__name__ __module__ __qualname__r8rnrrrrrirrrrrrr3r@rBrWrndarrayrrdrbooltuplerjrrrrrrrrrrrNr r'r%rr'sFMFMFMFMR///> f f fD   <<< + + + Z   "22 A A ARARARAh\7\7\7|9P9P9P9PxaeD&D&D&D&L" ########J6%Z6%j6% 6% ! 6%  6% tRZ$&& '6%6%6%6%p$1c$1bj$1T$1$1$1$1LF#F#F#F#P        ""     .&*" AKAKAKAKF2222L 2222@***X 7 7 7%#%#%#%#%#r'r)$rnumpyrrdonnx.numpy_helperrrbase_quantizerrr calibrater onnx_modelr quant_utilsr r r r r rrrrrrrrrrrrrregistryrrr r'r%rs  &&&&&&========!!!!!!!!!!!!(('''''o#o#o#o#o#Mo#o#o#o#o#r'