ktj@#vddlZddlZddlZddlZddlmZddlZddl m Z m Z m Z m Z ddlmZddlmZddlmZejeZGddZd Zed kreZejreejejZejZ ej!"e r(e#d e d e$d e d ej%eZ&ee&ej'ej(ej) Z*e*+e*j&,e ddSdS)N) GraphProto ModelProto NodeProto TensorProto)quantize_matmul_bnb4) ONNXModel)attribute_to_kwargceZdZdZdZdZddededefdZe d e e d e e e ffd Zd ejd ejfd Zdede e d efdZde e fdZdZdS)MatMulBnb4QuantizerzMPerform 4b quantization of constant MatMul weights using FP4 or NF4 data typerrNmodel quant_type block_sizec|pg}|tjtjfvsJt||_||_||_t||_dSN) r FP4NF4r r rrsetnodes_to_exclude)selfr rrrs l/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/onnxruntime/quantization/matmul_bnb4_quantizer.py__init__zMatMulBnb4Quantizer.__init__%s_+1r157J7NOOOOOu%% $$ #$4 5 5 graph_pathreturnctt|dz ddD]'}||}|jD]}|j|kr||fccS(dS)Nr)NN)rangelen initializername)r!rgidgraphtensors r__get_initializerz%MatMulBnb4Quantizer.__get_initializer-swZ1,b"55 ) )CsOE+ ) );$&&!5=(((((' )zrfpweightc t|jdkrtd|}|j\}}||z}|j}||zdz |z}|dzdz}t j|d} t j||j} t| || ||j ||| | fS)z4b quantize fp32/fp16 weightz9Current bnb4 block quantization only supports 2D tensors!ruint8)dtype) rshape ValueError transposecopyrnpzerosr*rr) rr& fpweight_trowscolsnumelr num_blocksquantized_numelpackedabsmaxs rbnb4_block_quantz$MatMulBnb4Quantizer.bnb4_block_quant6s x~  ! # #XYY Y''))..00 ^ dt _ j(1,;  19*/999*HN;;;VZT_VZ\`aaarnode graph_stackc|jdkr|Std|jd|j|jvr%td|jd|S|jd}t ||\}}|td|Stj |}t|j d krtd |S| |\}}tj |} |jd z| _|jD])} | j|kr|j| n*tj |} |jd z| _|j| | gi} |j \} }| | d <|| d<|j| d<|j| d<tjj d|jd| j| jg|jdg|jr |jd znddd| }td|jd|S)zdIf the node is MatMul with fp32 const weight, quantize the weight with int4, and return the new nodeMatMulzstart to quantize z ...zexclude to quantize z$ as specified by nodes_to_exclude...rNz2MatMul doesn't have const weight. Skip to quantizer(z)MatMul weight is not 2D. Skip to quantize_Bnb4_absmaxKNrr MatMulBnb4r com.microsoft)inputsoutputsr!domainzcomplete quantization of )rB)op_typeloggerdebugr!rinputr %_MatMulBnb4Quantizer__get_initializeronnx numpy_helperto_arrayrr+r9 from_arrayremover extendrrhelper make_nodeoutput)rr:r;inputBBBs_graphB_arrayr7r8B_quantrK absmax_tensorkwargsr2r3matmul_bnb4_nodes r_bnb4_matmul_node_weightz,MatMulBnb4Quantizer._bnb4_matmul_node_weightLss <8 # #K 9$)999::: 9- - - LL_ ___ ` ` `KA);;FKPP 8 9 LLM N N NK#,,Q// w}   " " LLD E E EK..w77#..v66v' ^  EzV##%%e,,,$)44V<< Vi/ ##Wm$<===] ds s #|#|;0  JqM7<1CD[^$(, 9W$$r"        @@@@AAArcng}|d}|jD]^}d|jD}|ri}|jD]}|jtjjkr7||j|j| |i}n}|jtjj krTg} |j D]@} || | | |gA|j| i}nt|}||tjj|j|j|jfd|ji|}||||`|d|j |||S)Nrczg|]8}|jtjjks|jtjjk6|9S)typerMAttributeProtoGRAPHGRAPHS).0attrs r z9MatMulBnb4Quantizer._process_subgraph..sG9 3 999TY$J]Jd=d=d=d=d=drr!r:)r: attributerbrMrcrdappendgr!_process_subgraphregraphsrRr updaterSrTrHrKrUr^ ClearFieldpop) rr; new_nodesr#r: graph_attrsr\rgkvvaluesubgraphs rrlz%MatMulBnb4Quantizer._process_subgraphs BJ O OD NK   N&&DyD$7$===#**46222"i)?)? )L)LMd&9&@@@ "(, PPH'..x888!LL$*@*@*M*M)NOOOO"i//55MM"%%%%{,L$*dk@D MS   T::4MM N N N N     )$$$ rc\|jg}|j}d}|D]}|jdkrd}|s4|t jddg|||j dS)NFrDTr) r r# opset_importrGrRrMrS make_opsetidrlclean_initializers)rr;rw has_ms_domainopsets rprocesszMatMulBnb4Quantizer.processsz''))* z..00  ! % %E|.. $  P   !9!9/1!M!M N O O O {+++ %%'''''rr)__name__ __module__ __qualname____doc__rrrintr staticmethodlistrtuplerrLnpt ArrayLiker/ndarrayr9rr^rlr|rarrr r sWW C C66j6c6s6666D,<{T^G^A_\  2:    ,5 Y5 T*EU5 Zc5 5 5 5 n$T*-=$$$$L ( ( ( ( (rr ctjd}|ddd|ddd|d d d tjtjgd |dd dd|ddd d|d |ddtd gd|S)NaBlockwise FP4/NF4 quantization for MatMul 2D weight matrices. A weight matrix is partitioned into blocks, where each block is a contiguous subset inside the flattened transposed weight matrix. Each block is quantized into a set of 4b integers with an absolute value scaling factor. ) descriptionz --input_modelTzPath to the input model file)requiredhelpz--output_modelzPath to the output model filez --quant_typeFrz&Quantization data type. 0: FP4, 1: NF4)rdefaultchoicesrz --block_size@zVBlock size for blockwise quantization. Note: bnb.nn.Linear4bit only uses block_size=64)rrrz-vz --verbose store_true)raction)verbosez--nodes_to_exclude+zBSpecify the nodes to be excluded from quantization with node names)nargsrbrrr) argparseArgumentParser add_argumentr rr set_defaultsstr parse_args)parsers rrrs5  $F $=[\\\ (4>]^^^ $(*=*AB 5   e   kE,OOO &&&   Q      r__main__zfile z already exists)rT)-rloggingosnumpyr/ numpy.typingtypingrrM onnx.onnx_pbrrrronnxruntime.capi._pybind_stater onnx_modelr quant_utilsr getLoggerr}rIr rargsrsetLevelDEBUG input_modelinput_model_path output_modeloutput_model_pathpathexistserror Exceptionloadr rrrquantr|save_model_to_filerarrrs  GGGGGGGGGGGG??????!!!!!!++++++  8 $ $^(^(^(^(^(^(^(^(B$$$N z :<