§
    kŠtj=5 ã            	      ój  — d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z d dlZd dlZd dlmZmZmZmZ d dlmZmZmZmZ ddlmZ ddlmZmZ ddlmZ dd	l m!Z!m"Z"  ej#        d
ej$        ¬¦  «          ej%        e&¦  «        Z' G d„ d¦  «        Z( G d„ de(¦  «        Z) G d„ de(¦  «        Z* G d„ de(¦  «        Z+ G d„ de(¦  «        Z, G d„ de(¦  «        Z- G d„ de(¦  «        Z.d„ Z/ G d„ d¦  «        Z0dRd!„Z1dSd%„Z2dTd/„Z3d0„ Z4 G d1„ d2¦  «        Z5 G d3„ d4¦  «        Z6 G d5„ d6¦  «        Z7d7„ Z8d8„ Z9d9„ Z:e&d:k    �rt e:¦   «         Z;e;j<        re' =                    ej>        ¦  «         e;j?        Z@e;jA        ZBe!e;jC                 ZCe;jD        r eEe;jD        ¦  «        nd;ZDe;jF        r eEe;jF        ¦  «        ndZFejG         H                    eB¦  «        r(e' I                    d<eB› d=�¦  «          eJd<eB› d=�¦  «        ‚e;jK        r'e;jL        d>k    re' M                    d?¦  «         d@e;_K         ejN        e@¦  «        ZOe;jL        d>k    r e,e;jP        e;jQ        eDeF¬A¦  «        ZR�n e;jL        dBk    r' e-e;jP        e;jK        e;jS        eCeDeFe;jQ        ¬C¦  «        ZRnÎe;jL        dDk    r e)eD¬E¦  «        ZRn¶e;jL        dFk    r e*eD¬E¦  «        ZRnže;jL        dGk    r e+e;jP        eD¬H¦  «        ZRn€e;jL        dIk    rbeCe!jT        k    re' M                    dJ¦  «         e!jU        ZCe@ZOe;jV        �e;jV        dKk    rdLZVndMZVndLZV e.e;jW        e;jX        e;jY        eV¬N¦  «        ZRn eZdOe;jL        › �¦  «        ‚ e7eOe;jQ        e;jS        e;j[        e;j\        eR¬P¦  «        Z]e] ^                    ¦   «          e]jO         _                    eBdQ¦  «         dS dS )Ué    )ÚannotationsN)Ú
GraphProtoÚ
ModelProtoÚ	NodeProtoÚTensorProto)Úquantize_matmul_2bitsÚquantize_matmul_4bitsÚquantize_matmul_8bitsÚquantize_qdq_matmul_4bitsé   )ÚCalibrationDataReader)Úgptq_quantizeÚrtn_quantize)Ú	ONNXModel)ÚQuantFormatÚattribute_to_kwargz2%(asctime)s %(name)s [%(levelname)s] - %(message)s)ÚformatÚlevelc                  ó   — e Zd Z	 	 	 ddd„ZdS )ÚWeightOnlyQuantConfigNÚ	algorithmÚstrÚquant_formatr   Úop_types_to_quantizeútuple[str, ...] | NoneÚ
quant_axesú"tuple[tuple[str, int], ...] | NoneÚcustomized_weight_configúdict | Nonec                ó˜   — || _         || _        |rt          |¦  «        ndh| _        |rt	          |¦  «        ndddœ| _        || _        dS )aq  This is the Base class for Weight Only blockwise quantization Configuration.

        Args:
            algorithm:
                weight only quantize algorithm name.
            quant_format: QuantFormat{QOperator, QDQ}.
                QOperator format quantizes the model with quantized operators directly.
                QDQ format quantize the model by inserting QuantizeLinear/DeQuantizeLinear on the tensor.
            op_types_to_quantize (optional):
                set of operator types to quantize. Default {MatMul}
            quant_axes (dict[str, int], optional):
                op:axis, which axis to quantize for an op. Default {MatMul: 0, Gather: 1}
            customized_weight_config:
                customized weight config for nodes if needed. It is dictionary with node name as key,
                and the value is a dict of customized config.
        ÚMatMulr   r   )r!   ÚGatherN)r   r   Úsetr   Údictr   r   )Úselfr   r   r   r   r   s         úm/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/onnxruntime/quantization/matmul_nbits_quantizer.pyÚ__init__zWeightOnlyQuantConfig.__init__&   sb   € ð0 #ˆŒØ(ˆÔØAUÐ$e¥CÐ(<Ñ$=Ô$=Ð$=Ð\dÐ[eˆÔ!Ø.8ÐX�$˜zÑ*Ô*Ð*ÈÐVWÐ>XÐ>XˆŒØ(@ˆÔ%Ð%Ð%ó    )NNN)
r   r   r   r   r   r   r   r   r   r   )Ú__name__Ú
__module__Ú__qualname__r'   © r(   r&   r   r   %   s?   € € € € € ð
 8<Ø9=Ø04ðAð Að Að Að Að Að Ar(   r   c                  ó4   ‡ — e Zd Zdej        ddfdˆ fd„Zˆ xZS )ÚRTNWeightOnlyQuantConfigNr   r   r   r   c                óœ   •— |t           j        k    s
J d¦   «         ‚|€i }t          ¦   «                              d|||¬¦  «         || _        dS )ap  
        This is a class for round-to-nearest (RTN) algorithm Weight Only Quant Configuration.
        RTN is the most straightforward way to quantize weight using scale maps.

        Args:
            ratios:
                percentile of clip. Defaults to {}.
            quant_format (QuantFormat{QOperator, QDQ}, optional):
                QOperator format quantizes the model with quantized operators directly.
                QDQ format quantize the model by inserting QuantizeLinear/DeQuantizeLinear on the tensor.
                Defaults to QuantFormat.QOperator.
            op_types_to_quantize (optional):
                set of operator types to quantize.
            customized_weight_config:
                customized weight config for nodes if needed. It is dictionary with node name as key,
                and the value is a dict of customized config.
        z"RTN only supports QOperator formatNÚRTN©r   r   r   r   ©r   Ú	QOperatorÚsuperr'   Úratios©r%   r5   r   r   r   Ú	__class__s        €r&   r'   z!RTNWeightOnlyQuantConfig.__init__F   sf   ø€ ð0 �{Ô4Ò4Ð4Ð4Ð6ZÑ4Ô4Ð4àˆ>ØˆFÝ‰Œ×ÒØØ%Ø!5Ø%=ð	 	ñ 	
ô 	
ð 	
ð ˆŒˆˆr(   ©r   r   r   r   ©r)   r*   r+   r   r3   r'   Ú__classcell__©r7   s   @r&   r.   r.   E   sS   ø€ € € € € ð Ø Ô*Ø7;Ø04ð"ð "ð "ð "ð "ð "ð "ð "ð "ð "ð "r(   r.   c                  ó4   ‡ — e Zd Zdej        ddfdˆ fd„Zˆ xZS )ÚKQuantWeightOnlyQuantConfigNr   r   r   r   c                óœ   •— |t           j        k    s
J d¦   «         ‚|€i }t          ¦   «                              d|||¬¦  «         || _        dS )aF  
        This is a class for k-quant algorithm Weight Only Quant Configuration.

        Args:
            ratios:
                percentile of clip. Defaults to {}.
            quant_format (QuantFormat{QOperator, QDQ}, optional):
                QOperator format quantizes the model with quantized operators directly.
                QDQ format quantize the model by inserting QuantizeLinear/DeQuantizeLinear on the tensor.
                Defaults to QuantFormat.QOperator.
            op_types_to_quantize (optional):
                set of operator types to quantize.
        z&k-quant only supports QOperator formatNÚk_quantr1   r2   r6   s        €r&   r'   z$KQuantWeightOnlyQuantConfig.__init__l   sf   ø€ ð( �{Ô4Ò4Ð4Ð4Ð6^Ñ4Ô4Ð4àˆ>ØˆFÝ‰Œ×ÒØØ%Ø!5Ø%=ð	 	ñ 	
ô 	
ð 	
ð ˆŒˆˆr(   r8   r9   r;   s   @r&   r=   r=   k   sS   ø€ € € € € ð Ø Ô*Ø7;Ø04ðð ð ð ð ð ð ð ð ð ð r(   r=   c                  ó<   ‡ — e Zd Zddddddej        dfdˆ fd
„Zˆ xZS )ÚGPTQWeightOnlyQuantConfigNg{®Gáz„?é€   FTÚcalibration_data_readerúCalibrationDataReader | Noner   r   c	                óØ   •— |t           j        k    s
J d¦   «         ‚t          ¦   «                              d||¬¦  «         || _        || _        || _        || _        || _        || _	        dS )a  
        This is a class for GPTQ algorithm Weight Only Quant Configuration.
        GPTQ algorithm provides more accurate quantization but requires more computational resources.

        Args:
            calibration_data_reader:
                a calibration data reader. It enumerates calibration data and generates inputs for the original model.
            percdamp:
                percent of the average Hessian diagonal to use for dampening.
            block_size (int, optional):
                channel number in one block to execute a GPTQ quantization iteration.
            actorder (bool, optional):
                whether rearrange Hessian matrix considering the diag's value.
            mse (bool, optional):
                whether get scale and zero point with mse error.
            perchannel (bool, optional):
                whether quantize weight per-channel.
            quant_format (QuantFormat{QOperator, QDQ}, optional):
                QOperator format quantizes the model with quantized operators directly.
                QDQ format quantize the model by inserting QuantizeLinear/DeQuantizeLinear on the tensor.
                Defaults to QuantFormat.QOperator.
            op_types_to_quantize (optional):
                set of operator types to quantize.
        z#GPTQ only supports QOperator formatÚGPTQ)r   r   r   N)
r   r3   r4   r'   rC   ÚpercdampÚ
block_sizeÚactorderÚmseÚ
perchannel)
r%   rC   rG   rH   rI   rJ   rK   r   r   r7   s
            €r&   r'   z"GPTQWeightOnlyQuantConfig.__init__Ž   s~   ø€ ðF �{Ô4Ò4Ð4Ð4Ð6[Ñ4Ô4Ð4å‰Œ×ÒØØ%Ø!5ð 	ñ 	
ô 	
ð 	
ð
 (?ˆÔ$Ø ˆŒØ$ˆŒØ ˆŒØˆŒØ$ˆŒˆˆr(   )rC   rD   r   r   r9   r;   s   @r&   rA   rA   �   sa   ø€ € € € € ð AEØØØØØØ Ô*Ø7;ð/%ð /%ð /%ð /%ð /%ð /%ð /%ð /%ð /%ð /%ð /%r(   rA   c                  ó8   ‡ — e Zd Zdddej        ddfd
ˆ fd	„Zˆ xZS )ÚHQQWeightOnlyQuantConfigrB   é   r   Nr   r   r   r   c                ó°   •— |t           j        k    s
J d¦   «         ‚t          ¦   «                              d|||¬¦  «         || _        || _        || _        dS )a  
        This is a class for HQQ algorithm Weight Only Quant Configuration.
        HQQ algorithm quant weight without needing calibrate data.

        Args:
            block_size (int, optional):
                channel number in one block to execute a HQQ quantization iteration.
            bits (int, optional):
                how many bits to represent weight.
            axis (int, optional):
                0 or 1. which axis to quantize. https://arxiv.org/pdf/2309.15531.pdf
            quant_format (QuantFormat{QOperator, QDQ}, optional):
                QOperator format quantizes the model with quantized operators directly.
                QDQ format quantize the model by inserting QuantizeLinear/DeQuantizeLinear on the tensor.
                Defaults to QuantFormat.QOperator.
            op_types_to_quantize (optional):
                set of operator types to quantize.
            quant_axes (dict[str, int], optional):
                op:axis, which axis to quantize for an op. Default {MatMul: 0, Gather: 1}
        z"HQQ only supports QOperator formatÚHQQ©r   r   r   r   N)r   r3   r4   r'   rH   ÚbitsÚaxis)r%   rH   rR   rS   r   r   r   r7   s          €r&   r'   z!HQQWeightOnlyQuantConfig.__init__Á   sj   ø€ ð: �{Ô4Ò4Ð4Ð4Ð6ZÑ4Ô4Ð4å‰Œ×ÒØØ%Ø!5Ø!ð	 	ñ 	
ô 	
ð 	
ð %ˆŒØˆŒ	ØˆŒ	ˆ	ˆ	r(   )r   r   r   r   r9   r;   s   @r&   rM   rM   À   sY   ø€ € € € € ð ØØØ Ô*Ø7;Ø9=ð'ð 'ð 'ð 'ð 'ð 'ð 'ð 'ð 'ð 'ð 'r(   rM   c                  ó<   ‡ — e Zd Zdddej        ddddfdˆ fd„Zˆ xZS )ÚDefaultWeightOnlyQuantConfigrB   FNrN   rH   ÚintÚis_symmetricÚboolÚaccuracy_levelú
int | Noner   r   r   r   rR   Úchannel_wised_quantizec	                óÞ   •— t          ¦   «                              d|||¬¦  «         || _        || _        || _        || _        || _        |r|t          j        k    rt          d¦  «        ‚dS dS )a   
        This is a class for weight only affine quantization configuration.

        Args:
            block_size (int, optional):
                channel number in one block to execute an affine quantization iteration.
            is_symmetric (bool, optional):
                whether quantize weight symmetrically.
            accuracy_level (int, optional):
                Accuracy level of the 4-bit quantized MatMul computation.
                Refer to the MatMulNBits contrib op's 'accuracy_level' attribute for details.
                (https://github.com/microsoft/onnxruntime/blob/main/docs/ContribOperators.md#commicrosoftmatmulnbits)
            quant_format (QuantFormat{QOperator, QDQ}, optional):
                QOperator format quantizes the model with quantized operators directly.
                QDQ format quantize the model by inserting QuantizeLinear/DeQuantizeLinear on the tensor.
                Defaults to QuantFormat.QOperator.
            op_types_to_quantize (optional):
                set of operator types to quantize.
            quant_axes (dict[str, int], optional):
                op:axis, which axis to quantize for an op. Default {MatMul: 0, Gather: 1}
            bits (int, optional):
                number of bits per element after quantization. Default 4.
        ÚDEFAULTrQ   zAQuantFormat.QOperator is not supported channel_wised_quantize yetN)
r4   r'   rH   rW   rR   rY   r[   r   r3   ÚNotImplementedError)
r%   rH   rW   rY   r   r   r   rR   r[   r7   s
            €r&   r'   z%DefaultWeightOnlyQuantConfig.__init__ì   s’   ø€ õD 	‰Œ×ÒØØ%Ø!5Ø!ð	 	ñ 	
ô 	
ð 	
ð %ˆŒØ(ˆÔØˆŒ	Ø,ˆÔØ&<ˆÔ#Ø!ð 	k lµkÔ6KÒ&KÐ&KÝ%Ð&iÑjÔjÐjð	kð 	kÐ&KÐ&Kr(   )rH   rV   rW   rX   rY   rZ   r   r   r   r   rR   rV   r[   rX   r9   r;   s   @r&   rU   rU   ë   sj   ø€ € € € € ð Ø"Ø%)Ø Ô*Ø7;Ø9=ØØ',ð.kð .kð .kð .kð .kð .kð .kð .kð .kð .kð .kr(   rU   c                  ó2   ‡ — e Zd Z	 	 	 dˆ fd„	Zd„ Zd„ Zˆ xZS )ÚNVAWQWeightOnlyQuantConfigÚcnnú./cacheÚawq_litec                óÈ  •— 	 ddl }ddlm} || _         || _        n-# t          $ r  t	          d¦  «         t          d¦  «        d‚w xY w	 ddlm} || _        n-# t          $ r  t	          d¦  «         t          d¦  «        d‚w xY w	 dd	lm}m	}	 || _        |	| _	        n-# t          $ r  t	          d
¦  «         t          d¦  «        d‚w xY wt          ¦   «                              dt          j        dd¬¦  «         | j                              | j         j                             ¦   «         rdnd¦  «        }
|                      |||ddd|
ddddd¬¦  «        }|| _        || _        dS )a=  
        Configuration for the nvidia_awq quantization method.

        Args:
            tokenizer_dir (str): pathof the tokenizer dir.
            dataset_name (str): Name of the dataset.
            cache_dir (str): Directory for caching.
            calibration_method (str): calib method for nvidia_awq.
        r   N)Ú
DataLoaderzfError: The 'torch' library is required but not installed. Please install it using 'pip install torch'.z torch is not installed. Exiting.)Úload_datasetzlError: The 'datasets' library is required but not installed. Please install it using 'pip install datasets'.z#datasets is not installed. Exiting.)Ú
AutoConfigÚAutoTokenizerztError: The 'transformers' library is required but not installed. Please install it using 'pip install transformers'.z'transformers is not installed. Exiting.Ú
nvidia_awqrQ   ÚcudaÚcpué    r   i   TFrB   )Údataset_nameÚ
model_nameÚ	cache_dirÚ
calib_sizeÚ
batch_sizerH   ÚdeviceÚuse_fp16Úuse_buffer_shareÚadd_past_kv_inputsÚmax_calib_rows_to_loadÚadd_position_ids)ÚtorchÚtorch.utils.datare   ÚImportErrorÚprintÚdatasetsrf   Útransformersrg   rh   r4   r'   r   ÚQDQrr   rj   Úis_availableÚget_calib_inputsrC   Úcalibration_method)r%   Útokenizer_dirrm   ro   r�   rx   re   rf   rg   rh   rr   Úcalib_inputsr7   s               €r&   r'   z#NVAWQWeightOnlyQuantConfig.__init__  s  ø€ ð"
	LØˆLˆLˆLØ3Ð3Ð3Ð3Ð3Ð3àˆDŒJØ(ˆDŒOˆOøÝð 	Lð 	Lð 	LÝØxñô ð õ Ð@ÑAÔAÀtÐKð		Løøøð	OØ-Ð-Ð-Ð-Ð-Ð-à ,ˆDÔÐøÝð 	Oð 	Oð 	OÝØ~ñô ð õ ÐCÑDÔDÈ$ÐNð		Oøøøð		SØ>Ð>Ð>Ð>Ð>Ð>Ð>Ð>à(ˆDŒOØ!.ˆDÔÐøÝð 	Sð 	Sð 	SÝð Gñô ð õ ÐGÑHÔHÈdÐRð		Søøøõ 	‰Œ×ÒØ"Ý$œØ!%Øð	 	ñ 	
ô 	
ð 	
ð ”×"Ò"¨T¬Z¬_×-IÒ-IÑ-KÔ-KÐ#V 6 6ÐQVÑWÔWˆà×,Ò,Ø%Ø$ØØØØØØØ"Ø#Ø#&Ø!ð -ñ 
ô 
ˆð (4ˆÔ$Ø"4ˆÔÐÐs!   ƒ œ*AÁ
A Á*BÂB Â*Cc	           	     ól  — | j         }	|}
|}t          |t          ¦  «        r:|	                     |||	j        ¬¦  «        }
|	                     |||	j        ¬¦  «        }|
                     ¦   «         |                     ¦   «         dœ}|r[|                     ¦   «                              d¦  «        dz
  }|                     |dk    d¦  «         |                     ¦   «         |d<   |rÑ|r|	j	        n|	j
        }|
j        \  }}|j        }|j        |j        |j        z  }}t!          |j        ¦  «        D ]…}|	                     |||r|nd|||¬¦  «        }|	                     |||r|nd|||¬¦  «        }|                     d|› d�|                     ¦   «         d|› d	�|                     ¦   «         i¦  «         Œ†|S )
N)rr   Údtype)Ú	input_idsÚattention_maskéÿÿÿÿr   r   Úposition_idszpast_key_values.z.keyz.value)rx   Ú
isinstanceÚlistÚtensorÚint64Ú
contiguousÚlongÚcumsumÚmasked_fill_Úfloat16Úfloat32ÚshapeÚmax_position_embeddingsÚnum_key_value_headsÚhidden_sizeÚnum_attention_headsÚrangeÚnum_hidden_layersÚzerosÚupdate)r%   ÚconfigÚinput_ids_argÚattention_mask_argru   rr   rs   rt   rw   rx   r†   r‡   Úinputsr‰   Útorch_dtyperq   Úsequence_lengthÚmax_sequence_lengthÚ	num_headsÚ	head_sizeÚiÚpast_keyÚ
past_values                          r&   Úmake_model_inputz+NVAWQWeightOnlyQuantConfig.make_model_inputn  s  € ð ”
ˆà!ˆ	Ø+ˆå�m¥TÑ*Ô*ð 	`ØŸš ]¸6ÈÌ˜ÑUÔUˆIØ"Ÿ\š\Ð*<ÀVÐSXÔS^˜\Ñ_Ô_ˆNð #×-Ò-Ñ/Ô/Ø,×7Ò7Ñ9Ô9ð
ð 
ˆð
 ð 	?Ø)×.Ò.Ñ0Ô0×7Ò7¸Ñ;Ô;¸aÑ?ˆLØ×%Ò% n¸Ò&9¸1Ñ=Ô=Ð=Ø%1×%<Ò%<Ñ%>Ô%>ˆF�>Ñ"àð 	Ø+3ÐF˜%œ-˜-¸¼ˆKØ*3¬/Ñ'ˆJ˜Ø"(Ô"@ÐàÔ*ØÔ" fÔ&@Ñ@ð !ˆIõ ˜6Ô3Ñ4Ô4ð ð �Ø Ÿ;š;ØØØ+;ÐBÐ'Ð'ÀØØ!Ø%ð 'ñ ô �ð #Ÿ[š[ØØØ+;ÐBÐ'Ð'ÀØØ!Ø%ð )ñ ô �
ð —’à2¨1Ð2Ð2Ð2°H×4GÒ4GÑ4IÔ4IØ4¨1Ð4Ð4Ð4°j×6KÒ6KÑ6MÔ6Mðñô ð ð ð ˆr(   c                ó¤  — | j         }| j        }| j        }|                     |d|d¬¦  «        }|                     |d|d¬¦  «        }|                     ddi¦  «         |j        |_        ||k    s
J d¦   «         ‚d|v r1 |ddd	¬
¦  «                             t          |¦  «        ¦  «        }d}n'd|v r |dd¬¦  «        }d}nt          d|› d�¦  «        ‚||         d |…         }| 
                    |ddd|¬¦  «        }|                     |¦  «        }|d         }|d         }| j        } |||d¬¦  «        } |||d¬¦  «        }t          |j        ¦  «        t          |j        ¦  «        k    sJ ‚t          |¦  «        t          |¦  «        k    sJ ‚||z  }g }t          |¦  «        D ]%\  }}|                     |¦  «         ||dz
  k    r nŒ&g }t          |¦  «        D ]%\  }}|                     |¦  «         ||dz
  k    r nŒ&t#          d|› dt          |¦  «        › dt          |¦  «        › d�¦  «         g }t          |¦  «        D ]a} ||          }!||          }"|                      ||!|"|
|||	|¦  «        }#d„ |#                     ¦   «         D ¦   «         }#|                     |#¦  «         Œbt#          dt          |¦  «        › d�¦  «         |S ) NT)Úuse_auth_tokenro   Útrust_remote_codeÚ	pad_tokenz[PAD]z8calib size should be no more than max_calib_rows_to_loadra   Úcnn_dailymailz3.0.0Útrain)ÚnameÚsplitÚarticleÚpilezmit-han-lab/pile-val-backupÚ
validation)r±   Útextz	dataset "z" not supportedÚpt)Úreturn_tensorsÚpaddingÚ
truncationÚ
max_lengthr†   r‡   F)rq   Úshuffler   z/
--Quantize-Script-- number_of_batched_samples=z, batch-input-ids-list-len=z, batched_attention_mask=ú
c                ób   — i | ],\  }}||                      ¦   «                              ¦   «         “Œ-S r,   )rk   Únumpy)Ú.0Ú
input_nameÚtorch_tensors      r&   ú
<dictcomp>z?NVAWQWeightOnlyQuantConfig.get_calib_inputs.<locals>.<dictcomp>  s7   € ÐlÐlÐlÑAYÀÈ\�j ,×"2Ò"2Ñ"4Ô"4×":Ò":Ñ"<Ô"<ÐlÐlÐlr(   z0
--Quantize-Script-- number of batched inputs = )rg   rh   rf   Úfrom_pretrainedÚadd_special_tokensÚ	eos_tokenr­   Úselectr™   Ú
ValueErrorÚbatch_encode_plusÚtore   ÚlenÚdatasetÚ	enumerateÚappendr{   r©   Úitems)$r%   rm   rn   ro   rp   rq   rH   rr   rs   rt   ru   rv   rw   Úauto_configÚauto_tokenizerrf   r�   Ú	tokenizerÚdataset2ÚcolumnÚbatch_encodedÚbatch_encoded_input_idsÚbatch_encoded_attention_maskÚdata_loaderÚcalib_dataloader_input_idsÚcalib_dataloader_attention_maskÚnumber_of_batched_samplesÚbatched_input_idsÚidxÚdataÚbatched_attention_maskÚbatched_inputs_listr¦   r†   r‡   r    s$                                       r&   r€   z+NVAWQWeightOnlyQuantConfig.get_calib_inputs¯  sí  € ð  ”oˆØÔ+ˆØÔ(ˆà×,Ò,Ø t°yÐTXð -ñ 
ô 
ˆð #×2Ò2Ø t°yÐTXð 3ñ 
ô 
ˆ	ð 	×$Ò$ k°7Ð%;Ñ<Ô<Ð<Ø'Ô1ˆ	ÔàÐ3Ò3Ð3Ð3Ð5oÑ3Ô3Ð3à�LÐ Ð Ø#�| O¸'ÈÐQÑQÔQ×XÒXÕY^Ð_uÑYvÔYvÑwÔwˆHØˆFˆFØ�|Ð#Ð#Ø#�|Ð$AÈÐVÑVÔVˆHØˆFˆFåÐF¨ÐFÐFÐFÑGÔGÐGà˜FÔ# K Z KÔ0ˆØ!×3Ò3Ø T°4ÀDÐU_ð 4ñ 
ô 
ˆð &×(Ò(¨Ñ0Ô0ˆØ"/°Ô"<ÐØ'4Ð5EÔ'FÐ$ð ”oˆà%0 [Ð1HÐU_ÐinÐ%oÑ%oÔ%oÐ"Ø*5¨+Ø(°ZÈð+
ñ +
ô +
Ð'õ Ð-Ô5Ñ6Ô6½#Ð>]Ô>eÑ:fÔ:fÒfÐfÐfÐfÝÐ-Ñ.Ô.µ#Ð6UÑ2VÔ2VÒVÐVÐVÐVà$.°*Ñ$<Ð!àÐÝ"Ð#=Ñ>Ô>ð 	ð 	‰IˆC�Ø×$Ò$ TÑ*Ô*Ð*ØÐ0°1Ñ4Ò5Ð5Ø�ð 6ð "$ÐÝ"Ð#BÑCÔCð 	ð 	‰IˆC�Ø"×)Ò)¨$Ñ/Ô/Ð/ØÐ0°1Ñ4Ò5Ð5Ø�ð 6õ 	ðyÐ?Xð yð yÝ(+Ð,=Ñ(>Ô(>ðyð yÝY\Ð]sÑYtÔYtðyð yð yñ	
ô 	
ð 	
ð
 !ÐÝÐ0Ñ1Ô1ð 	/ð 	/ˆAØ)¨!Ô,ˆIØ3°AÔ6ˆNà×*Ò*ØØØØ"ØØØ Ø ñ	ô 	ˆFð mÐlÐ]c×]iÒ]iÑ]kÔ]kÐlÑlÔlˆFØ×&Ò& vÑ.Ô.Ð.Ð.åÐ^Å#ÐFYÑBZÔBZÐ^Ð^Ð^Ñ_Ô_Ð_Ø"Ð"r(   )ra   rb   rc   )r)   r*   r+   r'   r©   r€   r:   r;   s   @r&   r`   r`     su   ø€ € € € € ð ØØ%ðN5ð N5ð N5ð N5ð N5ð N5ð`?ð ?ð ?ðBa#ð a#ð a#ð a#ð a#ð a#ð a#r(   r`   c                óX   — t          |t          j        | |z  ¦  «        z  ¦  «        | k    S ©N)rV   ÚnpÚceil)Úval1Úval2s     r&   Úis_divisibleræ     s(   € Ýˆt•b”g˜d T™kÑ*Ô*Ñ*Ñ+Ô+¨tÒ3Ð3r(   c                  ó^   — e Zd Zdd„Ze	 	 	 ddd„¦   «         Zed„ ¦   «         Z	 dd„Zdd„ZdS ) ÚHQQWeightOnlyQuantizerr�   rM   c                ó   — || _         d S rá   ©r�   ©r%   r�   s     r&   r'   zHQQWeightOnlyQuantizer.__init__  ó   € ð ˆŒˆˆr(   r   NFÚmin_maxú	list[int]rS   rV   Ú
opt_paramsr   c                ó  ‡— dd l Š|€dddddœn|}|d         |d         |d	         |d
         f\  }}}	}
| j        r‰j        n‰j        }|                      |¦  «        }|                     |¦  «        }|                     |¦  «        }|fˆfd„	}d}t          |
¦  «        D ]ß}‰                     ||z  |z   ¦  «                             |d         |d         ¦  «        }||z
  |z  } |||z
  |¦  «        }‰                     |||z
  |z  z
  |d¬¦  «        }||	z  }t          ‰ 
                    ||z
  ¦  «                             ¦   «         ¦  «        }|r#t          |t          j        |d¦  «        ¦  «         ||k     r|}Œß ~~~~||fS )Nr   gffffffæ?g      $@g)\�Âõ(ð?é   )Úlp_normÚbetaÚkappaÚitersrò   ró   rô   rõ   c           
     ó¨  •— |dk    rN‰                      | ¦  «        ‰j        j                             ‰                     | ¦  «        d|z  z
  ¦  «        z  S ‰                      | ¦  «        ‰j        j                             ‰                     | ¦  «        d|z  ‰                     ‰                     | ¦  «        dz   |dz
  ¦  «        z  z
  ¦  «        z  S )Nr   ç      ð?g:Œ0âŽyE>)ÚsignÚnnÚ
functionalÚreluÚabsÚpow)Úxró   Úprx   s      €r&   Ú	shrink_opz:HQQWeightOnlyQuantizer.optimize_weights.<locals>.shrink_op8  sµ   ø€ Ø�AŠvˆvØ—z’z !‘}”} u¤xÔ':×'?Ò'?ÀÇ	Â	È!ÁÄÈsÐUYÉzÑ@YÑ'ZÔ'ZÑZÐZà—z’z !‘}”} u¤xÔ':×'?Ò'?Ø—I’I˜a‘L”L C¨$¡J°%·)²)¸E¿IºIÀa¹L¼LÈ4Ñ<OÐQRÐUVÑQVÑ2WÔ2WÑ#WÑWñ(ô (ñ ð r(   g     ˆÃ@r   T©rS   Úkeepdimé   )rx   Úis_cudar’   r“   rÉ   r™   ÚroundÚclampÚmeanÚfloatrü   r{   râ   )rŒ   ÚscaleÚzerorí   rS   rï   Úverboserò   ró   rô   rõ   r…   Úw_fr   Ú
best_errorr¦   Úw_qÚw_rÚw_eÚcurrent_errorrx   s                       @r&   Úoptimize_weightsz'HQQWeightOnlyQuantizer.optimize_weights  sÔ  ø€ ð 	ˆˆˆàR\ÐRd ¨c¸DÈ2ÐNÐNÐNÐjtˆ
à�yÔ!Ø�vÔØ�wÔØ�wÔð	'
Ñ#ˆ��u˜eð "(¤ÐB�”�°U´]ˆØ�iŠi˜ÑÔˆØ—’˜‘”ˆØ�wŠw�u‰~Œ~ˆà!(ð 	ð 	ð 	ð 	ð 	ð 	ð ˆ
Ý�u‘”ð 	ð 	ˆAØ—+’+˜c E™k¨DÑ0Ñ1Ô1×7Ò7¸À¼
ÀGÈAÄJÑOÔOˆCØ˜‘: Ñ&ˆCØ�)˜C #™I tÑ,Ô,ˆCØ—:’:˜c S¨3¡Y°%Ñ$7Ñ7¸dÈD�:ÑQÔQˆDØ�E‰MˆDå! %§)¢)¨C°#©IÑ"6Ô"6×";Ò";Ñ"=Ô"=Ñ>Ô>ˆMØð 5Ý�a�œ -°Ñ3Ô3Ñ4Ô4Ð4Ø˜zÒ)Ð)Ø*�
�
àà��c˜3à�dˆ{Ðr(   c                ó  — | j         d         |j         d         k    r|j        }| j        } |dv rO|                      ¦   «         dz  |z  }t          |¦  «        D ]#}| dd …xx         ||d |…         ||z  z  z  cc<   Œ$d S t	          d¦  «        ‚)Nr   ©é   rN   é   r  zOnly 2,4,8 bits are supported.)r”   ÚTÚelement_sizer™   r^   )Úpack_tensorÚori_int_tensorrR   Úcompress_ratioÚjs        r&   Úpack_on_row_fast_248bitz.HQQWeightOnlyQuantizer.pack_on_row_fast_248bitT  s»   € àÔ˜QÔ >Ô#7¸Ô#:Ò:Ð:Ø+Ô-ˆNØ%œ-ˆKØ�9ÐÐØ(×5Ò5Ñ7Ô7¸!Ñ;¸tÑCˆNÝ˜>Ñ*Ô*ð Uð U�Ø˜A˜B˜B��” >°!Ð2C°^Ð2CÔ#DÈÐQRÉÑ#TÑT��‘�ðUð Uõ &Ð&FÑGÔGÐGr(   rN   Té@   r   c                óö  — dd l }|                     ¦   «         }	|	j        }
||
|         |z  z
  |z  }|dk    r%|j        j                             |	d|fdd¦  «        }	n&|j        j                             |	ddd|fdd¦  «        }	|	j        }|�6|r4|dk    r|	                     d|g¦  «        n|	                     |dg¦  «        }	|du r+|	                     ¦   «         |	                     ¦   «         }}d}n:|	                     |d¬¦  «        d         }|	                     |d¬¦  «        d         }d|z  dz
  }d}||g}|||z
  z   	                    d	¬
¦  «        }||z
  }|dk     
                    ¦   «                              ¦   «         dk    r"|||dk    <   ||z   	                    d	¬
¦  «        }| |z  }|r|                     |¦  «        }|r|                      |	||||¬¦  «        \  }}|                     |	|z  |z   ¦  «         	                    |d         |d         ¦  «        }|                     |¦  «                             ¦   «         }d|z  }|dk    r9|                     |d         d¦  «        }|                     |d         d¦  «        }n8|                     d|d         ¦  «        }|                     d|d         ¦  «        }~	~~||                     |j        ¦  «        |                     |j        ¦  «        fS )Nr   r   Úconstantrˆ   FTr  r  g     ˆÓ@)Úmax)rŒ   r	  r
  rí   rS   r÷   )rx   r  r”   rù   rú   ÚpadÚreshapeÚminr!  r  ÚsumÚitemr  r  rV   rÉ   r…   )r%   rŒ   rR   Úchannel_wiseÚ
group_sizeÚoptimizeÚ
round_zerorS   rx   ÚweightÚ	ori_shapeÚpad_lenr”   Ú_minÚ_maxÚmax_vÚmin_vrí   r	  Úmin_max_axisr
  r  s                         r&   Úquantize_internalz(HQQWeightOnlyQuantizer.quantize_internala  s  € ð 	ˆˆˆà—’‘”ˆØ”Lˆ	à 	¨$¤°*Ñ <Ñ<À
ÑJˆØ�1Š9ˆ9Ø”XÔ(×,Ò,¨V°a¸°\À:ÈqÑQÔQˆFˆFà”XÔ(×,Ò,¨V°a¸¸A¸wÐ5GÈÐUVÑWÔWˆFØ”ˆð Ð"¨Ð"Ø:>À!º)¸)�V—^’^ R¨Ð$4Ñ5Ô5Ð5È&Ï.Ê.ÐZdÐfhÐYiÑJjÔJjˆFð ˜5Ð Ð ØŸš™œ v§z¢z¡|¤|�$ˆDØˆHˆHà—:’: 4°�:Ñ6Ô6°qÔ9ˆDØ—:’: 4°�:Ñ6Ô6°qÔ9ˆDà�4‘˜!‘ˆØˆØ˜%�.ˆð ˜$ ™+Ñ&×-Ò-°#Ð-Ñ6Ô6ˆà˜d‘{ˆØ˜AÒ×"Ò"Ñ$Ô$×)Ò)Ñ+Ô+¨aÒ/Ð/Ø.3ˆL˜¨Ò*Ñ+Ø˜\Ñ)×0Ò0°SÐ0Ñ9Ô9ˆEØˆu�u‰}ˆàð 	%Ø—;’;˜tÑ$Ô$ˆDð ð 	sØ×/Ò/°vÀUÐQUÐ_fÐmqÐ/ÑrÔr‰KˆE�4ð �kŠk˜& 5™.¨4Ñ/Ñ0Ô0×6Ò6°w¸q´zÀ7È1Ä:ÑNÔNˆØ�kŠk˜%Ñ Ô ×$Ò$Ñ&Ô&ˆà�e‘ˆØ�1Š9ˆ9Ø—M’M %¨¤(¨BÑ/Ô/ˆEØ—<’<  a¤¨"Ñ-Ô-ˆDˆDà—M’M " e¨B¤iÑ0Ô0ˆEØ—<’<  E¨"¤IÑ.Ô.ˆDà�D˜$à�E—H’H˜Vœ\Ñ*Ô*¨D¯GªG°F´LÑ,AÔ,AÐAÐAr(   Únoder   Úgraph_stackúlist[GraphProto]Úreturnúlist[NodeProto]c                ó¸  — |j         dk    rt          d¦  «        ‚ddl}t                               d|j        › d�¦  «         |j        d         }t          ||¦  «        \  }}|€t                               d¦  «         |gS t          j	         
                    |¦  «        }|j        }t          |¦  «        d	k    r¡t          |¦  «        d	k     rt                               d
¦  «         |gS |dd…         }	t          d„ |	D ¦   «         ¦  «        r+t                               dt          |	¦  «        ¦  «         |gS |                     |d         |d         ¦  «        }nd}|                     |¦  «        }
|j                             ¦   «         r|
                     ¦   «         }
| j        j        }|                      |
j        || j        j        ¬¦  «        \  }}}|                     ¦   «         }|                     ¦   «         }|                     ¦   «         }d|z  }|                     |j        d         |j        d         |z  f|j        |j        ¬¦  «        }|                      |||¦  «         |                     ¦   «                              ¦   «         }|                     ¦   «                              ¦   «         }|                     d¦  «        }|                     d¦  «        }|
j        \  }}| j        j        }||z  }||z   dz
  |z  }|                     |||¦  «        }t          j	                             |                     ¦   «                              ¦   «         ¦  «        }|j        dz   tA          |¦  «        z   |_        |j        D ])}|j        |k    r|j         !                    |¦  «          nŒ*t          j	                             |¦  «        }|j        dz   |_        |j"         #                    ||g¦  «         |j        d         |j        |j        g}t          j	                             |¦  «        }|j        dz   |_        |j"         #                    |g¦  «         | $                    |j        ¦  «         i }|j        \  }}||d<   ||d<   ||d<   | j        j        |d<   tK          |j        d         |¦  «        }|�t          |¦  «        nd}|duo	|du p||k     } | s|j&        d         n|j&        d         dz   }!t          j'        j(        	 d ||!g|j        r|j        dz   tA          |¦  «        z   ndddœ|¤Ž}"|"g}#| rw|# #                    tS          |j        d         |||j        r|j        dz   tA          |¦  «        z   n|j        dz   tA          |¦  «        z   |!|j&        d         ¬¦  «        ¦  «         t                               d|j        › d�¦  «         |#S )!á½  
        Target node:        QOperator node:            QDQ nodes:
        MatMul              MatMulNBits                DeQuantizeLinear -> MatMul
        Gather              GatherBlockQuantized       Gather, Gather, Gather (optional) -> DequantizeLinear
        If the node is target node with fp32 or fp16 const weight, quantize the weight to int4 and
        return the new nodes.
        If QOperator format, return the corresponding QOperator nodes.
        If QDQ format, return the corresdponging QDQ nodes.
        Gather (quantized data) + Gather (scales) + Gather (optional, zero points) -> DequantizeLinear is
        not supported yet because Gather does not support int4 data.
        r"   z/Gather quantization is not supported yet in HQQr   Nústart to quantize ú ...r   ú2MatMul doesn't have const weight. Skip to quantizer  ú<MatMul weight has fewer than 2 dimensions. Skip to quantize.éþÿÿÿc              3  ó"   K  — | ]
}|d k    V — ŒdS ©r   Nr,   ©r¿   Úds     r&   ú	<genexpr>z2HQQWeightOnlyQuantizer.quantize.<locals>.<genexpr>Â  ó&   è è € Ð+Ð+˜a�1˜’6Ð+Ð+Ð+Ð+Ð+Ð+r(   úcMatMul weight has non-unit batch dims %s; N-D batched quantization not supported. Skip to quantize.rˆ   )rR   r(  r  )r…   rr   Ú_QÚ_scalesÚ_zero_pointsÚKÚNrR   rH   Ú_pre_reshapeÚMatMulNBitsÚ úcom.microsoft©r    Úoutputsr°   Údomain©Úa_input_nameÚb_original_shapeÚtarget_graphÚname_prefixÚpre_reshape_outputÚfinal_outputúcomplete quantization of ©rM  )*Úop_typer^   rx   ÚloggerÚinfor°   ÚinputÚget_initializerÚonnxÚnumpy_helperÚto_arrayr”   rÊ   Úanyr‹   r#  Ú
from_numpyrj   r   r�   rR   r3  r  rH   rŽ   r›   Úuint8rr   r  rk   r¾   Ú
from_arrayr   ÚremoveÚinitializerÚextendrÍ   Ú_get_static_rankÚoutputÚhelperÚ	make_nodeÚ_build_nbits_output_reshape)$r%   r4  r5  rx   Úinput_bÚb_pbÚbs_graphÚb_arrayrU  ÚleadingÚb_array_torchrR   Úquant_weight_torchÚscales_torchÚzero_points_torchÚpacked_sizeÚpacked_torchÚscalesÚzero_pointsÚrowsÚcolsrH   Ú	blob_sizeÚk_blocksÚb_quantr_  Úscales_tensorÚinput_namesÚ	zp_tensorÚkwargsÚa_static_rankÚrank_b_origÚneeds_reshapeÚmatmul_q_outputÚmatmul_q_nodeÚoutput_nodess$                                       r&   ÚquantizezHQQWeightOnlyQuantizer.quantize¢  s  € ð Œ<˜8Ò#Ð#Ý%Ð&WÑXÔXÐXàˆˆˆå�ŠÐ8¨¬Ð8Ð8Ð8Ñ9Ô9Ð9Ø”*˜Q”-ˆÝ(¨°+Ñ>Ô>‰ˆˆhØˆ<Ý�KŠKÐLÑMÔMÐMØ�6ˆMåÔ#×,Ò,¨TÑ2Ô2ˆØ"œ=ÐÝÐÑ Ô  AÒ%Ð%ÝÐ#Ñ$Ô$ qÒ(Ð(Ý—’ÐZÑ[Ô[Ð[Ø�v�Ø& s¨ sÔ+ˆGÝÐ+Ð+ 7Ð+Ñ+Ô+Ñ+Ô+ð Ý—’ð(å˜‘M”Mñô ð ð
 �v�à—o’oÐ&6°rÔ&:Ð<LÈRÔ<PÑQÔQˆGˆGà#ÐØ×(Ò(¨Ñ1Ô1ˆØŒ:×"Ò"Ñ$Ô$ð 	1Ø)×.Ò.Ñ0Ô0ˆMàŒ{ÔˆØ>B×>TÒ>TØŒO $°4´;Ô3Ið ?Uñ ?
ô ?
Ñ;Ð˜LÐ*;ð 0×:Ò:Ñ<Ô<ÐØ#×.Ò.Ñ0Ô0ˆØ-×8Ò8Ñ:Ô:Ðà˜4‘iˆà—{’{ØÔ% aÔ(Ð*<Ô*BÀ1Ô*EÈÑ*TÐUØ”+Ø%Ô,ð #ñ 
ô 
ˆð
 	×$Ò$ \Ð3EÀtÑLÔLÐLØ×!Ò!Ñ#Ô#×)Ò)Ñ+Ô+ˆØ'×+Ò+Ñ-Ô-×3Ò3Ñ5Ô5ˆà—’ Ñ#Ô#ˆØ!×)Ò)¨"Ñ-Ô-ˆØ"Ô(‰
ˆˆdØ”[Ô+ˆ
Ø +Ñ-ˆ	Ø˜:Ñ%¨Ñ)¨jÑ8ˆØ#×+Ò+¨D°(¸IÑFÔFˆåÔ#×.Ò.¨|×/?Ò/?Ñ/AÔ/A×/GÒ/GÑ/IÔ/IÑJÔJˆØ”y 4Ñ'­#¨d©)¬)Ñ3ˆŒØ”^ð 	ð 	ˆEØŒz˜WÒ$Ð$Ø”×%Ò% eÑ,Ô,Ð,Ø�ð %õ Ô)×4Ò4°VÑ<Ô<ˆØ!œY¨Ñ2ˆÔØÔ×#Ò# W¨mÐ$<Ñ=Ô=Ð=à”z !”} g¤l°MÔ4FÐGˆÝÔ%×0Ò0°Ñ=Ô=ˆ	Øœ ^Ñ3ˆ	ŒØÔ×#Ò# Y KÑ0Ô0Ð0Ø×Ò˜9œ>Ñ*Ô*Ð*àˆØ”]‰
ˆˆdØˆˆs‰Øˆˆs‰Øˆˆv‰Ø#œ{Ô5ˆˆ|Ñõ )¨¬°A¬¸ÑDÔDˆØ/?Ð/K•cÐ*Ñ+Ô+Ð+ÐQRˆØ(°Ð4Ðo¸-È4Ð:OÐ:nÐS`ÐcnÒSnˆØ0=Ðb˜$œ+ aœ.˜.À4Ä;ÈqÄ>ÐTbÑCbˆÝœÔ-Øð
àØ$Ð%Ø15´ÐB�”˜TÑ!¥C¨¡I¤IÑ-Ð-ÀØ"ð
ð 
ð ð
ð 
ˆð &�ˆØð 	ð ×ÒÝ+Ø!%¤¨A¤Ø%5Ø!)ØBFÄ)Ð o ¤¨TÑ!1µC¸±I´IÑ!=Ð!=ÐRVÔR[Ð^bÑRbÕehÐimÑenÔenÑRnØ'6Ø!%¤¨Q¤ðñ ô ñ	ô 	ð 	õ 	�ŠÐ?°´	Ð?Ð?Ð?Ñ@Ô@Ð@àÐr(   )r�   rM   )r   NF)rí   rî   rS   rV   rï   r   )rN   Tr  TTr   ©r4  r   r5  r6  r7  r8  )	r)   r*   r+   r'   Ústaticmethodr  r  r3  rŒ  r,   r(   r&   rè   rè     s±   € € € € € ðð ð ð ð ð Ø"&Øð2ð 2ð 2ð 2ñ „\ð2ðh ð	Hð 	Hñ „\ð	Hð fgð?Bð ?Bð ?Bð ?BðBEð Eð Eð Eð Eð Er(   rè   Ú
graph_pathr6  r7  útuple[TensorProto, GraphProto]c                ó˜   — t          t          |¦  «        dz
  dd¦  «        D ]'}||         }|j        D ]}|j        | k    r||fc c S ŒŒ(dS )Nr   rˆ   )NN)r™   rÊ   ri  r°   )r°   r�  ÚgidÚgraphrŒ   s        r&   r`  r`  *  sw   € Ý•S˜‘_”_ qÑ(¨"¨bÑ1Ô1ð %ð %ˆØ˜3”ˆØÔ'ð 	%ð 	%ˆFØŒ{˜dÒ"Ð"Ø˜u�}Ð$Ð$Ð$Ð$Ð$ð #ð	%ð ˆ:r(   Útensor_namer   rZ   c                óz  — t          t          |¦  «        dz
  dd¦  «        D ]˜}||         }t          |j        ¦  «        t          |j        ¦  «        z   t          |j        ¦  «        z   D ]O}|j        | k    rB|j        j        }| 	                    d¦  «        rt          |j
        j        ¦  «        c c S   dS ŒPŒ™dS )u†  Return the static rank of a tensor if its shape is known, else None.

    Searches graph inputs, value_info, and outputs in the graph stack (inner-most
    graph first).  A known shape requires ``HasField('shape')`` to be true on the
    tensor_type; the rank is then ``len(shape.dim)``.  Individual dim sizes may
    still be symbolic â€” only the rank (dimension count) matters here.
    r   rˆ   r”   N)r™   rÊ   r‹   r_  Ú
value_inforl  r°   ÚtypeÚtensor_typeÚHasFieldr”   Údim)r”  r�  r’  r“  ÚviÚtts         r&   rk  rk  3  sÎ   € õ •S˜‘_”_ qÑ(¨"¨bÑ1Ô1ð ð ˆØ˜3”ˆÝ�u”{Ñ#Ô#¥d¨5Ô+;Ñ&<Ô&<Ñ<½tÀEÄLÑ?QÔ?QÑQð 	ð 	ˆBØŒw˜+Ò%Ð%Ø”WÔ(�Ø—;’;˜wÑ'Ô'ð -Ý˜rœxœ|Ñ,Ô,Ð,Ð,Ð,Ð,Ð,Ø�t�t�tð	 &ð	ð ˆ4r(   rT  rU  ÚtuplerV  r   rW  rX  rY  r8  c                ó	  — t          |¦  «        }|d         }|dz   |z   }|dz   }	|dz   }
|dz   }|dz   }|dz   }|dz   }|d	z   }|d
z   }|j                             t          j                             t          j        dt          j        ¬¦  «        |	¬¦  «        t          j                             t          j        dt          j        ¬¦  «        |
¬¦  «        t          j                             t          j        dt          j        ¬¦  «        |¬¦  «        t          j                             t          j        dgt          j        ¬¦  «        |¬¦  «        t          j                             t          j        |t          j        ¬¦  «        |¬¦  «        t          j                             t          j        |gt          j        ¬¦  «        |¬¦  «        t          j                             t          j        dgt          j        ¬¦  «        |¬¦  «        g¦  «         |dz   }|dz   }|dz   }|dz   }|dz   }|dz   }|dz   }|dz   }|dz   }|dz   }|dz   }|dz   }t          j	         
                    d| g|g|dz   ¬¦  «        t          j	         
                    d|g|g|dz   ¬¦  «        t          j	         
                    d||	g|g|d z   ¬¦  «        t          j	         
                    d!||g|g|d"z   ¬¦  «        t          j	         
                    d#||g|g|d$z   ¬¦  «        t          j	         
                    d!||	g|g|d%z   ¬¦  «        t          j	         
                    d&||g|g|d'z   ¬¦  «        t          j	         
                    d(|g|g|d)z   t          j	                             |t          j        dgdg¬*¦  «        ¬+¦  «        t          j	         
                    d#||
g|g|d,z   ¬¦  «        t          j	         
                    d&||g|g|d-z   ¬¦  «        t          j	         
                    d.||||g|g|d/z   ¬¦  «        t          j	         
                    d0|||g|g|d1z   d¬2¦  «        t          j	         
                    d&||g|g|d3z   ¬¦  «        g}|S )4uß  Build the reshape chain that restores the ONNX MatMul-broadcast output shape.

    MatMulNBits produces shape ``[...A_batch_dims, M, N]`` (rank = rank(A)). To match
    the original ``MatMul(A, B_orig)`` output, where B_orig has all-unit leading
    dims, we need:

        a_rank_eff = max(rank(A), 2)   # ONNX promotes 1-D A to rank-2
        out_shape = [1] * max(rank(B_orig) - a_rank_eff, 0) + A.shape[:-1] + [N]

    This is built dynamically via Shape/Gather/Max/Sub/Max/ConstantOfShape/Slice/Concat
    so it works regardless of A's static rank (handles rank(A) == 1, rank(A) == 2
    â€” the common transformer case â€” as well as rank(A) >= rank(B_orig) where no
    leading-1 prepending is needed). All ops used are valid from opset 11 onward.

    Args:
        a_input_name: name of the activation input edge (A) feeding MatMulNBits.
        b_original_shape: the original (pre-squeeze) shape of B, e.g. ``(1, K, N)``.
        target_graph: graph proto to append helper initializers into.
        name_prefix: unique prefix for generated node/initializer names.
        pre_reshape_output: name of the MatMulNBits output edge (the input of the
            generated Reshape).
        final_output: name of the final edge produced by the generated Reshape
            (must match the original MatMul output edge).

    Returns:
        List of nodes to append to the consumer's ``output_nodes`` after the
        MatMulNBits node. Initializers are appended to ``target_graph`` in place.
    rˆ   Ú_Ú_zeroÚ_oneÚ_twoÚ_one_vecÚ_rank_bÚ_n_vecÚ	_zero_vecÚ
_one_valuer   ©r…   ©r°   r   r  Ú_a_shapeÚ_a_shape_of_shapeÚ_a_rankÚ_a_rank_effÚ
_extra_rawÚ_extra_countÚ_extra_count_vecÚ_extra_onesÚ
_a_rank_m1Ú_a_rank_m1_vecÚ_a_prefix_shapeÚ_target_shapeÚShapeÚ_shape_aÚ_shape_of_a_shaper"   Ú_gather_rankÚMaxÚ_max_rank_effÚSubÚ_subr/  ÚReshapeÚ_reshape_extraÚConstantOfShapeÚ_const_ones)r°   Ú	data_typeÚdimsÚvals)r°   ÚvalueÚ_sub_oneÚ_reshape_rank_m1ÚSliceÚ_slice_a_prefixÚConcatÚ_concat_target)r°   rS   Ú_reshape_out)rÊ   ri  rj  ra  rb  rg  râ   Úarrayr�   rm  rn  Úmake_tensorr   ÚINT64)rT  rU  rV  rW  rX  rY  r‡  Ún_dimrÿ   Ú	init_zeroÚinit_oneÚinit_twoÚinit_one_vecÚinit_rank_bÚ
init_n_vecÚinit_zero_vecÚinit_one_value_templateÚa_shapeÚa_shape_of_shapeÚa_rankÚ
a_rank_effÚ	extra_rawÚextra_countÚextra_count_vecÚ
extra_onesÚa_rank_minus_oneÚa_rank_minus_one_vecÚa_prefix_shapeÚtarget_shapeÚnodess                                 r&   ro  ro  F  sâ  € õH Ð&Ñ'Ô'€KØ˜RÔ €Eð 	�cÑ˜LÑ(€AØ�G‘€IØ�6‰z€HØ�6‰z€HØ�z‘>€LØ�i‘-€KØ�X‘€JØ˜‘O€MØ ,Ñ.ÐàÔ×#Ò#åÔ×(Ò(­¬°!½2¼8Ð)DÑ)DÔ)DÈ9Ð(ÑUÔUÝÔ×(Ò(­¬°!½2¼8Ð)DÑ)DÔ)DÈ8Ð(ÑTÔTÝÔ×(Ò(­¬°!½2¼8Ð)DÑ)DÔ)DÈ8Ð(ÑTÔTÝÔ×(Ò(­¬°1°#½R¼XÐ)FÑ)FÔ)FÈ\Ð(ÑZÔZÝÔ×(Ò(­¬°+ÅRÄXÐ)NÑ)NÔ)NÐU`Ð(ÑaÔaÝÔ×(Ò(­¬°5°'ÅÄÐ)JÑ)JÔ)JÐQ[Ð(Ñ\Ô\ÝÔ×(Ò(­¬°1°#½R¼XÐ)FÑ)FÔ)FÈ]Ð(Ñ[Ô[ð	
ñ
ô 
ð 
ð �*‰n€GØÐ.Ñ.ÐØ�‰]€FØ�]Ñ"€JØ�LÑ €IØ�nÑ$€KØÐ,Ñ,€OØ�]Ñ"€JØ˜<Ñ'ÐØÐ/Ñ/ÐØÐ*Ñ*€NØ�Ñ&€Lõ 	Œ×Ò˜g¨ ~¸°yÀqÈ:Á~ÐÑVÔVõ
 	Œ×Ò˜g¨ yÐ3CÐ2DÈ1ÐObÑKbÐÑcÔcÝŒ×Ò˜hÐ)9¸9Ð(EÈÀxÐVWÐZhÑVhÐÑiÔiõ 	Œ×Ò˜e f¨hÐ%7¸*¸ÈAÐP_ÑL_ÐÑ`Ô`ÝŒ×Ò˜e k°:Ð%>ÀÀÐRSÐV\ÑR\ÐÑ]Ô]ÝŒ×Ò˜e i°Ð%;¸k¸]ÐQRÐU[ÑQ[ÐÑ\Ô\ÝŒ×Ò˜i¨+°|Ð)DÀÐFWÐ^_ÐbrÑ^rÐÑsÔsÝŒ×ÒØØÐØˆLØ�]Ñ"Ý”+×)Ò)Ø,Ý%Ô+Ø�SØ�Sð	 *ñ ô ð 	ñ 	
ô 	
õ 	Œ×Ò˜e f¨hÐ%7Ð:JÐ9KÐRSÐV`ÑR`ÐÑaÔaÝŒ×ÒØÐ(¨,Ð7Ð:NÐ9OÐVWÐZlÑVlð 	ñ 	
ô 	
õ 	Œ×ÒØØ�mÐ%9¸=ÐIØÐØÐ&Ñ&ð	 	ñ 	
ô 	
õ 	Œ×ÒØØ˜¨Ð4ØˆNØÐ%Ñ%Øð 	ñ 	
ô 	
õ 	Œ×ÒØØ Ð.ØˆNØ�^Ñ#ð	 	ñ 	
ô 	
ðY2€Eðf €Lr(   c                ój  — ||z  }| dz	  dz  }| dz  }t          j        |t           j        ¬¦  «        }||dd d…<   ||dd d…<   |                     ||f¦  «        }|j        }|                     d¦  «        }	|	dd d…         dz  dz  |	dd d…         dz  z  } |                      t           j        ¦  «        S )	NrN   é   r¨  r   r  r   rˆ   éð   )râ   Úemptyrf  r#  r  Úastype)
Úpackedr}  r~  ÚtotalÚhighÚlowÚ	int4_valsÚint4_matrixÚint4_matrix_transposedÚflats
             r&   Útranspose_packed_int4_matrixró  É  sÌ   € à�4‰K€EØ�a‰K˜4Ñ€DØ
�4‰-€CÝ”˜¥b¤hÐ/Ñ/Ô/€IØ€Iˆaˆd�ˆd�OØ€Iˆaˆd�ˆd�OØ×#Ò# T¨4 LÑ1Ô1€Kð )œ]Ðð "×)Ò)¨"Ñ-Ô-€DØ�A�D�q�DŒz˜Q‰ $Ñ&¨4°°°1°¬:¸Ñ+<Ñ=€FØ�=Š=�œÑ"Ô"Ð"r(   c                  ó–   — e Zd Zdd„Zd d„Zd!d„Zed"d„¦   «         Zed#d„¦   «         Zed$d„¦   «         Z	ed%d„¦   «         Z
d!d„Zd!d„ZdS )&ÚDefaultWeightOnlyQuantizerr�   rU   c                ó   — || _         d S rá   rê   rë   s     r&   r'   z#DefaultWeightOnlyQuantizer.__init__Ý  s   € ØˆŒˆˆr(   Ú
fp32weightúnpt.ArrayLiker7  ú)tuple[np.ndarray, np.ndarray, np.ndarray]c           
     óH  — | j         j        }d|z  }t          |j        ¦  «        dk    rt	          d¦  «        ‚|j        \  }}| j         j        }||z   dz
  |z  }| j         j        t          j        k    rû||z   dz
  |z  }||z  }	|	|z
  }
|
dk    rt          j
        |d|
fdfd¦  «        }t          j        |||fd¬	¦  «        }t          j        |||z   dz
  |z  fd¬	¦  «        }t          j        ||f|j        ¬	¦  «        }|dk    r!t          |||||||| j         j        ¦  «         nø|dk    r!t          |||||||| j         j        ¦  «         nÑt!          |||||||| j         j        ¦  «         n°| j         j        r|n| j         j        }||z   dz
  |z  }|d
k    s
J d¦   «         ‚t          j        ||z  dz   dz  d¬	¦  «        }t          j        ||z  dz   dz  d¬	¦  «        }t          j        ||f|j        ¬	¦  «        }t%          |||||||| j         j        ¦  «         |||fS )z54b/8b quantize fp32 weight to int4 using C++ kernels.r  r  z9Current int4 block quantization only supports 2D tensors!r   r   )r   r   r   rf  r¨  rN   z+QDQ format only support 4 bits quantization)r�   rR   rÊ   r”   rÇ   rH   r   r   r3   râ   r"  r›   r…   r   rW   r
   r	   r[   r   )r%   r÷  ÚqbitsÚkpackr}  r~  rH   r€  r  Úpadded_rowsr-  rë  Ú
zero_pointr{  s                 r&   Úqbits_block_quantz,DefaultWeightOnlyQuantizer.qbits_block_quantà  s¥  € ð ”Ô ˆØ�U‘
ˆÝˆzÔÑ Ô  AÒ%Ð%ÝÐXÑYÔYÐYØÔ%‰
ˆˆdà”[Ô+ˆ
Ø˜:Ñ%¨Ñ)¨jÑ8ˆàŒ;Ô#¥{Ô'<Ò<Ð<Ø# eÑ+¨aÑ/°EÑ9ˆIØ" ZÑ/ˆKØ! DÑ(ˆGØ˜Š{ˆ{ÝœV J°!°W°¸vÐ0FÈ
ÑSÔS�
õ ”X˜t X¨yÐ9ÀÐIÑIÔIˆFÝœ 4¨8°eÑ+;¸aÑ+?ÀEÑ*IÐ"KÐSZÐ[Ñ[Ô[ˆJÝ”X˜t XÐ.°jÔ6FÐGÑGÔGˆFØ˜ŠzˆzÝ%Ø˜J¨°
¸JÈÈdÐTXÔT_ÔTlñô ð ð ð ˜!’�Ý%Ø˜J¨°
¸JÈÈdÐTXÔT_ÔTlñô ð ð õ &Ø˜J¨°
¸JÈÈdÐTXÔT_ÔTlñô ð ð ð
 "&¤Ô!CÐ_˜˜ÈÌÔI_ˆJØ˜zÑ)¨AÑ-°*Ñ<ˆHà˜A’:�:�:ÐL‘:”:�:Ý”X˜t d™{¨Q™°1Ñ4¸GÐDÑDÔDˆFÝœ 4¨(¡?°QÑ#6¸1Ñ"<ÀGÐLÑLÔLˆJÝ”X˜x¨Ð.°jÔ6FÐGÑGÔGˆFÝ%Ø˜
 F¨J¸
ÀDÈ$ÐPTÔP[ÔPhñô ð ð ˜ 
Ð+Ð+r(   r4  r   r5  r6  r8  c                ó  — | j         j        }|dk    r%| j         j        rt          j        nt          j        }n$| j         j        rt          j        nt          j        }|j        d         }t          ||¦  «        \  }}|€t                               d¦  «         |gS t          j        |¦  «                             ¦   «         }|j        }	t!          |	¦  «        dk    r¡t!          |	¦  «        dk     rt                               d¦  «         |gS |	dd…         }
t#          d„ |
D ¦   «         ¦  «        r+t                               d	t%          |
¦  «        ¦  «         |gS |                     |	d         |	d
         ¦  «        }nd}	|j        dk    }|r|                     t,          j        ¦  «        }|                      |¦  «        \  }}}|r|                     t2          j        ¦  «        }| j         j        t8          j        k    rzt          j                             t          j         ||j!        d|› �z   ¬¦  «        ¦  «        }t          j                             t          j         ||j!        dz   ¬¦  «        ¦  «        }n€tD          j#         $                    |j!        d|› �z   ||j        | %                    ¦   «         d¦  «        }t          j                             t          j         ||j!        dz   ¬¦  «        ¦  «        }| j         j        t8          j&        k    o| j         j'        o| j         j        }|r¯|j        \  }}tQ          |||¦  «        }|                     |df¦  «        }tD          j#         $                    |j!        d|› �z   |||g| %                    ¦   «         d¦  «        }t          j                             t          j         ||j!        dz   ¬¦  «        ¦  «        }|j        D ])}|j!        |k    r|j         )                    |¦  «          nŒ*|j*         +                    ||g¦  «         g }| j         j        t8          j        k    �rÊ|j        d         |j!        |j!        g}| j         j        s]tD          j,         -                    ||j!        dz   ¦  «        }| .                    |j!        ¦  «         |j*         +                    |g¦  «         i }|j        \  }}||d<   ||d<   ||d<   | j         j/        |d<   | j         j0        r| j         j0        |d<   tc          |j        d         |¦  «        }|	�t!          |	¦  «        nd}|	duo	|du p||k     }|s|j2        d         n|j2        d         dz   }tE          j#        j3        	 d)||g|j!        r|j!        d|› �z   ndddœ|¤Ž}| .                    |¦  «         |r]| +                    ti          |j        d         |	||j!        r|j!        d|› �z   n|j!        d|› �z   ||j2        d         ¬¦  «        ¦  «         �nu|j!        |j!        g}|j!        dz   g}|d         g} |d         d z   g}!|j        d         |r|!d         n|d         g}"tc          |j        d         |¦  «        }|	�t!          |	¦  «        nd}|	duo	|du p||k     }|s|j2        d         n|j2        d         dz   }#|#g}$| j         j        swtD          j#         $                    |j!        d!z   ||j        | %                    ¦   «         d¦  «        }| .                    |j!        ¦  «         |j*         +                    |g¦  «         |j        \  }}|rdnd| j         j'        r|n| j         j/        d"œ}%tE          j#        j3        	 d*|||j!        r|j!        d|› �z   ndd$œ|%¤Ž}&tD          j#         3                    d%|"|$|j!        r|j!        d&|› �z   nd¬$¦  «        }'|r>tD          j#         3                    d'| |!ddg¬(¦  «        }(| +                    |&|(|'g¦  «         n| +                    |&|'g¦  «         |r]| +                    ti          |j        d         |	||j!        r|j!        d|› �z   n|j!        d|› �z   |#|j2        d         ¬¦  «        ¦  «         |S )+a  
        Quantize weight B of MatMul node to int4 or int8.
        Supports 2D constant matrix, and N-D constant matrices whose leading dimensions are all 1
        (which are squeezed to 2D before quantization). Axis 0 blockwise quantization only.
        r  r   Nr=  r  r>  r?  c              3  ó"   K  — | ]
}|d k    V — ŒdS rA  r,   rB  s     r&   rD  z=DefaultWeightOnlyQuantizer.quantize_matmul.<locals>.<genexpr>*  rE  r(   rF  rˆ   Úbfloat16rG  r©  rH  Ú_DQ_QTÚ
_DQ_scalesr   rI  rJ  rK  rR   rH   rY   rL  rM  rN  rO  rP  rS  Ú_outputÚ_transposedÚ_DQ_zero_points)rS   rH   ÚDequantizeLinear)r    rQ  r°   r!   Ú	_matmul_QÚ	Transpose)r    rQ  Úpermr[  )r  )5r�   rR   rW   r   ÚINT8ÚUINT8ÚINT4ÚUINT4r_  r`  r]  r^  ÚirÚ
from_protor¾   r”   rÊ   rd  r‹   r#  r…   rê  râ   r“   rÿ  Ú	ml_dtypesr  r   r   r3   ÚserdeÚserialize_tensorÚTensorr°   ra  rm  rÎ  Útobytesr~   r[   ró  rh  ri  rj  rb  rg  rÍ   rH   rY   rk  rl  rn  ro  ))r%   r4  r5  rR   Úqtyperp  Úb_tensorÚb_graphÚ	b_ndarrayrU  rt  r  rë  r{  r|  r�  r‚  Úqdq_opt_for_intel_npu_enabledr}  r~  r_  r‹  rƒ  r„  r…  r†  r‡  rˆ  Ú
qop_outputÚmatmul_qbit_nodeÚdq_input_namesÚdq_output_namesÚtp_input_namesÚtp_output_namesÚmatmul_input_namesÚqdq_matmul_outÚmatmul_output_namesÚ	dq_kwargsÚdq_nodeÚmatmul_nodeÚtp_nodes)                                            r&   Úquantize_matmulz*DefaultWeightOnlyQuantizer.quantize_matmul  s>	  € ð Œ{ÔˆØ�1Š9ˆ9Ø(,¬Ô(@ÐW•KÔ$Ð$ÅkÔFWˆEˆEà(,¬Ô(@ÐW•KÔ$Ð$ÅkÔFWˆEØ”*˜Q”-ˆÝ+¨G°[ÑAÔAÑˆ�'ØÐÝ�KŠKÐLÑMÔMÐMØ�6ˆMå”M (Ñ+Ô+×1Ò1Ñ3Ô3ˆ	Ø$œ?ÐÝÐÑ Ô  AÒ%Ð%ÝÐ#Ñ$Ô$ qÒ(Ð(Ý—’ÐZÑ[Ô[Ð[Ø�v�Ø& s¨ sÔ+ˆGÝÐ+Ð+ 7Ð+Ñ+Ô+Ñ+Ô+ð Ý—’ð(å˜‘M”Mñô ð ð
 �v�à!×)Ò)Ð*:¸2Ô*>Ð@PÐQSÔ@TÑUÔUˆIˆIà#Ðà”? jÒ0ˆØð 	5Ø!×(Ò(­¬Ñ4Ô4ˆIà&*×&<Ò&<¸YÑ&GÔ&GÑ#ˆ�˜Øð 	7Ø—]’]¥9Ô#5Ñ6Ô6ˆFàŒ;Ô#¥{Ô'<Ò<Ð<Ý”h×/Ò/µ´	¸&ÀxÄ}ÐWbÐ\`ÐWbÐWbÑGbÐ0cÑ0cÔ0cÑdÔdˆGÝœH×5Ò5µb´iÀÈXÌ]Ð]fÑMfÐ6gÑ6gÔ6gÑhÔhˆMˆMå”k×-Ò-Ø” ¨  Ñ.°°y´ÈÏÊÑHXÔHXÐZ^ñô ˆGõ œH×5Ò5µb´iÀÈXÌ]Ð]iÑMiÐ6jÑ6jÔ6jÑkÔkˆMð ŒKÔ$­¬Ò7ð )Ø”Ô2ð)à”Ô(ð 	&ð
 )ð 	lØ"œ‰JˆD�$Ý1°&¸$ÀÑEÔEˆFØ—^’^ T¨1 IÑ.Ô.ˆFÝ”k×-Ò-Ø” ¨  Ñ.°¸¸d°|ÀVÇ^Â^ÑEUÔEUÐW[ñô ˆGõ œH×5Ò5µb´iÀÈXÌ]Ð]iÑMiÐ6jÑ6jÔ6jÑkÔkˆMà”]ð 	ð 	ˆEØŒz˜WÒ$Ð$Ø”×$Ò$ UÑ+Ô+Ð+Ø�ð %ð 	Ô×"Ò" G¨]Ð#;Ñ<Ô<Ð<àˆàŒ;Ô#¥{Ô'<Ò<Ñ<Øœ: aœ=¨'¬,¸Ô8JÐKˆKØ”;Ô+ð 8Ý Ô-×8Ò8¸ÀhÄmÐVdÑFdÑeÔe�	Ø×"Ò" 9¤>Ñ2Ô2Ð2ØÔ#×*Ò*¨I¨;Ñ7Ô7Ð7ØˆFØ"œ‰JˆD�$ØˆF�3‰KØˆF�3‰KØ!ˆF�6‰NØ#'¤;Ô#9ˆF�<Ñ ð Œ{Ô)ð FØ+/¬;Ô+E�Ð'Ñ(õ -¨T¬Z¸¬]¸KÑHÔHˆMØ3CÐ3O�#Ð.Ñ/Ô/Ð/ÐUVˆKØ,°DÐ8Ðs¸mÈtÐ>SÐ>rÐWdÐgrÒWrˆMØ/<Ða˜œ Qœ˜À$Ä+ÈaÄ.ÐSaÑBaˆJÝ#œ{Ô4Øð à"Ø#˜Ø04´	ÐA�T”Y  d  Ñ,Ð,¸rØ&ð ð  ð ð ð  Ðð ×ÒÐ 0Ñ1Ô1Ð1Øð 
Ø×#Ò#Ý/Ø%)¤Z°¤]Ø)9Ø%,ØAEÄÐ$m T¤Y°°d°°Ñ%<Ð%<ÐQYÔQ^ÐalÐfjÐalÐalÑQlØ+5Ø%)¤[°¤^ðñ ô ñ	ô 	ð 	ùð &œl¨MÔ,>Ð?ˆNØ&œ|¨iÑ7Ð8ˆOØ-¨aÔ0Ð1ˆNØ.¨qÔ1°MÑAÐBˆOà”
˜1”Ø&CÐ[� Ô"Ð"ÈÐYZÔI[ð"Ðõ -¨T¬Z¸¬]¸KÑHÔHˆMØ3CÐ3O�#Ð.Ñ/Ô/Ð/ÐUVˆKØ,°DÐ8Ðs¸mÈtÐ>SÐ>rÐWdÐgrÒWrˆMØ3@Ðe˜Tœ[¨œ^˜^ÀdÄkÐRSÄnÐWeÑFeˆNØ#1Ð"2ÐØ”;Ô+ð 8Ý œK×3Ò3Ø”MÐ$5Ñ5°u¸f¼lÈK×L_ÒL_ÑLaÔLaÐcgñô �	ð ×%Ò% i¤nÑ5Ô5Ð5ØÔ#×*Ò*¨I¨;Ñ7Ô7Ð7Ø"œ‰JˆD�$à:ÐA˜˜ÀØ&*¤kÔ&HÐd˜d˜dÈdÌkÔNdðð ˆIõ ”kÔ+Ø"ðà%Ø'Ø37´9ÐD�T”Y ¨  Ñ/Ð/À"ð	ð ð
 ðð ˆGõ œ+×/Ò/ØØ)Ø+Ø7;´yÐH�T”YÐ!3¨TÐ!3Ð!3Ñ3Ð3Àbð	 0ñ ô ˆKð -ð 	<Ýœ+×/Ò/ØØ)Ø+Ø˜Q˜ð	 0ñ ô �ð ×#Ò# W¨g°{Ð$CÑDÔDÐDÐDà×#Ò# W¨kÐ$:Ñ;Ô;Ð;Øð 
Ø×#Ò#Ý/Ø%)¤Z°¤]Ø)9Ø%,ØDHÄIÐ$s T¤Y°¸°°Ñ%?Ð%?ÐT\ÔTaÐdrÐlpÐdrÐdrÑTrØ+9Ø%)¤[°¤^ðñ ô ñ	ô 	ð 	ð Ðr(   rÝ   ú
np.ndarrayútuple[np.ndarray, np.ndarray]c                ó¬  — t          j        | dd¬¦  «        }t          j        | dd¬¦  «        }t          j        t          j        |¦  «        t          j        |¦  «        k    ||¦  «        }|dz  }t          j        |dk    d| |z  ¦  «                             ¦   «                              dd¦  «                             t           j        ¦  «        }||fS )Nr   T©rS   Úkeepdimsg       Àr   iøÿÿÿé   )	râ   r!  r$  Úwhererü   r  Úcliprê  Úint8)rÝ   Úmax_valÚmin_valÚabs_maxr	  Úquantized_slices         r&   Úquant_slice_symmetricz0DefaultWeightOnlyQuantizer.quant_slice_symmetricË  s´   € å”&˜ A°Ð5Ñ5Ô5ˆÝ”&˜ A°Ð5Ñ5Ô5ˆÝ”(�2œ6 '™?œ?­R¬V°G©_¬_Ò<¸gÀwÑOÔOˆà˜$‘ˆÝœ( 5¨A¢:¨q°$¸±,Ñ?Ô?×EÒEÑGÔG×LÒLÈRÐQRÑSÔS×ZÒZÕ[]Ô[bÑcÔcˆà Ð%Ð%r(   c                óL  — t          j        |                      dd¬¦  «        d¦  «        }t          j        |                      dd¬¦  «        d¦  «        }||z
  dz  }t          j        |dk    d| |z  ¦  «                             ¦   «                              dd¦  «                             t           j	        ¦  «        }t          j        |dk    d| |z  |z   ¦  «                             ¦   «                              dd¦  «                             t           j	        ¦  «        }|||fS )Nr   Tr-  r   g      .@r  rç  )
râ   Úminimumr$  Úmaximumr!  r0  r  r1  rê  rf  )rÝ   r4  r3  r	  rþ  r6  s         r&   Úquant_slice_asymmetricz1DefaultWeightOnlyQuantizer.quant_slice_asymmetricÖ  sü   € å”*˜TŸXšX¨1°t˜XÑ<Ô<¸aÑ@Ô@ˆÝ”*˜TŸXšX¨1°t˜XÑ<Ô<¸aÑ@Ô@ˆà˜7Ñ" dÑ*ˆÝ”X˜e qšj¨!¨g¨X¸Ñ-=Ñ>Ô>×DÒDÑFÔF×KÒKÈAÈrÑRÔR×YÒYÕZ\ÔZbÑcÔcˆ
Ýœ( 5¨A¢:¨q°$¸±,ÀÑ2KÑLÔL×RÒRÑTÔT×YÒYÐZ[Ð]_Ñ`Ô`×gÒgÕhjÔhpÑqÔqˆà  zÐ1Ð1r(   c                óì   — |                       d¦  «        }t          |¦  «        dz  dk    rt          j        |d¦  «        }|ddd…         dz  |ddd…         dz  dz  z  }|                     d¦  «        S )	z2Pack int8 data to int4 and store in uint8 ndarray.rˆ   r  r   Nrç  r   rN   rf  )r#  rÊ   râ   rÍ   rê  )rÝ   Ú	data_flatÚquant_data_int4s      r&   Úpack_int8_to_int4z,DefaultWeightOnlyQuantizer.pack_int8_to_int4á  s}   € ð —L’L Ñ$Ô$ˆ	Ýˆy‰>Œ>˜AÑ Ò"Ð"Ýœ	 )¨QÑ/Ô/ˆIØ$ S S q Sœ>¨CÑ/°Y¸q¸tÀ!¸t´_ÀsÑ5JÈqÑ4PÑQˆà×%Ò% gÑ.Ô.Ð.r(   Úquantize_axisrV   rH   rW   rX   ú0tuple[np.ndarray, np.ndarray, np.ndarray | None]c                óÔ  — d}| j         |         }d}t          | j         ¦  «        D ]\  }}||k     r||z  }Œ||k    r||z  }Œ||z   dz
  |z  }	t          | j         ¦  «        }
|	|
|<   |                      |||f¦  «        }t	          j        ||	|f| j        ¬¦  «        }|rt	          j        |||fd¬¦  «        }n2t	          j        |||fd¬¦  «        }t	          j        ||	|fd¬¦  «        }t          d||¦  «        D ]›}t          ||z   |¦  «        }|dd…||…dd…f         }|rt           
                    |¦  «        \  }}nt                               |¦  «        \  }}}||dd…||…dd…f<   ||z  }||dd…||dz   …dd…f<   |s||dd…||dz   …dd…f<   Œœt                               |¦  «        }d}|st                               |¦  «        }|                     |
¦  «        }|||fS )zXQuantize ndarray data to int4 using numpy, return (quantized data, scales, zero points).r   r¨  r2  rf  r   N)r”   rÌ   r‹   r#  râ   r›   r…   r™   r$  rõ  r7  r;  r?  )rÝ   r@  rH   rW   ÚmÚkÚnr¦   rš  r€  Úscales_shapeÚdata_reshaper{  Úquant_data_int8Úzero_point_int8Úend_idxÚsliceÚquantized_slice_int8Úscale_sliceÚzero_point_slice_int8r  r>  Úzero_point_int4s                          r&   Úquantize_ndarrayz+DefaultWeightOnlyQuantizer.quantize_ndarrayë  sw  € ð ˆØŒJ�}Ô%ˆØˆÝ ¤
Ñ+Ô+ð 	ð 	‰FˆAˆsØ�=Ò Ð Ø�S‘��Ø�]Ò"Ð"Ø�S‘�øà˜
‘N QÑ&¨:Ñ5ˆÝ˜DœJÑ'Ô'ˆØ&.ˆ�]Ñ#à—|’| Q¨¨1 IÑ.Ô.ˆÝ”˜1˜h¨Ð*°$´*Ð=Ñ=Ô=ˆØð 	HÝ œh¨¨1¨a y¸Ð?Ñ?Ô?ˆOˆOå œh¨¨1¨a y¸Ð@Ñ@Ô@ˆOÝ œh¨¨8°QÐ'7¸wÐGÑGÔGˆOõ �q˜!˜ZÑ(Ô(ð 	Kð 	KˆAÝ˜!˜j™.¨!Ñ,Ô,ˆGØ     A g I¨q¨q¨q Ô1ˆEàð Ý4N×4dÒ4dÐejÑ4kÔ4kÑ1Ð$ k kõ /×EÒEÀeÑLÔLñ IÐ$ kÐ3Hð 0DˆO˜A˜A˜A˜q ˜y¨!¨!¨!˜OÑ,Ø�Z‘ˆAØ(3ˆF�1�1�1�a˜1˜q™5�k 1 1 1Ð$Ñ%Øð KØ5J�    1¨¨A© ;°°°Ð 1Ñ2øõ 5×FÒFÀÑWÔWˆØˆØð 	\Ý8×JÒJÈ?Ñ[Ô[ˆOØ—’ Ñ-Ô-ˆØ ¨Ð7Ð7r(   c                ó$  — | j         j        t          j        k    s
J d¦   «         ‚| j         j        rt
          j        nt
          j        }|j        d         }t          ||¦  «        \  }}|€t                               d¦  «         |gS t          j                             |¦  «        }t          |j        ¦  «        }| j         j                             dd¦  «        }	| j         j        }
|	|k     r|	| k    s
J d¦   «         ‚|
dk    r|
dz
  |
z  dk    s
J d	¦   «         ‚|	|z   |z  }	|                      ||	|
| j         j        ¦  «        \  }}}|j        D ])}|j        |k    r|j                             |¦  «          nŒ*t          j                             |j        d
z   ||j        |                     ¦   «         d¦  «        }t          j                             ||j        dz   ¦  «        }|j        |j        d         |j        g}|j                             ||g¦  «         | j         j        swt          j                             |j        dz   ||j        |                     ¦   «         d¦  «        }|                     |j        ¦  «         |j                             |g¦  «         	 t          j                             |d¦  «        }n# t>          $ r d}Y nw xY w||	|
dœ}t          j        j         	 d||j!        d         g|j        r
|j        d
z   ndddœ|¤Ž}|gS )z,Quantize weight data of Gather node to int4.z0Gather only supports QOperator format currently.r   Nz4Gather doesn't have const weight. Skip quantization.r"   r   z&Invalid quantize axis for Gather node.é   z#Invalid block size for Gather node.Ú_Q4TrH  rI  rS   )Úgather_axisr@  rH   ÚGatherBlockQuantizedrN  rO  rP  )rU  )"r�   r   r   r3   rW   r   r  r  r_  r`  r]  r^  ra  rb  rc  rÊ   r”   r   ÚgetrH   rP  r°   rh  rm  rÎ  r  rg  ri  rj  rÍ   Úget_node_attr_valuerÇ   rn  rl  )r%   r4  r5  r  Údata_argÚdata_tensorprotoÚdata_graphprotoÚdata_ndarrayÚ	data_rankr@  rH   Úquantized_datar{  r|  r_  Úquantized_data_tensorprotoÚscales_tensorprotorƒ  Úzp_tensorprotorT  r…  Úgather_q4_nodes                         r&   Úquantize_gatherz*DefaultWeightOnlyQuantizer.quantize_gather#  sc  € àŒ{Ô'­;Ô+@Ò@Ð@Ð@ÐBtÑ@Ô@Ð@à$(¤KÔ$<ÐS•Ô Ð Å+ÔBSˆØ”:˜a”=ˆÝ,;¸HÀkÑ,RÔ,RÑ)Ð˜/ØÐ#Ý�KŠKÐNÑOÔOÐOØ�6ˆMåÔ(×1Ò1Ð2BÑCÔCˆÝ˜Ô*Ñ+Ô+ˆ	ØœÔ.×2Ò2°8¸QÑ?Ô?ˆØ”[Ô+ˆ
à˜yÒ(Ð(¨]¸y¸jÒ-HÐ-HÐ-HÐJrÑ-HÔ-HÐHØ˜RÒÐ j°1¡n¸
Ñ%BÀaÒ%GÐ%GÐ%GÐJoÑ%GÔ%GÐHà&¨Ñ2°iÑ?ˆØ.2×.CÒ.CØ˜-¨°T´[Ô5Mñ/
ô /
Ñ+ˆ˜ ð %Ô*ð 	ð 	ˆEØŒz˜XÒ%Ð%ØÔ%×,Ò,¨UÑ3Ô3Ð3Ø�ð &õ &*¤[×%<Ò%<ØÔ! EÑ)¨5°,Ô2DÀn×F\ÒF\ÑF^ÔF^Ð`dñ&
ô &
Ð"õ "Ô.×9Ò9¸&ÐBRÔBWÐZcÑBcÑdÔdÐØ1Ô6¸¼
À1¼ÐGYÔG^Ð_ˆØÔ#×*Ò*Ð,FÐHZÐ+[Ñ\Ô\Ð\ØŒ{Ô'ð 	AÝ!œ[×4Ò4Ø Ô%¨Ñ6¸¸v¼|È[×M`ÒM`ÑMbÔMbÐdhñô ˆNð ×Ò˜~Ô2Ñ3Ô3Ð3ØÔ'×.Ò.°Ð/?Ñ@Ô@Ð@ð	Ýœ+×9Ò9¸$ÀÑGÔGˆKˆKøÝð 	ð 	ð 	ØˆKˆKˆKð	øøøð 'Ø*Ø$ð
ð 
ˆõ œÔ.Ø"ð
àØ”[ ”^Ð$Ø&*¤iÐ7�”˜UÑ"Ð"°RØ"ð
ð 
ð ð
ð 
ˆð ÐÐs   Ê J> Ê>KËKc                ól  — t                                d|j        › d�¦  «         | j        j        }|j        dk    rT|dk    r7| j        j        t          j        k    rt            	                    d¦  «         |gS |  
                    ||¦  «        }nu|j        dk    rD| j        j        dk    rt            	                    d¦  «         |gS |                      ||¦  «        }n&t            	                    d	|j        › d
�¦  «         |gS t                                d|j        › d| j        j        › d�¦  «         |S )r:  r;  r<  r!   r  z>MatMul only supports QOperator format for 8 bits quantization.r"   rN   z)Gather only supports 4 bits quantization.zUnsupported operator z1 for weight only quantization. Skip quantization.rZ  z with z	 bits ...)r]  r^  r°   r�   rR   r\  r   r   r~   Úerrorr)  rb  )r%   r4  r5  rR   Úresultss        r&   rŒ  z#DefaultWeightOnlyQuantizer.quantizec  s-  € õ 	�ŠÐ8¨¬Ð8Ð8Ð8Ñ9Ô9Ð9àŒ{ÔˆØŒ<˜8Ò#Ð#Ø�qŠyˆy˜Tœ[Ô5½¼ÒHÐHÝ—’Ð]Ñ^Ô^Ð^Ø�v�Ø×*Ò*¨4°Ñ=Ô=ˆGˆGØŒ\˜XÒ%Ð%ØŒ{Ô 1Ò$Ð$Ý—’ÐHÑIÔIÐIØ�v�à×*Ò*¨4°Ñ=Ô=ˆGˆGå�LŠLÐp°´ÐpÐpÐpÑqÔqÐqØ�6ˆMå�ŠÐ\°´	Ð\Ð\ÀÄÔAQÐ\Ð\Ð\Ñ]Ô]Ð]Øˆr(   N)r�   rU   )r÷  rø  r7  rù  r�  )rÝ   r*  r7  r+  )rÝ   r*  r7  rù  )rÝ   r*  r7  r*  )
rÝ   r*  r@  rV   rH   rV   rW   rX   r7  rA  )r)   r*   r+   r'   rÿ  r)  rŽ  r7  r;  r?  rP  rb  rŒ  r,   r(   r&   rõ  rõ  Ü  s   € € € € € ðð ð ð ð0,ð 0,ð 0,ð 0,ðdwð wð wð wðr ð&ð &ð &ñ „\ð&ð ð2ð 2ð 2ñ „\ð2ð ð/ð /ð /ñ „\ð/ð ð58ð 58ð 58ñ „\ð58ðn> ð > ð > ð > ð@ð ð ð ð ð r(   rõ  c                  ó   — e Zd Zd
d„Zdd„Zd	S )ÚNVAWQWeightOnlyQuantizerr�   r`   c                ó   — || _         d S rá   rê   rë   s     r&   r'   z!NVAWQWeightOnlyQuantizer.__init__†  rì   r(   ÚmodelúModelProto | strr7  r   c                ó   — 	 ddl m} n-# t          $ r  t          d¦  «         t          d¦  «        d‚w xY wt                               d¦  «         | j        j        } ||| j        j        |¬¦  «        }t                               d¦  «         |S )	zé
        Perform nvidia_awq quantization using ModelOpt's int4 quantize function.

        Args:
            model (ModelProto): The ONNX model to quantize.

        Returns:
            ModelProto: The quantized ONNX model.
        r   )rŒ  zlPlease ensure that the 'modelopt' package is installed. Please install it using pip install nvidia_modelopt.zXmodelopt is not installed. Please install it using pip install nvidia_modelopt. Exiting.Nz#Starting nvidia_awq quantization...)r�   rC   ú"Completed nvidia_awq quantization.)	Úmodelopt.onnx.quantization.int4rŒ  rz   r{   r]  r^  r�   rC   r�   )r%   ri  Úquantize_int4rƒ   Úquantized_models        r&   Úquantize_awqz%NVAWQWeightOnlyQuantizer.quantize_awqŒ  sÈ   € ð	ØQÐQÐQÐQÐQÐQÐQøÝð 	ð 	ð 	ÝØ~ñô ð õ Øjñô àðð		øøøõ 	�ŠÐ9Ñ:Ô:Ð:ð ”{Ô:ˆð (˜-ØØ#œ{Ô=Ø$0ð
ñ 
ô 
ˆõ 	�ŠÐ8Ñ9Ô9Ð9ØÐs   ‚	 ‰*3N)r�   r`   )ri  rj  r7  r   )r)   r*   r+   r'   rp  r,   r(   r&   rg  rg  …  s<   € € € € € ðð ð ð ð!ð !ð !ð !ð !ð !r(   rg  c                  óV   — e Zd ZdZddddddej        ddddfdd„Zd d„Zd„ Zd„ Z	d„ Z
dS )!ÚMatMulNBitsQuantizera¼  
    Target node:        QOperator node:            QDQ nodes:
    MatMul              MatMulNBits                DeQuantizeLinear -> MatMul
    Gather              GatherBlockQuantized       Gather, Gather, Gather (optional) -> DequantizeLinear

    Perform 2/4/8 bits quantization of constant weights for target nodes.
    If algo_config.quant_format is QOperator:
      - nodes are replaced by the corresponding QOperator nodes.
      - quantized weights are stored in the contrib ops.
    If algo_config.quant_format is QDQ:
      - the quantized weight is stored in a standard onnx node. For MatMul, it is DequantizeLinear. For Gather,
        it is the three Gathers, one for quantized data, one for scales and one for optional zero points.
      - The nodes are replaced by the corresponding QDQ nodes.
      - currently Gather is not supported in QDQ because Gather does not support int4 yet.
    Note:
      - for quantized gather, the memory usage of "DequantizeLinear + Gather" is the same as the original Gather
        during runtime. Therefor it is not recommended.
      - when a node is in nodes_to_exclude, and the node configuration in algo_config.customized_weight_config will be ignored.
    rN   rB   FNri  rj  rR   rV   rH   rW   rX   rY   rZ   Únodes_to_includeúlist[str] | Noner   r   r   r   r[   Úalgo_configúWeightOnlyQuantConfig | Nonec           
     óú  — |€g }t          |t          ¦  «        r!t          t          j        |¦  «        ¦  «        nt          |¦  «        | _        t          |t          ¦  «        r|nd | _        || _        || _        || _	        || _
        t          |¦  «        | _        |rt          |¦  «        nd | _        d | _        |€t          |||||	|
||¬¦  «        }|| _        t#          | j        d¦  «        r| j        j        dv s
J d¦   «         ‚|j        dk    rt'          | j        ¦  «        | _        d S |j        dk    rt)          | j        ¦  «        | _        d S |j        dk    rt+          | j        ¦  «        | _        d S d S )N)rH   rW   rY   r   r   r   rR   r[   rR   r  z(Only support 2, 4 or 8 bits quantizationrP   r]   ri   )rŠ   r   r   ra  Úloadri  Ú
model_pathrR   rH   rW   rY   r#   Únodes_to_excluders  Únode_quantizerrU   ru  Úhasattrr   rè   rõ  rg  )r%   ri  rR   rH   rW   rY   rz  rs  r   r   r   r[   ru  s                r&   r'   zMatMulNBitsQuantizer.__init__Å  s¢  € ð Ð#Ø!ÐÝ4>¸uÅcÑ4JÔ4JÐ`•Y�tœy¨Ñ/Ô/Ñ0Ô0Ð0ÕPYÐZ_ÑP`ÔP`ˆŒ
Ý#-¨eµSÑ#9Ô#9ÐC˜%˜%¸tˆŒØˆŒ	Ø$ˆŒØ(ˆÔØ,ˆÔÝ #Ð$4Ñ 5Ô 5ˆÔØ9IÐ S¥Ð$4Ñ 5Ô 5Ð 5ÈtˆÔØ"ˆÔàÐÝ6Ø%Ø)Ø-Ø)Ø%9Ø%ØØ'=ð	ñ 	ô 	ˆKð 'ˆÔÝ�4Ô# VÑ,Ô,ð 	bØÔ#Ô(¨IÐ5Ð5Ð5Ð7aÑ5Ô5Ð5àÔ  EÒ)Ð)Ý"8¸Ô9IÑ"JÔ"JˆDÔÐÐØÔ" iÒ/Ð/Ý"<¸TÔ=MÑ"NÔ"NˆDÔÐÐØÔ" lÒ2Ð2Ý":¸4Ô;KÑ"LÔ"LˆDÔÐÐð 3Ð2r(   r5  r6  c                óˆ  — g }|d         }|j         D �]ë}d„ |j        D ¦   «         }|�ri }|j        D ]å}|j        t          j        j        k    r7|                     |j        ¦  «         |j        |  	                    |¦  «        i}n}|j        t          j        j
        k    rTg }	|j        D ]@}
|                     |
¦  «         |	                     |  	                    |¦  «        g¦  «         ŒA|j        |	i}nt          |¦  «        }|                     |¦  «         Œæt          j        j        |j        |j        |j        fd|j        i|¤Ž}g }|j        | j        v r't*                               d|j        › d�¦  «         |g}nj| j        r|j        | j        v s|j        | j        j        v r| j                             ||¦  «        }n&t*                               d|j        › d�¦  «         |g}|                     |¦  «         �Œí|                     d¦  «         |j                              |¦  «         |                     ¦   «          |S )	Nrˆ   c                óz   — g | ]8}|j         t          j        j        k    s|j         t          j        j        k    ¯6|‘Œ9S r,   )r—  ra  ÚAttributeProtoÚGRAPHÚGRAPHS)r¿   Úattrs     r&   ú
<listcomp>z:MatMulNBitsQuantizer._process_subgraph.<locals>.<listcomp>ü  sG   € ð ð ð àØ”9¥Ô 3Ô 9Ò9Ð9¸T¼YÍ$ÔJ]ÔJdÒ=dÐ=dð à=dÐ=dÐ=dr(   r°   zexclude to quantize z$ as specified by nodes_to_exclude...zskip to quantize r<  r4  )r4  Ú	attributer—  ra  r  r€  rÍ   Úgr°   Ú_process_subgraphr�  Úgraphsrj  r   rœ   rm  rn  r\  r_  rl  rz  r]  r^  rs  ru  r   r{  rŒ  Ú
ClearFieldÚpop)r%   r5  Ú	new_nodesr“  r4  Úgraph_attrsr…  r‚  ÚkvrÅ  ÚsubgraphÚ	out_nodess               r&   r†  z&MatMulNBitsQuantizer._process_subgraph÷  sw  € Øˆ	Ø˜B”ˆà”Jð %	(ñ %	(ˆDðð à œNðñ ô ˆKð
 ñ Ø�Ø œNð &ð &�DØ”y¥DÔ$7Ô$=Ò=Ð=à#×*Ò*¨4¬6Ñ2Ô2Ð2Ø"œi¨×)?Ò)?ÀÑ)LÔ)LÐM˜˜Øœ¥dÔ&9Ô&@Ò@Ð@Ø "˜Ø(,¬ð Pð P˜Hà'×.Ò.¨xÑ8Ô8Ð8Ø!ŸLšL¨$×*@Ò*@ÀÑ*MÔ*MÐ)NÑOÔOÐOÐOØ"œi¨Ð/˜˜å/°Ñ5Ô5˜Ø—M’M "Ñ%Ô%Ð%Ð%Ý”{Ô,Ø”L $¤*¨d¬kðð Ø@DÄ	ðØMSðð �ð ˆIØŒy˜DÔ1Ð1Ð1Ý—’Ðb°4´9ÐbÐbÐbÑcÔcÐcØ!˜F�	�	ØÔ'ð #¨D¬I¸Ô9NÐ,NÐ,NØ” Ô 0Ô EÐEÐEà Ô/×8Ò8¸¸{ÑKÔK�	�	å—’Ð?°´	Ð?Ð?Ð?Ñ@Ô@Ð@Ø!˜F�	Ø×Ò˜YÑ'Ô'Ð'Ñ'à×Ò˜Ñ Ô Ð ØŒ
×Ò˜)Ñ$Ô$Ð$Ø�ŠÑÔÐØˆr(   c                óz  ‡ — i }‰ j         j         j        j        D ] }|j        dv r•t	          ˆ fd„|j        D ¦   «         ¦  «        sud‰ j        ‰ j        rdnddœ}‰ j        j	        rK|j
        ‰ j        j	        v r8‰ j        j	        |j
                                      ¦   «         D ]\  }}||v r|||<   Œ|||j
        <   Œ¡|S )z3Generate weight only quant configuration for nodes.©r!   c              3  óP   •K  — | ] }‰j                              |¦  «        d u V — Œ!d S rá   )ri  r`  )r¿   r¦   r%   s     €r&   rD  z@MatMulNBitsQuantizer._generate_q4_node_config.<locals>.<genexpr>,  s8   øè è € ÐUÐUÀQ˜4œ:×5Ò5°aÑ8Ô8¸DÐ@ÐUÐUÐUÐUÐUÐUr(   rN   ÚsymÚasym)rR   r(  Úscheme)ri  r“  r4  r\  Úallr_  rH   rW   ru  r   r°   rÎ   )r%   Úq4_node_configr4  Útemplate_config_q4ÚkeyrÅ  s   `     r&   Ú_generate_q4_node_configz-MatMulNBitsQuantizer._generate_q4_node_config'  sü   ø€ àˆØ”JÔ$Ô*Ô/ð 	Cð 	CˆDØŒ|˜zÐ)Ð)ÝÐUÐUÐUÐUÈ$Ì*ÐUÑUÔUÑUÔUð Cà !Ø&*¤oØ+/Ô+<Ð"H % %À&ð*ð *Ð&ð Ô(ÔAð@à œI¨Ô)9Ô)RÐRÐRà*.Ô*:Ô*SÐTXÔT]Ô*^×*dÒ*dÑ*fÔ*fð @ð @™J˜C Ø"Ð&8Ð8Ð8Ø:?Ð 2°3Ñ 7øØ0B�N 4¤9Ñ-øØÐr(   c                ó´  ‡ — ˆ fd„}i }‰ j         �
‰ j         |d<   ‰                      ¦   «         }‰ j        j        }t                               d|› d�¦  «         |dv rQ‰ j        j        |d<   ||d<   	 ‰ j        D ]}d	||<   Œt          d‰ j	        �‰ j	        n‰ j
        j
        |d
œ|¤Ž‰ _
        n�|dk    r‡‰ j        j        |d<   ‰ j        j        |d<   ‰ j        j        |d<   ‰ j        j        |d<   ‰ j        j        |d<   d|d<    |¦   «         }t!          d‰ j	        �‰ j	        n‰ j
        j
        ||dœ|¤Ž‰ _
        t                               d|› d�¦  «         dS )u  4b quantize a model with RTN or GPTQ algorithm. Please refer to
        https://github.com/intel/neural-compressor/blob/master/docs/source/quantization_weight_only.md
        for more details on weight only quantization using IntelÂ® Neural Compressor.
        c               3  ó^   •K  — t          j        ‰j        j        ¦  «        } | D ]}|d fV — Œ	d S rá   )ÚcopyÚdeepcopyru  rC   )Údata_readerrÝ   r%   s     €r&   Úinc_dataloaderz<MatMulNBitsQuantizer.int4_quant_algo.<locals>.inc_dataloaderB  sH   øè è € Ýœ-¨Ô(8Ô(PÑQÔQˆKØ#ð !ð !�Ø˜D�jÐ Ð Ð Ð ð!ð !r(   NrY   zstart to quantize model with z algorithm...)r0   r?   r5   r   Úfp32)ri  Úweight_configrF   rG   Ú	blocksizerI   rJ   rK   rˆ   Ú	n_samples)ri  r¡  Ú
dataloaderz$complete quantization of model with z algorithm.r,   )rY   r™  ru  r   r]  r^  r5   rz  r   ry  ri  rG   rH   rI   rJ   rK   r   )r%   rŸ  r…  Úweight_only_node_configr   rE  r¤  s   `      r&   Úint4_quant_algoz$MatMulNBitsQuantizer.int4_quant_algo<  sÚ  ø€ ð	!ð 	!ð 	!ð 	!ð 	!ð
 ˆØÔÐ*Ø'+Ô':ˆFÐ#Ñ$Ø"&×"?Ò"?Ñ"AÔ"AÐàÔ$Ô.ˆ	Ý�ŠÐL°IÐLÐLÐLÑMÔMÐMØÐ*Ð*Ð*Ø#Ô/Ô6ˆF�8ÑØ"+ˆF�;Ñðð Ô*ð 4ð 4�Ø-3Ð'¨Ñ*Ð*å%ð Ø)-¬Ð)D�d”o�oÈ$Ì*ÔJZØ5ðð ð ðð ˆDŒJˆJð
 ˜&Ò Ð Ø!%Ô!1Ô!:ˆF�:ÑØ"&Ô"2Ô"=ˆF�;ÑØ!%Ô!1Ô!:ˆF�:ÑØ Ô,Ô0ˆF�5‰MØ#'Ô#3Ô#>ˆF�<Ñ Ø"$ˆF�;ÑØ'˜Ñ)Ô)ˆJå&ð Ø)-¬Ð)D�d”o�oÈ$Ì*ÔJZØ5Ø%ðð ð ð	ð ˆDŒJõ 	�ŠÐQ¸9ÐQÐQÐQÑRÔRÐRÐRÐRr(   c                óÐ  — | j         j        dv �r| j                             ¦   «         g}| j         j        t
          j        k    r| j                             dd¦  «         | j         j        t
          j        k    sd| j         j	        v rl| j         
                    ¦   «         }|D ]P}|j        dv rE|j        dk     r:t                               d¦  «         | j                             |j        d¦  «         ŒQ|                      |¦  «         | j                             ¦   «          d S | j         j        dk    rŸt                               d	¦  «         | j                             | j        €| j        j        n| j        ¦  «        | _        t                               d
¦  «         t+          | j        ¦  «        | _        | j                             ¦   «          d S |                      ¦   «          d S )N)rP   r]   rO  r   r"   )Nzai.onnxrN  é   zžThe opset of the input model is under 21 and doesn't support int4 data type. Force to update it to opset 21, but the generated model may not be a valid model.ri   z%Processing nvidia_awq quantization...rl  )ru  r   ri  r“  r   r   r3   Úset_opset_importr~   r   Úopset_importrR  Úversionr]  Úwarningr†  Úclean_initializersr^  r{  rp  ry  r   r¦  )r%   r5  rª  Úopsets       r&   ÚprocesszMatMulNBitsQuantizer.processn  sÏ  € ØÔÔ%Ð);Ð;Ñ;àœ:×+Ò+Ñ-Ô-Ð.ˆKð ÔÔ,µÔ0EÒEÐEØ”
×+Ò+¨O¸QÑ?Ô?Ð?àÔÔ,µ´Ò?Ð?À8ÈtÔO_ÔOtÐCtÐCtØ#œz×6Ò6Ñ8Ô8�Ø)ð Fð F�EØ”|Ð'<Ð<Ð<ÀÄÐQSÒASÐASÝŸšðpñô ð ð œ
×3Ò3°E´LÀ"ÑEÔEÐEøà×"Ò" ;Ñ/Ô/Ð/ØŒJ×)Ò)Ñ+Ô+Ð+Ð+Ð+ØÔÔ'¨<Ò7Ð7å�KŠKÐ?Ñ@Ô@Ð@ØÔ,×9Ò9Ø$(¤OÐ$;�”
Ô Ð ÀÄñô ˆDŒJõ �KŠKÐ<Ñ=Ô=Ð=Ý" 4¤:Ñ.Ô.ˆDŒJØŒJ×)Ò)Ñ+Ô+Ð+Ð+Ð+ð × Ò Ñ"Ô"Ð"Ð"Ð"r(   )ri  rj  rR   rV   rH   rV   rW   rX   rY   rZ   rs  rt  r   r   r   r   r[   rX   ru  rv  )r5  r6  )r)   r*   r+   Ú__doc__r   r3   r'   r†  r™  r¦  r¯  r,   r(   r&   rr  rr  °  s©   € € € € € ðð ð. ØØ"Ø%)ØØ-1Ø Ô*Ø7;Ø9=Ø',Ø48ð0Mð 0Mð 0Mð 0Mð 0Mðd.ð .ð .ð .ð`ð ð ð*0Sð 0Sð 0Sðd #ð  #ð  #ð  #ð  #r(   rr  c                ó.   — |                       ¦   «         dv S )N)ÚtrueÚ1)Úlower)rÅ  s    r&   Úort_convert_str_to_boolrµ  ‘  s   € Ø�;Š;‰=Œ=˜MÐ)Ð)r(   c                óT   — |                       d¦  «        \  }}|t          |¦  «        fS )Nú:)r±   rV   )Úsr˜  rÅ  s      r&   Úparse_key_value_pairr¹  –  s%   € Ø—’˜‘”�J€CˆØ•�E‘
”
ˆ?Ðr(   c            
     ó´  — t          j        d¬¦  «        } |                      ddd¬¦  «         |                      ddd¬¦  «         |                      d	d
dt          d¬¦  «         |                      ddt          g d¢d¬¦  «         |                      ddt          d¬¦  «         |                      dd
dddt
          dd
gd¬¦  «         |                      dd
t          d¬¦  «         |                      ddd
d ¬!¦  «         |                      d
¬"¦  «         |                      d#d$t          d
g d%¬&¦  «         |                      d'd$t          d
d(¬)¦  «         |                      d*d+t          d+d,gd-¬¦  «         |                      d.t          d$d/d0gd1¬2¦  «         |                      d3t          d$d
d4¬5¦  «         |                      d6d7¦  «        }|                     d8t          d9d:¬;¦  «         |                     d<t          d
d=¬>¦  «         |                     d?t          d
d@dAgdB¬C¦  «         |                     dDt          dEdF¬;¦  «         |  	                    ¦   «         S )GNa
  Blockwise int4 quantization for MatMul 2D weight matrices.

A weight matrix is partitioned into into blocks, where each block is a
continguous subset inside each column. Each block is quantized into a
set of 4b integers with a scaling factor and an optional offset.
)Údescriptionz--input_modelTzPath to the input model file)ÚrequiredÚhelpz--output_modelzPath to the output model filez--block_sizeFrl   zBlock size for quantization)r¼  Údefaultr—  r½  z--quant_methodr¾  )r¾  ÚhqqÚrtnr?   Úgptqri   uW   the algorithm used to quantize weight, 
rtn and gptq leverage IntelÂ® Neural Compressor)r¾  r—  Úchoicesr½  z--bitsrN   z#the target bits to represent weight)r¾  r—  r½  z--symmetricú?zWIndicate whether to quantize the model symmetrically, symmetric is not supported by hqq)r¼  r¾  ÚconstÚnargsr—  rÂ  r½  z--accuracy_levelzíAccuracy level of the 4-bit quantized MatMul computation. Refer to the MatMulNBits contrib op's 'accuracy_level' attribute for details (https://github.com/microsoft/onnxruntime/blob/main/docs/ContribOperators.md#commicrosoftmatmulnbits).)r¼  r—  r½  z-vz	--verboseÚ
store_true)r¼  Úaction)r  z--nodes_to_excludeú+zBSpecify the nodes to be excluded from quantization with node names)rÅ  r—  r¼  r¾  r½  z--nodes_to_includezKSpecify the specific nodes to be included from quantization with node names)rÅ  r—  r¼  r½  z--quant_formatr3   r~   z±QuantFormat {QOperator, QDQ}QOperator format quantizes the model with quantized operators directly.QDQ format quantize the model by inserting DeQuantizeLinear before the MatMul.z--op_types_to_quantizer!   r"   zPop_types_to_quantize {MatMul, Gather}. Operators to quantize. Default is MatMul.)r—  rÅ  rÂ  r½  z--quant_axeszªKey-value pairs in op_type:axis_to_quantize separated by space.Specify the axis to quantize for an op. Default {MatMul:0, Gather:1}Example: --quant_axes MatMul:0 Gather:1)r—  rÅ  r¼  r½  ri   z-Arguments specific to nvidia_awq quantizationz--calib_dataset_namera   z/Name of the calibration dataset for nvidia_awq.)r—  r¾  r½  z--tokenizer_dirzPath of the tokenizer dir.)r—  r¼  r½  z--calibration_methodÚawqÚawq_clipz<Support two options, awq implementation and weight clipping.)r—  r¼  rÂ  r½  z--cache_dirrb   z%Cache directory for calibration data.)
ÚargparseÚArgumentParserÚadd_argumentrV   r   rµ  Úset_defaultsr¹  Úadd_argument_groupÚ
parse_args)ÚparserÚnv_awq_configs     r&   rÐ  rÐ  ›  s<  € ÝÔ$ððñ ô €Fð ×Ò˜°$Ð=[ÐÑ\Ô\Ð\Ø
×ÒÐ(°4Ð>]ÐÑ^Ô^Ð^Ø
×Ò˜°ÀÍÐSpÐÑqÔqÐqØ
×ÒØØÝØJÐJÐJØgð ñ ô ð ð ×Ò˜¨!µ#Ð<aÐÑbÔbÐbØ
×ÒØØØØØÝ$Ø�u�Øfð ñ 	ô 	ð 	ð ×ÒØØÝðqð	 ñ ô ð ð ×Ò˜˜k°EÀ,ÐÑOÔOÐOØ
×Ò ÐÑ&Ô&Ð&Ø
×ÒØØÝØØØQð ñ ô ð ð ×ÒØØÝØØZð ñ ô ð ð ×ÒØØÝØ˜eÐ$ðYð ñ ô ð ð ×ÒØ ÝØØ˜8Ð$Ø_ð ñ ô ð ð ×ÒØÝ!ØØð2ð ñ ô ð ð ×-Ò-¨lÐ<kÑlÔl€MØ×ÒØÝØØ>ð	 ñ ô ð ð ×ÒØÝØØ)ð	 ñ ô ð ð ×ÒØÝØØ˜
Ð#ØKð ñ ô ð ð ×ÒØÝØØ4ð	 ñ ô ð ð ×ÒÑÔÐr(   Ú__main__r�  zfile z already existsr¿  zAsymmetric is not supportted by hqq, will force to symmetric=FalseF)rH   rR   r   r   r¾  )rH   rW   rY   r   r   r   rR   rÀ  )r   r?   rÁ  )rH   r   ri   zFQOperator is not applicable to nvidia_awq. overriding the value to QDQrÉ  rc   rÊ  )rm   r‚   ro   r�   z!Unsupported quantization method: )ri  rR   rY   rz  rs  ru  T)r�  r6  r7  r�  )r”  r   r�  r6  r7  rZ   )rT  r   rU  r�  rV  r   rW  r   rX  r   rY  r   r7  r8  )`Ú
__future__r   rË  rœ  ÚloggingÚosr  r¾   râ   Únumpy.typingÚtypingÚnptra  Úonnx_irr  Úonnx.onnx_pbr   r   r   r   Úonnxruntime.capi._pybind_stater   r	   r
   r   Ú	calibrater   Úneural_compressorr   r   Ú
onnx_modelr   Úquant_utilsr   r   ÚbasicConfigÚINFOÚ	getLoggerr)   r]  r   r.   r=   rA   rM   rU   r`   ræ   rè   r`  rk  ro  ró  rõ  rg  rr  rµ  r¹  rÐ  Úargsr  ÚsetLevelÚDEBUGÚinput_modelÚinput_model_pathÚoutput_modelÚoutput_model_pathr   r   r�  r   ÚpathÚexistsrd  Ú	ExceptionÚ	symmetricÚquant_methodr¬  rx  ri  rH   rR   Úquant_configrY   r3   r~   r�   Úcalib_dataset_namer‚   ro   rÇ   rz  rs  Úquantr¯  Úsave_model_to_filer,   r(   r&   ú<module>rô     s/  ðð #Ð "Ð "Ð "Ð "Ð "à €€€Ø €€€Ø €€€Ø 	€	€	€	à Ð Ð Ð Ø Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø €€€Ø Ð Ð Ð Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GÐ Gðð ð ð ð ð ð ð ð ð ð ð ð -Ð ,Ð ,Ð ,Ð ,Ð ,Ø :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ø !Ð !Ð !Ð !Ð !Ð !Ø 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8à €Ô ÐOÐW^ÔWcÐ dÑ dÔ dÐ dØ	ˆÔ	˜8Ñ	$Ô	$€ðAð Að Að Að Añ Aô Að Að@#ð #ð #ð #ð #Ð4ñ #ô #ð #ðLð ð ð ð Ð"7ñ ô ð ðD0%ð 0%ð 0%ð 0%ð 0%Ð 5ñ 0%ô 0%ð 0%ðf(ð (ð (ð (ð (Ð4ñ (ô (ð (ðV/kð /kð /kð /kð /kÐ#8ñ /kô /kð /kðds#ð s#ð s#ð s#ð s#Ð!6ñ s#ô s#ð s#ðl4ð 4ð 4ðPð Pð Pð Pð Pñ Pô Pð Pðfð ð ð ðð ð ð ð&ð ð ð ðF#ð #ð #ð&fð fð fð fð fñ fô fð fðR(ð (ð (ð (ð (ñ (ô (ð (ðV^#ð ^#ð ^#ð ^#ð ^#ñ ^#ô ^#ð ^#ðB*ð *ð *ð
ð ð ð
lð lð lð^ ˆzÒÑØˆ:‰<Œ<€DØ„|ð 'Ø�Š˜œÑ&Ô&Ð&àÔ'ÐØÔ)ÐØ˜tÔ0Ô1€LØ?CÔ?XÐi˜5˜5 Ô!:Ñ;Ô;Ð;Ð^iÐØ+/¬?ÐD���t”Ñ'Ô'Ð'À€Jà	„w‡~‚~Ð'Ñ(Ô(ð DØ�ŠÐ?Ð.Ð?Ð?Ð?Ñ@Ô@Ð@ØˆiÐBÐ 1ÐBÐBÐBÑCÔCÐCà„~ð ˜$Ô+¨uÒ4Ð4Ø�ŠÐZÑ[Ô[Ð[ØˆŒàˆDŒIÐ&Ñ'Ô'€EØÔ˜EÒ!Ð!Ø/Ð/Ø”¨T¬YÐMaÐnxð
ñ 
ô 
ˆ‰ð 
Ô	˜iÒ	'Ð	'Ø3Ð3Ø”ØœØÔ.Ø%Ø!5Ø!Ø”ð
ñ 
ô 
ˆˆð 
Ô	˜eÒ	#Ð	#Ø/Ð/ÐEYÐZÑZÔZˆˆØ	Ô	˜iÒ	'Ð	'Ø2Ð2ÐH\Ð]Ñ]Ô]ˆˆØ	Ô	˜fÒ	$Ð	$Ø0Ð0¸D¼OÐbvÐwÑwÔwˆˆØ	Ô	˜lÒ	*Ð	*Ø˜;Ô0Ò0Ð0Ø�NŠNÐcÑdÔdÐdØ&œ?ˆLà ˆØÔ"Ð.ØÔ&¨%Ò/Ð/Ø%/Ð"Ð"à%/Ð"Ð"à!+Ðà1Ð1ØÔ0ØÔ,Ø”nØ1ð	
ñ 
ô 
ˆˆð ˆjÐP¸TÔ=NÐPÐPÑQÔQÐQà Ð ØØŒYØÔ*ØÔ.ØÔ.Ø ðñ ô €Eð 
‡M‚M�O„O€OØ	„K×"Ò"Ð#4°dÑ;Ô;Ð;Ð;Ð;ðQ Ðr(   