§
    kŠtjçh  ã                  óè   — d Z ddlmZ ddlZddlZddlmZ ddlZ ej	        e
¦  «        ZerddlZd„ Zd„ Zdd	„Z ej        d
¬¦  «        dd„¦   «         Zd„ Zdd„Zdd d„Zd„ Z G d„ d¦  «        ZdS )!aw  CUDA weight-only quantization helpers.

This module contains small Python utilities for producing the weight layouts
consumed by CUDA weight-only kernels. The blockwise quantizers wrap the same C++
pybind entry points used by runtime prepacking, and the mixed-GEMM weight packer
is a PyTorch reimplementation of the runtime CUDA packing, so tests and model
builders can generate byte-identical quantized weights. The PyTorch packer runs
on CUDA when a device is available and falls back to CPU otherwise, which is the
only option on platforms where the standalone CUDA packer is not built (Windows).
A GPU-gated parity test validates it against that standalone CUDA packer.

Two storage families are exposed:

* raw MatMulNBits blockwise storage, laid out by output channel as ``[N, K/pack]``;
* CUDA mixed-GEMM prepacked storage. MatMulNBits prepacked initializers keep the
    schema shape ``[N, K/block_size, block_size*bits/8]`` and require the node
    attribute ``weight_prepacked=1``. QMoE/CUTLASS callers use the kernel-facing
    shape ``[K, N/pack]``.

All public helpers take one logical expert weight matrix with shape ``[N, K]``.
é    )ÚannotationsN)ÚTYPE_CHECKINGc                 óV   — 	 ddl } n"# t          $ r}t          d¦  «        |‚d}~ww xY w| S )zQImport torch lazily so importing onnxruntime.quantization does not require torch.r   NzMCUDA weight-only quantization requires torch. Please install torch to use it.)ÚtorchÚImportError)r   Úes     úe/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/onnxruntime/quantization/cuda_quantizer.pyÚ
_get_torchr
   +   sO   € ðrØˆˆˆˆøÝð rð rð rÝÐiÑjÔjÐpqÐqøøøøðrøøøð €Ls   ‚ ‡
&‘!¡&c                 óª   — 	 ddl m}  n"# t          $ r}t          d¦  «        |‚d}~ww xY w	 | j        S # t          $ r}t          d¦  «        |‚d}~ww xY w)a  Return the standalone CUDA mixed-GEMM weight packer (parity oracle).

    Production packing uses the PyTorch implementation (``_pack_weights_for_cuda_mixed_gemm``).
    This standalone packer lives in ``onnxruntime.capi.onnxruntime_cuda_quant_preprocess``, a
    separate extension module that links the CUDA runtime (built only on non-Windows CUDA
    builds). It is imported lazily here (never at ``import onnxruntime`` time) and is used by
    the parity test to validate the PyTorch packer byte-for-byte.
    r   )Ú!onnxruntime_cuda_quant_preprocessz’The standalone CUDA weight packer (onnxruntime_cuda_quant_preprocess) is unavailable; it is built only on non-Windows onnxruntime-gpu CUDA builds.NzNonnxruntime_cuda_quant_preprocess is missing pack_weights_for_cuda_mixed_gemm.)Úonnxruntime.capir   r   Ú pack_weights_for_cuda_mixed_gemmÚAttributeError)Ú_cuda_quantr   s     r	   Ú%_get_pack_weights_for_cuda_mixed_gemmr   5   sœ   € ðØUÐUÐUÐUÐUÐUÐUøÝð ð ð ÝðKñ
ô 
ð ð	øøøøðøøøðsØÔ;Ð;øÝð sð sð sÝÐjÑkÔkÐqrÐrøøøøðsøøøs$   ‚	 ‰
(“#£(¬3 ³
A½AÁAÚreturnÚboolc                 óH   — 	 t          ¦   «          n# t          $ r Y dS w xY wdS )a  Return True if mixed-GEMM weight prepacking is available.

    Prepacking is implemented with PyTorch (CUDA when available, CPU otherwise), so it is
    available whenever torch is importable. Callers use this to skip prepack code paths
    (and tests) when torch is unavailable.
    FT)r
   r   © ó    r	   Úhas_cuda_weight_prepackingr   L   s9   € ðÝ‰ŒˆˆøÝð ð ð Øˆuˆuðøøøàˆ4s   ‚ ‘
žé   )ÚmaxsizeÚNonec                 ó:   — t                                d¦  «         d S )NzÐCUDA device is not available; packing mixed-GEMM weights on CPU with PyTorch. This is correct but significantly slower for large Mixture-of-Experts models. Pack on a CUDA-enabled machine for best performance.)Ú_loggerÚwarningr   r   r	   Ú_warn_cpu_prepack_oncer   Z   s'   € å‡O‚Oð	?ñô ð ð ð r   c                 ó¬   — t          ¦   «         } | j                             ¦   «         r | j        d¦  «        S t	          ¦   «           | j        d¦  «        S )zRPick the torch device for mixed-GEMM weight packing (CUDA if available, else CPU).ÚcudaÚcpu)r
   r    Úis_availableÚdevicer   )r   s    r	   Ú_prepack_devicer$   c   sQ   € å‰LŒL€EØ„z×ÒÑ Ô ð $ØˆuŒ|˜FÑ#Ô#Ð#ÝÑÔÐØˆ5Œ<˜ÑÔÐr   ÚbitsÚintÚsmc                óî	  ‡‡— t          ¦   «         }d}|dk    rdnd}|                      ¦   «         dk    r|                      d¦  «        } g d¢g d¢dœ}d}d	|z  Š| j        d         | j        d
         | j        d         }
}	}|	‰z  dk    s	|
|z  dk    rt	          d|	› d|
› d‰› d|› d�	¦  «        ‚|dk     r8||› d|› �         Šˆˆfd„t          |	¦  «        D ¦   «         }| dd…|dd…f         } | j        }|dk    �r|                      |j        ¦  «        }|dz	                       ddd
¦  «                             d¦  «        }|dz  dz	                       ddd
¦  «                             d¦  «        } |j	        ||gd¬¦  «         
                    |j        d         d|j        d
         ¦  «        }|dd…dd…ddd…f         |dd…dd…d
dd…f         dz  z   }|                     |j        ¦  «         
                    |¦  «        } n*|                      ddd
¦  «         
                    |¦  «        } ||z  }|d
k    rƒ|dk     r}d|z  }d|z  }|	|z  dk    s	|	|z  dk    rt	          d|	› d�¦  «        ‚|  
                    |d||	|z  |dz  |z  ¦  «        } |                      dd
ddd¦  «         
                    |¦  «        } |dk    rª|                      |j        ¦  «        }|d|dk                         |j        ¦  «        z  d	z   z  }| 
                    dd¦  «        dd…g d¢f          
                    |¦  «        }|                     |j        ¦  «                             |j        ¦  «        } �na|                      |j        ¦  «        }|dz	                       d¦  «        }|dz  dz	                       d¦  «        } |j	        ||gd¬¦  «         
                    |j        d         |j        d
         d¦  «        }| 
                    dd¦  «        dd…g d¢f          
                    |j        ¦  «        }|                     |j        ¦  «        }|d |d!k                         |j        ¦  «        z  dz   z  }|dd…dd…ddd…f         |dd…dd…d
dd…f         dz  z   }|                     |j        ¦  «                             |j        ¦  «        } |                      d¦  «                             ¦   «         S )"a  PyTorch port of the runtime CUDA ``preprocess_weights_for_mixed_gemm``.

    ``tensor`` is a signed int8 weight in ``(K, N/pack)`` packed row-major layout on any
    device. Returns the CUTLASS mixed-GEMM layout with the same shape/dtype/device. This
    mirrors ``preprocess_weights_for_mixed_gemm_cuda`` (permute_B_rows -> subbyte_transpose
    -> interleave_column_major -> add_bias_and_interleave) so its output is byte-identical
    to the standalone CUDA packer, for both the SM80 (Ampere) and SM90 (Hopper) layouts.
    é   é   é   é   r   )r   r   r+   é	   r,   é   é
   é   r*   é   é   é   é   é   é   é   ) r   r   r+   r-   r)   é   é   é   r,   r.   r/   r0   é   é   é   é   r*   r1   r2   r3   é   é   é   é   r4   r5   r6   r7   é   é   é   é   )Ú16_8Ú16_4é€   r   zweight shape (rows=z, packed_cols=zF) is incompatible with mixed-GEMM packing (rows must be a multiple of z, packed cols a multiple of z).éd   Ú_c                ó8   •— g | ]}|‰z  ‰z  ‰|‰z           z   ‘ŒS r   r   )Ú.0ÚrÚb_rows_per_mmaÚpmaps     €€r	   ú
<listcomp>z<_preprocess_weights_for_mixed_gemm_torch.<locals>.<listcomp>®   s3   ø€ ÐnÐnÐnÐYZ�A˜Ñ'¨>Ñ9¸DÀÀ^ÑASÔ<TÑTÐnÐnÐnr   N©ÚdiméÿÿÿÿéZ   i   é    z
num_rows (z0) is incompatible with column-interleave tiling.r.   i ÿÿÿé   )r   r,   r   r.   )r   r,   r*   r4   r   r.   r1   r5   iðÿÿÿr5   )r
   rS   Ú	unsqueezeÚshapeÚ
ValueErrorÚrangeÚviewÚuint8ÚpermuteÚcatÚreshapeÚint8ÚtoÚint64Úint16ÚsqueezeÚ
contiguous)Útensorr%   r'   r   Úbits_aÚbits_bÚpermutation_mapÚmma_shape_nÚnum_expertsÚnum_rowsÚnum_colsÚrow_idxÚoriginal_shapeÚuÚhighÚlowÚmergedÚ
interleaveÚrows_per_tileÚelts_in_int32ÚtrO   rP   s                        @@r	   Ú(_preprocess_weights_for_mixed_gemm_torchry   l   sƒ  øø€ õ ‰LŒL€EØ€FØ˜!’)�)ˆQˆQ €Fà‡z‚z�|„|�qÒÐØ×!Ò! !Ñ$Ô$ˆð GÐFÐFð!
ð !
ð !
ð$ð $€OðJ €KØ˜vÑ%€Nà&,¤l°1¤o°v´|ÀA´ÈÌÐUVÌ˜8�€KØ�.Ñ  AÒ%Ð%¨°KÑ)?À1Ò)DÐ)DÝðo (ð oð o¸(ð oð oØ3Aðoð oØ_jðoð oð oñ
ô 
ð 	
ð 
ˆC‚x€xØ &Ð3Ð3¨6Ð3Ð3Ô4ˆØnÐnÐnÐnÐnÕ^cÐdlÑ^mÔ^mÐnÑnÔnˆØ˜˜˜˜7 A A A˜Ô&ˆð ”\€NØ�‚{�{Ø�KŠK˜œÑ$Ô$ˆØ�Q‘×Ò  1 aÑ(Ô(×2Ò2°1Ñ5Ô5ˆØ�Q‘˜1‰}×%Ò% a¨¨AÑ.Ô.×8Ò8¸Ñ;Ô;ˆØ�”˜C ˜;¨AÐ.Ñ.Ô.×6Ò6°q´w¸q´zÀ2ÀqÄwÈqÄzÑRÔRˆØ˜˜˜˜1˜1˜1˜a˜d ˜d˜
Ô# f¨Q¨Q¨Q°°°°1°4°a°4¨ZÔ&8¸2Ñ&=Ñ=ˆØ—’˜UœZÑ(Ô(×0Ò0°Ñ@Ô@ˆˆà—’  1 aÑ(Ô(×0Ò0°Ñ@Ô@ˆð ˜6Ñ!€JØ�A‚~€~˜"˜rš'˜'Ø 6Ñ)ˆØ˜f™ˆØ�mÑ# qÒ(Ð(¨H°}Ñ,DÈÒ,IÐ,IÝÐd¨(ÐdÐdÐdÑeÔeÐeØ—’Ø˜˜Z¨°]Ñ)BÀMÐTUÑDUÐYfÑDfñ
ô 
ˆð —’  1 a¨¨AÑ.Ô.×6Ò6°~ÑFÔFˆð �‚{€{Ø�IŠI�e”kÑ"Ô"ˆØ	ˆT�Q˜’W—L’L ¤Ñ-Ô-Ñ-°Ñ3Ñ3ˆØ�IŠI�b˜!ÑÔ˜Q˜Q˜Q   ˜_Ô-×5Ò5°nÑEÔEˆØ—’�e”kÑ"Ô"×'Ò'¨¬
Ñ3Ô3ˆ‰à�KŠK˜œÑ$Ô$ˆØ�Q‘×!Ò! "Ñ%Ô%ˆØ�Q‘˜1‰}×'Ò'¨Ñ+Ô+ˆØ�”˜C ˜;¨BÐ/Ñ/Ô/×7Ò7¸¼À¼
ÀAÄGÈAÄJÐPRÑSÔSˆØ—’  AÑ&Ô& q q qÐ*BÐ*BÐ*BÐ'BÔC×KÒKÈFÌLÑYÔYˆØ—’˜5œ;Ñ'Ô'ˆØ�#˜ !šŸš¨¬Ñ4Ô4Ñ4°qÑ8Ñ8ˆØ˜˜˜˜1˜1˜1˜a˜d ˜d˜
Ô# f¨Q¨Q¨Q°°°°1°4°a°4¨ZÔ&8¸2Ñ&=Ñ=ˆØ—’˜5œ;Ñ'Ô'×,Ò,¨U¬ZÑ8Ô8ˆà�>Š>˜!ÑÔ×'Ò'Ñ)Ô)Ð)r   éP   ÚnÚkÚ
force_archú
np.ndarrayc                ó2  — t          ¦   «         }t          |¦  «        }t          |¦  «        }|dvrt          d|› d�¦  «        ‚|dvrt          d|› d�¦  «        ‚d|z  }t          ¦   «         } |j        t          j        | ¦  «        ¦  «                             |j        ¦  «         	                    |||z  ¦  «         
                    |¦  «        }|dk    rð|dz   
                    |j        ¦  «        }	|dz	   
                    |j        ¦  «        }
 |j        ||f|j        |¬	¦  «        }|	|d
d
…dd
d…f<   |
|d
d
…dd
d…f<   |dz
                       dd¦  «                             ¦   «         }|d
d
…dd
d…f         dz  |d
d
…dd
d…f         dz  dz  z   
                    |j        ¦  «                             |j        ¦  «        }nu| 
                    |j        ¦  «        dz
                       dd¦  «                             ¦   «         }| 
                    |j        ¦  «                             |j        ¦  «        }t!          |                     ¦   «         ||¦  «        }| 	                    d¦  «                             ¦   «                              ¦   «         S )aQ  PyTorch implementation of the CUDA ``pack_weights_for_cuda_mixed_gemm``.

    ``q_weights`` is ORT's unsigned MatMulNBits/QMoE storage ``(N, K/pack)`` (uint8). Returns
    a flat ``int8`` numpy array with the CUTLASS mixed-GEMM layout, byte-identical to the
    standalone CUDA packer. Runs on CUDA when available, otherwise on CPU.
    ©r*   r+   zbits must be 4 or 8, got ú.©rz   rU   z/force_arch must be 80 (SM80) or 90 (SM90), got r+   r*   r7   ©Údtyper#   Nr   r,   r   rI   rT   )r
   r&   rZ   r$   Ú	as_tensorÚnpÚascontiguousarrayr\   r]   r`   rb   rd   ÚemptyÚ	transposerf   ra   ry   r!   Únumpy)Ú	q_weightsr{   r|   r%   r}   r   Úpackr#   Úqrs   rr   ÚunpackedÚsigned_tÚpacked_tÚouts                  r	   Ú!_pack_weights_for_cuda_mixed_gemmr’   Ý   sƒ  € õ ‰LŒL€EÝˆt‰9Œ9€DÝ�Z‘”€JØ�6ÐÐÝÐ<°TÐ<Ð<Ð<Ñ=Ô=Ð=Ø˜Ð!Ð!ÝÐXÈ:ÐXÐXÐXÑYÔYÐYØ�‰9€DÝÑÔ€FàˆŒ�Ô,¨YÑ7Ô7Ñ8Ô8×=Ò=¸e¼kÑJÔJ×RÒRÐSTÐVWÐ[_ÑV_Ñ`Ô`×cÒcÐdjÑkÔk€Að ˆq‚y€yØ�4‰x�mŠm˜EœKÑ(Ô(ˆØ�Q‘�{Š{˜5œ;Ñ'Ô'ˆØ�5”;  1˜v¨U¬[ÀÐHÑHÔHˆØˆ����A�D�q�D�ÑØ ˆ����A�D�q�D�ÑØ˜q‘L×+Ò+¨A¨qÑ1Ô1×<Ò<Ñ>Ô>ˆØ˜a˜a˜a   A ˜gÔ&¨Ñ-°8¸A¸A¸A¸q¸tÀ!¸t¸GÔ3DÀtÑ3KÐPQÑ2QÑR×VÒVÐW\ÔWbÑcÔc×hÒhÐinÔisÑtÔtˆˆà—D’D˜œÑ%Ô%¨Ñ+×6Ò6°q¸!Ñ<Ô<×GÒGÑIÔIˆØ—;’;˜uœ{Ñ+Ô+×0Ò0°´Ñ<Ô<ˆå
2°8×3FÒ3FÑ3HÔ3HÈ$ÐPZÑ
[Ô
[€CØ�;Š;�r‰?Œ?×ÒÑ Ô ×&Ò&Ñ(Ô(Ð(r   c                 ób   — 	 ddl m} m} n"# t          $ r}t          d¦  «        |‚d}~ww xY w| |fS )zCReturn MatMulNBits blockwise quantizers from the ORT pybind module.r   )Úquantize_matmul_4bitsÚquantize_matmul_8bitszfCUDA blockwise quantization requires quantize_matmul_4bits and quantize_matmul_8bits from onnxruntime.N)Úonnxruntime.capi._pybind_stater”   r•   r   )r”   r•   r   s      r	   Ú_get_quantize_matmul_nbitsr—     s{   € ðð	
ð 	
ð 	
ð 	
ð 	
ð 	
ð 	
ð 	
ð 	
øõ ð ð ð ÝØtñ
ô 
àð	øøøøðøøøð
 !Ð"7Ð7Ð7s   ‚ ‹
*•%¥*c                  óþ   — e Zd ZdZeddœd!d„¦   «         Ze	 d"ddœd#d„¦   «         Zed$d„¦   «         Zeddddddœd%d„¦   «         Ze	 d"dddddœd&d„¦   «         Z	e	 d"dddddœd&d„¦   «         Z
eddœd'd„¦   «         Zd S )(ÚCudaQuantizerz¼CUDA quantizer utilities for MoE/QMoE and MatMulNBits-style weight-only kernels.

    The methods are stateless; callers may use the class directly without
    constructing an object.
    T©Úunsigned_full_rangeÚweightsútorch.Tensorr%   r&   r›   r   r   ú!tuple[torch.Tensor, torch.Tensor]c               ó^  — t          ¦   «         }|                      ¦   «                              ¦   «                              |j        ¦  «                             ¦   «         } t          |¦  «        }|dvrt          d|› d�¦  «        ‚| j        \  }}d|z  }||z  dk    rt          d|› d|› d�¦  «        ‚|d	k    r|rd
\  }}}	}
nd\  }}}	}
n|rd\  }}}	}
nd\  }}}	}
|  	                    ¦   «          
                    dd¬¦  «        t          |	¦  «        z  } |j        | |j        |j        ¦  «        j        ¬¦  «        } |j         |j        | |z  ¦  «        ||¦  «                             |j        ¦  «                             ¦   «         }||
z                        |j        ¦  «        }|d	k    rC|dd…ddd…f         dz  |dd…ddd…f         dz  d	z  z  }|                     |j        ¦  «        }n|}|                     ¦   «         |                     d¦  «                             ¦   «         fS )aS  Quantize one QMoE expert with symmetric per-channel storage.

        ``weights`` has logical shape ``[N, K]``. Returns raw QMoE storage
        ``[N, K/pack]`` and scales ``[N]``. By default, this emits the ORT CUDA
        QMoE storage contract: unsigned bytes/nibbles with an implicit zero-point
        offset, so each stored value is ``q + zero_point`` even though the numeric
        quantization is symmetric. By default it uses the full ``[-8, 7]`` /
        ``[-128, 127]`` range. Set ``unsigned_full_range=False`` to use the legacy
        ``[-7, 7]`` / ``[-127, 127]`` range.
        r€   z=QMoE per-channel quantization only supports 4 or 8 bits, got r�   r+   r   úK (ú) must be divisible by z# for QMoE per-channel quantization.r*   ©éøÿÿÿr5   r+   r+   ©éùÿÿÿr5   r5   r+   ©é€ÿÿÿrW   rI   rI   ©é�ÿÿÿrW   rW   rI   r   T)rS   Úkeepdim©ÚminNr,   r7   rT   )r
   Údetachr!   rb   Úfloat32rf   r&   rZ   rY   ÚabsÚamaxÚfloatÚclampÚfinfoÚepsÚroundrd   r]   re   )rœ   r%   r›   r   r{   r|   rŒ   ÚqminÚqmaxÚscale_divisorÚ
zero_pointÚscalesÚ	quantizedÚqweights                 r	   Ú#qmoe_symmetric_per_channel_quantizez1CudaQuantizer.qmoe_symmetric_per_channel_quantize  sS  € õ" ‘”ˆà—.’.Ñ"Ô"×&Ò&Ñ(Ô(×+Ò+¨E¬MÑ:Ô:×EÒEÑGÔGˆÝ�4‰yŒyˆØ�vÐÐÝÐdÐ]aÐdÐdÐdÑeÔeÐeàŒ}‰ˆˆ1Ø�D‰yˆØˆt‰8�qŠ=ˆ=ÝÐf 1ÐfÐf¸TÐfÐfÐfÑgÔgÐgà�1Š9ˆ9Ø"ð DØ8CÑ5��d˜M¨:¨:à8CÑ5��d˜M¨:¨:à"ð LØ8KÑ5��d˜M¨:¨:à8KÑ5��d˜M¨:Ø—’‘”×#Ò#¨°4Ð#Ñ8Ô8½5ÀÑ;OÔ;OÑOˆØ�”˜V¨¨¬°U´]Ñ)CÔ)CÔ)GÐHÑHÔHˆØ�E”K  ¤¨G°fÑ,<Ñ =Ô =¸tÀTÑJÔJ×MÒMÈeÌkÑZÔZ×eÒeÑgÔgˆ	Ø Ñ+×/Ò/°´Ñ<Ô<ˆ	à�1Š9ˆ9Ø     A D q D Ô)¨CÑ/°Y¸q¸q¸qÀ!À$ÀQÀ$¸wÔ5GÈ#Ñ5MÐRSÑ4SÑTˆGØ—j’j ¤Ñ-Ô-ˆGˆGàˆGà×!Ò!Ñ#Ô# V§^¢^°BÑ%7Ô%7×%BÒ%BÑ%DÔ%DÐDÐDr   rz   Úprepackr}   c               ó  — t          ¦   «         }t                               | ||¬¦  «        \  }}|s||fS | j        \  }}	dt	          |¦  «        z  }
||
z  dk    rt          d|› d|
› d�¦  «        ‚t          |                     ¦   «         ||	t	          |¦  «        |¦  «        }t          j	        |¦  «         
                    t          j        ¦  «                             |	||
z  ¦  «        } |j        t          j        |¦  «        ¦  «        |fS )a7  Quantize per-channel QMoE weights and optionally CUTLASS-prepack them.

        When ``prepack`` is true, returned weights have shape ``[K, N/pack]``.
        Otherwise, returned weights keep raw per-channel storage ``[N, K/pack]``.
        Prepacking uses PyTorch (CUDA when available, CPU otherwise).
        rš   r+   r   úN (r¡   z! for CUDA QMoE prepacked weights.)r
   r™   r½   rY   r&   rZ   r’   rŠ   r†   Úasarrayr\   r]   r`   Ú
from_numpyr‡   )rœ   r%   r¾   r}   r›   r   r¼   rº   r{   r|   rŒ   Úpackeds               r	   Úqmoe_per_channel_quantizez'CudaQuantizer.qmoe_per_channel_quantizeK  s  € õ ‘”ˆå'×KÒKØØØ 3ð Lñ 
ô 
‰ˆ�ð
 ð 	#Ø˜F�?Ð"àŒ}‰ˆˆ1Ø•C˜‘I”I‰~ˆØˆt‰8�qŠ=ˆ=ÝÐd 1ÐdÐd¸TÐdÐdÐdÑeÔeÐeå2°7·=²=±?´?ÀAÀqÍ#ÈdÉ)Ì)ÐU_Ñ`Ô`ˆÝ”˜FÑ#Ô#×(Ò(­¬Ñ2Ô2×:Ò:¸1¸aÀ4¹iÑHÔHˆØˆuÔ¥Ô 4°VÑ <Ô <Ñ=Ô=¸vÐEÐEr   Ú
block_sizeÚ	symmetricÚ
abs_scalesú/tuple[torch.Tensor, torch.Tensor, torch.Tensor]c               óÊ  — t          ¦   «         }t          |¦  «        }t          |¦  «        }|                      ¦   «                              ¦   «                              |j        ¦  «                             ¦   «                              ¦   «         }|j        \  }}	|dvrt          d|› d�¦  «        ‚|dk    rt          d|› d�¦  «        ‚|	|z   dz
  |z  }
d|z  }||z   dz
  |z  }|�r~|dk    r|rd	nd
\  }}}}n|rdnd\  }}}}|
|z  }||	k    rt          j        |dd||	z
  ffd¦  «        }|                     ||
|¦  «        }t          j        t          j        |¦  «        d¬¦  «                             t          j        ¦  «        t          j        |¦  «        z  }t          j        |t          j        t          j        ¦  «        j        ¦  «        }t          j        t          j        ||dd…dd…t          j        f         z  ¦  «        ||¦  «                             t          j        ¦  «        }||z                        t          j        ¦  «        }|dk    rªt          j        ||
|ft          j        ¬¦  «        }|dd…dd…ddd…f         dz  |dd…dd…d|dd…dd…ddd…f         j        d         …f<   |dd…dd…d|dd…dd…ddd…f         j        d         …fxx         |dd…dd…ddd…f         dz  dz  z  cc<   n|}t          j        ||dk    r|
dz   dz  n|
ft          j        ¬¦  «        } |j        |¦  «         |j        |¦  «         |j        |¦  «        fS t          j        |j        ¦  «        }t          j        ||
|ft          j        ¬¦  «        }t          j        ||
ft          j        ¬¦  «        }t          j        ||dk    r|
dz   dz  n|
ft          j        ¬¦  «        }t9          ¦   «         \  }}|dk    r|n|} ||||||||	|¦  «         |rt          j        |¦  «        } |j        |¦  «         |j        |¦  «         |j        |¦  «        fS )zMQuantize ``weights`` with MatMulNBits pybinds and return unflattened storage.r€   z6Blockwise quantization only supports 4 or 8 bits, got r�   r   z;Blockwise quantization requires a positive block_size, got r   r+   r*   r¢   r¤   r¦   r¨   )r   r   Úconstantr,   )ÚaxisN)r„   r7   )r
   r&   r­   r!   rb   r®   rf   rŠ   rY   rZ   r†   Úpadr`   Úmaxr¯   ÚastypeÚmaximumr³   r´   ÚclipÚrintÚnewaxisrd   r]   ÚzerosrÂ   r‡   ÚTr—   )rœ   r%   rÅ   rÆ   rÇ   r›   r   Úwr{   r|   Ú
num_blocksrŒ   Ú	blob_sizer¶   r·   r¸   r¹   Úpadded_kÚblockedrº   r»   r¼   Úzero_pointsÚw_tr”   r•   Úquantizes                              r	   Ú$_matmulnbits_blockwise_quantize_implz2CudaQuantizer._matmulnbits_blockwise_quantize_implm  sÇ  € õ ‘”ˆå�4‰yŒyˆÝ˜‘_”_ˆ
Ø�NŠNÑÔ× Ò Ñ"Ô"×%Ò% e¤mÑ4Ô4×?Ò?ÑAÔA×GÒGÑIÔIˆØŒw‰ˆˆ1Ø�vÐÐÝÐ]ÐVZÐ]Ð]Ð]Ñ^Ô^Ð^Ø˜Š?ˆ?ÝÐhÐ[eÐhÐhÐhÑiÔiÐià˜*‘n qÑ(¨ZÑ7ˆ
Ø�D‰yˆØ $Ñ&¨Ñ*¨tÑ3ˆ	àñ 	fØ�qŠyˆyØI\Ð8o¸¸ÐboÑ5��d˜M¨:¨:ð .AÐ[Ð)Ð)ÐF[ñ 6��d˜M¨:ð " JÑ.ˆHØ˜1Š}ˆ}Ý”F˜1˜v¨¨8°a©<Ð'8Ð9¸:ÑFÔF�à—i’i  :¨zÑ:Ô:ˆGÝ”V�BœF 7™OœO°!Ð4Ñ4Ô4×;Ò;½B¼JÑGÔGÍ"Ì*ÐUbÑJcÔJcÑcˆFÝ”Z ­¬µ´Ñ(<Ô(<Ô(@ÑAÔAˆFÝœ¥¤¨°&¸¸¸¸A¸A¸A½r¼zÐ9IÔ2JÑ(JÑ KÔ KÈTÐSWÑXÔX×_Ò_Õ`bÔ`hÑiÔiˆIØ" ZÑ/×7Ò7½¼ÑAÔAˆIà�qŠyˆyÝœ( A z°9Ð#=ÅRÄXÐNÑNÔN�ØBKÈAÈAÈAÈqÈqÈqÐRSÐRVÐUVÐRVÈJÔBWÐZ]ÑB]�˜˜˜˜1˜1˜1Ð> 	¨!¨!¨!¨Q¨Q¨Q°°°1°¨*Ô 5Ô ;¸AÔ >Ð>Ð>Ñ?Ø˜˜˜˜1˜1˜1Ð> 	¨!¨!¨!¨Q¨Q¨Q°°°1°¨*Ô 5Ô ;¸AÔ >Ð>Ð>Ð?Ð?Ô?ÀIÈaÈaÈaÐQRÐQRÐQRÐTUÐTXÐWXÐTXÈjÔDYÐ\_ÑD_ÐdeÑCeÑeÐ?Ð?Ñ?Ð?à#�åœ( AÀÈÂ	À	¨
°Q©¸1Ñ'<Ð'<ÈzÐ#ZÕbdÔbjÐkÑkÔkˆKØ#�5Ô# GÑ,Ô,Ð.>¨eÔ.>¸vÑ.FÔ.FÐHXÈÔHXÐYdÑHeÔHeÐeÐeåÔ" 1¤3Ñ'Ô'ˆÝ”(˜A˜z¨9Ð5½R¼XÐFÑFÔFˆÝ”˜1˜j˜/µ´Ð<Ñ<Ô<ˆÝ”h ¸DÀAºI¸I J°¡N°qÑ#8Ð#8È:ÐVÕ^`Ô^fÐgÑgÔgˆå7QÑ7SÔ7SÑ4ÐÐ4Ø,0°AªI¨IÐ(Ð(Ð;PˆØˆ�˜#˜v {°JÀÀ1ÀiÑPÔPÐPàð 	$Ý”V˜F‘^”^ˆFàˆuÔ Ñ(Ô(Ð*:¨%Ô*:¸6Ñ*BÔ*BÐDTÀEÔDTÐU`ÑDaÔDaÐaÐar   F)rÆ   Úreturn_zero_pointsrÇ   Úflatten_qweightr›   rÞ   rß   úStuple[torch.Tensor, torch.Tensor] | tuple[torch.Tensor, torch.Tensor, torch.Tensor]c               óÊ   — t                                | |||||¬¦  «        \  }}	}
|r3|                     |j        d         d¦  «                             ¦   «         }|r||	|
fS ||	fS )a»  Quantize one expert with ONNX Runtime's MatMulNBits blockwise encoding.

        ``weights`` has logical shape ``[N, K]``. Returns raw flattened storage
        ``[N, ceil(K/block_size)*ceil(block_size/pack)]`` and block scales
        ``[N, ceil(K/block_size)]`` by default. Set ``flatten_qweight=False`` for
        the MatMulNBits initializer shape
        ``[N, ceil(K/block_size), ceil(block_size/pack)]``.
        Set ``return_zero_points=True`` to also return packed block zero-points.
        Symmetric quantization uses the full ``[-8, 7]`` / ``[-128, 127]`` range
        by default. Set ``unsigned_full_range=False`` to use the legacy
        ``[-7, 7]`` / ``[-127, 127]`` range.
        ©rÆ   rÇ   r›   r   rT   )r™   rÝ   r`   rY   rf   )rœ   r%   rÅ   rÆ   rÞ   rÇ   rß   r›   r¼   rº   rÚ   s              r	   Úmatmulnbits_blockwise_quantizez,CudaQuantizer.matmulnbits_blockwise_quantize±  sˆ   € õ0 (5×'YÒ'YØØØØØ!Ø 3ð (Zñ (
ô (
Ñ$ˆ�˜ð ð 	IØ—o’o g¤m°AÔ&6¸Ñ;Ô;×FÒFÑHÔHˆGØð 	0Ø˜F KÐ/Ð/à˜ˆÐr   )rÆ   rÞ   rÇ   r›   c          	     óò  — t          ¦   «         }t          |¦  «        }t          |¦  «        }t          |¦  «        }|dvrt          d|› d�¦  «        ‚|dk    rdnd}	||	vrt          d|	› d|› d	|› d�¦  «        ‚| j        \  }
}||z  d
k    rt          d|› d|› d�¦  «        ‚t                               | |||||¬¦  «        \  }}}t          } ||                     |
d¦  «                             ¦   «         |
|||¦  «        }t          j
        |¦  «                             t          j        ¦  «                             |j        ¦  «        } |j        t          j        |¦  «        ¦  «        }|r|||fS ||fS )a  Quantize and CUDA-prepack one MatMulNBits weight initializer.

        ``weights`` has logical shape ``[N, K]``. Returns ``B`` with the standard
        MatMulNBits initializer shape ``[N, K/block_size, block_size*bits/8]``
        and scales with shape ``[N, K/block_size]``.

        The ``force_arch`` value selects the mixed-GEMM weight layout and must match
        the ``weight_prepacked`` attribute set on the MatMulNBits node:

        * ``force_arch=80`` (default): SM80/Ampere layout, consumed by the SM80 kernel
          (also used on newer GPUs via the compatibility path). Use ``weight_prepacked=1``.
        * ``force_arch=90``: SM90/Hopper layout, consumed by the native SM90 TMA/WGMMA
          kernel. Use ``weight_prepacked=2``. Requires ``block_size`` in {64, 128}.
        r‚   z3force_arch must be 80 (SM80) or 90 (SM90), but got r�   rz   )rV   é@   rI   )rå   rI   zblock_size must be one of z for force_arch=z
, but got r   r    ú#) must be divisible by block_size (ú) for CUDA-prepacked weights.râ   rT   )r
   r&   rZ   rY   r™   rÝ   r’   r`   rŠ   r†   rÁ   r\   r]   rÂ   r‡   )rœ   r%   rÅ   r}   rÆ   rÞ   rÇ   r›   r   Úallowed_block_sizesr{   r|   r¼   rº   rÚ   r   rÃ   s                    r	   Ú(matmulnbits_prepacked_blockwise_quantizez6CudaQuantizer.matmulnbits_prepacked_blockwise_quantizeØ  sÅ  € õ4 ‘”ˆå�4‰yŒyˆÝ˜‘_”_ˆ
Ý˜‘_”_ˆ
Ø˜XÐ%Ð%ÝÐ`ÐS]Ð`Ð`Ð`ÑaÔaÐað 0:¸RÒ/?Ð/?˜m˜mÀYÐØÐ0Ð0Ð0ÝØuÐ-@ÐuÐuÐR\ÐuÐuÐhrÐuÐuÐuñô ð ð Œ}‰ˆˆ1Øˆz‰>˜QÒÐÝÐr 1ÐrÐrÈÐrÐrÐrÑsÔsÐså'4×'YÒ'YØØØØØ!Ø 3ð (Zñ (
ô (
Ñ$ˆ�˜õ ,MÐ(Ø1Ð1°'·/²/À!ÀRÑ2HÔ2H×2NÒ2NÑ2PÔ2PÐRSÐUVÐX\Ð^hÑiÔiˆÝ”˜FÑ#Ô#×(Ò(­¬Ñ2Ô2×:Ò:¸7¼=ÑIÔIˆØ!�Ô!¥"Ô"6°vÑ">Ô">Ñ?Ô?ˆØð 	/Ø˜6 ;Ð.Ð.à�vˆ~Ðr   c               óž  — t          |¦  «        }t          |¦  «        }| j        \  }}	d|z  }
|	|z  dk    rt          d|	› d|› d�¦  «        ‚||
z  dk    rt          d|› d|
› d�¦  «        ‚t                               | |||||¬	¦  «        \  }}}t
          } ||                     |d
¦  «                             ¦   «         ||	||¦  «        }t          j	        |¦  «         
                    t          j        ¦  «                             |	||
z  ¦  «        }t          ¦   «         } |j        t          j        |¦  «        ¦  «        }|r|||fS ||fS )a>  Quantize one expert and CUTLASS-prepack it for CUDA QMoE fpA_intB GEMM.

        ``weights`` has logical shape ``[N, K]``. Returns ``qweight`` with shape
        ``[K, N/pack]`` and block scales with shape ``[N, K/block_size]``.
        Set ``return_zero_points=True`` to also return packed block zero-points.
        r+   r   r    ræ   rç   rÀ   r¡   z! for QMoE blockwise quantization.râ   rT   )r&   rY   rZ   r™   rÝ   r’   r`   rŠ   r†   rÁ   r\   r]   r
   rÂ   r‡   )rœ   r%   rÅ   r}   rÆ   rÞ   rÇ   r›   r{   r|   rŒ   r¼   rº   rÚ   r   rÃ   r   s                    r	   Ú!qmoe_prepacked_blockwise_quantizez/CudaQuantizer.qmoe_prepacked_blockwise_quantize  sy  € õ$ �4‰yŒyˆÝ˜‘_”_ˆ
ØŒ}‰ˆˆ1Ø�D‰yˆØˆz‰>˜QÒÐÝÐr 1ÐrÐrÈÐrÐrÐrÑsÔsÐsØˆt‰8�qŠ=ˆ=ÝÐd 1ÐdÐd¸TÐdÐdÐdÑeÔeÐeå'4×'YÒ'YØØØØØ!Ø 3ð (Zñ (
ô (
Ñ$ˆ�˜õ ,MÐ(Ø1Ð1°'·/²/À!ÀRÑ2HÔ2H×2NÒ2NÑ2PÔ2PÐRSÐUVÐX\Ð^hÑiÔiˆÝ”˜FÑ#Ô#×(Ò(­¬Ñ2Ô2×:Ò:¸1¸aÀ4¹iÑHÔHˆÝ‘”ˆØ!�Ô!¥"Ô"6°vÑ">Ô">Ñ?Ô?ˆØð 	/Ø˜6 ;Ð.Ð.à�vˆ~Ðr   c               óD  — t          ¦   «         }|                      ¦   «                              ¦   «                              ¦   «         } | j        }t          |¦  «        }t          |¦  «        }|dk    r|rdnd\  }}}n$|dk    r|rdnd\  }}}nt          d|› d�¦  «        ‚|d	         }	|	|z   d
z
  |z  }
|
|z  |	z
  }|dk    rFt          |¦  «        }||d	<    |j        || j	        | j
        ¬¦  «        } |j        | |gd	¬¦  «        }n| } |j        g |dd	…         ¢|
‘|‘R Ž } |j         |j        |¦  «        d	¬¦  «        d         } |j        ||z  d¬¦  «        } |j        ||                     d	¦  «        z  ¦  «        } |j        |||¦  «        }|dk    �r |                     |j        ¦  «        j        g |dd	…         ¢|
|z  ‘R Ž }|dk    r|dd| …f         }|dz                        |j        ¦  «        }t          |¦  «        }|d	         d
z   dz  |d	<    |j        ||j        | j
        ¬¦  «        }|dddd…f         dz  |ddd…f<   |j        d	         d
k    r;|dd|dd
dd…f         j        d	         …fxx         |dd
dd…f         dz  dz  z  cc<   nZ |                     |j        ¦  «        j        g |dd	…         ¢|
|z  ‘R Ž }|dk    r|dd| …f         }|                     |¦  «        }|                     ¦   «         |                     ¦   «         fS )aI  Quantize one expert with a pure-PyTorch symmetric blockwise encoding.

        This helper is useful for non-CUDA reference paths. Unlike the pybind-backed
        helpers above, it pads the last dimension when it is not divisible by
        ``block_size`` and returns storage with the same leading shape as ``weights``.
        r*   )r£   r5   r+   )r¥   r5   r5   r+   )r§   rW   rI   )r©   rW   rW   z;CUDA blockwise quantization only supports 4 or 8 bits, got r�   rT   r   r   rƒ   rR   Ng:Œ0âŽyE>r«   .r,   r7   )r
   r­   r!   rf   rY   r&   rZ   ÚlistrÓ   r„   r#   r_   r\   rÍ   r¯   r²   rµ   rX   rb   ra   r]   )rœ   r%   rÅ   r›   r   rp   r¶   r·   r¸   Úlast_dimrÖ   Úpad_sizeÚ	pad_shapeÚpaddingÚweights_paddedÚreshaped_weightsÚblock_max_absrº   r»   Úquantized_flatÚquantized_uint4Úpacked_shaper¼   s                          r	   Úsymmetric_blockwise_quantizez*CudaQuantizer.symmetric_blockwise_quantizeD  s»  € õ ‘”ˆà—.’.Ñ"Ô"×&Ò&Ñ(Ô(×3Ò3Ñ5Ô5ˆØ œˆÝ�4‰yŒyˆÝ˜‘_”_ˆ
Ø�1Š9ˆ9Ø6IÐ(Y¨
¨
ÈzÑ%ˆD�$˜˜Ø�QŠYˆYØ<OÐ(eÐ(8Ð(8ÐUeÑ%ˆD�$˜˜åÐbÐ[_ÐbÐbÐbÑcÔcÐcà! "Ô%ˆØ Ñ+¨aÑ/°JÑ>ˆ
Ø 
Ñ*¨XÑ5ˆØ�aŠ<ˆ<Ý˜^Ñ,Ô,ˆIØ$ˆI�b‰MØ!�e”k )°7´=ÈÌÐXÑXÔXˆGØ&˜UœY¨°Ð'9¸rÐBÑBÔBˆNˆNà$ˆNà.˜>Ô.Ð\°¸sÀ¸sÔ0CÐ\ÀZÐ\ÐQ[Ð\Ð\Ð\ÐØ!˜œ	 ) %¤)Ð,<Ñ"=Ô"=À2ÐFÑFÔFÀqÔIˆØ�”˜]¨]Ñ:ÀÐEÑEÔEˆà�E”KÐ 0°6×3CÒ3CÀBÑ3GÔ3GÑ GÑHÔHˆ	Ø�E”K 	¨4°Ñ6Ô6ˆ	à�1Š9‰9Ø:˜YŸ\š\¨%¬*Ñ5Ô5Ô:Ði¸NÈ3ÈBÈ3Ô<OÐiÐQ[Ð^hÑQhÐiÐiÐiˆNØ˜!Š|ˆ|Ø!/°°Z°x°i°Z°Ô!@�à-°Ñ1×5Ò5°e´kÑBÔBˆOÝ Ñ/Ô/ˆLØ .¨rÔ 2°QÑ 6¸1Ñ<ˆL˜ÑØ!�e”k ,°e´kÈ'Ì.ÐYÑYÔYˆGØ-¨c°1°4°a°4¨iÔ8¸3Ñ>ˆG�C˜˜˜�F‰OØÔ$ RÔ(¨1Ò,Ð,Ø˜ÐC˜¨s°A°D°q°D¨yÔ9Ô?ÀÔCÐCÐCÐDÐDÔDÈÐY\Ð^_Ð^bÐabÐ^bÐYbÔIcÐfiÑIiÐnoÑHoÑoÐDÐDÑDøà3�i—l’l 5¤:Ñ.Ô.Ô3Ðb°^ÀCÀRÀCÔ5HÐbÈ*ÐWaÑJaÐbÐbÐbˆGØ˜!Š|ˆ|Ø! # z¨¨	 z /Ô2�Ø—l’l >Ñ2Ô2ˆGà�{Š{‰}Œ}˜fŸjšj™lœlÐ*Ð*r   N)rœ   r�   r%   r&   r›   r   r   rž   ©rz   )rœ   r�   r%   r&   r¾   r   r}   r&   r›   r   r   rž   )rœ   r�   r%   r&   rÅ   r&   rÆ   r   rÇ   r   r›   r   r   rÈ   )rœ   r�   r%   r&   rÅ   r&   rÆ   r   rÞ   r   rÇ   r   rß   r   r›   r   r   rà   )rœ   r�   r%   r&   rÅ   r&   r}   r&   rÆ   r   rÞ   r   rÇ   r   r›   r   r   rà   )
rœ   r�   r%   r&   rÅ   r&   r›   r   r   rž   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ústaticmethodr½   rÄ   rÝ   rã   ré   rë   rø   r   r   r	   r™   r™     s¯  € € € € € ðð ð ð
 %)ð	1Eð 1Eð 1Eð 1Eð 1Eñ „\ð1Eðf ð
 ð	Fð %)ðFð Fð Fð Fð Fñ „\ðFðB ðAbð Abð Abñ „\ðAbðF ð Ø#(ØØ $Ø$(ð$ð $ð $ð $ð $ñ „\ð$ðL ð
 ð	;ð Ø#(ØØ$(ð;ð ;ð ;ð ;ð ;ñ „\ð;ðz ð
 ð	+ð Ø#(Ø Ø$(ð+ð +ð +ð +ð +ñ „\ð+ðZ ð %)ð>+ð >+ð >+ð >+ð >+ñ „\ð>+ð >+ð >+r   r™   )r   r   )r   r   )r%   r&   r'   r&   rù   )
r{   r&   r|   r&   r%   r&   r}   r&   r   r~   )rý   Ú
__future__r   Ú	functoolsÚloggingÚtypingr   rŠ   r†   Ú	getLoggerrú   r   r   r
   r   r   Ú	lru_cacher   r$   ry   r’   r—   r™   r   r   r	   ú<module>r     sl  ððð ð, #Ð "Ð "Ð "Ð "Ð "à Ð Ð Ð Ø €€€Ø  Ð  Ð  Ð  Ð  Ð  à Ð Ð Ð à
ˆ'Ô
˜HÑ
%Ô
%€àð Ø€L€L€Lðð ð ðsð sð sð.ð ð ð ð €Ô˜QÐÑÔðð ð ñ  Ôððð ð ðn*ð n*ð n*ð n*ðb!)ð !)ð !)ð !)ð !)ðH8ð 8ð 8ðs+ð s+ð s+ð s+ð s+ñ s+ô s+ð s+ð s+ð s+r   