§
    ‚Štjp6  ã                   ó¶   — d dl mZ ddlmZmZmZmZ ddlmZ ddl	m
Z
  e¦   «         rd dlZerddlmZ dd	lmZ  ej        e¦  «        Z G d
„ de¦  «        ZdS )é    )ÚTYPE_CHECKINGé   )Úis_accelerate_availableÚis_torch_availableÚis_torch_xpu_availableÚloggingé   )ÚHfQuantizer)Úget_module_from_nameN)ÚPreTrainedModel)ÚFineGrainedFP8Configc                   óØ   ‡ — e Zd ZU dZdZded<   ˆ fd„Zd„ Zddd	ed
e	fd„Z
ddd	eddd
efˆ fd„Zdd„Z	 	 dd„Zd„ Zd„ Zd„ Zed
e	fd„¦   «         Zed
e	fd„¦   «         Zd„ Zd„ Zd„ Zˆ xZS )ÚFineGrainedFP8HfQuantizerz†
    FP8 quantization implementation supporting both standard and MoE models.
    Supports both e4m3fn formats based on platform.
    Fr   Úquantization_configc                 ó<   •—  t          ¦   «         j        |fi |¤Ž d S )N)ÚsuperÚ__init__)Úselfr   ÚkwargsÚ	__class__s      €úo/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/quantizers/quantizer_finegrained_fp8.pyr   z"FineGrainedFP8HfQuantizer.__init__   s)   ø€ Ø�‰ŒÔÐ,Ð7Ð7°Ð7Ð7Ð7Ð7Ð7ó    c                 óz  — t          ¦   «         st          d¦  «        ‚| j        j        rd S t          j                             ¦   «         sLt          ¦   «         s>| j        r(t           
                    d¦  «         d| j        _        d S t          d¦  «        ‚t          j                             ¦   «         rdt          j                             ¦   «         }|\  }}|dk     s|dk    r5|dk     r/t           
                    d|› d|› d	�¦  «         d| j        _        d S |                     d
¦  «        }|€t           
                    d¦  «         d S t          |t          ¦  «        rU| j        sPt!          |¦  «        dk    r?d|                     ¦   «         v sd|                     ¦   «         v rt%          d¦  «        ‚d S d S d S d S )NzMLoading an FP8 quantized model requires accelerate (`pip install accelerate`)z„Using FP8 quantized models requires a GPU or XPU, we will default to dequantizing the model to bf16 since no GPU or XPU is availableTzANo GPU or XPU found. A GPU or XPU is needed for FP8 quantization.é   é	   ziFP8 quantized models is only supported on GPUs with compute capability >= 8.9 (e.g 4090/H100), actual = `ú.zƒ`. We will default to dequantizing the model to bf16. Feel free to use a different quantization method like bitsandbytes or torchaoÚ
device_mapz×You have loaded an FP8 model on CPU and have a CUDA or XPU device available, make sure to set your model on a GPU or XPU device in order to run your model. To remove this warning, pass device_map = 'cuda' or 'xpu'. r	   ÚcpuÚdiskzìYou are attempting to load an FP8 model with a device_map that contains a cpu/disk device.This is not supported when the model is quantized on the fly. Please use a quantized checkpoint or remove the cpu/disk device from the device_map.)r   ÚImportErrorr   Ú
dequantizeÚtorchÚcudaÚis_availabler   Úpre_quantizedÚloggerÚwarning_onceÚRuntimeErrorÚget_device_capabilityÚgetÚ
isinstanceÚdictÚlenÚvaluesÚ
ValueError)r   Úargsr   Úcompute_capabilityÚmajorÚminorr   s          r   Úvalidate_environmentz.FineGrainedFP8HfQuantizer.validate_environment   s  € Ý&Ñ(Ô(ð 	oÝÐmÑnÔnÐnàÔ#Ô.ð 	ØˆFåŒz×&Ò&Ñ(Ô(ð 	hÕ1GÑ1IÔ1Ið 	hØÔ!ð hÝ×#Ò#ð [ñô ð ð 7;�Ô(Ô3Ø�å"Ð#fÑgÔgÐgåŒ:×"Ò"Ñ$Ô$ð 
	Ý!&¤×!AÒ!AÑ!CÔ!CÐØ-‰LˆE�5Ø˜’	�	˜u¨šz˜z¨e°aªi¨iÝ×#Ò#ð[Ø#(ð[ð [Ø+0ð[ð [ð [ñô ð ð
 7;�Ô(Ô3Ø�à—Z’Z Ñ-Ô-ˆ
ØÐÝ×Òð6ñô ð ð ð õ
 ˜
¥DÑ)Ô)ð 
	àÔ&ð	å˜
‘O”O aÒ'Ð'Ø˜j×/Ò/Ñ1Ô1Ð1Ð1°V¸z×?PÒ?PÑ?RÔ?RÐ5RÐ5Rå ðkñô ð ð
	ð 
	ð	ð 	à'Ð'Ø5RÐ5Rr   Úmodelr   Ú
param_nameÚreturnc                 ó‚   — ddl m}m} t          ||¦  «        \  }}t	          |||f¦  «        r| j        s|dk    rdS dS dS )Nr   )Ú
FP8ExpertsÚ	FP8LinearÚbiasFT)Úintegrations.finegrained_fp8r9   r:   r   r+   r%   )r   r5   r6   r   r9   r:   ÚmoduleÚtensor_names           r   Úparam_needs_quantizationz2FineGrainedFP8HfQuantizer.param_needs_quantizationN   si   € àHÐHÐHÐHÐHÐHÐHÐHå2°5¸*ÑEÔEÑˆ�Ý�f˜y¨*Ð5Ñ6Ô6ð 	ØÔ!ð  [°FÒ%:Ð%:Ø�uà�tØˆur   Úparamztorch.Tensorc                 óz   •— |                       ||¦  «        rdS t          ¦   «                              |||¦  «        S )z4Return the element size (in bytes) for `param_name`.r	   )r?   r   Úparam_element_size)r   r5   r6   r@   r   s       €r   rB   z,FineGrainedFP8HfQuantizer.param_element_sizeZ   s<   ø€ à×(Ò(¨°
Ñ;Ô;ð 	à�1Ý‰wŒw×)Ò)¨%°¸UÑCÔCÐCr   c                 óÖ   — | j         j        }|sdS ddlm}  ||¦  «        }g }|D ]6}|}|D ]}|                     |¦  «        \  }}	Œ|                     |¦  «         Œ7|| j         _        dS )u.  Rewrite the skip-list to the model's own module tree.
        For models that were already released, if they have a list of modules to not quantize
        we need to apply the weight renaming / weight conversion opÃ©rations to get the actual
        layer name of the model in `transformers`.
        Nr   )Úget_model_conversion_mapping)r   Úmodules_to_not_convertÚconversion_mappingrD   Úrename_source_keyÚappend)
r   r5   ÚskiprD   Ú	renamingsÚremappedÚnameÚrenamedÚrenameÚ_s
             r   Ú!_normalize_modules_to_not_convertz;FineGrainedFP8HfQuantizer._normalize_modules_to_not_converta   s¥   € ð Ô'Ô>ˆØð 	ØˆFàEÐEÐEÐEÐEÐEà0Ð0°Ñ7Ô7ˆ	ØˆØð 	%ð 	%ˆDØˆGØ#ð ?ð ?�Ø#×5Ò5°gÑ>Ô>‘
�˜˜Ø�OŠO˜GÑ$Ô$Ð$Ð$Ø:BˆÔ Ô7Ð7Ð7r   c                 óÎ   — ddl m} |                      |¦  «         |                      || j        j        |j        ¦  «        | _         ||| j        | j        | j        ¬¦  «        }d S )Nr   )Úreplace_with_fp8_linear)rE   r   r%   )r<   rR   rP   Úget_modules_to_not_convertr   rE   Ú_keep_in_fp32_modulesr%   )r   r5   r   rR   s       r   Ú$_process_model_before_weight_loadingz>FineGrainedFP8HfQuantizer._process_model_before_weight_loadingv   s…   € ð
 	KÐJÐJÐJÐJÐJà×.Ò.¨uÑ5Ô5Ð5Ø&*×&EÒ&EØ�4Ô+ÔBÀEÔD_ñ'
ô '
ˆÔ#ð (Ð'ØØ#'Ô#>Ø $Ô 8ØÔ,ð	
ñ 
ô 
ˆˆˆr   c           
      óº  — | j         j        dk    r¹ddlm}  |¦   «         }d„ |                     ¦   «         D ¦   «         }|D ]ˆ}|                     d¦  «        \  }}}	|                     |¦  «        }
t          |
|	¦  «        }t          |
|	t          j
                             |j                             |¦  «        d¬¦  «        ¦  «         Œ‰ddlm}  ||¦  «         |S )	NÚue8m0r   )Ú_get_ue8m0_dtypec                 ól   — g | ]1\  }}|                      d ¦  «        r|j        t          j        k    ¯/|‘Œ2S )Ú
_scale_inv)ÚendswithÚdtyper"   Úfloat32)Ú.0rL   r@   s      r   ú
<listcomp>zQFineGrainedFP8HfQuantizer._process_model_after_weight_loading.<locals>.<listcomp>’   sK   € ð ð ð á�D˜%Ø—=’= Ñ.Ô.ðð 49´;Å%Ä-Ò3OÐ3Oð à3OÐ3OÐ3Or   r   F)Úrequires_grad)Ú!_disable_deepgemm_on_multi_device)r   Ú	scale_fmtr<   rX   Únamed_parametersÚ
rpartitionÚget_submoduleÚgetattrÚsetattrr"   ÚnnÚ	ParameterÚdataÚtora   )r   r5   r   rX   rW   Úfloat32_scalesrL   Úmodule_namerO   Úattrr=   Úscalera   s                r   Ú#_process_model_after_weight_loadingz=FineGrainedFP8HfQuantizer._process_model_after_weight_loading‰   s  € ð
 Ô#Ô-°Ò8Ð8ØGÐGÐGÐGÐGÐGà$Ð$Ñ&Ô&ˆEðð à#(×#9Ò#9Ñ#;Ô#;ðñ ô ˆNð
 'ð eð e�Ø'+§¢°sÑ';Ô';Ñ$�˜Q Ø×,Ò,¨[Ñ9Ô9�Ý ¨Ñ-Ô-�Ý˜ ¥e¤h×&8Ò&8¸¼¿ºÀuÑ9MÔ9MÐ]bÐ&8Ñ&cÔ&cÑdÔdÐdÐdð 	UÐTÐTÐTÐTÐTà)Ð)¨%Ñ0Ô0Ð0Øˆr   c                 óX  ‡— d|j         j        v rdddddddddddddddœ}||_        ddlm} t          |dd ¦  «        }|j                             |¦  «        Š‰rOdD ]L}t          ||d ¦  «        pi }ˆfd	„|                     ¦   «         D ¦   «         }||k    rt          |||¦  «         ŒM|S )
NÚQwen3ÚcolwiseÚrowwise)z layers.*.self_attn.q_proj.weightz*layers.*.self_attn.q_proj.weight_scale_invz layers.*.self_attn.k_proj.weightz*layers.*.self_attn.k_proj.weight_scale_invz layers.*.self_attn.v_proj.weightz*layers.*.self_attn.v_proj.weight_scale_invz layers.*.self_attn.o_proj.weightz*layers.*.self_attn.o_proj.weight_scale_invzlayers.*.mlp.gate_proj.weightz'layers.*.mlp.gate_proj.weight_scale_invzlayers.*.mlp.up_proj.weightz%layers.*.mlp.up_proj.weight_scale_invzlayers.*.mlp.down_proj.weightz'layers.*.mlp.down_proj.weight_scale_invr   )r9   Ú_experts_implementation)Úbase_model_tp_planÚbase_model_ep_planc                 óD   •— i | ]\  }}|‰                      ||¦  «        “ŒS © )r*   )r^   ÚkÚvÚlayer_overridess      €r   ú
<dictcomp>z<FineGrainedFP8HfQuantizer.update_tp_plan.<locals>.<dictcomp>Ä   s/   ø€ Ð[Ð[Ð[ÁÀÀA  ?×#6Ò#6°q¸!Ñ#<Ô#<Ð[Ð[Ð[r   )
r   Ú__name__rv   r<   r9   rf   Ú_impl_tp_layer_overridesr*   Úitemsrg   )	r   ÚconfigÚ	text_planr9   ÚimplÚ	plan_attrÚ	base_planÚupdated_planr|   s	           @r   Úupdate_tp_planz(FineGrainedFP8HfQuantizer.update_tp_plan¤   s  ø€ Ø�fÔ&Ô/Ð/Ð/à4=Ø>GØ4=Ø>GØ4=Ø>GØ4=Ø>GØ1:Ø;DØ/8Ø9BØ1:Ø;Dðð ˆIð" )2ˆFÔ%ð 	>Ð=Ð=Ð=Ð=Ð=å�vÐ8¸$Ñ?Ô?ˆØ$Ô=×AÒAÀ$ÑGÔGˆØð 	=ØIð =ð =�	Ý# F¨I°tÑ<Ô<ÐBÀ�	Ø[Ð[Ð[Ð[ÈÏÊÑIZÔIZÐ[Ñ[Ô[�Ø 9Ò,Ð,Ý˜F I¨|Ñ<Ô<Ð<øàˆr   c                 ó   — dS ©NTry   ©r   s    r   Úis_serializablez)FineGrainedFP8HfQuantizer.is_serializableÊ   s   € Øˆtr   c                 ó   — dS )NFry   rŠ   s    r   Úis_trainablez&FineGrainedFP8HfQuantizer.is_trainableÍ   s   € àˆur   c                 ó   — dS r‰   ry   rŠ   s    r   Úis_compileablez(FineGrainedFP8HfQuantizer.is_compileableÑ   s   € àˆtr   c                 ó$   — ddl m}  || ¦  «        S )Nr   )ÚFp8Quantize)r<   r‘   )r   r‘   s     r   Úget_quantize_opsz*FineGrainedFP8HfQuantizer.get_quantize_opsÕ   s$   € Ø>Ð>Ð>Ð>Ð>Ð>àˆ{˜4Ñ Ô Ð r   c                 óz   — ddl m} ddlm} | j        r'| j        j        r |g d¢d || ¦  «        g¬¦  «        gS g S )Nr   )ÚWeightConverter©ÚFp8Dequantize)zweight$Úweight_scale_invÚactivation_scaleÚweight©Úsource_patternsÚtarget_patternsÚ
operations)Úcore_model_loadingr”   r<   r–   r%   r   r!   )r   r”   r–   s      r   Úget_weight_conversionsz0FineGrainedFP8HfQuantizer.get_weight_conversionsÚ   s�   € Ø8Ð8Ð8Ð8Ð8Ð8Ø@Ð@Ð@Ð@Ð@Ð@àÔð 		 $Ô":Ô"Eð 		ð  �Ø$WÐ$WÐ$WØ$,Ø - ¨dÑ 3Ô 3Ð4ðñ ô ðð ð ˆ	r   c                 óx  — ddl m}m} ddlm}  |dd¬¦  «        }|gt          |¦  «        z   }| j        r| j        j        s||  	                    ¦   «         z   S g }|D ]µ}t          ||¦  «        s|                     |¦  «         Œ(d„ |j        D ¦   «         }|red„ |D ¦   «         }	d	„ |D ¦   «         }
d
„ |j        D ¦   «         }|	|
z   |z   } || ¦  «        gt          |j        ¦  «        z   } |||j        |¬¦  «        }|                     |¦  «         Œ¶|                     |  	                    ¦   «         ¦  «         |S )uÕ  When loading with ``dequantize=True``, attach an :class:`Fp8Dequantize` op to
        every existing :class:`WeightConverter` so that per-block scales are folded into
        the weight *before* any later merge/concat ops collapse the per-expert structure.

        For each model-supplied converter that has a ``.weight`` source, we:
          1. anchor the existing weight patterns with ``$`` so they don't accidentally
             also match the ``.weight_scale_inv`` keys (the regex is searched, so the
             unanchored prefix would match both, sending scales to the wrong bucket);
          2. add anchored ``*.weight_scale_inv`` sources next to each weight pattern so
             the loader collects scale tensors alongside the weight tensors into the
             *same* converter bucket (both keys rewrite to the same target);
          3. prepend a fresh :class:`Fp8Dequantize` op so dequant runs first, before
             any merge/concat collapses the per-expert structure.

        The generic ``weight$ + weight_scale_inv â†’ weight`` converter from
        :meth:`get_weight_conversions` is still appended at the end as a fallback for
        plain ``nn.Linear`` weights with no model-specific converter.
        r   )r”   ÚWeightRenamingr•   z^(.+)\.scale$z\1.weight_scale_inv)r›   rœ   c                 ó<   — g | ]}|                      d ¦  «        ¯|‘ŒS ©ú.weight©r[   ©r^   Úps     r   r_   zGFineGrainedFP8HfQuantizer.update_weight_conversions.<locals>.<listcomp>  s)   € ÐWÐWÐW AÀÇÂÈIÑAVÔAVÐW˜aÐWÐWÐWr   c                 ó   — g | ]}|d z   ‘ŒS )ú$ry   r¦   s     r   r_   zGFineGrainedFP8HfQuantizer.update_weight_conversions.<locals>.<listcomp>  s   € Ð"CÐ"CÐ"C¨q 1 s¡7Ð"CÐ"CÐ"Cr   c                 óD   — g | ]}|d t          d¦  «         …         dz   ‘ŒS )Nr¤   z.weight_scale_inv$)r-   r¦   s     r   r_   zGFineGrainedFP8HfQuantizer.update_weight_conversions.<locals>.<listcomp>  s2   € Ð eÐ eÐ eÐQR Ð#4¥c¨)¡n¤n _Ð#4Ô!5Ð8LÑ!LÐ eÐ eÐ er   c                 ó<   — g | ]}|                      d ¦  «        °|‘ŒS r£   r¥   r¦   s     r   r_   zGFineGrainedFP8HfQuantizer.update_weight_conversions.<locals>.<listcomp>  s)   € ÐVÐVÐV˜qÀÇ
Â
È9Ñ@UÔ@UÐV˜ÐVÐVÐVr   rš   )rž   r”   r¡   r<   r–   Úlistr%   r   r!   rŸ   r+   rH   r›   r�   Ú_original_target_patternsÚextend)r   Úweight_conversionsr”   r¡   r–   Úscale_renameÚupdatedÚconvÚweight_sourcesÚanchored_weightÚscale_sourcesÚotherÚnew_sourcesÚnew_opss                 r   Úupdate_weight_conversionsz3FineGrainedFP8HfQuantizer.update_weight_conversionsê   s²  € ð& 	IÐHÐHÐHÐHÐHÐHÐHØ@Ð@Ð@Ð@Ð@Ð@ð &�~Ð6FÐXnÐoÑoÔoˆØ*˜^­dÐ3EÑ.FÔ.FÑFÐàÔ"ð 	F tÔ'?Ô'Jð 	FØ%¨×(CÒ(CÑ(EÔ(EÑEÐEàˆØ&ð 	!ð 	!ˆDõ ˜d OÑ4Ô4ð Ø—’˜tÑ$Ô$Ð$ØØWÐW¨Ô)=ÐWÑWÔWˆNØð 
Ø"CÐ"C°NÐ"CÑ"CÔ"C�Ø eÐ eÐVdÐ eÑ eÔ e�ØVÐV DÔ$8ÐVÑVÔV�Ø-°Ñ=ÀÑE�Ø(˜=¨Ñ.Ô.Ð/µ$°t´Ñ2GÔ2GÑG�Ø&�Ø$/Ø$(Ô$BØ&ðñ ô �ð
 �NŠN˜4Ñ Ô Ð Ð à�Š�t×2Ò2Ñ4Ô4Ñ5Ô5Ð5Øˆr   )r5   r   )r~   Ú
__module__Ú__qualname__Ú__doc__Úrequires_calibrationÚ__annotations__r   r4   ÚstrÚboolr?   ÚfloatrB   rP   rU   rp   r‡   r‹   Úpropertyr�   r�   r’   rŸ   r¹   Ú__classcell__)r   s   @r   r   r      s·  ø€ € € € € € ðð ð
 !ÐØ/Ð/Ð/Ñ/ð8ð 8ð 8ð 8ð 8ð.ð .ð .ð`
Ð.?ð 
ÈSð 
Ð_cð 
ð 
ð 
ð 
ðDÐ(9ð DÀsð DÐSað DÐfkð Dð Dð Dð Dð Dð DðCð Cð Cð Cð*
à ð
ð 
ð 
ð 
ð&ð ð ð6$ð $ð $ðLð ð ð ð˜dð ð ð ñ „Xðð ð ð ð ð ñ „Xðð!ð !ð !ð
ð ð ð 7ð 7ð 7ð 7ð 7ð 7ð 7r   r   )Útypingr   Úutilsr   r   r   r   Úbaser
   Úquantizers_utilsr   r"   Úmodeling_utilsr   Úutils.quantization_configr   Ú
get_loggerr~   r&   r   ry   r   r   ú<module>rË      sø   ðØ  Ð  Ð  Ð  Ð  Ð  à `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ø Ð Ð Ð Ð Ð Ø 2Ð 2Ð 2Ð 2Ð 2Ð 2ð ÐÑÔð Ø€L€L€Làð AØ0Ð0Ð0Ð0Ð0Ð0Ø@Ð@Ð@Ð@Ð@Ð@à	ˆÔ	˜HÑ	%Ô	%€ðOð Oð Oð Oð O ñ Oô Oð Oð Oð Or   