§
    ‚ŠtjÃ  ã                  ó¬  — d dl mZ d dlZd dlZd dlmZ d dlmZ d dlZd dl	m
Z
 d dl	mZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZmZmZmZ ddlmZmZm Z  ddl!m"Z" ddl#m$Z$m%Z% ddl&m'Z'  ej(        e)¦  «        Z*ej+        Z, ej-        e,¦  «        j.        Z/ ej-        e,¦  «        j0        Z1ej2        dYd„¦   «         Z3d„ Z4 ed¬¦  «         G d„ d¦  «        ¦   «         Z5ej2        dZd„¦   «         Z6ej7        j8        d[d„¦   «         Z9dZd„Z:d\d!„Z;	 	 	 	 d]d^d-„Z< ed.d/¬0¦  «        	 	 	 	 d_d`d;„¦   «         Z= ed.d/¬0¦  «        	 	 	 	 	 dadbd>„¦   «         Z> G d?„ d@e
j?        ¦  «        Z@ G dA„ dBe@¦  «        ZAdcdH„ZBdcdI„ZC G dJ„ dKe
jD        ¦  «        ZE G dL„ dMe$¦  «        ZF eF¦   «         ZGdddP„ZH	 dedfdT„ZI G dU„ dVe¦  «        ZJ G dW„ dXe¦  «        ZKdS )gé    )ÚannotationsN)ÚCallable)Ú	dataclass)Ú
functionalé   )ÚACT2FN)ÚConversionOps)Úget_module_from_nameÚshould_convert_module)Úlogging)Údeprecate_kwarg)ÚKERNELS_MAX_VERSIONÚKERNELS_MIN_VERSIONÚis_kernels_availableÚis_torchdynamo_compilingé   )Ú deepgemm_fp8_fp4_experts_forwardÚdeepgemm_fp8_fp4_linearÚ(deepgemm_fp8_fp4_megamoe_experts_forward)Úlazy_load_kernel)ÚExpertsInterfaceÚuse_experts_implementation)Úto_localÚreturnútorch.dtypec                 ó~   — t          t          d¦  «        st          dt          j        › d�¦  «        ‚t          j        S )uS  Return ``torch.float8_e8m0fnu`` or raise a clear error on torch without FP8 support.

    UE8M0 scales are always stored/consumed as this single dtype â€” the kernels (Triton
    finegrained + DeepGEMM) read it natively, and supporting the same scales in mixed
    container dtypes would be a mess â€” so fail loudly rather than fall back.Úfloat8_e8m0fnuzbscale_fmt='ue8m0' requires torch.float8_e8m0fnu, which is only available in PyTorch >= 2.7 (found z.). Upgrade torch to use UE8M0 FP8 checkpoints.)ÚhasattrÚtorchÚRuntimeErrorÚ__version__r   © ó    úg/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/integrations/finegrained_fp8.pyÚ_get_ue8m0_dtyper%   6   sR   € õ •5Ð*Ñ+Ô+ð 
ÝðgÝ%*Ô%6ðgð gð gñ
ô 
ð 	
õ ÔÐr#   c                óœ   — |D ]$}t          | |¦  «        rt          | |¦  «        c S Œ%t          t          | ¦  «        j        › d|› �¦  «        ‚)Nz has none of: )r   ÚgetattrÚAttributeErrorÚtypeÚ__name__)ÚobjÚnamesÚnames      r$   Ú_first_attrr.   E   sc   € Øð &ð &ˆÝ�3˜ÑÔð 	&Ý˜3 Ñ%Ô%Ð%Ð%Ð%ð	&å
�D ™IœIÔ.ÐEÐE¸eÐEÐEÑ
FÔ
FÐFr#   T)Úfrozenc                  ó2   — e Zd ZU dZded<   ded<   ded<   dS )ÚFineGrainedFP8zNEntry points exposed by the `kernels-community/finegrained-fp8` Triton kernel.r   ÚmatmulÚbatched_matmulÚgrouped_matmulN)r*   Ú
__module__Ú__qualname__Ú__doc__Ú__annotations__r"   r#   r$   r1   r1   L   s<   € € € € € € àXÐXàÐÐÑØÐÐÑØÐÐÑÐÐr#   r1   c                 óþ  — t          ¦   «         s6t          ¦   «         s(t          dt          › dt          › dt          › d�¦  «        ‚t          d¦  «        } | €t          d¦  «        ‚t          | dd¦  «        }t          | d	d¦  «        }t          | d
d¦  «        }d„ d|fd	|fd
|ffD ¦   «         }|r>t          dd                     |¦  «        › dt          › dt          › dt          › d�	¦  «        ‚t          |||¬¦  «        S )zÈ
    Load the finegrained-fp8 Triton kernel once and return its entry points.

    Raises `ImportError` if the `kernels` package is missing, or the kernel or required
    symbols cannot be found.
    z\finegrained-fp8 kernel requires the `kernels` package. Please install a compatible version (z <= version < z), e.g. `pip install kernels==ú`zfinegrained-fp8Nu‰   Failed to load the finegrained-fp8 kernel â€” check that `kernels-community/finegrained-fp8` has a build matching the current torch/CUDA.Ú	matmul_2dÚmatmul_batchedÚmatmul_groupedc                ó   — g | ]	\  }}|­|‘Œ
S ©Nr"   )Ú.0r-   Úattrs      r$   ú
<listcomp>z0_load_finegrained_fp8_kernel.<locals>.<listcomp>p   s-   € ð ð ð áˆD�$ð
 ˆ<ð 	ð ˆ<ˆ<r#   z4finegrained-fp8 kernel is missing required symbols: ú, z'. Please install a compatible version ()r2   r3   r4   )	r   r   ÚImportErrorr   r   r   r'   Újoinr1   )Úkernelr2   r3   r4   Úmissings        r$   Ú_load_finegrained_fp8_kernelrH   U   s™  € õ $Ñ%Ô%ð Ý#Ñ%Ô%ð 	ÝðEÝ8KðEð EÝ[nðEð Eå.AðEð Eð Eñô ð õ Ð/Ñ0Ô0€FØ€~Ýð;ñ
ô 
ð 	
õ
 �V˜[¨$Ñ/Ô/€FÝ˜VÐ%5°tÑ<Ô<€NÝ˜VÐ%5°tÑ<Ô<€Nðð ð ˜&Ð!Ø˜~Ð.Ø˜~Ð.ð
ðñ ô €Gð ð 
ÝðAÀ4Ç9Â9ÈWÑCUÔCUð Að AÝ4GðAð AÝWjðAð Aå*=ðAð Að Añ
ô 
ð 	
õ ØØ%Ø%ðñ ô ð r#   ÚNonec                 ó"   — t          ¦   «         } d S r?   )rH   )Ú_s    r$   Ú _populate_finegrained_fp8_kernelrL   ‡   s   € å$Ñ&Ô&€AØˆ4r#   c                 óV   — t          ¦   «         rt          ¦   «          t          ¦   «         S r?   )r   rL   rH   r"   r#   r$   Úload_finegrained_fp8_kernelrN   �   s(   € ÝÑ!Ô!ð +Ý(Ñ*Ô*Ð*Ý'Ñ)Ô)Ð)r#   ÚaÚintÚbc                ó   — | |z   dz
  |z  S )zCeiling division.r   r"   )rO   rQ   s     r$   Ú_cdivrS   “   s   € à�‰E�A‰I˜!ÑÐr#   Únum_expertsÚproj_outÚproj_inÚweight_dtypeÚsf_dtypeÚweight_k_divÚ	sf_gran_nú
int | NoneÚ	sf_gran_kÚ
min_sf_outú!tuple[nn.Parameter, nn.Parameter]c	                ó|  — t          j        | |||z  |¬¦  «        }	t          j        |	|	                     ¦   «         ¬¦  «        }
t          |�t          ||¦  «        nd|¦  «        }|�t          ||¦  «        nd}t          j        | |||¬¦  «        }t          j        ||                     ¦   «         ¬¦  «        }|
|fS )u×  Allocate `(weight, weight_scale_inv)` parameters for one expert projection.

    `weight_k_div` halves the K dim for FP4-packed storage (2 e2m1 values per byte).
    `sf_gran_n` / `sf_gran_k` set per-block (None â†’ per-row/per-tensor) SF granularity.
    `min_sf_out` floors the SF tensor's output dim â€” used by the fused gate_up
    projection to keep room for both halves (pass `2`) even when `proj_out < sf_gran_n`
    would otherwise collapse the SF dim to 1.
    ©Údtype©Úrequires_gradNr   )r   ÚemptyÚnnÚ	ParameterÚis_floating_pointÚmaxrS   )rT   rU   rV   rW   rX   rY   rZ   r\   r]   Úweight_tÚweightÚsf_outÚsf_inÚsf_tÚsfs                  r$   Ú_alloc_expert_projro   ˜   sÀ   € õ& Œ{˜;¨°'¸\Ñ2IÐQ]Ð^Ñ^Ô^€HÝŒ\˜(°(×2LÒ2LÑ2NÔ2NÐOÑOÔO€FÝ¨yÐ/D•�x Ñ+Ô+Ð+È!ÈZÑXÔX€FØ)2Ð)>�E�'˜9Ñ%Ô%Ð%ÀA€EÝŒ;�{ F¨E¸ÐBÑBÔB€DÝ	Œ�d¨$×*@Ò*@Ñ*BÔ*BÐ	CÑ	CÔ	C€BØ�2ˆ:Ðr#   Úoutput_dtypezv5.16)ÚversionÚinputútorch.Tensorrj   Úweight_scale_invÚ
block_sizeúlist[int] | NoneÚbiasútorch.Tensor | NoneÚactivation_scaleútorch.dtype | Nonec                ó�   — t          ¦   «         }|                     | |||| j        |¬¦  «        }|�|                     |¦  «         |S )u0  Triton FP8/FP4 linear: fused act-quant + matmul, then optional bias add.

    ``activation_scale=None`` â†’ dynamic per-K-block scales (inline); set it for
    static per-tensor quant. ``weight_scale_inv`` accepts fp32 or UE8M0; the
    dispatcher routes FP4 (``int8``-packed) weights automatically.
    ©ry   )rN   r2   ra   Úadd_)	rr   rj   rt   ru   rw   ry   rp   Úfinegrained_fp8Úoutputs	            r$   Úfinegrained_fp8_linearr€   ´   s\   € õ  2Ñ3Ô3€OØ×#Ò#ØØØØØŒØ)ð $ñ ô €Fð ÐØ�Š�DÑÔÐØ€Mr#   Úallow_deepgemmÚboolc                óð  — |o“|du o�|j         j        dk    ot          j                             ¦   «         j        dk    oX|j        t          j        k    p|duo|d         |d         cxk    odk    nc o#t          j	         
                    dd¦  «        d	k    }|rK	 t          | |||||¬
¦  «        S # t          $ r(}	t                               d|	› d�¦  «         Y d}	~	nd}	~	ww xY wt          | |||||¦  «        S )u  End-to-end FP8/FP4 linear used by `FP8Linear` and the eager `FP8Experts` loop.

    Dispatch order â€” both backends handle FP8 and FP4 weights with fp32 or UE8M0 scales:
      1. DeepGEMM (`deepgemm_fp8_fp4_linear`) â€” 3-6Ã— faster on the shapes it supports.
         Preferred for FP4, UE8M0 SFs, and 128Ã—128 block FP8.
      2. Triton finegrained-fp8 fallback â€” used when DeepGEMM is unavailable, when the
         caller passes ``activation_scale`` (DeepGEMM is dynamic-only), or for any
         shape DeepGEMM declined.

    Args:
        input: (..., K) bf16/fp16 activations.
        weight: (N, K) `float8_e4m3fn` or (N, K // 2) `int8` (FP4-packed).
        weight_scale_inv: per-block weight scales â€” `float32` (V3-style) or `float8_e8m0fnu`
            (V4-style; reinterpreted as int32 at the DeepGEMM kernel boundary).
        block_size: [block_n, block_k] for FP8 block-wise quant, or None/[N, K] for per-tensor.
            Ignored for FP4 weights (the kernel infers SF granularity from the dtype).
        bias: optional bias added to the matmul output.
        activation_scale: pass a per-tensor scalar to use static activation quant; leave `None`
            for dynamic (per-token) quant.
        allow_deepgemm: set ``False`` to force the Triton fallback for this call. Used when the
            model spans multiple CUDA devices in one process â€” DeepGEMM's cached kernels are bound
            to a single CUDA context and produce garbage across devices (see the multi-device guard
            in ``quantizer_finegrained_fp8.py``).
    NÚcudaé	   r   r   é€   Ú$TRANSFORMERS_DISABLE_DEEPGEMM_LINEARÚ0Ú1)ru   ry   rw   zDDeepGEMM unavailable for this call, falling back to Triton. Reason: zW Set `TRANSFORMERS_DISABLE_DEEPGEMM_LINEAR=1` to skip DeepGEMM for FP8 linear entirely.)Údevicer)   r   r„   Úget_device_propertiesÚmajorra   Úint8ÚosÚenvironÚgetr   rD   ÚloggerÚwarning_oncer€   )
rr   rj   rt   ru   rw   ry   rp   r�   Údeepgemm_preferredÚes
             r$   Ú
fp8_linearr•   Ò   s  € ðV 	ð 	OØ Ð$ð	OàŒMÔ &Ò(ð	Oõ ŒJ×,Ò,Ñ.Ô.Ô4¸Ò9ð	Oð Œ\�UœZÒ'Ðm¨J¸dÐ,BÐ,lÀzÐRSÄ}ÐXbÐcdÔXeÐGlÐGlÒGlÐGlÐilÒGlÐGlÐGlÐGlð		Oõ
 ŒJ�NŠNÐAÀ3ÑGÔGÈ3ÒNð ð ð ð	Ý*ØØØ Ø%Ø!1Øðñ ô ð øõ ð 	ð 	ð 	õ ×ÒðiÐWXð ið ið iñô ð ð ð ð ð ð øøøøð	øøøõ " %¨Ð1AÀ:ÈtÐUeÑfÔfÐfs   ÂB/ Â/
C!Â9CÃC!c                  ó6   ‡ — e Zd ZdZ	 	 	 	 ddˆ fd„Zdd„Zˆ xZS )Ú	FP8LinearFNÚdynamicÚfloatÚin_featuresrP   Úout_featuresru   útuple[int, int] | NoneÚactivation_schemeÚstrÚ	scale_fmtÚhas_biasr‚   c                óî  •— t          ¦   «                              ||¦  «         || _        || _        || _        t
          j                             t          j        ||t          ¬¦  «        ¦  «        | _
        | j        €8t          j        t          j        dt
          j        ¬¦  «        ¦  «        | _        n—|dk    rt          ¦   «         nt
          j        }|| j        d         z   dz
  | j        d         z  }|| j        d         z   dz
  | j        d         z  }	t          j        t          j        ||	|¬¦  «        |j        ¬¦  «        | _        | j        dk    r8t          j        t          j        dt
          j        ¬¦  «        ¦  «        | _        n|                      dd ¦  «         | j        r2t          j        t          j        | j        ¦  «        ¦  «        | _        d S |                      d	d ¦  «         d S )
Nr`   ç      ð?Úue8m0r   r   rb   Ústaticry   rw   )ÚsuperÚ__init__r    ru   r�   r   re   rf   rd   Ú
_FP8_DTYPErj   ÚtensorÚfloat32rt   r%   rg   ry   Úregister_parameterr›   rw   )Úselfrš   r›   ru   r�   rŸ   r    rX   Úscale_out_featuresÚscale_in_featuresÚ	__class__s             €r$   r¦   zFP8Linear.__init__   s»  ø€ õ 	‰Œ×Ò˜ lÑ3Ô3Ð3à ˆŒØ$ˆŒØ!2ˆÔÝ”h×(Ò(­¬°\À;ÕV`Ð)aÑ)aÔ)aÑbÔbˆŒàŒ?Ð"å$&¤Lµ´¸cÍÌÐ1WÑ1WÔ1WÑ$XÔ$XˆDÔ!Ð!à-6¸'Ò-AÐ-AÕ'Ñ)Ô)Ð)ÅuÄ}ˆHØ".°´ÀÔ1CÑ"CÀaÑ"GÈDÌOÐ\]ÔL^Ñ!^ÐØ!,¨t¬¸qÔ/AÑ!AÀAÑ!EÈ$Ì/ÐZ[ÔJ\Ñ \ÐÝ$&¤LÝ”Ð.Ð0AÈÐRÑRÔRØ&Ô8ð%ñ %ô %ˆDÔ!ð
 Ô! XÒ-Ð-Ý$&¤Lµ´¸cÍÌÐ1WÑ1WÔ1WÑ$XÔ$XˆDÔ!Ð!à×#Ò#Ð$6¸Ñ=Ô=Ð=àŒ=ð 	2Ýœ¥U¤[°Ô1BÑ%CÔ%CÑDÔDˆDŒIˆIˆIà×#Ò# F¨DÑ1Ô1Ð1Ð1Ð1r#   rr   rs   r   c           	     ó"  — | j                              ¦   «         dk    r t          j        || j         | j        ¦  «        S t          | j         ¦  «        }t          | j        ¦  «        }t          |||| j        | j	        | j        | j
         ¬¦  «        S )Nr   )ru   ry   rw   r�   )rj   Úelement_sizeÚFÚlinearrw   r   rt   r•   ru   ry   Ú_deepgemm_disabled)r«   rr   rj   Ú	scale_invs       r$   ÚforwardzFP8Linear.forwardF  s‹   € ØŒ;×#Ò#Ñ%Ô%¨Ò)Ð)Ý”8˜E 4¤;°´	Ñ:Ô:Ð:å˜$œ+Ñ&Ô&ˆÝ˜TÔ2Ñ3Ô3ˆ	åØØØØ”Ø!Ô2Ø”Ø#Ô6Ð6ð
ñ 
ô 
ð 	
r#   ©Nr˜   r™   F)rš   rP   r›   rP   ru   rœ   r�   rž   rŸ   rž   r    r‚   )rr   rs   r   rs   )r*   r5   r6   r³   r¦   rµ   Ú__classcell__©r®   s   @r$   r—   r—     sm   ø€ € € € € ð Ðð .2Ø!*Ø Øð$2ð $2ð $2ð $2ð $2ð $2ð $2ðL
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r#   r—   c                  ó6   ‡ — e Zd ZdZ	 	 	 	 ddˆ fd„Zdd„Zˆ xZS )ÚFP8GroupedLinearuï  FP8 drop-in for block-diagonal grouped linears.

    The underlying nn.Linear stores a single `(n_groups * out_per_group, in_per_group)`
    weight; logically that's `n_groups` independent `(out_per_group, in_per_group)`
    sub-matrices, each consuming a disjoint slice of the input's last-but-one dim.
    Forward expects input of shape `(..., n_groups, in_per_group)` and returns
    `(..., n_groups, out_per_group)` â€” same contract as the vanilla bf16 grouped
    linear it replaces.

    Nr˜   r™   FÚin_features_per_grouprP   r›   Ún_groupsru   rœ   r�   rž   rŸ   r    r‚   c                ód   •— t          ¦   «                              ||||||¬¦  «         || _        d S )N©rš   r›   ru   r�   rŸ   r    )r¥   r¦   r¼   )	r«   r»   r›   r¼   ru   r�   rŸ   r    r®   s	           €r$   r¦   zFP8GroupedLinear.__init__d  sE   ø€ õ 	‰Œ×ÒØ-Ø%Ø!Ø/ØØð 	ñ 	
ô 	
ð 	
ð !ˆŒˆˆr#   Úxrs   r   c                óÂ  — |j         d d…         }|j         d         }| j                             ¦   «         dk    rà| j                             | j        d|¦  «                             dd¦  «        }|                     d| j        |¦  «                             dd¦  «        }t          j        ||¦  «                             dd¦  «        } |j        g |¢| j        ‘d‘R Ž }| j	        r3| 
                    | j                             | j        d¦  «        ¦  «         |S t          | j        ¦  «        }t          | j        ¦  «        }|                     | j        d|¦  «        }|                     dd¦  «                             d|¦  «        }|                     | j        |                     d¦  «        | j        z  |                     d¦  «        ¦  «        }|                     d¦  «        | j        z  }t          j        | j        f||j        t          j        ¬¦  «        }t          j        d| j        dz   |j        t          j        ¬¦  «        |z  }	t)          ¦   «         }
|
                     ||||	|| j        ¬¦  «        } |j        | j        g|¢d‘R Ž                      dd¦  «        }| j	        r3| 
                    | j                             | j        d¦  «        ¦  «         |S )Néþÿÿÿéÿÿÿÿr   r   r   )rŠ   ra   ©ÚoffsetsÚtokens_per_expertru   )Úshaperj   r°   Úviewr¼   Ú	transposeÚreshaper   Úbmmr    r}   rw   r   rt   ÚmovedimÚsizeÚfullrŠ   Úint32ÚarangerN   r4   ru   )r«   r¿   Úinput_shapeÚ
hidden_dimÚwÚyr´   Útokens_per_grouprÅ   rÄ   r~   s              r$   rµ   zFP8GroupedLinear.forwardx  s”  € Ø”g˜c˜r˜c”lˆØ”W˜R”[ˆ
àŒ;×#Ò#Ñ%Ô%¨Ò)Ð)Ø”× Ò  ¤°°JÑ?Ô?×IÒIÈ!ÈQÑOÔOˆAØ—	’	˜"˜dœm¨ZÑ8Ô8×BÒBÀ1ÀaÑHÔHˆAÝ”	˜!˜Q‘”×)Ò)¨!¨QÑ/Ô/ˆAØ�”	Ð:˜;Ð:¨¬Ð:°rÐ:Ð:Ð:ˆAØŒ}ð :Ø—’�t”y—~’~ d¤m°RÑ8Ô8Ñ9Ô9Ð9ØˆHå�T”[Ñ!Ô!ˆÝ˜TÔ2Ñ3Ô3ˆ	à�FŠF�4”= " jÑ1Ô1ˆØ�IŠI�b˜!ÑÔ×$Ò$ R¨Ñ4Ô4ˆØ—N’N 4¤=°)·.².ÀÑ2CÔ2CÀtÄ}Ñ2TÐV_×VdÒVdÐefÑVgÔVgÑhÔhˆ	àŸ6š6 !™9œ9¨¬Ñ5ÐÝ!œJ¨¬Ð'7Ð9IÐRSÔRZÕbgÔbmÐnÑnÔnÐÝ”,˜q $¤-°!Ñ"3¸A¼HÍEÌKÐXÑXÔXÐ[kÑkˆå5Ñ7Ô7ˆØ×*Ò*ØØØØØ/Ø”ð +ñ 
ô 
ˆð ˆAŒI�d”mÐ6 kÐ6°2Ð6Ð6Ð6×>Ò>¸qÀ"ÑEÔEˆØŒ=ð 	6Ø�FŠF�4”9—>’> $¤-°Ñ4Ô4Ñ5Ô5Ð5Øˆr#   r¶   )r»   rP   r›   rP   r¼   rP   ru   rœ   r�   rž   rŸ   rž   r    r‚   )r¿   rs   r   rs   )r*   r5   r6   r7   r¦   rµ   r·   r¸   s   @r$   rº   rº   X  sn   ø€ € € € € ð	ð 	ð  .2Ø!*Ø Øð!ð !ð !ð !ð !ð !ð !ð($ð $ð $ð $ð $ð $ð $ð $r#   rº   r«   útorch.nn.ModuleÚhidden_statesÚtop_k_indexÚtop_k_weightsc                ó\  — | j         dk    rt          d¦  «        ‚t          ¦   «         }|                     d¦  «        }|                     d¦  «        }|                     d¦  «        }|                     |d¬¦  «        }|                     d¦  «        }	|                     d¦  «        }
|
| j        k                         d¦  «        }t          | j	        r| j
        n| j        ¦  «        }t          | j	        r| j        n| j        ¦  «        }t          | j        ¦  «        }t          | j        ¦  «        }|                     |||| j        |
¬¦  «        }| j	        r|                      |¦  «        }n|                      |¦  «        }|                     |||| j        |
¬¦  «        }||	                     |j        ¦  «                             d¦  «        z  }|                     |d¦  «         |                     |||¦  «                             d¬¦  «        }|                     |j        ¦  «        S )	Nr¤   z‘batched_mm experts dispatch does not support activation_scheme='static'. Use the default eager dispatch or switch to activation_scheme='dynamic'.rÂ   r   ©Údim)ru   Ú
expert_idsç        r   )r�   ÚNotImplementedErrorrN   rÌ   Úrepeat_interleaverÉ   rT   Ú	unsqueezer   Úhas_gateÚgate_up_projÚup_projÚgate_up_proj_scale_invÚup_proj_scale_invÚ	down_projÚdown_proj_scale_invr3   ru   Ú_apply_gateÚact_fnÚtora   Úmasked_fill_rÇ   Úsum)r«   rÖ   r×   rØ   r~   Ú	num_top_kÚ
num_tokensrÑ   Úselected_hidden_statesÚsample_weightsrÜ   Úsentinel_maskÚ	weight_upÚweight_scale_upÚweight_downÚweight_scale_downrU   Úweighted_outÚfinal_hidden_statess                      r$   Úfp8_batched_mm_experts_forwardrø   Ÿ  s?  € ð Ô Ò)Ð)Ý!ðWñ
ô 
ð 	
õ
 2Ñ3Ô3€Oà× Ò  Ñ$Ô$€IØ×#Ò# AÑ&Ô&€JØ×#Ò# BÑ'Ô'€Jð +×<Ò<¸YÈAÐ<ÑNÔNÐØ"×*Ò*¨2Ñ.Ô.€NØ×$Ò$ RÑ(Ô(€Jð
   4Ô#3Ò3×>Ò>¸rÑBÔB€Må¨d¬mÐM˜Ô*Ð*ÀÄÑNÔN€IÝ¸d¼mÐg˜tÔ:Ð:ÐQUÔQgÑhÔh€OÝ˜4œ>Ñ*Ô*€KÝ  Ô!9Ñ:Ô:Ðð ×-Ò-ØØØØ”?Øð .ñ ô €Hð „}ð )à×#Ò# HÑ-Ô-ˆˆð —;’;˜xÑ(Ô(ˆð ×-Ò-ØØØØ”?Øð .ñ ô €Hð ˜n×/Ò/°´Ñ?Ô?×IÒIÈ"ÑMÔMÑM€Lð ×Ò˜m¨SÑ1Ô1Ð1ð '×+Ò+¨J¸	À:ÑNÔN×RÒRÐWXÐRÑYÔYÐà×!Ò! -Ô"5Ñ6Ô6Ð6r#   c                ó  — | j         dk    rt          d¦  «        ‚t          ¦   «         }|j        }|                     d¦  «        }|                     d¦  «        }|                     d¦  «        }|                     d¦  «        }	|                     d¦  «        }
t          j        |
¦  «        \  }}|||z           }|	|         }|j        dk    r| 	                    ¦   «         n| 
                    ¦   «         }t          j        || j        d| j        dz
  ¬¦  «        }t          j        |dt          j        ¬¦  «        }|| j        k                         d¦  «        }t!          | j        r| j        n| j        ¦  «        }t!          | j        r| j        n| j        ¦  «        }t!          | j        ¦  «        }t!          | j        ¦  «        }|                     |||||| j        ¬	¦  «        }| j        r|                      |¦  «        }n|                      |¦  «        }|                     |||||| j        ¬	¦  «        }||                     |j        ¦  «                             d¦  «        z  }|                     |d
¦  «         t          j        |¦  «        }t          j         |                     d¦  «        |¬¦  «        ||<   ||         }| !                    |||¦  «         "                    d¬¦  «        }|                     |j        ¦  «        S )Nr¤   z‘grouped_mm experts dispatch does not support activation_scheme='static'. Use the default eager dispatch or switch to activation_scheme='dynamic'.rÂ   r   Úcpur   )ÚbinsÚminrh   )rÛ   ra   rÃ   rÝ   )rŠ   rÚ   )#r�   rÞ   rN   rŠ   rÌ   rÉ   r   Úsortr)   r™   rP   ÚhistcrT   ÚcumsumrÎ   rà   r   rá   râ   rã   rä   rå   ræ   rç   r4   ru   rè   ré   rê   ra   rë   Ú
empty_likerÏ   rÇ   rì   )r«   rÖ   r×   rØ   r~   rŠ   rí   rî   rÑ   rð   rÜ   Úexpert_ids_gÚpermÚselected_hidden_states_gÚsample_weights_gÚhistc_inputrÅ   rÄ   rñ   rò   ró   rô   rõ   rU   rö   Úinv_permr÷   s                              r$   Úfp8_grouped_mm_experts_forwardr  é  s  € ð Ô Ò)Ð)Ý!ðWñ
ô 
ð 	
õ
 2Ñ3Ô3€OàÔ!€FØ× Ò  Ñ$Ô$€IØ×#Ò# AÑ&Ô&€JØ×#Ò# BÑ'Ô'€Jð #×*Ò*¨2Ñ.Ô.€NØ×$Ò$ RÑ(Ô(€Jõ œ JÑ/Ô/Ñ€L�$Ø,¨T°YÑ->Ô?ÐØ% dÔ+Ðð
 +1¬+¸Ò*>Ð*>�,×$Ò$Ñ&Ô&Ð&ÀL×DTÒDTÑDVÔDV€KÝœ K°dÔ6FÈAÐSWÔScÐfgÑSgÐhÑhÔhÐÝŒlÐ,°!½5¼;ÐGÑGÔG€Gð " TÔ%5Ò5×@Ò@ÀÑDÔD€Må¨d¬mÐM˜Ô*Ð*ÀÄÑNÔN€IÝ¸d¼mÐg˜tÔ:Ð:ÐQUÔQgÑhÔh€OÝ˜4œ>Ñ*Ô*€KÝ  Ô!9Ñ:Ô:Ðð ×-Ò-Ø ØØØØ+Ø”?ð .ñ ô €Hð „}ð )à×#Ò# HÑ-Ô-ˆˆð —;’;˜xÑ(Ô(ˆð ×-Ò-ØØØØØ+Ø”?ð .ñ ô €Hð Ð.×1Ò1°(´.ÑAÔA×KÒKÈBÑOÔOÑO€Lð ×Ò˜m¨SÑ1Ô1Ð1õ Ô Ñ%Ô%€HÝ”\ $§)¢)¨A¡,¤,°vÐ>Ñ>Ô>€HˆT�NØ Ô)€Lð '×+Ò+¨J¸	À:ÑNÔN×RÒRÐWXÐRÑYÔYÐà×!Ò! -Ô"5Ñ6Ô6Ð6r#   c                  óf   ‡ — e Zd ZU dZddddœiZded<   	 	 	 	 	 d#d$ˆ fd„Zd%d„Zd&d„Z	 d'd(d"„Z	ˆ xZ
S ))Ú
FP8ExpertsFÚdeepgemm_megamoeÚmegamoe_expertsÚmegamoe_router)Úmoe_tp_expertsÚ	ep_routerzdict[str, dict[str, str]]Ú_impl_tp_layer_overridesNr˜   r™   Tru   rœ   r�   rž   rŸ   r    r‚   rá   c                ó^  •— t          ¦   «                              ¦   «          |du s
J d¦   «         ‚|| _        || _        || _        || _        |j        | _        || _        t          |dd¦  «        | _
        t          |dd¦  «        | _        t          |dd ¦  «        | _        t          |dd ¦  «        | _        t          t          |d	d
¦  «                 | _        t          |dd ¦  «        | _        t          |dd¦  «        dk    }|dk    rt%          ¦   «         nt&          j        }|rt&          j        |ddddœ}	n t,          ||�|d         nd |�|d         nd dœ}	| j        rGt/          | j
        d| j        z  | j        fddi|	¤Ž\  | _        | _        |                      dd ¦  «         nAt/          | j
        | j        | j        fi |	¤Ž\  | _        | _        |                      dd ¦  «         t/          | j
        | j        | j        fi |	¤Ž\  | _        | _        |                      dd ¦  «         | j        dk    rzt?          j         t'          j!        | j
        t&          j        ¬¦  «        ¦  «        | _"        t?          j         t'          j!        | j
        t&          j        ¬¦  «        ¦  «        | _#        d S d S )NFzWFP8Experts does not support bias for now, please open an issue if you want this featureÚnum_local_expertsrT   Úmoe_intermediate_sizeÚintermediate_sizeÚswiglu_alphaÚswiglu_limitÚhidden_activationÚ
hidden_actÚexpert_dtypeÚfp8Úfp4r£   r   r   é    )rW   rX   rY   rZ   r\   r   )rW   rX   rZ   r\   r]   Úgate_up_proj_biasÚup_proj_biasÚdown_proj_biasr¤   r`   )$r¥   r¦   Úconfigr    rá   ru   Úhidden_sizerÑ   r�   r.   rT   Úintermediate_dimr'   r  r  r   ré   Úlimitr%   r   r©   r�   r§   ro   râ   rä   rª   rã   rå   ræ   rç   re   rf   ÚonesÚgate_up_proj_activation_scaleÚdown_proj_activation_scale)r«   r  ru   r�   rŸ   r    rá   Úis_fp4rX   Úalloc_kwargsr®   s             €r$   r¦   zFP8Experts.__init__[  sè  ø€ õ 	‰Œ×ÒÑÔÐà˜5Ð Ð Ð Øeñ !Ô Ð ð ˆŒØ ˆŒØ ˆŒØ$ˆŒØ Ô,ˆŒØ!2ˆÔÝ& vÐ/BÀMÑRÔRˆÔÝ +¨FÐ4KÐM`Ñ aÔ aˆÔÝ# F¨N¸DÑAÔAˆÔÝ# F¨N¸DÑAÔAˆÔÝ�[¨Ð1DÀlÑSÔSÔTˆŒÝ˜V ^°TÑ:Ô:ˆŒ
õ ˜ °Ñ7Ô7¸5Ò@ˆØ)2°gÒ)=Ð)=Õ#Ñ%Ô%Ð%Å5Ä=ˆØð 	å %¤
Ø$Ø !ØØðð ˆLˆLõ !+Ø$Ø.8Ð.D˜Z¨œ]˜]È$Ø.8Ð.D˜Z¨œ]˜]È$ð	ð ˆLð Œ=ð 		:Ý=OØÔ  ! dÔ&;Ñ";¸T¼_ð>ð >ØYZð>Ø^jð>ð >Ñ:ˆDÔ˜tÔ:ð ×#Ò#Ð$7¸Ñ>Ô>Ð>Ð>å3EØÔ  $Ô"7¸¼ð4ð 4ØLXð4ð 4Ñ0ˆDŒL˜$Ô0ð ×#Ò# N°DÑ9Ô9Ð9å3EØÔ˜dœo¨tÔ/Dð4
ð 4
ØHTð4
ð 4
Ñ0ˆŒ˜Ô0ð 	×ÒÐ 0°$Ñ7Ô7Ð7àÔ! XÒ-Ð-Ý13´½e¼jÈÔIYÕafÔanÐ>oÑ>oÔ>oÑ1pÔ1pˆDÔ.Ý.0¬l½5¼:ÀdÔFVÕ^cÔ^kÐ;lÑ;lÔ;lÑ.mÔ.mˆDÔ+Ð+Ð+ð .Ð-r#   Úgate_uprs   r   c                óÄ  — |                      dd¬¦  «        \  }}| j        �d|                     | j        ¬¦  «        }|                     | j         | j        ¬¦  «        }|t	          j        || j        z  ¦  «        z  }|dz   |z  S | j        �=|                     | j        ¬¦  «        }|                     | j         | j        ¬¦  «        }|                      |¦  «        |z  S )Nr   rÂ   rÚ   )rh   ©rü   rh   r¢   )Úchunkr  Úclampr  r   Úsigmoidr"  ré   )r«   r(  ÚgateÚupÚglus        r$   rè   zFP8Experts._apply_gate¡  sÒ   € Ø—=’= ¨�=Ñ+Ô+‰ˆˆbØÔÐ(à—:’: $Ô"3�:Ñ4Ô4ˆDØ—’˜tÔ0Ð0°dÔ6G�ÑHÔHˆBØ�œ t¨dÔ.?Ñ'?Ñ@Ô@Ñ@ˆCØ˜‘H Ñ#Ð#ØŒZÐ#Ø—:’: $¤*�:Ñ-Ô-ˆDØ—’˜tœz˜k¨t¬z�Ñ:Ô:ˆBØ�{Š{˜4Ñ Ô  2Ñ%Ð%r#   rÖ   r×   rØ   c                óê  — t          j        |t           j        ¬¦  «        }t          j        ¦   «         5  t           j        j                             || j        dz   ¬¦  «        }|                     ddd¦  «        }t          j	        | 
                    d¬¦  «        d¦  «                             d¬	¦  «                             d
¦  «        }d d d ¦  «         n# 1 swxY w Y   |D �]v}|| j        k    rŒt          j        ||         ¦  «        \  }}	||	         }
| j        dk    r| j        |         nd }|                      |
| j        r| j        |         n| j        |         | j        r| j        |         n| j        |         |¬¦  «        }| j        r|                      |¦  «        n|                      |¦  «        }| j        dk    r| j        |         nd }|                      || j        |         | j        |         |¬¦  «        }||	|d f         }||                     |j        ¦  «        z  }|                     d|	|                     |j        ¦  «        ¦  «         �Œx|                     |j        ¦  «        S )Nr`   r   )Únum_classesr   r   )rÂ   rÁ   rÚ   F)Úas_tuplerÂ   r¤   r|   )r   Ú
zeros_liker©   Úno_gradre   r   Úone_hotrT   ÚpermuteÚgreaterrì   ÚnonzerorÇ   Úwherer�   r$  r²   rá   râ   rã   rä   rå   rè   ré   r%  ræ   rç   rê   ra   Ú
index_add_)r«   rÖ   r×   rØ   r÷   Úexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgate_up_act_scalerU   Údown_act_scaleÚrouting_weightsrö   s                   r$   rµ   zFP8Experts.forward®  sË  € õ
 $Ô.¨}ÅEÄMÐRÑRÔRÐåŒ]‰_Œ_ð 	jð 	jÝœ(Ô-×5Ò5°kÈtÔO_ÐbcÑOcÐ5ÑdÔdˆKØ%×-Ò-¨a°°AÑ6Ô6ˆKÝœ {§¢¸8 Ñ'DÔ'DÀaÑHÔH×PÒPÐZ_ÐPÑ`Ô`×eÒeÐfhÑiÔiˆJð	jð 	jð 	jñ 	jô 	jð 	jð 	jð 	jð 	jð 	jð 	jøøøð 	jð 	jð 	jð 	jð
 %ð 	eñ 	eˆJØ˜TÔ-Ò-Ð-Øå#(¤;¨{¸:Ô/FÑ#GÔ#GÑ ˆI�yØ)¨)Ô4ˆMàBFÔBXÐ\dÒBdÐBd�Ô2°:Ô>Ð>Ðjnð ð —{’{ØØ15´Ð\�Ô! *Ô-Ð-ÀDÄLÐQ[ÔD\Ø;?¼=Ðp�Ô+¨JÔ7Ð7ÈdÔNdÐeoÔNpØ!2ð	 #ñ ô ˆHð 6:´]Ð]�t×'Ò'¨Ñ1Ô1Ð1ÈÏÊÐT\ÑH]ÔH]ˆHà?CÔ?UÐYaÒ?aÐ?a�Ô/°
Ô;Ð;Ðgkð ð —{’{ØØ”˜zÔ*ØÔ(¨Ô4Ø!/ð	 #ñ ô ˆHð ,¨I°yÀ$Ð,FÔGˆOØ# o×&8Ò&8¸¼Ñ&HÔ&HÑHˆLØ×*Ò*¨1¨i¸¿ºÐI\ÔIbÑ9cÔ9cÑdÔdÐdÑdØ"×%Ò% mÔ&9Ñ:Ô:Ð:s   ´BCÃCÃCrr   rj   rt   ry   rx   c                óž   — |                      ¦   «         dk    rt          j        ||d ¦  «        S t          |||| j        || j         ¬¦  «        S )Nr   )ry   r�   )r°   r±   r²   r•   ru   r³   )r«   rr   rj   rt   ry   s        r$   r²   zFP8Experts.linearØ  s`   € ð ×ÒÑ Ô  1Ò$Ð$Ý”8˜E 6¨4Ñ0Ô0Ð0åØØØØŒOØ-Ø#Ô6Ð6ð
ñ 
ô 
ð 	
r#   )Nr˜   r™   FT)
ru   rœ   r�   rž   rŸ   rž   r    r‚   rá   r‚   )r(  rs   r   rs   )rÖ   rs   r×   rs   rØ   rs   r   rs   r?   )
rr   rs   rj   rs   rt   rs   ry   rx   r   rs   )r*   r5   r6   r³   r  r8   r¦   rè   rµ   r²   r·   r¸   s   @r$   r	  r	  G  sä   ø€ € € € € € ð Ðð 	Ø/Ø)ð
ð 
ð;Ðð ð ð ñ ð .2Ø!*Ø ØØðDnð Dnð Dnð Dnð Dnð Dnð DnðL&ð &ð &ð &ð(;ð (;ð (;ð (;ð^ 15ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r#   r	  c                  ó    — e Zd ZdZeeeedœZdS )ÚFP8ExpertsInterfacez?Interface for registering custom FP8 experts forward functions.)Ú
batched_mmÚ
grouped_mmÚdeepgemmr
  N)	r*   r5   r6   r7   rø   r  r   r   Ú_global_mappingr"   r#   r$   rG  rG  ì  s.   € € € € € ØIÐIð 5Ø4Ø4ØDð	ð €O€O€Or#   rG  Úmodelú	nn.Modulec                ó„  — d„ |                       ¦   «         D ¦   «         }t          ¦   «         }|D ]U}t          |                     ¦   «         d¦  «        }|�/|j        j        dk    r|                     |j        j        ¦  «         ŒVt          |¦  «        dk    rdS |D ]	}d|_	        Œ
t                               d¦  «         dS )a4  Internal, temporary helper (not public API): flag every FP8 module to skip DeepGEMM when the
    model spans >1 CUDA device in one process.

    DeepGEMM loads each kernel via `cuKernelGetFunction`, which binds the `CUfunction` handle to the
    CUDA context live at load time; driving that cached handle from another device launches it against
    the wrong context and produces garbage. (Build-time fix: compile DeepGEMM with
    `DG_JIT_USE_RUNTIME_API=1` for a context-free `cudaKernel_t` loader; until our wheel picks that up
    we avoid single-process multi-device.) Setting `_deepgemm_disabled` routes both the linear and
    experts paths through Triton/grouped_mm. A model that fits on one device keeps DeepGEMM even with
    other GPUs visible; TP/EP put one device per process, so this is a no-op there.
    c                óJ   — g | ] }t          |t          t          f¦  «        ¯|‘Œ!S r"   )Ú
isinstancer—   r	  )r@   Úms     r$   rB   z5_disable_deepgemm_on_multi_device.<locals>.<listcomp>  s,   € ÐXÐXÐX˜µ¸AÅ	Í:Ð?VÑ1WÔ1WÐX�1ÐXÐXÐXr#   Nr„   r   TaB  This FP8 model spans multiple CUDA devices in one process; routing its FP8 linear and experts layers through Triton/grouped_mm instead of DeepGEMM (DeepGEMM's cached kernels are bound to a single CUDA context and corrupt across devices). Run tensor/expert parallel (one device per process) to use the faster DeepGEMM path.)ÚmodulesÚsetÚnextÚ
parametersrŠ   r)   ÚaddÚindexÚlenr³   r‘   r’   )rL  Úfp8_modulesÚcuda_devicesrQ  Úparams        r$   Ú!_disable_deepgemm_on_multi_devicer\  ú  s×   € ð YÐX˜eŸmšm™oœoÐXÑXÔX€KÝ‘5”5€LØð 1ð 1ˆÝ�Q—\’\‘^”^ TÑ*Ô*ˆØÐ ¤Ô!2°fÒ!<Ð!<Ø×Ò˜Uœ\Ô/Ñ0Ô0Ð0øÝ
ˆ<ÑÔ˜AÒÐØˆØð $ð $ˆØ#ˆÔÐÝ
×Òð	4ñô ð ð ð r#   FÚmodules_to_not_convertúlist[str] | Nonec                óð  — |j         r| S d}|                      ¦   «         D �]¸\  }}t          ||¦  «        sŒd}t          j        d¦  «        5  |                     d¦  «        rˆt          |dd¦  «        }t          |dd¦  «        }	t          |d| j                             ¦   «         ¦  «        }
t          t          t          |	|¬	¦  «        } ||
|j        |j        |j        |	|¬
¦  «        }n¼t          |¦  «        t           j        u r6t%          |j        |j        |j        |j        |j        |j        du¬¦  «        }nkt-          |t           j        ¦  «        rQdt          |¦  «        j        v r;t1          |j        |j        |j        |j        |j        |j        |j        du¬¦  «        }|�|                      ||¦  «         d}ddd¦  «         n# 1 swxY w Y   �Œº|st6                               d¦  «         | S )a®  
    A helper function to replace all `torch.nn.Linear` modules by `FP8Linear` modules.

    Parameters:
        model (`torch.nn.Module`):
            Input model or `torch.nn.Module` as the function is run recursively.
        modules_to_not_convert (`list[`str`]`, *optional*, defaults to `None`):
            Names of the modules to not convert. In practice we keep the `lm_head` in full precision for numerical stability reasons.
        quantization_config (`FineGrainedFP8Config`):
            The quantization config object that contains the quantization parameters.
        pre_quantized (`book`, defaults to `False`):
            Whether the model is pre-quantized or not
    FNÚmetaz.expertsrá   Tr    r  )Úexperts_classÚexperts_interfacer    rá   )r  ru   r�   rŸ   r    rá   r¾   ÚGroupedLinear)r»   r›   r¼   ru   r�   rŸ   r    z�You are loading your model using fp8 but no linear modules were found in your model. Please double check your model architecture.)Ú
dequantizeÚnamed_modulesr   r   rŠ   Úendswithr'   r  Úget_text_configr   r	  ÚALL_FP8_EXPERTS_FUNCTIONSÚweight_block_sizer�   rŸ   r)   re   ÚLinearr—   rš   r›   rw   rP  r*   rº   r¼   Úset_submoduler‘   Úwarning)rL  r]  Úquantization_configÚpre_quantizedÚhas_been_replacedÚmodule_nameÚmoduleÚ
new_modulerá   r    r  Ú	new_classs               r$   Úreplace_with_fp8_linearrt    sx  € ð" Ô%ð ØˆàÐØ$×2Ò2Ñ4Ô4ð 4)ñ 4)Ñˆ�VÝ$ [Ð2HÑIÔIð 	Øàˆ
ÝŒ\˜&Ñ!Ô!ð /	)ð /	)Ø×#Ò# JÑ/Ô/ð +Ý" 6¨:°tÑ<Ô<�Ý" 6¨:°uÑ=Ô=�Ý  ¨°5´<×3OÒ3OÑ3QÔ3QÑRÔR�Ý6Ý",Ý&?Ø%Ø%ð	ñ ô �	ð '˜YØ!Ø2ÔDØ&9Ô&KØ1Ô;Ø%Ø%ðñ ô �
�
õ �f‘”¥¤Ð*Ð*å&Ø &Ô 2Ø!'Ô!4Ø2ÔDØ&9Ô&KØ1Ô;Ø#œ[°Ð4ðñ ô �
�
õ ˜F¥B¤IÑ.Ô.ð °?ÅdÈ6ÁlÄlÔF[Ð3[Ð3[õ .Ø*0Ô*<Ø!'Ô!4Ø#œ_Ø2ÔDØ&9Ô&KØ1Ô;Ø#œ[°Ð4ðñ ô �
ð Ð%Ø×#Ò# K°Ñ<Ô<Ð<Ø$(Ð!ð_/	)ð /	)ð /	)ñ /	)ô /	)ð /	)ð /	)ð /	)ð /	)ð /	)ð /	)øøøð /	)ð /	)ð /	)ð /	)ùðb ð 
Ý�Šð<ñ	
ô 	
ð 	
ð €Ls   ÁE4GÇG	ÇG	c                  óH   — e Zd ZdZd„ Zdd„Zdd„Zdd„Zedd„¦   «         Z	dS )ÚFp8Quantizez^
    A quantization operation that creates two tensors, weight and scale out of a weight.
    c                ó   — || _         d S r?   ©Úhf_quantizer©r«   ry  s     r$   r¦   zFp8Quantize.__init__p  ó   € Ø(ˆÔÐÐr#   Úvaluers   r   útuple[int, int]c                ó(  — d }| j         j        �Zt          | j         j        t          ¦  «        r | j         j                             d¦  «        }nt          | j         j        dd ¦  «        }|€|j        d         |j        d         f}t          |¦  «        S )Nri  rÁ   rÂ   )ry  rm  rP  Údictr�   r'   rÆ   Útuple)r«   r|  ru   s      r$   Ú_resolve_block_sizezFp8Quantize._resolve_block_sizes  s‹   € Øˆ
ØÔÔ0Ð<Ý˜$Ô+Ô?ÅÑFÔFð gØ!Ô.ÔB×FÒFÐGZÑ[Ô[�
�
å$ TÔ%6Ô%JÐL_ÐaeÑfÔf�
ØÐØœ+ bœ/¨5¬;°r¬?Ð;ˆJÝ�ZÑ Ô Ð r#   Úkeyrž   údict[str, torch.Tensor]c                ó°  — |j         dk     r||iS |                      |¦  «        \  }}|j        d         |j        d         }}||z  dk    s	||z  dk    r||iS |j        d d…         }||z  }||z  }	|j        }
|                     t          j        ¦  «        } |j        g |¢|‘|‘|	‘|‘R Ž }|                     ¦   «                              d¬¦  «        }t	          j	        |dk    |t	          j
        |¦  «        ¦  «        }t          |z  }t	          j	        |dk    |t	          j
        |¦  «        ¦  «        }d|z                       t          j        ¦  «        }| j        j        j        dk    r±t	          j        d	t	          j        t	          j        |                     t	          j        t          j        ¦  «        j        ¬
¦  «        ¦  «        ¦  «        ¦  «        }|                     t+          ¦   «         ¦  «        }d|                     t          j        ¦  «        z  }|                     d¦  «                             d¦  «        }||z  }t	          j        |t.          t          ¬¦  «                             t0          ¦  «        }|                     |
¦  «        }|                     d¦  «        r|                     dd¦  «        d         dz   n|dz   }||||iS )Nr   rÁ   rÂ   r   )éýÿÿÿrÂ   rÚ   r¢   r£   ç       @)rü   r…  r*  ú.weightú.r   ú.weight_scale_invÚ
_scale_inv)Úndimr�  rÆ   rê   r   r©   rÉ   ÚabsÚamaxr:  Ú	ones_likeÚ_FP8_MAXry  rm  rŸ   ÚpowÚceilÚlog2r,  ÚfinfoÚtinyr%   rà   Ú_FP8_MINr§   rf  Úrsplit)r«   r‚  r|  Úblock_mÚblock_nÚrowsÚcolsÚleading_shapeÚ
rows_tilesÚ
cols_tilesÚoriginal_shapeÚ
value_fp32ÚreshapedÚmax_absÚsafe_max_absÚscalesÚ
inv_scalesÚscales_broadcastÚscaledÚ	quantizedÚ	scale_keys                        r$   Ú_quantize_onezFp8Quantize._quantize_one~  s™  € ð Œ:˜Š>ˆ>Ø˜�<ÐØ×3Ò3°EÑ:Ô:Ñˆ�Ø”[ ”_ e¤k°"¤oˆdˆØ�'‰>˜QÒÐ $¨¡.°AÒ"5Ð"5Ø˜�<Ðð œ C R CÔ(ˆØ˜W‘_ˆ
Ø˜W‘_ˆ
ØœˆØ—X’X�eœmÑ,Ô,ˆ
à%�:Ô%Ð_ }Ð_°jÐ_À'Ð_È:Ð_ÐW^Ð_Ð_Ð_ˆà—,’,‘.”.×%Ò%¨(Ð%Ñ3Ô3ˆÝ”{ 7¨Q¢;°½¼ÈÑ9QÔ9QÑRÔRˆå˜LÑ(ˆÝ”˜W qš[¨&µ%´/À&Ñ2IÔ2IÑJÔJˆØ˜F‘l×&Ò&¥u¤}Ñ5Ô5ˆ
ð ÔÔ0Ô:¸gÒEÐEÝœ 3­¬
µ5´:¸j×>NÒ>NÕSXÔS^Õ_dÔ_lÑSmÔSmÔSrÐ>NÑ>sÔ>sÑ3tÔ3tÑ(uÔ(uÑvÔvˆJØ#ŸšÕ'7Ñ'9Ô'9Ñ:Ô:ˆJØ˜:Ÿ=š=­¬Ñ7Ô7Ñ7ˆFà!×+Ò+¨BÑ/Ô/×9Ò9¸"Ñ=Ô=ÐØÐ,Ñ,ˆÝ”K ­H½(ÐCÑCÔC×FÒFÅzÑRÔRˆ	Ø×%Ò% nÑ5Ô5ˆ	ØCFÇ<Â<ÐPYÑCZÔCZÐr�C—J’J˜s AÑ&Ô& qÔ)Ð,?Ñ?Ð?Ð`cÐfrÑ`rˆ	Ø�Y 	¨:Ð6Ð6r#   Ú
input_dictc                óÎ   — i }|                      ¦   «         D ]M\  }}t          |t          ¦  «        r|d         n|}|                     |                      ||¦  «        ¦  «         ŒN|S )Nr   )ÚitemsrP  ÚlistÚupdater©  )r«   rª  ÚkwargsÚresultr‚  r|  r¨   s          r$   ÚconvertzFp8Quantize.convert¦  sn   € ð +-ˆØ$×*Ò*Ñ,Ô,ð 	;ð 	;‰JˆC�Ý!+¨Eµ4Ñ!8Ô!8ÐC�U˜1”X�X¸eˆFØ�MŠM˜$×,Ò,¨S°&Ñ9Ô9Ñ:Ô:Ð:Ð:Øˆr#   r	   c                ó*   — t          | j        ¦  «        S r?   )ÚFp8Dequantizery  ©r«   s    r$   Ú
reverse_opzFp8Quantize.reverse_op°  s   € å˜TÔ.Ñ/Ô/Ð/r#   N)r|  rs   r   r}  )r‚  rž   r|  rs   r   rƒ  )rª  rs   r   rƒ  ©r   r	   )
r*   r5   r6   r7   r¦   r�  r©  r±  Úpropertyrµ  r"   r#   r$   rv  rv  k  sŠ   € € € € € ðð ð)ð )ð )ð	!ð 	!ð 	!ð 	!ð&7ð &7ð &7ð &7ðPð ð ð ð ð0ð 0ð 0ñ „Xð0ð 0ð 0r#   rv  c                  óf   — e Zd ZdZd„ Zdd„ZdZdd
„Z	 ddd„Zdd„Z		 	 d d!d„Z
ed"d„¦   «         ZdS )#r³  u¬  Dequantize FP8 weights using their per-block ``weight_scale_inv``.

    Designed to run as the *first* op in any :class:`WeightConverter` chain when
    loading with ``dequantize=True`` â€” :meth:`update_weight_conversions` on the
    FP8 quantizer attaches it to each existing model-specific converter so that
    per-expert (weight, scale) pairs are folded into full-precision tensors before
    the chain's merge / concat ops collapse the per-expert structure.

    Pattern semantics
        Input ``input_dict`` carries one entry per source pattern; each value is a
        list of tensors (one per ``*`` match). For every weight pattern that has a
        sibling ``*.weight_scale_inv`` pattern in the dict, this op pairs them up by
        index, dequantizes per-pair, and emits the dequantized list under the
        original *weight* key. Scale entries are dropped from the output so the
        remaining ops only see weights.
    c                ó   — || _         d S r?   rx  rz  s     r$   r¦   zFp8Dequantize.__init__Ç  r{  r#   Úweight_patternrž   r   c                óØ   — |                      d¦  «        }|r
|d d…         n|}|                      d¦  «        r|d t          d¦  «         …         dz   }n|dk    rd}n|dz   }|r|dz   n|S )Nú$rÂ   r‡  r‰  rj   rt   rŠ  )rf  rX  )r«   rº  ÚanchoredÚbaseÚscales        r$   Ú_scale_pattern_forz Fp8Dequantize._scale_pattern_forÊ  s�   € à!×*Ò*¨3Ñ/Ô/ˆØ&.ÐBˆ~˜c˜r˜cÔ"Ð"°NˆØ�=Š=˜Ñ#Ô#ð 	(ØÐ*�C 	™NœN˜?Ð*Ô+Ð.AÑAˆEˆEØ�XÒÐØ&ˆEˆEà˜<Ñ'ˆEØ&Ð1ˆu�s‰{ˆ{¨EÐ1r#   )rÝ   g      à?r¢   g      ø?r†  g      @g      @g      @g       €g      à¿g      ð¿g      ø¿g       Àg      Àg      Àg      ÀÚpackedrs   c                ó¸  — t          j        | j        t           j        |j        ¬¦  «        }|                     ¦   «                              t           j        ¦  «        }|dz                       ¦   «         }|dz	  dz                       ¦   «         }t          j	        ||         ||         gd¬¦  «        } |j
        g |j        dd…         ¢d|j        d         z  ‘R Ž S )uR   Two ``e2m1`` FP4 values per byte â†’ float32 tensor twice as wide on the last dim.)ra   rŠ   é   é   rÂ   rÚ   Nr   )r   r¨   Ú_FP4_E2M1_LUTr©   rŠ   Ú
contiguousrÇ   Úuint8ÚlongÚstackrÉ   rÆ   )r«   rÁ  ÚlutÚu8ÚlowÚhighÚunpackeds          r$   Ú_unpack_fp4zFp8Dequantize._unpack_fp4Û  sÂ   € åŒl˜4Ô-µU´]È6Ì=ÐYÑYÔYˆØ×ÒÑ Ô ×%Ò%¥e¤kÑ2Ô2ˆØ�C‰x�oŠoÑÔˆØ�q‘˜C‘×%Ò%Ñ'Ô'ˆÝ”;  C¤¨#¨d¬)Ð4¸"Ð=Ñ=Ô=ˆØˆxÔÐI ¤¨c¨r¨cÔ!2ÐI°A¸¼ÀRÔ8HÑ4HÐIÐIÐIÐIr#   Nr§  r£  rp   rz   c                ó  — t          t          dd ¦  «        }|j        t          j        k    s|�!|j        |k    r|                      |¦  «        }n|                     t          j        ¦  «        }|j        dd …         \  }}	 |j        dd …         \  }}	n# t          $ r d\  }}	Y nw xY w||z  s||	z  rt          d|› d|› d|› d|	› d�	¦  «        ‚||z  }
||	z  }|€7|j        j
        r|                     ¦   «         dk    r|j        nt          j        }|j        t          j        k    r5|                     t          j        ¦  «        d	z
                       ¦   «         }n|                     t          j        ¦  «        }|j        }|                     d
||
|	|¦  «        }|                     d
||	¦  «                             d
¦  «                             d¦  «        }||z                       |¦  «                             |¦  «        S )NÚfloat4_e2m1fn_x2rÁ   )r   r   zWeight shape (rC   z) not divisible by scale grid (z).r   g     À_@rÂ   )r'   r   ra   r�   rÏ  rê   r©   rÆ   Ú	ExceptionÚ
ValueErrorrg   r°   Úbfloat16rÇ  Úexp2rÉ   rà   )r«   r§  r£  rp   Ú	fp4_dtypeÚquantized_fp32r™  rš  Ú
scale_rowsÚ
scale_colsr—  r˜  Ús_fp32rž  ÚqÚss                   r$   Ú_dequantize_onezFp8Dequantize._dequantize_oneä  s   € õ
 �EÐ#5°tÑ<Ô<ˆ	ØŒ?�eœjÒ(Ð(¨YÐ-BÀyÄÐZcÒGcÐGcØ!×-Ò-¨iÑ8Ô8ˆNˆNà&Ÿ\š\­%¬-Ñ8Ô8ˆNØ#Ô)¨"¨#¨#Ô.‰
ˆˆdð	*Ø%+¤\°"°#°#Ô%6Ñ"ˆJ˜
˜
øÝð 	*ð 	*ð 	*à%)Ñ"ˆJ˜
˜
˜
ð	*øøøð �*Ñð 	  zÑ 1ð 	ÝØj ÐjÐj¨ÐjÐjÈjÐjÐjÐ\fÐjÐjÐjñô ð ð ˜*Ñ$ˆØ˜*Ñ$ˆð
 Ðà &¤Ô >ÐqÀ6×CVÒCVÑCXÔCXÐ\]ÒC]ÐC]�”�ÕchÔcqð ð Œ<�5œ;Ò&Ð&Ø—i’i¥¤Ñ.Ô.°Ñ6×<Ò<Ñ>Ô>ˆFˆFà—Y’Y�uœ}Ñ-Ô-ˆFØ'Ô-ˆØ×"Ò" 2 z°7¸JÈÑPÔPˆØ�NŠN˜2˜z¨:Ñ6Ô6×@Ò@ÀÑDÔD×NÒNÈqÑQÔQˆØ�A‘�zŠz˜,Ñ'Ô'×/Ò/°Ñ?Ô?Ð?s   ÂB ÂB&Â%B&rL  útorch.nn.Module | NoneÚfull_layer_nameú
str | Nonec                óx   — |�|€d S t          ||¦  «        \  }}t          ||d ¦  «        }t          |dd ¦  «        S )Nra   )r
   r'   )r«   rL  rß  rq  Útensor_namer[  s         r$   Ú_get_target_dtypezFp8Dequantize._get_target_dtype  sH   € Øˆ=˜OÐ3Ø�4Ý2°5¸/ÑJÔJÑˆ�Ý˜ ¨TÑ2Ô2ˆÝ�u˜g tÑ,Ô,Ð,r#   rª  ú,dict[str, list[torch.Tensor] | torch.Tensor]c                ó  ‡ ‡— ‰                       ||¦  «        Šd|v rv|�|nd}|d         }t          |t          ¦  «        r|d         n|}d|v rA|d         }t          |t          ¦  «        r|d         n|}|‰                      ||‰¬¦  «        iS ||iS i }|                     ¦   «         D ]Ü\  }	}
d|	v sd|	v rŒ‰                      |	¦  «        }||vr|
||	<   Œ-t          |
t          ¦  «        r|
n|
g}||         }t          |t          ¦  «        r|n|g}t          |¦  «        t          |¦  «        k    r3t          d|	› dt          |¦  «        › d	t          |¦  «        › d
�¦  «        ‚ˆˆ fd„t          ||¦  «        D ¦   «         ||	<   ŒÝ|S )Nzweight$rj   r   rt   ©rp   ry   z/Fp8Dequantize: weight/scale count mismatch for z (z weights vs z	 scales).c                óF   •— g | ]\  }}‰                      ||‰¬ ¦  «        ‘ŒS )ræ  )rÝ  )r@   rÒ   rÜ  rp   r«   s      €€r$   rB   z)Fp8Dequantize.convert.<locals>.<listcomp>C  s5   ø€ ÐrÐrÐrÑUYÐUVÐXY˜4×/Ò/°°1À<Ð/ÑPÔPÐrÐrÐrr#   )	rã  rP  r­  rÝ  r¬  rÀ  rX  rÓ  Úzip)r«   rª  rß  rL  r¯  Ú
target_keyr§  r£  r°  r‚  r|  r¨  Úweightsrp   s   `            @r$   r±  zFp8Dequantize.convert  s  øø€ ð ×-Ò-¨e°_ÑEÔEˆð
 ˜
Ð"Ð"ð -<Ð,G˜˜ÈXˆJØ" 9Ô-ˆIÝ(2°9½dÑ(CÔ(CÐR˜	 !œ˜ÈˆIØ! ZÐ/Ð/Ø#Ð$6Ô7�Ý&0°½Ñ&>Ô&>ÐJ˜ œ˜ÀF�Ø" D×$8Ò$8¸ÀFÐYeÐ$8Ñ$fÔ$fÐgÐgØ 	Ð*Ð*ð @BˆØ$×*Ò*Ñ,Ô,ð 	sð 	s‰JˆC�Ø! SÐ(Ð(Ð,>À#Ð,EÐ,EØØ×/Ò/°Ñ4Ô4ˆIØ 
Ð*Ð*à#��s‘ØÝ)¨%µÑ6Ô6ÐC�e�e¸U¸GˆGØ 	Ô*ˆFÝ)¨&µ$Ñ7Ô7ÐE�V�V¸f¸XˆFÝ�7‰|Œ|�s 6™{œ{Ò*Ð*Ý ðIÀcð Ið IÝ˜G™œðIð IÝ25°f±+´+ðIð Ið Iñô ð ð sÐrÐrÐrÐrÕ]`ÐahÐjpÑ]qÔ]qÐrÑrÔrˆF�3‰KˆKØˆr#   r	   c                ó*   — t          | j        ¦  «        S r?   )rv  ry  r´  s    r$   rµ  zFp8Dequantize.reverse_opF  s   € õ
 ˜4Ô,Ñ-Ô-Ð-r#   )rº  rž   r   rž   )rÁ  rs   r   rs   r?   )r§  rs   r£  rs   rp   rz   r   rs   )rL  rÞ  rß  rà  r   rz   )NN)rª  rä  rß  rà  rL  rÞ  r   rä  r¶  )r*   r5   r6   r7   r¦   rÀ  rÅ  rÏ  rÝ  rã  r±  r·  rµ  r"   r#   r$   r³  r³  µ  sÝ   € € € € € ðð ð")ð )ð )ð
2ð 
2ð 
2ð 
2ð m€MðJð Jð Jð Jð aeð+@ð +@ð +@ð +@ð +@ðZ-ð -ð -ð -ð '+Ø(,ð	,ð ,ð ,ð ,ð ,ð\ ð.ð .ð .ñ „Xð.ð .ð .r#   r³  )r   r   )r   r1   )r   rI   )rO   rP   rQ   rP   r   rP   )r   NNr   )rT   rP   rU   rP   rV   rP   rW   r   rX   r   rY   rP   rZ   r[   r\   r[   r]   rP   r   r^   )NNNN)rr   rs   rj   rs   rt   rs   ru   rv   rw   rx   ry   rx   rp   rz   r   rs   )NNNNT)rr   rs   rj   rs   rt   rs   ru   rv   rw   rx   ry   rx   rp   rz   r�   r‚   r   rs   )
r«   rÕ   rÖ   rs   r×   rs   rØ   rs   r   rs   )rL  rM  r   rI   )NNF)r]  r^  )LÚ
__future__r   Ú	functoolsrŽ   Úcollections.abcr   Údataclassesr   r   Útorch.nnre   r   r±   Úactivationsr   Úcore_model_loadingr	   Úquantizers.quantizers_utilsr
   r   Úutilsr   Úutils.deprecationr   Úutils.import_utilsr   r   r   r   rJ  r   r   r   Úhub_kernelsr   Úmoer   r   Útensor_parallelr   Ú
get_loggerr*   r‘   Úfloat8_e4m3fnr§   r“  rü   r•  rh   r�  Úcacher%   r.   r1   rH   Ú_dynamoÚallow_in_graphrL   rN   rS   ro   r€   r•   rj  r—   rº   rø   r  ÚModuler	  rG  rh  r\  rt  rv  r³  r"   r#   r$   ú<module>r      s!  ðð #Ð "Ð "Ð "Ð "Ð "à Ð Ð Ð Ø 	€	€	€	Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ø $Ð $Ð $Ð $Ð $Ð $à  Ð  Ð  Ð  Ð  Ð  Ø .Ð .Ð .Ð .Ð .Ð .Ø UÐ UÐ UÐ UÐ UÐ UÐ UÐ UØ Ð Ð Ð Ð Ð Ø /Ð /Ð /Ð /Ð /Ð /ðð ð ð ð ð ð ð ð ð ð ð ðð ð ð ð ð ð ð ð ð ð
 *Ð )Ð )Ð )Ð )Ð )Ø =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ø %Ð %Ð %Ð %Ð %Ð %ð 
ˆÔ	˜HÑ	%Ô	%€ð Ô €
Øˆ5Œ;�zÑ"Ô"Ô&€Øˆ5Œ;�zÑ"Ô"Ô&€ð „ð ð  ð  ñ „ð ðGð Gð Gð €�$ÐÑÔðð ð ð ð ñ ô ñ Ôðð „ð.ð .ð .ñ „ð.ðb „Ôðð ð ñ Ôðð
*ð *ð *ð *ðð ð ð ð Ø Ø Øðð ð ð ð ð8 €�¨Ð1Ñ1Ô1ð
 $(Ø $Ø,0Ø'+ðð ð ð ñ 2Ô1ðð: €�¨Ð1Ñ1Ô1ð
 $(Ø $Ø,0Ø'+ØðDgð Dgð Dgð Dgñ 2Ô1ðDgðN;
ð ;
ð ;
ð ;
ð ;
�”	ñ ;
ô ;
ð ;
ð|Dð Dð Dð Dð D�yñ Dô Dð DðNG7ð G7ð G7ð G7ðT[7ð [7ð [7ð [7ð|b
ð b
ð b
ð b
ð b
�”ñ b
ô b
ð b
ðJð ð ð ð Ð*ñ ô ð ð 0Ð/Ñ1Ô1Ð ðð ð ð ð> ejðPð Pð Pð Pð PðfG0ð G0ð G0ð G0ð G0�-ñ G0ô G0ð G0ðTV.ð V.ð V.ð V.ð V.�Mñ V.ô V.ð V.ð V.ð V.r#   