§
    ‚Štj:*  ã                   óú   — d dl Z d dlZd dlZd dlmZ d dlmZmZmZ  e¦   «         rddl	m
Z
 ddlmZ  e¦   «         rd dlmZ d dlmZ  ej        e¦  «        Zd	„ Zd
„ Z G d„ de
¦  «        Z G d„ de
¦  «        ZdS )é    N)Úlogging)Úis_torch_accelerator_availableÚis_torch_availableÚis_torchao_availableé   )ÚConversionOps)Úget_module_from_name)Úunflatten_tensor_state_dict)Úis_metadata_torchaoc                 óþ   — ddl m} ddlm} t	          | |¦  «        r$| j        j        › d|                      ¦   «         › d�S t	          | |¦  «        r,| j        j        › d| j        › dt          | j	        ¦  «        › d�S d S )Nr   )ÚAffineQuantizedTensor)ÚLinearActivationQuantizedTensorú(ú)z(activation=ú	, weight=)
Útorchao.dtypesr   Ú7torchao.quantization.linear_activation_quantized_tensorr   Ú
isinstanceÚ	__class__Ú__name__Ú_quantization_typeÚinput_quant_funcÚoriginal_weight_tensor)Úweightr   r   s      ú_/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/integrations/torchao.pyr   r   &   sÜ   € Ø4Ð4Ð4Ð4Ð4Ð4ØgÐgÐgÐgÐgÐgå�&Ð/Ñ0Ô0ð MØÔ"Ô+ÐLÐL¨f×.GÒ.GÑ.IÔ.IÐLÐLÐLÐLå�&Ð9Ñ:Ô:ð PØÔ"Ô+ð  Pð  P¸Ô9Pð  Pð  PÕ[mÐntô  oLñ  \Mô  \Mð  Pð  Pð  Pð  	PðPð Pó    c                 óÖ   — t          | j        ¦  «        }|€)d| j        j        d         › d| j        j        d         › d�S d| j        j        d         › d| j        j        d         › d|› �S )Nzin_features=é   z, out_features=r   z, weight=Noner   )r   r   Úshape)Úselfr   s     r   Ú_linear_extra_reprr!   1   sx   € Ý ¤Ñ,Ô,€FØ€~Øf˜dœkÔ/°Ô2ÐfÐfÀ4Ä;ÔCTÐUVÔCWÐfÐfÐfÐfàj˜dœkÔ/°Ô2ÐjÐjÀ4Ä;ÔCTÐUVÔCWÐjÐjÐbhÐjÐjÐjr   c                   óŽ   — e Zd Zd„ Zd„ Z	 	 	 d	deeej        f         dej	        j
        dz  dedz  deeej        f         fd„ZdS )
ÚTorchAoQuantizec                 ó   — || _         d S ©N©Úhf_quantizer©r    r'   s     r   Ú__init__zTorchAoQuantize.__init__:   ó   € Ø(ˆÔÐÐr   c                 ót  — ddl m} t          |                     ¦   «         ¦  «        j        }| j        j        rr|j        dk    rgt          ¦   «         rt          j
                             ¦   «         nd}|                     |¦  «          |||g|¢R i |¤Ž |                     d¦  «         dS  |||g|¢R i |¤Ž dS )a7  Run quantize_, moving to CUDA first if CPU offloading is active.

        Some torchao quantization ops (e.g. int4 packing) only have CUDA kernels.
        When a layer is destined for CPU (e.g. CPU offloading), we temporarily move
        it to CUDA for quantization, then move the result back to CPU.
        r   )Ú	quantize_ÚcpuÚcudaN)Útorchao.quantizationr,   ÚnextÚ
parametersÚdevicer'   Úoffload_to_cpuÚtyper   ÚtorchÚacceleratorÚcurrent_acceleratorÚto)r    ÚmoduleÚconfigÚargsÚkwargsr,   Útarget_devicer2   s           r   Ú	_quantizezTorchAoQuantize._quantize=   sæ   € ð 	3Ð2Ð2Ð2Ð2Ð2å˜V×.Ò.Ñ0Ô0Ñ1Ô1Ô8ˆØÔÔ+ð 	7°Ô0BÀeÒ0KÐ0KÝ@^Ñ@`Ô@`Ðl•UÔ&×:Ò:Ñ<Ô<Ð<ÐflˆFØ�IŠI�fÑÔÐØˆI�f˜fÐ6 tÐ6Ð6Ð6¨vÐ6Ð6Ð6Ø�IŠI�eÑÔÐÐÐàˆI�f˜fÐ6 tÐ6Ð6Ð6¨vÐ6Ð6Ð6Ð6Ð6r   NÚ
input_dictÚmodelÚfull_layer_nameÚreturnc                 ó˜  — t          |                     ¦   «         ¦  «        d         \  }}t          |t          ¦  «        r|d         n|}t	          ||¦  «        \  }}	t
          j                             ||j        ¬¦  «        |j	        |	<   | 
                    ¦   «         }
t          |¦  «        t          |
¦  «        k    }| j        j        j        }|r,|r*t          |j                             d¬¦  «        dd¦  «         ddlm} | j        j                             ¦   «         }t          ||¦  «        �r|                     dd	¦  «        \  }}d }||j        v r-|                     d
¦  «        r
J d¦   «         ‚|j        |         }nÉ||j        v r-|                     d
¦  «        r
J d¦   «         ‚|j        |         }n“|j        D ]p}|                     d
¦  «        sŒt3          j        |dd …         |¦  «        r|j        |         } nHt3          j        |dd …         |¦  «        r|j        |         } nŒq|j                             dd ¦  «        }|�ä|dk    r{|r|r|j                             ¦   «         }|                      ||d„ ¦  «         |                     |¦  «         d|_         | !                    d¬¦  «        D ]	}d|_         Œ
|r|rd|ini S  |||i¦  «        }|                      ||d ¬¦  «         |                     |¦  «         d|_         | !                    d¬¦  «        D ]	}d|_         Œ
i S ||iS |r|r|j                             ¦   «         }|                      || j        j                             ¦   «         ¦  «         |                     |¦  «         d|_         | !                    d¬¦  «        D ]	}d|_         Œ
|r|rd|ini S )Nr   )Úrequires_gradT)ÚdecoderÚtie_word_embeddingsF)ÚFqnToConfigú.r   zre:zHparam fqn should not start with`re:`, which is used for specifying regexzImodule fqn should not start with`re:`, which is used for specifying regexé   Ú_defaultr   c                 ó   — dS )NT© )ÚxÚfqns     r   ú<lambda>z)TorchAoQuantize.convert.<locals>.<lambda>�   s   € ¸d€ r   ©Úrecursezlm_head.weight)Ú	filter_fn)"ÚtupleÚitemsr   Úlistr	   r5   ÚnnÚ	ParameterrD   Ú_parametersÚget_input_embeddingsÚidr'   Úquantization_configÚuntie_embedding_weightsÚsetattrr:   Úget_text_configr/   rG   Úget_apply_tensor_subclassÚrsplitÚfqn_to_configÚ
startswithÚmodule_fqn_to_configÚreÚ	fullmatchÚgetr   Úcloner>   ÚdiscardÚ_is_hf_initializedr1   )r    r?   r@   rA   Úmissing_keysr<   Ú_Úvaluer9   Útensor_nameÚinput_embedÚis_embedding_paramr\   rG   r:   Ú
module_fqnÚtop_level_param_nameÚcÚmaybe_module_fqn_patternÚlm_headÚparamÚcustom_param_fqn_configs                         r   ÚconvertzTorchAoQuantize.convertO   sw  € õ ˜×)Ò)Ñ+Ô+Ñ,Ô,¨QÔ/‰ˆˆ5Ý& u­dÑ3Ô3Ð>��a”�¸ˆå2°5¸/ÑJÔJÑˆ�å*/¬(×*<Ò*<¸UÐRWÔReÐ*<Ñ*fÔ*fˆÔ˜;Ñ'ð ×0Ò0Ñ2Ô2ˆÝ ™ZœZ­2¨k©?¬?Ò:ÐØ"&Ô"3Ô"GÔ"_Ðà"ð 	^Ð'9ð 	^Ý�E”L×0Ò0¸Ð0Ñ>Ô>Ð@UÐW\Ñ]Ô]Ð]à4Ð4Ð4Ð4Ð4Ð4àÔ"Ô6×PÒPÑRÔRˆÝ�f˜kÑ*Ô*ñ 6	,Ø/>×/EÒ/EÀcÈ1Ñ/MÔ/MÑ,ˆJÐ,ØˆAØ &Ô"6Ð6Ð6Ø%×0Ò0°Ñ7Ô7ð ð Ø^ñô Ð7ð Ô/°Ô@��Ø˜vÔ3Ð3Ð3Ø%×0Ò0°Ñ7Ô7ð ð Ø_ñô Ð7ð Ô/°
Ô;��ð 17Ô0Dð Jð JÐ,à3×>Ò>¸uÑEÔEð 	Ø åœÐ&>¸q¸r¸rÔ&BÀOÑTÔTð Ø"Ô7Ð8PÔQ˜Ø˜ÝœÐ&>¸q¸r¸rÔ&BÀJÑOÔOð à"Ô7Ð8PÔQ˜Ø˜ðð
 Ô3×7Ò7¸
ÀDÑIÔI�Aàˆ}Ø'¨8Ò3Ð3Ø)ð 8Ð.Eð 8Ø"(¤-×"5Ò"5Ñ"7Ô"7˜à—N’N 6¨1Ð/BÐ/BÑDÔDÐDØ ×(Ò(¨Ñ9Ô9Ð9Ø04�FÔ-ð
 "(×!2Ò!2¸5Ð!2Ñ!AÔ!Að 8ð 8˜Ø37˜Ô0Ð0Ø:LÐpÐQhÐpÐ,¨gÐ6Ð6ÐnpÐpð /:¨kÐ;OÐQRÐ:SÑ.TÔ.TÐ+Ø—N’N 6Ð+BÈd�NÑSÔSÐSØ ×(Ò(¨Ñ9Ô9Ð9Ø04�FÔ-Ø!'×!2Ò!2¸5Ð!2Ñ!AÔ!Að 8ð 8˜Ø37˜Ô0Ð0Ø�IØ# UÐ+Ð+àð 	,Ð"9ð 	,Ø”m×)Ò)Ñ+Ô+ˆGØ�Š�v˜tÔ0ÔD×^Ò^Ñ`Ô`ÑaÔaÐaØ×Ò˜_Ñ-Ô-Ð-Ø$(ˆÔ!Ø×&Ò&¨uÐ&Ñ5Ô5ð 	,ð 	,ˆEØ'+ˆEÔ$Ð$Ø.@ÐdÐE\ÐdÐ  'Ð*Ð*ÐbdÐdr   )NNN)r   Ú
__module__Ú__qualname__r)   r>   ÚdictÚstrr5   ÚTensorrV   ÚModulerw   rL   r   r   r#   r#   9   s·   € € € € € ð)ð )ð )ð7ð 7ð 7ð* )-Ø&*Øð\eð \eà˜˜eœlÐ*Ô+ð\eð ŒxŒ Ñ%ð\eð ˜t™ð	\eð 
ˆc�5”<ÐÔ	 ð\eð \eð \eð \eð \eð \er   r#   c                   ó    — e Zd Zd„ Z	 	 	 	 d	deeej        f         dee         dz  dej	        j
        dz  dedz  deeej        f         f
d„ZdS )
ÚTorchAoDeserializec                 ó   — || _         d S r%   r&   r(   s     r   r)   zTorchAoDeserialize.__init__¯   r*   r   Nr?   Úsource_patternsr@   rA   rB   c           
      ó¦  — t          |                     ¦   «         ¦  «        d         |v}i }d                     |                     d¦  «        dd…         ¦  «        }	|r3t	          |d         t           ¦  «        r|d         d         }
nx|d         }
no|                     ¦   «         D ]Z}t          ||         ¦  «        dk    r)t          d|› dt          ||         ¦  «        › d	�¦  «        ‚||         d         ||	› d|› �<   Œ[|r||
iS t          | j        j	        ¦  «        st          d
¦  «        ‚t          || j        j	        ¦  «        \  }}|rJ ‚||         }t          ||¦  «        \  }}t	          |t          j        j        ¦  «        rt          j        t"          |¦  «        |_        d|_        |                     d¬¦  «        D ]	}d|_        Œ
||iS )a&  
        Consolidates tensor subclass components before reconstructing the object

        For example:
            input_dict: {
                "_weight_qdata": torch.Tensor,
                "_weight_scale": torch.Tensor,
            }
            full_layer_name: "model.layers.0.self_attn.k_proj.weight"

            Given this, we reconstruct a Float8Tensor instance using the qdata and scale
            and return it as a dictionary with the full_layer_name as the key and the recovered
            Float8Tensor instance as the value.
        r   rH   Néÿÿÿÿr   r   zExpected a single tensor for z	 but got z tensors insteadz$Invalid torchao safetensors metadataTFrP   )rU   ÚkeysÚjoinÚsplitr   ÚlenÚ
ValueErrorr   r'   Úmetadatar
   r	   r5   rV   ÚLinearÚtypesÚ
MethodTyper!   Ú
extra_reprri   r1   )r    r?   r�   r@   rA   rj   r<   Úis_unsafe_serializationÚ
param_dataÚ
layer_namer   ÚsuffixÚunflattened_state_dictÚleftover_state_dictÚ	new_paramr9   rk   ru   s                     r   rw   zTorchAoDeserialize.convert²   s  € õ. #' z§¢Ñ'8Ô'8Ñ"9Ô"9¸!Ô"<ÀOÐ"SÐàˆ
Ø—X’X˜o×3Ò3°CÑ8Ô8¸¸"¸Ô=Ñ>Ô>ˆ
Ø"ð 	MÝ˜* XÔ.µÑ5Ô5ð .Ø# HÔ-¨aÔ0��à# HÔ-��à$Ÿ/š/Ñ+Ô+ð Mð M�Ý�z &Ô)Ñ*Ô*¨aÒ/Ð/Ý$Ør¸ÐrÐrÍÈZÐX^ÔM_ÑI`ÔI`ÐrÐrÐrñô ð ð 8BÀ&Ô7IÈ!Ô7L�
˜jÐ3Ð3¨6Ð3Ð3Ñ4Ð4ð #ð 	EØ# VÐ,Ð,Ý$ TÔ%6Ô%?Ñ@Ô@ð 	EÝÐCÑDÔDÐDå6QØ˜Ô)Ô2ñ7
ô 7
Ñ3ÐÐ 3ð 'Ð&Ð&Ð&Ø*¨?Ô;ˆ	å(¨°Ñ@Ô@‰	ˆ�å�f�eœhœoÑ.Ô.ð 	MÝ %Ô 0Õ1CÀVÑ LÔ LˆFÔØ$(ˆÔ!Ø×&Ò&¨uÐ&Ñ5Ô5ð 	,ð 	,ˆEØ'+ˆEÔ$Ð$à Ð+Ð+r   )NNNN)r   rx   ry   r)   rz   r{   r5   r|   rU   rV   r}   rw   rL   r   r   r   r   ®   s¨   € € € € € ð)ð )ð )ð -1Ø(,Ø&*Øð<,ð <,à˜˜eœlÐ*Ô+ð<,ð ˜cœ TÑ)ð<,ð ŒxŒ Ñ%ð	<,ð
 ˜t™ð<,ð 
ˆc�5”<ÐÔ	 ð<,ð <,ð <,ð <,ð <,ð <,r   r   )rd   r‹   r5   Útransformers.utilsr   Útransformers.utils.import_utilsr   r   r   Úcore_model_loadingr   Úquantizers.quantizers_utilsr	   Ú1torchao.prototype.safetensors.safetensors_supportr
   Ú/torchao.prototype.safetensors.safetensors_utilsr   Ú
get_loggerr   Úloggerr   r!   r#   r   rL   r   r   ú<module>r�      s  ðð 
€	€	€	Ø €€€à €€€à &Ð &Ð &Ð &Ð &Ð &Ø tÐ tÐ tÐ tÐ tÐ tÐ tÐ tÐ tÐ tð ÐÑÔð 3Ø2Ð2Ð2Ð2Ð2Ð2Ø >Ð >Ð >Ð >Ð >Ð >ð ÐÑÔð Tðð ð ð ð ð ð TÐSÐSÐSÐSÐSà	ˆÔ	˜HÑ	%Ô	%€ðPð Pð Pðkð kð kðreð reð reð reð re�mñ reô reð reðj@,ð @,ð @,ð @,ð @,˜ñ @,ô @,ð @,ð @,ð @,r   