§
    ‚ŠtjÐ[  ã                  ó  — U d Z ddlmZ ddlZddlZddlmZ ddlm	Z	 ddl
mZmZ ddlZddlmZ ddlmZ d	d
lmZ d	dlmZ d	dlmZ d	dlmZmZ d	dlmZmZmZmZmZm Z m!Z!m"Z"m#Z# erd	dl$m%Z% d	dlm&Z&  ej'        e(¦  «        Z)e	 G d„ d¦  «        ¦   «         Z*d=d„Z+ee*ge,ee-e.ef         f         f         Z/d>d„Z0	 d?d@d)„Z1dAd+„Z2dAd,„Z3dAd-„Z4dAd.„Z5dAd/„Z6dAd0„Z7dAd1„Z8dAd2„Z9dAd3„Z:dAd4„Z;dAd5„Z<dAd6„Z=dAd7„Z>dAd8„Z?dAd9„Z@dAd:„ZAejB        ejC        ejD        ejE        ejF        ejG        ejH        ejI        ejJ        ejK        ejL        ejM        ejN        ejO        ejP        gZQejR        ejS        ejT        ejU        ejV        ejW        gZXejY        ejZ        gZ[ej\        ej]        gZ^ej_        ej`        eja        gZbejc        e2ejd        e3eje        e3ejf        e4ejg        e5ejh        e7eji        e8ejj        e9ejk        e:ejl        e>ejm        eAejn        e=ejo        e=ie- p                    eQe6¦  «        ¥e- p                    eXe;¦  «        ¥e- p                    e[e<¦  «        ¥e- p                    e^e?¦  «        ¥e- p                    ebe@¦  «        ¥Zqd;erd<<   dS )Bz,
Optimizer utilities for the Trainer class.
é    )ÚannotationsN)ÚCallable)Ú	dataclass)ÚTYPE_CHECKINGÚAny)Úversion)Únné   )Ú	Adafactor)ÚLayerWiseDummyOptimizer)Úcheck_target_module_exists)ÚOptimizerNamesÚParallelMode)	Úis_apollo_torch_availableÚis_bitsandbytes_availableÚis_galore_torch_availableÚis_grokadamw_availableÚis_lomo_availableÚis_schedulefree_availableÚis_torch_optimi_availableÚis_torchao_availableÚ	strtobool)ÚPreTrainedModel)ÚTrainingArgumentsc                  óF   — e Zd ZU dZded<   ded<   ded<   ded<   d	ed
<   dS )ÚOptimizerContextz0Context object passed to all optimizer handlers.r   ÚargszPreTrainedModel | NoneÚmodelúdict[str, Any]Úoptimizer_kwargsÚadam_kwargsúdict[str, str]Ú
optim_argsN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú__annotations__© ó    ú\/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/trainer_optimizer.pyr   r   6   sT   € € € € € € à:Ð:àÐÐÑØ!Ð!Ð!Ñ!Ø$Ð$Ð$Ñ$ØÐÐÑØÐÐÑÐÐr*   r   Úoptim_args_strú
str | NoneÚreturnr"   c                ó¤   — | si S i }|                       dd¦  «                             d¦  «        D ]}|                     d¦  «        \  }}|||<   Œ |S )z8Parse optimizer arguments from a comma-separated string.ú Ú ú,ú=)ÚreplaceÚsplit)r,   r#   ÚmappingÚkeyÚvalues        r+   Ú_parse_optim_argsr9   A   sg   € àð Øˆ	Ø€JØ!×)Ò)¨#¨rÑ2Ô2×8Ò8¸Ñ=Ô=ð  ð  ˆØ—]’] 3Ñ'Ô'‰
ˆˆUØˆ
�3‰ˆØÐr*   Úoptimizer_cls_or_factoryr   Úboolc                ór   — t          | t          ¦  «        r!t          | t          j        j        ¦  «        rdS dS )a4  
    Check if the returned value from a handler is a factory rather than an Optimizer class.

    Factory callables are used for complex optimizers like Muon or Dion that need to:
    - Split parameters between multiple internal optimizers
    - Handle complex sharding logic
    - Access the full model structure for parameter grouping

    Args:
        optimizer_cls_or_factory: The first element returned by an optimizer handler.

    Returns:
        `bool`: True if it's not an Optimizer class (i.e., likely a factory), False if it's an Optimizer class.
    FT)Ú
isinstanceÚtypeÚ
issubclassÚtorchÚoptimÚ	Optimizer)r:   s    r+   Úis_optimizer_factoryrC   P   s:   € õ  Ð*­DÑ1Ô1ð µjÐAYÕ[`Ô[fÔ[pÑ6qÔ6qð ØˆuØˆ4r*   Tr   r   r   r   Úoptimizer_nameÚstrÚoptimizer_mappingr   Úoptim_kwargsr    Úis_layerwise_supportedútuple[Any, dict[str, Any]]c                óŽ  ‡‡— |                      ¦   «                              d¦  «        }|r*| j        t          j        k    r|rt          d|› d�¦  «        ‚||         }| j        €t          d|› d�¦  «        ‚t          | j        t          t          f¦  «        st          d| j        › �¦  «        ‚|€t          d|› d	�¦  «        ‚t          | j        t          ¦  «        o| j                             d
d¦  «        dk    }	g Š|                     ¦   «         D ]{\  }
}t          | j        |
d¬¦  «        \  }}t          |t          j        ¦  «        s%|r"|s t"                               |
› d|› d�¦  «         Œ^|s|	sŒc‰                     |
dz   ¦  «         Œ|t)          ‰¦  «        dk    rt          d|› d| j        › d�¦  «        ‚ˆfd„|                     ¦   «         D ¦   «         }ˆfd„|                     ¦   «         D ¦   «         }d|id|i|¥g}|r¢| j        dk    rt          d|› d�¦  «        ‚i Š|D ]} |d|gigfi |¤Ž‰|<   Œ|D ]} |d|gi|¥gfi |¤Ž‰|<   Œˆfd„}|                     ¦   «         D ]}|j        r|                     |¦  «         Œt4          }|                     d‰i¦  «         |                     d|i¦  «         ||fS )z¶
    Helper function to set up low-rank optimizers like GaLore and Apollo.

    These optimizers apply low-rank projections to specific target modules (typically linear layers).
    Ú	layerwisezLayer-wise z" does not support DDP at this timeNz1You need to define `optim_target_modules` to use z optimizerszX`optim_target_modules` must be a list of strings, a regex string, or 'all-linear'. Got: z'You need to pass a model to initialize z optimizer.Ú_ú-z
all-linearT)Úreturn_is_regexz matched but ignored. z only supports linear layers.z.weightr   zNo target modules found for z (z).c                ó"   •— g | ]\  }}|‰v ¯	|‘ŒS r)   r)   ©Ú.0ÚnÚpÚtarget_params_namess      €r+   ú
<listcomp>z-_setup_low_rank_optimizer.<locals>.<listcomp>œ   s(   ø€ ÐXÐXÐX™4˜1˜a¸qÐDWÐ?WÐ?W�QÐ?WÐ?WÐ?Wr*   c                ó"   •— g | ]\  }}|‰v¯	|‘ŒS r)   r)   rP   s      €r+   rU   z-_setup_low_rank_optimizer.<locals>.<listcomp>�   s(   ø€ Ð`Ð`Ð`™t˜q !À1ÐL_ÐC_ÐC_˜ÐC_ÐC_ÐC_r*   Úparamsr
   z
Layerwise z( does not support gradient accumulation!c                ó‚   •— | j         �6‰|                               ¦   «          ‰|                               ¦   «          d S d S )N)ÚgradÚstepÚ	zero_grad)ÚparamÚoptimizer_dicts    €r+   Úoptimizer_hookz1_setup_low_rank_optimizer.<locals>.optimizer_hook®   sG   ø€ ØŒzÐ%Ø˜uÔ%×*Ò*Ñ,Ô,Ð,Ø˜uÔ%×/Ò/Ñ1Ô1Ð1Ð1Ð1ð &Ð%r*   r]   )ÚlowerÚendswithÚparallel_moder   ÚDISTRIBUTEDÚNotImplementedErrorÚoptim_target_modulesÚ
ValueErrorr=   ÚlistrE   Ú	TypeErrorr4   Únamed_modulesr   r	   ÚLinearÚloggerÚwarningÚappendÚlenÚnamed_parametersÚgradient_accumulation_stepsÚ
parametersÚrequires_gradÚ"register_post_accumulate_grad_hookr   Úupdate)r   r   rD   rF   rG   r    rH   Úis_layerwiseÚoptimizer_clsÚ
all_linearÚmodule_nameÚmoduleÚtarget_module_existsÚis_regexÚtarget_paramsÚnon_target_paramsÚparam_groupsr\   r^   r]   rT   s                      @@r+   Ú_setup_low_rank_optimizerr~   e   sÝ  øø€ ð "×'Ò'Ñ)Ô)×2Ò2°;Ñ?Ô?€LØð d˜Ô*­lÔ.FÒFÐFÐKaÐFÝ!Ð"b°Ð"bÐ"bÐ"bÑcÔcÐcà% nÔ5€MàÔ Ð(ÝÐhÈ^ÐhÐhÐhÑiÔiÐiå�dÔ/µ$½°Ñ=Ô=ð 
Ýð0ØÔ-ð0ð 0ñ
ô 
ð 	
ð
 €}ÝÐ^À>Ð^Ð^Ð^Ñ_Ô_Ð_õ 	�4Ô,­cÑ2Ô2Ðr°tÔ7P×7XÒ7XÐY\Ð^aÑ7bÔ7bÐfrÒ7rð ð ÐØ$×2Ò2Ñ4Ô4ð <ð <Ñˆ�VÝ)CØÔ% {ÀDð*
ñ *
ô *
Ñ&Ð˜hõ ˜&¥"¤)Ñ,Ô,ð 	Ø#ð t¨Hð tÝ—’ +ÐrÐrÀ^ÐrÐrÐrÑsÔsÐsØà#ð 	¨Jð 	Øà×"Ò" ;°Ñ#:Ñ;Ô;Ð;Ð;å
ÐÑÔ 1Ò$Ð$ÝÐg¸ÐgÐgÈ$ÔJcÐgÐgÐgÑhÔhÐhàXÐXÐXÐX 5×#9Ò#9Ñ#;Ô#;ÐXÑXÔX€MØ`Ð`Ð`Ð` u×'=Ò'=Ñ'?Ô'?Ð`Ñ`Ô`Ðð 
Ð$Ð%Ø	�=Ð1 LÐ1ð€Lð
 ð DØÔ+¨qÒ0Ð0ÝÐb¨.ÐbÐbÐbÑcÔcÐcàˆØ&ð 	]ð 	]ˆEØ$1 M°H¸u¸gÐ3FÐ2GÐ$\Ð$\ÐK[Ð$\Ð$\ˆN˜5Ñ!Ð!Ø"ð 	mð 	mˆEØ$1 M°H¸u¸gÐ3VÈÐ3VÐ2WÐ$lÐ$lÐ[kÐ$lÐ$lˆN˜5Ñ!Ð!ð	2ð 	2ð 	2ð 	2ð 	2ð
 ×%Ò%Ñ'Ô'ð 	Ið 	IˆEØÔ"ð IØ×8Ò8¸ÑHÔHÐHøå/ˆØ×ÒÐ!1°>Ð BÑCÔCÐCà×Ò˜X |Ð4Ñ5Ô5Ð5ØÐ*Ð*Ð*r*   Úctxc                óX   — | j                              dddœ¦  «         t          | j         fS )zGet Adafactor optimizer.F©Úscale_parameterÚrelative_step)r    rs   r   )r   s    r+   Ú_get_adafactorr„   Ã   s/   € àÔ×Ò°EÈEÐ RÐ RÑSÔSÐSÝ�cÔ*Ð*Ð*r*   c                óÊ   — ddl m} | j                             | j        ¦  «         | j        j        t          j        k    r| j                             ddi¦  «         || j        fS )z/Get PyTorch AdamW optimizer (regular or fused).r   ©ÚAdamWÚfusedT)	Útorch.optimr‡   r    rs   r!   r   rA   r   ÚADAMW_TORCH_FUSED©r   r‡   s     r+   Ú_get_adamw_torchrŒ   É   sh   € à!Ð!Ð!Ð!Ð!Ð!àÔ×Ò ¤Ñ0Ô0Ð0Ø
„x„~�Ô9Ò9Ð9ØÔ×#Ò# W¨d OÑ4Ô4Ð4Ø�#Ô&Ð&Ð&r*   c                óš   — 	 ddl m} | j                             | j        ¦  «         || j        fS # t
          $ r t          d¦  «        ‚w xY w)z'Get Torch XLA syncfree AdamW optimizer.r   r†   z7Trainer failed to import syncfree AdamW from torch_xla.)Útorch_xla.amp.syncfreer‡   r    rs   r!   ÚImportErrorre   r‹   s     r+   Ú_get_adamw_torch_xlar�   Ó   so   € ðTØ0Ð0Ð0Ð0Ð0Ð0àÔ×#Ò# C¤OÑ4Ô4Ð4Ø�cÔ*Ð*Ð*øÝð Tð Tð TÝÐRÑSÔSÐSðTøøøó	   ‚-0 °A
c                óš   — 	 ddl m} | j                             | j        ¦  «         || j        fS # t
          $ r t          d¦  «        ‚w xY w)zGet NPU Fused AdamW optimizer.r   )ÚNpuFusedAdamWz3Trainer failed to import FusedAdamW from torch_npu.)Útorch_npu.optimr“   r    rs   r!   r�   re   )r   r“   s     r+   Ú_get_adamw_torch_npu_fusedr•   Þ   so   € ðPØ1Ð1Ð1Ð1Ð1Ð1àÔ×#Ò# C¤OÑ4Ô4Ð4Ø˜cÔ2Ð2Ð2øÝð Pð Pð PÝÐNÑOÔOÐOðPøøør‘   c                ó  — t          ¦   «         st          d¦  «        ‚ddlm}m}m} | j        j        }d|v }d|v rdnd}d}| j        }d	|v r|}�nod
|v r|}d| j        j	        | j        j
        fi}�nMd|v r|}| j        }�n>d|v �r9ddlm}	 |	}t          | j                             d| j        j	        ¦  «        ¦  «        t          | j                             d| j        j
        ¦  «        ¦  «        t          | j                             dd¦  «        ¦  «        ft          | j                             dd¦  «        ¦  «        t          | j                             d| j        j        ¦  «        ¦  «        dœ}d| j        v rt!          | j        d         ¦  «        |d<   d| j        v rt!          | j        d         ¦  «        |d<   d|i}
d|vr||
d<   | j                             |¦  «         | j                             |
¦  «         || j        fS )z;Get bitsandbytes optimizer (AdamW, Lion, RMSprop variants).ziYou need to install `bitsandbytes` in order to use bitsandbytes optimizers: `pip install -U bitsandbytes`r   )r‡   ÚLionÚRMSpropÚpagedÚ8bité   é    NÚadamÚlionÚbetasÚrmspropÚademamix)ÚAdEMAMixÚbeta1Úbeta2Úbeta3g§èH.ÿï?Úalphag      @Úeps)rŸ   r¦   r§   Út_alphaÚt_beta3Ú
optim_bitsÚis_paged)r   r�   Úbitsandbytes.optimr‡   r—   r˜   r   rA   r!   Ú
adam_beta1Ú
adam_beta2r#   r¢   ÚfloatÚgetÚadam_epsilonÚintr    rs   )r   r‡   r—   r˜   Ú
optim_namer«   rª   ru   Úadditional_optim_kwargsr¢   Ú
bnb_kwargss              r+   Ú_get_bitsandbytes_optimizerr¶   é   sV  € å$Ñ&Ô&ð 
ÝØwñ
ô 
ð 	
ð 8Ð7Ð7Ð7Ð7Ð7Ð7Ð7Ð7Ð7à””€JØ˜*Ð$€HØ 
Ð*Ð*��°€JØ€MØ!œoÐà�ÐÐØˆ‰Ø	�:Ð	Ð	ØˆØ#*¨S¬XÔ-@À#Ä(ÔBUÐ,VÐ"WÐÑØ	�jÐ	 Ð	 ØˆØ"%¤.ÐÑØ	�zÐ	!Ñ	!Ø/Ð/Ð/Ð/Ð/Ð/à ˆõ �c”n×(Ò(¨°#´(Ô2EÑFÔFÑGÔGÝ�c”n×(Ò(¨°#´(Ô2EÑFÔFÑGÔGÝ�c”n×(Ò(¨°&Ñ9Ô9Ñ:Ô:ðõ
 ˜3œ>×-Ò-¨g°sÑ;Ô;Ñ<Ô<Ý˜œ×+Ò+¨E°3´8Ô3HÑIÔIÑJÔJð#
ð #
Ðð ˜œÐ&Ð&Ý14°S´^ÀIÔ5NÑ1OÔ1OÐ# IÑ.Ø˜œÐ&Ð&Ý14°S´^ÀIÔ5NÑ1OÔ1OÐ# IÑ.à 
Ð+€JØ˜
Ð"Ð"Ø!)ˆ
�:ÑàÔ×ÒÐ 7Ñ8Ô8Ð8ØÔ×Ò 
Ñ+Ô+Ð+Ø˜#Ô.Ð.Ð.r*   c                ó,  — 	 ddl m} | j                             | j        ¦  «         | j                             t          | j                             dd¦  «        ¦  «        t          t          | j                             dd¦  «        ¦  «        t          t          | j                             dd¦  «        ¦  «        t          t          | j                             dd	¦  «        ¦  «        d
œ¦  «         || j        fS # t          $ r t          d¦  «        ‚w xY w)z!Get AnyPrecision AdamW optimizer.r   )ÚAnyPrecisionAdamWÚuse_kahan_summationÚFalseÚmomentum_dtypeÚfloat32Úvariance_dtypeÚcompensation_buffer_dtypeÚbfloat16)r¹   r»   r½   r¾   z4Please install https://github.com/pytorch/torchdistx)Útorchdistx.optimizersr¸   r    rs   r!   r   r#   r°   Úgetattrr@   r�   re   )r   r¸   s     r+   Ú_get_adamw_anyprecisionrÂ     s  € ðQØ;Ð;Ð;Ð;Ð;Ð;àÔ×#Ò# C¤OÑ4Ô4Ð4ØÔ×#Ò#å'0°´×1CÒ1CÐDYÐ[bÑ1cÔ1cÑ'dÔ'dÝ")­%°´×1CÒ1CÐDTÐV_Ñ1`Ô1`Ñ"aÔ"aÝ")­%°´×1CÒ1CÐDTÐV_Ñ1`Ô1`Ñ"aÔ"aÝ-4Ý˜3œ>×-Ò-Ð.IÈ:ÑVÔVñ.ô .ð	ð ñ		
ô 		
ð 		
ð ! #Ô"6Ð6Ð6øÝð Qð Qð QÝÐOÑPÔPÐPðQøøøs   ‚C6C9 Ã9Dc                ó  — | j                              ¦   «         }| j        rXdD ](}|| j        v rt          | j        |         ¦  «        ||<   Œ)d| j        v r$| j        d                              ¦   «         dv |d<   t
          j        j        |fS )zGet SGD optimizer.)ÚmomentumÚ	dampeningÚweight_decayÚnesterov©ÚtrueÚ1Úyes)r    Úcopyr#   r¯   r_   r@   rA   ÚSGD©r   Úkwargsr7   s      r+   Ú_get_sgdrÐ   0  s•   € àÔ!×&Ò&Ñ(Ô(€FØ
„~ð \Ø<ð 	9ð 	9ˆCØ�c”nÐ$Ð$Ý# C¤N°3Ô$7Ñ8Ô8��s‘øØ˜œÐ'Ð'Ø!$¤°
Ô!;×!AÒ!AÑ!CÔ!CÐG[Ð![ˆF�:ÑÝŒ;Œ?˜FÐ"Ð"r*   c                ó¾   — | j                              ¦   «         }| j        r+dD ](}|| j        v rt          | j        |         ¦  «        ||<   Œ)t          j        j        |fS )zGet Adagrad optimizer.)Úlr_decayrÆ   r§   )r    rÌ   r#   r¯   r@   rA   ÚAdagradrÎ   s      r+   Ú_get_adagradrÔ   <  se   € àÔ!×&Ò&Ñ(Ô(€FØ
„~ð 9Ø6ð 	9ð 	9ˆCØ�c”nÐ$Ð$Ý# C¤N°3Ô$7Ñ8Ô8��s‘øÝŒ;Ô Ð&Ð&r*   c                ó  — | j                              ¦   «         }| j        rXdD ](}|| j        v rt          | j        |         ¦  «        ||<   Œ)d| j        v r$| j        d                              ¦   «         dv |d<   t
          j        j        |fS )zGet RMSprop optimizer.)rÄ   r¦   r§   rÆ   ÚcenteredrÈ   )r    rÌ   r#   r¯   r_   r@   rA   r˜   rÎ   s      r+   Ú_get_rmspropr×   F  s–   € àÔ!×&Ò&Ñ(Ô(€FØ
„~ð \Ø?ð 	9ð 	9ˆCØ�c”nÐ$Ð$Ý# C¤N°3Ô$7Ñ8Ô8��s‘øØ˜œÐ'Ð'Ø!$¤°
Ô!;×!AÒ!AÑ!CÔ!CÐG[Ð![ˆF�:ÑÝŒ;Ô Ð&Ð&r*   c                óÔ  — t          ¦   «         st          d¦  «        ‚ddlm}m}m} t          j        |t          j        |t          j	        |t          j
        |t          j        |t          j        |i}t          | j                             dd¦  «        ¦  «        t          | j                             dd¦  «        ¦  «        t!          | j                             dd	¦  «        ¦  «        | j                             d
d¦  «        dœ}t#          | j        | j        | j        j        ||| j        ¦  «        \  }}| j        j        t          j	        k    r|                     dddœ¦  «         ||fS )zGet GaLore optimizer.z�You need to install `galore_torch` in order to use GaLore optimizers. Install it with `pip install git+https://github.com/jiaweizzhao/GaLore`r   )ÚGaLoreAdafactorÚGaLoreAdamWÚGaLoreAdamW8bitÚranké€   Úupdate_proj_gapéÈ   Úscaleg      Ð?Ú	proj_typeÚstd)rÜ   rÞ   rà   rá   Fr�   )r   r�   Úgalore_torchrÙ   rÚ   rÛ   r   ÚGALORE_ADAMWÚGALORE_ADAMW_8BITÚGALORE_ADAFACTORÚGALORE_ADAMW_LAYERWISEÚGALORE_ADAMW_8BIT_LAYERWISEÚGALORE_ADAFACTOR_LAYERWISEr²   r#   Úpopr¯   r~   r   r   rA   r    rs   )r   rÙ   rÚ   rÛ   rF   Úgalore_optim_kwargsru   r    s           r+   Ú_get_galore_optimizerrì   R  sn  € å$Ñ&Ô&ð 
ÝðVñ
ô 
ð 	
ð KÐJÐJÐJÐJÐJÐJÐJÐJÐJõ 	Ô# [ÝÔ(¨/ÝÔ'¨ÝÔ-¨{ÝÔ2°OÝÔ1°?ðÐõ �C”N×&Ò& v¨sÑ3Ô3Ñ4Ô4Ý˜sœ~×1Ò1Ð2CÀSÑIÔIÑJÔJÝ�s”~×)Ò)¨'°4Ñ8Ô8Ñ9Ô9Ø”^×'Ò'¨°UÑ;Ô;ð	ð Ðõ '@ØŒ�#”)˜SœXœ^Ð->Ð@SÐUXÔUiñ'ô 'Ñ#€MÐ#ð „x„~�Ô8Ò8Ð8Ø×Ò°EÈEÐ RÐ RÑSÔSÐSØÐ*Ð*Ð*r*   c           
     ó–  — t          ¦   «         st          d¦  «        ‚ddlm} t          j        |t          j        |i}t          | j         	                    dd¦  «        ¦  «        | j         	                    dd¦  «        | j         	                    dd	¦  «        t          | j         	                    d
d¦  «        ¦  «        t          | j         	                    dd¦  «        ¦  «        | j         	                    dd¦  «        dœ}|                     | j        ¦  «         t          | j        | j        | j        j        ||| j        ¦  «        S )zGet Apollo optimizer.zŒYou need to install `apollo_torch` in order to use APOLLO optimizers. Install it with `pip install git+https://github.com/zhuhanqing/APOLLO`r   )ÚAPOLLOAdamWrÜ   rÝ   ÚprojÚrandomÚ
scale_typeÚchannelrÞ   rß   rà   ç      ð?rá   râ   )rÜ   rï   rñ   rÞ   rà   rá   )r   r�   Úapollo_torchrî   r   ÚAPOLLO_ADAMWÚAPOLLO_ADAMW_LAYERWISEr²   r#   rê   r¯   rs   r!   r~   r   r   rA   r    )r   rî   rF   Úapollo_optim_kwargss       r+   Ú_get_apollo_optimizerrø   s  sA  € å$Ñ&Ô&ð 
ÝðUñ
ô 
ð 	
ð )Ð(Ð(Ð(Ð(Ð(õ 	Ô# [ÝÔ-¨{ðÐõ �C”N×&Ò& v¨sÑ3Ô3Ñ4Ô4Ø”×"Ò" 6¨8Ñ4Ô4Ø”n×(Ò(¨°yÑAÔAÝ˜sœ~×1Ò1Ð2CÀSÑIÔIÑJÔJÝ�s”~×)Ò)¨'°3Ñ7Ô7Ñ8Ô8Ø”^×'Ò'¨°UÑ;Ô;ðð Ðð ×Ò˜sœÑ/Ô/Ð/å$ØŒ�#”)˜SœXœ^Ð->Ð@SÐUXÔUiñô ð r*   c                óð   — t          ¦   «         st          d¦  «        ‚| j        €t          d¦  «        ‚ddlm}m} d| j        j        v r|n|}| j	         
                    d| j        i¦  «         || j	        fS )zGet LOMO optimizer.zjYou need to install `lomo_optim` in order to use LOMO optimizers. Install it with `pip install lomo-optim`NzMYou need to pass a `model` in order to correctly initialize a LOMO optimizer.r   )ÚAdaLomoÚLomoÚadar   )r   r�   r   re   Ú
lomo_optimrú   rû   r   rA   r    rs   )r   rú   rû   ru   s       r+   Ú_get_lomo_optimizerrþ   �  s›   € åÑÔð 
Ýð7ñ
ô 
ð 	
ð
 „yÐÝÐhÑiÔiÐià(Ð(Ð(Ð(Ð(Ð(Ð(Ð(à$¨¬¬Ð6Ð6�G�G¸D€MØÔ×Ò ¨#¬)Ð 4Ñ5Ô5Ð5Ø˜#Ô.Ð.Ð.r*   c                ó  — t          ¦   «         st          d¦  «        ‚ddlm} | j                             t          | j                             dd¦  «        ¦  «        t          | j                             dd¦  «        ¦  «        t          | j                             dd	¦  «        ¦  «        t          | j                             d
d	¦  «        ¦  «        t          | j                             dd¦  «        ¦  «        dœ¦  «         || j        fS )zGet GrokAdamW optimizer.z5Please install grokadamw with `pip install grokadamw`r   )Ú	GrokAdamWÚ
alpha_initg\�Âõ(\ï?Úlambç       @Úgammagš™™™™™¹?Úgrokking_signal_decay_rateÚgradient_clippingró   )r  r  r  r  r  )	r   re   Ú	grokadamwr   r    rs   r¯   r#   r°   )r   r   s     r+   Ú_get_grokadamwr  ¢  sû   € å!Ñ#Ô#ð RÝÐPÑQÔQÐQà#Ð#Ð#Ð#Ð#Ð#àÔ×Òå ¤× 2Ò 2°<ÀÑ FÔ FÑGÔGÝ˜#œ.×,Ò,¨V°SÑ9Ô9Ñ:Ô:Ý˜3œ>×-Ò-¨g°sÑ;Ô;Ñ<Ô<Ý*/°´×0BÒ0BÐC_ÐadÑ0eÔ0eÑ*fÔ*fÝ!& s¤~×'9Ò'9Ð:MÈsÑ'SÔ'SÑ!TÔ!Tð	
ð 	
ñô ð ð �cÔ*Ð*Ð*r*   c           	     ól  — t          ¦   «         rGt          j        t          j                             d¦  «        ¦  «        t          j        d¦  «        k     rt          d¦  «        ‚t          j        t          j                             d¦  «        ¦  «        t          j        d¦  «        k    rt          d¦  «        ‚t          j        t          j                             d¦  «        ¦  «        t          j        d¦  «        k    r	dd	lm}m} ndd	l	m}m} | j
        j        t          j        k    r|}n|}| j                             | j                             d
d¦  «        t%          | j                             dd¦  «        ¦  «        dœ¦  «         | j                             | j        ¦  «         || j        fS )z%Get TorchAO 4-bit or 8-bit optimizer.Útorchaoz0.4.0z®You need to have `torchao>=0.4.0` in order to use torch 4-bit optimizers. Install it with `pip install torchao` or follow the instructions here: https://github.com/pytorch/aor@   z2.4z¹You need to have `torch>2.4` in order to use torch 4-bit optimizers. Install it with `pip install --upgrade torch` it is available on pipy. Otherwise, you need to install torch nightly.z0.11.0r   )Ú	AdamW4bitÚ	AdamW8bitÚ
block_sizeé   Úbf16_stochastic_roundrº   )r  r  )r   r   ÚparseÚ	importlibÚmetadatar�   Útorchao.optimr  r  Útorchao.prototype.low_bit_optimr   rA   r   ÚADAMW_TORCH_4BITr    rs   r#   r°   r   r!   )r   r  r  ru   s       r+   Ú_get_torchao_optimizerr  µ  s§  € åÑ!Ô!ð 
¥W¤]µ9Ô3E×3MÒ3MÈiÑ3XÔ3XÑ%YÔ%YÕ\cÔ\iÐjqÑ\rÔ\rÒ%rÐ%rÝð,ñ
ô 
ð 	
õ
 „}•YÔ'×/Ò/°Ñ8Ô8Ñ9Ô9½W¼]È5Ñ=QÔ=QÒQÐQÝð<ñ
ô 
ð 	
õ „}•YÔ'×/Ò/°	Ñ:Ô:Ñ;Ô;½w¼}ÈXÑ?VÔ?VÒVÐVØ6Ð6Ð6Ð6Ð6Ð6Ð6Ð6Ð6àHÐHÐHÐHÐHÐHÐHÐHà
„x„~�Ô8Ò8Ð8Ø!ˆˆà!ˆàÔ×Òàœ.×,Ò,¨\¸3Ñ?Ô?Ý%.¨s¬~×/AÒ/AÐBYÐ[bÑ/cÔ/cÑ%dÔ%dð	
ð 	
ñô ð ð Ô×Ò ¤Ñ0Ô0Ð0Ø˜#Ô.Ð.Ð.r*   c           	     óÖ  — t          ¦   «         st          d¦  «        ‚ddlm}m} i }d}| j        j        t          j        k    r0t          d¦  «        st          d¦  «        ‚ddlm	} |}| j
        }d}nP| j        j        t          j        k    r
|}| j
        }n,| j        j        t          j        k    r|}nt          d	¦  «        ‚| j        j        |d
<   |r| j        j        |d<   |                     t#          | j                             dd¦  «        ¦  «        t#          | j                             dd¦  «        ¦  «        dœ¦  «         | j                             |¦  «         || j        fS )zGet ScheduleFree optimizer.zwYou need to install `schedulefree` in order to use schedulefree optimizers. Install it with `pip install schedulefree.`r   )ÚAdamWScheduleFreeÚSGDScheduleFreeTz1.4.0z‚You need to install `schedulefree>=1.4.0` in order to use RAdamScheduleFree optimizer. Install it with `pip install schedulefree.`)ÚRAdamScheduleFreeFzInvalid schedulefree optimizerrÆ   Úwarmup_stepsÚweight_lr_powerr  Úrg        )r  r  )r   r�   Úschedulefreer  r  r   rA   r   ÚSCHEDULE_FREE_RADAMr  r!   ÚSCHEDULE_FREE_ADAMWÚSCHEDULE_FREE_SGDre   rÆ   r  rs   r¯   r#   r°   r    )r   r  r  r´   Úrequire_warmupr  ru   s          r+   Ú_get_schedule_free_optimizerr#  Ø  s¦  € å$Ñ&Ô&ð 
Ýð:ñ
ô 
ð 	
ð @Ð?Ð?Ð?Ð?Ð?Ð?Ð?à ÐØ€Nà
„x„~�Ô;Ò;Ð;Ý(¨Ñ1Ô1ð 	Ýð>ñô ð ð 	3Ð2Ð2Ð2Ð2Ð2à)ˆØ"%¤/ÐØˆˆØ	ŒŒ�>Ô=Ò	=Ð	=Ø)ˆØ"%¤/ÐÐØ	ŒŒ�>Ô;Ò	;Ð	;Ø'ˆˆåÐ9Ñ:Ô:Ð:à.1¬hÔ.CÐ˜NÑ+Øð HØ25´(Ô2GÐ Ñ/Ø×"Ò"å$ S¤^×%7Ò%7Ð8IÈ3Ñ%OÔ%OÑPÔPÝ�s”~×)Ò)¨#¨sÑ3Ô3Ñ4Ô4ð	
ð 	
ñô ð ð Ô×ÒÐ 7Ñ8Ô8Ð8Ø˜#Ô.Ð.Ð.r*   c                óü  — t          ¦   «         st          d¦  «        ‚ddlm} | j                             dd¦  «        }|�t          |¦  «        }| j                             dd¦  «        }|�t          |¦  «        }| j        j	        | j
        d<   t          | j                             dd	¦  «        ¦  «        ||d
œ}| j                             | j
        ¦  «         | j                             |¦  «         || j        fS )z,Get StableAdamW optimizer from torch-optimi.zwYou need to install `torch-optimi` in order to use stable_adamw optimizers. Install it with `pip install torch-optimi`.r   )ÚStableAdamWÚmax_lrNÚ	kahan_sumrÆ   Údecouple_lrF)r(  r&  r'  )r   r�   Úoptimir%  r#   rê   r¯   r;   r   rÆ   r!   r    rs   )r   r%  r&  r'  Ústable_adamw_kwargss        r+   Ú_get_stable_adamwr+    s  € å$Ñ&Ô&ð 
Ýð:ñ
ô 
ð 	
ð #Ð"Ð"Ð"Ð"Ð"àŒ^×Ò ¨$Ñ/Ô/€FØÐÝ�v‘”ˆà”×"Ò" ;°Ñ5Ô5€IØÐÝ˜‘O”Oˆ	à&)¤hÔ&;€C„O�NÑ#å˜CœN×.Ò.¨}¸eÑDÔDÑEÔEØØðð Ðð Ô×Ò ¤Ñ0Ô0Ð0ØÔ×ÒÐ 3Ñ4Ô4Ð4Ø˜Ô,Ð,Ð,r*   zdict[str, OptimizerHandler]Ú_OPTIMIZER_HANDLERS)r,   r-   r.   r"   )r:   r   r.   r;   )T)r   r   r   r   rD   rE   rF   r   rG   r   r    r   rH   r;   r.   rI   )r   r   r.   rI   )sr'   Ú
__future__r   Úimportlib.metadatar  ÚloggingÚcollections.abcr   Údataclassesr   Útypingr   r   r@   Ú	packagingr   r	   Úoptimizationr   Útrainer_pt_utilsr   Útrainer_utilsr   Útraining_argsr   r   Úutilsr   r   r   r   r   r   r   r   r   Úmodeling_utilsr   r   Ú	getLoggerr$   rj   r   r9   ÚtupleÚdictrE   ÚOptimizerHandlerrC   r~   r„   rŒ   r�   r•   r¶   rÂ   rÐ   rÔ   r×   rì   rø   rþ   r  r  r#  r+  Ú	ADAMW_BNBÚ
ADAMW_8BITÚPAGED_ADAMWÚPAGED_ADAMW_8BITÚADEMAMIXÚADEMAMIX_8BITÚPAGED_ADEMAMIXÚPAGED_ADEMAMIX_8BITÚLIONÚ	LION_8BITÚ
PAGED_LIONÚPAGED_LION_8BITÚRMSPROP_BNBÚRMSPROP_8BITÚRMSPROP_32BITÚ_BITSANDBYTES_OPTIMIZERSrä   rå   ræ   rç   rè   ré   Ú_GALORE_OPTIMIZERSrõ   rö   Ú_APOLLO_OPTIMIZERSr  ÚADAMW_TORCH_8BITÚ_TORCHAO_OPTIMIZERSr  r   r!  Ú_SCHEDULE_FREE_OPTIMIZERSÚ	ADAFACTORÚADAMW_TORCHrŠ   ÚADAMW_TORCH_XLAÚADAMW_TORCH_NPU_FUSEDÚADAMW_ANYPRECISIONrÍ   ÚADAGRADÚRMSPROPÚ	GROKADAMWÚSTABLE_ADAMWÚLOMOÚADALOMOÚfromkeysr,  r(   r)   r*   r+   ú<module>r_     s�  ððð ð ð #Ð "Ð "Ð "Ð "Ð "à Ð Ð Ð Ø €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !Ø %Ð %Ð %Ð %Ð %Ð %Ð %Ð %à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à #Ð #Ð #Ð #Ð #Ð #Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð ð 1Ø/Ð/Ð/Ð/Ð/Ð/Ø0Ð0Ð0Ð0Ð0Ð0à	ˆÔ	˜8Ñ	$Ô	$€ð ðð ð ð ð ñ ô ñ „ððð ð ð ð Ð-Ð.°°c¸4ÀÀSÀ¼>Ð6IÔ0JÐJÔKÐ ðð ð ð ð8 $(ðV+ð V+ð V+ð V+ð V+ð|+ð +ð +ð +ð'ð 'ð 'ð 'ðTð Tð Tð TðPð Pð Pð Pð//ð //ð //ð //ðdQð Qð Qð Qð*	#ð 	#ð 	#ð 	#ð'ð 'ð 'ð 'ð	'ð 	'ð 	'ð 	'ð+ð +ð +ð +ðBð ð ð ð:/ð /ð /ð /ð$+ð +ð +ð +ð& /ð  /ð  /ð  /ðF)/ð )/ð )/ð )/ðX-ð -ð -ð -ðD ÔØÔØÔØÔ#ØÔØÔ ØÔ!ØÔ&ØÔØÔØÔØÔ"ØÔØÔØÔ ðÐ ð& ÔØÔ$ØÔ#ØÔ)ØÔ.ØÔ-ðÐ ð ÔØÔ)ðÐ ð Ô#ØÔ#ðÐ ð Ô&ØÔ&ØÔ$ðÐ ð Ô˜nØÔÐ 0ØÔ$Ð&6ØÔ"Ð$8ØÔ(Ð*DØÔ%Ð'>ØÔ˜ØÔ˜LØÔ˜LØÔ˜nØÔÐ!2ØÔÐ,ØÔÐ/ð4ð ‡m‚mÐ,Ð.IÑJÔJð4ð ‡m‚mÐ&Ð(=Ñ>Ô>ð4ð  ‡m‚mÐ&Ð(=Ñ>Ô>ð!4ð" ‡m‚mÐ'Ð)?Ñ@Ô@ð#4ð$ ‡m‚mÐ-Ð/KÑLÔLð%4Ð ð ð ð ñ ð ð r*   