§
    ŠŠtjY÷  ã                   óP
  — U d dl Z d dlZd dlZd dlZd dlmZmZmZ d dlm	Z	m
Z
mZ d dlmZ d dlmZmZmZ d dlZd dlmZ d dlmZ d dlmZ d dlmZmZmZmZmZm Z  d dl!m"Z" d d	l#m$Z$m%Z%m&Z'm(Z(m)Z)m*Z*m+Z+m,Z, d d
l-m.Z.m/Z/ d dl0m1Z1 d dl2m3Z3 d dl4m5Z5 d dl6m7Z8 d dl9m:Z: g d¢Z;dZ<dZ=dZ>dZ?e@eA         ZBe3ez  ejC        z  eDz  eEz  eAz  ZFeFeGeF         z  eHeF         z  eIeAdf         z  ZJeIeAeJf         ZKeGeK         ZLeIeAeKeLz  f         ZM e@¦   «         ZNe@e         eOd<   e jP        d„ ¦   «         ZQe
 G d„ d¦  «        ¦   «         ZRe
 G d„ deR¦  «        ¦   «         ZS	 	 	 dMdejT        deAd eAd!eUd"eUd#eBfd$„ZV G d%„ d&¦  «        ZWdNd'„ZXddd(œdejT        d)eHejY        jZ        d*f         d+eUd,e@ejT                 dz  d-eRdz  d#eSfd.„Z[d/eIeAeJf         d0eMd1eSd#dfd2„Z\d3ejT        ejY        jZ        z  d4eAd#efd5„Z]dejT        d1eSd#ej^        dz  fd6„Z_d7eIeAef         d1eSd#eIeAef         fd8„Z` eja        ¦   «         dejT        d1eSd#eIeAeJf         fd9„¦   «         Zb eja        ¦   «         dejT        d7eIeAeJf         d1eSd#e5fd:„¦   «         Zcd;ejY        jZ        d#dfd<„Zdd7eMd#eIeAeJf         fd=„Zed;ejY        jZ        d7eIeAeJf         d1eSd#eMfd>„Zf eja        ¦   «         dejT        d?eHejY        jZ        d*f         d1eSd#eMfd@„¦   «         ZgdejT        d;ejY        jZ        d0eMd1eSd#eMf
dA„Zh eja        ¦   «         dejT        d?eHejY        jZ        d*f         d7eMd1eSd#df
dB„¦   «         Ziddd(œdejT        d,e@ejT                 dz  d-eRdz  d#eIeAeJf         fdC„Zjddd(œdejT        d?ejY        jZ        eejY        jZ                 z  d,e@ejT                 dz  d-eRdz  d#eMf
dD„Zkddd(œdejT        d?ejY        jZ        eejY        jZ                 z  d,e@ejT                 dz  d-eRdz  d#eHeIeAeJf         eMf         f
dE„ZldejT        d7eIejT        eIeAeJf         f         eIeAeJf         z  d#eIeAeJf         fdF„ZmddGœdejT        d/eIeAeJf         d-eRdz  d#e5fdH„ZnddGœdejT        d?ejY        jZ        eejY        jZ                 z  d0eMd-eRdz  d#df
dI„ZoddGœdejT        d?ejY        jZ        eejY        jZ                 z  d/eIeAeJf         d0eMd-eRdz  d#e5fdJ„ZpeddGœdejT        d-eRdz  d#dfdK„¦   «         ZqeddGœdejT        d?eHejY        jZ        d*f         d-eRdz  d#dfdL„¦   «         ZrdS )Oé    N)ÚCallableÚ	GeneratorÚIterable)ÚasdictÚ	dataclassÚfield)Úchain)ÚAnyÚcastÚno_type_check)ÚShardedTensor)Ú_broadcast_state_dictÚ_distribute_state_dictÚ_flatten_state_dictÚ_gather_state_dictÚ_offload_state_dict_to_cpuÚ_unflatten_state_dict)Ú_CHECKPOINT_PREFIX)ÚFullOptimStateDictConfigÚFullStateDictConfigÚFullyShardedDataParallelÚOptimStateDictConfigÚShardedOptimStateDictConfigÚShardedStateDictConfigÚStateDictConfigÚStateDictType)Ú._get_module_fsdp_state_if_fully_sharded_moduleÚFSDP_WRAPPED_MODULE)ÚHYBRID_SHARDING_STRATEGIES)ÚDTensor)Ú_IncompatibleKeys)ÚDistributedDataParallel)Útree_map_only)ÚFQNS_TÚPrimitiveTypeÚ	ValueTypeÚDictValueTypeÚListDictValueTypeÚOptimizerStateTypeÚStateDictOptionsÚget_model_state_dictÚget_optimizer_state_dictÚget_state_dictÚset_model_state_dictÚset_optimizer_state_dictÚset_state_dictÚ_flat_paramÚparam_groupsÚparamsÚstater&   Ú_patched_state_dictc               #   óÄ   K  — t          j        ¦   «         } t          j        ¦   «          	 d V — | rt          j        ¦   «          d S d S # | rt          j        ¦   «          w w xY w©N)ÚgcÚ	isenabledÚdisableÚenable)Ú
is_enableds    úe/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/torch/distributed/checkpoint/state_dict.pyÚ_gc_contextr>   R   sh   è è € å”‘”€JÝ„J�L„L€LðØˆˆˆàð 	ÝŒI‰KŒKˆKˆKˆKð	ð 	øˆ:ð 	ÝŒI‰KŒKˆKˆKð	øøøs   ªA ÁAc                   ó„   — e Zd ZU dZdZeed<   dZeed<   dZeed<   dZ	eed<   dZ
eed<   dZeed	<   dZeed
<   dZeed<   dS )r*   ap  
    This dataclass specifies how get_state_dict/set_state_dict will work.

    - ``full_state_dict``: if this is set to True, all the tensors in the
      returned state_dict will be gathered. No ShardedTensor and DTensor
      will be in the returned state_dict.

    - ``cpu_offload``: offload all the tensors to cpu. To prevent CPU OOM, if
      ``full_state_dict`` is also true, then only the rank0 will get the
      state_dict and all other ranks will get empty state_dict.

    - ``ignore_frozen_params``: if the value is True, the returned state_dict
      won't contain any frozen parameters -- the ``requires_grad`` is False.
      The default value is False.

    - ``keep_submodule_prefixes`` (deprecated): when ``submodules`` is not None, this option
      indicates whether to keep the submodule prefixes from the state_dict keys.
      or example, if the submodule is ``module.pretrain`` and the full FQN of
      the parameter is ``pretrain.layer1.weight`` of the param. When this option
      is True, the parameter's key in the returned state_dict will be
      ``pretrain.layer1.weight``. If the options is False, the key will be
      ``layer1.weight``.
      Note that if ``keep_submodule_prefixes`` is False, there may be conflicted
      FQNs, hence there should be only one submodule in ``submodules``.

    - ``strict``: the ``strict`` option when ``set_state_dict`` calls
      model.load_state_dict().

    - ``broadcast_from_rank0``: when the option is True, rank0 should receive a
       full state_dict and will broadcast the tensors in the state_dict/
       optim_state_dict one by one to other ranks. Other ranks will receive
       the tensors and shard according to the local shards in the model and
       optimizer. ``full_state_dict`` must be set to True when using this option.
       This option currently only supports DTensor, not the legacy ShardedTensor.
    FÚfull_state_dictÚcpu_offloadÚignore_frozen_paramsTÚkeep_submodule_prefixesÚstrictÚbroadcast_from_rank0Úflatten_optimizer_state_dictÚ_fqn_modifiersÚdsd_fqn_modifiersN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r@   ÚboolÚ__annotations__rA   rB   rC   rD   rE   rF   rH   Ústr© ó    r=   r*   r*   ]   s¤   € € € € € € ð"ð "ðH "€O�TÐ!Ð!Ñ!Ø€K�ÐÐÑØ!&Ð˜$Ð&Ð&Ñ&Ø$(Ð˜TÐ(Ð(Ñ(Ø€FˆDÐÐÑØ!&Ð˜$Ð&Ð&Ñ&Ø).Ð  $Ð.Ð.Ñ.Ø-Ð�sÐ-Ð-Ñ-Ð-Ð-rQ   r*   c                   óN  — e Zd ZU  ee¬¦  «        Zeeej        z  e	ej        z  f         e
d<    ee¬¦  «        Zeeej        z  e	ej        z  f         e
d<    ee¬¦  «        Zee         e
d<   dZee
d<   dZee
d<   ej        Zee
d<    ee¬¦  «        Zeej                 e
d	<   d
S )Ú_StateDictInfo)Údefault_factoryÚfqn_param_mappingÚshared_params_mappingÚsubmodule_prefixesTÚhandle_modelÚhandle_optimÚfsdp_contextÚfsdp_modulesN)rI   rJ   rK   r   ÚdictrU   rO   ÚtorchÚTensorr$   rN   rV   ÚsetrW   rX   rM   rY   Ú
contextlibÚnullcontextrZ   r   Úlistr[   ÚnnÚModulerP   rQ   r=   rS   rS   �   s  € € € € € € ð
 	ˆ˜dÐ#Ñ#Ô#ð �tØˆeŒlÑØ�”Ñð	ôð $ð $ñ $ð 	ˆ˜dÐ#Ñ#Ô#ð ˜4ØˆeŒlÑØ�”Ñð	ôð $ð $ñ $ð $) 5¸Ð#=Ñ#=Ô#=Ð˜˜CœÐ=Ð=Ñ=Ø€L�$ÐÐÑØ€L�$ÐÐÑØ'Ô3€L�(Ð3Ð3Ñ3Ø$) E¸$Ð$?Ñ$?Ô$?€L�$�r”y”/Ð?Ð?Ñ?Ð?Ð?rQ   rS   rG   TÚmodelÚnamerH   Úskip_ddp_prefixÚskip_compiler_prefixÚreturnc                 óx  ‡— |                      t          d¦  «        }d|vr|hS |                     d¦  «        }g }| }t          |¦  «        D �]?\  }}	t	          |t
          ¦  «        r8|	dk    rt          d|	› d�¦  «        ‚|j        }|s|                     |	¦  «         ŒSt	          |t          ¦  «        r·|t          |¦  «        dz
  k     rZ||dz            t          k    rFd                     |¦  «        Št          |t          ¦  «        }
‰r‰› d�Šˆfd„|
j        D ¦   «         c S t          |t          ¦  «        }|	t          k    r%|                     |	¦  «         t          ||	¦  «        }�Œt	          |t           j        j        j        ¦  «        r9|	dk    rt          d	|	› d�¦  «        ‚|j        }|s|                     |	¦  «         �Œ|t+          ||¦  «        rM t          ||¦  «        ¦   «                              |	¦  «        x}r t+          ||¦  «        rt          ||¦  «        }|                     |	¦  «         |	t.          j        j        j        k    r'|t          |¦  «        dz
  k    rt5          d
¦  «        ‚�Œ/t          ||	¦  «        }�ŒAd                     |¦  «                              t          d¦  «        hS )aá  
    This API is used to convert the name of a parameter to the FQNs. For FSDP
    without `use_orig_params`, the name of FlatParameter can be mapped to
    multiple original parameters. As a result, the return type of this function
    is `set[str]`.

    Args:
        module (nn.Module): the root model.
        name (str): the name
        skip_ddp_prefix (bool): whether to skip DDP's `module` prefix

    Returns:
        The canonical FQNs based on the model traversal.
    Ú ú.ÚmodulezExpected 'module', got 'ú'é   c                 ó   •— h | ]}‰› |› �’Œ	S rP   rP   )Ú.0ÚfqnÚprefixs     €r=   ú	<setcomp>z_get_fqns.<locals>.<setcomp>É   s$   ø€ ÐEÐEÐE¨S˜6Ð( 3Ð(Ð(ÐEÐEÐErQ   Ú	_orig_modzExpected '_orig_mod', got 'z-Expect `_extra_state` to be the last obj name)Úreplacer   ÚsplitÚ	enumerateÚ
isinstanceÚDDPÚAssertionErrorrm   ÚappendÚFSDPÚlenÚ_FLAT_PARAMÚjoinÚgetattrÚ_fqnsr   r]   Ú_dynamoÚ
eval_frameÚOptimizedModuleru   ÚhasattrÚgetrc   ÚmodulesÚ_EXTRA_STATE_KEY_SUFFIXÚRuntimeError)re   rf   rH   rg   rh   Ú	obj_namesÚfqn_obj_namesÚcurr_objÚiÚcurr_obj_nameÚ
flat_paramÚremoved_fqnrs   s               @r=   Ú	_get_fqnsr’   ž   sè  ø€ ð. �<Š<Õ*¨BÑ/Ô/€DØ
�$€€Øˆvˆà—
’
˜3‘”€IØ€MØ€HÝ% iÑ0Ô0ð &<ñ &<Ñˆˆ=Ý�h¥Ñ$Ô$ð %	<Ø Ò(Ð(Ý$Ð%PÀÐ%PÐ%PÐ%PÑQÔQÐQØ”ˆHØ"ð 4Ø×$Ò$ ]Ñ3Ô3Ð3øÝ˜¥$Ñ'Ô'ð 	<Ø•3�y‘>”> AÑ%Ò%Ð%¨)°A¸±EÔ*:½kÒ*IÐ*IØŸš -Ñ0Ô0�Ý$ X­{Ñ;Ô;�
Øð *Ø &˜\˜\˜\�FØEÐEÐEÐE°JÔ4DÐEÑEÔEÐEÐEÐEÝ˜xÕ)<Ñ=Ô=ˆHØÕ 3Ò3Ð3Ø×$Ò$ ]Ñ3Ô3Ð3Ý" 8¨]Ñ;Ô;�ùÝ˜¥%¤-Ô":Ô"JÑKÔKð 	<Ø Ò+Ð+Ý$Ð%SÀ=Ð%SÐ%SÐ%SÑTÔTÐTØÔ)ˆHØ'ð 4Ø×$Ò$ ]Ñ3Ô3Ð3ùõ �xÐ!2Ñ3Ô3ð BØ"F¥'¨(Ð4EÑ"FÔ"FÑ"HÔ"H×"LÒ"LØ!ñ#ô #ð �;ð Bõ ˜x¨Ñ5Ô5ð BÝ#*¨8°[Ñ#AÔ#A˜Ø× Ò  Ñ/Ô/Ð/Ø¥¤
Ô 1Ô IÒIÐIØ�˜I™œ¨Ñ*Ò*Ð*Ý&Ð'VÑWÔWÐWñ +õ # 8¨]Ñ;Ô;�‘à�HŠH�]Ñ#Ô#×+Ò+Õ,>ÀÑCÔCÐDÐDrQ   c                   ó   — e Zd ZdS )Ú_EXTRA_STATEN)rI   rJ   rK   rP   rQ   r=   r”   r”   ç   s   € € € € € Ø€DrQ   r”   c              #   ó”   ‡‡‡K  — t          ¦   «         Šdt          j        dt          dt          fˆˆˆfd„Š ‰| d¦  «        E d {V —† d S )Nrm   Úcurr_fqnri   c              3   ó¸  •K  — ‰                      | ¦  «         |r|› d�nd}|                      ¦   «         D ]i\  }}|‰v rŒ
t          | ‰¦  «        r7| t          | ‰¦  «        ¦   «                              ¦   «         v r|d d…         }n|› |› �} ‰||¦  «        E d {V —† Œjt          |                      d¬¦  «        |                      d¬¦  «        ¦  «        D ]\  }}|| j        v rŒ|› |› �}||fV — Œt          | j	        dt          j        j        ¦  «        t          j        j        k    r.|› t          j        j        j        › �}|t!          ¦   «         fV — d S d S )Nrl   rk   éÿÿÿÿF)ÚrecurseÚget_extra_state)ÚaddÚnamed_childrenr†   r�   Úvaluesr	   Únamed_buffersÚnamed_parametersÚ_non_persistent_buffers_setÚ	__class__rc   rd   rš   rˆ   rm   r‰   r”   )	rm   r–   rf   Ú	submoduleÚnew_fqnÚobjrH   r™   Úvisited_moduless	         €€€r=   r™   z+_iterate_valid_model_state.<locals>.recurseî   sÊ  øè è € Ø×Ò˜FÑ#Ô#Ð#à%-Ð5�h�>�>�>�>°2ˆØ%×4Ò4Ñ6Ô6ð 	3ð 	3‰OˆD�)Ø˜OÐ+Ð+Øõ ˜Ð 1Ñ2Ô2ð.àÐ>�G FÐ,=Ñ>Ô>Ñ@Ô@×GÒGÑIÔIÐIÐIð # 3 B 3œ-��à%Ð- tÐ-Ð-�Ø�w˜y¨'Ñ2Ô2Ð2Ð2Ð2Ð2Ð2Ð2Ð2Ð2åØ× Ò ¨Ð Ñ/Ô/°×1HÒ1HÐQVÐ1HÑ1WÔ1Wñ
ô 
ð 	ð 	‰IˆD�#ð �vÔ9Ð9Ð9ØØ!Ð) 4Ð)Ð)ˆGØ˜3�,ÐÐÐÐõ �FÔ$Ð&7½¼Ô9RÑSÔSÝŒyÔ(ò)ð )ð "ÐN¥2¤:Ô#4Ô#LÐNÐNˆGØ�<™>œ>Ð)Ð)Ð)Ð)Ð)Ð)ð	)ð )rQ   rk   )r_   rc   rd   rO   r   )re   rH   r™   r¥   s    `@@r=   Ú_iterate_valid_model_stater¦   ë   s~   øøøè è € Ý&)¡e¤e€Oð *�œ	ð  *­Sð  *µYð  *ð  *ð  *ð  *ð  *ð  *ð  *ð  *ðD ˆw�u˜bÑ!Ô!Ð!Ð!Ð!Ð!Ð!Ð!Ð!Ð!Ð!rQ   )Ú
submodulesÚoptionsÚoptims.Ú
optim_onlyr§   r¨   c                óp  — |rt          j        dt          d¬¦  «         |r|st          d¦  «        ‚|pt	          ¦   «         }i }i }t          | ¦  «        D ]½\  }}t          |t          ¦  «        rŒt          | |¦  «        }	| 	                    |¦  «        }
|
�Et          t          t                   ||         ¦  «                             |	¦  «         ||         ||<   n|	                     ¦   «         ||<   |	D ]}
t          |t          ¦  «        s|||
<   ŒŒ¾t          |                     ¦   «         ¦  «        D ]'\  }}|D ]}
t          t"          j        |¦  «        ||
<   Œ Œ(t          ¦   «         }|rt          |¦  «        }|                      ¦   «         D ][\  }}||vrŒ
t          | |¦  «        }	t)          |	¦  «        dk    rt+          d¦  «        ‚|                     d„ |	D ¦   «         ¦  «         Œ\|j        r|j        st1          d	¦  «        ‚t3          j        | ¦  «        }|r¶|j        rJt7          |j        |j        ¬
¦  «        }t;          |j        |j        p|j        ¬
¦  «        }t<          j        }n6tA          |j        ¬¦  «        }tC          |j        ¬¦  «        }t<          j"        }tF          j$        d„ ¦   «         }tK          j&        || |||¬¦  «        }ntF          j'        }tQ          di tS          |¦  «        ¤||||t          t          tT          j+                 |¦  «        | t)          |¦  «        dk    dœ¤ŽS )zW
    Verify the model and options passed by the user and generates _StateDictInfo.
    z¼Getting submodules only model/optim state_dict is deprecated and will be removed in 2.5. This feature can be achieved by manually filtering out the state_dict returned from get_state_dict.é   ©Ú
stacklevelz;Optimizers are not passed in but optim_only is set to True.Nro   z)Submodule FQN should only have 1 instancec              3   ó    K  — | ]	}|› d �V — Œ
dS )rl   NrP   )rq   rr   s     r=   ú	<genexpr>z"_verify_options.<locals>.<genexpr>L  s(   è è € Ð%@Ð%@°C¨ i i iÐ%@Ð%@Ð%@Ð%@Ð%@Ð%@rQ   z?full_state_dict must be True when broadcast_from_rank0 is True.)Úoffload_to_cpuÚ
rank0_only)r±   c              3   ó  K  — t          j        ¦   «         5  t          j        ddt          ¬¦  «         t	          j        | |||¬¦  «        5  d V — d d d ¦  «         n# 1 swxY w Y   d d d ¦  «         d S # 1 swxY w Y   d S )NÚignorezFSDP.state_dict_type)ÚmessageÚcategory©rm   Ústate_dict_typeÚstate_dict_configÚoptim_state_dict_config)ÚwarningsÚcatch_warningsÚfilterwarningsÚFutureWarningr}   r¸   r·   s       r=   Ú$fsdp_state_dict_type_without_warningz=_verify_options.<locals>.fsdp_state_dict_type_without_warningj  s#  è è € õ Ô(Ñ*Ô*ð 
ð 
ÝÔ'ØÐ&<Å}ðñ ô ð õ Ô)Ø!Ø$3Ø&7Ø,Cð	ñ ô ð ð ð �E�E�Eðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð	
ð 
ð 
ñ 
ô 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
øøøð 
ð 
ð 
ð 
ð 
ð 
s5   –5A4ÁAÁA4ÁA 	Á A4Á#A 	Á$A4Á4A8Á;A8r·   r   )rU   rV   rW   rZ   r[   rX   rY   rP   ),r»   Úwarnr¾   rŠ   r*   r¦   ry   r”   r’   r‡   r   r_   rO   ÚupdateÚcopyrb   Úitemsr]   r^   Únamed_modulesr~   r{   rE   r@   Ú
ValueErrorr}   r[   r   rA   r   r   ÚFULL_STATE_DICTr   r   ÚSHARDED_STATE_DICTr`   ÚcontextmanagerÚ	functoolsÚpartialra   rS   r   rc   rd   )re   r©   rª   r§   r¨   rU   rV   rf   ÚparamÚfqnsrr   Úparam_Úfqns_rW   rm   r[   r¹   rº   r¸   r¿   rZ   s                        r=   Ú_verify_optionsrÏ     sñ  € ð ð 
ÝŒðIõ Øð	
ñ 	
ô 	
ð 	
ð ð 
˜&ð 
ÝØIñ
ô 
ð 	
ð Ð+Õ)Ñ+Ô+€GàKMÐØOQÐÝ1°%Ñ8Ô8ð /ð /‰ˆˆeÝ�e�\Ñ*Ô*ð 	Øå˜ Ñ%Ô%ˆØ×#Ò# EÑ*Ô*ˆØˆ?Ý••S”Ð,¨UÔ3Ñ4Ô4×;Ò;¸DÑAÔAÐAØ+<¸UÔ+CÐ! %Ñ(Ð(ð (,§y¢y¡{¤{Ð˜eÑ$Øð 	/ð 	/ˆCÝ˜e¥\Ñ2Ô2ð /Ø).Ð! #Ñ&øð	/õ Ð3×9Ò9Ñ;Ô;Ñ<Ô<ð Dð D‰ˆ�Øð 	Dð 	DˆCÝ)-­e¬l¸FÑ)CÔ)CÐ! #Ñ&Ð&ð	Dõ $'¡5¤5ÐØð AÝ˜‘_”_ˆ
Ø!×/Ò/Ñ1Ô1ð 	Að 	A‰LˆD�&Ø˜ZÐ'Ð'ØÝ˜U DÑ)Ô)ˆDÝ�4‰yŒy˜AŠ~ˆ~Ý$Ð%PÑQÔQÐQØ×%Ò%Ð%@Ð%@¸4Ð%@Ñ%@Ô%@Ñ@Ô@Ð@Ð@àÔ#ð 
¨GÔ,Cð 
ÝØMñ
ô 
ð 	
õ Ô$ UÑ+Ô+€Lð ð /.àÔ"ð 	?Ý 3Ø&Ô2¸wÔ?Rð!ñ !ô !Ðõ '?Ø&Ô2Ø#Ô/ÐO°7Ô3Oð'ñ 'ô 'Ð#õ ,Ô;ˆOˆOå 6Ø&Ô2ð!ñ !ô !Ðõ 'BØ&Ô2ð'ñ 'ô 'Ð#õ ,Ô>ˆOå	Ô	"ð	ð 	ñ 
#Ô	"ð	õ$ !Ô(Ø0ØØ+Ø/Ø$;ð
ñ 
ô 
ˆˆõ "Ô-ˆåð 	ð 	Ý
�‰/Œ/ð	à+Ø3Ø-Ø!Ý�$�rœyœ/¨<Ñ8Ô8Ø#�^Ý˜&‘k”k A’oð	ð 	ð 	ð 	ð 	rQ   Úmodel_state_dictÚoptim_state_dictÚinfoc                 ó¸  — |j         D ]"}t          |¦  «        }|€t          d¦  «        ‚Œ#|j        rP| sN|j        sG|j        s@|j        r|j        s2|j        r+|j	        s$t          dt          j        ¦   «         ›d�¦  «        ‚|j        r)|s'|j        r|j        s|j	        st          d|› �¦  «        ‚| D ]%}t          |v rt          |› dt          › d�¦  «        ‚Œ&d S )Nz)Expected a fsdp_state with a fsdp module.z}The option indicates that model state_dict is required to save or load, but model state_dict is empty.rank = dist.get_rank()=rl   zgThe option indicates that model state_dict is required to save, or load but optim state_dict is empty. z
 contains z6. This can happen if the model is not the root module.)r[   r   r{   rX   rW   rB   rA   r@   rD   rE   rŠ   ÚdistÚget_rankrY   r   )rÐ   rÑ   rÒ   rm   Ú
fsdp_stateÚkeys         r=   Ú_verify_state_dictrØ   “  sŽ  € ð
 Ô#ð Nð NˆÝCÀFÑKÔKˆ
ØÐÝ Ð!LÑMÔMÐMð ð 	Ôð
à ð
ð Ô'ð
ð Ô)ð	
ð
 Ô!ð
ð
 '+Ô&:ð
ð ŒKð
ð Ô)ð
õ ð*å”m‘o”oð*ð *ð *ñ
ô 
ð 	
ð Ôð 	à ð	àÔ%ð	à*.Ô*>ð	ð Ô.ð	õ
 ðMØ:JðMð Mñô ð ð
  ð ð ˆÝ˜#ÐÐÝØð *ð *¥+ð *ð *ð *ñô ð ð ðð rQ   r¤   Úapic                 óŠ   — t          | |¦  «        }|t          v r)t          j        t          | j        |¦  «        | ¬¦  «        }|S )N)Úself)r�   r5   rÉ   rÊ   r¡   )r¤   rÙ   Úcalls      r=   Ú_state_dict_fnrÝ   Á  sC   € Ý�3˜ÑÔ€DØÕ"Ð"Ð"ÝÔ ¥¨¬¸Ñ!<Ô!<À3ÐGÑGÔGˆØ€KrQ   c                 óæ   — t          | t          ¦  «        r| }n*|j        r!t          t          |j        d         ¦  «        }nd S |j        t
          v rd S |j        }t          |t          ¦  «        rd S |S )Nr   )ry   r}   r[   r   Úsharding_strategyr   Úprocess_groupÚtuple)re   rÒ   Úfsdp_modulerà   s       r=   Ú_get_fsdp_process_grouprã   È  s   € õ �%�ÑÔð ØˆˆØ	Ô	ð Ý�4 Ô!2°1Ô!5Ñ6Ô6ˆˆàˆtàÔ$Õ(BÐBÐBØˆtàÔ-€MÝ�-¥Ñ'Ô'ð ØˆtØÐrQ   Ú
state_dictc                 óÀ   — |j         r@|j        rt          j                             ¦   «         sdnd}t          | |j        |¬¦  «        S |j        rt          | ¦  «        S | S )NrP   )r   )rA   Ú
ranks_only)r@   rA   r]   ÚdistributedÚis_initializedr   r   )rä   rÒ   ræ   s      r=   Ú_maybe_full_or_cpu_state_dictré   Û  sƒ   € ð Ôð ð Ô$ðÝ,1Ô,=×,LÒ,LÑ,NÔ,NðˆBˆBàð 	õ
 "Ø DÔ$4Àð
ñ 
ô 
ð 	
ð 
Ô	ð Ý)¨*Ñ5Ô5Ð5àÐrQ   c           	      óÐ  — |j         si S |                     ¦   «         5   t          | d¦  «        ¦   «         }d d d ¦  «         n# 1 swxY w Y   t          |                     ¦   «         ¦  «        D ]°}t          | |¦  «        }t          |¦  «        dk    r%t          d|› dt          |¦  «        › d|› �¦  «        ‚t          t          |¦  «        ¦  «        }||k    rDdt          fd„} |||¦  «        st          d|› d	|› �¦  «        ‚|                     |¦  «        ||<   Œ±|j        r^i }|D ]W}|j        D ]M}|                     |¦  «        sŒ|j        r||         ||<   Œ+|t          |¦  «        d …         }	||         ||	<   ŒNŒX|}|j        rL|                      ¦   «         D ]7\  }}
|
j        rŒt          | |¦  «        }|D ]}|                     |¦  «         ŒŒ8t'          ||¦  «        S )
Nrä   ro   úExpected 1 FQN for key 'ú', got z: ri   c                 óT  — t          |¦  «        t          | ¦  «        k    rdS |                     d¦  «        }|                      d¦  «        }d}t          |¦  «        D ]I\  }}|||         k    r1|dz  }|t          |¦  «        k    r|t          |¦  «        dz
  k    c S ŒB|dv rŒG dS dS )NFrl   r   ro   )rm   ru   T)r~   rw   rx   )r×   rr   Ú	fqn_splitÚ	key_splitÚfqn_idxÚkey_idxÚkey_names          r=   Úverifyz%_get_model_state_dict.<locals>.verify  sÃ   € Ý�s‘8”8�s 3™xœxÒ'Ð'Ø ˜5ØŸIšI c™NœN�	ØŸIšI c™NœN�	Ø�Ý)2°9Ñ)=Ô)=ð %ð %Ñ%�G˜XØ 9¨WÔ#5Ò5Ð5Ø 1™˜Ø"¥c¨)¡n¤nÒ4Ð4Ø#*­c°)©n¬n¸qÑ.@Ò#@Ð@Ð@Ð@ð 5à!Ð%<Ð<Ð<Ø à$˜u˜uØ�trQ   zAn unexpected key, z, exists. FQN is )rX   rZ   rÝ   rb   Úkeysr’   r~   r{   ÚnextÚiterrM   rŠ   ÚpoprW   Ú
startswithrC   rB   rŸ   Úrequires_gradré   )re   rÒ   rä   r×   rÌ   rr   ró   Únew_state_dictrs   r£   rË   s              r=   Ú_get_model_state_dictrû   í  s¢  € ð Ôð Øˆ	à	×	Ò	Ñ	Ô	ð ;ð ;Ø8•^ E¨<Ñ8Ô8Ñ:Ô:ˆ
ð;ð ;ð ;ñ ;ô ;ð ;ð ;ð ;ð ;ð ;ð ;øøøð ;ð ;ð ;ð ;õ �J—O’OÑ%Ô%Ñ&Ô&ð 2ð 2ˆÝ˜ Ñ$Ô$ˆÝˆt‰9Œ9˜Š>ˆ>Ý ØJ¨3ÐJÐJµs¸4±y´yÐJÐJÀDÐJÐJñô ð õ •4˜‘:”:ÑÔˆØ�#Š:ˆ:ð¥Dð ð ð ð ð" �6˜#˜sÑ#Ô#ð VÝ"Ð#T¸Ð#TÐ#TÈsÐ#TÐ#TÑUÔUÐUØ(Ÿnšn¨SÑ1Ô1ˆJ�s‰OøàÔð $Ø/1ˆàð 	>ð 	>ˆCØÔ1ð >ð >�Ø—~’~ fÑ-Ô-ð ØØÔ/ð >Ø*4°S¬/�N 3Ñ'Ð'à!¥# f¡+¤+ - -Ô0�GØ.8¸¬o�N 7Ñ+Ð+ð>ð $ˆ
àÔ ð $Ø×0Ò0Ñ2Ô2ð 	$ð 	$‰JˆC�ØÔ"ð ØÝ˜U CÑ(Ô(ˆDØð $ð $�Ø—’˜sÑ#Ô#Ð#Ð#ð$õ )¨°TÑ:Ô:Ð:s   žAÁAÁ
Ac           	      óÀ  — |j         r	|s|j        st          i i ¦  «        S i }t          | |j        ¦  «        D ]«\  }}t          | ||j        ¦  «        }t          | ||j        dd¬¦  «        }t          ||¦  «        D ]f\  }}	|j        rt          j        ¦   «         dk    r>||	k    r8| 	                    |d ¦  «        }
|
€|j
        rt          d|› d�¦  «        ‚n|
||	<   |||	<   ŒgŒ¬d}|j        s|j        �r‰t          ¦   «         }|                     ¦   «         D ]H}t          j        |¦  «        r2|                     ¦   «         dk    r|                     |j        ¦  «         ŒIt          j        d¦  «        |v r)|                     t          j        d¦  «        ¦  «         d}t+          |¦  «        dk    r2|                     t          j                             ¦   «         ¦  «         n"t+          |¦  «        dk    rt1          d	¦  «        ‚|j        r1t3          ||| 	                    ¦   «         |j
        |j        ¬
¦  «         n+|j        r$t7          ||| 	                    ¦   «         ¬¦  «         |                     |¦  «         |                     ¦   «         5  t=          t           t?          | d¦  «        ||j
        |¬¦  «        ¦  «        cd d d ¦  «         S # 1 swxY w Y   d S )NF)rg   rh   r   zMissing key: rl   ÚmetaTro   zMultiple devices found)ÚdevicerD   rA   ©rþ   Úload_state_dict)rä   rD   Úassign) rX   rE   r!   r¦   rH   r’   ÚziprÔ   rÕ   r÷   rD   rŠ   r@   r_   r�   r]   Ú	is_tensorÚdimr›   rþ   Úremover~   Údistributed_c10dÚ_get_pg_default_devicerÅ   r   rA   r   rÁ   rZ   r   rÝ   )re   rä   rÒ   Úlocal_state_dictr×   ÚvaluerÌ   Úfqns_with_prefixrr   Úfqn_with_prefixÚ
load_valuer  Údevicess                r=   Ú_load_model_state_dictr  0  ss  € ð Ôð ) Zð )¸Ô8Qð )Ý   RÑ(Ô(Ð(àÐÝ0°¸Ô8NÑOÔOð 6ð 6‰
ˆˆUÝ˜  TÔ%;Ñ<Ô<ˆÝ$ØØØÔ"Ø!Ø!&ð
ñ 
ô 
Ðõ %(¨Ð.>Ñ$?Ô$?ð 
	6ð 
	6Ñ ˆC�àÔ-ð=Ý15´±´ÀAÒ1EÐ1EØ˜Ò(Ð(Ø'Ÿ^š^¨C°Ñ6Ô6�
ØÐ%Ø”{ð CÝ*Ð+A¸3Ð+AÐ+AÐ+AÑBÔBÐBðCð 3=�J˜Ñ/Ø05Ð˜_Ñ-Ð-ð
	6ð €FØÔ ð , DÔ$8ñ ,Ý‘%”%ˆØ%×,Ò,Ñ.Ô.ð 	*ð 	*ˆEÝŒ˜uÑ%Ô%ð *¨%¯)ª)©+¬+¸ª/¨/Ø—’˜EœLÑ)Ô)Ð)øõ Œ<˜ÑÔ 7Ð*Ð*Ø�NŠN�5œ<¨Ñ/Ô/Ñ0Ô0Ð0ØˆFÝˆw‰<Œ<˜1ÒÐØ�KŠK�Ô-×DÒDÑFÔFÑGÔGÐGÐGÝ�‰\Œ\˜AÒÐÝÐ5Ñ6Ô6Ð6àÔ$ð 		WÝ!ØØ Ø—{’{‘}”}Ø”{Ø Ô,ðñ ô ð ð ð Ô!ð 	WÝ" :Ð/?ÈÏÊÉÌÐVÑVÔVÐVØ×ÒÐ*Ñ+Ô+Ð+à	×	Ò	Ñ	Ô	ð 
ð 
ÝÝØ4�N˜5Ð"3Ñ4Ô4Ø%¨d¬kÀ&ðñ ô ñ
ô 
ð
ð 
ð 
ð 
ñ 
ô 
ð 
ð 
ð 
ð 
ð 
ð 
øøøð 
ð 
ð 
ð 
ð 
ð 
s   Ê4KËKËKÚoptimc                 óB  — | j         rdS | j        D ]}|t                   D ]}|j        �  dS ŒŒ| j        D ]2}|t                   D ]"}|j        rt          j        |¦  «        |_        Œ#Œ3g }| j        D ]Z}d|v rT|                     |d         ¦  «         t          |d         t
          j	        ¦  «        rt          j
        d¦  «        nd|d<   Œ[|                      d¬¦  «         | j        D ]}d|v r|                     d¦  «        |d<   Œ|                      d¬¦  «         dS )zH
    Initialize optim states by calling the step() with zero grads.
    NÚlrg        )Úclosurer   T)Úset_to_none)r4   r2   Ú_PARAMSÚgradrù   r]   Ú
zeros_liker|   ry   r^   ÚtensorÚstepr÷   Ú	zero_grad)r  Úparam_grouprË   Úlrss       r=   Ú_init_optim_stater  u  sx  € ð „{ð àˆð Ô)ð ð ˆØ ¥Ô)ð 	ð 	ˆEØŒzÐ%Ø���ð &ð	ð Ô)ð 5ð 5ˆØ ¥Ô)ð 	5ð 	5ˆEØÔ"ð 5Ý"Ô-¨eÑ4Ô4�”
øð	5ð €CØÔ)ð ð ˆØ�;ÐÐØ�JŠJ�{ 4Ô(Ñ)Ô)Ð)õ ˜k¨$Ô/µ´Ñ>Ô>ð•”˜SÑ!Ô!Ð!àð ˜Ñøð
 
‡J‚J�t€JÑÔÐð Ô)ð +ð +ˆØ�;ÐÐØ #§¢¨¡
¤
ˆK˜ÑøØ	‡O‚O €OÑ%Ô%Ð%Ð%Ð%rQ   c           
      ó¼  ‡	‡
— dt           t          t          f         dt          dt           t          t          f         fˆ	ˆ
fd„Š	d„ Š
i }t	          t
          | t                   ¦  «                             ¦   «         D ]U\  }}t          › d|› �}|                      ‰	t	          t           t          t          f         |¦  «        |¦  «        ¦  «         ŒVt	          t          | t                   ¦  «        D ]k}|                     t          ¦  «        }t	          t          t                   |¦  «        D ].}|                     ¦   «         D ]\  }}||t          › d|› d|› �<   ŒŒ/Œl|S )a3  
    This API flattens the optimizer state_dict to support optimizer resharding for
    MPMD, e.g., pipeline parallelism.

    Without the API, the original optimizer state_dict looks like:
    {
        "state": {
            "layer1.weight": {
                "step": 10, "exp_avg": SomeTensor, "exp_avg_sq": SomeTensor
            },
            "layer2.weight": {
                "step": 10, "exp_avg": SomeTensor, "exp_avg_sq": SomeTensor
            },
        },
        "param_groups": [
            {
                "lr": 0.0,
                "betas": (0.9, 0.95), ...,
                "params": ["layer1.weight", "layer2.weight"]
            }
        ]
    }

    With this API, the optimizer state_dict looks like:
    {
        "state.layer1.weight.step": 10,
        "state.layer2.weight.step": 10,
        "state.layer1.weight.exp_avg": SomeTensor,
        "state.layer2.weight.exp_avg": SomeTensor,
        "state.layer1.weight.exp_avg_sq": SomeTensor,
        "state.layer2.weight.exp_avg_sq": SomeTensor,
        "param_groups.layer1.weight.lr": 0.1,
        "param_groups.layer2.weight.lr": 0.1,
        "param_groups.layer1.weight.betas": (0.9, 0.95),
        "param_groups.layer2.weight.betas": (0.9, 0.95),
    }

    The "state" section supports arbitrary levels of nesting for optimizers like Shampoo.
    Únested_dictrs   ri   c                 óþ   •— i }|                       ¦   «         D ]d\  }}t          |¦  «        }|r|› d|› �n|}t          |t          ¦  «        r |                      ‰||¦  «        ¦  «         ŒT ‰|¦  «         |||<   Œe|S )a  
        Recursively flatten a nested dictionary with dot-separated keys.

        Args:
            nested_dict: The dictionary to flatten
            prefix: The prefix to prepend to all keys

        Returns:
            Flattened dictionary with dot-separated keys
        rl   )rÃ   rO   ry   r\   rÁ   )	r  rs   Ú	flattenedr×   r	  Ústr_keyÚfull_keyÚ_flatten_state_nested_dictÚ_raise_if_type_not_supporteds	          €€r=   r#  z=_flatten_optim_state_dict.<locals>._flatten_state_nested_dictÈ  s®   ø€ ð +-ˆ	à%×+Ò+Ñ-Ô-ð 	,ð 	,‰JˆC�å˜#‘h”hˆGØ06ÐC˜&Ð,Ð, 7Ð,Ð,Ð,¸GˆHå˜%¥Ñ&Ô&ð ,à× Ò Ð!;Ð!;¸EÀ8Ñ!LÔ!LÑMÔMÐMÐMð -Ð,¨UÑ3Ô3Ð3Ø&+�	˜(Ñ#Ð#àÐrQ   c                 ó    — t          | t          j        t          t          t
          f¦  «        s t          dt          | ¦  «        › d�¦  «        ‚d S )Nz[Flattening optimizer state_dict only supports tensor, int, float, dict states now. Type is rl   )ry   r]   r^   ÚintÚfloatr\   ÚNotImplementedErrorÚtype)Úvs    r=   r$  z?_flatten_optim_state_dict.<locals>._raise_if_type_not_supportedæ  sX   € Ý˜!�eœl­Cµ½Ð=Ñ>Ô>ð 	Ý%ð&å ™7œ7ð&ð &ð &ñô ð ð	ð 	rQ   rl   )r\   rO   r
   r&   r   r'   Ú_STATErÃ   rÁ   r(   Ú_PGr÷   r  rb   )rä   Úretrr   r4   Ústate_prefixr  rÌ   Úkr*  r#  r$  s            @@r=   Ú_flatten_optim_state_dictr0  Ÿ  sˆ  øø€ ðRÝ�#�s˜(”^ðÝ-0ðå	�c•9ˆnÔ	ðð ð ð ð ð ð ð<ð ð ð !#€Cõ �=¨*µVÔ*<Ñ=Ô=×CÒCÑEÔEð 
ð 
‰
ˆˆUÝ Ð(Ð( 3Ð(Ð(ˆØ�
Š
Ø&Ð&¥t­Dµµc°¬N¸EÑ'BÔ'BÀLÑQÔQñ	
ô 	
ð 	
ð 	
õ
 Õ-¨z½#¬Ñ?Ô?ð ,ð ,ˆØ�Š�wÑ'Ô'ˆÝ��Sœ	 4Ñ(Ô(ð 	,ð 	,ˆCØ#×)Ò)Ñ+Ô+ð ,ð ,‘��1Ø*+�•sÐ&Ð&˜SÐ&Ð& 1Ð&Ð&Ñ'Ð'ð,ð	,ð €JrQ   c                 ó¾  — dt           dt          t           t          f         dt          t           t          f         fd„}i }g }t          |t          |i}| j        D �]}|                     t          g i¦  «         |t                   D �]'}|j        |         D �]}	|	|j	        v r)d}
|D ]#}|t          k    rŒt          › d|	› d|› �}||v rd}
 nd}
|
sŒ:|d         t                   }t          |t          ¦  «        st          d	t          |¦  «        › �¦  «        ‚|                     |	¦  «         |j        sŒži ||	<   | j        |         D ]d}t          › d|	› d|› �}||vr+ |||¦  «        }|t!          t"          ||	         ¦  «        |<   Œ@||         t!          t"          ||	         ¦  «        |<   Œe�Œ�Œ)t!          t          t                    |d         t                   ¦  «        d
         }|D ]s}|t          k    rŒ|t          › d|› d|› �         }||d         vr||d         |<   Œ9|d         |         |k    r(t%          d|› d|› d|› d|d         |         › d�	¦  «        ‚Œt�Œ|S )zý
    This API unflattens the state_dict generated by _flatten_optim_state_dict().
    Supports arbitrary levels of nesting in the state section through recursive reconstruction.

    See the docstring of _flatten_optim_state_dict() for more detail.
    Úflattened_keyÚflattened_dictri   c                 óf  — | › d�}i }|                      ¦   «         D ]”\  }}|                     |¦  «        sŒ|t          |¦  «        d…         }|                     d¦  «        }|}|dd…         D ]5}	|	|vri ||	<   t	          ||	         t
          ¦  «        st          ‚||	         }Œ6|||d         <   Œ•|S )zØ
        Reconstructs a potentially nested value from flattened keys.
        For non-nested values, returns the value directly.
        For nested values, reconstructs the nested structure with string keys.
        rl   Nr˜   )rÃ   rø   r~   rw   ry   r\   r{   )
r2  r3  rs   r  r×   r	  Úremaining_keyÚpartsÚcurrentÚparts
             r=   Ú_reconstruct_nested_dictz=_unflatten_optim_state_dict.<locals>._reconstruct_nested_dict  sð   € ð "Ð$Ð$Ð$ˆà&(ˆð )×.Ò.Ñ0Ô0ð 	'ð 	'‰JˆC�ð —>’> &Ñ)Ô)ð àð  ¥ F¡¤  Ô.ˆMð "×'Ò'¨Ñ,Ô,ˆEà!ˆGð ˜c˜r˜cœ
ð (ð (�à˜wÐ&Ð&Ø$&�G˜D‘Må! '¨$¤-µÑ6Ô6ð )Ý(Ð(Ø! $œ-��ð "'ˆG�E˜"”IÑÐð ÐrQ   Frl   Tr˜   úExpected list, got r   zaAll the parameters in the same parameter group should have the same saved param_group value. But z is z while other(s) is )rO   r\   r&   r+  r,  r2   r|   r  rU   rV   ry   rb   r{   r)  rù   r4   r   r'   rŠ   )r  rä   rÒ   r9  r4   Úpg_stateÚ
return_osdr  rË   rr   Ú	in_paramsr/  Úflatten_keyr3   Ú
state_nameÚflattened_state_keyÚreconstructed_valueÚfirst_param_fqnr	  s                      r=   Ú_unflatten_optim_state_dictrC    s5  € ð0Ýð0Ý,0µµi°Ô,@ð0å	�c•9ˆnÔ	ð0ð 0ð 0ð 0ðd €EØ"$€HÝ&,¨eµS¸(Ð%C€JàÔ)ð Añ AˆØ�Š� "˜Ñ&Ô&Ð&Ø ¥Ô)ð 1	ñ 1	ˆEØÔ-¨eÔ4ð 0ñ 0�ð ˜$Ô4Ð4Ð4Ø %�IØ(ð ð ˜Ø¥š<˜<Ø$Ý),Ð&8Ð&8¨sÐ&8Ð&8°QÐ&8Ð&8˜Ø&¨*Ð4Ð4Ø(,˜IØøà $�Ià ð Øà! "œ¥gÔ.�Ý! &­$Ñ/Ô/ð OÝ(Ð)M½tÀF¹|¼|Ð)MÐ)MÑNÔNÐNØ—’˜cÑ"Ô"Ð"ð Ô*ð Øð  ��c‘
Ø"'¤+¨eÔ"4ð ð �JÝ-3Ð*HÐ*H°cÐ*HÐ*H¸JÐ*HÐ*HÐ'à*°*Ð<Ð<à.FÐ.FØ/°ñ/ô /Ð+ð
 0õ �]¨E°#¬JÑ7Ô7¸
ÑCÐCð GQØ/ôG��]¨E°#¬JÑ7Ô7¸
ÑCÐCññ?0õd �t¥Cœy¨(°2¬,µwÔ*?Ñ@Ô@ÀÔCˆØð 	ð 	ˆAØ•GŠ|ˆ|ØØ¥#Ð=Ð=¨Ð=Ð=¸!Ð=Ð=Ô>ˆEØ˜ œÐ$Ð$Ø"'�˜”˜Q‘�Ø˜"”˜a” EÒ)Ð)Ý"ðGØ=LðGð GØOPðGð GàðGð Gà4<¸R´LÀ´OðGð Gð Gñô ð ð *ñ	ð ÐrQ   Ú
optimizersc                 ó.  ‡— |j         si S t          i t          g i}|D �]?}t          |¦  «          t	          |d¦  «        ¦   «         }|j        rý|                     ¦   «         5  t          j        | ||t          | |¦  «        ¬¦  «        }d d d ¦  «         n# 1 swxY w Y   |sŒ†t          |t                                        ¦   «         ¦  «        D ]H}d|v rB|t                                        |¦  «        |t                   |                     dd¦  «        <   ŒI|t                   D ]#}d„ |t                   D ¦   «         }||t          <   Œ$�n‘t          t          j        d„ |j        D ¦   «         ¦  «        ¦  «        }t%          t'          |t)          t+          |¦  «        ¦  «        ¦  «        ¦  «        }	i Š|                      ¦   «         D ]}\  }
}t/          | |
¦  «        }t+          |¦  «        dk    r"t1          d	|
› d
t+          |¦  «        › �¦  «        ‚t3          t5          |¦  «        ¦  «        }||	vrŒk|	|         }|‰|<   |‰|<   Œ~t          |t                                        ¦   «         ¦  «        D ]8}
‰|
         }|t                                        |
¦  «        |t                   |<   Œ9|t                   D ]#}ˆfd„|t                   D ¦   «         |t          <   Œ$|s�ŒÃt7          t8          |t                   ¦  «                             |t                   ¦  «         t7          t<          |t                   ¦  «                             |t                   ¦  «         �ŒA|j         r"t7          tB          tE          |¦  «        ¦  «        }tG          ||¦  «        S )Nrä   ©Úgroupru   ú
_orig_mod.rk   c                 ó:   — g | ]}|                      d d¦  «        ‘ŒS )rH  rk   ©rv   ©rq   r/  s     r=   ú
<listcomp>z)_get_optim_state_dict.<locals>.<listcomp>©  s&   € ÐJÐJÐJ¸!˜!Ÿ)š) L°"Ñ5Ô5ÐJÐJÐJrQ   c              3   ó0   K  — | ]}|t                    V — Œd S r7   )r  )rq   Úgs     r=   r°   z(_get_optim_state_dict.<locals>.<genexpr>¬  s&   è è € Ð-UÐ-U¸Q¨aµ¬jÐ-UÐ-UÐ-UÐ-UÐ-UÐ-UrQ   ro   rë   rì   c                 ó    •— g | ]
}‰|         ‘ŒS rP   rP   )rq   ÚpidÚfqn_pid_mappings     €r=   rL  z)_get_optim_state_dict.<locals>.<listcomp>Æ  s   ø€ Ð!QÐ!QÐ!Q¸3 /°#Ô"6Ð!QÐ!QÐ!QrQ   )$rY   r+  r,  r  rÝ   r[   rZ   r}   rÑ   rã   rb   rô   r÷   rv   r  r	   Úfrom_iterabler2   r\   r  Úranger~   rŸ   r’   r{   rõ   rö   r   r'   rÁ   r(   ÚextendrF   r)   r0  ré   )re   rD  rÒ   rÑ   r  Úosdr/  rN  r3   Úparam_pid_mappingr×   rË   rÌ   rr   rP  rG  rQ  s                   @r=   Ú_get_optim_state_dictrW  ‰  sÐ  ø€ ð Ôð Øˆ	å,2°B½¸RÐ+@ÐØð 9Hñ 9HˆÝ˜%Ñ Ô Ð Ø1�n˜U LÑ1Ô1Ñ3Ô3ˆØÔð 0	RØ×"Ò"Ñ$Ô$ð ð ÝÔ+ØØØÝ1°%¸Ñ>Ô>ð	ñ ô �ðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð ð ØÝ˜#�fœ+×*Ò*Ñ,Ô,Ñ-Ô-ð Rð R�Ø !Ð#Ð#Ø?BÅ6¼{¿ºÈqÑ?QÔ?Q�C�”K §	¢	¨,¸Ñ ;Ô ;Ñ<øØ�”Xð $ð $�ØJÐJ¸qÅ¼zÐJÑJÔJ�Ø#�•'‘
�
ñ$õ �%Ô-Ð-UÐ-UÀ%ÔBTÐ-UÑ-UÔ-UÑUÔUÑVÔVˆFÝ $¥S¨µµs¸6±{´{Ñ1CÔ1CÑ%DÔ%DÑ EÔ EÐØ ˆOØ#×4Ò4Ñ6Ô6ð +ð +‘
��UÝ  ¨Ñ,Ô,�Ý�t‘9”9 ’>�>Ý(ØJ°3ÐJÐJ½sÀ4¹y¼yÐJÐJñô ð õ �4 ™:œ:Ñ&Ô&�ØÐ 1Ð1Ð1Øà'¨Ô.�Ø'*� Ñ$à'*� Ñ$Ð$õ ˜C¥œK×,Ò,Ñ.Ô.Ñ/Ô/ð 8ð 8�Ø% cÔ*�ð $'¥v¤;§?¢?°3Ñ#7Ô#7�•F”˜CÑ Ð à�Sœð Rð R�Ø!QÐ!QÐ!QÐ!QÀ%ÍÄ.Ð!QÑ!QÔ!Q�•g‘�àð 	Ùå�]Ð,­VÔ4Ñ5Ô5×<Ò<¸SÅ¼[ÑIÔIÐIÝÕÐ 0µÔ 5Ñ6Ô6×=Ò=¸cÅ#¼hÑGÔGÐGÑGàÔ(ð 
ÝÝÕ 9Ð:JÑ KÔ Kñ
ô 
Ðõ )Ð)9¸4Ñ@Ô@Ð@s   Á"'BÂB	ÂB	c           
      óÔ  — i }g }t           |t          |i}i }t          d„ t          t          |t                    ¦  «        D ¦   «         ¦  «        r|S |j        D �]}|                     t          g i¦  «         |t                   D �]À}	|j        |	         D �]®}
|
|j	        v rWd}t          t          |t                   ¦  «        D ]3}|
t          t          t                   |t                   ¦  «        v rd} nŒ4nd}|sŒh|d         t                   }t          |t          ¦  «        st          dt          |¦  «        › �¦  «        ‚|                     |
¦  «         |	j        rf|
t          t          |t                    ¦  «        v r*t          t          |t                    ¦  «        |
         ||
<   n|j        rt%          d|
› d�¦  «        ‚t          t          |t                   ¦  «        D ]\}|
t          t          t                   |t                   ¦  «        v r-t'          |t                   ¦  «        dz
  |t)          |¦  «        <   Œ]�Œ°�ŒÂt'          |t                   ¦  «        d	k    �rg }t          t          |t                   ¦  «        D ]S}t'          t          t          t                   |t                   ¦  «        ¦  «        d	k    r|                     |¦  «         ŒTt'          |¦  «        dk    rt+          d
¦  «        ‚t'          |t                   ¦  «        t'          |j        ¦  «        k    rt+          d¦  «        ‚t'          |t                   ¦  «        dz
  |t)          |¦  «        <   �Œt          t          |t                   ¦  «        D ]]}|                     t)          |¦  «        d¦  «        }|dk    rŒ,|                     ¦   «         D ]\  }}|t          k    rŒ|||         |<   ŒŒ^|S )að  
    Extract the corresponding optim state_dict from ``optim_state_dict`` for
    ``optim`` and return the result optim state_dict.

    Args:
        model (nn.Module): the root model.
        optim (torch.optim.Optimizer): the optimizer.
        optim_state_dict (Dict[str, ValueType]): the superset optim state_dict that
            contains the optim state_dict of ``optim``.
        info (_StateDictInfo): state dict information.

    Returns:
        The optim state_dict of ``optim``.
    c              3   ó@   K  — | ]}t          |t          ¦  «        V — Œd S r7   )ry   r&  rK  s     r=   r°   z*_split_optim_state_dict.<locals>.<genexpr>ð  s,   è è € Ð
UÐ
U !�:�a�ÑÔÐ
UÐ
UÐ
UÐ
UÐ
UÐ
UrQ   FTr˜   r:  z'Missing optimizer state for parameter 'z‡' in checkpoint. The parameter requires gradients but has no saved optimizer state. To load anyway, use StateDictOptions(strict=False).ro   r   zÛThere are param groups that have zero parameters. In such a case, DSD only support exactly one param group with zero parameters.But the loaded state_dict has zero or more than one param groups that have zero parameters.z`When there is a parameter group that has zero parameters, multiple optimizers are not supported.)r+  r,  Úallr   r'   r2   r|   r  rU   rV   r(   rb   rO   ry   r{   r)  rù   rD   rŠ   r~   ÚidrÅ   r‡   rÃ   )re   r  rÑ   rÒ   r4   r;  r<  Ú
pg_mappingr  rË   rr   r=  Úloaded_param_groupr3   r-  Úpg_idxr×   r	  s                     r=   Ú_split_optim_state_dictr_  Ö  s	  € ð* €EØ"$€HÝ&,¨eµS¸(Ð%C€JØ!#€Jå
Ð
UÐ
U¥t­MÐ;KÍFÔ;SÑ'TÔ'TÐ
UÑ
UÔ
UÑUÔUð  ØÐàÔ)ð 7Jñ 7JˆØ�Š� "˜Ñ&Ô&Ð&Ø ¥Ô)ð  	Vñ  	VˆEØÔ-¨eÔ4ð Vñ V�Ø˜$Ô4Ð4Ð4Ø %�IÝ.2Ý)Ð+;½CÔ+@ñ/ô /ð "ð "Ð*ð ¥$¥t­C¤yÐ2DÅWÔ2MÑ"NÔ"NÐNÐNØ(,˜IØ!˜Eð Oøð !%�IØ ð Øà! "œ¥gÔ.�Ý! &­$Ñ/Ô/ð OÝ(Ð)M½tÀF¹|¼|Ð)MÐ)MÑNÔNÐNØ—’˜cÑ"Ô"Ð"ØÔ&ð Ø�d¥=Ð2BÅ6Ô2JÑKÔKÐKÐKÝ%)­-Ð9IÍ&Ô9QÑ%RÔ%RÐSVÔ%W˜˜c™
˜
Øœð Ý*ðRÀcð Rð Rð Rñô ð õ
 +/Ý%Ð'7½Ô'<ñ+ô +ð Vð VÐ&ð �d¥4­¤9Ð.@ÅÔ.IÑJÔJÐJÐJÝ=@ÀÍCÄÑ=QÔ=QÐTUÑ=U˜
¥2Ð&8Ñ#9Ô#9Ñ:øñ	Vñ7VõB ˆ{�7Ô#Ñ$Ô$¨Ò)Ñ)àˆCÝ&*Õ+<Ð>NÍsÔ>SÑ&TÔ&Tð 3ð 3Ð"Ý•t�D¥œIÐ'9½'Ô'BÑCÔCÑDÔDÈÒIÐIØ—J’JÐ1Ñ2Ô2Ð2øÝ�3‰xŒx˜1Š}ˆ}Ý ð1ñô ð õ Ð#¥CÔ(Ñ)Ô)­S°Ô1CÑ-DÔ-DÒDÐDÝ ð=ñô ð õ 25°ZÅ´_Ñ1EÔ1EÈÑ1IˆJ•rÐ,Ñ-Ô-Ñ.ùåÕ-Ð/?ÅÔ/DÑEÔEð 	*ð 	*ˆØ—’¥ ;¡¤°Ñ4Ô4ˆØ�RŠ<ˆ<Øà%×+Ò+Ñ-Ô-ð 	*ð 	*‰JˆC�Ø•gŠ~ˆ~Øà$)ˆH�VÔ˜SÑ!Ð!ð		*ð ÐrQ   c                 ó8  ‡‡‡— |j         sd S |D �]‰}t          |¦  «         |rSt          |v rt          | |||¦  «        }n9t	          |t          t          t          t          f         |¦  «        |¦  «        }ni }|j	        �rÅ|  
                    ¦   «         D �][\  }}t          | |¦  «        }t          | |d¬¦  «        }	||	k    rŒ/t          |¦  «        dk    r"t          d|› dt          |¦  «        › �¦  «        ‚|                     ¦   «         Š|	                     ¦   «         Š|t                   D ]M}
t          t          t          t           f         |
¦  «        }ˆˆfd„|t"                   D ¦   «         }||t"          <   ŒNt          t$          |t                   ¦  «        }t'          |                     ¦   «         ¦  «        D ]2}‰|v r,|                     |¦  «        ||                     ‰‰¦  «        <   Œ3�Œ]|                     ¦   «         5  t/          j        | ||t3          | |¦  «        ¬¦  «        }d d d ¦  «         n# 1 swxY w Y   �n9|j        �r1d|_        t7          | |f|¦  «        }d|_        d Šˆfd	„}t9          t:          j        ||¦  «        }‰€t          d
¦  «        ‚t?          |¦  «        \  }}t?          |¦  «        \  }}|j         rtC          ||‰¬¦  «         ntE          ||‰¬¦  «         |D ]3}||vr-||vrt          d|› d�¦  «        ‚||         ||<   ||         ||<   Œ4tG          ||¦  «        }|t                   D ]:}t"          |vr/g t          t          t          t          f         |¦  «        t"          <   Œ; tI          |d¦  «        |¬¦  «         �Œ‹d S )NF)rh   ro   zExpected 1 FQN for 'rì   c                 ó<   •— g | ]}|                      ‰‰¦  «        ‘ŒS rP   rJ  )rq   r×   rr   Úfqn_with_compilers     €€r=   rL  z*_load_optim_state_dict.<locals>.<listcomp>d  s5   ø€ ð ð ð Ø@C˜Ÿš CÐ):Ñ;Ô;ðð ð rQ   rF  Tc                 ó€   •— |                       ¦   «         dk    r$‰€| j        Šn‰| j        k    rt          d¦  «        ‚| S )Nr   zDevice mismatch)r  rþ   rÅ   )Útrþ   s    €r=   Ú_devicez'_load_optim_state_dict.<locals>._devicez  sC   ø€ Ø—5’5‘7”7˜Q’;�;à�~Ø!"¤˜˜Ø 1¤8Ò+Ð+Ý(Ð):Ñ;Ô;Ð;Ø�rQ   zExpected device to be setrÿ   zExpected key 'z' in osd_mappingr   )rä   )%rY   r  r+  r_  rC  r   r\   rO   r&   r[   rŸ   r’   r~   r{   r÷   r,  r
   r  r'   rb   rô   rv   rZ   r}   Úoptim_state_dict_to_loadrã   r@   rW  r#   r]   r^   r   rE   r   r   r   rÝ   )re   rD  rä   rÒ   r  rÑ   Úoriginal_fqnÚ_rÌ   Úfqns_with_compilerrN  Úvalr3   Ú	osd_stater/  r  re  Úflatten_osdÚosd_mappingÚflatten_local_osdÚlocal_osd_mappingÚ	optim_keyÚpgrþ   rr   rb  s                          @@@r=   Ú_load_optim_state_dictrr  :  s¨  øøø€ ð Ôð Øˆàð ^Nñ ^NˆÝ˜%Ñ Ô Ð Øð 
	"Ý˜Ð#Ð#Ý#:Ø˜5 *¨dñ$ô $Ð Ð õ $?Ø�4¥¥S­) ^Ô 4°jÑAÔAÀ4ñ$ô $Ð Ð ð  "ÐØÔñ L	Að $)×#9Ò#9Ñ#;Ô#;ð Xñ X‘�˜aÝ  ¨Ñ5Ô5�Ý%.Ø˜<¸eð&ñ &ô &Ð"ð Ð-Ò-Ð-Øå�t‘9”9 ’>�>Ý(ØO¨|ÐOÐOÅCÈÁIÄIÐOÐOñô ð ð —h’h‘j”j�Ø$6×$:Ò$:Ñ$<Ô$<Ð!Ø)­#Ô.ð *ð *�AÝ�t¥C­ Hœ~¨qÑ1Ô1�Cðð ð ð ð ØGJÍ7Ä|ðñ ô �Fð $*�C�‘L�LÝ ¥Ð0@ÅÔ0HÑIÔI�	Ý˜iŸnšnÑ.Ô.Ñ/Ô/ð Xð X�AØ˜a�x�xØGPÇ}Â}ÐUVÑGWÔGW˜	 !§)¢)¨CÐ1BÑ"CÔ"CÑDøñXð ×"Ò"Ñ$Ô$ð ð Ý#'Ô#@ØØØ$Ý1°%¸Ñ>Ô>ð	$ñ $ô $Ð ðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ùð Ô!ñ )	AØ#(ˆDÔ Ý4°U¸U¸HÀdÑKÔKÐØ#'ˆDÔ ØˆFðð ð ð ð õ �eœl¨GÐ5EÑFÔFˆAØˆ~Ý$Ð%@ÑAÔAÐAÝ':Ð;KÑ'LÔ'LÑ$ˆK˜Ý3FÐGWÑ3XÔ3XÑ0ÐÐ0ØÔ(ð VÝ% kÐ3DÈVÐTÑTÔTÐTÐTå& {Ð4EÈfÐUÑUÔUÐUð
 )ð Jð J�	ØÐ$5Ð5Ð5Ø ¨Ð3Ð3Ý,ØH¨YÐHÐHÐHñô ð ð 4?¸yÔ3IÐ% iÑ0Ø3>¸yÔ3IÐ% iÑ0øÝ4Ø!Ð#4ñ ô  Ðð '¥sÔ+ð Að A�Ý "Ð$Ð$Ø>@•D��c¥9˜nÔ-¨rÑ2Ô2µ7Ñ;øð
 	1��uÐ/Ñ0Ô0Ð<LÐMÑMÔMÐMÑMð}^Nð ^Ns   È'H8È8H<	È?H<	c                óÀ   — t          ¦   «         5  t          | dd||¬¦  «        }t          | |¦  «        }t          |i |¦  «         |cddd¦  «         S # 1 swxY w Y   dS )aH  
    Return the model state_dict of ``model``.

    See ``get_state_dict`` for the detail usage.

    Args:
        model (nn.Module): the nn.Module to the model.
        submodules (deprecated): Optional[set[nn.Module]]: only return the model parameters
            that belong to the submodules.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be returned. See
            `StateDictOptions` for the details.

    Returns:
        The state_dict for ``model``.

    :rtype: typing.Dict[str, ValueType]
    rP   F©rª   r§   r¨   N)r>   rÏ   rû   rØ   )re   r§   r¨   rÒ   rÐ   s        r=   r+   r+   ¥  s½   € õ0 
‰Œð 
 ð 
 ÝØØØØ!Øð
ñ 
ô 
ˆõ 1°¸Ñ=Ô=ÐÝÐ+¨R°Ñ6Ô6Ð6Øð
 ð 
 ð 
 ð 
 ñ 
 ô 
 ð 
 ð 
 ð 
 ð 
 ð 
 ð 
 øøøð 
 ð 
 ð 
 ð 
 ð 
 ð 
 s   �7AÁAÁAc                ó$  — t          ¦   «         5  t          |t          j        j        ¦  «        r|fnt          |¦  «        }t          | |d||¬¦  «        }t          | ||¦  «        }t          i ||¦  «         |cddd¦  «         S # 1 swxY w Y   dS )aË  
    Return the combined state_dict for optimizers.

    See ``get_state_dict`` for the detail usage.

    Args:
        model (nn.Module): the nn.Module to the model.
        optimizers (Union[None, Optimizer, Iterable[Optimizer]]):
            The optimizers that are used to optimize ``model``.
        submodules (deprecated): Optional[set[nn.Module]]: only return the model parameters
            that belong to the submodules.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be returned. See
            `StateDictOptions` for the details.

    Returns:
        The state_dict for ``optimizers``.

    :rtype: OptimizerStateType
    Trt  N)	r>   ry   r]   r  Ú	Optimizerrá   rÏ   rW  rØ   )re   rD  r§   r¨   rÒ   rÑ   s         r=   r,   r,   Ê  sð   € õ6 
‰Œð  ð  õ ˜*¥e¤kÔ&;Ñ<Ô<ð#ˆZˆMˆMå�zÑ"Ô"ð 	õ
 ØØØØ!Øð
ñ 
ô 
ˆõ 1°¸
ÀDÑIÔIÐÝ˜2Ð/°Ñ6Ô6Ð6Øð ð  ð  ð  ñ  ô  ð  ð  ð  ð  ð  ð  øøøð  ð  ð  ð  ð  ð  s   �A)BÂB	ÂB	c                óH  — t          ¦   «         5  t          |t          j        j        ¦  «        r|fnt          |¦  «        }t          | |d||¬¦  «        }t          | |¦  «        }t          | ||¦  «        }t          |||¦  «         ||fcddd¦  «         S # 1 swxY w Y   dS )aô  
    Return the model state_dict and optimizers state_dict.

    ``get_state_dict`` can process any module that is parallelized by PyTorch
    FSDP/fully_shard, DDP/replicate, tensor_parallel/parallelize_module, and any
    combination of these parallelisms. The main functions of ``get_state_dict``
    are: 1.) returning a model and optimizer state_dict that can be resharded
    with a different number of trainers and/or different parallelisms.
    2.) hiding the parallelism-specific state_dict APIs. Users don't have to call
    these APIs.
    3.) sanity checking the result state_dict.

    The keys of the result state dictionary are the canonical FQNs (Fully
    Qualified Names).  A canonical FQN refers to the FQN based on a parameter's
    position in an nn.Module hierarchy. More specifically, a canonical FQN to a
    parameter is the FQN returned by ``module.named_parameters()`` or
    ``module.named_buffers()`` when the module is not distributed by any
    parallelisms. Since the optimizer internally uses parameter IDs to represent
    a parameter, there will be a conversion from the parameter IDs to the
    canonical FQNs when calling this API.

    ``get_state_dict`` can also process a module that is not parallelized. In
    such a case, ``get_state_dict`` only performs one function -- converting the
    optimizer parameter IDs to the canonical FQNs.

    Example:
        >>> # xdoctest: +SKIP
        >>> import torch
        >>> from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
        >>> from torch.nn.parallel import DistributedDataParallel as DDP
        >>> from torch.distributed.checkpoint.state_dict import get_state_dict

        >>> fsdp_model = FSDP(copy.deepcopy(model))
        >>> fsdp_optim = torch.optim.Adam(model.parameters(), lr=1e-3)
        >>> ddp_model = DDP(copy.deepcopy(model))
        >>> ddp_optim = torch.optim.Adam(model.parameters(), lr=1e-3)


        >>> ddp_state_dict, ddp_optim_state_dict = get_state_dict(ddp_model, ddp_optim)
        >>> fsdp_state_dict, fsdp_optim_state_dict = get_state_dict(
        ...     fsdp_model, fsdp_optim
        ... )

        >>> # if we simply call ddp_model.state_dict() and fsdp_model.state_dict(),
        >>> # the asserts will fail.
        >>> assert ddp_state_dict == fsdp_state_dict
        >>> assert ddp_optim_state == fsdp_optim_state_dict


    Args:
        model (nn.Module): the nn.Module to the model.
        optimizers (Union[None, Optimizer, Iterable[Optimizer]]):
            The optimizers that are used to optimize ``model``.
        submodules (deprecated): Optional[set[nn.Module]]: only return the model parameters
            that belong to the submodules.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be returned. See
            `StateDictOptions` for the details.

    Returns:
        ``Tuple`` that contain model state_dict and optimizer state_dict.

    :rtype: typing.Tuple[typing.Dict[str, ValueType], OptimizerStateType]
    Frt  N)
r>   ry   r]   r  rv  rá   rÏ   rû   rW  rØ   )re   rD  r§   r¨   rÒ   rÐ   rÑ   s          r=   r-   r-   ÷  s  € õP 
‰Œð 2ð 2õ ˜*¥e¤kÔ&;Ñ<Ô<ð#ˆZˆMˆMå�zÑ"Ô"ð 	õ
 ØØØØ!Øð
ñ 
ô 
ˆõ 1°¸Ñ=Ô=ÐÝ0°¸
ÀDÑIÔIÐÝÐ+Ð-=¸tÑDÔDÐDØÐ!1Ð1ð!2ð 2ð 2ð 2ñ 2ô 2ð 2ð 2ð 2ð 2ð 2ð 2øøøð 2ð 2ð 2ð 2ð 2ð 2s   �A;BÂBÂBc                 ó"  ‡	— |si S t          t          t          |                     ¦   «         ¦  «        ¦  «        t          j        ¦  «        �rt          j        dt          d¬¦  «         t          t          t          j        t          t          t          f         f         |¦  «        }i }|                     ¦   «         D ]ª\  }}|                      ¦   «         D ]�\  }}||k    rŒt          | |¦  «        }t!          |¦  «        dk    rt#          d¦  «        ‚t          t          |¦  «        ¦  «        › d�Š	|                     ˆ	fd„|                     ¦   «         D ¦   «         ¦  «         Œ‘Œ«|S t          t          t          t          f         |¦  «        S )NzÎPassing model_state_dict as a ``Dict[nn.Module, Dict[str, Any]]``is deprecated and will be removed in 2.5. If you need this feature, please preprocess the model_state_dict to achieve the same functionality.r¬   r­   ro   z/FQNs for a submodule should only have 1 elementrl   c                 ó"   •— i | ]\  }}‰|z   |“ŒS rP   rP   )rq   Úsubfqnr	  rs   s      €r=   ú
<dictcomp>z/_unflatten_model_state_dict.<locals>.<dictcomp>p  s#   ø€ ÐXÐXÐX±°¸�V˜f‘_ eÐXÐXÐXrQ   )ry   rõ   rö   rô   rc   rd   r»   rÀ   r¾   r   r\   rO   r&   rÃ   rÄ   r’   r~   r{   rÁ   )
re   rä   Úcast_state_dictrú   r¢   Úsub_state_dictrf   ÚmrÌ   rs   s
            @r=   Ú_unflatten_model_state_dictr  R  s“  ø€ ð ð Øˆ	å•$•t˜JŸOšOÑ-Ô-Ñ.Ô.Ñ/Ô/µ´Ñ;Ô;ñ 6ÝŒð"õ Øð	
ñ 	
ô 	
ð 	
õ �t¥B¤I­tµC½°NÔ/CÐ$CÔDÀjÑQÔQˆØ/1ˆØ)8×)>Ò)>Ñ)@Ô)@ð 	ð 	Ñ%ˆI�~Ø ×.Ò.Ñ0Ô0ð ð ‘��aØ˜	’>�>Øå  ¨Ñ-Ô-�Ý�t‘9”9 ’>�>Ý(ØIñô ð õ !¥ d¡¤Ñ,Ô,Ð/Ð/Ð/�Ø×%Ò%ØXÐXÐXÐXÀ×AUÒAUÑAWÔAWÐXÑXÔXñô ð ð ðð Ðå•D��i˜Ô(¨*Ñ5Ô5Ð5rQ   )r¨   c                óÜ   — t          | |¦  «        }t          ¦   «         5  t          | dd|¬¦  «        }t          |i |¦  «         t	          | ||¦  «        cddd¦  «         S # 1 swxY w Y   dS )a=  Load the model state_dict.

    The counterpart of ``get_model_state_dict`` to set the state_dict to the
    model. See ``set_state_dict`` for the detail usage.

    Args:
        model (nn.Module): the nn.Module to the model.
        model_state_dict: (Dict[str, ValueType]):
           the model state_dict to load. If the key of the ``model_state_dict``
           is nn.Module, the key is a submodule of ``model`` and the value should
           be the state_dict of the submodule. When loading the state_dict,
           the prefix of the submodule will be append to the state_dict.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be loaded. See
            `StateDictOptions` for the details.

    Returns:
        ``NamedTuple`` with ``missing_keys`` and ``unexpected_keys`` fields:
            * **missing_keys** is a list of str containing the missing keys
            * **unexpected_keys** is a list of str containing the unexpected keys

    :type model_state_dict: typing.Dict[str, ValueType]
    rP   F©rª   r¨   N)r  r>   rÏ   rØ   r  )re   rÐ   r¨   rÒ   s       r=   r.   r.   w  sØ   € õ: .IØÐñ.ô .Ðõ 
‰Œð Eð EÝ˜u b°UÀGÐLÑLÔLˆåÐ+¨R°Ñ6Ô6Ð6Ý% eÐ-=¸tÑDÔDð	Eð Eð Eð Eñ Eô Eð Eð Eð Eð Eð Eð Eøøøð Eð Eð Eð Eð Eð Es   Ÿ5A!Á!A%Á(A%c                ó"  — t          ¦   «         5  t          |t          j        j        ¦  «        r|fnt          |¦  «        }t          | |d|¬¦  «        }t          i ||¦  «         t          | |||¦  «         ddd¦  «         dS # 1 swxY w Y   dS )a¬  Load the optimizers state_dict.

    The counterpart of ``get_optimizer_state_dict`` to set the state_dict to the
    optimizers. See ``set_state_dict`` for the detail usage.

    WARN: ``set_optimizer_state_dict`` can only be called before ``backward()`` or after
        ``step()`` is called on the optimizers. Otherwise, the optimizer states won't be
        initialized correctly.

    Args:
        model (nn.Module): the nn.Module to the model.
        optimizers (Union[Optimizer, Iterable[Optimizer]]):
            The optimizers that are used to optimize ``model``.
        optim_state_dict: OptimizerStateType:
            the optimizer state_dict to load.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be loaded. See
            `StateDictOptions` for the details.

    Returns:
        None

    :type optim_state_dict: typing.OptimizerStateType
    Tr�  N)	r>   ry   r]   r  rv  rá   rÏ   rØ   rr  )re   rD  rÑ   r¨   rÒ   s        r=   r/   r/   ž  sõ   € õ> 
‰Œð 	Jð 	Jõ ˜*¥e¤kÔ&;Ñ<Ô<ð#ˆZˆMˆMå�zÑ"Ô"ð 	õ
 ˜u j¸TÈ7ÐSÑSÔSˆå˜2Ð/°Ñ6Ô6Ð6Ý˜u jÐ2BÀDÑIÔIÐIð	Jð 	Jð 	Jñ 	Jô 	Jð 	Jð 	Jð 	Jð 	Jð 	Jð 	Jð 	Jøøøð 	Jð 	Jð 	Jð 	Jð 	Jð 	Js   �A(BÂBÂBc                ód  — t          | |¦  «        }t          ¦   «         5  t          |t          j        j        ¦  «        r|fnt          |¦  «        }t          | || |¬¦  «        }t          |||¦  «         t          | |||¦  «         t          | ||¦  «        cddd¦  «         S # 1 swxY w Y   dS )aü  Load the model state_dict and optimizers state_dict.

    The counterpart of ``get_state_dict`` to set the state_dict to the model and
    optimizers.  The given ``model_state_dict`` and ``optim_state_dict`` do not
    have to be returned by ``get_state_dict`` but must meet the following
    requirements: 1) all FQNs are canonical FQNs as defined in ``get_state_dict``,
    2) if a tensor is sharded, it must be either a ShardedTensor or DTensor,
    3) optimizer state_dict cannot contain the parameter IDs; the keys should be
    the canonical FQNs.

    WARN: ``set_state_dict`` can only be called before ``backward()`` or after ``step()``
        is called on the optimizers. Otherwise, the optimizer states won't be initialized
        correctly.

    Args:
        model (nn.Module): the nn.Module to the model.
        optimizers (Union[Optimizer, Iterable[Optimizer]]):
            The optimizers that are used to optimize ``model``.
        model_state_dict: (Union[Dict[nn.Module, Dict[str, ValueType]], Dict[str, ValueType]]):
           the model state_dict to load. If the key of the ``model_state_dict``
           is nn.Module, the key is a submodule of ``model`` and the value should
           be the state_dict of the submodule. When loading the state_dict,
           the prefix of the submodule will be append to the state_dict.
        optim_state_dict: OptimizerStateType:
            the optimizer state_dict to load.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be loaded. See
            `StateDictOptions` for the details.

    Returns:
        ``NamedTuple`` with ``missing_keys`` and ``unexpected_keys`` fields:
            * **missing_keys** is a list of str containing the missing keys of the model state_dict.
            * **unexpected_keys** is a list of str containing the unexpected keys of the model state_dict.

    :type model_state_dict: typing.Dict[str, ValueType]
    :type optim_state_dict: typing.OptimizerStateType
    r�  N)r  r>   ry   r]   r  rv  rá   rÏ   rØ   rr  r  )re   rD  rÐ   rÑ   r¨   rÒ   s         r=   r0   r0   É  s-  € õ\ .IØÐñ.ô .Ðõ 
‰Œð Eð Eõ ˜*¥e¤kÔ&;Ñ<Ô<ð#ˆZˆMˆMå�zÑ"Ô"ð 	õ
 Ø�:Ð.>Ð*>Èð
ñ 
ô 
ˆõ 	Ð+Ð-=¸tÑDÔDÐDÝ˜u jÐ2BÀDÑIÔIÐIÝ% eÐ-=¸tÑDÔDðEð Eð Eð Eñ Eô Eð Eð Eð Eð Eð Eð Eøøøð Eð Eð Eð Eð Eð Es   ŸA9B%Â%B)Â,B)c                óF  ‡‡— t          j        t          | |¬¦  «        Šˆfd„}|| _        t          j        t          | |¬¦  «        Šdt
          t          t          f         fˆfd„}|| _        t           
                    |¦  «         t           
                    |¦  «         dS )aó  Patch the ``state_dict`` and ``load_state_dict`` attributes of ``model``.

    Patch the ``state_dict`` and ``load_state_dict`` attributes of ``model`` to
    be a partial function to call ``get_state_dict`` and ``set_state_dict``.

    Example:
        from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
        from torch.distributed.checkpoint.state_dict import patch_model_state_dict

        model = fsdp(model)
        patch_model_state_dict(model)

    Args:
        model (nn.Module): the nn.Module to the model.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be loaded. See
            `StateDictOptions` for the details.
    Returns:
        None
    )re   r¨   c                  ó   •—  ‰ ¦   «         S r7   rP   ©Ú_state_dict_calls   €r=   Ústate_dict_callz0_patch_model_state_dict.<locals>.state_dict_call,  ó   ø€ ØÐÑ!Ô!Ð!rQ   rä   c                 ó    •—  ‰| ¬¦  «         d S )N)rÐ   rP   ©rä   Ú_load_state_dict_calls    €r=   Úload_state_dict_callz5_patch_model_state_dict.<locals>.load_state_dict_call7  ó   ø€ ØÐ¨zÐ:Ñ:Ô:Ð:Ð:Ð:rQ   N)rÉ   rÊ   r+   rä   r.   r\   rO   r
   r   r5   r›   )re   r¨   rˆ  r�  rŒ  r‡  s       @@r=   Ú_patch_model_state_dictr�    sÑ   øø€ õ6 !Ô(ÝØØðñ ô Ðð"ð "ð "ð "ð "ð '€EÔå%Ô-ÝØØðñ ô Ðð;­­cµ3¨h¬ð ;ð ;ð ;ð ;ð ;ð ;ð 1€EÔå×Ò˜OÑ,Ô,Ð,Ý×ÒÐ0Ñ1Ô1Ð1Ð1Ð1rQ   c                ó¶  ‡‡— t          j        t          | ||¬¦  «        Šˆfd„}t          j        t          | ||¬¦  «        Šdt          t
          t          f         fˆfd„}t                               |¦  «         t                               |¦  «         t          |t          j        j        ¦  «        r|fnt          |¦  «        }|D ]}||_        ||_        ŒdS )a¤  Patch the ``state_dict`` and ``load_state_dict`` attributes of ``optimizers``.

    Patch the ``state_dict`` and ``load_state_dict`` attributes of ``optimizers`` to
    be a partial function to call ``get_state_dict`` and ``set_state_dict``.

    Note that if there are multiple optimizers, all of the optimizers will be patched.
    So users only need to call one of the state_dict() to get the full result.

    Example:
        from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
        from torch.distributed.checkpoint.state_dict import patch_model_state_dict

        model = fsdp(model)
        patch_model_state_dict(model)

    Args:
        model (nn.Module): the nn.Module to the model.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be loaded. See
            `StateDictOptions` for the details.
    Returns:
        None
    )re   rD  r¨   c                  ó   •—  ‰ ¦   «         S r7   rP   r†  s   €r=   rˆ  z4_patch_optimizer_state_dict.<locals>.state_dict_callh  r‰  rQ   rä   c                 ó    •—  ‰| ¬¦  «         d S )N)rÑ   rP   r‹  s    €r=   r�  z9_patch_optimizer_state_dict.<locals>.load_state_dict_callr  rŽ  rQ   N)rÉ   rÊ   r,   r/   r\   rO   r
   r5   r›   ry   r]   r  rv  rá   rä   r   )re   rD  r¨   rˆ  r�  r  rŒ  r‡  s         @@r=   Ú_patch_optimizer_state_dictr“  B  s  øø€ õ> !Ô(Ý ØØØð	ñ ô Ðð"ð "ð "ð "ð "õ &Ô-Ý ØØØð	ñ ô Ðð;­­cµ3¨h¬ð ;ð ;ð ;ð ;ð ;ð ;õ ×Ò˜OÑ,Ô,Ð,Ý×ÒÐ0Ñ1Ô1Ð1õ �j¥%¤+Ô"7Ñ8Ô8ð	ˆˆˆå�:ÑÔð ð
 ð 5ð 5ˆØ*ˆÔØ 4ˆÔÐð5ð 5rQ   )rG   TT)rG   )sr`   rÉ   r8   r»   Úcollections.abcr   r   r   Údataclassesr   r   r   Ú	itertoolsr	   Útypingr
   r   r   r]   Útorch.distributedrç   rÔ   Útorch.nnrc   Ú'torch.distributed._shard.sharded_tensorr   Ú#torch.distributed._state_dict_utilsr   r   r   r   r   r   Ú;torch.distributed.algorithms._checkpoint.checkpoint_wrapperr   Útorch.distributed.fsdpr   r   r   r}   r   r   r   r   r   Ú$torch.distributed.fsdp._common_utilsr   r   Ú"torch.distributed.fsdp._init_utilsr   Útorch.distributed.tensorr    Útorch.nn.modules.moduler!   Útorch.nn.parallelr"   rz   Útorch.utils._pytreer#   Ú__all__r   r,  r  r+  r_   rO   r$   r^   r&  r'  r%   rb   rá   r\   r&   r'   r(   r)   r5   rN   rÈ   r>   r*   rS   rd   rM   r’   r”   r¦   r  rv  rÏ   rØ   rÝ   ÚProcessGrouprã   ré   Úno_gradrû   r  r  r0  rC  rW  r_  rr  r+   r,   r-   r  r.   r/   r0   r�  r“  rP   rQ   r=   ú<module>r§     sP  ðà Ð Ð Ð Ð Ø Ð Ð Ð Ø 	€	€	€	Ø €€€Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø Ð Ð Ð Ð Ð Ø +Ð +Ð +Ð +Ð +Ð +Ð +Ð +Ð +Ð +à €€€Ø  Ð  Ð  Ð  Ð  Ð  Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ Aðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ðð ð ð ð ð ð	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ðð ð ð ð ð ð ð ð JÐ IÐ IÐ IÐ IÐ IØ ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø <Ð <Ð <Ð <Ð <Ð <Ø -Ð -Ð -Ð -Ð -Ð -ðð ð €ð" €Ø€Ø
€Ø	€à	ˆSŒ€Ø˜-Ñ'¨%¬,Ñ6¸Ñ<¸uÑDÀsÑJ€à�D˜Ô'Ñ'¨%°Ô*>Ñ>ÀÀcÈ;ÐFVÔAWÑWð 
ð �S˜)�^Ô$€Ø˜Ô'Ð Ø˜#˜}Ð/@Ñ@Ð@ÔAÐ ð &) S¡U¤UÐ �S˜”]Ð *Ð *Ñ *ð Ôðð ñ Ôðð ð,.ð ,.ð ,.ð ,.ð ,.ñ ,.ô ,.ñ „ð,.ð^ ð@ð @ð @ð @ð @Ð%ñ @ô @ñ „ð@ð& .Ø Ø!%ðFEð FEØŒ9ðFEà
ðFEð ðFEð ð	FEð
 ðFEð ðFEð FEð FEð FEðR	ð 	ð 	ð 	ð 	ñ 	ô 	ð 	ð%"ð %"ð %"ð %"ðZ )-Ø'+ð}ð }ð }ØŒ9ð}à�%”+Ô'¨Ð,Ô-ð}ð ð}ð
 �B”I” Ñ%ð}ð  Ñ$ð}ð ð}ð }ð }ð }ð@+Ø˜3 	˜>Ô*ð+à(ð+ð ð+ð 
ð	+ð +ð +ð +ð\˜œ	 E¤KÔ$9Ñ9ð Àð Èð ð ð ð ðØŒ9ðØ*ðà	Ô˜Ñðð ð ð ð&Ø�S˜#�X”ðØ&4ðà	ˆ#ˆsˆ(„^ðð ð ð ð$ €„�„ð?;ØŒ9ð?;Ø*ð?;à	ˆ#ˆyˆ.Ôð?;ð ?;ð ?;ñ „ð?;ðD €„�„ðA
ØŒ9ðA
à�S˜)�^Ô$ðA
ð ðA
ð ð	A
ð A
ð A
ñ „ðA
ðH'&˜Uœ[Ô2ð '&°tð '&ð '&ð '&ð '&ðT_Ð*<ð _ÀÀcÈ9ÀnÔAUð _ð _ð _ð _ðDEØŒ;Ô ðEà�S˜)�^Ô$ðEð ðEð ð	Eð Eð Eð EðP €„�„ðIAØŒ9ðIAà�e”kÔ+¨SÐ0Ô1ðIAð ðIAð ð	IAð IAð IAñ „ðIAðXaØŒ9ðaàŒ;Ô ðað )ðað ð	að
 ðað að að aðH €„�„ðgNØŒ9ðgNà�e”kÔ+¨SÐ0Ô1ðgNð #ðgNð ð	gNð
 
ðgNð gNð gNñ „ðgNðZ )-Ø'+ð	" ð " ð " ØŒ9ð" ð �B”I” Ñ%ð" ð  Ñ$ð	" ð
 
ˆ#ˆyˆ.Ôð" ð " ð " ð " ðR )-Ø'+ð* ð * ð * ØŒ9ð* à”Ô%¨°´Ô1FÔ(GÑGð* ð �B”I” Ñ%ð	* ð
  Ñ$ð* ð ð* ð * ð * ð * ðb )-Ø'+ðX2ð X2ð X2ØŒ9ðX2à”Ô%¨°´Ô1FÔ(GÑGðX2ð �B”I” Ñ%ð	X2ð
  Ñ$ðX2ð ˆ4��Y�ÔÐ!3Ð3Ô4ðX2ð X2ð X2ð X2ðv"6ØŒ9ð"6à�R”Y  S¨) ^Ô 4Ð4Ô5¸¸SÀ)¸^Ô8LÑLð"6ð 
ˆ#ˆyˆ.Ôð"6ð "6ð "6ð "6ðR (,ð	$Eð $Eð $EØŒ9ð$Eà˜3 	˜>Ô*ð$Eð  Ñ$ð	$Eð
 ð$Eð $Eð $Eð $EðX (,ð(Jð (Jð (JØŒ9ð(Jà”Ô%¨°´Ô1FÔ(GÑGð(Jð )ð(Jð
  Ñ$ð(Jð 
ð(Jð (Jð (Jð (Jðb (,ð=Eð =Eð =EØŒ9ð=Eà”Ô%¨°´Ô1FÔ(GÑGð=Eð ˜3 	˜>Ô*ð	=Eð
 )ð=Eð  Ñ$ð=Eð ð=Eð =Eð =Eð =EðD ð (,ð12ð 12ð 12ØŒ9ð12ð  Ñ$ð12ð 
ð	12ð 12ð 12ñ „ð12ðl ð
 (,ð	;5ð ;5ð ;5ØŒ9ð;5ð �e”kÔ+¨SÐ0Ô1ð;5ð  Ñ$ð	;5ð
 
ð;5ð ;5ð ;5ñ „ð;5ð ;5ð ;5rQ   