§
    ‚ŠtjÕ*  ã                  óT  — d dl mZ d dlZd dlZd dlmZmZ ddlmZ ddl	m
Z
mZmZmZ ddlmZ erd dlmZ dd	lmZ  e
¦   «         rd dlZ e
¦   «         r ed
¦  «        rd dlmZ d dlmZmZ  ej        e¦  «        Zd,d„Zd-d„Zd.d„Zd/d„Z d0d„Z!d1d„Z"d2d!„Z#d3d&„Z$d4d)„Z%d*„ Z&d+„ Z'dS )5é    )ÚannotationsN)ÚTYPE_CHECKINGÚAnyé   ©Ú replace_layer_number_by_wildcard)Úis_torch_availableÚis_torch_greater_or_equalÚloggingÚ	strtobool)ÚQuantizationMethodé   )ÚDistributedConfigú2.6)Úfully_shard)ÚCPUOffloadPolicyÚMixedPrecisionPolicyÚreturnÚboolc                 ó^  — t          ¦   «         sdS t          j                             ¦   «         ot          j                             ¦   «         oat          t          j                             dd¦  «        ¦  «        dk    o0t          t          j                             dd¦  «        ¦  «        dk    S )uM   Check if FSDP is active via Accelerate (env var based) â€” covers FSDP1 only.FÚACCELERATE_USE_FSDPÚFalser   ÚFSDP_CPU_RAM_EFFICIENT_LOADING)	r	   ÚtorchÚdistributedÚis_availableÚis_initializedr   ÚosÚenvironÚget© ó    ú[/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/distributed/fsdp.pyÚis_fsdp_enabledr$   (   s—   € åÑÔð Øˆuõ 	Ô×&Ò&Ñ(Ô(ð 	VÝÔ×,Ò,Ñ.Ô.ð	Vå•b”j—n’nÐ%:¸GÑDÔDÑEÔEÈÒJð	Võ •b”j—n’nÐ%EÀwÑOÔOÑPÔPÐTUÒUð	r"   Úmoduleú	nn.Modulec                óÚ   — t          ¦   «         sdS t          j                             ¦   «         sdS t	          | dd¦  «        rdS 	 ddlm} n# t          $ r Y dS w xY wt          | |¦  «        S )z.Check if a module is managed by FSDP (1 or 2).FÚ_is_fsdp_managed_moduleTr   )ÚFullyShardedDataParallel)	r	   r   r   r   ÚgetattrÚtorch.distributed.fsdpr)   ÚImportErrorÚ
isinstance)r%   r)   s     r#   Úis_fsdp_managed_moduler.   5   sš   € åÑÔð ØˆuÝÔ×)Ò)Ñ+Ô+ð Øˆuõ ˆvÐ0°%Ñ8Ô8ð ØˆtðØCÐCÐCÐCÐCÐCÐCøÝð ð ð Øˆuˆuðøøøå�fÐ6Ñ7Ô7Ð7s   ÁA Á
AÁAÚdistributed_configúDistributedConfig | Noneúdict[str, Any]c                ó¢   — | €i S i }| j         rt          ¦   «         |d<   | j        r)t          t          j        t          j        d¬¦  «        |d<   |S )zMBuild ``fully_shard`` policy kwargs from ``DistributedConfig`` runtime flags.NÚoffload_policy)Úparam_dtypeÚreduce_dtypeÚoutput_dtypeÚ	mp_policy)Úfsdp_cpu_offloadr   Úfsdp_mixed_precisionr   r   Úbfloat16Úfloat32)r/   Úfsdp_policy_kwargss     r#   Ú_get_fsdp_policy_kwargsr=   G   sk   € àÐ!Øˆ	àÐØÔ*ð BÝ/?Ñ/AÔ/AÐÐ+Ñ,ØÔ.ð 
Ý*>ÝœÝœØð+
ñ +
ô +
Ð˜;Ñ'ð
 Ðr"   Úmodelú)tuple[nn.Module | None, nn.Module | None]c                ó¢   — d }d }t          | d¦  «        r|                      ¦   «         }t          | d¦  «        r|                      ¦   «         }||fS )NÚget_input_embeddingsÚget_output_embeddings)ÚhasattrrA   rB   )r>   Úinput_embedÚoutput_heads      r#   Ú_get_input_output_embeddingsrF   X   s_   € Ø€KØ€KÝˆuÐ,Ñ-Ô-ð 3Ø×0Ò0Ñ2Ô2ˆÝˆuÐ-Ñ.Ô.ð 4Ø×1Ò1Ñ3Ô3ˆØ˜Ð#Ð#r"   Úno_reshard_targetsúlist[tuple[str, nn.Module]]c                óJ  ‡
— t          | ¦  «        dk    rdS t          |¦  «        \  }}d„ ||fD ¦   «         Š
g g }}| D ]/\  }}|                     |¦  «         |                     |¦  «         Œ0t          d„ |D ¦   «         ¦  «        }t          ˆ
fd„|D ¦   «         ¦  «        }	|o|	S )Nr   Fc                ó   — h | ]}|®|’ŒS ©Nr!   )Ú.0r%   s     r#   ú	<setcomp>z(is_norm_and_head_pair.<locals>.<setcomp>f   s   € ÐZÐZÐZ˜vÀvÐGY�FÐGYÐGYÐGYr"   c              3  óL   K  — | ]}|d k    p|                      d¦  «        V — Œ dS )Únormz.normN)Úendswith)rL   Únames     r#   ú	<genexpr>z(is_norm_and_head_pair.<locals>.<genexpr>m   s7   è è € ÐTÐTÀd˜ šÐA¨4¯=ª=¸Ñ+AÔ+AÐTÐTÐTÐTÐTÐTr"   c              3  ó    •K  — | ]}|‰v V — Œ	d S rK   r!   )rL   r%   Úhead_moduless     €r#   rR   z(is_norm_and_head_pair.<locals>.<genexpr>n   s(   øè è € ÐGÐG°V˜& LÐ0ÐGÐGÐGÐGÐGÐGr"   )ÚlenrF   ÚappendÚany)rG   r>   rD   rE   ÚnamesÚmodulesrQ   r%   Úhas_final_normÚhas_output_headrT   s             @r#   Úis_norm_and_head_pairr\   b   sÒ   ø€ Ý
ÐÑÔ !Ò#Ð#ØˆuÝ;¸EÑBÔBÑ€K�ØZÐZ¨+°{Ð)CÐZÑZÔZ€Là˜ˆ7€EØ*ð ð ‰ˆˆfØ�Š�TÑÔÐØ�Š�vÑÔÐÐåÐTÐTÈeÐTÑTÔTÑTÔT€NÝÐGÐGÐGÐG¸wÐGÑGÔGÑGÔG€OØÐ-˜oÐ-r"   Ú	fsdp_planúdict[str, str]c                ó°  — t          |dd¦  «        pi }|s| S t          |¦  «        \  }}d„ |                     ¦   «         D ¦   «         }|                     |¦  «        }|                     |¦  «        }|�|€| S |                      ¦   «         }|                     |d¦  «         |                      |¦  «        dk    r|                     |d¦  «         d||<   |S )a›  
    Rewrite the plan so tied embed/lm_head weights are wrapped once.
    Example:
        {"model.embed_tokens": "free_full_weight",
        "model.layers.*": "free_full_weight",
        "model.norm": "keep_full_weight",
        "lm_head": "keep_full_weight"}
    ->
        {"model.layers.*": "free_full_weight",
        "model.norm": "keep_full_weight",
        "model.embed_tokens": "keep_full_weight"}
    Úall_tied_weights_keysNc                ó   — i | ]\  }}||“Œ	S r!   r!   )rL   rQ   r%   s      r#   ú
<dictcomp>z4_resolve_tied_embed_lm_head_plan.<locals>.<dictcomp>‡   s   € ÐMÐMÐM¡| t¨V�f˜dÐMÐMÐMr"   Úkeep_full_weight)r*   rF   Únamed_modulesr    ÚcopyÚpop)	r]   r>   Ú	tied_keysrD   rE   Úname_by_moduleÚembed_moduleÚhead_moduleÚadapted_plans	            r#   Ú _resolve_tied_embed_lm_head_planrl   r   sô   € õ  ˜Ð6¸Ñ=Ô=ÐCÀ€IØð ØÐå;¸EÑBÔBÑ€K�ØMÐM°u×7JÒ7JÑ7LÔ7LÐMÑMÔM€NØ!×%Ò% kÑ2Ô2€LØ ×$Ò$ [Ñ1Ô1€KàÐ˜{Ð2ØÐà—>’>Ñ#Ô#€LØ×Ò�\ 4Ñ(Ô(Ð(à‡}‚}�[Ñ!Ô!Ð%7Ò7Ð7Ø×Ò˜ dÑ+Ô+Ð+Ø%7ˆ�\Ñ"àÐr"   ú?tuple[list[tuple[str, nn.Module]], list[tuple[str, nn.Module]]]c                óî   — g }g }|                       ¦   «         D ]Y\  }}||v r|nt          |¦  «        }||v r;||         dk    r|                     ||f¦  «         ŒB|                     ||f¦  «         ŒZ||fS )zUExpand plan keys into reshard and no-reshard ``(module_name, module)`` shard targets.rc   )rd   r   rV   )r>   r]   Úreshard_targetsrG   Úmodule_namer%   Úplan_keys          r#   Úexpand_fsdp_planrr   ˜   sª   € ð
 46€OØ68Ðà$×2Ò2Ñ4Ô4ð >ð >Ñˆ�VØ"-°Ð":Ð":�;�;Õ@`ÐalÑ@mÔ@mˆØ�yÐ Ð Ø˜Ô"Ð&8Ò8Ð8Ø"×)Ò)¨;¸Ð*?Ñ@Ô@Ð@Ð@à×&Ò&¨°VÐ'<Ñ=Ô=Ð=øàÐ.Ð.Ð.r"   Úmodule_namesú	list[str]údict[str, str] | NoneÚNonec                ó\  ‡— |sdS t                                | ¦  «        }i }i }|                     ¦   «         D ]3\  Š}|dvr||‰<   Œ‰|vr t          ˆfd„|D ¦   «         ¦  «        s||‰<   Œ4|rt                               d|› �¦  «         |rt                               d|› �¦  «         dS dS )zs
    Verify the FSDP plan of the model, log a warning if plan keys were not applied or strategies are invalid.
    N>   Úfree_full_weightrc   c              3  ó>   •K  — | ]}t          |¦  «        ‰k    V — Œd S rK   r   )rL   rQ   Úkeys     €r#   rR   z#verify_fsdp_plan.<locals>.<genexpr>¹   s4   øè è € Ð/vÐ/vÐbfÕ0PÐQUÑ0VÔ0VÐZ]Ò0]Ð/vÐ/vÐ/vÐ/vÐ/vÐ/vr"   z4The following FSDP entries have unknown strategies: z9The following FSDP rules were not applied to any module: )ÚdictÚfromkeysÚitemsrW   ÚloggerÚwarning)rs   r]   Úname_lookupÚunused_rulesÚinvalid_strategiesÚstrategyrz   s         @r#   Úverify_fsdp_planr„   «   sú   ø€ ð ð Øˆå—-’- Ñ-Ô-€KØ#%€LØ)+Ðà"ŸšÑ*Ô*ð )ð )‰ˆˆXØÐCÐCÐCØ&.Ð˜sÑ#Ð#Ø˜Ð#Ð#­CÐ/vÐ/vÐ/vÐ/vÐjuÐ/vÑ/vÔ/vÑ,vÔ,vÐ#Ø (ˆL˜Ñøàð dÝ�ŠÐbÐN`ÐbÐbÑcÔcÐcØð cÝ�ŠÐaÐS_ÐaÐaÑbÔbÐbÐbÐbðcð cr"   Ú	fsdp_meshú(torch.distributed.device_mesh.DeviceMeshc                óÜ  — t          ¦   «         st          d¦  «        ‚t          d¦  «        st          d¦  «        ‚t	          t          | dd¦  «        pi ¦  «        }|s$t          t          | ¦  «        j        › d�¦  «        ‚t          | j	        dd¦  «        }t          |¦  «        }t          || ¦  «        }t          | |¦  «        \  }}|D ]3\  }}	t          |	f|dd	œ|¤Ž t                               d
|› d�¦  «         Œ4t!          || ¦  «        reg g }}
|D ]/\  }}	|
                     |¦  «         |                     |	¦  «         Œ0t          |f|dd	œ|¤Ž t                               d|
› d�¦  «         n6|D ]3\  }}	t          |	f|dd	œ|¤Ž t                               d
|› d�¦  «         Œ4t          | fd|i|¤Ž t                               dt'          |¦  «        › d�¦  «         d| _        | S )z/
    Apply FSDP2 (fully_shard) to a model.
    z$PyTorch is required for FSDP supportr   zFSDP2 requires torch>=2.6Ú
_fsdp_planNzr does not have a FSDP2 plan declared. Set `base_model_fsdp_plan` on the config and `_fsdp_plan` on the head class.r/   T)ÚmeshÚreshard_after_forwardzApplied fully_shard to z (reshard=True)FzGrouped tail z (reshard=False)r‰   z'FSDP2 applied to model via _fsdp_plan: z entries)r	   r,   r
   ÚOSErrorr{   r*   Ú
ValueErrorÚtypeÚ__name__Úconfigr=   rl   rr   r   r~   Údebugr\   rV   ÚinforU   r(   )r>   r…   r]   r/   r<   Úadapted_fsdp_planro   rG   rp   r%   rX   rY   rQ   s                r#   Ú!apply_fully_sharded_data_parallelr“   Â   sw  € õ ÑÔð BÝÐ@ÑAÔAÐAå$ UÑ+Ô+ð 3ÝÐ1Ñ2Ô2Ð2å•W˜U L°$Ñ7Ô7Ð=¸2Ñ>Ô>€IØð 
ÝÝ�E‰{Œ{Ô#ð Wð Wð Wñ
ô 
ð 	
õ
 ! ¤Ð/CÀTÑJÔJÐÝ0Ð1CÑDÔDÐå8¸ÀEÑJÔJÐÝ*:¸5ÐBSÑ*TÔ*TÑ'€OÐ'à.ð Mð MÑˆ�VÝ�FÐ] À$Ð]Ð]ÐJ\Ð]Ð]Ð]Ý�ŠÐK¨{ÐKÐKÐKÑLÔLÐLÐLõ Ð/°Ñ7Ô7ð 
KØ˜RˆwˆØ.ð 	#ð 	#‰LˆD�&Ø�LŠL˜ÑÔÐØ�NŠN˜6Ñ"Ô"Ð"Ð"Ý�GÐ_ )À5Ð_Ð_ÐL^Ð_Ð_Ð_Ý�ŠÐ< UÐ<Ð<Ð<Ñ=Ô=Ð=Ð=à.ð 	Kð 	K‰LˆD�&Ý˜Ðb YÀeÐbÐbÐOaÐbÐbÐbÝ�LŠLÐI°4ÐIÐIÐIÑJÔJÐJÐJõ �Ð<Ð<˜IÐ<Ð);Ð<Ð<Ð<å
‡K‚KÐR½#¸i¹.¼.ÐRÐRÐRÑSÔSÐSð %)€EÔ!ð €Lr"   c                 ój   — ddl m}  dt          t          j        | ¦  «        j        ¦  «        v rddiS i S )zÀ
    Returns checkpoint kwargs for FSDP model saving.

    Checks if the `adapter_only` parameter is supported by `save_fsdp_model` from accelerate
    and returns the appropriate kwargs.
    r   ©Úsave_fsdp_modelÚadapter_onlyT)Úaccelerate.utilsr–   ÚlistÚinspectÚ	signatureÚ
parametersr•   s    r#   Úget_fsdp_ckpt_kwargsr�   ü   sG   € ð 1Ð0Ð0Ð0Ð0Ð0à��gÔ/°Ñ@Ô@ÔKÑLÔLÐLÐLØ Ð%Ð%àˆ	r"   c                óN  — ddl m} ddlm} t	          | j        |¦  «        r || ¦  «        |j        j        _        t          | dd¦  «        t          j        k    rH| j        j        j        j        r4|j        j                             | j        j        j        d¬¦  «         dS dS dS )aG  
    Updates the FSDP plugin for PEFT LoRA/QLoRA compatibility.

    When using FSDP with PEFT LoRA, the auto wrap policy needs to be updated to additionally wrap
    LoRA trainable layers separately. When using FSDP with QLoRA, the mixed precision policy needs
    to be updated to use the quantization storage data type.
    r   )Ú
PeftConfig)Úfsdp_auto_wrap_policyÚquantization_methodNT)Úoverride)ÚpeftrŸ   Úpeft.utils.otherr    r-   Úactive_peft_configÚstateÚfsdp_pluginÚauto_wrap_policyr*   r   ÚBITS_AND_BYTESÚhf_quantizerÚquantization_configÚbnb_4bit_quant_storageÚis_floating_pointÚset_mixed_precision)r>   ÚacceleratorrŸ   r    s       r#   Úupdate_fsdp_plugin_peftr°     sÕ   € ð  ÐÐÐÐÐØ6Ð6Ð6Ð6Ð6Ð6å�%Ô*¨JÑ7Ô7ð VØ9NÐ9NÈuÑ9UÔ9UˆÔÔ%Ô6å�Ð,¨dÑ3Ô3Õ7IÔ7XÒXÐXØÔÔ2ÔIÔ[ð 	Yð 	ÔÔ%×9Ò9ØÔÔ2ÔIÐTXð 	:ñ 	
ô 	
ð 	
ð 	
ð 	
ð 	YÐXÐXÐXr"   )r   r   )r%   r&   r   r   )r/   r0   r   r1   )r>   r&   r   r?   )rG   rH   r>   r&   r   r   )r]   r^   r>   r&   r   r^   )r>   r&   r]   r^   r   rm   )rs   rt   r]   ru   r   rv   )r>   r&   r…   r†   r   r&   )(Ú
__future__r   rš   r   Útypingr   r   Úintegrations.tensor_parallelr   Úutilsr	   r
   r   r   Úutils.quantization_configr   Útorch.nnÚnnÚconfiguration_utilsr   r   Ú"torch.distributed._composable.fsdpr   r+   r   r   Ú
get_loggerrŽ   r~   r$   r.   r=   rF   r\   rl   rr   r„   r“   r�   r°   r!   r"   r#   ú<module>r»      s  ðð #Ð "Ð "Ð "Ð "Ð "à €€€Ø 	€	€	€	Ø %Ð %Ð %Ð %Ð %Ð %Ð %Ð %à KÐ KÐ KÐ KÐ KÐ KØ UÐ UÐ UÐ UÐ UÐ UÐ UÐ UÐ UÐ UÐ UÐ UØ :Ð :Ð :Ð :Ð :Ð :ð ð 7ØÐÐÐÐÐà6Ð6Ð6Ð6Ð6Ð6àÐÑÔð Ø€L€L€LàÐÑÔð NÐ5Ð5°eÑ<Ô<ð NØ>Ð>Ð>Ð>Ð>Ð>ØMÐMÐMÐMÐMÐMÐMÐMà	ˆÔ	˜HÑ	%Ô	%€ð
ð 
ð 
ð 
ð8ð 8ð 8ð 8ð$ð ð ð ð"$ð $ð $ð $ð.ð .ð .ð .ð #ð #ð #ð #ðL/ð /ð /ð /ð&cð cð cð cð.5ð 5ð 5ð 5ðtð ð ð
ð 
ð 
ð 
ð 
r"   