tj$DddlZddlmZddlmZmZddlZddlmZddl m Z ddl m Z ddl mZm Z dZGd d e Zd ed ed efdZGddZe e ddejdeejjdzd ejfdZdejd efdZdS)N)Iterable)AnyNoReturn)_State)DistributedDataParallel) _get_registrycontractc feZdZUejed<dfd Zefdej de ej de ej de ddf d Z dd Zdej de ej ddfd Zdd Zdd Zdej deedfdee efdefdZdej deejdejdejfdZxZS)_ReplicateState _ddp_weakrefreturnNcttj|_d|_tj|_|j|_g|_d|_ d|_ i|_ g|_ dS)NF) super__init__nn ParameterListmodulehas_initialized _param_list _orig_module _param_names_no_sync _init_args _init_kwargs_comm_hook_args)self __class__s e/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/torch/distributed/_composable/replicate.pyrz_ReplicateState.__init__sz !#!1!3!3 %*-/-=-?-?!K')# 26,.*,rignored_modulesignored_paramsprefixct|rdS||vrdS|tkr|dnt}|dD]A\}}||vr8|j||j||B|D]"\}} || ||||#dS)N.F)recurse)r$)_is_fully_sharded_ROOT_MODULE_PREFIXnamed_parametersrappendrnamed_children_collect_params) rrr"r#r$recurse_prefixnpname child_modules r r-z_ReplicateState._collect_params"s V $ $  F _ $ $ F#&999vLLLL?R ++E+:: A ADAq&& ''***!((N)?A)?)?@@@"("7"7"9"9   D,  (0$00 !      r!c\tjdfd}|dS)NT) recursivecjtjjijd_i_dS)N)rAssertionErrorinitrregister_comm_hook)rsr _lazy_initz-_ReplicateState.lazy_init.._lazy_initCsT&$$ DIt <$*; < < <  # # % % % DO "D   r!)torch_disable_dynamo)rr:s` r lazy_initz_ReplicateState.lazy_initBsF   . . . # # # # / . #  r!c |jrdSd|_||_d|D}|D]8}t|r'||9ddlm}|||||||d|vr\|d:|d}t|tj r|j dkrd|d<n |g|d<nd|d<| dt|jfi||_t#j|jt&|j_dS) NTc@h|]}|D]}|Sr6) parameters).0mr0s r z'_ReplicateState.init..Ys-MMMallnnMM!MMMMr!r)_localize_dtensor)r# device_idcpu device_ids)rrmodulesr(updater@%torch.distributed.tensor.parallel.ddprDr- isinstancer;devicetypepoprr_ddpweakrefref replicatestater)rrr"kwargsr# submodulerDrEs r r8z_ReplicateState.initNs~    F# MM_MMM)) > >I ++ >%%i&:&:&<&<===KKKKKK&@@@@ V_nEEE & k".";/ i6679>U;R;R+/F<((-6;F<(('+|$ JJ{ # # #+D,<GGGG 4;K 4J4J  $$111r!cv|jD]\}}|jj|i||jdSN)rrOr9clear)r comm_args comm_kwargss r r9z"_ReplicateState.register_comm_hookxsS&*&: D D "I{ (DI () C{ C C C C ""$$$$$r!c"||_||_dSrW)rrrargsrTs r record_init_argsz _ReplicateState.record_init_args}s"r!r].rTc|js|jr||j |j_|jt _|jj|i|SrW) rrr=rrOrequire_backward_grad_syncr_active_ddp_module _pre_forward)rrr]rTs r forward_pre_hookz _ReplicateState.forward_pre_hooks\ ? d/  NN   37=/@ ,59Y2%ty%t6v666r!inputoutputcNdt_|j|SrW)rrarO _post_forward)rrrdres r forward_post_hookz!_ReplicateState.forward_post_hooks# 6:2y&&v...r!rN)__name__ __module__ __qualname__rPrQ__annotations__rr)rModuleset Parameterstrr-r=r8r9r^tuplerdictrcr;Tensorrh __classcell__)rs@r r r s+ - - - - - -&*  RYBL)     @    (K (KRY(K  (K(K(K(KT%%%% ####7i7',S#X7@DS#X7 7777/ /U\"/ /  ////////r!r r]rTrc td)NzGDDP does not support deepcopy. Please use state dict for serialization.)r7)r]rTs r unimplemented_deepcopyrws Q  r!c,eZdZdZdeddfdZddZdS)DDPc>|jd}|j|g|Ri|S)z Override ``__new__`` to remove the DDP class and directly construct the original class for cases like indexing into a container module. )__mro____new__)clsr]rTorig_clss r r}z DDP.__new__s2;q>x:4:::6:::r!requires_gradient_syncrNcF| t|_dS)a Sets if the module should sync gradients. This can be used to implement gradient accumulation without communication. Args: requires_gradient_sync (bool): Whether to reduce gradients for the module's parameters. N)rRrSr)rrs r set_requires_gradient_synczDDP.set_requires_gradient_syncs".D)C &&&r!cnt|j||fdSrW)rRrSrr+r\s r r9zDDP.register_comm_hooks/-44dF^DDDDDr!ri)rjrkrlr}boolrr9r6r!r ryryse;;; D D$ D D D DEEEEEEr!ry) state_clsrr"c :tjdd|vrLt|dttjfs%t dt|dt|rt d|i}nt|}t |}| |j d|d}|L|}||kr2d d lm}m}| |||||j|j||fi||j}d t.i} td |jt2|f| } | |_|S) zReplicates a module Args: module (torch.nn.Module): module to replicate Example:: >>> # xdoctest: +REQUIRES(module:torch._C._distributed_c10d) >>> module = nn.Linear(3, 3) >>> replicate(module) ztorch.distributed.replicaterEz6Expected device_id to be int or torch.device, but got zGCannot apply `replicate()` on a Module already managed by `fully_shard`NT) with_kwargs device_meshr)rD_reconstruct_dtensor __deepcopy__ry)r;_C_log_api_usage_oncerKintrL RuntimeErrorrMr(rorRrSregister_forward_pre_hookrcget_get_root_meshrJrDrregister_forward_hookrhr^rrwrjry) rr"rTrSr root_meshrDrr~dctnew_clss r rRrRs  H  !>???f&-U\/BCC 7{ 34477      U   o.. OOF # #E $$U%;$NNN**]++K..00   # #          , ,-A B B B  ( (): ; ; ;   !8999E6?==f===  C 1 2C'''#sS99GF Mr!c0t|}|dSd|vS)z+Check if module is marked with fully_shard.NF fully_shard)r )rregistrys r r(r(s%V$$Hu H $$r!rW)rPcollections.abcrtypingrrr;torch.nnr#torch.distributed._composable_statertorch.nn.parallelrr r r)r rwryrnrRrr(r6r!r rs$$$$$$ 666666555555--------@/@/@/@/@/f@/@/@/F#EEEEEEEE4 O$$$9=CC ICeho.5CY CCC%$CL%bi%D%%%%%%r!