VhV$ZddlZddlmZddlmZmZmZddlZddlm Z ddl m Z ddl m Z ddlmZmZdZGd d e Zd ed ed efdZGddZee dde j(deeejj(d e j(fdZde j(d efdZy)N)Iterable)AnyNoReturnOptional)_State)DistributedDataParallel) _get_registrycontractc eZdZUejed<dfd Zefdejde ejde ejde ddf d Z dd Zdejde ejddfd Zdd Zdd Zdejdeedfdee efdefdZdejdeej.dej.dej.fdZxZS)_ReplicateState _ddp_weakrefreturnNct|tj|_d|_tj|_|j|_g|_d|_ d|_ i|_ g|_ y)NF) super__init__nn ParameterListmodulehas_initialized _param_list _orig_module _param_names_no_sync _init_args _init_kwargs_comm_hook_args)self __class__s W/home/dcms/DCMS/lib/python3.12/site-packages/torch/distributed/_composable/replicate.pyrz_ReplicateState.__init__sk !#!1!1!3 %*-/-=-=-?!KK')# 59,.*,rignored_modulesignored_paramsprefixcbt|ry||vry|tk7r|dnt}|jdD]D\}}||vs |jj ||j j ||F|j D]\}} |j| |||| y)N.F)recurse)r%)_is_fully_sharded_ROOT_MODULE_PREFIXnamed_parametersrappendrnamed_children_collect_params) rrr#r$r%recurse_prefixnpname child_modules r!r.z_ReplicateState._collect_params"s V $  _ $ #&99vhaL?R ++E+: ADAq&  ''*!!((N+;A3)?@ A #)"7"7"9  D,  ()$0 !  r"cPtjdfd}|y)NT) recursivecjJjjijjd_i_y)N)rinitrregister_comm_hook)rsr! _lazy_initz-_ReplicateState.lazy_init.._lazy_initCsK??. .. DIIt <$*;*; <  # # % DO "D r")torch_disable_dynamo)rr:s` r! lazy_initz_ReplicateState.lazy_initBs'    . # / #  r"c |jryd|_||_|Dchc]}|jD]}|}}}|jD]-}t |s|j |j/ddlm}||||j|||d|vrV|d;|d} t| tjr| jdk(rd|d<n | g|d<nd|d<|jdt|jfi||_t#j$|j t&j)|j_ycc}}w)NTr)_localize_dtensor)r$ device_idcpu device_ids)rr parametersmodulesr)update%torch.distributed.tensor.parallel.ddpr?r. isinstancer;devicetypepoprr_ddpweakrefref replicatestater) rrr#kwargsmr1r$ submoduler?r@s r!r8z_ReplicateState.initMs@    # %4MallnM!M!MM) >I +%%i&:&:&<= > L&@ V_nE & k".";/ i69>>U;R+/F<(-6;F<('+|$ JJ{ #+D,<,<GG 4;KK 4J  $1;NsEc|jD]!\}}|jj|i|#|jjyN)rrKr9clear)r comm_args comm_kwargss r!r9z"_ReplicateState.register_comm_hookwsI&*&:&: D "I{ (DII ( () C{ C D ""$r"c ||_||_yrT)rrrargsrPs r!record_init_argsz _ReplicateState.record_init_args|s"r"rZ.rPc|js |jr|j|j |j_|jj |i|SrT)rrr=rrKrequire_backward_grad_sync _pre_forward)rrrZrPs r!forward_pre_hookz _ReplicateState.forward_pre_hooksL ??d// NN 37==/@ ,%tyy%%t6v66r"inputoutputc8|jj|SrT)rK _post_forward)rrr`ras r!forward_post_hookz!_ReplicateState.forward_post_hooks yy&&v..r"rN)__name__ __module__ __qualname__rLrM__annotations__rr*rModuleset Parameterstrr.r=r8r9r[tuplerdictr_r;Tensorrd __classcell__)r s@r!rrs++ -&*  RYYBLL)     @ (K (KRYY(K  (KT% #7ii7',S#X7@DS#X7 7/ /U\\"/ /  /r"rrZrPrctd)NzGDDP does not support deepcopy. Please use state dict for serialization.)AssertionError)rZrPs r!unimplemented_deepcopyrts Q r"c*eZdZdZdeddfdZddZy)DDPcL|jd}|j|g|i|S)z Override ``__new__`` to remove the DDP class and directly construct the original class for cases like indexing into a container module. )__mro____new__)clsrZrPorig_clss r!rzz DDP.__new__s.;;q>x:4:6::r"requires_gradient_syncrNc:| tj|_y)a Sets if the module should sync gradients. This can be used to implement gradient accumulation without communication. Args: requires_gradient_sync (bool): Whether to reduce gradients for the module's parameters. N)rNrOr)rr}s r!set_requires_gradient_synczDDP.set_requires_gradient_syncs.D)C &r"cdtj|jj||fyrT)rNrOrr,rYs r!r9zDDP.register_comm_hooks#--44dF^Dr"re)rfrgrhrzboolrr9r7r"r!rvrvs#; D D$ DEr"rv) state_clsrr#c tjjdd|vr=t|dttj fst dt|dt|r t d|i}n t|}tj|}|j|jd|jdd}|Fd d lm}|j#|}||k7r*d d lm}m}|j||j+||j+|j,|j.||fi||j0} d t2i} td | j4t6| f| } | |_|S)zReplicates a module Args: module (torch.nn.Module): module to replicate Example:: >>> # xdoctest: +REQUIRES(module:torch._C._distributed_c10d) >>> module = nn.Linear(3, 3) >>> replicate(module) ztorch.distributed.replicater@z6Expected device_id to be int or torch.device, but got zGCannot apply `replicate()` on a Module already managed by `fully_shard`NT) with_kwargs device_meshr)_mesh_resources)r?_reconstruct_dtensor __deepcopy__rv)r;_C_log_api_usage_oncerGintrH RuntimeErrorrIr)rkrNrOregister_forward_pre_hookr_gettorch.distributed.device_meshr get_root_meshrFr?rregister_forward_hookrdr[r rtrfrv) rr#rPrOrr root_meshr?rr{dctnew_clss r!rNrNs}  HH  !>?f&-U\\/BC{ 3457    U  o. OOF #E $$U%;%;$N**]D1KA#11+>   #   , ,-A B  ( (): ;   !8!89E6?=f=   C 1 2CS'#sS9GF Mr"c&t|}|yd|vS)z+Check if module is marked with fully_shard.F fully_shard)r )rregistrys r!r)r)s V$H H $$r"rT)rLcollections.abcrtypingrrrr;torch.nnr#torch.distributed._composable_statertorch.nn.parallelrr r r*rrtrvrjrNrr)r7r"r!rs$** 65-}/f}/@#EE4 O$<@E IIEhuxx78EYY E%EP%bii%D%r"