VhddlZddlZddlZddlZddlmZmZddlmZm Z m Z m Z ddl Z ddl mZddlmcmcmcmZddlmZddlmcmZddlmZmZmZddlmZddlm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'ddl(m)Z)ddl*m+Z+m,Z,m-Z-m.Z.dd l/m0Z0m1Z1m2Z2dd l3m4Z4dd l5m6Z6d d l7m8Z8m9Z9m:Z:m;Z;mZ>m?Z?ejeAZBde deCfdZDdeEdeEfdZFdejde deeHeEeEeEffdZIdejdeeHeEeEeEffdZJe d=dejde deCdeCdeCdeCddfdZKe dejde ddfdZLdejde ddfdZMdejde deCdeCddf dZNe dejde deOeEefd eEd!e deOeEeff d"ZPe de dejddfd#ZQe dejde deOeEefd eEdeOeEeff d$ZRdejde deOeEefd eEddf d%ZSdejde ddfd&ZTde dejddfd'ZUe dejde deOeEefd eEdeOeEeff d(ZVdejde ddfd)ZWdejde deOeEefd eEddf d*ZXde dejddfd+ZYe dejde deOeEefd eEdeOeEeff d,ZZe dejde ddfd-Z[e dejde deOeEefd eEddf d.Z\ejde defd/Z^e e jdejdeOeEefd eEd0edeOeEeff d1Z`e e jdejddfd2Zae de ddfd3Zbe e jdejdeOeEefd eEd0eddf d4Zce e jdejd5eHedeEedeEfd0eddfd6Zed7e fd8Zfe d7e d9eEd:e d;eOeEefddf d<Zgy)>N) GeneratorIterator)AnyCallablecast no_type_check)init_from_local_shardsShard ShardedTensor)_mesh_resources) _FSDPState._get_module_fsdp_state_if_fully_sharded_module_has_fsdp_params_is_composable_module_handleclean_tensor_name FSDP_PREFIXFSDP_WRAPPED_MODULE)SimpleProfiler)!_cast_buffers_to_dtype_and_device_get_orig_buffer_dtypes _lazy_init%_reset_flat_param_grad_info_if_needed)FullStateDictConfigShardingStrategy StateDictType)DTensor)_replace_by_prefix)_ext_all_gather_dtensor_ext_chunk_dtensor_ext_chunk_tensor_ext_post_unflatten_transform"_ext_pre_load_state_dict_transform)_unshard_fsdp_state_params FLAT_PARAM fsdp_statereturnct|jtjk(xrt|xs |j SN)sharding_strategyrNO_SHARDr_use_orig_params)r's X/home/dcms/DCMS/lib/python3.12/site-packages/torch/distributed/fsdp/_state_dict_utils.py_should_unshard_paramsr/:s:$$(8(A(AA H J ' F:+F+F  module_namec|jtd}|jtd}|r|d}|jtjd}|S)N.)replacerrcheckpoint_wrapper_CHECKPOINT_PREFIX)r1s r._convert_to_wrapped_module_namer8AsZ%% ;K%%)<(=CK$ Q' %%&8&K&KRPK r0modulec#Kt||syt||jD]\}}t|}||}|||fywr*)rrparam_module_namesr8r9r' param_namer1fqns r._param_name_infosr?Ksc J /#1F$+ K6kB  j\*:{** +sA A c#Kt||jD]\}}t|}||}|||fywr*)rshared_param_module_namesr8r<s r._shared_param_name_infosrBXsV$2F$!+ K6kB  j\*:{** +s<> writeback rank0_onlyoffload_to_cpu with_gradsc||jvsJdt|||||||j|<|j|jy)z state_dict hooks cannot use the pure context call as the checkpoint flow requires to enter the context in the pre-hook but leave the context in the post-hook. This API enters the context of ``_unshard_fsdp_state_params``. z`Entering the ``_unshard_fsdp_state_params`` context but _unshard_params_ctx[module] is not None.)rCrDrErFN)_unshard_params_ctxr% __enter__)r9r'rCrDrErFs r._enter_unshard_params_ctxrJcsg 77 7  7.H% .J""6*""6*446r0cz|j|jddd|jj|y)zAA helper function to exit ``_unshard_fsdp_state_params`` context.N)rH__exit__popr9r's r._exit_unshard_params_ctxrOs5""6*33D$E""&&v.r0c|jjr|jjt|||jrt |j yy)zAPerforms the pre-state_dict tasks shared by all state_dict types.N)_device_handle is_available synchronizer_is_rootr _all_handlesrNs r._common_pre_state_dict_hookrVsN   --/!!--/z6"-j.E.EFr0c<t|syt||d||y)z Performs the pre-state_dict tasks shared by all state_dict types that require ``_unshard_fsdp_state_params()``. FULL_STATE_DICT and SHARDED_STATE_DICT use this hook. NF)rCrErD)r/rJ)r9r'rErDs r.#_common_unshard_pre_state_dict_hookrXs& "* -% r0 state_dictprefix param_hookct||tz||r t||st|r t |||S|j t jk(xr$tt|jj}|xr|jdk7}|r}|jsq|jD];}|jt j"dd}|j%||d=|j%|t&t |||St)||D]`\}} } ||}|r|j%|!||vs2Jd|d|j+d|d| d | d |jd ||||bt|r t ||t-j.d } g} g} |jD]}t1|}||}||vr|r|j%|-||}|jj2r#|j.| k7r|j5| ||<||j6vsz| j9|| j9||| rt;|s|j=n|j>j@du}|rqtC|| }tE| ||jFtI| | D]?\}}||}tJjMd ||jN|jQ||<A|S) z The post-state_dict flow that shared by all state_dict types that require ``_unshard_fsdp_state_params()``. FULL_STATE_DICT and SHARDED_STATE_DICT use this hook. rr4r3Nz FSDP assumes z2 is in the state_dict but the state_dict only has z . prefix=z, module_name=z , param_name=z rank=cpuz%FSDP is casting the dtype of %s to %s))rrrr/rO_state_dict_typerFULL_STATE_DICTrr_state_dict_configrDrankr- _buffer_namesr5r6r7rMr&r?keystorchdevicerrEto_ignored_buffer_namesappendr$_mixed_precision_enabled_for_buffersmixed_precision buffer_dtyperrcompute_deviceziploggerinfodtypeclone)r9r'rYrZr[rDno_fsdp_return clean_keyr>r=r1 cpu_devicebuffer_clean_fqnsbuffersbuffer#mixed_precision_enabled_for_buffers buffer_dtypes clean_fqns r.$_common_unshard_post_state_dict_hookr{s>z6{m#51%%j&=&=> 3'!44CC+Z-J-JKVV r0cpdtttfdtdtddffd }t||||S)a! Hook that runs after model.state_dict() is called before returning result to user. For FSDP, we may have to clone the tensors in state_dict as params go back to sharded version after _unshard_fsdp_state_params ends, and also remove the ``FSDP_WRAPPED_MODULE`` prefix. rYrZr>r(Nc V|}t|}|j|}t||dds0 ||jj ||<d||_yy#t $r>}tjd|djd|dt|Yd}~yd}~wwxYw)N_has_been_clonedFTz#Failed to clone() tensor with name z on rank z. This may mean that this state_dict entry could point to invalid memory regions after returning from state_dict() call if this parameter is managed by FSDP. Please check clone implementation of z . Error: ) r removeprefixrdetachrqr BaseExceptionwarningswarnrastr)rYrZr>rs clean_prefixer's r.r[z._full_post_state_dict_hook..param_hook<s  (0 **<8 z#(:EB ",S/"8"8":"@"@"B 337 30C!  9#i GXY)*-Ys1vh @ s.A!! B(*4B##B(dictrrr{r9r'rYrZr[s ` r._full_post_state_dict_hookr.sNcN  4 0 J  r0ct||t|r,tjd5t ||ddddt |st |||tzyy#1swY+xYw)NrJTrC)rr/rprofilerJrrr)r9r'rYrZs r._full_pre_load_state_dict_hookr[sl z6"j)  # #$? @ J %fjD I J * %:vv;-/HI & J Js A&&A/ct|r+tjd5t||dddyy#1swYyxYwNrO)r/rrrOr9r'rrs r._full_post_load_state_dict_hookrjsDj)  # #$> ? 9 $VZ 8 9 9* 9 9s 8Acvt||r!t||js tdt ||y)z Hook that runs before model.state_dict() is called. Right now, pre-state_dict hook is not supported by the PyTorch core. So this API is called from `_local_post_state_dict_hook()` to simulate the case. zN``local_state_dict`` can only be used when parameters are flatten and sharded.N)rruses_sharded_strategy RuntimeErrorrVrs r._local_pre_state_dict_hookrrs< V,z62HH     3r0cZt||t|t||s|St||sJdt||j}|j j }|j |jz}|j |jz }|dkDr8|d|j|}tj||g|jg}ng}t|||j} |jjr| j!} | ||t"<|S)z This hook create a ShardedTensor from the local flat_param and replace the state_dict[f"{prefix}{FLAT_PARAM}] with the ShardedTensor. No copy will happen. The underlying storage is the same. zShould have returned earlyrN) process_group)rrrr flat_param_unpadded_unsharded_sizenumelra_shard_numel_paddedviewr from_tensor_and_offsetsr rr`rEr]r&) r9r'rYrZr full_numel shard_offsetvalid_data_size local_shardssharded_tensors r._local_post_state_dict_hookrs0zfXk]#;VD J / *f -K/KK - F3>>J44:: load_tensorrrshardss r._local_pre_load_state_dict_hookrsz6"z6fXk]+CD H[M* .C *#J7  $ 7 S/K k= 1> 1  F3>>J  !! ! &&(:+I+IIO  % % 'F6{WWW{Qi&&   ) )A -$$&)9)9);; %j&6&6&8%9:%%0%6%6%8$9< ;%% a1O1O-PQK !JsOr0ct||r!t||js tdt ||t ||ddy)zz Hook that runs before model.state_dict() is called. Check ``_full_pre_load_state_dict_hook`` for the detail. zP``sharded_state_dict`` can only be used when parameters are flatten and sharded.Fr~N)rrrrrVrXrs r._sharded_pre_state_dict_hookrsR V,z62HH     3( r0cldtttfdtdtffd }t||||S)z The hook replaces the unflattened, unsharded parameter in the state_dict with a unflattened, sharded parameter (a ShardedTensor). rYrZr>c||}jjsRt|jjj j jj}n-t|jjj}jjr|j}|||<y)N)rra world_sizenum_devices_per_nodepgfsdp_extension)rra device_meshr) r` _use_dtensorr"rarrQ device_countr_fsdp_extensionr!r}rEr])rYrZr>paramrr's r.r[z1_sharded_post_state_dict_hook..param_hooks3,,99.__%00%/%>%>%K%K%M++)99 N0__&33)99 N  ( ( 7 7+//1N( 3r0rrs ` r._sharded_post_state_dict_hookr s?)tCH~)s)), 0 J  r0ct||r+tjd5t||dddyy#1swYyxYwr)rrrrOrs r."_sharded_post_load_state_dict_hookr4sF F+  # #$> ? 9 $VZ 8 9 9, 9 9s 9Act||t|st|||tzt ||syt ||}|j s tdtt|jj|jj}t||D]@\}}}t|s |t|}n||} |j|} |j$j&sBt)| |j*\} } t-| dks!Jdt-| d|j.d| j1j3} | j1d } t5j6| |j8z | z| z} t-| d k(r| d j:j=}t?j@t>jBjD5|jG|jH}ddd| |j3z }|d kDrEtKjL|d |g}n,tOjP| | jR|jH }tOjT| |j8z|jR|jH }t?j@t>jBjV5tYjZ|||j\ ddd|j_d d | ja| j1}|||<| jb|jdjfk7r%| jG|jdjf} tijj|jd}tm| ||j*}|jo|||}tq|||j*}|||<Ct?j@d 5ts||ddddy#t$rt j#d|dYwxYw#1swY8xYw#1swYWxYw#1swYyxYw)z The hook combines the unflattened, sharded parameters (ShardedTensor) to a new FlatParameter and shards the new FlatParameter to the local chunk. NzUload_sharded_state_dict can only be called when parameters are flattened and sharded.zDid not find param with FQN zD, skipping it. The weight will not be filled if you expect it to be.z&Expects 0 or 1 shard per rank but got z shards on rank r4rr)rpre)grouprJTr):rrrrrrrrrrmr_fqns_param_extensionsr?rMKeyErrorrnwarningr`rr$rrrasizermathceilrrflattenrrTypeH2Drfrlrrrdzerosrpempty ALLGATHERdistall_gather_into_tensorrnarrowreshaperer} device_typer rr getr#rJ)r9r'rYrZhandlefqn_to_param_extr>_fqn_from_global_rootrr param_numel dim_0_size chunk_size local_tensor num_paddingr root_meshexts r.!_sharded_pre_load_state_dict_hookr=sz6" * %:vv;-/HI J / J /F  ' ' )   F   # #V%6%6%H%HI'vz:D< Qj)&,Xk]3%#@ &,XcU#3  NN#78E,,99>z11ME6v;? v;-'7 7HK ? **,,,.KaJ *z'<'<<=  6{a%ay//779 #++N,?,?,C,CDN#/??:3L3L#MLN(<+=+=+?? ?#$55;7G#HL${{ekk*:S:S  [[Z222"((!00F  ''(;(;(E(EF ++L 0H0H ]]1a5==ejjlKF/5J+ ,||z66BBB!8!8!D!DE'55j6M6MNI2y*"<"<L ##C(4&s+< #z'A'A  0r'| jAj4} | j<} n | j<} t$j'd|tC| | j4| | jD| S#1swY_xYw)z _post_state_dict_hook() is called after the state_dict() of this FSDP module is executed. ``fsdp_state._state_dict_type`` is used to decide what postprocessing will be done. RWhen using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.Nz0FSDP finished processing state_dict(), prefix=%srz>FQN=%s: type=%s, shape=%s, local_shape=%s, dtype=%s, device=%s)#rr+rr,rrr contextlib nullcontextrr_rLOCAL_STATE_DICTrSHARDED_STATE_DICTrr^rTrnrosorteditems startswithrrdTensorshaper rrrerto_localtyperp) r9rYrZrr'context_post_state_dict_hook_fnprocessed_state_dictkeyr local_shaperers r._post_state_dict_hookrs@GJ##'7'@'@@4Z@   ((*   ) )+E  * *,G  , ,.K$  U7 8S8ST J F    FO!"6"<"<">? KC~~f%*VU\\*J$ll fm4"&K#002F&,Qi&6&6&<&< !'!1!1!8!80"(//"3"9"9K#]]F#]]F TLLLLL 2 K  s AHHct|}|jtjk(r!t |}t j dnt|tj}|5tjttjttjt i}||j"||g|i|dddy#1swYyxYw)z This is called before the core state dict saving logic of ``module``. ``fsdp_state._state_dict_type`` is used to decide what postprocessing will be done. rN)rr+rr,rrr_set_use_dtensorrrrr_rrrrrr^)r9rrr'r_pre_state_dict_hook_fns r._pre_state_dict_hookrs@GJ##'7'@'@@4Z@   $((*    ) )+D  * *,F  , ,.J#  = ; ;<         s +AC  Cct|ddr>|j}|tjk(r t dddd|j _yy)Nr}z&Found state_dict_type LOCAL_STATE_DICTz3DeviceMesh is not compatible with LOCAL_STATE_DICT.zKPlease set state_dict_type to SHARDED_STATE_DICT to get DTensor state_dict.T)rr^rrrr`r)r'state_dict_types r.rrsTz>40$55 m<< <8E]  :>J ) ) 61r0cnt|}|jtjk(r!t |}t j dnt|tj}t|||jrtj|5tjt tj"t$tj&t(i}|j*j-r|j*j/||j0||||dddy#1swYyxYw)z This is called before ``module._load_from_state_dict()``. ``fsdp_state._state_dict_type`` is used to decide what preprocessing will be done. rN)rr+rr,rrrrrrrrTrresetrr_rrrrrrQrRrSr^)r9rYrZrr'r_pre_load_state_dict_hook_fns r._pre_load_state_dict_hookr !s@GJ##'7'@'@@4Z@   $((*z6"    ) )+I  * *,K  , ,.O( $  $ $ 1 1 3  % % 1 1 3A$Z%@%@A J F     s B D++D4incompatible_keysct|}|jtjk(r!t |}t j dntj}|5tjttjttjti}||j ||ddd|d}|d}t#t%|D]}t'||||<t#t%|D]}t'||||<|j(rt+j,dyy#1swYxYw)Nrrrz&FSDP model load_state_dict profiling: )rr+rr,rrrrrrr_rrrrrr^rangerrrTrdump_and_reset) r9r rr'r_post_load_state_dict_hook_fn missing_keysunexpected_keysis r._post_load_state_dict_hookrLsC@GJ##'7'@'@@4Z@   ((*  W  ) )+J  * *,L  , ,.P) % C%j&A&AB6:V W%Q'L'*O 3|$ %=+LO< Q=3' (C.q/ABC%%&NO) W Ws AD>>EstateczdtifdtifdtddifdtiffD]\}}}t ||||y)zR Registers pre-save, post-save, pre-load, and post-load state dict hooks. register_state_dict_pre_hook_register_state_dict_hook"_register_load_state_dict_pre_hook with_moduleT"register_load_state_dict_post_hookN)rrr r_register_state_dict_hooks_base)rhook_registration_fn_strhookhook_registration_fn_kwargss r._register_all_state_dict_hooksr!usj ()=rB $&;R@ 0 % D ! ./I2N H  C $(C ( +T3N   r0hook_registration_fn_namerr ct|st|||fi|y|j}|rt|j||fi|yy)z2Registers ``hook`` using ``hook_registration_fn``.N)rr_handle_fully_sharded_module)rr"rr rs r.rrsX % 101$V:UV  LGF002K L 3  r0)FFFF)hrloggingrrcollections.abcrrtypingrrrrrdtorch.distributed distributedr;torch.distributed.algorithms._checkpoint.checkpoint_wrapper algorithms _checkpointr6torch.nnnntorch.nn.functional functionalr'torch.distributed._shard.sharded_tensorr r r torch.distributed.device_meshr $torch.distributed.fsdp._common_utilsr rrrrrrr#torch.distributed.fsdp._debug_utilsr%torch.distributed.fsdp._runtime_utilsrrrrtorch.distributed.fsdp.apirrrtorch.distributed.tensorrtorch.distributed.utilsr_fsdp_extensionsr r!r"r#r$_unshard_param_utilsr%r& getLogger__name__rnboolr/rr8Moduletupler?rBrJrOrVrXrr{rrrrrrrrrrrrcontextmanagerrno_gradrrrr listrr!rrr0r.rDsE /55 XX :   ?  -6I   8 $zd + II +#- + eCcM"# ++ II+ eCcM"#+  7 II777 7  7  7 778/RYY/J/4// G II G G  G II   ,f IIffS#Xf  f  f  #s(^ ffR II  2) II))S#X)  )  #s(^ ))X J II J JS#X J  J  J9 II9#-9 944 II4  4,. II..S#X.  .  #s(^ ..b II #-   -" II-"-"S#X-"  -"  -"` II  :# II##S#X#  #  #s(^ ##L9 II9#-9 99aF IIaFaFS#XaF  aF  aFaFH 6:6)66: II: S#X:  :   :  #s(^ : : z  II      F > > > >& II& S#X&  &   &  & & R$P II$PT#YS 12$P $P $P$PN * & " "&c3h   r0