Vh` UddlZddlZddlZddlZddlZddlmZddlmZddl m Z m Z m Z m Z ddlZddlmZddlmZddlmZmZddlmZdd lmZdd lmZmZmZmZmZm Z m!Z!m"Z"dd l#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*dd l+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1dd l2m3Z3ddl4m5Z5ddl6m7Z7ejpe9Z:ejveGdde%Z?Gdde?Z@ d-deAeBe fdedeCde$fdZDdeEe$deEe$fdZFdeAeBe fd eCde'fd!ZG d-deEe'd"eCdeHeEe'effd#ZIdedefd$ZJd%ed&edeCfd'ZKd(ejd)edeCfd*ZMd+eEe'dedeCfd,ZNy).N)ChainMap)reduce)AnycastOptionalUnion)narrow_tensor_by_index)dedup_save_plans)FLATTEN_MAPPINGflatten_state_dict)_flatten_sharded_tensors) set_element)BytesStorageMetadataChunkStorageMetadataMetadata MetadataIndexSTATE_DICT_TYPE STORAGE_TYPES StorageMetaTensorStorageMetadata)LoadPlan LoadPlannerReadItemSavePlan SavePlanner WriteItem WriteItemType)_compare_save_plans"_create_default_metadata_only_plan_create_read_items_create_write_items_init_state_dict_merge_delta_local_plans)find_state_dict_object)DTensor)_versionlogger)DefaultSavePlannerDefaultLoadPlannercreate_default_local_load_plancreate_default_global_load_plancreate_default_local_save_plancreate_default_global_save_planc veZdZUeed< ddededeedededdf d Z dd ed ee d eddfd Z de fdZ de e dee e effdZde e dee e e e effdZde e dee e effdZde de fdZde de fdZdedeej0ej4ffdZdedefdZdedefdZy)r)mappingsNr flatten_sharded_tensorsdedup_replicated_tensorsdedup_save_to_lowest_rankenable_plan_cachingreturnc||_||_i|_||_|tj d|j j|_||_ y)NzDefaultSavePlanner's `dedup_replicated_tensors` argument is being deprecated, and no longer has any effect. Please remove this argument from your call.) r r1r0r3r(warning __class____name___cached_plans_key_enable_plan_caching)selfr r1r2r3r4s \/home/dcms/DCMS/lib/python3.12/site-packages/torch/distributed/checkpoint/default_planner.py__init__zDefaultSavePlanner.__init__HsV#5'>$ )B& # / NN"  '+nn&=&=$7! state_dict storage_metais_coordinatorc|jrt|\}|_|jr t|}||_||_yN)r r0r1r r@rB)r<r@rArBs r=set_up_plannerz!DefaultSavePlanner.set_up_planner]s@  " "(::(F %J  ' '1*=J$,r?ct|j|j}|jr!t j ||j }||_|jr|jtjvrIt|tj|jr"tjdtgdS|tj|j<|jS)N planner_datazINo change in the local plan. Skipping sending the plan to the coordinatorFusable)r-r@rBr dataclassesreplacer0planr;r:r_cached_save_planrr(infor)r<rMs r=create_local_planz$DefaultSavePlanner.create_local_planjs-doot?R?RS  " "&&t$--HD  $ $&&+*G*GG'+778N8NO _ 511HL --d.D.DEyyr? all_plansc$t||j}t|\}}|jrA|Dcgc]}|j}}t t |}tj||}t||s td||fScc}w)NrGzFailed to validate global plan) r r3r.r rHdictrrKrL_validate_global_plan ValueError)r<rQ global_planmetadatapplanner_data_dictmerged_mappingss r=_create_global_planz&DefaultSavePlanner._create_global_plans%Y0N0NO ? J X  " " :E EA E  E"8->#?@O"**8/RH$[(;=> >H$$!FsB c&g}|jtjvrftj|tj|j<|j |\}}|tj |j<|||fSttj|j|}tj|tj|j<|j |\}}|j|j vrittj |j|D]?\}}t||r|jtgd/|j|A|tj |j<|||fS)zw Create global plan with caching. Returns a tuple of global_plan_delta, global_plan, metadata. FrI) r:r_cached_all_planscopydeepcopyr[_cached_global_planr#ziprappendr)r<rQglobal_plan_deltarVrW merged_plans cached_plannew_plans r= _create_global_plan_with_cachingz3DefaultSavePlanner._create_global_plan_with_cachingsv-/  ! !)F)F FDHMMEK ) )$*@*@ A%)$<$(>?!+x77r?cg}|jr|j|\}}}n|j|\}}|}||_||_||jfSrD)r;rgr[rVrW)r<rQrcrVrWs r=create_global_planz%DefaultSavePlanner.create_global_plansl-/  $ $55i@  !%)$<$rrrErrPlisttuplerr[rgrirnrprrrTensorrrrwrrrsrur?r=r)r)Es$((,37*/$) 8 8"&8#+4. 8 $( 8 " 8 80/3$ -# -{+ - -  -8.%h% tH~x' (%*,8h,8 tH~tH~x7 8,8\0h0 tH~x' (0,(xH9y9U5<<;S5T9>=>S>9cr?r)c FeZdZUdZeed<eed< ddedededdfd Z dd ed e e d eddfd Z de fdZ dee dee fdZde de fdZdedej&ddfdZdefdZdedej.ddfdZdedej.fdZdedej.fdZy)r*ak DefaultLoadPlanner that adds multiple features on top of LoadPlanner. In particular it adds the following: flatten_state_dict: Handle state_dict with nested dicts flatten_sharded_tensors: For FSDP in 2D parallel mode allow_partial_load: If False, will raise a runtime error if a key is present in state_dict, but not in the checkpoint. original_state_dictr0r r1allow_partial_loadr5NcJ||_||_i|_i|_||_yrD)r r1rr0r)r<r r1rs r=r>zDefaultLoadPlanner.__init__ s+ #5'>$#%  "4r?r@rWrBct|||_|jr t|}|jrt |\}|_||_||_||_yrD) r"rr1r r r0r@rWrB)r<r@rWrBs r=rEz!DefaultLoadPlanner.set_up_plannersW $#-  ' '1*=J  " "(::(F %J $  ,r?c|jJ|jrt|jj }t|jj j }||z }|r[dt _t|j\}}t|j }||zr||c|_|_ dt _t|j|j|j S)N2_3) rWr setr@keysstate_dict_metadatar'_derived_versionrr0r+r)r< current_keys load_keys missing_keysold_state_dict old_mappingsold_keyss r=rPz$DefaultLoadPlanner.create_local_plan+s}}(((  " "$t3356LDMM==BBDEI$|3L,1)/A,,0, ~2245l*5C\2DOT]-1)- OOT]]0G0G,G  r?rVct|SrD)r,)r<rVs r=riz%DefaultLoadPlanner.create_global_planRs .{;;r?rfc|SrDr)r<rfs r=rpzDefaultLoadPlanner.finish_planUsr? read_itemvaluec(|jrNt|j|j|jj t j|dyt j|d|j|jj <y)NF) weights_only) r rrr0 dest_indexfqnrloadr@)r<rrs r= load_byteszDefaultLoadPlanner.load_bytesXsl  " " (( i22667 5u5  9> E9DOOI0044 5r?c\|j|j}|j||SrD) lookup_tensorrtransform_tensorr<rtensors r=resolve_tensorz!DefaultLoadPlanner.resolve_tensords+##I$8$89$$Y77r?rcyrDrrs r= commit_tensorz DefaultLoadPlanner.commit_tensorhs r?rtc.t|j|Sryrzr{s r=rz DefaultLoadPlanner.lookup_tensorkr|r?cDt||j|jSry)r dest_offsetslengthsrs r=rz#DefaultLoadPlanner.transform_tensoros%fi.D.DiFWFWXXr?)TTFr)r9rr__doc__rrr rr>rrrErrPrrirprrrrrrrrrrrrr?r=r*r*s7)($((,#( 5  5"& 5! 5  5(,$ -#-8$- -  -&% 8% N=>U\\>Y(YELLYr?r*c `eZdZdZd fd ZdededefdZ d de de ed eddffd Z xZ S) _EmptyStateDictLoadPlannera Extension of DefaultLoadPlanner, which rebuilds state_dict from the saved metadata. Useful for loading in state_dict without first initializing a model, such as when converting a DCP checkpoint into a Torch save file. . N.B. `state_dict` must be an empty dictionary when used with this LoadPlanner .. warning:: Because the entire state dict is initialized, It's recommended to only utilize this LoadPlanner on a single rank or process to avoid OOM. Nc2||_t||i|yrD)rsuperr>)r<rargskwargsr8s r=r>z#_EmptyStateDictLoadPlanner.__init__s  $)&)r?keyrWr5c 4jy|jvr g}|jj|}|D]D}|r/|jdj |dt |g4|j|Ft fd|Dryy)NT.c3:K|]}|jvywrD)r).0unflattened_keyr<s r= zA_EmptyStateDictLoadPlanner._should_include_key..sT$))+TsF)rrHgetrbjoinstrany)r<rrWunflattened_keysrHrs` r=_should_include_keyz._EmptyStateDictLoadPlanner._should_include_keys 99  $))  &(,,005 + 9O ''HH.r2C4HIJ !''8 9 TCST Tr?r@rBc|rJ|J|jjD]\}}|j||st|tr5t j |j|jj}||jvrt||j|||||<t|5|||y)N)dtype)ritemsr isinstancerremptysize propertiesrrHrrrE)r<r@rWrBkvr8s r=rEz)_EmptyStateDictLoadPlanner.set_up_planners ~###00668 "DAq++Ax8!23KKall.@.@AH)))J(=(=a(@!D ! 1  " z8^Dr?rDr) r9rrrr>rrrrrrrE __classcell__)r8s@r=rrtsj *sh44(,$ E#E8$E E  EEr?rr@rWstrictr5c g} |jD]\}}||jvr|rtd|d&|j|}t|trVt |ddI|j |j k7r,td|j d|j d|t|tr,|jj|t|||z }|t|||z }t|S)Nz&Missing key in checkpoint state_dict: rrzSize mismatch between saved z and current: z for ) rr RuntimeErrorrrgetattrrrUr% device_meshget_coordinater r)r@rWrrequestsrobjmds r=r+r+sH$$&9S h22 2"%KC5PQ#RSS  ) )# . r0 1VT*6388:%.rwwi~chhj\QVWZV[\  c7 #--/;.sB<< *3C8 8H/92 H r?rQc|S)z Create global load plan used by DefaultLoadPlanner. The default load behavior involved no global coordination and this function currently doesn't change the local plans. r)rQs r=r,r,s  r?rBcg}|jD]O\}}t|tr+|jj 1|t ||z }A|t ||z }Qt |S)a Create the ``SavePlan`` used by DefaultSavePlanner. On non-coordinator ranks, this function ignores tensors and non-tensor objects, only producing writes for ShardedTensor objects. On the coordinator rank, produce writes for all values. )rrr%rrr!r)r@rBrrrs r=r-r-swH$$& 6S c7 #--/;/S99 +C5 5H 6 H r?rewrite_index_hintsc .i}g}|D]}g}|jD]}|jtjk(s|jj |vsJ|jtj k(r3t||jj <|j||jJtt|j|jj t|jj|jjg}|}|rKtj |jt#|j$} tj || }|j||jj&Jd|jj d|j$j|jj&|jtj |||t)|fS)a6 Create the global plan and metadata used by DefaultSavePlanner. Metadata is produced by concatenating the metadata of all ``WriteItem`` from the supplied plans. The only global planning change is to update index hints in all ``MetadataIndex`` objects if ``rewrite_index_hints`` is True. )rrchunks)rtzZ Cannot create MD for tensor without bounds. FQN: z )r)rr~rSHARDrtrrrrb tensor_datarr setdefaultrrrKrLlenrchunkr) rQrr new_plansrM new_itemsitem tensor_mdnew_item new_indexs r=r.r.s$&BI#E JJ @D99 3 33zz~~R///yyM111%9%;4::>>"  &''333 )MM -'+'7'7'B'B!%!1!1!6!6#%   & + 3 3 #i.>.>*?!I +224yIH  *''--9@**..)*<9  ''(8(8(>(>?A @B ,,TCDG#EH x| $$r?c:t|}t|g\}}|S)zTReturn the ``Metadata`` if DefaultSavePlanner was used to checkpoint ``state_dict``.)rr.)r@rM_rs r=_create_default_local_metadatar;s! -j 9D +TF 3EAr Ir?box0box1ct|j}t|D]d}|j||j||j|zk\ry|j||j||j|zk\sdyy)z9Check if two boxes overlap. Tuples are (offset, lengths).FT)roffsetsrangesizes)rrndimsis r=_check_box_overlaprBsz  E 5\ <<?dll1o 1 = = <<?dll1o 1 = =  r?outer_box_size inner_boxctt|D]Q}|j|dkry|j|dkry|j||j|z||kDsQyy)NrFT)rrrr)rrrs r=_check_box_boundsrRsu3~& '   Q ! # ??1  !   Q )//!"4 4~a7H H  r?rVc d}|jjD]5\}}t|trt |j dk(r1d}t |jD]\}}t|j |s$tjd||j |d}|ttj|jdz }|j|dzdD])}t||stjd|||d}+ttj|j d} || k7stjd|| |d}8|S)NTrz~ key:%s has out of bounds chunk: tensor-size:%s chunk: %s Fr&z$key:%s has overlapping chunks: %s %szq key:%s invalid fill tensor-volume: %s chunks-volume: %s )rrrrrr enumeraterrr(r7roperatormulrr) rVrWall_goodrr chunks_volume chunk_idxchunk0chunk1 tensor_volumes r=rTrT`sNH2288:) U e1 2  uzz?a   !*5<VV %H  %! %0x||UZZ; M ) NN HS)V Or?)T)Or^rKrloggingr collectionsr functoolsrtypingrrrrrtorch.distributed._shard._utilsr .torch.distributed.checkpoint._dedup_save_plansr )torch.distributed.checkpoint._nested_dictr r 2torch.distributed.checkpoint._sharded_tensor_utilsr &torch.distributed.checkpoint._traverser%torch.distributed.checkpoint.metadatarrrrrrrr$torch.distributed.checkpoint.plannerrrrrrrr,torch.distributed.checkpoint.planner_helpersrrr r!r"r#"torch.distributed.checkpoint.utilsr$torch.distributed.tensorr%r' getLoggerr9r(Loggerr__all__r)r*rrSrrr+rr,r-rr.rrSizerrTrr?r=rs   -- BKX>   F,+**844 vvrsYsYl>E!3>EDDH'S#X'*2'<@' 'T H~  (^ S#X04 :!%2%H~2%2% 4>8 #$2%j8 1 9M RV  JJ +?   -tH~--d-r?