Vhf-ddlmZddlZddlmZmZmZmZddlm Z m Z m Z ejjZ GddejZy))AnyN)ChunkStorageMetadata MetadataIndexTensorPropertiesTensorStorageMetadata)TensorWriteData WriteItem WriteItemTypeceZdZUdZddgZeejed<e ed<e deejdee e dfddfdZ ed&d Ze d Ze d Ze d Ze dZe dZe dZe dZedej0j2fdZedefdZdefdZd'deddfdZdeejfdZdeej>fdZ deej>fdZ!edee"fdZ#de fdZ$de%de&dee'fdZ(dee"fdZ)d e*dejfd!Z+de fd"Z,d#Z-de%fd$Z.de%fd%Z/y )(LocalShardsWrapperz A wrapper class to hold local shards of a DTensor. This class is used largely for checkpointing purposes and implicity subtypes the _Checkpointable protocol. _local_shards _storage_meta local_shards local_offsets.returnc tdkDsJtt|k(sJtfdddDsJtdj}tdkDr(ddD] }|dxx|jdz cc<"t j d}t j|}t|Dcgc]3\}}tt j||j5}}}t jj|t j|} | _ t|||| _| Scc}}w)Nrc3VK|] }|jdjk("yw)rN)device).0tensorrs X/home/dcms/DCMS/lib/python3.12/site-packages/torch/distributed/tensor/_shards_wrapper.py z-LocalShardsWrapper.__new__...s) 8>FMM\!_33 3 s&)offsetssizes) propertiessizechunks)lenalllistrrcreate_from_tensortorchSizeziprTensor_make_wrapper_subclassr rr) clsrrcat_tensor_shapeshardwrapper_properties wrapper_shapeoffset chunks_metars ` r__new__zLocalShardsWrapper.__new__(si< 1$$$< C $6666 BNqrBR     Q 4 4 67 | q %ab) 7 #uzz|A6# 7.@@aQ #34 "%\=!A  v ! 6*jjl    LL / /  JJ' ( '/)  ' s8ENc|xsi}tjjjj|j tjjj j|jtjj|jtjj|jtjj|jtjj|j tj"j|j$i}||vr ||||St'|d)N) is not supported for LocalShardsWrapper!)r$ops_c10d_functionalall_gather_into_tensordefaulthandle_all_gather_into_tensor wait_tensorhandle_wait_tensoraten_to_copyhandle_to_copyview handle_viewequal handle_equaldetach handle_detachclone handle_cloneNotImplementedError)r)functypesargskwargs dispatchers r__torch_dispatch__z%LocalShardsWrapper.__torch_dispatch__Ps2 II & & = = E EsGhGh II & & 2 2 : :Cr4r5r6r7)rIrJrPt cat_tensors rr8z0LocalShardsWrapper.handle_all_gather_into_tensoris1g!!#A&q)YY!%a!5!5!7 8AQVVBZ 8a $r3- yy))@@HH  ab %+   9sB)cZtjjj|dSNr)r$r4r5r9)rIrJs rr:z%LocalShardsWrapper.handle_wait_tensorus#yy))55d1g>>rMc |djDcgc](}tjj|g|ddi|*}}t ||dj Scc}wNrr)rr;r<r7r rrIrJr+res_shards_lists rr=z!LocalShardsWrapper.handle_to_copy{sn a--/  MM ! !% =$qr( =f =  "/473H3H3JKK  s-A"c|djDcgc]&}tjj||dfi|(}}t ||dj Scc}wrX)rr;r>r7r rrYs rr?zLocalShardsWrapper.handle_viewsia--/  II  eT!W 7 7  "/473H3H3JKK  s+A c6|d|d}}t|jt|jk7rytdt|j|jDsy|j |j k(syy)z| LocalShardsWrapper equal impl also checks for equality of storage metadata and the order of shards rrFc3bK|]'\}}tjj||)ywN)r;r@r7)rxys rrz2LocalShardsWrapper.handle_equal..s* )-ADJJ  q! $ s-/T)r rr!r&storage_metadata)rIrJabs rrAzLocalShardsWrapper.handle_equalsAwQ1 q~~ C(8$9 9 14Q^^5Eq~~GW1X  !!#q'9'9';;rMc|d}|jDcgc]!}tjj|#}}||_d|j j _|Scc}w)NrF)rr;rBr7r rr requires_grad)rIrJself_lsr+deatched_local_shardss rrCz LocalShardsWrapper.handle_detachshq'4;4H4H4J! +0DKK   &! ! !69>((6 ! s&A$c|d}|jdd}|r!|tjk7rt|d|jDcgc]}|j |}}t ||jScc}w)Nr memory_formatr3)ri)getr$preserve_formatrFr rDr r)rIrJrfdesired_memory_formatr+cloned_local_shardss rrEzLocalShardsWrapper.handle_clonesq' & ?D A %:e>S>S%S%())RS  !..  KK&;K <  ""5w7L7L7NOO  s A>c4|jdjSrV)r rselfs rrzLocalShardsWrapper.devices!!!$+++rMc4|jdjSrV)r is_metaros rrrzLocalShardsWrapper.is_metas!!!$,,,rMcB|jjjSr^)rr pin_memoryros r is_pinnedzLocalShardsWrapper.is_pinneds!!,,777rMrec||jj_|jDcgc]}|j |c}|Scc}wr^)rrrer requires_grad_)rprer+s rrwz!LocalShardsWrapper.requires_grad_s@6C%%3:>:L:LM  m ,M  NsAc|jS)z Returns a list of :class:`torch.Tensor' corresponding to the local shards for this rank. Returns an empty list if the current rank does not host any shards for this Tensor. )r ros rrzLocalShardsWrapper.local_shardss !!!rMch|jjDcgc]}|jc}Scc}w)z Returns a list of :class:`torch.Size' corresponding to the local sizes for the shards on this rank. Returns an empty list if the current rank does not host any shards for this Tensor. )rrrrpchunks rrQzLocalShardsWrapper.local_sizess( *.););)B)BC CCC/ch|jjDcgc]}|jc}Scc}w)z Returns a list of :class:`torch.Size' corresponding to the local offsets for the shards on this rank. Returns an empty list if the current rank does not host any shards for this Tensor. )rrrrzs rrz LocalShardsWrapper.local_offsetss( ,0+=+=+D+DE% EEEr|c.|jjS)z Returns a :class:`List[ChunkStorageMetadata]` object corresponding to the metadata for each tensor shard rrros r local_chunkszLocalShardsWrapper.local_chunkss !!(((rMc|jS)z Returns a :class:`TensorStorageMetadata` object corresponding to the metadata for the local tensor on current rank )rros rraz#LocalShardsWrapper.storage_metadatas !!!rMfqnobjectcrt|j|jDcgc]\}}tt ||j t jtt|j |j|jj|jc}}Scc}}w)z} For compatibility with DCP, we support creation of WriteItems such that they can be saved properly. r)r{rr)indextype tensor_data)r&rrr rrr SHARDrrrrrr)rprrrrs r__create_write_items__z)LocalShardsWrapper.__create_write_items__s(#&d&7&7&94;L;L"M  #C8"((+. &$ll $11<<     sBB3c.|jjS)z~ For compatibility with DCP, we support creation of chunk lists such that they can be saved properly. rros r__create_chunk_list__z(LocalShardsWrapper.__create_chunk_list__s !!(((rMrc|jut|j|jkDrS|jj|jj |j k(r|j|jS|j Ot|j|jjD]"\}}|j |j k(s |cStd|j d|jd)z For compatibility with DCP, we support finding shard based on index Return a 'torch.Tensor' shard based on 'MetadataIndex'. zCould not find shard at 'z ' for FQN: '') rr r rrrr.r& ValueErrorr)rprr+r{s r__get_tensor_shard__z'LocalShardsWrapper.__get_tensor_shard__s ;; "D&&'%++5&&--ekk:BBellR))%++66 << # #D$6$68J8J8Q8Q R ! u==ELL0 L !' ~\%))A N  rMc|d}|jD]&}||j|jzz }(|SrV)rnelement element_size)rp object_sizer+s r_get_tensor_size_bytesz)LocalShardsWrapper._get_tensor_size_bytes,sF &&( CE 5>>+e.@.@.BB BK CrMct|Sr^)idros r__hash__zLocalShardsWrapper.__hash__3s $xrMc:d|jd|jSNzLocalShardsWrapper: r rros r__repr__zLocalShardsWrapper.__repr__8#$T%7%7$8$:L:L9MNNrMc:d|jd|jSrrros r__str__zLocalShardsWrapper.__str__;rrM)N)T)0__name__ __module__ __qualname____doc__ __slots__r"r$r'__annotations__r staticmethodtupleintr1 classmethodrLr8r:r=r?rArCrEproperty_Crboolrrrurwrr%rQrrrrastrrr rrrrrrrrrrMrr r sU !/2I %%(($ -$>B5c?>S$ $$N,  ??LLLL  P P,,,---848DrsG   IINN _O_OrM