
    VhA                         U d dl Z d dlZd dlZd dlmZ d dlZd dlmZmZ d dl	m
Z
mZmZmZ d dlmZmZmZmZmZmZ d dlmZ d dlmZ dd	gZd
Zeed<   dZdZ G d de      Z G d d	e      Z y)    N)Optional)FsspecReaderFsspecWriter)BytesStorageMetadataMetadataSTORAGE_TYPESStorageMeta)LoadPlanLoadPlannerReadItemSavePlanSavePlanner	WriteItem)WriteResult)Future_HuggingFaceStorageWriter_HuggingFaceStorageReaderzmodel.safetensors.index.json_metadata_fnzmodel-{cpt_idx}-of-{num_shards}z.safetensorsc            	           e Zd ZdZ	 ddedeeef   dee   ddf fdZde	de	f fd	Z
d
ee	   dee	   fdZde	dedeee      f fdZdedeee      ddfdZdeeef   dee   deeee   f   fdZdededefdZedefd       Z xZS )r   z
    A writer that writes to a huggingface repository in the huggingface format.
    Uses in Fsspec back-end to communicate with the huggingface hub.
    Npathfqn_to_index_mappingtokenreturnc                     ddl m} |j                  t        j                         vr t        j
                  |j                  |       t        |   ||       || _        y)a  
        Initialize the huggingface writer pointing to path.

        Args:
            path: hf directory where the checkpoint will be written to. Should begin with hf://.
            token: The token to use to authenticate with huggingface hub.
            fqn_to_index_mapping: A mapping from tensor FQN to the index of the file that the tensor should be written to.
                              Indices are from 1 to N, where N is the number of files.

        r   HfFileSystemr   r   N)	huggingface_hubr   protocolfsspecavailable_protocolsregister_implementationsuper__init___fqn_to_index_mapping)selfr   r   r   r   	__class__s        X/home/dcms/DCMS/lib/python3.12/site-packages/torch/distributed/checkpoint/_hf_storage.pyr$   z"_HuggingFaceStorageWriter.__init__*   sO      	1  (B(B(DD**<+@+@,Od%05I"    planc                 d    t         |   |       t        j                  || j                        S )N)storage_data)r#   prepare_local_plandataclassesreplacer%   )r&   r*   r'   s     r(   r-   z,_HuggingFaceStorageWriter.prepare_local_planB   s)    "4(""4d6P6PQQr)   plansc                 0    t        |      dk(  sJ d       |S )N   z.distributed checkpointing is not yet supported)len)r&   r0   s     r(   prepare_global_planz-_HuggingFaceStorageWriter.prepare_global_planF   s    5zQP PPr)   plannerc                    |j                   }| j                  ||j                        }t        |j	                               }t        j                         }|j                         D ]O  \  }}| j                  ||      }	|j                  | j                  j                  | j                  |	      |	|f       Q t        
| 5  ||d      S )NT)safe_tensors)r,   _split_by_storage_planitemsmaxkeysqueueQueue_gen_file_nameputfsconcat_pathr   r#   _write_data)r&   r*   r5   storage_planbucketshighest_index
file_queue
file_indexwrite_items	file_namer'   s             r(   
write_dataz$_HuggingFaceStorageWriter.write_dataJ   s     (,'8'8--lDJJGGLLN+"'++-
'.}} 	#J++JFINN$$TYY	:I{S	 w"7JT"JJr)   metadataresultsc                 "   i }i }d}|D ]z  }|j                  |D ci c]-  }|j                  j                  |j                  j                  / c}       |t        |D cg c]  }|j                  j                   c}      z  }| d|i|d<   ||d<   | j                  j                  | j                  t               }| j                  j                  |d      5 }	t        j                  ||	d       d d d        y c c}w c c}w # 1 sw Y   y xY w)Nr   
total_sizerK   
weight_mapw   )indent)updateindexfqnr,   relative_pathsumlengthr@   rA   r   r   create_streamjsondump)
r&   rK   rL   metadata_to_write
storage_mdrN   wr_listwrmetadata_pathmetadata_files
             r(   finishz _HuggingFaceStorageWriter.finish^   s   

 	JGGNOr<<<O #H"r55HIIJ		J
 *6z(B*%*4,'++DII,IWW""=#6 	B-II'qA	B 	B PH
	B 	Bs   2C;
D DDrC   r9   c                     i }|D ]<  }|j                   j                  }||   }||vr|g||<   )||   j                  |       > |S N)rT   rU   append)r&   rC   r9   rD   itemkeyidxs          r(   r8   z0_HuggingFaceStorageWriter._split_by_storage_plann   s[      	*D**..Cs#C'! $v##D)	* r)   rT   largest_indexc                 ~    t         j                  | j                  d      | j                  d            t        z   S )N   )cpt_idx
num_shards)	FILE_NAMEformatzfillSUFFIX)r&   rT   ri   s      r(   r>   z(_HuggingFaceStorageWriter._gen_file_name}   sD     '((+=/8P8PQR8S   	
r)   c                     t         S rd   )r   )r&   s    r(   r`   z'_HuggingFaceStorageWriter.metadata_path   s    r)   rd   )__name__
__module____qualname____doc__strdictintr   r$   r   r-   listr4   r   r   r   rJ   r   rb   r   r8   r>   propertyr`   __classcell__r'   s   @r(   r   r   $   s=     $	JJ #38nJ }	J
 
J0Rx RH Rh DN KK K 
[!	"	K(Bx B$tK7H2I Bd B  cN37	?	c4	?"	#
C 
 
 
 s  r)   c                   ^     e Zd ZdZddedee   ddf fdZdedede	d   fd	Z
defd
Z xZS )r   z
    A reader that reads from a huggingface repository in the huggingface format.
    Uses in Fsspec back-end to communicate with the huggingface hub.
    Nr   r   r   c                     ddl m} |j                  t        j                         vr t        j
                  |j                  |       t        |   ||       i | _        y)z
        Initialize the huggingface reader pointing to path.

        Args:
            path: hf directory where the checkpoint will be read from. Should begin with hf://.
            token: The token to use to authenticate with huggingface hub.
        r   r   r   N)	r   r   r   r    r!   r"   r#   r$   r,   )r&   r   r   r   r'   s       r(   r$   z"_HuggingFaceStorageReader.__init__   sO     	1  (B(B(DD**<+@+@,Od%0,.r)   r*   r5   c                    ddl m} i }|j                  D ]F  }| j                  |j                  j
                     }|j                  |g       j                  |       H |j                         D ]  \  }}| j                  j                  | j                  |      }| j                  j                  |d      5 }	 ||	j                               }
|D ]|  }|
|j                  j
                     }|j                  |      j                         }|j!                  |j#                                |j%                  |       |j'                  ||       ~ 	 d d d         t)               }|j+                  d        |S # 1 sw Y   xY w)Nr   )loadrb)safetensors.torchr   r9   r,   storage_indexrU   
setdefaultre   r@   rA   r   rY   read
dest_indexresolve_tensordetachresize_sizecopy_commit_tensorr   
set_result)r&   r*   r5   r   per_file	read_itemrI   reqsnew_pathstreamloaded_tensorsreqtensortarget_tensorfuts                  r(   	read_dataz#_HuggingFaceStorageReader.read_data   sT   *.0 	AI)))*A*A*E*EFI	2.55i@	A  (~~/ 
	>OItww**499i@H&&x6 >&!%fkkm!4 >C+CNN,>,>?F$+$:$:3$?$F$F$HM!))&++-8!''/))#}=>> >
	> ht
> >s   6BE66F 	c                    | j                   j                  | j                  t              }| j                   j	                  |d      5 }t        j                  |      }d d d        i }d   j                         D ]  }t               ||<    t        ||d         }t        |dd       t               |_        | j                  |j                  _        |S # 1 sw Y   yxY w)NrrO   )state_dict_metadatar,   storage_meta)r@   rA   r   r   rY   rZ   r   r;   r   r   getattrr	   r   load_id)r&   r   ra   rK   r   rg   s         r(   read_metadataz'_HuggingFaceStorageReader.read_metadata   s    ww""499l;WW""4- 	0yy/H	0 9;L)..0 	>C';'=$	> 3(<BX
 8^T2:$/MH!(,%	0 	0s   CCrd   )rs   rt   ru   rv   rw   r   r$   r
   r   r   r   r   r   r|   r}   s   @r(   r   r      sP    
/S /# /$ /h   2x r)   )!r.   rZ   r<   typingr   r    /torch.distributed.checkpoint._fsspec_filesystemr   r   %torch.distributed.checkpoint.metadatar   r   r   r	   $torch.distributed.checkpoint.plannerr
   r   r   r   r   r   $torch.distributed.checkpoint.storager   torch.futuresr   __all__r   rw   __annotations__rn   rq   r   r    r)   r(   <module>r      so         V   =   '(C
D2c 2-		c cL> >r)   