
    Vh-                         d dl Z d dlZd dlmZmZ d dlmZ d dlmZ d dl	m
Z
mZmZ d dlmZ d dlmZ d dlmZmZmZ erd d	lmZ d
dgZ G d de      Z G d d
e      Z G d de      Zy)    N)	GeneratorSequence)contextmanager)Path)OptionalTYPE_CHECKINGUnion)	url_to_fs)StreamTransformExtension)FileSystemBaseFileSystemReaderFileSystemWriter)AbstractFileSystemFsspecWriterFsspecReaderc            
          e Zd ZddZedeeej                  f   dede	e
j                  ddf   fd       Zdeeej                  f   dedeeej                  f   fdZdeeej                  f   deeej                  f   fd	Zdeeej                  f   d
eeej                  f   ddfdZdeeej                  f   ddfdZedeeej                  f   defd       Zdeeej                  f   defdZdeeej                  f   ddfdZy)
FileSystemreturnNc                     d | _         y N)fs)selfs    _/home/dcms/DCMS/lib/python3.12/site-packages/torch/distributed/checkpoint/_fsspec_filesystem.py__init__zFileSystem.__init__    s	    04    pathmodec              #   *  K   | j                   J t        j                  |      }| j                   j                  |      5 }	 | 	 d d d        y #  t	        fddD              r	 | j                  |        #  Y  xY w xY w# 1 sw Y   y xY ww)Nc              3   &   K   | ]  }|v  
 y wr    ).0chr   s     r   	<genexpr>z+FileSystem.create_stream.<locals>.<genexpr>1   s     2brTz2s   zw+a)r   osfspathopenanyrm_file)r   r   r   streams     ` r   create_streamzFileSystem.create_stream#   s      ww"""yy
 WW\\$% 				 		2E22T* 		 		sH   A BBA
	BB*A<;B<B >BBBBsuffixc                 B    t         j                  j                  ||      S r   )r$   r   join)r   r   r+   s      r   concat_pathzFileSystem.concat_path8   s     ww||D&))r   c                 .    t        |fi |\  | _        }|S r   )r
   r   )r   r   kwargs_s       r   	init_pathzFileSystem.init_path=   s     t.v.
r   new_pathc                 <    | j                   j                  ||       y r   )r   rename)r   r   r3   s      r   r5   zFileSystem.renameC   s     	tX&r   c                 >    | j                   j                  |d       y )NT)exist_ok)r   makedirsr   r   s     r   mkdirzFileSystem.mkdirH   s    -r   checkpoint_idc                 \    t        |t              ry	 t        |       y# t        $ r Y yw xY w)NFT)
isinstancer   r
   
ValueErrorclsr;   s     r   validate_checkpoint_idz!FileSystem.validate_checkpoint_idK   s5    mT*	m$   		s    	++c                 8    | j                   j                  |      S r   )r   existsr9   s     r   rC   zFileSystem.existsW   s    ww~~d##r   c                 :    | j                   j                  |       y r   )r   rmr9   s     r   r(   zFileSystem.rm_fileZ   s    

4r   )r   N)__name__
__module____qualname__r   r   r	   strr$   PathLiker   ioIOBaser*   r.   r2   r5   r:   classmethodboolrA   rC   r(   r    r   r   r   r      s   5 #r{{*+36	299dD(	) (*#r{{*+*58*	sBKK	 *
#r{{*+	sBKK	 '#r{{*+'7<S"++=M7N'	'
.%R[[ 01 .d . 	5bkk9I3J 	t 	 	$5bkk!12 $t $E#r{{"23  r   r   c                        e Zd ZdZ	 	 	 	 	 	 ddeeej                  f   dedede	de	ded	e
ee      d
df fdZedeeej                  f   d
efd       Z xZS )r   a`  
    Basic implementation of StorageWriter using FFspec.

    This implementation makes the following assumptions and simplifications:

    * The checkpoint path is an empty or non-existing directory.
    * File creation is atomic

    The checkpoint consist of one file per write request plus
    a `.metadata` file with the serialized metadata.

    Nr   single_file_per_rank
sync_filesthread_countper_thread_copy_ahead	overwrite_extensionsr   c           	          t         	|   |||||||       t               | _         | j                  j                  |fi || _        y)a=  
        Initialize the writer pointing to `path`.

        Args:
            path: directory where the checkpoint will be written to.
            single_file_per_rank: Produce one file per rank instead of one file per tensor/blob. Default to True.
            sync_files : force files to be synced to permanent storage. Default to True.
            thread_count: Number of IO threads to use to write. Default to 1.
            per_thread_copy_ahead: How many bytes to copy from the GPU ahead of saving then. Default 10Mb.
            overwrite: Whether to allow overwriting existing checkpoints. Defaults to True.
            _extensions: Extensions to apply to output streams (EXPERIMENTAL)

        N. B. If sync_files is disabled, there's no guarantee that the checkpoint will be consistent in the case of a failure.
        )rT   rU   Nsuperr   r   r   r2   r   )
r   r   rP   rQ   rR   rS   rT   rU   r0   	__class__s
            r   r   zFsspecWriter.__init__m   sT    2 	 !# 	 	
 ,%DGG%%d5f5	r   r;   c                 ,    t         j                  |      S r   r   rA   r?   s     r   rA   z#FsspecWriter.validate_checkpoint_id       00??r   )TT   i TN)rF   rG   rH   __doc__r	   rI   r$   rJ   rN   intr   r   r   r   rM   rA   __classcell__rY   s   @r   r   r   _   s      &*%/DH#6C$%#6 ##6 	#6
 #6  ##6 #6 h'?@A#6 
#6J @5bkk9I3J @t @ @r   c                   ~     e Zd Zdeeej                  f   ddf fdZedeeej                  f   de	fd       Z
 xZS )r   r   r   Nc                     t         |   |       t               | _         | j                  j                  |fi || _        y r   rW   )r   r   r0   rY   s      r   r   zFsspecReader.__init__   s6    ,%DGG%%d5f5	r   r;   c                 ,    t         j                  |      S r   r[   r?   s     r   rA   z#FsspecReader.validate_checkpoint_id   r\   r   )rF   rG   rH   r	   rI   r$   rJ   r   rM   rN   rA   r`   ra   s   @r   r   r      sX    6U3#34 64 6
 @5bkk9I3J @t @ @r   )rK   r$   collections.abcr   r   
contextlibr   pathlibr   typingr   r   r	   fsspec.corer
   'torch.distributed.checkpoint._extensionr   'torch.distributed.checkpoint.filesystemr   r   r   fsspecr   __all__r   r   r   r    r   r   <module>rn      so    
 	 / %  1 1 ! L  ) < <@5@# 5@p@# @r   