
    Vh                         d dl Z d dlZd dlmZ d dlmZmZmZ d dlm	Z	 d dl
mZ dZ ee      ZdZ ee      Zg dZ G d	 d
e j"                        Z G d de      Z G d de      Z G d d      Zy)    N)Sequence)castIOOptional)Buffer)
try_importpyzstd	zstandard)	ExtensionStreamTransformExtension	ZStandardExtensionRegistryc                       e Zd ZdZeej                  defd              Zeej                  dedd fd              Z	ej                  defd       Z
y)r   a  
    Extensions provide modular additions to functionality within distributed checkpointing,
    which affect the layout or format of the written artifacts.  Extensions may be
    built into pytorch, or provided externally.

    When writing, the caller provides a list of extension instances of the appropriate
    type.  Each extension can output a descriptor which is used to reconstitute the
    extension at read-time.
    returnc                       yz<
        See ExtensionRegistry.from_descriptor_list
        N r       W/home/dcms/DCMS/lib/python3.12/site-packages/torch/distributed/checkpoint/_extension.pyregistry_namezExtension.registry_name*       r   versionc                      yr   r   r   s    r   from_descriptorzExtension.from_descriptor1   r   r   c                      y)z
        Return descriptor name to be included in metadata.  The form should be
        "extension_name[@local-domain][/version]".
        Nr   selfs    r   get_descriptorzExtension.get_descriptor8   r   r   N)__name__
__module____qualname____doc__staticmethodabcabstractmethodstrr   r   r   r   r   r   r   r      s     3   
     
 	  r   r   c                       e Zd ZdZej
                  dee   dee   fd       Zej
                  dee   dee   fd       Z	y)r   an  
    An extension which performs transformation on a byte stream, such as compression
    or encryption.

    Implementations should try to be memory friendly and performant.  For example, don't
    read the whole input, then transform it, and write it back.  If at all possible, do it in
    chunks.  But, don't read/transform/write one byte at a time, either.
    outputr   c                      y)z
        Takes a writeable output stream, and generates a new stream which implements the
        output transform.  Input data written to the returned stream will be transformed
        and written to the `output` argument stream.
        Nr   r   r)   s     r   transform_toz%StreamTransformExtension.transform_toJ   r   r   inputc                      y)z
        Takes a readable input stream, and generates a new stream which implements the
        input transform.  When the returned stream is read, data will be read from the
        'input' stream, transformed, and returned.
        Nr   r   r-   s     r   transform_fromz'StreamTransformExtension.transform_fromR   r   r   N)
r    r!   r"   r#   r%   r&   r   bytesr,   r0   r   r   r   r   r   @   sg     	2e9 E   	BuI "U)  r   r   c                        e Zd Zedefd       Zededd fd       Zedefd       Zd fdZ	defdZ
dee   dee   fd	Zd
ee   dee   fdZ xZS )r   r   c                  &    t         d uxs t        d uS N)r
   r	   r   r   r   is_availablezZStandard.is_available\   s    $:d(::r   r   c                     | j                  d      d   dk7  rt        d|       t        j                         st        dt         dt
         d      t               S )N.r   1zUnknown extension version=zOStream with ZStandard compression cannot be processed because no module named '' or '')	partition
ValueErrorr   r5   zstandard_module_namepyzstd_module_namer   s    r   r   zZStandard.from_descriptor`   sl    S!!$+:'<==%%'$$9#:&AS@TTUW  {r   c                       y)Nzstream.zstdr   r   r   r   r   zZStandard.registry_namek   s    r   c                 ~    t         |           t        j                         st	        dt
         dt         d      y )Nz<ZStandard extension is unavailable because no module named 'r9   r:   )super__init__r   r5   r<   r=   r>   )r   	__class__s    r   rB   zZStandard.__init__o   sK    %%'NOdNeekl~k  @A  B  (r   c                 (    | j                          dS )Nz/1)r   r   s    r   r   zZStandard.get_descriptorv   s    $$&'r**r   r)   c                     t         %t         j                         }|j                  |      S  G d dt        j                        }t        t        t            ||            S )Nc                   L    e Zd Zdee   ddfdZdefdZdede	e
   fdZd	dZy)
&ZStandard.transform_to.<locals>.Writerr)   r   Nc                 D    || _         t        j                         | _        y r4   )r)   r	   ZstdCompressor
compressorr+   s     r   rB   z/ZStandard.transform_to.<locals>.Writer.__init__   s    $"("7"7"9r   c                      yNTr   r   s    r   	writeablez0ZStandard.transform_to.<locals>.Writer.writeable       r   bc                     | j                   j                  |      }|r| j                  j                  |       t	        t        |            S r4   )rJ   compressr)   writelen
memoryview)r   rO   outdatas      r   rR   z,ZStandard.transform_to.<locals>.Writer.write   s:    //2215KK%%g.:a=))r   c                     | j                   j                         }|r| j                  j                  |       | j                  j                          y r4   )rJ   flushr)   rR   )r   rU   s     r   rW   z,ZStandard.transform_to.<locals>.Writer.flush   s9    ////1KK%%g.!!#r   r   N)r    r!   r"   r   r1   rB   boolrM   r   r   intrR   rW   r   r   r   WriterrG   ~   sA    :r%y :T :4 *v *(3- *$r   r[   )r
   rI   stream_writerio	RawIOBaser   r   r1   )r   r)   rJ   r[   s       r   r,   zZStandard.transform_toy   sN     "113J++F33	$R\\ 	$( BuIvf~..r   r-   c                     t         %t         j                         }|j                  |      S  G d dt        j                        }t        t        t            ||            S )Nc                   P    e Zd Zdee   ddfdZdefdZdede	e
   fdZdefdZy)	(ZStandard.transform_from.<locals>.Readerr-   r   Nc                 D    || _         t        j                         | _        y r4   )r-   r	   EndlessZstdDecompressordecompressorr/   s     r   rB   z1ZStandard.transform_from.<locals>.Reader.__init__   s    "
$*$B$B$D!r   c                      yrL   r   r   s    r   readablez1ZStandard.transform_from.<locals>.Reader.readable   rN   r   rO   c                     | j                   j                  r| j                  j                  d      }nd}t	        |      }t        |      }| j                   j                  ||      }|y t        |      }||d | |S )Ni  r   )rd   needs_inputr-   readrT   rS   
decompress)r   rO   indatabviewblenrU   counts          r   readintoz1ZStandard.transform_from.<locals>.Reader.readinto   sv     $$00!ZZ__-=>F F"15z++66vtD?G 'fur   c                      y)NFr   r   s    r   seekablez1ZStandard.transform_from.<locals>.Reader.seekable   s    r   )r    r!   r"   r   r1   rB   rY   rf   r   r   rZ   ro   rq   r   r   r   Readerra      sK    Ebi ED E$ & Xc] ,$ r   rr   )r
   ZstdDecompressorstream_readerr]   r^   r   r   r1   )r   r-   rd   rr   s       r   r0   zZStandard.transform_from   sO     $557L--e44	R\\ 	B BuIve}--r   rX   )r    r!   r"   r$   rY   r5   r'   r   r   rB   r   r   r1   r,   r0   __classcell__)rC   s   @r   r   r   [   s    ;$ ; ;     3  + +/2e9 /E /6&.BuI &."U) &.r   r   c                   F    e Zd ZddZdee   ddfdZdee   dee   fdZ	y)	r   r   Nc                 `    t         fD ci c]  }|j                         | c}| _        y c c}w r4   )r   r   
extensionsr   clss     r   rB   zExtensionRegistry.__init__   s0     2;7
),C$7
 7
s   +rz   c                 >    || j                   |j                         <   y r4   )rx   r   ry   s     r   registerzExtensionRegistry.register   s    /2))+,r   descriptorsc                 ^     dt         dt        f fd}|D cg c]
  } ||       c}S c c}w )a.  
        Given a seuquence of descriptor strings as returned by
        Extension.get_descriptor at save time, creates a sequence of
        Extension instances.  The name[@local-domain] preceding the
        version number is used to look up an implementation class in
        the registry, and the version is passed to the class's
        from_descriptor static method.  If the registry contains no
        match, this will throw ValueError.  If the from_descriptor
        method raises an exception, that will pass through to the
        caller.
        descr   c                     | j                  d      \  }}}|d}j                  j                  |      }|st        d|      |j	                  |      S )N/r   zUnknown extension name=)r;   rx   getr<   r   )r   name_r   extr   s        r   r   z?ExtensionRegistry.from_descriptor_list.<locals>.from_descriptor   s]    #~~c2D!W//%%d+C #;dW!=>>&&w//r   )r'   r   )r   r}   r   r   s   `   r   from_descriptor_listz&ExtensionRegistry.from_descriptor_list   s1    	0# 	0) 	0 3>>$%>>>s   *rX   )
r    r!   r"   rB   typer   r|   r   r'   r   r   r   r   r   r      s:    
3DO 3 3? ?(9BU ?r   r   )r%   r]   collections.abcr   typingr   r   r   typing_extensionsr   torch._utilsr   r>   r	   r=   r
   __all__ABCr   r   r   r   r   r   r   <module>r      s     	 $ % % % #  	&	'# ,-	 By 6_.( _.D ?  ?r   