
    BVhc                         d Z ddlZddlmZ ddlmZ ddlmZ ddlmZ ddlm	Z
 	 	 	 	 ddZ G d	 d
ej                        Zy)z1The implementation of `tf.data.Dataset.snapshot`.    N)dataset_ops)structured_function)dtypes)tensor_spec)gen_experimental_dataset_opsc                     d}|| j                  |      } d }d }n|}t        | |||||      }||j                  ||      }|S )z%See `Dataset.snapshot()` for details.N)namec                 0    | t        j                         z  S N)multiprocessing	cpu_count)index_s     V/home/dcms/DCMS/lib/python3.12/site-packages/tensorflow/python/data/ops/snapshot_op.py<lambda>z_snapshot.<locals>.<lambda>)   s    0I0I0K(K     c                     |S r    )r   elems     r   r   z_snapshot.<locals>.<lambda>*   s    4 r   )input_datasetpathcompressionreader_func
shard_funcr	   )	enumerate_SnapshotDatasetmap)	r   r   r   r   r   r	   project_funclocal_shard_funcdatasets	            r   	_snapshotr!      ss     ,!+++6M
 L'L!! "' kk,Tk2G	.r   c                   :     e Zd ZdZ	 	 	 	 	 d fd	Zd Zd Z xZS )r   zBA dataset that allows saving and re-use of already processed data.c	           	         |d }|| _         || _        || _        t        j                  || j                         dz   t        j                  t        j                  |j                              |      | _	        t        j                  || j                         dz   ||      | _
        | j                  j                  j                  t        j                  g t        j                               sj| j                  j                  j                  t        j                  g t        j"                              s#t%        d| j                  j                   d      || _        t)        j*                  |j,                  || j                  j.                  j0                  | j                  j.                  j0                  f|| j                  j.                  | j                  j.                  d| j2                  }	t4        
| m  ||	       y )	Nc                 l    | j                  d t        j                         t        j                        S )Nc                     | S r   r   )xs    r   r   z=_SnapshotDataset.__init__.<locals>.<lambda>.<locals>.<lambda>J   s    A r   )cycle_lengthnum_parallel_calls)
interleaver   r   r   AUTOTUNE)datasetss    r   r   z+_SnapshotDataset.__init__.<locals>.<lambda>I   s.    X%8%8
&002(11 &9 &3 r   z.reader_func)input_structureuse_legacy_functionz.shard_func)r    r-   z_Invalid `shard_func`. `shard_func` must return `tf.int64` scalar tensor but its return type is .)r   r   r   )_input_dataset_path_compressionr   StructuredFunctionWrapper_transformation_namer   DatasetSpecelement_spec_reader_func_shard_funcoutput_structureis_compatible_withr   
TensorSpecr   int32int64	TypeError_nameged_opssnapshot_dataset_v2_variant_tensorfunctioncaptured_inputs_common_argssuper__init__)selfr   r   r   r   r   pending_snapshot_expiry_secondsr-   r	   variant_tensor	__class__s             r   rF   z_SnapshotDataset.__init__>   s    3k
 (DDJ#D+EE!!#n4#//##M$>$>?A/1D +DD!!#m3/	1D ..AAr6<<02..AA""2v||46 I))::;1> ? ? DJ00%%""22!!11	
  %%..##,, 

N 
G]N3r   c                 2    | j                   | j                  gS r   )r6   r7   rG   s    r   
_functionsz_SnapshotDataset._functionss   s    t//00r   c                      y)NzDataset.snapshot()r   rL   s    r   r3   z%_SnapshotDataset._transformation_namev   s    r   )NNNFN)__name__
__module____qualname____doc__rF   rM   r3   __classcell__)rJ   s   @r   r   r   ;   s'    J  /3#(34j1 r   r   )AUTONNN)rR   r   tensorflow.python.data.opsr   r   tensorflow.python.frameworkr   r   tensorflow.python.opsr   r?   r!   UnaryUnchangedStructureDatasetr   r   r   r   <module>rY      sB    8  2 : . 3 I
 !B< {AA < r   