
    BVhG                     j   d Z ddlZddlZddlZddlmZ ddlmZ ddlm	Z	 ddlm
Z
 ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z  d Z!d Z"d Z#d Z$d Z%d Z& G d de jN                        Z(y)z)Test utilities for tf.data functionality.    N)
lookup_ops)random_access)dataset_ops)	test_mode)nest)	structure)context)combinations)config)constant_op)dtypes)errors)ops)sparse_tensor)	array_ops)gen_dataset_ops)gen_experimental_dataset_ops)tensor_array_ops)ragged_tensor)testc                  8    t        j                  ddgddg      S )z8Returns the default test combinations for tf.data tests.      eagergraphtf_api_versionmoder
   combine     ]/home/dcms/DCMS/lib/python3.12/site-packages/tensorflow/python/data/kernel_tests/test_base.pydefault_test_combinationsr$   +   s    			aV7G:L	MMr"   c                  4    t        j                  ddgd      S )zHReturns the default test combinations for eager mode only tf.data tests.r   r   r   r   r   r!   r"   r#   eager_only_combinationsr&   0       			aV'	BBr"   c                  4    t        j                  ddgd      S )zHReturns the default test combinations for graph mode only tf.data tests.r   r   r   r   r   r!   r"   r#   graph_only_combinationsr)   5   r'   r"   c                  4    t        j                  dddg      S )z@Returns the default test combinations for v1 only tf.data tests.r   r   r   r   r   r!   r"   r#   v1_only_combinationsr+   :       			Qgw5G	HHr"   c                  4    t        j                  dddg      S )z@Returns the default test combinations for v2 only tf.data tests.r   r   r   r   r   r!   r"   r#   v2_only_combinationsr.   ?   r,   r"   c                  0    t        j                  dd      S )zFReturns the default test combinations for v2 eager only tf.data tests.r   r   r   r   r!   r"   r#   v2_eager_only_combinationsr0   D   s    			QW	==r"   c                        e Zd ZdZ fdZd Zd ZddZd ZddZ	d	 Z
	 	 	 	 	 	 	 dd
Zd Z	 ddZdej                  fdZd Zd Zd Zd Zd Zd ZddZdej                  fdZd Zd Z xZS )DatasetTestBasezBase class for dataset tests.c                 L    t         |           t        j                  d       y )NT)supersetUpr   toggle_test_mode)self	__class__s    r#   r5   zDatasetTestBase.setUpL   s    	GMOt$r"   c                     | j                  t        j                        5  | j                  |       d d d        y # 1 sw Y   y xY wN)assertRaisesr   CancelledErrorevaluate)r7   ops     r#   assert_op_cancelledz#DatasetTestBase.assert_op_cancelledP   s7    			600	1 
mmB  s	   ;Ac                 4   t        |t              rn| j                  t        |j	                               t        |j	                                      |j	                         D ]  }| j                  ||   ||           yt        j                  |      rs| j                  |j                  |j                         | j                  |j                  |j                         | j                  |j                  |j                         y| j                  ||       y)z"Asserts that two values are equal.N)
isinstancedictassertItemsEquallistkeysassertValuesEqualr   	is_sparseassertAllEqualindicesvaluesdense_shape)r7   expectedactualks       r#   rF   z!DatasetTestBase.assertValuesEqualT   s    (D!
D143FG}} 7!x{F1I67		 	 	*
(**FNN;
(//6==9
(..0B0BC
(F+r"   Nc                 \   d }t        j                         st        j                         rt	        |      } ||j
                        S |r2t        j                  ||      }| j                  |j                         nt        j                  |      }|j                          |fd      S )au  Returns a callable that returns the next element of the dataset.

    Example use:
    ```python
    # In both graph and eager modes
    dataset = ...
    get_next = self.getNext(dataset)
    result = self.evaluate(get_next())
    ```

    Args:
      dataset: A dataset whose elements will be returned.
      requires_initialization: Indicates that when the test is executed in graph
        mode, it should use an initializable iterator to iterate through the
        dataset (e.g. when it contains stateful nodes). Defaults to False.
      shared_name: (Optional.) If non-empty, the returned iterator will be
        shared under the given name across multiple sessions that share the same
        devices (e.g. when using a remote server).
    Returns:
      A callable that returns the next element of `dataset`. Any `TensorArray`
      objects `dataset` outputs are stacked.
    c                       fd}|S )Nc                  j            } t        | t        j                        r| j                         S | S r:   )rA   r   TensorArraystack)rgns    r#   _wrapperz=DatasetTestBase.getNext.<locals>.ta_wrapper.<locals>._wrappery   s,    Da)556
(r"   r!   )rU   rV   s   ` r#   
ta_wrapperz+DatasetTestBase.getNext.<locals>.ta_wrapperx   s     or"   c                       S r:   r!   )get_nexts   r#   <lambda>z)DatasetTestBase.getNext.<locals>.<lambda>   s     r"   )r	   executing_eagerlyr   inside_functioniter_next_internalr   make_initializable_iteratorr=   initializermake_one_shot_iteratorrY   )r7   datasetrequires_initializationshared_namerW   iteratorrY   s         @r#   getNextzDatasetTestBase.getNexta   s    .   "c&9&9&;gh//00	 ::7KPh**+55g>""$h())r"   c                 <   |r| j                  ||       y t        t        |            D ]p  }t        j                  ||   ||          t        t        j                  ||         t        j                  ||               D ]  \  }}| j                  ||        r y r:   )rC   rangelenr   assert_same_structurezipflattenrF   )r7   result_valuesexpected_valuesassert_items_equaliresult_valueexpected_values          r#   _compareOutputToExpectedz(DatasetTestBase._compareOutputToExpected   s     M?;3}%& =
  q!1?13EF*-
,,}Q'
($,,q7I*J+L =
&,~|<==r"   c                 J    | j                  ||      }| j                  |      S )Nrc   )rf   getIteratorOutput)r7   rb   rc   rY   s       r#   getDatasetOutputz DatasetTestBase.getDatasetOutput   s-    ||)@  BH!!(++r"   c                     g }	 	 |j                  | j                   |                    '# t        j                  $ r Y |S w xY w)z?Evaluates `get_next` until end of input, returning the results.)appendr=   r   OutOfRangeError)r7   rY   resultss      r#   rv   z!DatasetTestBase.getIteratorOutput   sJ    G
t}}XZ01  ## Ns   %+ AAc	                 D   | j                  |duxs |dud       |rp| j                  |du d       | j                  |d   |d         5  | j                  ||      }	t        |      D ]  }
| j	                   |	               	 ddd       y|r%| j                  |t        j                  |             | j                  |d       t        |      D ]  }
| j                  ||      }	g }t        t        |            D ](  }
	 |j                  | j	                   |	                    * | j                  |||       | j!                  t        j                        5  | j	                   |	              ddd       | j!                  t        j                        5  | j	                   |	              ddd        y# 1 sw Y   yxY w# t        j                  $ r- t        dt        |      t        |      t        |      fz        w xY w# 1 sw Y   xY w# 1 sw Y   NxY w)a  Asserts that a dataset produces the expected output / error.

    Args:
      dataset: A dataset to check for the expected output / error.
      expected_output: A list of elements that the dataset is expected to
        produce.
      expected_shapes: A list of TensorShapes which is expected to match
        output_shapes of dataset.
      expected_error: A tuple `(type, predicate)` identifying the expected error
        `dataset` should raise. The `type` should match the expected exception
        type, while `predicate` should either be 1) a unary function that inputs
        the raised exception and returns a boolean indicator of success or 2) a
        regular expression that is expected to match the error message
        partially.
      requires_initialization: Indicates that when the test is executed in graph
        mode, it should use an initializable iterator to iterate through the
        dataset (e.g. when it contains stateful nodes). Defaults to False.
      num_test_iterations: Number of times `dataset` will be iterated. Defaults
        to 1.
      assert_items_equal: Tests expected_output has (only) the same elements
        regardless of order.
      expected_error_iter: How many times to iterate before expecting an error,
        if an error is expected.
    NzDExactly one of expected_output or expected error should be provided.r   r   ru   zHDataset ended early, producing %d elements out of %d. Dataset output: %s)
assertTrueassertRaisesWithPredicateMatchrf   rh   r=   assertEqualr   get_legacy_output_shapesassertGreaterri   ry   r   rz   AssertionErrorstrrs   r;   )r7   rb   expected_outputexpected_shapesexpected_errorrc   num_test_iterationsro   expected_error_iterrY   _results               r#   assertDatasetProducesz%DatasetTestBase.assertDatasetProduces   s)   B 	OOd"AoT&ANP 
oo
T
!
P ..~a/@/=a/@B $<<-D   F*+ 	$A
--

#	$	$ 
";;GDF*A.&' "
+B  DhfS)* @!	@
--hj1
2@ ##FO=OPV334 "hj!"V334 "hj!" ""$  %% 	@#6{C0#f+>?@ @	@" "" "s1   	:F:%GH	H:GA H	H	H	c                 ,   | j                  t        j                  t        j                  |      t        j                  |                   t        j                  t        j                  |            }| j                  |      }| j                  |      }	 	 | j                   |             }| j                   |             }t        j                  |      }t        j                  |      }t        |      t        |      k(  sJ t        t        |            D ]  }t        j                   ||         st#        j$                  ||         r| j'                  ||   ||          L||   t(        j*                  k(  r| j-                  ||   ||          {| j/                  ||   ||           # t        j                  $ rN | j                  t        j                        5  | j                   |              ddd       Y y# 1 sw Y   Y yxY ww xY w)zCChecks that datasets are equal. Supports both graph and eager mode.N)r}   r   are_compatibler   get_structurer   rl   get_legacy_output_typesrf   r=   r   rz   r;   ri   rh   r   rG   r   	is_raggedrF   r   stringrH   assertAllClose)	r7   dataset1dataset2flattened_typesnext1next2op1op2rp   s	            r#   assertDatasetsEqualz#DatasetTestBase.assertDatasetsEqual   s   OO  %%h/%%h/	12
 ll++H57O LL"ELL"E
mmEG$
 MM%'"cLLcLLcXS!!!SX .!""3q6*m.E.Ec!f.M

 
 QQ
0Q6==0


c!fc!f
-


c!fc!f
-.  ## v556 	!
--
 	!	!s*   F2 22H$H;HH	
HHc                    |g }| j                  |      }| j                  |      }	 | j                   |              t        dt        |      z        # |$ r}|j                  }|D ]  \  }	}
}|j                  |	|
|      } | j                  |t        j                  |            5  | j                   |              ddd       n# 1 sw Y   nxY wY d}~yY d}~yd}~ww xY w)zEChecks that datasets raise the same error on the first get_next call.Nz>Expected dataset to raise an error of type %s, but it did not.)	rf   r=   
ValueErrorreprmessagereplaceassertRaisesRegexreescape)r7   r   r   exception_classreplacementsr   r   eexpected_messageoldnewcounts               r#   assertDatasetsRaiseSameErrorz,DatasetTestBase.assertDatasetsRaiseSameError  s     lLL"ELL"E
mmEG
J

 ! !  ) E/#sE+33CeDE !!/299=M3NO eg    s0   -A C$AC)C	 	C	C	CC$c                    |g }|4t         j                  j                  t        j                  ||            S t         j                  j                  t        |D cg c]  }| j                  |||       c}            S c c}w )z5Returns a singleton dataset with the given structure.dtype)r   Datasetfrom_tensorsr   zerosrk   tuplestructuredDataset)r7   dataset_structureshaper   substructures        r#   r   z!DatasetTestBase.structuredDataset)  s     }e   --
//%u
-/ /   $$
"3 $$\5%@  s   Bc           	          | j                  ||       | j                  t        j                        5  | j	                  t        j                  |t        |                   d d d        y # 1 sw Y   y xY w)N)index)%verifyRandomAccessInfiniteCardinalityr;   r   rz   r=   r   atri   )r7   rb   rL   s      r#   verifyRandomAccessz"DatasetTestBase.verifyRandomAccess8  s\    ..wA			611	2 D
mmM$$WCMBCD D Ds   0A++A4c           
         t        |      }t        t        |            dz  }t        j                  |       |D ]:  }| j                  ||   | j                  t        j                  ||                   < t        t        |            }|D ]:  }| j                  ||   | j                  t        j                  ||                   < y)z/Tests randomly accessing elements of a dataset.r   N)ri   rD   rh   randomshufflerH   r=   r   r   setsorted)r7   rb   rL   len_expectedrI   rp   s         r#   r   z5DatasetTestBase.verifyRandomAccessInfiniteCardinality=  s     x=L5&'!+G
NN7 G
(1+--(8(8!(DEGG
 &/"G G
(1+--(8(8!(DEGGr"   c                    t         j                  j                  | j                         d      }t	        |d      5 }|j                  dj                  d |D              dz          d d d        t        j                  |t        j                  t        j                  j                  t        j                  t        j                  j                        S # 1 sw Y   nxY w)Ntext_file_initializerw
c              3   2   K   | ]  }t        |        y wr:   )r   ).0vs     r#   	<genexpr>z6DatasetTestBase.textFileInitializer.<locals>.<genexpr>P  s     -1A-s   )ospathjoinget_temp_diropenwriter   TextFileInitializerr   int64TextFileIndexLINE_NUMBER
WHOLE_LINE)r7   valsfilefs       r#   textFileInitializerz#DatasetTestBase.textFileInitializerM  s    77<<))+-DED	dC 6Aggdii---456))$*4*B*B*N*N*0,,*4*B*B*M*MO O6 6s   +CCc           	          t        j                  t        t        t	        |                  t
        j                        }t        j                  |      }t        j                  ||      S )Nr   )	r   constantrD   rh   ri   r   r   r   KeyValueTensorInitializer)r7   r   keys_tensorvals_tensors       r#   keyValueTensorInitializerz)DatasetTestBase.keyValueTensorInitializerV  sL    &&U3t9fll4K&&t,K//[IIr"   c                     t         j                  j                  t        |            }t         j                  j	                  |      }t         j                  j                  ||f      }t        j                  |      S r:   )r   r   rh   ri   from_tensor_slicesrk   data_lookup_opsDatasetInitializer)r7   r   rE   rJ   dss        r#   datasetInitializerz"DatasetTestBase.datasetInitializer\  s]    $$SY/D  33D9F				 	 $	0B--b11r"   c                     |dk(  r| j                  |      S |dk(  r| j                  |      S |dk(  r| j                  |      S t        d|z         )a  Returns a lookup table initializer for the given source and values.

    Args:
      init_source: One of ["textfile", "keyvalue", "dataset"], indicating what
        type of initializer to use.
      vals: The initializer values. The keys will be `range(len(vals))`.
    textfilekeyvaluetensorrb   zUnrecognized init_source: )r   r   r   r   )r7   init_sourcer   s      r#   lookupTableInitializerz&DatasetTestBase.lookupTableInitializerb  sa     j %%d++	(	(++D11			!$$T**3kABBr"   c                     t        j                  |j                  |      }t        j                  t        j                  |      |j                        S )z'Converts a dataset to a graph and back.)allow_stateful)r   dataset_to_graph_variant_tensorr   from_variantr   dataset_from_graphelement_spec)r7   rb   r   r   s       r#   graphRoundTripzDatasetTestBase.graphRoundTrips  sH    ,,@E##$77> r"   c           
          |g }|t        j                  ||      S t        |D cg c]  }| j                  |||       c}      S c c}w )z,Returns an element with the given structure.r   )r   r   r   structuredElement)r7   element_structurer   r   r   s        r#   r   z!DatasetTestBase.structuredElement{  s\     }e __U%00/ 
 
 ue
< 	 	 s   Ac                 4   |r, |d      }| j                  |      }| j                  ||       ydD ]T  } ||      }| j                  |      }| j                  ||       t        t	        |            D ]  }||   ||   k7  s  y V | j                  d       y)a-  Tests whether a dataset produces its elements deterministically.

    `dataset_fn` takes a delay_ms argument, which tells it how long to delay
    production of the first dataset element. This gives us a way to trigger
    out-of-order production of dataset elements.

    Args:
      dataset_fn: A function taking a delay_ms argument.
      expect_determinism: Whether to expect deterministic ordering.
      expected_elements: The elements expected to be produced by the dataset,
        assuming the dataset produces elements in deterministic order.
    d   N)
   r   i  i N  i z+Failed to observe nondeterministic ordering)rw   rH   assertCountEqualrh   ri   fail)r7   
dataset_fnexpect_determinismexpected_elementsrb   rM   delay_msrp   s           r#   checkDeterminismz DatasetTestBase.checkDeterminism  s     3g$$W-f
+V4 3 8$g$$W-f
-v6S[! !!9)!,,
	 	II;<r"   c                    t        j                  d      }t        j                  d      }t        j                  |d   t        |      D cg c]  }t	        j
                          c}       t        |dz
        D cg c]  }dt        |      z    }}|r|j                  d       |S |j                  dt        |dz
        z          |S c c}w c c}w )a  Configures number of logical devices for multi-device tests.

    It returns a list of device names. If invoked in GPU-enabled runtime, the
    last device name will be for a GPU device. Otherwise, all device names will
    be for a CPU device.

    Args:
      num_devices: The number of devices to configure.

    Returns:
      A list of device names to use for a multi-device test.
    CPUGPUr   r   z/device:CPU:z/device:GPU:0)r   list_physical_devices set_logical_device_configurationrh   r	   LogicalDeviceConfigurationr   ry   )r7   num_devicescpusgpusr   rp   devicess          r#   "configureDevicesForMultiDeviceTestz2DatasetTestBase.configureDevicesForMultiDeviceTest  s     ''.D''.D
++DG6;K6H612**,6  16kAo0FG1~A&GGGnn_% N nn^c+/&::;N6 Hs   C
=C)FN)F)NNNFr   Fr   r:   )__name__
__module____qualname____doc__r5   r?   rF   rf   rs   rw   rv   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  __classcell__)r8   s   @r#   r2   r2   I   s    %%,,*\=,
 -1,0+/4901/401D"L.J 15	. 8<$llD
G OJ2C" 8<$ll	=@r"   r2   ))r  r   r   r   'tensorflow.python.data.experimental.opsr   r   r   tensorflow.python.data.opsr   r   tensorflow.python.data.utilr   r   tensorflow.python.eagerr	   tensorflow.python.frameworkr
   r   r   r   r   r   r   tensorflow.python.opsr   r   r   r   tensorflow.python.ops.raggedr   tensorflow.python.platformr   r$   r&   r)   r+   r.   r0   TestCaser2   r!   r"   r#   <module>r     s    0 	  	 Q A 2 0 , 1 + 4 . 3 . . + 5 + 1 > , 2 6 +N
C
C
I
I
>
vdmm vr"   