
    AVh              
          d Z ddlZddlZddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl	m
Z
 dd	l	mZ dd
l	mZ ddl	mZ ddlmZ ddlmZ ddlmZ ddlmZ  edddg       G d d ej*                  ddg                   Z ed       G d d ej*                  dg d                   Z edddg       G d d ej*                  dg d                   Z edddg       G d d ej*                  dg d                    Z ed!d!d"g       G d# d" ej*                  d"g d$                   Z G d% d&      Zd' Zd( Z	 d+d)Z	 d+d*Zy),z.Feature configuration for tf.io.parse_example.    N)constant_op)dtypes)ops)tensor)tensor_shape)	array_ops)	check_ops)math_ops)
sparse_ops)ragged_math_ops)ragged_tensor)
tf_logging)	tf_exportzio.VarLenFeatureVarLenFeature)v1c                       e Zd ZdZy)r   zgConfiguration for parsing a variable-length input feature.

  Fields:
    dtype: Data type of input.
  N)__name__
__module____qualname____doc__     T/home/dcms/DCMS/lib/python3.12/site-packages/tensorflow/python/ops/parsing_config.pyr   r   ,   s    
 r   dtypezio.RaggedFeaturec                   @    e Zd ZdZ ej
                  ddg      Z ej
                  ddg      Z ej
                  ddg      Z ej
                  ddg      Z	 ej
                  ddg      Z
 ej
                  dd	g      Zeeee	e
efZd
dej                  df fd	Z xZS )RaggedFeaturea  Configuration for passing a RaggedTensor input feature.

  `value_key` specifies the feature key for a variable-length list of values;
  and `partitions` specifies zero or more feature keys for partitioning those
  values into higher dimensions.  Each element of `partitions` must be one of
  the following:

    * `tf.io.RaggedFeature.RowSplits(key: string)`
    * `tf.io.RaggedFeature.RowLengths(key: string)`
    * `tf.io.RaggedFeature.RowStarts(key: string)`
    * `tf.io.RaggedFeature.RowLimits(key: string)`
    * `tf.io.RaggedFeature.ValueRowIds(key: string)`
    * `tf.io.RaggedFeature.UniformRowLength(length: int)`.

  Where `key` is a feature key whose values are used to partition the values.
  Partitions are listed from outermost to innermost.

  * If `len(partitions) == 0` (the default), then:

    * A feature from a single `tf.Example` is parsed into a 1D `tf.Tensor`.
    * A feature from a batch of `tf.Example`s is parsed into a 2D
      `tf.RaggedTensor`, where the outer dimension is the batch dimension, and
      the inner (ragged) dimension is the feature length in each example.

  * If `len(partitions) == 1`, then:

    * A feature from a single `tf.Example` is parsed into a 2D
      `tf.RaggedTensor`, where the values taken from the `value_key` are
      separated into rows using the partition key.
    * A feature from a batch of `tf.Example`s is parsed into a 3D
      `tf.RaggedTensor`, where the outer dimension is the batch dimension,
      the two inner dimensions are formed by separating the `value_key` values
      from each example into rows using that example's partition key.

  * If `len(partitions) > 1`, then:

    * A feature from a single `tf.Example` is parsed into a `tf.RaggedTensor`
      whose rank is `len(partitions)+1`, and whose ragged_rank is
      `len(partitions)`.

    * A feature from a batch of `tf.Example`s is parsed into a `tf.RaggedTensor`
      whose rank is `len(partitions)+2` and whose ragged_rank is
      `len(partitions)+1`, where the outer dimension is the batch dimension.

  There is one exception: if the final (i.e., innermost) element(s) of
  `partitions` are `UniformRowLength`s, then the values are simply reshaped (as
  a higher-dimensional `tf.Tensor`), rather than being wrapped in a
  `tf.RaggedTensor`.

  #### Examples

  >>> import google.protobuf.text_format as pbtext
  >>> example_batch = [
  ...   pbtext.Merge(r'''
  ...     features {
  ...       feature {key: "v" value {int64_list {value: [3, 1, 4, 1, 5, 9]}}}
  ...       feature {key: "s1" value {int64_list {value: [0, 2, 3, 3, 6]}}}
  ...       feature {key: "s2" value {int64_list {value: [0, 2, 3, 4]}}}
  ...     }''', tf.train.Example()).SerializeToString(),
  ...   pbtext.Merge(r'''
  ...     features {
  ...       feature {key: "v" value {int64_list {value: [2, 7, 1, 8, 2, 8, 1]}}}
  ...       feature {key: "s1" value {int64_list {value: [0, 3, 4, 5, 7]}}}
  ...       feature {key: "s2" value {int64_list {value: [0, 1, 1, 4]}}}
  ...     }''', tf.train.Example()).SerializeToString()]

  >>> features = {
  ...     # Zero partitions: returns 1D tf.Tensor for each Example.
  ...     'f1': tf.io.RaggedFeature(value_key="v", dtype=tf.int64),
  ...     # One partition: returns 2D tf.RaggedTensor for each Example.
  ...     'f2': tf.io.RaggedFeature(value_key="v", dtype=tf.int64, partitions=[
  ...         tf.io.RaggedFeature.RowSplits("s1")]),
  ...     # Two partitions: returns 3D tf.RaggedTensor for each Example.
  ...     'f3': tf.io.RaggedFeature(value_key="v", dtype=tf.int64, partitions=[
  ...         tf.io.RaggedFeature.RowSplits("s2"),
  ...         tf.io.RaggedFeature.RowSplits("s1")])
  ... }

  >>> feature_dict = tf.io.parse_single_example(example_batch[0], features)
  >>> for (name, val) in sorted(feature_dict.items()):
  ...   print('%s: %s' % (name, val))
  f1: tf.Tensor([3 1 4 1 5 9], shape=(6,), dtype=int64)
  f2: <tf.RaggedTensor [[3, 1], [4], [], [1, 5, 9]]>
  f3: <tf.RaggedTensor [[[3, 1], [4]], [[]], [[1, 5, 9]]]>

  >>> feature_dict = tf.io.parse_example(example_batch, features)
  >>> for (name, val) in sorted(feature_dict.items()):
  ...   print('%s: %s' % (name, val))
  f1: <tf.RaggedTensor [[3, 1, 4, 1, 5, 9],
                        [2, 7, 1, 8, 2, 8, 1]]>
  f2: <tf.RaggedTensor [[[3, 1], [4], [], [1, 5, 9]],
                        [[2, 7, 1], [8], [2], [8, 1]]]>
  f3: <tf.RaggedTensor [[[[3, 1], [4]], [[]], [[1, 5, 9]]],
                        [[[2, 7, 1]], [], [[8], [2], [8, 1]]]]>

  Fields:
    dtype: Data type of the `RaggedTensor`.  Must be one of:
      `tf.dtypes.int64`, `tf.dtypes.float32`, `tf.dtypes.string`.
    value_key: (Optional.) Key for a `Feature` in the input `Example`, whose
      parsed `Tensor` will be the resulting `RaggedTensor.flat_values`.  If
      not specified, then it defaults to the key for this `RaggedFeature`.
    partitions: (Optional.) A list of objects specifying the row-partitioning
      tensors (from outermost to innermost).  Each entry in this list must be
      one of:
        * `tf.io.RaggedFeature.RowSplits(key: string)`
        * `tf.io.RaggedFeature.RowLengths(key: string)`
        * `tf.io.RaggedFeature.RowStarts(key: string)`
        * `tf.io.RaggedFeature.RowLimits(key: string)`
        * `tf.io.RaggedFeature.ValueRowIds(key: string)`
        * `tf.io.RaggedFeature.UniformRowLength(length: int)`.
      Where `key` is a key for a `Feature` in the input `Example`, whose parsed
      `Tensor` will be the resulting row-partitioning tensor.
    row_splits_dtype: (Optional.) Data type for the row-partitioning tensor(s).
      One of `int32` or `int64`.  Defaults to `int32`.
    validate: (Optional.) Boolean indicating whether or not to validate that
      the input values form a valid RaggedTensor.  Defaults to `False`.
  	RowSplitskey
RowLengths	RowStarts	RowLimitsValueRowIdsUniformRowLengthlengthNr   Fc                    |+t        |t              st        d|       |st        d      t        j                  |      }|t        j
                  t        j                  t        j                  fvrt        d|      t        j                  |      }|t        j                  t        j
                  fvrt        d|      t        |t        t        f      s%t        d| dt        |      j                   d      |D ]3  }t        || j                        rt        d| j                   d	|       t        |t              st        d
|      t         t"        | K  | |||||      S )Nz+Argument `value_key` must be a string; got z&Argument `value_key` must not be emptyz7Argument `dtype` must be int64, float32, or bytes; got z7Argument `row_splits_dtype` must be int32 or int64; gotzBArgument `partitions` must be a list or tuple. Receivedpartitions=z	 of type .z:Argument `partitions` must be a list of partition objects z; got: z(Argument `validate` must be a bool; got )
isinstancestr
ValueErrorr   as_dtypeint64float32stringint32listtuple	TypeErrortyper   _PARTITION_TYPESboolsuperr   __new__)clsr   	value_key
partitionsrow_splits_dtypevalidate	partition	__class__s          r   r6   zRaggedFeature.__new__   s    	3'9)EG 	GABBOOE"EV\\6>>6==AAP$ % %'78fll;;P*-/ 0 0j4-0 $$.<yj)22316 7 7   O		3#7#78 ##&#7#7"8	}N O 	OO h%@MNN,S%J-=xI Ir   )r   r   r   r   collections
namedtupler   r   r    r!   r"   r#   r3   r   r.   r6   __classcell__r=   s   @r   r   r   6   s    
tn %k$$[5':)%{%%lUG<*$k$$[5':)$k$$[5':)&&&}ug>++[++,>
K  Y	;&(
 %||I Ir   r   )r   r8   r9   r:   r;   zio.SparseFeatureSparseFeaturec                   $     e Zd ZdZd fd	Z xZS )rB   aZ
  Configuration for parsing a sparse input feature from an `Example`.

  Note, preferably use `VarLenFeature` (possibly in combination with a
  `SequenceExample`) in order to parse out `SparseTensor`s instead of
  `SparseFeature` due to its simplicity.

  Closely mimicking the `SparseTensor` that will be obtained by parsing an
  `Example` with a `SparseFeature` config, a `SparseFeature` contains a

  * `value_key`: The name of key for a `Feature` in the `Example` whose parsed
    `Tensor` will be the resulting `SparseTensor.values`.

  * `index_key`: A list of names - one for each dimension in the resulting
    `SparseTensor` whose `indices[i][dim]` indicating the position of
    the `i`-th value in the `dim` dimension will be equal to the `i`-th value in
    the Feature with key named `index_key[dim]` in the `Example`.

  * `size`: A list of ints for the resulting `SparseTensor.dense_shape`.

  For example, we can represent the following 2D `SparseTensor`

  ```python
  SparseTensor(indices=[[3, 1], [20, 0]],
               values=[0.5, -1.0]
               dense_shape=[100, 3])
  ```

  with an `Example` input proto

  ```python
  features {
    feature { key: "val" value { float_list { value: [ 0.5, -1.0 ] } } }
    feature { key: "ix0" value { int64_list { value: [ 3, 20 ] } } }
    feature { key: "ix1" value { int64_list { value: [ 1, 0 ] } } }
  }
  ```

  and `SparseFeature` config with 2 `index_key`s

  ```python
  SparseFeature(index_key=["ix0", "ix1"],
                value_key="val",
                dtype=tf.float32,
                size=[100, 3])
  ```

  Fields:
    index_key: A single string name or a list of string names of index features.
      For each key the underlying feature's type must be `int64` and its length
      must always match that of the `value_key` feature.
      To represent `SparseTensor`s with a `dense_shape` of `rank` higher than 1
      a list of length `rank` should be used.
    value_key: Name of value feature.  The underlying feature's type must
      be `dtype` and its length must always match that of all the `index_key`s'
      features.
    dtype: Data type of the `value_key` feature.
    size: A Python int or list thereof specifying the dense shape. Should be a
      list if and only if `index_key` is a list. In that case the list must be
      equal to the length of `index_key`. Each for each entry `i` all values in
      the `index_key`[i] feature must be in `[0, size[i])`.
    already_sorted: A Python boolean to specify whether the values in
      `value_key` are already sorted by their index position. If so skip
      sorting. False by default (optional).
  c                 4    t         t        |   | |||||      S N)r5   rB   r6   )r7   	index_keyr8   r   sizealready_sortedr=   s         r   r6   zSparseFeature.__new__%  s&    ,Y	5$@ @r   )Fr   r   r   r   r6   r@   rA   s   @r   rB   rB      s    
?B@ @r   )rF   r8   r   rG   rH   zio.FixedLenFeatureFixedLenFeaturec                   $     e Zd ZdZd fd	Z xZS )rJ   a  Configuration for parsing a fixed-length input feature.

  To treat sparse input as dense, provide a `default_value`; otherwise,
  the parse functions will fail on any examples missing this feature.

  Fields:
    shape: Shape of input data.
    dtype: Data type of input.
    default_value: Value to be used if an example is missing this feature. It
        must be compatible with `dtype` and of the specified `shape`.
  c                 0    t         t        |   | |||      S rE   )r5   rJ   r6   )r7   shaper   default_valuer=   s       r   r6   zFixedLenFeature.__new__9  s     #.UE=* *r   rE   rI   rA   s   @r   rJ   rJ   *  s    
* *r   )rM   r   rN   zio.FixedLenSequenceFeatureFixedLenSequenceFeaturec                   $     e Zd ZdZd fd	Z xZS )rO   a  Configuration for parsing a variable-length input feature into a `Tensor`.

  The resulting `Tensor` of parsing a single `SequenceExample` or `Example` has
  a static `shape` of `[None] + shape` and the specified `dtype`.
  The resulting `Tensor` of parsing a `batch_size` many `Example`s has
  a static `shape` of `[batch_size, None] + shape` and the specified `dtype`.
  The entries in the `batch` from different `Examples` will be padded with
  `default_value` to the maximum length present in the `batch`.

  To treat a sparse input as dense, provide `allow_missing=True`; otherwise,
  the parse functions will fail on any examples missing this feature.

  Fields:
    shape: Shape of input data for dimension 2 and higher. First dimension is
      of variable length `None`.
    dtype: Data type of input.
    allow_missing: Whether to allow this feature to be missing from a feature
      list item. Is available only for parsing `SequenceExample` not for
      parsing `Examples`.
    default_value: Scalar value to be used to pad multiple `Example`s to their
      maximum length. Irrelevant for parsing a single `Example` or
      `SequenceExample`. Defaults to "" for dtype string and 0 otherwise
      (optional).
  c                 2    t         t        |   | ||||      S rE   )r5   rO   r6   )r7   rM   r   allow_missingrN   r=   s        r   r6   zFixedLenSequenceFeature.__new__\  s#    (#6UE=-9 9r   )FNrI   rA   s   @r   rO   rO   >  s    
29 9r   )rM   r   rR   rN   c                       e Zd ZdZ	 	 	 	 	 	 	 	 	 ddZed        Zed        Zed        Z	ed        Z
d Zd	 Zd
 Zd Zd Zd Zd Zd Zd Zd Zy)_ParseOpParamsa  Raw parameters used by `gen_parsing_ops`.

  Attributes:
    sparse_keys: A list of string keys in the examples' features. The results
      for these keys will be returned as `SparseTensor` objects.
    sparse_types: A list of `DTypes` of the same length as `sparse_keys`. Only
      `tf.float32` (`FloatList`), `tf.int64` (`Int64List`), and `tf.string`
      (`BytesList`) are supported.
    dense_keys: A list of string keys in the examples' features. The results for
      these keys will be returned as `Tensor`s
    dense_types: A list of DTypes of the same length as `dense_keys`. Only
      `tf.float32` (`FloatList`), `tf.int64` (`Int64List`), and `tf.string`
      (`BytesList`) are supported.
    dense_defaults: A dict mapping string keys to `Tensor`s. The keys of the
      dict must match the dense_keys of the feature.
    dense_shapes: A list of tuples with the same length as `dense_keys`. The
      shape of the data for each dense feature referenced by `dense_keys`.
      Required for any input tensors identified by `dense_keys`.  Must be either
      fully defined, or may contain an unknown first dimension. An unknown first
      dimension means the feature is treated as having a variable number of
      blocks, and the output shape along this dimension is considered unknown at
      graph build time.  Padding is applied for minibatch elements smaller than
      the maximum number of blocks for the given feature along this dimension.
    ragged_keys: A list of string keys in the examples' features.  The
      results for these keys will be returned as `RaggedTensor` objects.
    ragged_value_types: A list of `DTypes` of the same length as `ragged_keys`,
      specifying the value type for each ragged feature.  Must be one of:
      `tf.float32`, `tf.int64`, `tf.string`.
    ragged_split_types: A list of `DTypes` of the same length as `ragged_keys`,
      specifying the row_splits type for each ragged feature.  Must be one of:
      `tf.int32`, `tf.int64`.
    dense_shapes_as_proto: dense_shapes converted to TensorShapeProto.
    dense_defaults_vec: A vector of `Tensor`s containing the default values,
      corresponding 1:1 with `dense_keys`.
    num_features: The total number of feature keys.
  Nc
                    |t        j                         n|}|g n|}|g n|}|g n|}|g n|}|g gt        |      z  n|}|g n|}|g n|}|	g n|	}	|| _        |D 
cg c]  }
t	        j
                  |
       c}
| _        || _        |D 
cg c]  }
t	        j
                  |
       c}
| _        |D cg c]  }t        j                  |       c}| _        || _        || _        |D 
cg c]  }
t	        j
                  |
       c}
| _        |	D 
cg c]  }
t	        j
                  |
       c}
| _        | j!                          y c c}
w c c}
w c c}w c c}
w c c}
w rE   )r>   OrderedDictlensparse_keysr   r*   sparse_types
dense_keysdense_typesr   as_shapedense_shapesdense_defaultsragged_keysragged_value_typesragged_split_types	_validate)selfrX   rY   rZ   r[   r^   r]   r_   r`   ra   tss               r   __init__z_ParseOpParams.__init__  sm    &4%;! #+"K%-2<L!)zJ#+"K'3'; D
O$AM #+"K/7 =O  07 =O "D5AB+BD DO4?@q*@D;GHa..q1HD(D"D;MNavq1ND;MNavq1NDNN C@H ONs   !EE8E.EEc           
          |        }|rrt        |j                               D ]V  }||   }t        |t        |            s(t	        dt        |      j                   d| d| d      |j                  ||       X |j                          |S )af  Builds _ParseOpParams for a given set of features and allowed types.

    Args:
      features: A `dict` mapping feature keys to objects of a type in `types`.
      types: Type of features to allow, among `FixedLenFeature`,
        `VarLenFeature`, `SparseFeature`, and `FixedLenSequenceFeature`.

    Returns:
      A `_ParseOpParams` containing the raw parameters for `gen_parsing_ops`.

    Raises:
      ValueError: if `features` contains an item not in `types`, or an invalid
          feature.
      ValueError: if sparse and dense key sets intersect.
      ValueError: if input lengths do not match up.
    zUnsupported  z
 for key '')	sortedkeysr'   r0   r)   r2   r   _add_featurerb   )r7   featurestypesparamsr   features         r   from_featuresz_ParseOpParams.from_features  s    $ UF( *#3-'5<0T']334AgYjQOQ QC)* Mr   c                 \    | j                   D cg c]  }|j                          c}S c c}w rE   )r]   as_proto)rc   rM   s     r   dense_shapes_as_protoz$_ParseOpParams.dense_shapes_as_proto  s#    *.*;*;<ENN<<<s   )c                     t        | j                        t        | j                        z   t        | j                        z   S rE   )rW   rZ   rX   r_   )rc   s    r   num_featuresz_ParseOpParams.num_features  s1    t#d&6&6"77#d>N>N:OOOr   c           
          t        | j                  | j                  | j                        D cg c]  \  }}}| j	                  |||       c}}}S c c}}}w rE   )ziprZ   r]   r[   _make_dense_default)rc   kre   rd   s       r   dense_defaults_vecz!_ParseOpParams.dense_defaults_vec  sW     4??D,=,=t?O?OP Aq! 	  Aq)  s   Ac                    | j                   j                  |      }|j                  |j                  dkD  r|j                  d   j                  z|.t        j                  |t        j                  k(  rdnd|      }|S dt        j                  dd|      z   }t        j                  |||      }t        j                  |g       }|S |t        j                  g |      }|S t        |t         j"                        sHdt        j                  dd|      z   }t        j                  |||      }t        j                  ||      }|S )	zConstruct the default value tensor for a specified dense feature.

    Args:
      key: The key string identifying the dense feature.
      shape: The dense feature's shape.
      dtype: The dense feature's dtype.

    Returns:
      A Tensor.
    r    r   padding_z[^A-Za-z0-9_.\-/]_)r   namekey_)r^   getndimsdimsvaluer   convert_to_tensorr   r-   resubr   reshaper   constantr'   r   Tensor)rc   r   rM   r   rN   key_names         r   ry   z"_ParseOpParams._make_dense_default  s8    ''++C0MEKK!O

1# 
	--6==(Bau>&  ';S# FF--X7!))-<  
	#,,Ru=  -7BFF#7cBB--X7!))-?r   c                    t        |t              r| j                  ||       yt        |t              r| j	                  ||       yt        |t
              r| j                  ||       yt        |t              r| j                  ||       yt        |t              r| j                  ||       yt        d| d| d      )z1Adds the specified feature to this ParseOpParams.zInvalid feature :r&   N)r'   r   _add_varlen_featurerB   _add_sparse_featurerJ   _add_fixed_len_featurerO   _add_fixed_len_sequence_featurer   _add_ragged_featurer)   rc   r   rp   s      r   rl   z_ParseOpParams._add_feature  s    '=)
sG,	G]	+
sG,	G_	-
!!#w/	G4	5
**38	G]	+
sG,)#ay:;;r   c                 v    |j                   st        d| d|       | j                  ||j                          y)zAdds a VarLenFeature.Missing type for feature . Received feature=N)r   r)   _add_sparse_keyr   s      r   r   z"_ParseOpParams._add_varlen_feature  s=    ==%cU*=gY
GI Igmm,r   c           	         || j                   v rC| j                  | j                   j                  |         }||k7  rt        d| d| d| d      y| j                   j	                  |       | j                  j	                  |       y)z3Adds a sparse key & dtype, checking for duplicates.Conflicting type  vs  for feature r&   N)rX   rY   indexr)   append)rc   r   r   original_dtypes       r   r   z_ParseOpParams._add_sparse_key  s    
d(()9)9)?)?)DEn	5	 /tE7-uANP 	P 
! c"
u%r   c                    |j                   st        d| d      |j                  st        d| d      |j                  st        d| d| d      |j                   }t	        |t
              r|g}n#t        |      dkD  rt        j                  d       t        |      D ]"  }| j                  |t        j                         $ | j                  |j                  |j                         y)	zAdds a SparseFeature.z$Missing index_key for SparseFeature r&   z$Missing value_key for SparseFeature r   r      zsSparseFeature is a complicated feature config and should only be used after careful consideration of VarLenFeature.N)rF   r)   r8   r   r'   r(   rW   r   warningrj   r   r   r+   )rc   r   rp   
index_keysrF   s        r   r   z"_ParseOpParams._add_sparse_feature  s     =gYaHII=gYaHII==23%7J!!% & &""J*c"<j	Z1	 ; < J' 4	
9fll34**GMM:r   c                    |j                   st        d| d| d      |j                  t        d| d| d      t        j                  |j                        }|j                  r7|j
                  r+|j                  d   j                  t        d| d| d      |j                  ,|j                         st        d	| d
|j                  d      | j                  j                  |       | j                  j                  t        j                  |j                               | j                  j                  |j                          |j                  |j                  | j                  |<   yy)zAdds a FixedLenFeature.r   r   r&   NMissing shape for feature r   z%First dimension of shape for feature zC unknown. Consider using FixedLenSequenceFeature. Received feature=z$All dimensions of shape for feature z need to be known but received )r   r)   rM   r   r\   r   r   r   is_fully_definedrZ   r   r]   r[   rN   r^   )rc   r   rp   feature_tensor_shapes       r   r   z%_ParseOpParams._add_fixed_len_feature4  su   ==23%7J!!% & &}}3C58K!!% & &'00?.44!!!$**2>se D"")!- . . 	! 113=cU C--4]],=Q@ A AOO3\227==ABGMM*(!(!6!6d# )r   c                    |j                   st        d| d| d      |j                  t        d| d| d      | j                  j	                  |       | j
                  j	                  t        j                  |j                               | j                  j	                  |j                          |j                  rd| j                  |<   |j                  |j                  | j                  |<   yy)zAdds a FixedLenSequenceFeature.r   r   r&   Nr   )r   r)   rM   rZ   r   r]   r   r\   r[   rR   r^   rN   r   s      r   r   z._ParseOpParams._add_fixed_len_sequence_featureL  s    ==23%7J!!% & &}}3C58K!!% & &OO3\227==ABGMM*!%d#(!(!6!6d# )r   c           	         || j                   v r| j                  | j                   j                  |         }| j                  | j                   j                  |         }||k7  rt	        d| d| d| d      ||k7  rt	        d| d| d| d      y| j                   j                  |       | j                  j                  |       | j                  j                  |       y)z3Adds a ragged key & dtype, checking for duplicates.r   r   r   r&   zConflicting partition type N)r_   r`   r   ra   r)   r   )rc   r   
value_type
split_typeoriginal_value_typeoriginal_split_types         r   _add_ragged_keyz_ParseOpParams._add_ragged_key\  s    
d 33D4D4D4J4J34OP 33D4D4D4J4J34OP	
	*,-@,A&<}SE< = 	=	
	*67J6K4&<}SE< = 	= 
+ c"
$$Z0
$$Z0r   c                 H   |j                   |n|j                   }| j                  ||j                  |j                         |j                  D ]R  }t        |t        j                        r| j                  |j                  t        j                  |j                         T y)zAdds a RaggedFeature.N)r8   r   r   r:   r9   r'   r   r#   r   r   r+   )rc   r   rp   r8   r<   s        r   r   z"_ParseOpParams._add_ragged_featurel  s~    ((0g6G6GIGMM73K3KL'' 7		=#A#ABY]]FLL$55	77r   c           
         t        | j                        t        | j                        k7  r8t        dt        | j                         dt        | j                         d      t        | j                        t        | j                        k7  r8t        dt        | j                         dt        | j                         d      t        | j
                        t        | j                        k7  r8t        dt        | j
                         dt        | j                         d      t        | j                        t        | j                        k7  r8t        dt        | j                         dt        | j                         d      t        | j                        t        | j                        k7  r8t        dt        | j                         dt        | j                         d      t        | j                        }t        | j                        }t        | j                        }|j                  |      s7t        d| j                   d	| j                   d
|j                  |             |j                  |      s7t        d| j                   d| j                   d
|j                  |             |j                  |      s7t        d| j                   d	| j                   d
|j                  |             y)z-Validates the features in this ParseOpParams.z0len(self.dense_shapes) != len(self.dense_keys): r   r&   z/len(self.dense_types) != len(self.dense_keys): z1len(self.sparse_types) != len(self.sparse_keys): z7len(self.ragged_value_types) != len(self.ragged_keys): z7len(self.ragged_split_types) != len(self.ragged_keys): z6Dense and sparse keys must not intersect; dense_keys: z, sparse_keys: z, intersection: z6Dense and ragged keys must not intersect; dense_keys: z, ragged_keys: z8Ragged and sparse keys must not intersect; ragged_keys: N)rW   r]   rZ   r)   r[   rY   rX   r`   r_   ra   set
isdisjointintersection)rc   dense_key_setsparse_key_setragged_key_sets       r   rb   z_ParseOpParams._validateu  s   
4T__!55Id//01c$//6J5K1N O O
4DOO 44Hd../0S5I4J!M N N
4T%5%5!66Jd//01c$:J:J6K5LAO P P
4""#s4+;+;'<<
C(()*$s43C3C/D.EQHI I 4""#s4+;+;'<<
C(()*$s43C3C/D.EQHI I (M))*N))*N##N3
B___T-=-=,>>N''78:; ; ##N3
B___T-=-=,>>N''78:; ; $$^4
Dod.>.>-??O((89;< < 5r   )	NNNNNNNNN)r   r   r   r   rf   classmethodrq   propertyrt   rv   r{   ry   rl   r   r   r   r   r   r   r   rb   r   r   r   rT   rT   a  s    #L   " "&"&"H  : = = P P  $L<-	&;*707 1 7%<r   rT   c           
      2   t        |      }i }t        | j                               D ]  }| |   }t        |t              rt        |j
                  t              r||j
                     }n|j
                  D cg c]  }||   	 }}||j                     }t        j                  |||j                  |j                        ||<   t        |t              s|j                  |n|j                  }||   }	t        |	t        j                        r|	j                  dkD  r|	j                   }
|	j"                  }	nd}
t%        |j&                        D ]  }t)        |	||||j*                  |
      }	 |
jt        j                  j-                  |	|
|j*                        }	n=t%        |j&                        D ]%  }t/        |	|||j0                  |j*                        }	' |	||<    |j3                  |       t5        |      t5        |       z
  D ]  }||=  |S c c}w )a  Creates tensors for SparseFeatures and RaggedFeatures.

  Constructs new dict based on `tensor_dict`.

  For each key in `features` whose value is a `SparseFeature`:

    * Looks up that SparseFeature's value_key and index_keys in tensor_dict.
    * Uses those tensors to construct a single SparseTensor.
    * Stores that SparseTensor in the output dict under the same key.

  For each key in `features` whose value is a `RaggedFeature`:

    * Looks up that RaggedFeature's value_key and partition keys in tensor_dict.
    * Uses those tensors to construct a single RaggedTensor.
    * Stores that RaggedTensor in the output dict under the same key.

  For any other key in `features`:

    * Copies that key and its value from tensor_dict to the output dictionary.

  Args:
    features: A `dict` mapping feature keys to `SparseFeature` or
      `RaggedFeature` values.  Values of other types will be ignored.
    tensor_dict: A `dict` mapping feature keys to `Tensor`, `SparseTensor`, and
      `RaggedTensor` values.  Expected to contain keys of the `SparseFeature`s'
      `index_key`s and `value_key`s and mapping them to `SparseTensor`s.

  Returns:
    A `dict` mapping feature keys to `Tensor`, `SparseTensor`, and
    `RaggedTensor` values. Similar to `tensor_dict` except each `SparseFeature`
    in `features` results in a single `SparseTensor`; and each `RaggedFeature`
    in `features` results in a single `RaggedTensor`.
  )
vocab_sizerH   Nr   r;   )dictrj   rk   r'   rB   rF   r(   r8   r   sparse_mergerG   rH   r   r   RaggedTensorragged_rank
row_splitsvaluesreversedr9   _add_batched_ragged_partitionr;   from_row_splits_add_ragged_partitionr:   updater   )rm   tensor_dictupdatesr   rp   sp_idsrF   	sp_valuesr8   rtouter_splitsr<   s               r   )_construct_tensors_for_composite_featuresr     s   D [!+'HMMO$ )csmG'=)	G%%s	+W../:A:K:KLY+i(LLg//0i,,

\\ //	1gcl
 
G]	+ **2#8I8Iiy!b	B22	3 >>A ,yy",!'"4"45 	;I,RK-0'2B2B-9;"	; #))99,)9)9 : ;" "'"4"45 	QI$RK%,%=%=w?O?OQ"	Q gclS)\ W H- cC	[ Ms   8Hc           	      H   t        |t        j                        rt        | t        j                        rCt        j                  |j                  |      }t        j                  j                  | ||      S t        j                  | t        j                  d|j                  gt        j                  |       dd gd            S t        j                  ||j                     |      }t        |t        j                         r"t        j                  j#                  | ||      S t        |t        j$                        r"t        j                  j'                  | ||      S t        |t        j(                        r"t        j                  j+                  | ||      S t        |t        j,                        r"t        j                  j/                  | ||      S t        |t        j0                        r"t        j                  j3                  | ||      S t5        d|      )	a  Creates a RaggedTensor from a values tensor and a partition tensor.

  Args:
    values: The values tensor for the new RaggedTensor.
    partition: The partition configuration object.  Specifies the key that
      should be used to look up the partition tensor (unless partition is a
      RaggedFeature.UniformRowLength, in which case there is no partition
      tensor).
    tensor_dict: The dictionary mapping keys to tensors.
    row_splits_dtype: The dtype for the partition tensor.
    validate: Whether to validate that the values form a valid RaggedTensor.

  Returns:
    A new RaggedTensor formed from the values and partition tensors.
  r~   r   r   Nr   axisUnhandled partition type )r'   r   r#   r   r   r   r   r$   from_uniform_row_lengthr   r   concatrM   r
   castr   r   r   r   from_row_lengthsr    from_row_startsr!   from_row_limitsr"   from_value_rowidsr)   )r   r<   r   r:   r;   r$   partition_ts          r   r   r     s   " 	=99:&-445$$Y%5%5=MNf''??
&8 @ - - vy'7'7	  !9??6#:12#>
?a(I J J --IMM :<LMK)]445''77
+ 8 2 2	I}77	8''88
+ 9 2 2	I}66	7''77
+ 8 2 2	I}66	7''77
+ 8 2 2	I}88	9''99
+ : 2 2
0>
??r   c           	      	   t        |t        j                        r8| j                  dkD  rt	        j
                  |j                  | j                  j                        }t        j                  j                  t        j                  j                  | j                  ||      | j                  |z  |      S t        j                  | j                  t        j                   d|j                  gt        j"                  | j                        dd gd            }t        j                  j                  || j                  |j                  z  |      S ||j$                     }|j                  j                  | j                  j                  k7  r*t'        j(                  || j                  j                        }g }	|B|r4|	j+                  t-        j.                  ||j                  d|z               |j                  }t	        j0                  |	      5  t        |t        j2                  t        j4                  f      rt        |t        j2                        r|ddddf   }|j                  t        j6                  | j9                         |j;                               z   }
|j=                  t        j                  j?                  | j                  |
|            cddd       S t        |t        j@                        r|j                  t        j6                  | j9                         |j;                               z   }|j=                  t        j                  jC                  | j                  ||            cddd       S t        |t        jD                        rN|j=                  t        j                  jG                  | j                  |j                  |            cddd       S t        |t        jH                        rt'        jJ                  tM        jN                  |dz   d      d      }|j                  t        j6                  t'        jP                  |d	
      |j;                               z   }t        j                  jG                  t        j                  jS                  | j                  ||      ||      cddd       S tU        d|      # 1 sw Y   yxY w)a  Adds a batched ragged partition tensor to a batched ragged tensor.

  Args:
    rt: A RaggedTensor with shape [batch_size, ...].
    partition: The partition configuration object.  Specifies the key that
      should be used to look up the partition tensor (unless partition is a
      RaggedFeature.UniformRowLength, in which case there is no partition
      tensor).  The specified tensor must have shape [batch_size, ...].
    tensor_dict: The dictionary mapping keys to tensors.
    feature_key: The name of the feature being parsed (for error messages).
    validate: Whether to validate that the values form a valid RaggedTensor.
    outer_splits: If not None, then we have two batch dimensions, and this
      is the row-splits for the collapsed batch dimension.  Every partition
      tensor must have an outer row_splits that matches this value.

  Returns:
    A new RaggedTensor where each batch item `rt[i]` has been partitioned
    using the `partition_t[i]`.
  r   r   r   Nr   r   z1Feature %s: values and partitions are not aligned)messageT)	exclusiver   )+r'   r   r#   r   r   r   r$   r   r   r   r   r   r   r   r   r   r   rM   r   r
   r   r   r	   assert_equalcontrol_dependenciesr   r!   repeat
row_startsrow_lengthswith_valuesr   r    r   r   r   r"   maximumr   
reduce_maxcumsumr   r)   )r   r<   r   feature_keyr;   r   r$   reshaped_valsr   checksadjusted_limitsadjusted_startsnrowsadjusted_rowidss                 r   r   r   %  s)   * 	=99:	~~$$Y%5%5r}}7J7JKf''77

$
$
<
<ii( = 4
--6
!	 8    ''		93C3C	  !9??299#=ab#A
B4L Mm''77
)*:*::X 8 O O IMM*+!4!44--R]]-@-@AK&mmI**
..E  $$K
' @)m55+557 8	I}66	7!!QR%(#**Y-=-=
--/;224.6 6o$$

$
$
4
4ii8 5 =>@ @ 
I}66	7#**Y-=-=
--/;224.6 6o$$

$
$
4
4ii8 5 =>@ @  
I}77	8$$

$
$
5
5ii++h 6 @A#@ @( 
I}88	9

$
$[1_1
=qBe#**Y-=-=
//%4
0+2I2I2K.M Mo''88

$
$
6
6ii8 7 =
	 9 3@ @> 0>
???@ @s(   C	S(BSAS-CSSS'c           	      :   |>t        ||      D cg c]'  \  }}t        j                  j                  ||d      ) }}}| j                  dk(  r|S t        ||      D cg c]'  \  }}t        j                  j                  ||d      ) c}}S c c}}w c c}}w )z4Builds RaggedTensors from the outputs of a parse op.Fr   r   )rx   r   r   r   r   )serialized_shaperagged_valuesragged_row_splitsragged_inner_splitsvalsplits         r   _build_ragged_tensorsr   w  s    
 $  /BCS% 	""2232NM  q   /@AS% 	""2232N s   ,B!,BrE   ) r   r>   r   tensorflow.python.frameworkr   r   r   r   r   tensorflow.python.opsr   r	   r
   r   tensorflow.python.ops.raggedr   r   tensorflow.python.platformr    tensorflow.python.util.tf_exportr   r?   r   r   rB   rJ   rO   rT   r   r   r   r   r   r   r   <module>r      s   5  	 3 . + . 4 + + * , 8 6 1 6 ?4F"GH*K**?WIF  I eIKLNeI eIP #5"GHG@KEGG@ IG@T %9;L$MN*,k,,:< * O*& '+-FGI94k448: 9I9By< y<x	Xv*@\ :>O@j /3r   