
    2Vh7                     h    d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZ d dl	m
Z
  G d de
      Zd Zy)	    N)tree)array_slicing)data_adapter_utils)DataAdapterc                       e Zd ZdZ	 	 	 	 	 	 ddZd Zd Zd Zd Zd Z	e
d	        Ze
d
        Ze
d        Ze
d        Zy)ArrayDataAdapterzBAdapter for array-like objects, e.g. TF/JAX Tensors, NumPy arrays.Nc                    t        ||f      st        d|       |t        d      t        j                  |      rt	        t
        t        t        f      r	 t        j                  |       nlt        j                        dk(  xs, t        j                        dk(  xr j                  d   dk(  }|st        d      t        j                  fd	|      |6t        j                  |      rt        d
      t        j                  ||      t        j                  ||      }	t        j                  |	       t!        d t        j"                  |	      D              j%                         }
|
| _        |	| _        |s%|r!t+        t-        j.                  |
|z              nd}t+        t-        j.                  |
|z              | _        || _        |
|z  | _        || _        y # t        $ r t        d| d d      w xY w)NzIExpected all elements of `x` to be array-like. Received invalid types: x=z?You cannot `class_weight` and `sample_weight` at the same time.zgYou should provide one `sample_weight` array per output in `y`. The two structures did not match:
- y: z
- sample_weight: 
      a  For a model with multiple outputs, when providing a single `sample_weight` array, it should only have one scalar score per sample (i.e. shape `(num_samples,)`). If you want to use non-scalar sample weights, pass a `sample_weight` argument with one array per model output.c                     S N )_sample_weights    c/home/dcms/DCMS/lib/python3.12/site-packages/keras/src/trainers/data_adapters/array_data_adapter.py<lambda>z+ArrayDataAdapter.__init__.<locals>.<lambda>@   s    -     zA`class_weight` is only supported for Models with a single output.c              3   :   K   | ]  }|j                   d      yw)r   N)shape).0is     r   	<genexpr>z,ArrayDataAdapter.__init__.<locals>.<genexpr>O   s     C!''!*Cs       )can_convert_arrays
ValueErrorr   	is_nested
isinstancelisttupledictassert_same_structurelenr   map_structurer   class_weight_to_sample_weightspack_x_y_sample_weightcheck_data_cardinalitysetflattenpop_num_samples_inputsintmathceil_size_batch_size_partial_batch_size_shuffle)selfxyr   
batch_sizestepsshuffleclass_weightis_samplewiseinputsnum_sampless      `       r   __init__zArrayDataAdapter.__init__   s!    "1a"78--.C1 
 $' (  ~~a mdE4-@A221mD %((;(;$<$A %M//0A5 8)//2a7 " )(H  %)$6$6/%M #~~a    /MM<M $::1aO11&9Cdll6.BCCGGI' @ETYY{U':;<2J;#;<=
%#.#; e & ($$%3 '00=bB s   G, ,Hc                 p    t        j                  | j                  d      }dd}| j                  ||      S )Nnumpytarget_backendc                 2    | |   }| j                  |      }|S r   )convert_to_numpy	sliceableindicesr5   s      r   slice_and_convert_to_numpyzGArrayDataAdapter.get_numpy_iterator.<locals>.slice_and_convert_to_numpyb   s!    '"A**1-AHr   r   r   convert_to_sliceabler,   _get_iterator)r4   r<   rH   s      r   get_numpy_iteratorz#ArrayDataAdapter.get_numpy_iterator]   s6    33LL
	
 !!"<fEEr   c                    	
 ddl m  j                  
 j                   j                  	t         j                  z        j                  j                  j                  d      }	
fd}|j                  |      j                  d      } fd} fd}|j                  |      }
dk(  r%|j                  j                  j                        } || j                        }j                  j                         }j                  j                   j"                  j$                  |j&                  _        |j+                  |      }|j                  j                  j,                        S )Nr   )
tensorflowr   c                     j                  j                        }r dk7  rj                  j                  |      }|S )N)dtypebatch)rangeint64randomr9   )r   rG   r=   r9   tfs     r   permutationz4ArrayDataAdapter.get_tf_dataset.<locals>.permutation~   s>     hh{"((h;G7g-))++G4Nr   c                 l   z  }j                  | dg|g      }j                  |g      }j                  j                  j	                  |      }j
                  rSj                  j                  j                  j                  | |gj
                  g            }|j                  |      }|S )a  Convert a Tensor of indices into a dataset of batched indices.

            This step can be accomplished in several ways. The most natural is
            to slice the Tensor in a Dataset map. (With a condition on the upper
            index to handle the partial batch.) However it turns out that
            coercing the Tensor into a shape which is divisible by the batch
            size (and handling the last partial batch separately) allows for a
            much more favorable memory access pattern and improved performance.

            Args:
                indices: Tensor which determines the data order for an entire
                    epoch.

            Returns:
                A Dataset of batched indices.
            r   )slicereshapedataDatasetfrom_tensor_slicesr2   from_tensorsconcatenate)	rG   num_in_full_batchfirst_k_indicesflat_datasetindex_remainderr7   num_full_batchesr4   rU   s	        r   slice_batch_indicesz<ArrayDataAdapter.get_tf_dataset.<locals>.slice_batch_indices   s    " !1: = hhw6G5HIO jj"2J!?O 77??==oNL''"$''//">">HH"3!4t7O7O6P#
  ,77Hr   c                 j   t        j                  j                  d      }t        j                  |      }j
                  j                  j                  | j
                  j                  j                  |      j                         f      }fd}|j                  |j
                  j                        }j
                  j                         }d|j                  _        j                  r/j
                  j                   j"                  j$                  |_        |j)                  |      }|S )a%  Slice inputs into a Dataset of batches.

            Given a Dataset of batch indices and the unsliced inputs,
            this step slices the inputs in a parallelized fashion
            and produces a dataset of input batches.

            Args:
                indices_dataset: A Dataset of batched indices.
                inputs: A python data structure that contains the inputs,
                    targets, and possibly sample weights.

            Returns:
                A Dataset of input batches matching the batch indices.
            rN   rA   c                 >      fd}t        j                  ||      S )Nc                     t        | t        j                        rt        j                  |       S t        | t        t
        t        f      ry j                  |       rj                  | d      S | S )Nr   )axis)	r   r   TensorflowSparseWrapperslice_tensorflow_sparse_wrapperr   r    r!   	is_tensorgather)r5   r   rU   s    r   grab_onez[ArrayDataAdapter.get_tf_dataset.<locals>.slice_inputs.<locals>.grab_batch.<locals>.grab_one   sf    !!]%J%JK,LLq   "!dE4%89#||A!yyAAy66Hr   )r   traverse)r   rZ   rm   rU   s   `  r   
grab_batchzIArrayDataAdapter.get_tf_dataset.<locals>.slice_inputs.<locals>.grab_batch   s    	 }}Xt44r   )num_parallel_callsF)r   rJ   r,   r   lists_to_tuplesrZ   r[   zipr]   repeatmapAUTOTUNEOptionsexperimental_optimizationapply_default_optimizationsr3   experimentalExternalStatePolicyIGNORE"experimental_external_state_policywith_options)indices_datasetr<   datasetro   optionsr4   rU   s        r   slice_inputsz5ArrayDataAdapter.get_tf_dataset.<locals>.slice_inputs   s     #77\F ))&1Fggoo)) "''//">">v"F"M"M"OPG5 kkrww/?/? " G ggoo'G --I }}GG((<<CC : **73GNr   rQ   )keras.src.utils.module_utilsrN   r3   r1   r+   r-   rZ   r[   rR   rt   prefetchflat_maprT   r9   r,   rv   ry   AutoShardPolicyDATAexperimental_distributeauto_shard_policyr}   ru   )r4   r~   rV   rd   r   r   r   r7   rc   r=   r9   rU   s   `      @@@@@r   get_tf_datasetzArrayDataAdapter.get_tf_dataseti   s,   A--%%
''t00J>? ''////2	 *--k:CCAF 	 D5	n *223FGg-11"))2C2CDO=''//#GG  0055 	''9 &&w/ 0 011r   c                 p    t        j                  | j                  d      }dd}| j                  ||      S )NjaxrA   c                 2    | |   }| j                  |      }|S r   )convert_to_jax_compatiblerE   s      r   slice_and_convert_to_jaxzCArrayDataAdapter.get_jax_iterator.<locals>.slice_and_convert_to_jax   s!    '"A33A6AHr   r   rI   )r4   r<   r   s      r   get_jax_iteratorz!ArrayDataAdapter.get_jax_iterator   s6    33LL
	
 !!":FCCr   c                    dd l ddlm  G fddj                  j                  j
                        } G fddj                  j                  j                        }| j                  dk(  rL |j                  j                  j                  t        | j                        | j                  d	            }n| j                  rij                  j                  j                  j                  j                  j                  t        | j                              | j                  d	      }nhj                  j                  j                  j                  j                  j                  t        | j                              | j                  d	      }d
 }t        j                  | j                   d      } ||      }j                  j                  j#                  |||      S )Nr   convert_to_tensorc                   $    e Zd Zd Z fdZd Zy);ArrayDataAdapter.get_torch_dataloader.<locals>.ArrayDatasetc                     || _         y r   )array)r4   r   s     r   r>   zDArrayDataAdapter.get_torch_dataloader.<locals>.ArrayDataset.__init__  s	    "
r   c                 R    fd}t        j                  || j                        S )Nc                 D    |    }| j                  |      } |      }|S r   )convert_to_torch_compatible)rF   r5   r   rG   s     r   slice_and_convertzcArrayDataAdapter.get_torch_dataloader.<locals>.ArrayDataset.__getitems__.<locals>.slice_and_convert
  s,    !'*A!==a@A)!,AHr   )r   r$   r   )r4   rG   r   r   s    ` r   __getitems__zHArrayDataAdapter.get_torch_dataloader.<locals>.ArrayDataset.__getitems__	  s"     ))*;TZZHHr   c                 2    t        | j                  d         S Nr   )r#   r   r4   s    r   __len__zCArrayDataAdapter.get_torch_dataloader.<locals>.ArrayDataset.__len__  s    4::a=))r   N)__name__
__module____qualname__r>   r   r   r   s   r   ArrayDatasetr     s    #I*r   r   c                   $    e Zd Zd Z fdZd Zy)AArrayDataAdapter.get_torch_dataloader.<locals>.RandomBatchSamplerc                     || _         y r   )sampler)r4   r   s     r   r>   zJArrayDataAdapter.get_torch_dataloader.<locals>.RandomBatchSampler.__init__  s	    &r   c              3      K   | j                   D ]/  }j                  t        |            D cg c]  }||   	 c} 1 y c c}w wr   )r   randpermr#   )r4   rQ   r   torchs      r   __iter__zJArrayDataAdapter.get_torch_dataloader.<locals>.RandomBatchSampler.__iter__  sA     !\\ IE-2^^CJ-GH58HHIHs   ,AAAc                 ,    t        | j                        S r   )r#   r   r   s    r   r   zIArrayDataAdapter.get_torch_dataloader.<locals>.RandomBatchSampler.__len__  s    4<<((r   N)r   r   r   r>   r   r   )r   s   r   RandomBatchSamplerr     s    'I)r   r   rQ   F)r7   	drop_lastc                     | S r   r   )rQ   s    r   no_op_collatez<ArrayDataAdapter.get_torch_dataloader.<locals>.no_op_collate7  s    Lr   r   rA   )batch_sampler
collate_fn)r   keras.src.backend.torch.corer   utilsrZ   r[   Samplerr3   BatchSamplerrR   r+   r1   RandomSamplerSequentialSamplerr   rJ   r,   
DataLoader)	r4   r   r   r   r   r<   r   r   r   s	          @@r   get_torch_dataloaderz%ArrayDataAdapter.get_torch_dataloader   s   B	*5;;++33 	* 		)!1!1!9!9 		) ==G#.  --$++,#//# . M ]]!KK,,99  ..uT5F5F/GH++ : M "KK,,99  2259J9J3KL++ : M	 33LL
 v&{{**=] + 
 	
r   c              #   :  K   d }| j                   r8| j                   dk7  r)t        j                  j                  | j                        }t        | j                        D ]  }|| j                  z  }t        |dz   | j                  z  | j                        }| j                   dk(  r&t        j                  j                  ||z
        |z   }n| j                   r||| }nt        ||      }t        j                  ||      }t        j                  ||        y w)NrQ   r   )rG   )r3   nprT   rV   r+   rR   r0   r1   minrX   	functoolspartialr   r$   )	r4   slice_and_convert_fnr<   global_permutationr   startstoprG   slice_indices_and_convert_fns	            r   rK   zArrayDataAdapter._get_iteratorB  s     !==T]]g5!#!6!6t7H7H!Itzz" 	KA(((EA!1!1143D3DED}}'))//u=E,U48t,+4+<+<$g,( $$%A6JJ	Ks   DDc                     | j                   S r   )r0   r   s    r   num_batcheszArrayDataAdapter.num_batchesV  s    zzr   c                     | j                   S r   )r1   r   s    r   r7   zArrayDataAdapter.batch_sizeZ  s    r   c                      | j                   dkD  S r   r2   r   s    r   has_partial_batchz"ArrayDataAdapter.has_partial_batch^  s    ''!++r   c                 "    | j                   xs d S r   r   r   s    r   partial_batch_sizez#ArrayDataAdapter.partial_batch_sizeb  s    ''/4/r   )NNNNFN)r   r   r   __doc__r>   rL   r   r   r   rK   propertyr   r7   r   r   r   r   r   r   r      s    L
 L \
FI2V
D@
DK(       , , 0 0r   r   c                     t        t        j                  t        j                  t        j
                  |                   S )zCheck if array like-inputs can be handled by `ArrayDataAdapter`

    Args:
        inputs: Structure of `Tensor`s, NumPy arrays, or tensor-like.

    Returns:
        `True` if `arrays` can be handled by `ArrayDataAdapter`, `False`
        otherwise.
    )allr   r)   r$   r   can_slice_array)arrayss    r   r   r   g  s0     T''(E(EvNO r   )r   r.   r@   r   	keras.srcr    keras.src.trainers.data_adaptersr   r   -keras.src.trainers.data_adapters.data_adapterr   r   r   r   r   r   <module>r      s.        : ? EX0{ X0v
r   