
    BVhTt                        d Z ddlZddlmZ ddlmZ ddlmZ ddlm	Z	 ddl
mZ ddl
mZ dd	l
mZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m!Z! ddl m"Z" ddl#m$Z% d Z&d!dZ'd Z(	 	 	 	 	 	 	 	 d"dZ)	 	 	 d#dZ*	 	 	 d#dZ+ G d dejX                        Z-d Z. G d d ejX                        Z/y)$zBPart of the Keras training engine related to distributed training.    N)distribute_lib)	input_lib)reduce_util)context)constant_op)errors)ops)backend)	callbacks)distribute_coordinator_utils)distributed_training_utils_v1)partial_batch_padding_handler)training_arrays_v1)training_utils_v1)Progbar)ModeKeys)	array_ops)control_flow_ops)
tf_loggingc                     | j                  |      }|j                  |j                  |j                  |j                  fS N)_make_execution_functioninputsoutputs
updates_opsession_kwargs)modelmode	exec_funcs      f/home/dcms/DCMS/lib/python3.12/site-packages/tensorflow/python/keras/engine/training_distributed_v1.py_per_replica_execution_functionr!   (   s>    ,,T2)


I--y/C/C

"
"
$ $    c                     |j                   rt        j                  || |||       y t        j                  || |||       y )N)r   targets)_compile_distribution
dist_utilsclone_model_on_replicas_build_distributed_network)strategyr   r   r   r$   s        r    _build_modelr*   .   s?    
  &&xfg? ))%4*13r"   c                       fd}|S )a1  Create step fn.

  Args:
    model: a Keras Model instance.
    mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT.
    strategy: a `tf.distribute.Strategy` instance.
    output_labels: the output labels for the step function.

  Returns:
    A step function to run by `tf.distribute.Strategy`.
  c                    t        |t        t        f      rt        |      dk(  r|\  }}nd}t        |t              rj
                  D cg c]  }||   	 }}t        ||       j                  j                  t        t        j                        f      \  }}}}t        j                  ||||      \  }}	}
}t        j                  ||	f|
dt              z   dz   d|}t!        |j"                        D ]R  \  }}|dk(  rt$        j&                  j(                  }nt$        j&                  j*                  }| j-                  |||       T |j.                  S c c}w )z"A step fn that returns update ops.   Nargsdistributed_	_function)updatesnameloss)
isinstancetuplelistlendict_feed_input_namesr*   extendedcall_for_each_replicar!   r&   get_distributed_modelunwrap_valuesr
   functionstrzipr   ds_reduce_utilReduceOpSUMMEANset_last_step_outputr   )ctxr   r$   
input_namegrouped_inputsgrouped_outputsgrouped_updatesgrouped_session_args
all_inputsall_outputsall_updatesall_session_argscombined_fnlabeloutput	reduce_opr   r   output_labelsr)   s                   r    _step_fnz%_make_train_step_fn.<locals>._step_fnD   s   &5$-(S[A-=ofgg &$5:5L5LMzz"MfM5$8 &..DD	(//t<dC E E^_o $11(N2A2A2FHZk "" c$i'+5	
 K ]K,?,?@ 9v	&"++//	 #++00		ufi89 !!!A Ns   E# )r   r   r)   rU   rV   s   ```` r    _make_train_step_fnrX   7   s    -"^ 
/r"   c
           
      ~	   t         j                  }
| j                  }t        ||j                  j
                        }t        j                  |dd      }t        j                  ||      }t        j                  |d      }|j                          | j                  xs g }t        | t         j                  ||      }i }t        j                  d      |d<   | j!                         D ]I  }|j#                         }t%        j&                  |j(                  |j*                        ||j,                  <   K |j                  j/                  ||||      }|j0                  }|j2                  }t5        |      }| j6                  rt        j8                  | |
       t;        j<                  || ||||d	|

      }|j                  j
                  g||j                  j
                  z  z  }||j                  j
                  z  r(|j?                  ||j                  j
                  z         tA        |      }|jC                  |
       | jE                  ||
      }tG        ||      D ]  }t        jH                  |        |jK                  |       i }d}d}d}||k  r||   } |d| d}!|jM                  |
d||!       || |k7  r4t        jN                         jQ                  |jS                  |              | }	 t        jT                  ||g      \  }"}#|!j_                  |#       |jM                  |
d||!       || z   }|dz  }|j`                  jb                  rn||k  r|rte        jf                  |	|      rt[        jh                  d|       | j6                  r$t        jj                  | t         j                         tm        | ||||      }$to        |$tp              s|$g}$ts        ||$      D ]  \  }%}&|&|d|%z   <    |ju                  ||       |j`                  jb                  s n d| _;        |jy                  |
       | j6                  r$t        jj                  | t         j                         |j{                  ddd       | j|                  S # tV        jX                  $ r t[        j\                  d|z  |z         Y bw xY w)a  Fit loop for training with TPU tf.distribute.Strategy.

  Args:
      model: Keras Model instance.
      dataset: Dataset that returns inputs and targets
      epochs: Number of times to iterate over the data
      verbose: Integer, Verbosity mode, 0, 1 or 2
      callbacks: List of callbacks to be called during training
      initial_epoch: Epoch at which to start training
          (useful for resuming a previous training run)
      steps_per_epoch: Total number of steps (batches of samples)
          before declaring one epoch finished and starting the
          next epoch. Ignored with the default value of `None`.
      val_dataset: Dataset for validation data.
      validation_steps: Number of steps to run validation for
          (only if doing validation from data tensors).
          Ignored with the default value of `None`.
      validation_freq: Only relevant if validation data is provided. Integer or
          `collections.abc.Container` instance (e.g. list, tuple, etc.). If an
          integer, specifies how many training epochs to run before a new
          validation run is performed, e.g. `validation_freq=2` runs
          validation every 2 epochs. If a Container, specifies the epochs on
          which to run validation, e.g. `validation_freq=[1, 2, 10]` runs
          validation at the end of the 1st, 2nd, and 10th epochs.

  Returns:
      Returns `None`.

  Raises:
      ValueError: in case of invalid arguments.
  int32steps_per_run)valuedtyper3      r)   learning_phaseg    cAr4   )
iterationsinitial_loop_valuesstepsdo_validationepochssteps_per_epochverbose
count_moder   r   N)batchsize	num_stepsbeginzYour dataset iterator ran out of data; interrupting training. Make sure that your dataset can generate at least `steps_per_epoch * epochs` batches (in this case, %d batches).endz#Running validation at fit epoch: %s)rc   rh   r   val_T)?r   TRAIN_distribution_strategyminr;   r[   r
   variabler&   get_iteratordistributed_scope	__enter__metrics_namesrX   r   constant_get_training_eval_metricsresultr   zerosshaper]   r3   "experimental_run_steps_on_iteratorrun_oplast_step_outputsboolr%   "_copy_weights_to_distributed_modelcbksconfigure_callbacksappendr8   _call_begin_hook#_maybe_load_initial_epoch_from_ckptrange_reset_metricson_epoch_begin_call_batch_hookget_sessionrunassignbatch_get_valuer   OutOfRangeErrorloggingwarningupdater   stop_trainingr   should_run_validationinfo_copy_weights_to_original_modelexperimental_tpu_test_loopr5   r7   rA   on_epoch_end_successful_loop_finish_call_end_hook__exit__history)'r   datasetrf   rh   r   initial_epochrg   val_datasetvalidation_stepsvalidation_freqr   current_strategyiteration_valuer[   iteratorscope
out_labelsstep_fnrb   mtensorrG   train_opoutput_tensorsre   steps_to_runtarget_stepsepoch
epoch_logs
step_indexprev_step_countcurrent_step
step_count
batch_logs_r   val_outsrR   val_outs'                                          r    experimental_tpu_fit_loopr   v   s   R 
$11(11??A/""- $$W.>?(

&
&3%//""(b*x~~7G *,'  + 4 4S 9f++- NaXXZF"+//&,,"MN 	!!DDxM- 	E 	/# ZZ(((.'(-
  11%>&&!%) $,,::;"#,,::;<, '00>>>*33AAAC\",T";;M4P-]F+ 7ee$U#JJOL

%-j'Lj  w
JG		 J/$A!!-"6"6z"BC$,,h-GH
7    uj*E
*jal		&	&/ 
%2 	//Gll8%@		$	$ 	225(..I+

 h $':
H5 -.%%,
6E>"- 5*-$$o7p #'%4 
  ..uhnnE..tT"	_ ##  > (	( +1	1 	2
 	s   3R

.R<;R<c           
          t         j                   j                  }t        j                  ||      }t        j
                  |d      }|j                           j                  } fd}	|j                         }
|j                  |	|
f      }i }t        ||      D ]V  \  }}|dk(  rt        j                  j                  }nt        j                  j                  }|j                  ||d      ||<   X t!        j"                  t%        |j'                                     }|dk\  rt)        |	      } j*                  rt        j,                          t        j.                          t1        j2                  | d
d||dt         j                        }|j5                         dgt7         j                        z  }||}nt9        d      d}||k  r|dd}|j;                  d||       	 t=        j>                  ||g      \  }}tK         j                        D ]#  \  }}|dk(  r||xx   ||   z  cc<   ||   ||<   % t1        jL                   ||      }|j;                  d||       |dk(  rjO                  |dz          |dz  }||k  r|dk\  rjO                  |       |jQ                         |jS                  ddd       t7        |      dk\  r|dxx   |z  cc<   t7        |      dk(  r|d   S |S # t@        jB                  $ r. djE                  |      }tG        jH                  d|z          |}Y w xY w)a  Test loop for evaluating with TPU tf.distribute.Strategy.

  Args:
      model: Keras Model instance.
      dataset: Dataset for input data.
      verbose: Integer, Verbosity mode 0 or 1.
      steps: Total number of steps (batches of samples)
          before declaring predictions finished.
          Ignored with the default value of `None`.
      callbacks: List of callbacks to be called during training

  Returns:
      Scalar loss (if the model has a single output and no metrics)
      or list of scalars (if the model has multiple outputs
      and/or metrics). The attribute `model.metrics_names` will give you
      the display labels for the outputs.
  r   r_   c                    t        | t        t        f      rt        |       dk(  r| \  } }nd}t	        j
                         j                  t        | |f       t        t        j                              \  }}}}t        j                  |g      5  |D cg c]  }t        j                  |       c}cddd       S c c}w # 1 sw Y   yxY w)z-A fn that returns output of single test step.r-   Nr.   )r5   r6   r7   r8   r   get_replica_context
merge_callr*   r!   r&   r=   r	   control_dependenciesr   identity)r   r$   r   r   r2   outr   r   s         r    _test_step_fnz1experimental_tpu_test_loop.<locals>._test_step_fn@  s    &5$-(S[A-=ofgg'')44E49 5 ;  ?((5t =Q!		!	!7)	, :189#i  %9: :9: :s   CC7CCCr.   r4   Naxisr^   targetFrc   rd           TNumber of steps could not be inferred from the data, please pass the steps argument.rj   rk   rm   ]Make sure that your dataset can generate at least `steps` batches (in this case, {} batches).@Your dataset iterator ran out of data; interrupting evaluation. rn   )*r   TESTrq   r&   rt   ru   rv   rw   get_nextr   rA   rB   rC   rD   rE   reducer   groupr7   valuesr   r%   r   r   r   r   r   r8   
ValueErrorr   r
   r   r   r   formatr   r   	enumerate	make_logsr   r   r   )r   r   rh   rc   r   r   r   r   r   r   test_input_dataper_replica_outputsr   rR   rS   rT   test_opprogbaroutsr   r   r   r   
batch_outswarning_msgir   s   `                         @r    r   r      s^   , 
$11$$W.>?(

&
&3%//""*: %%'/(,,?, - ..:':; ?meV ))--i !))..i,33Iv9= 4 ?N5? ""4(=(=(?#@A'\U#G
  11%>E"&&==) T"
E''(	($
L
 7 8 8 ,|#'3JtWlJG
--w.GHma e112 $5	
aQ:e$$ U#Q$ z4>JtUL*E!|nn\A%&AL5 	|#8 \NN< 4 ..tT"Y!^GGY!^7N	+E !! 88>u  oo 24?@ A!ls   3L >MMc           
           t         j                   t        j                  |      }d}|st	        j
                   j                        }t        j                  |      \  }}}	||_	        |j                  |j                  |j                        |_        |j                  |j                        }|j                         }|j!                  |d      }|	|j#                  |	      } j$                  }
t        j&                  ||
      }t        j(                  |
d      }|j+                            fd}|j-                         }|
j/                  ||f      }t        j0                  |
|      }|dk\  rt3        |	      } j4                  rt        j6                           t        j8                          t;        j<                  | d
d||d       }|j?                          tA         jB                        }tE        |      D cg c]  }g  }}||}ntG        d      d}||k  r|dd}|jI                   d||       	 tK        jL                  |      }tO        jP                  ||g      \  }}tE        |      D ]9  }||
j\                  z  }||
j\                  z   }||| }||   j_                  |       ; t;        j`                   ||       }|jI                   d||       |dk(  rjc                  |dz          |dz  }||k  r|dk\  rjc                  |       |je                          |jg                  ddd       tA        |      dk(  rti        jj                  |d   d      }n$|D cg c]  }ti        jj                  |d       }}|r|jm                  |      }|S c c}w # tR        jT                  $ r, djW                  |      }tY        jZ                  d|z          Y w xY wc c}w )a>  Predict loop for predicting with TPU tf.distribute.Strategy.

  Args:
      model: Keras Model instance.
      dataset: Dataset for input data.
      verbose: Integer, Verbosity mode 0 or 1.
      steps: Total number of steps (batches of samples)
          before declaring `_predict_loop` finished.
          Ignored with the default value of `None`.
      callbacks: List of callbacks to be called during training

  Returns:
      Array of predictions (if the model has a single output)
      or list of arrays of predictions
      (if the model has multiple outputs).
  NT)drop_remainderr   r_   c                 N   t        j                         j                  t        | f       t	        t        j                              \  }}}}t        j                  |g      5  |D cg c]  }t        j                  |       c}cddd       S c c}w # 1 sw Y   yxY w)z3A fn that returns output of single prediction step.r.   N)r   r   r   r*   r!   r&   r=   r	   r   r   r   )r   r   r   r2   r   r   r   s        r    _predict_step_fnz7experimental_tpu_predict_loop.<locals>._predict_step_fn  s     '')44E40 5 2  ?((5t =Q! 
	!	!7)	, :189#i  %9: :9: :s   )B.B
BBB$r.   r^   r   Frc   rd   r   r   rm   r   r   rn   r   )7r   PREDICTr&   is_dataset_shape_fully_definedpadding_utilPartialBatchPaddingHandler_feed_output_shapesr   _get_dataset_attributespadded_batch_sizer   padding_maskupdate_maskmap	pad_batchunbatchrj   prefetchrq   rt   ru   rv   r   r   flatten_per_replica_valuesr   r%   r   r   r   r   r   r8   output_namesr   r   r   r   r   r
   r   r   r   r   r   r   num_replicas_in_syncextendr   r   r   r   npconcatenate
apply_mask)!r   r   rh   rc   r   dataset_fully_shapedpadding_handler
batch_sizer   prefetch_bufferr   r   r   r   predict_input_datar   r   r   num_model_outputsunconcatenated_outsr   r   r   predict_opsr   r   r   output_start_indexoutput_end_indexsingle_model_outputprediction_resultr   r   s!   `                               @r    experimental_tpu_predict_loopr     s   * 
		$#BB7K/	 #==!!#O%.%F%Fw%O"J?(2O%#*>>/2N2N2A2M2M$OO  kk/334GooG mmJtm<G"  1g11$$W.>?(

&
&3%//
:  ((*(,,02 - 488+-. \U#G
  11%>E"&&) T"
 %,,-%*+<%=>>>
L
 7 8 8 ,|#'3JtWlJG$**>:k--{N.KLma $% 9/DDD
/DD
D &'9:JK!##$789 z:tDJtUL*E!|nn\A%&AL9 	|#< \NN< 4 ..tT"	"':1'=AF 0C(+s#  '223DE	q ? !! 88>u  oo 24?@ ADs   	N&/N+ 1O-+<O*)O*c                   \    e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZ	 	 	 	 	 	 	 ddZ	 	 	 	 ddZy)	$DistributionSingleWorkerTrainingLoopz;Training loop for distribution strategy with single worker.Nc                 2   t        j                  ||j                         t        j                  ||       t        j                  |j
                  |||t        j                  |      \  }}|j                  |||      }|j                  |||||||
|      }t        j                  |      s>|j
                  j                         5  |j                  ||||||
      \  }}}ddd       d}|	r}t        j                  |	      \  }}}t        j                  ||       t        j                  |j
                  |||t        j                        \  }}|j                  |||d|||
d      }n|rt!        d      t#        j$                  |j
                        rPt        j&                  ||||d	
      }|t!        d      t)        j*                         st-        ||||||||||
      S t/        j0                  ||||||||
||||d	      S # 1 sw Y   )xY w)z%Fit loop for Distribution Strategies.)input_callbacks	optimizer)validation_split)sample_weightclass_weightr   r  shufflerf   )r  r  r   r  r  NT)r  r  r   r  r  allow_partial_batchzHvalidation_split argument is not supported with distribution strategies.rg   
steps_namez^Number of steps could not be inferred from the data, please pass the steps_per_epoch argument.)rf   rh   r   r   r   rg   r   r   )r   rf   rh   r   
val_inputsr  r   rg   r   r   r  )r&   validate_callbacksr  validate_inputsprocess_batch_and_step_sizerq   r   rp   _validate_or_infer_batch_size#_distribution_standardize_user_datais_distributing_by_cloningr   _standardize_user_datar   unpack_validation_datar   r   r
   is_tpu_strategyinfer_steps_for_datasetr   executing_eagerlyr   r   fit_loop)selfr   xyr   rf   rh   r   r  validation_datar  r  r  r   rg   r   r   kwargsr   r   r   val_xval_yval_sample_weightss                           r    fitz(DistributionSingleWorkerTrainingLoop.fitA  so   & !!),1OO=q!$","H"H$$	)#+J 44OQ(J77	1#!) 8 G 007''--/ 66'%!- 7 !Q K

2
2?
C 'eU&  .&BB

&
&z;K
--a	 ==
*+" > $k 
 2 3 3 u;;<)AA
/6>OQo		  E F 	F &&((#'+-+
- 
	- &&#')'$& &g s   	HHc	                    t        j                  ||       t        j                  |j                  |||t        j
                        \  }}|j                  |||      }|j                  ||||d      }
t        j                  |j                        rJt        j                  ||
|d      }|t        d      t        j                         st        ||
|||      S t!        j"                  ||
||||      S )z*Evaluate loop for Distribution Strategies.T)r  r   r	  rc   r
  r   rh   rc   r   )r   r   rh   rc   r   )r&   r  r  rq   r   r   r  r  r
   r  r   r  r   r   r  r   r   	test_loop)r  r   r  r  r   rh   r  rc   r   r  r   s              r    evaluatez-DistributionSingleWorkerTrainingLoop.evaluate  s    q!$">>$$aUHMMKJ44ZJJ77	1# 	 8 "G u;;<77
%G5e	 ; < 	< &&()7G5IO 	O '' r"   c                    t        j                  |d       t        j                  |j                  |||t        j
                        \  }}|j                  |||      }|j                  ||d      }t        j                  |j                        rJt        j                  |||d      }|t        d      t        j                         st        |||||      S t!        j"                  ||||||	      S )
z)Predict loop for Distribution Strategies.N)r  r  T)r   r	  rc   r
  r   r#  )r   rh   rc   r   )r&   r  r  rq   r   r   r  r  r
   r  r   r  r   r   r  r   r   predict_loop)	r  r   r  r   rh   rc   r   r  r   s	            r    predictz,DistributionSingleWorkerTrainingLoop.predict  s    d+">>$$aUH<L<LNJ44ZJJ77	  8 "G u;;<77
%G5e	 ; < 	<&&(,7G5IO 	O** r"   )NNNr^   r^   Nr   NTNNr   NNr^   )NNNr^   NNN)Nr   NN)__name__
__module____qualname____doc__r!  r%  r(  rW   r"   r    r  r  >  sk    C !i&Z !'X  r"   r  c                       fd}|S )zHDecorator that handles multi worker training with distribution strategy.c                 V      fd}t        j                  | j                        S )Nc                 p    j                  dd       }t        j                  |      }|d<    fi S )Nr   )popr&   filter_distributed_callbacks)r   r   filtered_callbacksr  methodr   s      r    
_worker_fnz=_train_with_multi_worker.<locals>.wrapper.<locals>._worker_fn  sC    **[$/i%BB
U.f[E$V$$r"   )dcrun_distribute_coordinatorrq   )r   r  r4  r3  s   `` r    wrapperz)_train_with_multi_worker.<locals>.wrapper  s)    % (($$& &r"   rW   )r3  r7  s   ` r    _train_with_multi_workerr8    s    
& 
.r"   c                   (    e Zd ZdZd Zd Zd Zd Zy)#DistributionMultiWorkerTrainingLoopz=Training loop for distribution strategy with multiple worker.c                     || _         y r   )_single_worker_loop)r  single_worker_loops     r    __init__z,DistributionMultiWorkerTrainingLoop.__init__  s
    1Dr"   c                 L     t        | j                  j                        |i |S r   )r8  r<  r!  r  r/   r  s      r    r!  z'DistributionMultiWorkerTrainingLoop.fit  s.    A#D$<$<$@$@A	 r"   c                 L     t        | j                  j                        |i |S r   )r8  r<  r%  r@  s      r    r%  z,DistributionMultiWorkerTrainingLoop.evaluate  s.    F#D$<$<$E$EF	 r"   c                 :     | j                   j                  |i |S r   )r<  r(  r@  s      r    r(  z+DistributionMultiWorkerTrainingLoop.predict  s     +4##++T<V<<r"   N)r)  r*  r+  r,  r>  r!  r%  r(  rW   r"   r    r:  r:  
  s    E2=r"   r:  r   )d   r^   Nr   NNNr^   )r   NN)0r,  numpyr   tensorflow.python.distributer   r   r   rB   tensorflow.python.eagerr   tensorflow.python.frameworkr   r   r	   tensorflow.python.kerasr
   r   r   "tensorflow.python.keras.distributer   r5  r   r&   tensorflow.python.keras.enginer   r   r   r   +tensorflow.python.keras.utils.generic_utilsr   'tensorflow.python.keras.utils.mode_keysr   tensorflow.python.opsr   r   tensorflow.python.platformr   r   r!   r*   rX   r   r   r   TrainingLoopr  r8  r:  rW   r"   r    <module>rP     s    I  7 2 F + 3 . + + 5 Q Z X = < ? < + 2 <$3<B &)&'(,,-.2*./3./gX ()%))-	@J +,(,,0	Xvw+<+I+I wt$=*;*H*H =r"   