
    AVhy                     b   d Z ddlZddlmZ ddlmZ ddlmZ ddlmZ ddlm	Z	 ddlm
Z
 dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddlm!Z! ddl"m#Z# ddl"m$Z$ ddl"m%Z% ddl"m&Z& ddl'm(Z) ddl*m+Z+ dd l,m-Z- d! Z.d" Z/d# Z0d$ Z1d% Z2d0d&Z3d' Z4 e-d(g )       G d* d+e
jj                               Z6 e-d(g)       G d, d-e
jn                               Z8 G d. d/e
jr                        Z:y)1z;Class MirroredStrategy implementing tf.distribute.Strategy.    N)tf2)collective_util)cross_device_ops)cross_device_utils)device_util)distribute_lib)distribute_utils)	input_lib)
input_util)mirrored_run)multi_worker_util)numpy_dataset)reduce_util)values)values_util)tfconfig_cluster_resolver)context)record)config)constant_opdevice)dtypes)ops)	array_ops)control_flow_ops)control_flow_util)
while_loop)
tf_logging)nest)	tf_exportc                    g }| D ]X  }t        |t        j                        r|j                  n|}|j	                  t
        j                  j                  |             Z t        |D ch c]%  }|j                  |j                  |j                  f' c}      }t        d |D              }t        d |D              }|r|st        d      |dk(  r|st        d |D              rt        d      |dk(  S c c}w )af  Checks whether the devices list is for single or multi-worker.

  Args:
    devices: a list of device strings or tf.config.LogicalDevice objects, for
      either local or for remote devices.

  Returns:
    a boolean indicating whether these device strings are for local or for
    remote.

  Raises:
    ValueError: if device strings are not consistent.
  c              3   8   K   | ]  }|j                   d v   yw)N	localhostNjob.0ds     ^/home/dcms/DCMS/lib/python3.12/site-packages/tensorflow/python/distribute/mirrored_strategy.py	<genexpr>z0_is_device_list_single_worker.<locals>.<genexpr>I        >1!%%..>   c              3   8   K   | ]  }|j                   d v   ywr$   r&   r(   s     r+   r,   z0_is_device_list_single_worker.<locals>.<genexpr>J   r-   r.   a  Local device should have only 'localhost' in the job field in device string. E.g. 'job:localhost' in /job:localhost/replica:0/task:0/device:CPU:0Devices cannot have mixed list of device strings containing both localhost and other job types such as worker, ps etc.    c              3   8   K   | ]  }|j                   d u   y wN)taskr(   s     r+   r,   z0_is_device_list_single_worker.<locals>.<genexpr>V   s     
)a166T>
)r.   zhRemote device string must have task specified.E.g. 'task:0' in /job:worker/replica:0/task:0/device:CPU:0)
isinstancer   LogicalDevicenameappend	tf_device
DeviceSpecfrom_stringlenr'   r3   replicaallany
ValueError)devicesspecsr*   r6   num_workers	all_local	any_locals          r+   _is_device_list_single_workerrE   6   s     % 9a7#8#89166qD	LL%%11$789 ?AaeeQVVQYY/?@+>>>)>>>)y
 ( ) ) Ai

)5
)) C D D 
	' @s   )*C6c                 N   t        j                  |       } g }dD ]  }t        t        | j	                         j                  |g                   D ]L  }|dk(  r|j                  d||fz         |j                  t        |      D cg c]
  }d|||fz   c}       N  |S c c}w )z+Returns a device list given a cluster spec.chiefworkerr   z/job:%s/task:%d/device:CPU:0z/job:%s/task:%d/device:GPU:%i)r   normalize_cluster_specranger;   as_dictgetr7   extend)cluster_specnum_gpus_per_workerr@   	task_typetask_idgpu_ids         r+   _cluster_spec_to_device_listrT   ^   s    "99,G,'& i\11377	2FGH 		!5G8LLM 34
 ,y'6.JJ
 		 
.	
s   B"c                    t        |       rJ i }| D ]  }t        j                  j                  |      }|j                  |vrg ||j                  <   t        ||j                           |j                  k  rD||j                     j                  g        t        ||j                           |j                  k  rD||j                     |j                     j                  |        |S )a  Groups the devices list by task_type and task_id.

  Args:
    devices: a list of device strings for remote devices.

  Returns:
    a dict of list of device strings mapping from task_type to a list of devices
    for the task_type in the ascending order of task_id.
  )rE   r8   r9   r:   r'   r;   r3   r7   )r@   device_dictr*   d_specs       r+   _group_device_listrX   n   s     +733	3+ 3a!!--a0F zz$ "k&** k&**%
&&++
5&**$$R( k&**%
&&++
5 

FKK(//23 
    c                 Z    t         j                  j                  |       j                  dk(  S )NGPU)r8   r9   r:   device_typer   s    r+   _is_gpu_devicer]      s$    				)	)&	1	=	=	FFrY   c                    t        |       rt        d | D              S t        |       }d}|j                         D ]  \  }}|D ]  }|t        d |D              }n |t        d |D              k7  rt	        d      |D ]K  }t
        j                  j                  |      }|j                  dk(  s2|j                  |k\  sBt	        d         |S )a  Infers the number of GPUs on each worker.

  Currently to make multi-worker cross device ops work, we need all workers to
  have the same number of GPUs.

  Args:
    devices: a list of device strings, can be either local devices or remote
      devices.

  Returns:
    number of GPUs per worker.

  Raises:
    ValueError if workers have different number of GPUs or GPU indices are not
    consecutive and starting from 0.
  c              3   8   K   | ]  }t        |      sd   ywr0   Nr]   r(   s     r+   r,   z-_infer_num_gpus_per_worker.<locals>.<genexpr>   s     7Q^A%6q7   Nc              3   8   K   | ]  }t        |      sd   ywr`   ra   r(   s     r+   r,   z-_infer_num_gpus_per_worker.<locals>.<genexpr>   s     HqnQ6GHrb   c              3   8   K   | ]  }t        |      sd   ywr`   ra   r(   s     r+   r,   z-_infer_num_gpus_per_worker.<locals>.<genexpr>   s     L1.:KQLrb   z0All workers should have the same number of GPUs.r[   zFGPU `device_index` on a worker should be consecutive and start from 0.)
rE   sumrX   itemsr?   r8   r9   r:   r\   device_index)r@   rV   num_gpus_devices_in_taskdevice_in_taskr*   rW   s           r+   _infer_num_gpus_per_workerrl      s    " #7+7'777$W-KH)//1 >?+ >.HNHH( LLLLMN
N 	>A''33A6&  E)!!X- = > >		>>> OrY   c                 l    t        j                  d      }| |d |  }|xs t        j                  d      S )Nr[   CPU)r   list_logical_devices)rh   r@   s     r+   all_local_devicesrp      s:    ''.'ix G		6F//66rY   c                      g } t        j                         }|j                         j                         r,t	        |j                         t        j                               } | r| S t               S r2   )r   TFConfigClusterResolverrO   rL   rT   r   rh   rp   )r@   tfconfigs     r+   all_devicesrt      s\    '&>>@($$&*8+@+@+B+2+;+;+=?G4!2!44rY   zdistribute.MirroredStrategy)v1c                   (     e Zd ZdZdZd fd	Z xZS )MirroredStrategya  Synchronous training across multiple replicas on one machine.

  This strategy is typically used for training on one
  machine with multiple GPUs. For TPUs, use
  `tf.distribute.TPUStrategy`. To use `MirroredStrategy` with multiple workers,
  please refer to `tf.distribute.experimental.MultiWorkerMirroredStrategy`.

  For example, a variable created under a `MirroredStrategy` is a
  `MirroredVariable`. If no devices are specified in the constructor argument of
  the strategy then it will use all the available GPUs. If no GPUs are found, it
  will use the available CPUs. Note that TensorFlow treats all CPUs on a
  machine as a single device, and uses threads internally for parallelism.

  >>> strategy = tf.distribute.MirroredStrategy(["GPU:0", "GPU:1"])
  >>> with strategy.scope():
  ...   x = tf.Variable(1.)
  >>> x
  MirroredVariable:{
    0: <tf.Variable ... shape=() dtype=float32, numpy=1.0>,
    1: <tf.Variable ... shape=() dtype=float32, numpy=1.0>
  }

  While using distribution strategies, all the variable creation should be done
  within the strategy's scope. This will replicate the variables across all the
  replicas and keep them in sync using an all-reduce algorithm.

  Variables created inside a `MirroredStrategy` which is wrapped with a
  `tf.function` are still `MirroredVariables`.

  >>> x = []
  >>> @tf.function  # Wrap the function with tf.function.
  ... def create_variable():
  ...   if not x:
  ...     x.append(tf.Variable(1.))
  ...   return x[0]
  >>> strategy = tf.distribute.MirroredStrategy(["GPU:0", "GPU:1"])
  >>> with strategy.scope():
  ...   _ = create_variable()
  ...   print(x[0])
  MirroredVariable:{
    0: <tf.Variable ... shape=() dtype=float32, numpy=1.0>,
    1: <tf.Variable ... shape=() dtype=float32, numpy=1.0>
  }

  `experimental_distribute_dataset` can be used to distribute the dataset across
  the replicas when writing your own training loop. If you are using `.fit` and
  `.compile` methods available in `tf.keras`, then `tf.keras` will handle the
  distribution for you.

  For example:

  ```python
  my_strategy = tf.distribute.MirroredStrategy()
  with my_strategy.scope():
    @tf.function
    def distribute_train_epoch(dataset):
      def replica_fn(input):
        # process input and return result
        return result

      total_result = 0
      for x in dataset:
        per_replica_result = my_strategy.run(replica_fn, args=(x,))
        total_result += my_strategy.reduce(tf.distribute.ReduceOp.SUM,
                                           per_replica_result, axis=None)
      return total_result

    dist_dataset = my_strategy.experimental_distribute_dataset(dataset)
    for _ in range(EPOCHS):
      train_result = distribute_train_epoch(dist_dataset)
  ```

  Args:
    devices: a list of device strings such as `['/gpu:0', '/gpu:1']`.  If
      `None`, all available GPUs are used. If no GPUs are found, CPU is used.
    cross_device_ops: optional, a descendant of `CrossDeviceOps`. If this is not
      set, `NcclAllReduce()` will be used by default.  One would customize this
      if NCCL isn't available or if a special implementation that exploits
      the particular hardware is available.
  r   c                     t        | ||      }t        t        |   |       t        j
                  j                  d      j                  d       y )Nr@   r   V2rw   )MirroredExtendedsuperrw   __init__r   distribution_strategy_gaugeget_cellsetselfr@   r   extended	__class__s       r+   r}   zMirroredStrategy.__init__  sH    g0@BH	
D*84..77=AArY   NN)__name__
__module____qualname____doc___collective_key_baser}   __classcell__r   s   @r+   rw   rw      s    Od  rY   rw   c                   <     e Zd Zej                  ZdZd fd	Z xZS )MirroredStrategyV1r   c                     t        | ||      }t        t        |   |       t        j
                  j                  d      j                  d       y )Nry   V1rw   )r{   r|   r   r}   r   r~   r   r   r   s       r+   r}   zMirroredStrategyV1.__init__-  sH    g0@BH	
d,X6..77=AArY   r   )r   r   r   rw   r   r   r}   r   r   s   @r+   r   r   %  s"     $$'  rY   r   c                       e Zd ZdZd/ fd	Zd Zd Zd Zd Zd Z	d0dZ
ed	        Zd
 Zd Zd Zd Zej$                  j&                  fdZd Zd Zd Zd Z	 d0dZd Zd Z	 	 	 	 d1dZd Zd Zd Zd Z d Z!d Z"d0 fd	Z# fdZ$d Z%d  Z&d! Z'ed"        Z(ed#        Z)ed$        Z*ed%        Z+ed&        Z,ed'        Z-ed(        Z.ed)        Z/d* Z0ed+        Z1d, Z2d- Z3d. Z4 xZ5S )2r{   z#Implementation of MirroredStrategy.c                 V   t         t        |   |       t        j                         rl|rt        |      st        d      t        j                         j                         j                         rt        j                  d       |xs
 t               }n|xs
 t               }|sJ d       |j                  | _        t!        j"                  t         j$                  j&                        | _        || _        | j-                  |       t/        j0                         rd| _        d| _        y )NzVIn-graph multi-worker training with `MirroredStrategy` is not supported in eager mode.zInitializing local devices since in-graph multi-worker training with `MirroredStrategy` is not supported in eager mode. TF_CONFIG will be ignored when when initializing `MirroredStrategy`.zFGot an empty `devices` list and unable to recognize any local devices.implementationTF)r|   r{   r}   r   executing_eagerlyrE   RuntimeErrorr   rr   rO   rL   logginginforp   rt   r   r   OptionsCommunicationImplementationNCCL_communication_options_cross_device_ops_initialize_strategyr   #executing_eagerly_outside_functions(experimental_enable_get_next_as_optional_use_var_policy)r   container_strategyr@   r   r   s       r+   r}   zMirroredExtended.__init__9  s   	
D*+=>  "	6w? P Q 	Q &==?\^WY ,, ? @ 0.0(;=g + * +7 !3 G GD"1"9"9&BBGG#ID-Dg& ..06:d3 !DrY   c                     t        j                  t        j                                xs- t	        | j
                  D cg c]  }t        |       c}       S c c}w r2   )r   GraphOrParentsInXlaContextr   get_default_graphr=   _devicesr]   r   r*   s     r+   _use_merge_callz MirroredExtended._use_merge_call_  sT     !;;! ! 8(+(,61^A6)8 %886s   Ac                    |sJ d       t        d |D              }t        t        |            t        |      k(  s
J d|       | j                  |       | j	                         | _        | j                  s| j
                  | _        y y )Nz!Must specify at least one device.c              3   F   K   | ]  }t        j                  |        y wr2   )r   resolver(   s     r+   r,   z8MirroredExtended._initialize_strategy.<locals>.<genexpr>j  s     <qK''*<   !z-No duplicates allowed in `devices` argument: )tupler;   r   _initialize_single_worker#_make_collective_ops_with_fallbacks_collective_opsr   )r   r@   s     r+   r   z%MirroredExtended._initialize_strategyf  s     7777<G<<Gs7|G, H=DFH, 	""7+CCED!!#33d "rY   c                    t        j                  d| j                  z         | _        t	        j
                         s0t        d | j                  D              rt        j                         S t        d | j                  D              r0t        d | j                  D              rt        j                         S t        d | j                  D              r4t        j                  t        j                  j                        | _        nt!        j                          j#                  d	      }t!        j                          j%                  d	      }t'        |      t'        |      k  r3t        j                  t        j                  j                        | _        t        j(                  | j                  t'        | j                        | j                  | j                  
      S )Nr0   )group_key_startc              3   @   K   | ]  }d |j                         v  ywgpuNlowerr(   s     r+   r,   zGMirroredExtended._make_collective_ops_with_fallbacks.<locals>.<genexpr>y  s       =7#$QWWY=7   c              3   @   K   | ]  }d |j                         v   ywcpuNr   r(   s     r+   r,   zGMirroredExtended._make_collective_ops_with_fallbacks.<locals>.<genexpr>       
5!5AGGI
5r   c              3   @   K   | ]  }d |j                         v   ywr   r   r(   s     r+   r,   zGMirroredExtended._make_collective_ops_with_fallbacks.<locals>.<genexpr>  s       >3 >3r   c              3   @   K   | ]  }d |j                         v   ywr   r   r(   s     r+   r,   zGMirroredExtended._make_collective_ops_with_fallbacks.<locals>.<genexpr>  r   r   r   r[   )r\   )r@   
group_sizeoptionscollective_keys)r   CollectiveKeysr   _collective_keysr   r   r>   r   cross_device_ops_libReductionToOneDevicer=   r   r   r   RINGr   r   list_physical_devicesro   r;   CollectiveAllReduce)r   physical_gpuslogical_gpuss      r+   r   z4MirroredExtended._make_collective_ops_with_fallbacksu  so   .==D5557D 224 =7(,=7 :7 "6688 
5t}}
55# >3$(MM>3 ;3!6688

5t}}
55$3$;$;(DDII%Kd! oo'==%=Pm__&;;;Nl	]	c,/	/&5&=&=*FFKK'M#  33t}}%++--	/ /rY   c                    t        d |D              | _        t        j                  d|d         |ff| _        t        j                  | j                  d   d         | _        t        j                  j                  | j                  d   d         }|j                  :|j                  dk7  r+d|j                  |j                  |j                  fz  | _        t        j                   d|       y)z2Initializes the object for single-worker training.c              3   F   K   | ]  }t        j                  |        y wr2   r   canonicalizer(   s     r+   r,   z=MirroredExtended._initialize_single_worker.<locals>.<genexpr>  s     G!+2215Gr   /device:CPU:0r   Nr%   z/job:%s/replica:%d/task:%dz&Using MirroredStrategy with devices %r)r   r   r   r   _input_workers_devicesr   SingleDevice_host_input_devicer8   r9   r:   r'   r<   r3   _default_devicer   r   )r   r@   device_specs      r+   r   z*MirroredExtended._initialize_single_worker  s    GwGGDM		!	!/71:	>H#KD ,88##A&q)+D&&22##A&q)+K "{+'E9
//;..0@0@=B Bd LL97CrY   c           	         t        |      }g }g }dD ]\  }t        t        |j                  |g                   D ]3  }d||fz  }|j	                  |       |j	                  |||   |   f       5 ^ |d   | _        t        j                  |d         | _        t        |      | _
        || _        d| _        t        |      dkD  rZt        | j                  t        j                         r| j                  j"                  dkD  rt%        d      | j                  | _        nt        j(                         | _        t+        j,                  d|       y)	z1Initializes the object for multi-worker training.rG   z/job:%s/task:%dr   Tr0   zHIn-graph multi-worker training with `MirroredStrategy` is not supported.z-Using MirroredStrategy with remote devices %rN)rX   rK   r;   rM   r7   r   r   r   r   r   r   r   _is_multi_worker_trainingr4   r   r   r   _num_between_graph_workersr?   _inferred_cross_device_opsNcclAllReducer   r   )r   r@   rV   workersworker_devicesr'   r3   rI   s           r+   _initialize_multi_workerz)MirroredExtended._initialize_multi_worker  sB   $W-KGN" @KOOC456 @$"c4[0vv{3'7'=>?@@ #1:D+88DD'NDM"0D%)D"
7|a//-BBD

 
 
;
;a
? 	 )-(>(>d% )=(J(J(Ld%LL@'JrY   c                 R   |st        j                  | j                        S |j                  t        j
                  j                  k(  rn|j                  r"t        d | j                  D              | _        n!t        d | j                  D              | _        t        j                  | j                        S |j                  sAt        j                  | j                  D cg c]  \  }}||ft        |      z  f c}}      S t        j                  | j                        S c c}}w )Nc              3   N   K   | ]  }t        j                  ||      |ff  y wr2   r   r(   s     r+   r,   z?MirroredExtended._input_workers_with_options.<locals>.<genexpr>  s+      O;<))!Q/!6O   #%c              3   N   K   | ]  }t        j                  d |      |ff  yw)r   Nr   r(   s     r+   r,   z?MirroredExtended._input_workers_with_options.<locals>.<genexpr>  s,      * ++OQ?!F *r   )r
   InputWorkersr   experimental_replication_moder   InputReplicationModePER_REPLICA$experimental_place_dataset_on_devicer   r   experimental_fetch_to_devicer;   )r   r   host_devicecompute_devicess       r+   _input_workers_with_optionsz,MirroredExtended._input_workers_with_options  s   ##D$?$?@@--++77	8		5	5 O@DO O 	#
  *==* * 	# ##D$?$?@@11%%040K0K'
,_ ;.3+??@'
  	
 %%d&A&ABB'
s   !D#
c                 "    | j                         S r2   )r   r   s    r+   _input_workerszMirroredExtended._input_workers  s    ++--rY   c                 B    |dk(  r|d   S J J |J fd}|S )z4Return the initial value for variables on a replica.r   initial_valuec                  6   t        j                         st        j                         r%j	                         } t        j                  |       S t        j                        5  j                  } t        j                  |       cd d d        S # 1 sw Y   y xY wr2   )	r   r   r   inside_functionvaluer   identityr   r   )
init_valuer   primary_vars    r+   initial_value_fnzNMirroredExtended._get_variable_creator_initial_value.<locals>.initial_value_fn  ss    $$&#*=*=*?"((**##J/
/zz&! 2$22J%%j12 2 2s   $!BB )r   
replica_idr   r   kwargsr   s     ``  r+   #_get_variable_creator_initial_valuez4MirroredExtended._get_variable_creator_initial_value  sH     QO$$$$$2 rY   c                     |j                  dd      }| j                  nWt        |t        j                        r1t        j                  |j                        5   di |cddd       S |j                   fd}t        j                   j                         |t        j                  t        j                  fi |S # 1 sw Y   SxY w)z=Create a mirrored variable. See `DistributionStrategy.scope`.colocate_withNc            	      T   g }t              D ]  \  }}t        j                  |      5   j                  d|||r|d   nd d| | d<   |dkD  r+|d   j                  j                  d      d   }d||fz  | d<   t        j                  t        j                        5  t        j                         5   di | }d d d        d d d        t        t        j                        rJ |j                  |       d d d         |S # 1 sw Y   JxY w# 1 sw Y   NxY w# 1 sw Y   xY w)Nr   )r   r   r   r   :z%s/replica_%d/r6   r   )	enumerater   r   r   r6   splitr   device_policyDEVICE_PLACEMENT_SILENTr   stop_recordingr4   r   DistributedVariabler7   )	r   
value_listir*   var0namevr@   next_creatorr   s	         r+   _real_mirrored_creatorzAMirroredExtended._create_variable.<locals>._real_mirrored_creator  s?   jG$ $!QZZ] 	$LD$L$L %+5*Q-4% 	%&
!
 U!!}))//4Q7H .1=F6N$$W%D%DE ) &&( )((a))
  6#=#=>
>>


A
'	 	* 	) )) )	 	s<   A4DD3	D	<D5DD
DDDD'	r   )popr   r4   r   r   r   r   r	   create_mirrored_variable_container_strategyVARIABLE_CLASS_MAPPINGVARIABLE_POLICY_MAPPING)r   r  r   r   r	  r@   s   ``   @r+   _create_variablez!MirroredExtended._create_variable  s    JJ5Mg	M=#=#=	>::m**+ &%f%& & &&g2 44  "$://00< 5;< <=& &s   CCc                 0    t        j                  ||        y r2   )r	   &validate_colocate_distributed_variable)r   colocate_with_variables     r+    _validate_colocate_with_variablez1MirroredExtended._validate_colocate_with_variable.  s    ;;&rY   c                 x    t        j                  || j                  | j                         | j                        S )N)num_replicas_in_sync)input_lib_v1DatasetIteratorr   r  _num_replicas_in_sync)r   datasets     r+   _make_dataset_iteratorz'MirroredExtended._make_dataset_iterator2  s6    ''  "!77	9 9rY   c                    g }| j                   j                  }t        |      D ]3  }|j                  t	        j
                  ||| j                               5 t        j                  || j                   || j                               S N)num_input_pipelinesinput_pipeline_idr  )
r   rB   rK   r7   r   InputContextr  r  InputFunctionIteratorr  )r   input_fnreplication_modeinput_contextsrB   r  s         r+   _make_input_fn_iteratorz(MirroredExtended._make_input_fn_iterator9  s     N%%11K; <N77)#99; <<
 --h8K8K.<.2.F.F.HJ JrY   c                     |r2|j                   t        j                  j                  k(  rt	        d      t        j                  || j                  |      | j                         | j                  |      S )NzZInputReplicationMode.PER_REPLICA is only supported in `distribute_datasets_from_function`.)r  r   )
r   r   r   r   NotImplementedErrorr   get_distributed_datasetr   r  r  )r   r  r   s      r+    _experimental_distribute_datasetz1MirroredExtended._experimental_distribute_datasetH  sq    G99++7781 
 --((1  "!77 rY   c                 D    t        j                  || j                  |      S r2   )r   one_host_numpy_datasetr   )r   numpy_inputsessions      r+    _experimental_make_numpy_datasetz1MirroredExtended._experimental_make_numpy_datasetW  s"    //T,,g7 7rY   c                    | j                  |      }g }|j                  }t        |      D ]3  }|j                  t	        j
                  ||| j                               5 t        j                  |||| j                         |      S r  )
r   rB   rK   r7   r   r  r  r   &get_distributed_datasets_from_functionr  )r   
dataset_fnr   input_workersr#  rB   r  s          r+   "_distribute_datasets_from_functionz3MirroredExtended._distribute_datasets_from_function[  s    44W=MN++K; <N77)#99; << <<M>43K3K3M rY   c           	          g }t        | j                        D ]7  }|j                   |t        j                  || j                                     9 t        j                  |d      S )NT)always_wrap)rK   r  r7   r   ValueContextr	   regroup)r   value_fnper_replica_valuesr   s       r+   -_experimental_distribute_values_from_functionz>MirroredExtended._experimental_distribute_values_from_functioni  sk    D667 D


%
%j&*&@&@B!C DD ##$6DIIrY   c           
          |i }t        j                  |      }t        j                          fd}t	        j
                         j                          _        fd}t        j                  d      }t        j                  |||g|z   ddddd      } `t        j                  |      _        |dd  }	t        j                  j                  |	      }
j                   j#                         D ]=  \  }}|
|   }|t%        j&                  |      |
|<   &t)        |      dk(  sJ |d   |
|<   ? j+                  |
       S )	Nc                 \   ~ j                               }j                  j                         D ]#  \  }}	j                  |      j                  |<   % t	        j
                  j                        }t        j                  |g      5  | dz   g|z   cddd       S # 1 sw Y   yxY w)z4A wrapper around `fn` to create the while loop body.r0   N)get_nextlast_step_outputsrf   _local_resultsr    flattenr   control_dependencies)
r  args	fn_resultr6   outputflat_last_step_outputsctxfniteratorr   s
         r+   bodyzBMirroredExtended._experimental_run_steps_on_iterator.<locals>.bodyy  s    
S(++-.i11779 B.4&*&9&9&&Ad#B  $||C,A,AB##YK0 0Aw//0 0 0s   	B""B+c                     | k  S r2   r   )r  rA  
iterationss     r+   <lambda>zFMirroredExtended._experimental_run_steps_on_iterator.<locals>.<lambda>  s    A
N rY   r    r0   FT)r6   parallel_iterations	back_propswap_memoryreturn_same_structure)r    r?  r
   MultiStepContextr   r   _get_control_flow_context_outer_control_flow_contextr   constantr   r   grouprun_oppack_sequence_asr=  _last_step_outputs_reduce_opsrf   r	   r6  r;   _set_last_step_outputs)r   rF  rG  rJ  initial_loop_valuesrH  condr  loop_resultlast_step_tensor_outputslast_step_tensor_outputs_dictr6   	reduce_oprC  rE  s   ````          @r+   #_experimental_run_steps_on_iteratorz4MirroredExtended._experimental_run_steps_on_iteratorr  s_   ",,':;

$
$
&C	0" 	99; 	$ +DQA''qc''"$K 	(!''4CJ  +12$($9$97%9! <<BBD 8i,T2f 
	.>.F.Fv.N%d+6{a.4Qi%d+8 <=JrY   c                     t        |t        t        f      r|S |s| j                  }| j	                  |      j                  ||      S r2   )r4   floatintr   _get_cross_device_ops	broadcast)r   tensordestinationss      r+   _broadcast_tozMirroredExtended._broadcast_to  sA     &5#,'m]]l%%f-77MMrY   c                 N    t        j                  | j                         |||      S r2   )r   call_for_each_replicar  )r   rF  rA  r   s       r+   _call_for_each_replicaz'MirroredExtended._call_for_each_replica  s'    --  "Bf6 6rY   c                     ~~|r |j                  | j                  |             |r3t        | j                        }t	        ||      }| j                  |       y y r2   )CopyFrom_update_config_protorl   r   rT   r   )r   session_configrO   rQ   rR   rP   multi_worker_devicess          r+   
_configurezMirroredExtended._configure  s^    
 	7d77GH 7t}}E9
+-
##$89 rY   c                 >    t        j                  |      }d|_        |S NT)copydeepcopyisolate_session_state)r   config_protoupdated_configs      r+   rn  z%MirroredExtended._update_config_proto  s    ]]<0N+/N(rY   c                 :   | j                         sPt        | j                  t        j                        s t        j                  d| j                         | j                  S t        |t        j                        rKd|j                  D ch c]2  }t        j                  |j                        t        j                  k(  4 c}v }n0t        j                  |j                        t        j                  k(  }|rt        j                         S | j                  S c c}w )Na  Under XLA context, MirroredStrategy uses CollectiveAllReduce op. Although %r is provided to initialize MirroredStrategy, it is ignored in XLA. Please use CollectiveAllReduce(or default option) in the future, since other cross device ops are not well supported on XLA.T)r   r4   r   r   r   r   warningr   r   DistributedValuesr   as_dtypedtypeint32r   )r   r   r  value_int32s       r+   rd  z&MirroredExtended._get_cross_device_ops  s     !..,@@B  "&!7!7	
 !!!%112<ALL78&//!''
"fll
2 k OOEKK0FLL@k!6688###s   
7Dc                     t        |t        j                        s|S | j                  |      j	                  |||| j
                  j                  |            S )N)rg  axisr   )r4   r   r{  rd  _gatherr   merge)r   r   rg  r  r   s        r+   _gather_to_implementationz*MirroredExtended._gather_to_implementation  sW    eV556l%%e,44!++11':	 5 < <rY   c                      t        j                  |      rt        j                  j                  k(  r|S t        j                  |      rJ  fd}t        j                  ||      S )Nc                    t        | t        j                        s"t        j                  | j
                        S j                         r`t        j                  |       r%t        d t        j                        D              r%t        j                         j                  |       S j                  |       j                  | j                  j                              S )Nc              3   @   K   | ]  }d |j                         v   ywr   r   r(   s     r+   r,   zBMirroredExtended._reduce_to.<locals>.get_values.<locals>.<genexpr>  s%      L qwwy  Lr   )rg  r   )r4   r   r{  r   reduce_non_distributed_valuer  r   _devices_matchr>   get_devices_fromr   reducerd  r   r  )r   rg  r   r_  r   s    r+   
get_valuesz/MirroredExtended._reduce_to.<locals>.get_values   s    v778
 $@@ulD,F,FH 	H 
			"11%F
 L+<<\JL L#88:AAul, 	,''.55

#--33G<	 6 > >rY   )r	   is_mirroredr   ReduceOpMEANr    map_structure)r   r_  r   rg  r   r  s   `` `` r+   
_reduce_tozMirroredExtended._reduce_to  sX    $$U+[))...l++E222>* j%00rY   c                     d }|D ]7  \  }}|| j                  |      }|| j                  |      us.t        d       |j                  ||| j                  j	                  |            S )NzWInputs to batch_reduce_to must be either all on the host or all on the compute devices.)r   )rd  r?   batch_reducer   r  )r   r_  value_destination_pairsr   r   r   ri   s          r+   _batch_reduce_toz!MirroredExtended._batch_reduce_to  s    + Dq		!55e<4#=#=e#DD C D 	D	D ((++11': ) < <rY   c                 F   t        |t        j                        sJ g }t        |j                        D ]  \  }}d|z  }	t	        j
                  |j
                        5  t        j                  |      5  t	        j                  |	      5  |j                   ||gt        j                  ||      i t        j                  ||             d d d        d d d        d d d         t        j                  | ||      S # 1 sw Y   2xY w# 1 sw Y   6xY w# 1 sw Y   xY wNz	update_%d)r4   r   r  r   r   r   r   UpdateContext
name_scoper7   r	   select_replicaupdate_regroup)
r   varrF  rA  r   rU  updatesr  r  r6   s
             r+   _updatezMirroredExtended._update$  s	   c655666G#**% >11_d::ahh >''*>>>$> 	q =#221d; =!00F;=	>	> > > >> **4%@@> > > > > >s=   D4D
AC?	DD?DDDDD 	c                 V   |t        j                         }t        j                         s$t	        j
                         r| j                         rt        | !  |||      S t        j                         }|sJ d       | j                  |      j                  |||j                  |      S )z8Implements `StrategyExtendedV2._replica_ctx_all_reduce`.zN`StrategyExtended._replica_ctx_all_reduce` must be called in a replica context)r   r   r   r   r   enabledr   r|   _replica_ctx_all_reducer   get_replica_contextrd  _all_reduce_replica_id)r   r_  r   r   replica_contextr   s        r+   r  z(MirroredExtended._replica_ctx_all_reduce3  s     '')g  "KKMd224
 W,YwGG$88:O 	? %%e,88##	 rY   c                    | j                         rt        
| 	  |||||      S t        j                         }|sJ t        j                         }d|z  }t        |t        j                        r|j                  |      }t        j                  |j                        5  t        j                  |      5   ||g|i |}	d d d        d d d        	S # 1 sw Y   xY w# 1 sw Y   	S xY wr  )r   r|   _replica_ctx_updater   r  r   get_current_replica_id_as_intr4   r   r  _get_replicar   r   r  )r   r  rF  rA  r   rU  r  r   r6   resultr   s             r+   r  z$MirroredExtended._replica_ctx_updateL  s    W(b$FF$88:O?::<J#D#v112Z(c	CJJ	 (!5 (#'''f( (M( ( (Ms$    C!6CC!C	C!!C+c                    t        |t              sJ g }t        |      D ]  \  }}d|z  }	t        j                  |      5  t        j                  |      5  t        j                  |	      5  |j                   |t        j                  ||      i t        j                  ||             d d d        d d d        d d d         t        j                  | ||      S # 1 sw Y   2xY w# 1 sw Y   6xY w# 1 sw Y   xY wr  )r4   r   r   r   r   r   r  r  r7   r	   r  r  )
r   r   rF  rA  r   rU  r  r  r*   r6   s
             r+   _update_non_slotz!MirroredExtended._update_non_slot\  s    mU+++G-( >11_d::a= >.66q9 >3>>$;O > //48 =!00F;=	>> > > >> **4%@@	> > > > > >s=    C5C),A C	,C)4C5C&"C))C2.C55C>	c                     t        j                  |      r|j                         S t        j                  |      sJ t	        j
                  |j                               S )z5Read the aggregate value of a replica-local variable.)r	   is_sync_on_read_get_cross_replicar  r   r   _get)r   replica_local_vars     r+   read_varzMirroredExtended.read_varh  sR     ''(9:1133''(9:::/44677rY   c                 ,    t        j                  |      S r2   )r	   value_container)r   vals     r+   r  z MirroredExtended.value_containerq  s    ++C00rY   c                 ,    t        | j                        S r2   )r;   r   r   s    r+   r  z&MirroredExtended._num_replicas_in_synct  s    t}}rY   c                     | j                   S r2   r   r   s    r+   r   zMirroredExtended.worker_devicesx  s    ==rY   c                 B    | j                   D cg c]  }|g c}S c c}w r2   r  r   s     r+   worker_devices_by_replicaz*MirroredExtended.worker_devices_by_replica|  s    'AQC'''s   
c                     | j                   S r2   )r   r   s    r+   parameter_devicesz"MirroredExtended.parameter_devices  s    rY   c                      y)NFr   r   s    r+   experimental_between_graphz+MirroredExtended.experimental_between_graph      rY   c                      yrs  r   r   s    r+   experimental_should_initz)MirroredExtended.experimental_should_init      rY   c                      yrs  r   r   s    r+   should_checkpointz"MirroredExtended.should_checkpoint  r  rY   c                      yrs  r   r   s    r+   should_save_summaryz$MirroredExtended.should_save_summary  r  rY   c                     ~| j                   S r2   r  )r   var_lists     r+   non_slot_devicesz!MirroredExtended.non_slot_devices  s    ==rY   c                      y)z`make_dataset_iterator` and `make_numpy_iterator` use global batch size.

    `make_input_fn_iterator` assumes per-replica batching.

    Returns:
      Boolean.
    Tr   r   s    r+   _global_batch_sizez#MirroredExtended._global_batch_size  s     rY   c                      y)zAWhether this strategy indicates working in multi-worker settings.Fr   r   s    r+   _in_multi_worker_modez&MirroredExtended._in_multi_worker_mode  r  rY   c                     |S r2   r   )r   replica_id_in_sync_groups     r+   _get_local_replica_idz&MirroredExtended._get_local_replica_id  s    ##rY   c                     |S r2   r   )r   r   s     r+   _get_replica_id_in_sync_groupz.MirroredExtended._get_replica_id_in_sync_group  s    rY   r   r2   )NNNN)6r   r   r   r   r}   r   r   r   r   r   r   propertyr   r   r  r  r  r   r   
PER_WORKERr$  r(  r-  r2  r9  r`  rh  rk  rq  rn  rd  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r  r  r  r  r  r   r   s   @r+   r{   r{   6  s   +$!L84!/FD"'KRC. . .0'<R&9 &::EEJ7J ?C9vN6
 !%"	:$
$6<18<A2 
A81     ( (            $rY   r{   r2   );r   rt  tensorflow.pythonr   tensorflow.python.distributer   r   r   r   r   r   r	   r
   r   r   r   r   r   r   r   -tensorflow.python.distribute.cluster_resolverr   tensorflow.python.distribute.v1r  tensorflow.python.eagerr   r   tensorflow.python.frameworkr   r   r   r8   r   r   tensorflow.python.opsr   r   r   r   tensorflow.python.platformr   r   tensorflow.python.utilr     tensorflow.python.util.tf_exportr!   rE   rT   rX   r]   rl   rp   rt   Strategyrw   
StrategyV1r   StrategyExtendedV1r{   r   rY   r+   <module>r     s    B  ! 8 Q ; 4 7 9 2 3 5 : 6 4 / 4 S E + * . 3 ; . + + 2 3 , < ' 6
%P :G%P75 (R0Z~.. Z 1Zz ,-.22  / w	~88 w	rY   