
    AVhn                        d Z ddlZddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm	Z	 dd	lm
Z
 dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m!Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( dZ) e(dg       G d dejT                               Z+ G d d ejX                        Z-y)!zJClass implementing a multi-worker parameter server tf.distribute strategy.    N)cross_device_ops)device_util)distribute_lib)distribute_utils)	input_lib)
input_util)mirrored_run)multi_worker_util)numpy_dataset)	ps_values)valuescluster_resolver)tfconfig_cluster_resolver)context)device)ops)	array_ops)resource_variable_ops)variable_scope)
tf_logging)device_setter)nest)	tf_export/device:CPU:0z/distribute.experimental.ParameterServerStrategy)v1c                   X     e Zd ZdZd fd	Zd fd	Zd fd	Zd	 fd	Z fdZd Z	 xZ
S )
ParameterServerStrategyV1a	  An asynchronous multi-worker parameter server tf.distribute strategy.

  This strategy requires two roles: workers and parameter servers. Variables and
  updates to those variables will be assigned to parameter servers and other
  operations are assigned to workers.

  When each worker has more than one GPU, operations will be replicated on all
  GPUs. Even though operations may be replicated, variables are not and each
  worker shares a common view for which parameter server a variable is assigned
  to.

  By default it uses `TFConfigClusterResolver` to detect configurations for
  multi-worker training. This requires a 'TF_CONFIG' environment variable and
  the 'TF_CONFIG' must have a cluster spec.

  This class assumes each worker is running the same code independently, but
  parameter servers are running a standard server. This means that while each
  worker will synchronously compute a single gradient update across all GPUs,
  updates between workers proceed asynchronously. Operations that occur only on
  the first replica (such as incrementing the global step), will occur on the
  first replica *of every worker*.

  It is expected to call `call_for_each_replica(fn, ...)` for any
  operations which potentially can be replicated across replicas (i.e. multiple
  GPUs) even if there is only CPU or one GPU. When defining the `fn`, extra
  caution needs to be taken:

  1) It is generally not recommended to open a device scope under the strategy's
  scope. A device scope (i.e. calling `tf.device`) will be merged with or
  override the device for operations but will not change the device for
  variables.

  2) It is also not recommended to open a colocation scope (i.e. calling
  `tf.compat.v1.colocate_with`) under the strategy's scope. For colocating
  variables, use `strategy.extended.colocate_vars_with` instead. Colocation of
  ops will possibly create device assignment conflicts.

  Note: This strategy only works with the Estimator API. Pass an instance of
  this strategy to the `experimental_distribute` argument when you create the
  `RunConfig`. This instance of `RunConfig` should then be passed to the
  `Estimator` instance on which `train_and_evaluate` is called.

  For Example:
  ```
  strategy = tf.distribute.experimental.ParameterServerStrategy()
  run_config = tf.estimator.RunConfig(
      experimental_distribute.train_distribute=strategy)
  estimator = tf.estimator.Estimator(config=run_config)
  tf.estimator.train_and_evaluate(estimator,...)
  ```
  c                     |t        j                         }t        t        |   t        | |             t        j                  j                  d      j                  d       y)a   Initializes this strategy with an optional `cluster_resolver`.

    Args:
      cluster_resolver: Optional
        `tf.distribute.cluster_resolver.ClusterResolver` object. Defaults to a
        `tf.distribute.cluster_resolver.TFConfigClusterResolver`.
    Nr   V1ParameterServerStrategy)
r   TFConfigClusterResolversuperr   __init__ParameterServerStrategyExtendedr   distribution_strategy_gaugeget_cellset)selfr   	__class__s     f/home/dcms/DCMS/lib/python3.12/site-packages/tensorflow/python/distribute/parameter_server_strategy.pyr$   z"ParameterServerStrategyV1.__init__f   s\     2JJL	
#T3'#3	56 ..77=AA!#    c                     |r2|j                   t        j                  j                  k(  rt	        d      | j                          t        t        | #  ||       y )NzgInputReplicationMode.PER_REPLICA is only supported in `experimental_distribute_datasets_from_function`.)datasetoptions)	experimental_replication_moder   InputReplicationModePER_REPLICANotImplementedError_raise_pss_error_if_eagerr#   r   experimental_distribute_dataset)r)   r.   r/   r*   s      r+   r5   z9ParameterServerStrategyV1.experimental_distribute_datasetv   sa    G99++778> 
 	""$	
#
08?8? 0 Ar,   c                     |r2|j                   t        j                  j                  k(  rt	        d      | j                          t        t        | #  ||       y )NzInputReplicationMode.PER_REPLICA is only supported in `experimental_distribute_datasets_from_function` of tf.distribute.MirroredStrategy)
dataset_fnr/   )	r0   r   r1   r2   r3   r4   r#   r   !distribute_datasets_from_function)r)   r7   r/   r*   s      r+   r8   z;ParameterServerStrategyV1.distribute_datasets_from_function   s]    G99++778./ /
 	""$	
#TLw M 0r,   c                 T    | j                          t        t        |   ||||       y )N)argskwargsr/   )r4   r#   r   run)r)   fnr:   r;   r/   r*   s        r+   r<   zParameterServerStrategyV1.run   s-    ""$	
#T.
fg / 7r,   c                 H    | j                          t        t        |          S N)r4   r#   r   scope)r)   r*   s    r+   r@   zParameterServerStrategyV1.scope   s     ""$*D799r,   c                 B    t        j                         rt        d      y )Nzm`tf.compat.v1.distribute.experimental.ParameterServerStrategy` currently only works with the tf.Estimator API)r   executing_eagerlyr3   r)   s    r+   r4   z3ParameterServerStrategyV1._raise_pss_error_if_eager   s%      ";< < #r,   r?   ) NN)__name__
__module____qualname____doc__r$   r5   r8   r<   r@   r4   __classcell__r*   s   @r+   r   r   0   s(    2h# A
07
:<r,   r   c                       e Zd ZdZ	 	 	 d, fd	Z	 	 	 d,dZd Z	 d-dZd-dZe	d        Z
d Zd	 Zd
 Zej                  j                   fdZd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Z d Z!d Z"	 	 	 	 d.dZ#d Z$d Z%e	d         Z&e	d!        Z'e	d"        Z(e	d#        Z)d$ Z*e	d%        Z+e	d&        Z,e	d'        Z-e	d(        Z.e	d)        Z/d* Z0d+ Z1 xZ2S )/r%   zEImplementation of ParameterServerStrategy and CentralStorageStrategy.c                     t         t        |   |       | j                  |||       t	        j
                  t              | _        y )N)r   compute_devicesparameter_device)reduce_to_device)r#   r%   r$   _initialize_strategycross_device_ops_libReductionToOneDevice
_LOCAL_CPU_cross_device_ops)r)   container_strategyr   rM   rN   r*   s        r+   r$   z(ParameterServerStrategyExtended.__init__   sJ    
 

)49:LM)')  + 	11:N 	r,   c                 t    |r"|j                         r| j                  |       y | j                  |||       y )Nr   )cluster_spec_initialize_multi_worker_initialize_local)r)   r   rM   rN   s       r+   rP   z4ParameterServerStrategyExtended._initialize_strategy   s?     ,99;
##$45

+>N  Pr,   c           
          t        |t        j                        rt        j                         }n |j                         j                  dd      }| _        |j                         }|j                  }|j                  }|r|t        d      t        j                  |      }|j                         sJ d||fz   _        t!        j"                   j                         _        |dkD  rt'         fdt)        |      D              }n j                  f}|D cg c]  }t+        j,                  |       c} _        t1        |j                         j                  dg             }|dk(  rt        d      t3        j4                  | j                  d	|
       _        t'        t9        dj:                  t)        |                   _         j                   _        t        j@                  |||       _!        | _"        | _#        | _$        tK        jL                  d|j                         ||| jB                   j.                   j6                         yc c}w )a  Initialize devices for multiple workers.

    It creates variable devices and compute devices. Variables and operations
    will be assigned to them respectively. We have one compute device per
    replica. The variable device is a device function or device string. The
    default variable device assigns variables to parameter servers in a
    round-robin fashion.

    Args:
      cluster_resolver: a descendant of `ClusterResolver` object.

    Raises:
      ValueError: if the cluster doesn't have ps jobs.
    GPUr   NzMWhen `cluster_spec` is given, you must also specify `task_type` and `task_id`/job:%s/task:%dc              3   @   K   | ]  }d j                   |fz    yw)z%s/device:GPU:%dN)_worker_device).0ir)   s     r+   	<genexpr>zKParameterServerStrategyExtended._initialize_multi_worker.<locals>.<genexpr>   s)      $  3 3Q7
7$s   psz)The cluster spec needs to have `ps` jobs.T)ps_tasksworker_devicemerge_devicesclusterz/job:ps/task:{}zMulti-worker ParameterServerStrategy with cluster_spec = %r, task_type = %r, task_id = %r, num_ps_replicas = %r, is_chief = %r, compute_devices = %r, variable_device = %r)'
isinstancer   r"   r   num_gpusnum_acceleratorsget_num_gpus_per_workerrW   	task_typetask_id
ValueErrorr
   normalize_cluster_specas_dictr^   r   SingleDevice_input_host_devicetupleranger   canonicalize_compute_deviceslenr   replica_device_setter_variable_devicemapformat_parameter_devices_default_deviceis_chief	_is_chief_cluster_spec
_task_type_task_idlogginginfo)	r)   r   rh   rW   rl   rm   rM   dnum_ps_replicass	   `        r+   rX   z8ParameterServerStrategyExtended._initialize_multi_worker   s6   " 3KKM!!#h!22488Bh !)D#002L **I&&G 3 4 4$;;LIL!!!+y'.BBD+889L9LMD
 !| $?$ $o ,,.o .=>()  #>D ,..044T2>?O!BCC)?? ))	D $C(9(@(@(-o(>%@ AD
  ..D&//i079DN%DDODMLL	 !- 4 4 6	7)>)>I>s   Ic                 V   t        j                  d      | _        t        j                  | j                        | _        |S|st        j                         }n |j                         j                  dd      }|| _
        t        j                  |      }|D cg c]  }t        j                  |       }}|t        |      dk(  r|d   }nt        }|| _        || _        |f| _        d| _        d| _        d| _        d| _        t+        j,                  d|| j                         yc c}w )z&Initialize local devices for training.r   Nr[   r      TzParameterServerStrategy (CentralStorageStrategy if you are using a single machine) with compute_devices = %r, variable_device = %r)r   ru   r^   r   rq   rr   r   rh   ri   rj   rk   local_devices_from_num_gpusrw   rS   ry   rv   r|   r   r   r   r   r   r   )r)   rM   rN   r   rh   r   s         r+   rY   z1ParameterServerStrategyExtended._initialize_local  s   
 &22?CD+889L9LMD##%#446::5!D #+d#??Io<KLq{//2LOL 
_		"*1-%,D+D/1DDNDDODMLL	J..0% Ms   D&c                     |r|j                   r,t        j                  | j                  | j                  fg      S t        j                  | j                  | j                  ft        | j                        z  fg      S r?   )experimental_fetch_to_devicer   InputWorkersr^   rv   rw   )r)   r/   s     r+   _input_workers_with_optionsz;ParameterServerStrategyExtended._input_workers_with_optionsD  s    g::##!6!67
8: : ##  "S)>)>%??A BC Cr,   c                 "    | j                         S r?   )r   rC   s    r+   _input_workersz.ParameterServerStrategyExtended._input_workersM  s    ++--r,   c                 0    t        j                  ||        y r?   )r   validate_colocate)r)   colocate_with_variables     r+    _validate_colocate_with_variablez@ParameterServerStrategyExtended._validate_colocate_with_variableQ  s    &&'=tDr,   c                     t        j                  || j                  |      | j                         | j                  |      S )N)num_replicas_in_syncr/   )r   get_distributed_datasetr   _container_strategy_num_replicas_in_sync)r)   r.   r/   s      r+    _experimental_distribute_datasetz@ParameterServerStrategyExtended._experimental_distribute_datasetT  s>    --((1  "!77 r,   c                 x    t        j                  || j                  | j                         | j                        S )N)r   )input_lib_v1DatasetIteratorr   r   r   )r)   r.   s     r+   _make_dataset_iteratorz6ParameterServerStrategyExtended._make_dataset_iterator\  s6    ''  "!77	9 9r,   c                    | j                   r`t        j                  | j                   | j                  | j                        }t        j
                  | j                   | j                        }nd}d}t        j                  ||| j                        }t        j                  || j                  |g| j                               S )z*Distributes the dataset to each local GPU.r   r   num_input_pipelinesinput_pipeline_idr   )r   r
   id_in_clusterr   r   worker_countr   InputContextr   r   InputFunctionIteratorr   r   )r)   input_fnreplication_moder   r   input_contexts         r+   _make_input_fn_iteratorz7ParameterServerStrategyExtended._make_input_fn_iteratorc  s    
 +99


doot}}>-::


doo/ "///+!779M --h8K8K/<o.2.F.F.HJ Jr,   c                 D    t        j                  || j                  |      S r?   )r   one_host_numpy_datasetrr   )r)   numpy_inputsessions      r+    _experimental_make_numpy_datasetz@ParameterServerStrategyExtended._experimental_make_numpy_datasetx  s"    //T,,g7 7r,   c                    | j                   r`t        j                  | j                   | j                  | j                        }t        j
                  | j                   | j                        }nd}d}t        j                  ||| j                        }t        j                  || j                  |      |g| j                         |      S )Nr   r   r   )r/   )r   r
   r   r   r   r   r   r   r   r   &get_distributed_datasets_from_functionr   r   )r)   r7   r/   r   r   r   s         r+   "_distribute_datasets_from_functionzBParameterServerStrategyExtended._distribute_datasets_from_function|  s    +99


doot}}>-::


doo/ "///+!779M
 <<((1M?  "	 r,   c           	          g }t        | j                        D ]7  }|j                   |t        j                  || j                                     9 t        j                  |d      S )NT)always_wrap)rt   r   appendr   ValueContextr   regroup)r)   value_fnper_replica_values
replica_ids       r+   -_experimental_distribute_values_from_functionzMParameterServerStrategyExtended._experimental_distribute_values_from_function  sk    D667 M

>..z/3/I/IK LMM ##$6DIIr,   c                     t        |t        t        f      r|S t        j                  |      s| j
                  }| j                  j                  ||      S r?   )rg   floatintrQ   check_destinationsrv   rT   	broadcast)r)   tensordestinationss      r+   _broadcast_toz-ParameterServerStrategyExtended._broadcast_to  sI     &5#,'m22<@**l!!++FLAAr,   c                 ,    t        j                          S r?   )r   rB   rC   s    r+   _allow_variable_partitionz9ParameterServerStrategyExtended._allow_variable_partition  s    ((***r,   c                      j                   dkD  r|j                  dt        j                  j                        t        j                  j                  t        j                  j
                  t        j                  j                  t        j                  j                  fvrt        dz   dz   |d   z          fd}|S S )Nr   aggregationz#Invalid variable aggregation mode: z for variable: namec                     | j                  dd      }|t        j                  j                  g}g | d<    di | }t	        j
                  j                         |      }t        j                         st        j                         }| j                  dd      rg|j                  t        j                  j                         |j                  t        j                  j                        }||v r|j                  |       |j                  ||       |S t        j                  j                   |v r.t        j                  t        j                  j                   |       |S )z5Create an AggregatingVariable and fix up collections.collectionsN	trainableTrD   )popr   	GraphKeysGLOBAL_VARIABLESr   AggregatingVariabler   r   rB   get_default_graphrj   r   TRAINABLE_VARIABLESget_collection_refremoveadd_to_collectionsGLOBAL_STEP)	r;   r   vwrappedglr   next_creatorr)   s	         r+   var_creatorzHParameterServerStrategyExtended._create_var_creator.<locals>.var_creator  s     jj5778+ "} "6"//0H0H0JA0;= ((*##%! ZZT*s}}@@A$$S]]%F%FGAAvhhqk


{G
4  ]]&&+5

 
 !:!:G
Dr,   )	r   r   vsVariableAggregationNONESUMMEANONLY_FIRST_REPLICArn   )r)   r   r;   r   r   s   ``  @r+   _create_var_creatorz3ParameterServerStrategyExtended._create_var_creator  s    !!A%JJ}b.D.D.I.IJk	

 
 
%
%

 
 
$
$

 
 
%
%

 
 
3
3	 
 >L*+-3F^< = 	= B r,   c                     | j                   |fi |}d|v r|d   }t        |t        j                        r1t	        j
                  |j
                        5   |di |cd d d        S t	        j
                  d       5  t	        j                  |      5   |di |cd d d        cd d d        S t	        j                  d d      5  t	        j
                  | j                        5   |di |cd d d        cd d d        S # 1 sw Y   xY w# 1 sw Y   nxY w	 d d d        t# 1 sw Y   }xY w# 1 sw Y   nxY w	 d d d        y # 1 sw Y   y xY w)Ncolocate_withT)ignore_existingrD   )r   rg   r   rq   r   r   r   ry   )r)   r   r;   r   r   s        r+   _create_variablez0ParameterServerStrategyExtended._create_variable  sB   *$**<B6BK& _-m	M=#=#=	>ZZ,,- 	'&v&	' 	'::d '}- 	'&v&	' 	'' ' 
		4	6 %::d++, %$V$% %% %	' 	'	' 	' 	'' ' '
% % %% % %sT   D =D"D	D" E%D.-	E D	D	D""D+.D7	3EEc                 N    t        j                  | j                         |||      S r?   )r	   call_for_each_replicar   )r)   r=   r:   r;   s       r+   _call_for_each_replicaz6ParameterServerStrategyExtended._call_for_each_replica  s'    --d.F.F.H".2F< <r,   c                 2   | j                   sy |y t        j                  |      D ]o  }t        j                  j                  |      }|j                  | j                  k(  s<|j                  | j                  k7  sVt        d|d| j                         y )Nz!Cannot reduce to another worker: z, current worker is )r   rQ   get_devices_from	tf_device
DeviceSpecfrom_stringjobr   taskr   rn   r^   )r)   r   r   d_specs       r+   )_verify_destinations_not_different_workerzIParameterServerStrategyExtended._verify_destinations_not_different_worker  s    !22<@ &##//2f	t	&6;;$--+G##%& 	&&r,   c                     | j                  |       t        |t        j                        s|S | j                  j                  ||||      S )N)r   axisr/   )r   rg   r   DistributedValuesrT   _gather)r)   valuer   r   r/   s        r+   _gather_to_implementationz9ParameterServerStrategyExtended._gather_to_implementation  sO    22<@eV556l!!))!	 *  r,   c                     | j                  |       t        |t        j                        s"t	        j
                  |||| j                        S | j                  j                  ||||      S )N)r   r/   )	r   rg   r   r   rQ   reduce_non_distributed_valuer   rT   reduce)r)   	reduce_opr   r   r/   s        r+   
_reduce_toz*ParameterServerStrategyExtended._reduce_to  sm    22<@eV556!>>
UL$*D*DF F!!((5|W ) F Fr,   c                 r    |D ]  \  }}| j                  |        | j                  j                  |||      S r?   )r   rT   batch_reduce)r)   r   value_destination_pairsr/   _r   s         r+   _batch_reduce_toz0ParameterServerStrategyExtended._batch_reduce_to  sG    2 C<
44\BC!!..y/FQ Qr,   c                 4    d }t        j                  ||      S )z(Select any single value in `structured`.c                     t        | t        j                        st        | t        j                        r| j                  S | S r?   )rg   r   Mirrored
PerReplica_primaryxs    r+   
_select_fnzHParameterServerStrategyExtended._select_single_value.<locals>._select_fn  s.    	Av	':a9J9J+Kzzr,   )r   map_structure)r)   
structuredr	  s      r+   _select_single_valuez4ParameterServerStrategyExtended._select_single_value  s     j*55r,   c           	      .   t        |t        j                        r|j                         }t	        j
                  |      st        d|z        t        j                  |      5  t        j                  |j                        5   ||g| j                  |      i | j                  |      }|r|cd d d        cd d d        S t        j                  | j                  |      cd d d        cd d d        S # 1 sw Y   nxY wd d d        y # 1 sw Y   y xY w)Nz3You can not update `var` %r. It must be a Variable.)rg   r   r   rj   r   is_resource_variablern   r   r   r   UpdateContextr   r  r   r
  _local_results)r)   varr=   r:   r;   groupresults          r+   _updatez'ParameterServerStrategyExtended._update!  s    #y445GGIc 55c:
?#
EG G			3	 ?!=!=cjj!I ?# 711$7 7--f57f		? ? ? !!$"5"5v>? ? ? ? ? ? ?s0   # D.C61	DC6#	D6C?	;DDc                 P   t        j                  |j                        5  t        j                  |      5   ||i |}|r|cd d d        cd d d        S t	        j
                  | j                  |      cd d d        cd d d        S # 1 sw Y   nxY wd d d        y # 1 sw Y   y xY wr?   )r   r   r   r  r   r
  r  )r)   r   r=   r:   r;   r  r  s          r+   _update_non_slotz0ParameterServerStrategyExtended._update_non_slot0  s    	
 ?-;;MJ?4"6"f		? ? ? !!$"5"5v>? ? ? ? ? ? ?s.   BB	BB4	BB	BB%c                 z    t        |d      r.t        |t        j                        s|j	                         }||S |S )N_aggregating_container)hasattrrg   r   r   r  )r)   valwrappers      r+   value_containerz/ParameterServerStrategyExtended.value_container9  s;    -.sI99:**,g		Jr,   c                 ,    t        j                  |      S r?   )r   identity)r)   r  s     r+   read_varz(ParameterServerStrategyExtended.read_varA  s     c""r,   c                     |rIt        j                  t        j                  |      ||d| j                  i      }| j                  |       |r!|j                  | j                  |             yy)a!  Configures the strategy class with `cluster_spec`.

    The strategy object will be re-initialized if `cluster_spec` is passed to
    `configure` but was not passed when instantiating the strategy.

    Args:
      session_config: Session config object.
      cluster_spec: a dict, ClusterDef or ClusterSpec object specifying the
        cluster configurations.
      task_type: the current task type.
      task_id: the current task id.

    Raises:
      ValueError: if `cluster_spec` is given but `task_type` or `task_id` is
        not.
    r[   )rW   rl   rm   ri   N)cluster_resolver_libSimpleClusterResolverr
   ro   rk   rX   CopyFrom_update_config_proto)r)   session_configrW   rl   rm   r   s         r+   
_configurez*ParameterServerStrategyExtended._configureF  sn    *  .CC(??M!4#<#<=	?
 ##$45d77GH r,   c                    t        j                  |      }| j                  s	d|_        |S d|_        | j                  sJ | j
                  J |j                  d d = | j                  dv r8|j                  j                  d| j                  | j
                  fz  dg       |S | j                  dk(  r4|j                  j                  d| j                  | j
                  fz         |S )NTF)chiefworkerr\   z/job:ps	evaluator)	copydeepcopyr   isolate_session_stater   r   device_filtersextendr   )r)   config_protoupdated_configs      r+   r$  z4ParameterServerStrategyExtended._update_config_protoh  s    ]]<0N-1n*+0N(???==$$$ 	%%a(--##**??
KM
  
K	'##**
t>
>@r,   c                     | j                   duS )zAWhether this strategy indicates working in multi-worker settings.N)r   rC   s    r+   _in_multi_worker_modez5ParameterServerStrategyExtended._in_multi_worker_mode}  s    T))r,   c                 ,    t        | j                        S r?   )rw   rv   rC   s    r+   r   z5ParameterServerStrategyExtended._num_replicas_in_sync  s    t$$%%r,   c                     | j                   S r?   rv   rC   s    r+   worker_devicesz.ParameterServerStrategyExtended.worker_devices  s       r,   c                 B    | j                   D cg c]  }|g c}S c c}w r?   r6  )r)   r   s     r+   worker_devices_by_replicaz9ParameterServerStrategyExtended.worker_devices_by_replica  s    ../AQC///s   
c                     | j                   S r?   )r|   rC   s    r+   parameter_devicesz1ParameterServerStrategyExtended.parameter_devices  s    """r,   c                     t        |d       S )Nc                     | j                   S r?   )r   r  s    r+   <lambda>zBParameterServerStrategyExtended.non_slot_devices.<locals>.<lambda>  s
    qvv r,   )key)min)r)   var_lists     r+   non_slot_devicesz0ParameterServerStrategyExtended.non_slot_devices  s    x-..r,   c                      y)NTrD   rC   s    r+   experimental_between_graphz:ParameterServerStrategyExtended.experimental_between_graph  s     r,   c                     | j                   S r?   r   rC   s    r+   experimental_should_initz8ParameterServerStrategyExtended.experimental_should_init      >>r,   c                     | j                   S r?   rF  rC   s    r+   should_checkpointz1ParameterServerStrategyExtended.should_checkpoint  rH  r,   c                     | j                   S r?   rF  rC   s    r+   should_save_summaryz3ParameterServerStrategyExtended.should_save_summary  rH  r,   c                      y)z`make_dataset_iterator` and `make_numpy_iterator` use global batch size.

    `make_input_fn_iterator` assumes per-replica batching.

    Returns:
      Boolean.
    TrD   rC   s    r+   _global_batch_sizez2ParameterServerStrategyExtended._global_batch_size  s     r,   c                     |S r?   rD   )r)   replica_id_in_sync_groups     r+   _get_local_replica_idz5ParameterServerStrategyExtended._get_local_replica_id  s    ##r,   c                     |S r?   rD   )r)   r   s     r+   _get_replica_id_in_sync_groupz=ParameterServerStrategyExtended._get_replica_id_in_sync_group  s    r,   )NNNr?   )NNNN)3rE   rF   rG   rH   r$   rP   rX   rY   r   propertyr   r   r   r   r   r1   
PER_WORKERr   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r  r&  r$  r3  r   r7  r9  r;  rB  rD  rG  rJ  rL  rN  rQ  rS  rI   rJ   s   @r+   r%   r%      s   M !%# $	P  -1+/,0PZB *.(0TC . .E9 &::EEJ*7*JB+/f% <
&	FQ	6??# !%"	 ID** & & ! ! 0 0 # #/          $r,   r%   ).rH   r+  tensorflow.python.distributer   rQ   r   r   r   r   r   r	   r
   r   r   r   -tensorflow.python.distribute.cluster_resolverr   r!  r   tensorflow.python.distribute.v1r   tensorflow.python.eagerr   tensorflow.python.frameworkr   r   r   tensorflow.python.opsr   r   r   r   tensorflow.python.platformr   r   tensorflow.python.trainingr   tensorflow.python.utilr    tensorflow.python.util.tf_exportr   rS   
StrategyV1r   StrategyExtendedV1r%   rD   r,   r+   <module>rb     s    Q  R 4 7 9 2 3 5 : 6 2 / b S E + ; + + 7 6 < 4 ' 6
 @ABk< 9 9 k< Ck<^Un&G&G Ur,   