
    AVh                    B   d Z ddlZddlmZ ddlZddlmZ ddlmZ ddlm	Z	 ddlm
Z ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddlm!Z! ddlm"Z# ddlm$Z% ddl&m'Z' ddl(m)Z* ddl+m,Z, ddl-m.Z. ddl-m/Z0 dd l-m1Z1 d! Z2d" Z3 G d# d$e0jh                        Z4 G d% d&e4      Z5 G d' d(e4ejl                  e0jn                        Z7dJd)Z8 ejr                  e7e8        G d* d+ejt                        Z; e'jx                   e'jz                  e;ej|                  j~                                G d, d-e5e0j                        Z@ G d. d/eA      ZB G d0 d1e1j                        ZD G d2 d3e5e%j                  e.j                        ZG G d4 d5e,j                        ZI G d6 d7e,j                        ZJ G d8 d9eGe@      ZK G d: d;e,j                        ZL G d< d=eG      ZM	 	 	 dJd>ZN ejr                  eGeN       dJd?ZO ejr                  eKeO       dJd@ZP ejr                  e@eP       dJdAZQ ejr                  eMeQ        G dB dCeA      ZR G dD dEeR      ZS G dF dGeR      ZT G dH dI      ZUy)Kz0Various classes representing distributed values.    N)Optional)
struct_pb2)device_util)distribute_lib)packed_distributed_variable)reduce_util)values_util)context)record)composite_tensordevice)dtypes)ops)tensor)tensor_conversion_registry)tensor_shape)tensor_util)	type_spec)	array_ops)control_flow_ops)math_ops)resource_variable_ops)variable_scope)	variables)nested_structure_coder)base)saveable_object)core)
distribute)tracec                      j                   t        j                  j                  k(  r  j	                         |fi |S t        j                         j                  j                         s j                   t        j                  j                  k(  r6 j                  j                  s t        j                  |      rt        d      t        | j                          }t!        j"                          t        j$                         j'                   |f|d      S  fd}t        j$                         j)                  ||f|      S )zCUpdates variables with ON_WRITE synchronization in replica context.Cannot update non-float variables with tf.VariableAggregation.MEAN aggregation in replica context. Either change the variable dtype to float or update it in cross-replica context.Targskwargsgroupc                 B   j                   t        j                  j                  k(  r1j                  j
                  st        |t              rt        d      | j                  k(  sJ t        j                  | |j                         } j                  |fi |S )zCAggregate values and update all variables in cross replica context.r#   )aggregationvsVariableAggregationMEANdtypeis_floating
isinstance
PerReplica
ValueErrordistribute_strategyr	   apply_aggregation_update_cross_replica)strategyvaluer&   v	update_fnvars       S/home/dcms/DCMS/lib/python3.12/site-packages/tensorflow/python/distribute/values.pymerge_fnz*_on_write_update_replica.<locals>.merge_fnR   s     
B2277	7ii##E:)F%& 	& 00000

'
'%#
Na&S&&y!>v>>    )r%   r&   )r)   r*   r+   NONE_get_on_device_or_primaryr   get_strategyextended_use_merge_callr,   r-   r.   r   
is_tf_typer1   !apply_aggregation_replica_contextr	   mark_as_unsaveableget_replica_context_update
merge_call)r9   r8   r6   r&   aggregated_valuer;   s   ``    r:   _on_write_update_replicarI   4   s   __..333S224eFvFF		$	$	&	/	/	?	?	A "00555II!!{'='=e'D#$ $ 9s%""$--/77  8  ?* --/::x ; 0 0r<   c                    t        | t              rt        d      t        j                  |       s| S |t
        j                  j                  k(  r+fd}t        j                         j                  || f      S t        j                  j                  |      }t        j                         j                  j!                  ||       }|S )zBAggregate `value` to `destinations` as specified by `aggregation`.zDCannot use DistributedValues to update variables in replica context.c                 b    | j                   j                  | j                  |      d         S )Nr   )destinations)r@   broadcast_toexperimental_local_results)r5   r6   rL   s     r:   r;   z3apply_aggregation_replica_context.<locals>.merge_fnv   s7    ++

-
-e
4Q
7# , % %r<   )r%   )r/   DistributedValues	TypeErrorr   rB   r*   r+   ONLY_FIRST_REPLICAr   rE   rG   r   ReduceOpfrom_variable_aggregationr?   r@   _replica_ctx_all_reduce)r6   r)   rL   r;   	reduce_oprH   s     `   r:   rC   rC   k   s     ()
NP P				&LB**===%
 --/::x ; ! ! $$>>{KI%22 h&&y%8 r<   c                   T    e Zd ZdZd Zd Zd Zd Zed        Z	ed        Z
d Zd	 Zy
)rO   z/Base class for representing distributed values.c                 $    t        |      | _        y)z+Should only be called by subclass __init__.Ntuple_values)selfvaluess     r:   __init__zDistributedValues.__init__   s    =DLr<   c                 l    t        j                         }|| j                         S | j                  |   S z@Returns the value for the current device or raises a ValueError.)r	   get_current_replica_id_as_int_get_cross_replicarZ   r[   
replica_ids     r:   _getzDistributedValues._get   s4    ::<J$$&&\\*%%r<   c                 0    t        dt        |              )Nz}DistributedValues._get_cross_replica should be implemented by sub-classes which support cross-replica accesses. Type name is NotImplementedErrortyper[   s    r:   ra   z$DistributedValues._get_cross_replica   s"    
	T
|	% r<   c                     t        j                         }|jt        j                  t        j                               }| j
                  D ](  }t        j                  |j                        |k(  s&|c S  | j                  S | j
                  |   S )GReturns value in same replica or device if possible, else the _primary.)r	   r`   r   canonicalizecurrentrZ   r   _primary)r[   rc   current_devicer6   s       r:   r>   z+DistributedValues._get_on_device_or_primary   sy    ::<J"//0C0C0EFn<< %##ELL1^C, ]]\\*%%r<   c                      | j                   d   S )z#Returns a representative component.r   rZ   ri   s    r:   rn   zDistributedValues._primary   s     <<?r<   c                 :    t        d | j                  D              S )Nc              3   4   K   | ]  }|j                     y wNr   .0r7   s     r:   	<genexpr>z-DistributedValues._devices.<locals>.<genexpr>   s     0a0   rX   ri   s    r:   _deviceszDistributedValues._devices   s    04<<000r<   c                     dj                  d t        | j                        D              }| j                  j                  d|dS )N,
c              3   0   K   | ]  \  }}d ||fz    yw)z  %d: %sN rv   ir7   s      r:   rw   z,DistributedValues.__str__.<locals>.<genexpr>   s$      A $1
aVA   :{

}join	enumeraterZ   	__class____name__)r[   	debug_strs     r:   __str__zDistributedValues.__str__   s>    

 A(1$,,(?A AI NN33Y??r<   c                     dj                  d t        | j                        D              }| j                  j                  d|dS )Nr{   c              3   0   K   | ]  \  }}d ||fz    yw)z  %d: %rNr}   r~   s      r:   rw   z-DistributedValues.__repr__.<locals>.<genexpr>   s$      A $1
aVAr   r   r   r   )r[   
debug_reprs     r:   __repr__zDistributedValues.__repr__   s>     A(1$,,(?A AJ NN33Z@@r<   N)r   
__module____qualname____doc__r]   rd   ra   r>   propertyrn   ry   r   r   r}   r<   r:   rO   rO      sM    7!&&   1 1@
Ar<   rO   c                        e Zd ZdZ fdZed        Zd Zd Zd Z	d Z
d Zd	 Zd
 Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd%dZd Z d Z!d Z"d  Z#d! Z$d" Z%d# Z&d$ Z' xZ(S )&DistributedDelegatezAA map from device to values; acts as the same type as the values.c                     |j                  d      s|dv rt        t        |   |      S |dk(  r
t	               t        | j                         |      S )N_self_)_use_resource_variables_attribute_sentinel_distributed_containerrZ   )
startswithsuperr   __getattr__AttributeErrorgetattrrd   r[   namer   s     r:   r   zDistributedDelegate.__getattr__   s\     x D -G %G &9$?? y 499;%%r<   c                     | j                   S zReturns the per replica values.rq   ri   s    r:   r\   zDistributedDelegate.values        <<r<   c                 "    | j                         S )a  Returns the value for operations for the current device.

    Some implementations, e.g. `TPUMirroredVariable`, are not able to return the
    value type within a replica context. They can, however, return a value that
    can be used by the operations below.
    )rd   ri   s    r:   _get_as_operandz#DistributedDelegate._get_as_operand   s     99;r<   c                 (    | j                         |z   S rt   r   r[   os     r:   __add__zDistributedDelegate.__add__       !A%%r<   c                 (    || j                         z   S rt   r   r   s     r:   __radd__zDistributedDelegate.__radd__       t##%%%r<   c                 (    | j                         |z
  S rt   r   r   s     r:   __sub__zDistributedDelegate.__sub__   r   r<   c                 (    || j                         z
  S rt   r   r   s     r:   __rsub__zDistributedDelegate.__rsub__   r   r<   c                 (    | j                         |z  S rt   r   r   s     r:   __mul__zDistributedDelegate.__mul__   r   r<   c                 (    || j                         z  S rt   r   r   s     r:   __rmul__zDistributedDelegate.__rmul__   r   r<   c                 (    | j                         |z  S rt   r   r   s     r:   __truediv__zDistributedDelegate.__truediv__  r   r<   c                 (    || j                         z  S rt   r   r   s     r:   __rtruediv__z DistributedDelegate.__rtruediv__  r   r<   c                 (    | j                         |z  S rt   r   r   s     r:   __floordiv__z DistributedDelegate.__floordiv__	      !Q&&r<   c                 (    || j                         z  S rt   r   r   s     r:   __rfloordiv__z!DistributedDelegate.__rfloordiv__  s    $$&&&r<   c                 (    | j                         |z  S rt   r   r   s     r:   __mod__zDistributedDelegate.__mod__  r   r<   c                 (    || j                         z  S rt   r   r   s     r:   __rmod__zDistributedDelegate.__rmod__  r   r<   c                 (    | j                         |k  S rt   r   r   s     r:   __lt__zDistributedDelegate.__lt__  r   r<   c                 (    | j                         |k  S rt   r   r   s     r:   __le__zDistributedDelegate.__le__  r   r<   c                 (    | j                         |kD  S rt   r   r   s     r:   __gt__zDistributedDelegate.__gt__  r   r<   c                 (    | j                         |k\  S rt   r   r   s     r:   __ge__zDistributedDelegate.__ge__  r   r<   c                 (    | j                         |z  S rt   r   r   s     r:   __and__zDistributedDelegate.__and__!  r   r<   c                 (    || j                         z  S rt   r   r   s     r:   __rand__zDistributedDelegate.__rand__$  r   r<   c                 (    | j                         |z  S rt   r   r   s     r:   __or__zDistributedDelegate.__or__'  r   r<   c                 (    || j                         z  S rt   r   r   s     r:   __ror__zDistributedDelegate.__ror__*  r   r<   c                 (    | j                         |z  S rt   r   r   s     r:   __xor__zDistributedDelegate.__xor__-  r   r<   c                 (    || j                         z  S rt   r   r   s     r:   __rxor__zDistributedDelegate.__rxor__0  r   r<   c                 (    | j                         |   S rt   r   r   s     r:   __getitem__zDistributedDelegate.__getitem__3  s    !!$$r<   c                 8    t        | j                         ||      S rt   powr   )r[   r   modulos      r:   __pow__zDistributedDelegate.__pow__6  s    t##%q&11r<   c                 6    t        || j                               S rt   r   r   s     r:   __rpow__zDistributedDelegate.__rpow__9  s    q$&&())r<   c                 $    | j                          S rt   r   ri   s    r:   
__invert__zDistributedDelegate.__invert__<        """r<   c                 $    | j                          S rt   r   ri   s    r:   __neg__zDistributedDelegate.__neg__?  r   r<   c                 4    t        | j                               S rt   )absr   ri   s    r:   __abs__zDistributedDelegate.__abs__B  s    t##%&&r<   c                 l    	 | j                         j                  |      S # t        $ r	 t        cY S w xY wrt   )r   __div__r   NotImplementedr   s     r:   r   zDistributedDelegate.__div__E  s6    !!#++A..    ! 33c                 l    	 | j                         j                  |      S # t        $ r	 t        cY S w xY wrt   )r   __rdiv__r   r   r   s     r:   r   zDistributedDelegate.__rdiv__L  s6    !!#,,Q// r   c                 l    	 | j                         j                  |      S # t        $ r	 t        cY S w xY wrt   )r   
__matmul__r   r   r   s     r:   r   zDistributedDelegate.__matmul__S  s6    !!#..q11 r   c                 l    	 | j                         j                  |      S # t        $ r	 t        cY S w xY wrt   )r   __rmatmul__r   r   r   s     r:   r   zDistributedDelegate.__rmatmul__Z  s6    !!#//22 r   rt   ))r   r   r   r   r   r   r\   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   __classcell__r   s   @r:   r   r      s    I&4  &&&&&&&&''&&&'&'&&&&&&%2*##'r<   r   c                   0    e Zd ZdZed        Zed        Zy)r0   z2Holds a map from replica to unsynchronized values.c                 4    t        d | j                  D         S )Nc              3   F   K   | ]  }t        j                  |        y wrt   )r   type_spec_from_valueru   s     r:   rw   z(PerReplica._type_spec.<locals>.<genexpr>k  s     	B)
(
(
+	Bs   !)PerReplicaSpecrZ   ri   s    r:   
_type_speczPerReplica._type_spech  s    	BT\\	BD Dr<   c                     | j                   S r   rq   ri   s    r:   r\   zPerReplica.valuesm  r   r<   N)r   r   r   r   r   r   r\   r}   r<   r:   r0   r0   d  s-    :D D  r<   r0   c                 v   ~|T|j                  | j                        s9t        dj                  |j                  | j                  j                              |rt        d      t        j                         st        j                         st        d      t        j                         }| j                  |   S )z&Converts a `PerReplica` to a `Tensor`.zMIncompatible type conversion requested to type {!r} for variable of type {!r}z5PerReplica doesn't support being used as a reference.zIt looks like you are using a PerReplica object while not inside a replica context, which is not supported. Try running your op or function inside a replica context by using `strategy.run`)is_compatible_withr-   r1   formatr   rg   r   in_cross_replica_contexthas_strategyr	   r`   r\   )r9   r-   r   as_refrc   s        r:   _per_replica_to_tensorr  s  s    

u77		B
	vejj#))..9; ; 
?A A--/

%
%
'
 / 0 0
 ::<J::j!!r<   c                   P    e Zd ZdZdgZ ed       Zd Zd Zed        Z	d Z
d Zy	)
r   z&Type specification for a `PerReplica`._value_specsc                     t         S rt   r0   ri   s    r:   <lambda>zPerReplicaSpec.<lambda>  s    Z r<   c                 $    t        |      | _        y rt   )rY   r  )r[   value_specss     r:   r]   zPerReplicaSpec.__init__  s    k*Dr<   c                     | j                   S rt   r  ri   s    r:   
_serializezPerReplicaSpec._serialize  s    r<   c                     | j                   S rt   r
  ri   s    r:   _component_specszPerReplicaSpec._component_specs      r<   c                 z    t        j                         }||j                  dkD  rt        d      |j                  S )N   zJFlattening a PerReplica to components is not supported in replica context.)r   rE   num_replicas_in_syncr1   rZ   )r[   r6   replica_contexts      r:   _to_componentszPerReplicaSpec._to_components  sA    $88:O"'K'Ka'O  ==r<   c                     t        |      S rt   r  )r[   tensor_lists     r:   _from_componentszPerReplicaSpec._from_components  s    k""r<   N)r   r   r   r   	__slots__r   
value_typer]   r  r  r  r  r}   r<   r:   r   r     s@    .)/0*+  #r<   r   c                   "    e Zd ZdZd Zd Zd Zy)Mirroredz:Holds a map from replica to values which are kept in sync.c                 "    | j                         S rt   r>   ri   s    r:   ra   zMirrored._get_cross_replica  s    ))++r<   c                 h    | j                         }t        |dd       }|rt        |      r |       S |S )N_as_graph_element)rd   r   callable)r[   objconv_fns      r:   r  zMirrored._as_graph_element  s3    
))+Cc.5G8G$YJr<   c                      yNTr}   ri   s    r:   _is_mirroredzMirrored._is_mirrored      r<   N)r   r   r   r   ra   r  r$  r}   r<   r:   r  r    s    B,r<   r  c                   "    e Zd ZdZd Zd Zd Zy)DistributedVarOpz'A class that looks like `tf.Operation`.c                 <    || _         || _        || _        || _        y rt   )r   graph	tracebackrh   )r[   r   r)  r*  typs        r:   r]   zDistributedVarOp.__init__  s    DIDJDNDIr<   c                    t        || j                        st        | j                  |j                  k(  xrO | j                  |j                  k(  xr4 | j
                  |j
                  k(  xr | j                  |j                  k(  S rt   )r/   r   rg   r   r)  r*  rh   r   s     r:   __eq__zDistributedVarOp.__eq__  sm    a(II BDJJ!''$9 BNNakk)B.2ii166.ACr<   c                     t        | j                  | j                  t        | j                        | j
                  f      S rt   )hashr   r)  rY   r*  rh   ri   s    r:   __hash__zDistributedVarOp.__hash__  s+    DJJdnn(=tyyIJJr<   N)r   r   r   r   r]   r-  r0  r}   r<   r:   r'  r'    s    /CKr<   r'  c                   N    e Zd ZdZd Zd Zd ZddZd Zd Z	d	e
fd
Zd	efdZy)DistributedVariableTraceTypez)TraceType of DistributedVariable objects.c                 z    || _         t        |j                  j                               |j                  f| _        y rt   )distributed_variablerY   shapeas_listr-   
components)r[   r4  s     r:   r]   z%DistributedVariableTraceType.__init__  s4     4D177??AB+113DOr<   c                     | |k(  S rt   r}   r[   others     r:   is_subtype_ofz*DistributedVariableTraceType.is_subtype_of  s    5=r<   c                 4     t         fd|D              r S d S )Nc              3   (   K   | ]	  }|k(    y wrt   r}   )rv   r:  r[   s     r:   rw   zNDistributedVariableTraceType.most_specific_common_supertype.<locals>.<genexpr>  s     9tu}9s   )all)r[   otherss   ` r:   most_specific_common_supertypez;DistributedVariableTraceType.most_specific_common_supertype  s    9&994CtCr<   Nc                     | j                   S rt   )r4  )r[   placeholder_contexts     r:   placeholder_valuez.DistributedVariableTraceType.placeholder_value  s    $$$r<   c                     g S rt   r}   )r[   r6   s     r:   
to_tensorsz'DistributedVariableTraceType.to_tensors  s    Ir<   c                     |S rt   r}   )r[   r6   _s      r:   castz!DistributedVariableTraceType.cast  s    Lr<   returnc                 ,    t        | j                        S rt   )r/  r7  ri   s    r:   r0  z%DistributedVariableTraceType.__hash__  s      r<   c                 V    t        |t              sy| j                  |j                  k(  S NF)r/   r2  r7  r9  s     r:   r-  z#DistributedVariableTraceType.__eq__  s%    e9:??e....r<   rt   )r   r   r   r   r]   r;  r@  rC  rE  rH  intr0  boolr-  r}   r<   r:   r2  r2    s<    13
D%! !/T /r<   r2  c                       e Zd ZdZdC fd	Zd Zd ZdCdZed        Z	d Z
d Zed	        Zed
        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zed        ZdCdZed        Zd Zd Zed        Zed        Zed        Zde jB                  fdZ"dCdZ#ede$jJ                  fd        Z&ed!        Z'd" Z(d# Z)d$ Z*d% Z+d& Z,d' Z-dDd(Z.dDd)Z/dDd*Z0dEd+Z1dEd,Z2dEd-Z3dEd.Z4dEd/Z5dEd0Z6dEd1Z7d2 Z8d3 Z9d4 Z:d5 Z;d6 Z<d7 Z=d8 Z>d9 Z?dFd:Z@	 	 dGd;eAeBj                     d<eAeD   deEj                  fd=ZG	 	 	 dHd>ZHd? ZId@ ZJedA        ZKdB ZL xZMS )IDistributedVariablez&Holds a map from replica to variables.c                 h   |t         j                  j                  k(  r$|d   j                  j                  st        d      || _        || _        t        t        | +  |       | j                  j                  j                  d      d   | _        |D ]g  }t        |t         j"                        r1t%        |d      r%t'        j(                  |       |j*                  _        Nt'        j(                  |       |_        i t/        j0                         rt3        |dd      ryd| j                  z  }t%        |d   d      r>t5        j6                  t9        d	 |D        g       |
      }|D ]	  }||_         d | _        n$t5        j6                  ||
      | _        nd | _        d| _        d | _        || _         y )Nr   zcreating distributed tf.Variable with aggregation=MEAN and a non-floating dtype is not supported, please use a different aggregation or dtype:handle%_enable_packed_variable_in_eager_modeFz
%s/packed/_varsc              3   4   K   | ]  }|j                     y wrt   )rU  )rv   r6   s     r:   rw   z/DistributedVariable.__init__.<locals>.<genexpr>   s     11rx   r   )!variables_libr+   r,   r-   r.   r1   _distribute_strategy_aggregationr   rP  r]   rn   r   split_common_namer/   r   CompositeTensorhasattrweakrefrefrS  r   r   #executing_eagerly_outside_functionsr   packedPackedDistributedVariablesum_packed_var_keras_initialized_initializer_op_policy)
r[   r5   r\   r)   
var_policyr7   r   
packed_varr6   r   s
            r:   r]   zDistributedVariable.__init__  s   }88===1IOO''!" " !)D#D	
t-f5**005a8D  5 
A'77	8W
X>*1++d*;'#*;;t#4 5 ..0W956BD---d	G	$
 551&126TC
 	)E(%
	) ";;FNd $D
  D DLr<   c           	         t        j                  | j                        5  g }| j                  D ]O  }t	        j
                  |j
                        5  |j                  t        j                  ||             ddd       Q 	 ddd        t        |       | j                  | j                  t        j                  | j                  |            }||t        |       <   |S # 1 sw Y   xY w# 1 sw Y   lxY w)ab  Perform a deepcopy of the `DistributedVariable`.

    Unlike the deepcopy of a regular tf.Variable, this keeps the original
    strategy and devices of the `DistributedVariable`.  To avoid confusion
    with the behavior of deepcopy on a regular `Variable` (which does
    copy into new devices), we only allow a deepcopy of a `DistributedVariable`
    within its originating strategy scope.

    Args:
      memo: The memoization object for `deepcopy`.

    Returns:
      A deep copy of the current `DistributedVariable`.

    Raises:
      RuntimeError: If trying to deepcopy into a different strategy.
    N)r5   r\   r)   ri  )r   enter_or_assert_strategyrY  rZ   r   r   appendcopydeepcopyrh   rZ  rh  id)r[   memo
new_valuesr6   copied_variables        r:   __deepcopy__z DistributedVariable.__deepcopy__9  s    $ 
	0	01J1J	K 8j<< 8%ZZ% 	8


DMM%6
7	8 	888 !d4j**%%==t4	6O %DDN	8 	8	8 8s#   1C-&C!7
C-!C*&C--C6c                 L    | j                   d uxr t        j                          S rt   )re  r	   is_saving_non_distributedri   s    r:   _use_packed_variablez(DistributedVariable._use_packed_variable\  s+     4' 511335r<   c                    t        j                         r| j                  j                         S | j	                         r| j
                  j                         S | j                  j                         }| j                  dd D ]&  }t        j                  ||j                               }( t        j                  || j                  d   j                         |      }|S )zIdentifies if all the component variables are initialized.

    Args:
      name: Name of the final `logical_and` op.

    Returns:
      The op that evaluates to True or False depending on if all the
      component variables are initialized.
    r  rW  )	r	   rv  rn   is_initializedrw  re  rZ   r   logical_and)r[   r   resultr7   s       r:   rz  z"DistributedVariable.is_initializedb  s     ,,.]]))++  ",,..]]))+F
 \\!B @##FA,<,<,>?f@!!R //1>FMr<   c                     t        j                         r| j                  j                  S | j                  r| j                  }|S t        j                  t        d | j                  D                    }|S )Nc              3   4   K   | ]  }|j                     y wrt   )initializerru   s     r:   rw   z2DistributedVariable.initializer.<locals>.<genexpr>  s     4!4rx   )	r	   rv  rn   r  rg  r   r'   rY   rZ   )r[   init_ops     r:   r  zDistributedVariable.initializer{  sb    ,,.]]&&&$$g N !&&
4t||4
46gNr<   c                 >    | j                         j                         S rt   )r>   initialized_valueri   s    r:   r  z%DistributedVariable.initialized_value      ))+==??r<   c                 V    | j                   d uxr | j                   j                         S rt   )rh  r$  ri   s    r:   r$  z DistributedVariable._is_mirrored  s#    LL$G4<<+D+D+FGr<   c                 6    | j                         j                  S rt   )r>   initial_valueri   s    r:   r  z!DistributedVariable.initial_value  s    ))+999r<   c                 .    | j                   j                  S rt   )rn   
constraintri   s    r:   r  zDistributedVariable.constraint      ==###r<   c                 .    | j                   j                  S rt   )rn   r)  ri   s    r:   r)  zDistributedVariable.graph      ==r<   c                     | j                   S rt   )r\  ri   s    r:   _shared_namez DistributedVariable._shared_name  r  r<   c                 .    | j                   j                  S rt   )rn   
_unique_idri   s    r:   r  zDistributedVariable._unique_id  r  r<   c                 .    | j                   j                  S )z7Lets Optimizers know which graph this variable is from.)rn   
_graph_keyri   s    r:   r  zDistributedVariable._graph_key  s     ==###r<   c                 .    | j                   j                  S rt   )rn   r   ri   s    r:   r   zDistributedVariable.name  s    ==r<   c                 .    | j                   j                  S rt   )rn   r-   ri   s    r:   r-   zDistributedVariable.dtype  r  r<   c                 .    | j                   j                  S rt   )rn   r5  ri   s    r:   r5  zDistributedVariable.shape  r  r<   c                 .    | j                   j                  S rt   )rn   synchronizationri   s    r:   r  z#DistributedVariable.synchronization  s    ==(((r<   c                     | j                   S rt   rZ  ri   s    r:   r)   zDistributedVariable.aggregation  r  r<   c                 <    | j                         r| j                  S y rt   )rw  re  ri   s    r:   _packed_variablez$DistributedVariable._packed_variable  s      "r<   c                    t        j                         r| j                  j                  S t        j                         }|t        d      | j                         r| j                  j                  S | j                  |   j                  S )NztDistributedVariable.handle is not available outside the replica context or a `tf.distribute.Strategy.update()` call.)	r	   rv  rn   rS  r`   r1   rw  re  rZ   rb   s     r:   rS  zDistributedVariable.handle  s|    ,,.]]!!!::<JAB B 
	"	"	$&&&\\*%,,,r<   c                 @    | j                         j                  |      S rt   )r>   eval)r[   sessions     r:   r  zDistributedVariable.eval  s    ))+0099r<   c                 .    | j                   j                  S rt   )rn   _save_slice_infori   s    r:   r  z$DistributedVariable._save_slice_info  s    ==)))r<   c                 6    | j                   j                         S rt   )rn   _get_save_slice_infori   s    r:   r  z(DistributedVariable._get_save_slice_info  s    ==--//r<   c                 H    | j                   D ]  }|j                  |        y rt   )rZ   _set_save_slice_info)r[   save_slice_infor7   s      r:   r  z(DistributedVariable._set_save_slice_info  s#    \\ ._-.r<   c                 6    | j                         j                  S rt   )r>   r   ri   s    r:   r   zDistributedVariable.device  s    ))+222r<   c                 .    | j                   j                  S rt   )rn   	trainableri   s    r:   r  zDistributedVariable.trainable  s    =="""r<   c                     | j                   S rt   )rY  ri   s    r:   r2   z'DistributedVariable.distribute_strategy  s    $$$r<   rI  c                 6    | j                   j                         S rt   )rn   	get_shaperi   s    r:   r  zDistributedVariable.get_shape  s    ==""$$r<   c                 :    | j                   j                  |      S )N)export_scope)rn   to_proto)r[   r  s     r:   r  zDistributedVariable.to_proto  s    ==!!|!<<r<   c                    t        j                         r| j                  j                  S t	        j
                         rt        | j                  j                  j                  | j                  j                  j                  | j                  j                  j                  | j                  j                  j                        S | j                         j                  S rt   )r	   rv  rn   opr   r   r'  r   r)  r*  rh   rd   ri   s    r:   r  zDistributedVariable.op  s    ,,.]] ..0dmm..33T]]5E5E5K5K"mm..88$--:J:J:O:OQ Q99;>>r<   c                 .    | j                   j                  S rt   )rn   _in_graph_moderi   s    r:   r  z"DistributedVariable._in_graph_mode  s    =='''r<   c                     | j                   |   }| j                         r%| j                  j                  |j                        S |S )z8Returns the value on a device with the given replica_id.)rZ   rw  re  	on_devicer   )r[   rc   r6   s      r:   _get_replicaz DistributedVariable._get_replica  s<    LL$E  "''55lr<   c                     t        j                         r| j                  S t        j                         }|| j	                         S | j                  |      S r_   )r	   rv  rn   r`   ra   r  rb   s     r:   rd   zDistributedVariable._get  sL    ,,.]]::<J$$&&z**r<   c                    t        j                         r| j                  S t        j                         }|t	        j
                  t	        j                               }t        | j                        D ]:  \  }}t	        j
                  |j                        |k(  s)| j                  |      c S  | j                  d      S | j                  |      S )rk   r   )r	   rv  rn   r`   r   rl   rm   r   rZ   r   r  )r[   rc   ro   r   r6   s        r:   r>   z-DistributedVariable._get_on_device_or_primary  s    ,,.]]::<J"//0C0C0EFn- &(!U##ELL1^C""1%
%& q!!z**r<   c                    t        j                         r| j                  j                         S t	        j
                  | j                        5  t        j                  | j                               cd d d        S # 1 sw Y   y xY wrt   )
r	   rv  rn   
read_valuer   rl  rY  r   identityrd   ri   s    r:   r  zDistributedVariable.read_value   s^    ,,.]]%%''		0	01J1J	K -		,- - -s   #A;;Bc                     t        j                         r| j                  j                         S | j                  r| j                  j                  |       S | j                         j                         S rt   )r	   rv  rn   r6   rh  r>   ri   s    r:   r6   zDistributedVariable.value&  sU    ,,.]]  ""||\\%%))+1133r<   c                 |    t        j                         r| j                         j                         S t	        d      )NzNDistributedVariable.numpy() is only available when eager execution is enabled.)r
   executing_eagerlyr  numpyrg   ri   s    r:   r  zDistributedVariable.numpy-  s9      "__$$&& !C D Dr<   c                     t        j                         r| j                  j                  ||||      S | j                  r | j                  j                  | ||||      S t        j
                  | ||||      S Nuse_lockingr   r  )r	   rv  rn   
assign_subrh  on_write_assign_subr[   r6   r  r   r  s        r:   r  zDistributedVariable.assign_sub4  }    ,,.]]%%e[$
KK||\\$$

! % ! ! **e4JP Pr<   c                     t        j                         r| j                  j                  ||||      S | j                  r | j                  j                  | ||||      S t        j
                  | ||||      S r  )r	   rv  rn   
assign_addrh  on_write_assign_addr  s        r:   r  zDistributedVariable.assign_addA  r  r<   c                     t        j                         r| j                  j                  ||||      S | j                  r | j                  j                  | ||||      S t        j
                  | ||||      S r  )r	   rv  rn   assignrh  on_write_assignr  s        r:   r  zDistributedVariable.assignN  s}    ,,.]]!!%dJGG||\\  

! ! ! ! &&e4JP Pr<   c                     t        j                         r| j                  j                  |||      S | j                  r| j                  j                  | |||      S t        j                  | |||      S Nr  r   )r	   rv  rn   scatter_subrh  r[   sparse_deltar  r   s       r:   r  zDistributedVariable.scatter_sub[  s    ,,.]]&&|[$GG||\\%%
+D & B B""l$@ @r<   c                     t        j                         r| j                  j                  |||      S | j                  r| j                  j                  | |||      S t        j                  | |||      S r  )r	   rv  rn   scatter_addrh  r  s       r:   r  zDistributedVariable.scatter_addd  r  r<   c                     t        j                         r| j                  j                  |||      S | j                  r| j                  j                  | |||      S t        j                  | |||      S r  )r	   rv  rn   scatter_mulrh  r  s       r:   r  zDistributedVariable.scatter_mulm  r  r<   c                     t        j                         r| j                  j                  |||      S | j                  r| j                  j                  | |||      S t        j                  | |||      S r  )r	   rv  rn   scatter_divrh  r  s       r:   r  zDistributedVariable.scatter_divv  r  r<   c                     t        j                         r| j                  j                  |||      S | j                  r| j                  j                  | |||      S t        j                  | |||      S r  )r	   rv  rn   scatter_minrh  r  s       r:   r  zDistributedVariable.scatter_min  r  r<   c                     t        j                         r| j                  j                  |||      S | j                  r| j                  j                  | |||      S t        j                  | |||      S r  )r	   rv  rn   scatter_maxrh  r  s       r:   r  zDistributedVariable.scatter_max  r  r<   c                     t        j                         r| j                  j                  |||      S | j                  r| j                  j                  | |||      S t        j                  | |||      S r  )r	   rv  rn   scatter_updaterh  r  s       r:   r  z"DistributedVariable.scatter_update  ss    ,,.]])),TJJ||\\((
+D ) B B%%l$@ @r<   c                     t        |       S rt   )r2  r[   rG  s     r:   __tf_tracing_type__z'DistributedVariable.__tf_tracing_type__  s    '--r<   c                 J      j                   f fd	}t        j                  |iS )zOverrides Trackable method.

    This allows both name-based and object-based save and restore of
    DistributedVariables.

    Returns:
      A dictionary mapping attribute names to `SaveableObject` factories.
    c                 2    t        j                  |       S rt   )_DistributedVariableSaveablern   r   r[   s    r:   _saveable_factoryzODistributedVariable._gather_saveables_for_checkpoint.<locals>._saveable_factory  s    )$tDDr<   r\  	trackableVARIABLE_VALUE_KEYr[   r  s   ` r:    _gather_saveables_for_checkpointz4DistributedVariable._gather_saveables_for_checkpoint  s(      $00 E ((*;<<r<   c                     t        j                         r| j                  j                         S | j                  r| j                  j                  |       S t        dt        |              )NzDistributedVariable._as_graph_element requires a valid VariablePolicy. Please set the policy via the `var_policy` argument in the constructor, or override this method in sub-classes which support cross-replica accesses. Type name is )r	   rv  rn   r  rh  rg   rh   ri   s    r:   r  z%DistributedVariable._as_graph_element  s_    ,,.]],,..||\\++D11
	 T
|		% r<   c                     t        j                         r| j                  S | j                  r| j                  j	                  |       S t        dt        |              )NzDistributedVariable._get_cross_replica requires a valid VariablePolicy. Please set the policy via the `var_policy` argument in the constructor, or override this method in sub-classes which support cross-replica accesses. Type name is )r	   rv  rn   rh  ra   rg   rh   ri   s    r:   ra   z&DistributedVariable._get_cross_replica  sV    ,,.]]||\\,,T22
	 T
|		% r<   c                     t        j                          | j                  j                  j	                  | ||f|d      S )ag  Applies updates across replicas.

    Args:
      update_fn: A callable to pass to `strategy.extended.update` to update the
        variable. It should has the same signature as `Variable.assign()`.
      value: value to be passed to `update_fn`.
      **kwargs: remaining arguments to `update_fn`.

    Returns:
      Updated variable or `tf.Operation`.
    Tr$   )r	   rD   r2   r@   updater[   r8   r6   r&   s       r:   r4   z)DistributedVariable._update_cross_replica  sC     ""$##,,33iuhvT 4 C Cr<   c                     | j                   r | j                   j                  | ||fi |S t        dt        |              )a@  Applies updates in one replica.

    Args:
      update_fn: A callable to update the variable. It should has the same
        signature as `Variable.assign()`.
      value: value to be passed to `update_fn`.
      **kwargs: remaining arguments to `update_fn`.

    Returns:
      Updated variable or `tf.Operation`.
    zDistributedVariable._update_replica requires a valid VariablePolicy. Please set the policy via the `var_policy` argument in the constructor, or override this method in sub-classes which support cross-replica accesses. Type name is )rh  _update_replicarg   rh   r  s       r:   r  z#DistributedVariable._update_replica  sM     ||)T\\))$	5KFKK
	 T
|		% r<   c                    t        j                         r || j                  |fi |S t        j                  | j
                        5  t        j                         rWt        j                         }|$| j                  |      } |||fi |cddd       S  | j                  ||fi |cddd       S t        j                  | j
                          | j                  ||fi |cddd       S # 1 sw Y   yxY w)a  Applies updates depending on the context.

    The method calls `_update_replica` in replica context,
    `_update_cross_replica` in cross replica context, and `update_fn` in update
    context.

    If `read_value` is True, the method returns the updated Variable. If
    `read_value` is False, the method returns the update `tf.Operation`.

    Args:
      update_fn: A callable to pass to `strategy.extended.update` to update the
        variable. It should have the same signature as `Variable.assign()`.
      value: value to be passed to `update_fn`.
      **kwargs: keyword arguments to `update_fn`.

    Returns:
      Updated variable or `tf.Operation`.

    N)r	   rv  rn   r   rl  r2   r   get_update_replica_idr  r4   assert_replica_contextr  )r[   r8   r6   r&   update_replica_idreplica_values         r:   rF   zDistributedVariable._update  s    ( ,,.t}}e6v66		0	01I1I	J 	@		0	0	2*@@B(++,=>-=%:6:	@ 	@ *t)))UEfE	@ 	@ 	**4+C+CD#t##Iu??	@ 	@ 	@s   AC0C042C00C9c                      y)z6Pass resource_variable_ops.is_resource_variable check.Nr}   ri   s    r:    _should_act_as_resource_variablez4DistributedVariable._should_act_as_resource_variable  s    r<   c                 (   t        j                         r#t        j                  | j                  |||      S t        j                  | j                        5  t        j                  | j                         |||      cddd       S # 1 sw Y   yxY w) Converts a variable to a tensor.r-   r   r   N)	r	   rv  r   convert_to_tensorrn   r   rl  rY  rd   r[   r-   r   r   s       r:   _dense_var_to_tensorz(DistributedVariable._dense_var_to_tensor  sv    ,,.""
--u4@ @		0	01J1J	K >""
))+Uf>> > >s   'BBr-   r   c                 &    | j                  ||      S rt   r   )r[   r-   r   s      r:   __tf_tensor__z!DistributedVariable.__tf_tensor__  s     $$UD11r<   c           
          | j                   j                  d|||d|}| j                  D cg c]  }|| j                   k7  s| c}D ]  }|j                  j	                         r&|j                   |j                  d|||d|       C|| j                      ||<   || j                   j                     ||j                  <   |j                  |j                          || j                      || <   || j                   j                     || <   |j                  |        | j                  U|| j                   j                     || j                  j                  <   |j                  | j                  j                         |S c c}w )N)
object_map
tensor_mapoptionsr}   )
rn   _export_to_saved_model_graphrZ   experimental_variable_policy_expand_distributed_variablesextendrS  rm  re  packed_handle)r[   r  r  r  r&   resource_listr7   s          r:   r  z0DistributedVariable._export_to_saved_model_graph!  s    ?DMM>>  	M
 <Admm);a< '

.
.((*+*A** %% 		 #4==1
1)$--*>*>?
188QXX&' "$--0Jt!$--"6"67Jt#3=
--

4 j!!//04++99:) =s   FFc           	      X   | |vrt         j                  j                  | j                        j	                  dd      j                         }t        j                  |      5  t        j                  | j                  | j                  | j                  | j                  | j                  | j                        }ddd       || <   ||    }t        j                  |j                        5  |j                  | j!                                ddd       y# 1 sw Y   [xY w# 1 sw Y   yxY w)zFor implementing `Trackable`.CPUr   )device_typedevice_index)r  r5  r-   r   r2   r)   N)pydev
DeviceSpecfrom_stringr   replace	to_stringr   r   UninitializedVariabler  r5  r-   r  rY  rZ  r  r  )r[   r  	op_devicenew_vardestination_vars        r:   _copy_trackable_to_cpuz*DistributedVariable._copy_trackable_to_cpuC  s    :""..t{{;CC! D --6Y[ ::i  +'==nn****"" $ 9 9))++ !j !&O	O**	+ 0T__./0 0+ +0 0s   "AD+ D D D)c                 `    t        j                  | ||       t        j                  | ||       y)a  Update a SavedObject proto for the caller.

    If a DistributedVariable object supports this method, it will be called when
    saving with a pre-built `SavedObject` proto representing the object, plus an
    instance of `SaveOptions`. This method is then free to modify that proto
    instance.

    `DistributedVariable` with `AUTO` or `ON_WRITE` synchronization optionally
    write out information about their components to the
    `experimental_distributed_variable_components` field of a
    `SavedVariable` (depending on the `SaveOptions` variable policy).

    Args:
      proto: A pre-built `SavedObject` proto for this object. It is assumed this
        will be a `SavedVariable` instance.
      options: A `SaveOptions` instance.
    N)r   (write_object_proto_for_resource_variabler	   write_object_proto)r[   protor  s      r:   _write_object_protoz'DistributedVariable._write_object_protoX  s-    $ BBeW ""48r<   c                      yr#  r}   ri   s    r:   is_distributed_variablez+DistributedVariable.is_distributed_variablep  s    r<   c                 ~    |j                   }|j                  | |       t        j                  d|g| gd d        | S )Ncaptured_valuec                     | gS rt   r}   xs    r:   r  zIDistributedVariable.__tf_experimental_restore_capture__.<locals>.<lambda>{  s    QC r<   c                     | gS rt   r}   r&  s    r:   r  zIDistributedVariable.__tf_experimental_restore_capture__.<locals>.<lambda>|  s    A3 r<   )backward_functionforward_function)r)  replace_capturer   record_operation)r[   concrete_functioninternal_capturer)  s       r:   #__tf_experimental_restore_capture__z7DistributedVariable.__tf_experimental_restore_capture__t  sF    ##E	$ 01
+,tf'&( Kr<   rt   FNTFNNNF)NN)NNN)Nr   r   r   r   r]   rt  rw  rz  r   r  r  r$  r  r  r)  r  r  r  r   r-   r5  r  r)   r  rS  r  r  r  r  r   r  r2   r   TensorShaper  r  r   	Operationr  r  r  rd   r>   r  r6   r  r  r  r  r  r  r  r  r  r  r  r  r  r  ra   r4   r  rF   r  r   r   r   DTypestr
tensor_libTensorr  r  r  r   r"  r/  r   r   s   @r:   rP  rP    s   .9v!F52 
 
@H : : $ $     $ $ $ $       ) )    
 - -: * *0. 3 3 # # % %%11 %= 	#-- 	 	 ( (++-4DPPP@@@@@@@.=C ,@B	> 59*.2#FLL12"3-23=3D3D2 /3.2+/ D0*90  	r<   rP  c                   (     e Zd ZdZ fdZd Z xZS )r  z8Class for defining how to restore a DistributedVariable.c                     || _         | j                   j                  st        d      |j                  j                  |||      \  }}t        t
        |   |||       y )NzThe VariablePolicy of the argument `distributed_variable` must be set to create a _DistributedVariableSaveable. Please set it via the `var_policy` argument in the constructor of DistributedVariable.)_distributed_variablerh  r1   get_saveabler   r  r]   )r[   r4  primary_variabler   r   specr   s         r:   r]   z%_DistributedVariableSaveable.__init__  se    !5D%%--Q 
 (//<<.6LFD	
&6vtTJr<   c                 j    |\  }| j                   j                  j                  | j                   |      S z*Restore the same value into all variables.)r;  rh  get_restore_opsr[   restored_tensorsrestored_shapesr   s       r:   restorez$_DistributedVariableSaveable.restore  s3    GF%%--==""F, ,r<   r   r   r   r   r]   rE  r   r   s   @r:   r  r    s    @
K,r<   r  c                   (     e Zd ZdZ fdZd Z xZS )_MirroredSaveablez5Class for defining how to restore a MirroredVariable.c                     || _         t        j                  | j                   ||      \  }}t        t        |   |||       y rt   )_mirrored_variabler	   get_on_write_saveabler   rH  r]   )r[   mirrored_variabler=  r   r   r>  r   s         r:   r]   z_MirroredSaveable.__init__  sA    /D44T5L5L5EtMLFD	
T+FD$?r<   c                 J    |\  }t        j                  | j                  |      S r@  )r	   get_on_write_restore_opsrJ  rB  s       r:   rE  z_MirroredSaveable.restore  s"    GF//0G0GPPr<   rF  r   s   @r:   rH  rH    s    =@Qr<   rH  c                   \     e Zd ZdZd Zd Z fdZ fdZ fdZd Z	d Z
d	 Zdd
Z xZS )MirroredVariablezDHolds a map from replica to variables whose values are kept in sync.c                 ,    t         j                  |       S rt   )r  r$  ri   s    r:   r$  zMirroredVariable._is_mirrored  s      &&r<   c                     t        | ||fi |S rt   )rI   r  s       r:   r  z MirroredVariable._update_replica  s    #D)UEfEEr<   c                    t        j                         r | j                  j                  |i |S | j                  t
        j                  j                  k7  r[| j                  t
        j                  j                  k7  r4t        t         j                  j                  d| j                              t        t        |   |i |S )Nr  op_namer)   )r	   rv  rn   r  rZ  r*   r+   rQ   r=   rg   scatter_error_msgr   r   rP  r[   r%   r&   r   s      r:   r  zMirroredVariable.scatter_min      ,,.&T]]&&777R33FFFR33888

'
'
.
.#1B1B / DE E !44dEfEEr<   c                    t        j                         r | j                  j                  |i |S | j                  t
        j                  j                  k7  r[| j                  t
        j                  j                  k7  r4t        t         j                  j                  d| j                              t        t        |   |i |S )Nr  rT  )r	   rv  rn   r  rZ  r*   r+   rQ   r=   rg   rV  r   r   rP  rW  s      r:   r  zMirroredVariable.scatter_max  rX  r<   c                    t        j                         r | j                  j                  |i |S | j                  t
        j                  j                  k7  r[| j                  t
        j                  j                  k7  r4t        t         j                  j                  d| j                              t        t        |   |i |S )Nr  rT  )r	   rv  rn   r  rZ  r*   r+   rQ   r=   rg   rV  r   r   rP  rW  s      r:   r  zMirroredVariable.scatter_update  s    ,,.)T]]))4:6::R33FFFR33888

'
'
.
.&D4E4E / GH H !47HHHr<   c                 R    t        j                  t        j                  |             S rt   )r   r  r  ra   ri   s    r:   ra   z#MirroredVariable._get_cross_replica  s      h99$?@@r<   c                 >    | j                         j                         S rt   r>   r  ri   s    r:   r  z"MirroredVariable._as_graph_element  r  r<   c                 J      j                   f fd	}t        j                  |iS )zOverrides Trackable method.

    This allows both name-based and object-based save and restore of
    MirroredVariables.

    Returns:
      A dictionary mapping attribute names to `SaveableObject` factories.
    c                 2    t        j                  |       S rt   )rH  rn   r  s    r:   r  zLMirroredVariable._gather_saveables_for_checkpoint.<locals>._saveable_factory  s    tT]]D99r<   r  r  s   ` r:   r  z1MirroredVariable._gather_saveables_for_checkpoint  s'      $00 : ((*;<<r<   c                 j    |rt        d      t        j                  | j                         |||      S )r  zYou may be using variable created under distribute strategy in TF 1.x control flows. Try explicitly converting the variable to Tensor using variable.read_value(), or switch to TF 2.x.r  )r1   r   r  rd   r  s       r:   r   z%MirroredVariable._dense_var_to_tensor  s?      >? ?   		5tF< <r<   r2  )r   r   r   r   r$  r  r  r  r  ra   r  r  r   r   r   s   @r:   rP  rP    s9    L'FFFIA
@=<r<   rP  c                   (     e Zd ZdZ fdZd Z xZS )_SyncOnReadSaveablez7Class for defining how to restore a SyncOnReadVariable.c                     || _         t        j                  ||j                  |      \  }}t        t
        |   |||       y rt   )_sync_on_read_variabler	   get_on_read_saveablern   r   rb  r]   )r[   sync_on_read_variabler   r   r>  r   s        r:   r]   z_SyncOnReadSaveable.__init__  sD    "7D334==tELFD 

t-fdDAr<   c                 t    |\  }t        j                  | j                  || j                  j                        S r@  )r	   get_on_read_restore_opsrd  r)   rB  s       r:   rE  z_SyncOnReadSaveable.restore  s6    GF..##V##//1 1r<   rF  r   s   @r:   rb  rb    s    ?B1r<   rb  c                        e Zd ZdZd Z fdZd fd	Zd fd	Zd fd	Zd Z	d Z
d	 Zd
 Zd Zd Zd Zd Zd Z fdZd Zd Zd ZddZ xZS )SyncOnReadVariablezGHolds a map from replica to variables whose values are reduced on save.c                 2     || j                         |fi |S rt   r  r  s       r:   r  z"SyncOnReadVariable._update_replica  s    T335uGGGr<   c                     t        j                  | j                        5  t        t        |          cddd       S # 1 sw Y   yxY w)a  Returns the value of SyncOnReadVariable based on surrounding context.

    If called under a non-default replica-context, returns the corresponding
    variable on that replica.
    If called under default replica-context or cross-replica context, returns
    the synced value.
    N)r   rl  rY  r   rj  rd   r[   r   s    r:   rd   zSyncOnReadVariable._get  s<     
	0	01J1J	K 4%t134 4 4s	   =Ac                    t        j                         r| j                  j                  ||||      S t	        j
                  | j                        5  t	        j                         rIt        j                         s5t        j                          t        j                  | ||      cd d d        S t        t        |   ||||      cd d d        S # 1 sw Y   y xY wNr  )r	   rv  rn   r  r   rl  rY  r   in_replica_update_contextrD    on_read_assign_sub_cross_replicar   rj  r[   r6   r  r   r  r   s        r:   r  zSyncOnReadVariable.assign_sub!      ,,.]]%%e[$
KK		0	01J1J	K L

1
1
3335&&(;;%J0	L L '&&+[$
KL L L   AC1CCc                    t        j                         r| j                  j                  ||||      S t	        j
                  | j                        5  t	        j                         rIt        j                         s5t        j                          t        j                  | ||      cd d d        S t        t        |   ||||      cd d d        S # 1 sw Y   y xY wro  )r	   rv  rn   r  r   rl  rY  r   rq  rD    on_read_assign_add_cross_replicar   rj  rs  s        r:   r  zSyncOnReadVariable.assign_add.  rt  ru  c                    t        j                         r| j                  j                  ||||      S t	        j
                  | j                        5  t	        j                         rIt        j                         s5t        j                          t        j                  | ||      cd d d        S t        t        |   ||||      cd d d        S # 1 sw Y   y xY wro  )r	   rv  rn   r  r   rl  rY  r   rq  rD   on_read_assign_cross_replicar   rj  rs  s        r:   r  zSyncOnReadVariable.assign;  s    ,,.]]!!%dJGG		0	01J1J	K B

1
1
3335&&(77%J0	B B '5e[$6@BB B Bru  c                      t        d| d      )Nz:Variables with `synchronization=ON_READ` doesn't support ``rg   r[   methods     r:   _scatter_not_implementedz+SyncOnReadVariable._scatter_not_implementedH  s    

DVHANP Pr<   c                     t        j                         r | j                  j                  |i |S | j	                  d       y Nr  )r	   rv  rn   r  r  r[   r%   r&   s      r:   r  zSyncOnReadVariable.scatter_subL  8    ,,.&T]]&&777!!-0r<   c                     t        j                         r | j                  j                  |i |S | j	                  d       y Nr  )r	   rv  rn   r  r  r  s      r:   r  zSyncOnReadVariable.scatter_addQ  r  r<   c                     t        j                         r | j                  j                  |i |S | j	                  d       y Nr  )r	   rv  rn   r  r  r  s      r:   r  zSyncOnReadVariable.scatter_mulV  r  r<   c                     t        j                         r | j                  j                  |i |S | j	                  d       y Nr  )r	   rv  rn   r  r  r  s      r:   r  zSyncOnReadVariable.scatter_div[  r  r<   c                     t        j                         r | j                  j                  |i |S | j	                  d       y Nr  )r	   rv  rn   r  r  r  s      r:   r  zSyncOnReadVariable.scatter_min`  r  r<   c                     t        j                         r | j                  j                  |i |S | j	                  d       y Nr  )r	   rv  rn   r  r  r  s      r:   r  zSyncOnReadVariable.scatter_maxe  r  r<   c                     t        j                         r | j                  j                  |i |S | j	                  d       y Nr  )r	   rv  rn   r  r  r  s      r:   r  z!SyncOnReadVariable.scatter_updatej  s9    ,,.)T]]))4:6::!!"23r<   c                 b   t        j                         rt        d      t        j                         r| j
                  j                         S t        j                  | j                        5  t        j                         r|t        j                         sh| j                  t        j                  j                  k(  r(| j                  d      j                         cd d d        S | j!                         cd d d        S | j#                         j                         cd d d        S # 1 sw Y   y xY w)NzMcall `variable.value()` inside variable_sync_on_read_context is not supportedr   )r    in_variable_sync_on_read_contextrg   r	   rv  rn   r6   rl  rY  r   rq  rZ  r*   r+   rQ   r  ra   r>   ri   s    r:   r6   zSyncOnReadVariable.valueo  s    668  ,,.]]  ""		0	01J1J	K 8

1
1
3335 6 6 I II""1%++-	8 8
 &&(8 8 --/5578 8 8s   -A.D%%D%>D%%D.c                 ^    t        j                         rt        d      t        |          S )NzRcall `variable.read_value()` inside variable_sync_on_read_context is not supported)r   r  rg   r   r  rm  s    r:   r  zSyncOnReadVariable.read_value  s1    668  7r<   c                    | j                   t        j                  j                  k(  r| j	                  d      S | j                   t        j                  j
                  k(  rt        j                          t        j                  | j                        5  | j                  j                  t        j                  j                  | j                         | d       cd d d        S # 1 sw Y   y xY wNr   )axis)rZ  r*   r+   rQ   r  SUMr	   rD   r   rl  rY  reducer   rR   rS   ri   s    r:   ra   z%SyncOnReadVariable._get_cross_replica  s    B22EEE q!!B22666$$&		0	01J1J	K &&--



8
89J9J
K
 .      AC""C+c                    t        j                         r| j                  j                         S t	        j
                  | j                        5  t	        j                         r,t        j                  | j                               cd d d        S 	 d d d        | j                         j                         S # 1 sw Y   'xY wrt   )r	   rv  rn   r  r   rl  rY  r   r   r  ra   rd   ri   s    r:   r  z$SyncOnReadVariable._as_graph_element  s    ,,.]],,..		0	01J1J	K @		0	0	2$$T%<%<%>?@ @	2@ 99;((**@ @s   7B66B?c                 J      j                   f fd	}t        j                  |iS )zOverrides Trackable method.

    This allows both name-based and object-based save and restore of
    `SyncOnReadVariable`s.

    Returns:
      A dictionary mapping attribute names to `SaveableObject` factories.
    c                     t        |       S rt   )rb  r  s    r:   r  zNSyncOnReadVariable._gather_saveables_for_checkpoint.<locals>._saveable_factory  s     t,,r<   r  r  s   ` r:   r  z3SyncOnReadVariable._gather_saveables_for_checkpoint  s'      $00 - ((*;<<r<   c                    t        j                         r#t        j                  | j                  |||      S t        j                  | j                        5  t        j                         }|3t        j                         r| j                  t        j                  j                  k(  r1t        j                  | j                  d      |||      cddd       S | j                  t        j                  j                  k(  rt        j                           |j"                  j$                  j'                  t(        j*                  j-                  | j                        | j/                         j1                               }t        j                  ||||      cddd       S t        j                  | j/                         |||      cddd       S # 1 sw Y   yxY w)z*Converts a SyncOnReadVariable to a tensor.r  Nr   )r	   rv  r   r  rn   r   rl  rY  rE   r  rZ  r*   r+   rQ   r  r  rD   r5   r@   rT   r   rR   rS   rd   r  )r[   r-   r   r   r  reduceds         r:   r   z'SyncOnReadVariable._dense_var_to_tensor  sz   ,,.""
--u4@ @		0	01J1J	K >&::<o

%

9
9
; 6 6 I II&&"%d6K> >  6 6 : ::

(
(
* $$--EE$$>>%%'		&&(* 	
 $$5tF<> >$ ""
))+Uf>%> > >s   A;GB<G"&GGr0  r2  )r   r   r   r   r  rd   r  r  r  r  r  r  r  r  r  r  r  r6   r  ra   r  r  r   r   r   s   @r:   rj  rj    sj    OH	4LLBP1
1
1
1
1
1
4
8" +=>r<   rj  c                 *    | j                  |||      S Nr  r  r9   r-   r   r   s       r:   "_tensor_conversion_distributed_varr    s     
	!	!D	!	HHr<   c                 *    | j                  |||      S r  r  r  s       r:   _tensor_conversion_mirroredr        		!	!D	!	HHr<   c                 P    t        j                  | j                         |||      S r  )r   r  rd   )r6   r-   r   r   s       r:   _tensor_conversion_mirrored_valr    s$    			jjl%d6
; ;r<   c                 *    | j                  |||      S r  r  r  s       r:   _tensor_conversion_sync_on_readr    r  r<   c                   4    e Zd ZdZd Zd Zd Zd Zd Zd Z	y)	VariablePolicya  Policy defining synchronization and aggregation of a distributed variable.

  Given `synchronization` and `aggregation` parameters set on a `tf.Variable`
  during variable creation within `tf.distribute` scope, `tf.distribute` creates
  an appropriate policy object and assigns it to the distributed variable. All
  variable operations are delegated to the respective policy object.
  c                     || _         y rt   r  )r[   r)   s     r:   r]   zVariablePolicy.__init__  s
    #Dr<   c                 0    t        dt        |              )NzGVariablePolicy.value should be overridden by sub-classes. Type name is rf   ri   s    r:   r6   zVariablePolicy.value  "    
	T
|	% r<   c                 0    t        dt        |              )NzNVariablePolicy._is_mirrored should be overridden by sub-classes. Type name is rf   ri   s    r:   r$  zVariablePolicy._is_mirrored  r  r<   c                 0    t        dt        |              )NzSVariablePolicy._as_graph_element should be overridden by sub-classes. Type name is rf   r  s     r:   r  z VariablePolicy._as_graph_element	  r  r<   c                 0    t        dt        |              )NzTVariablePolicy._get_cross_replica should be overridden by sub-classes. Type name is rf   r[   r9   s     r:   ra   z!VariablePolicy._get_cross_replica  s"    
	&&*4j\	3 r<   c                 0    t        dt        |              )NzQVariablePolicy._update_replica should be overridden by sub-classes. Type name is rf   r[   r9   r8   r6   r&   s        r:   r  zVariablePolicy._update_replica  r  r<   N)
r   r   r   r   r]   r6   r$  r  ra   r  r}   r<   r:   r  r    s%    $r<   r  c                       e Zd ZdZd Zd Zd Zd Zd Zd Z		 	 	 dd	Z
	 	 	 dd
ZddZd Zd Zd Zd Zd Zd Zd Zd Zd Zy)OnReadPolicya{  Policy defined for `tf.VariableSynchronization.ON_READ` synchronization.

  This policy is created when `synchronization` is set to
  `tf.VariableSynchronization.ON_READ` and `aggregation` is set to any of the
  values allowed by the `tf.VariableAggregation` enum such as `NONE`, `SUM`,
  `MEAN` or `ONLY_FIRST_REPLICA`when creating a `tf.Variable` in `tf.distribute`
  scope.
  c                      yrL  r}   ri   s    r:   r$  zOnReadPolicy._is_mirrored&  s    r<   c                    t        j                  |j                        5  t        j                         r|t	        j
                         sh| j                  t        j                  j                  k(  r(|j                  d      j                         cd d d        S |j                         cd d d        S |j                         j                         cd d d        S # 1 sw Y   y xY w)Nr   )r   rl  r2   r   r	   rq  rZ  r*   r+   rQ   r  r6   ra   r>   r  s     r:   r6   zOnReadPolicy.value)  s    		0	01H1H	I 7

1
1
3335 6 6 I II!!!$**,	7 7
 %%'7 7 ,,.4467 7 7s   A.CC1CC!c                 (   t        j                  |j                        5  t        j                         r,t	        j
                  |j                               cd d d        S 	 d d d        |j                         j                         S # 1 sw Y   'xY wrt   )	r   rl  r2   r   r   r  ra   rd   r  r  s     r:   r  zOnReadPolicy._as_graph_element3  sr    		0	01H1H	I ?		0	0	2$$S%;%;%=>? ?	2? 88:''))? ?s   7BBc                    | j                   t        j                  j                  k(  r|j	                  d      S | j                   t        j                  j
                  k(  rt        j                          t        j                  |j                        5  |j                  j                  t        j                  j                  | j                         |d       cd d d        S # 1 sw Y   y xY wr  )rZ  r*   r+   rQ   r  r  r	   rD   r   rl  r2   r  r   rR   rS   r  s     r:   ra   zOnReadPolicy._get_cross_replica9  s    B22EEEa  B22666$$&		0	01H1H	I $$++



8
89J9J
K
 ,   r  c                 2     ||j                         |fi |S rt   r  r  s        r:   r  zOnReadPolicy._update_replicaD  s    S224eFvFFr<   c                      t        d| d      )Nz#ON_READ variables doesn't support `z` in cross replica contextr|  r}  s     r:   r  z%OnReadPolicy._scatter_not_implementedG  s!    
 CF8 L9 9 : :r<   Nc                 Z   t        j                  |j                        5  t        j                         rIt	        j
                         s5t	        j                          t	        j                  |||      cddd       S t	        j                  |||||      cddd       S # 1 sw Y   yxY w)z%Subtracts a value from this variable.rp  Nr  )	r   rl  r2   r   r	   rq  rD   rr  r  r[   r9   r6   r  r   r  s         r:   r  zOnReadPolicy.assign_subK       
	0	01H1H	I #

1
1
3335&&(;;:/	# # ..#!## # #   AB!>B!!B*c                 Z   t        j                  |j                        5  t        j                         rIt	        j
                         s5t	        j                          t	        j                  |||      cddd       S t	        j                  |||||      cddd       S # 1 sw Y   yxY w)zAdds a value to this variable.rp  Nr  )	r   rl  r2   r   r	   rq  rD   rw  r  r  s         r:   r  zOnReadPolicy.assign_add`  r  r  c                 Z   t        j                  |j                        5  t        j                         rIt	        j
                         s5t	        j                          t	        j                  |||      cd d d        S t	        j                  |||||      cd d d        S # 1 sw Y   y xY w)Nrp  r  )	r   rl  r2   r   r	   rq  rD   ry  r  r  s         r:   r  zOnReadPolicy.assignu  s    		0	01H1H	I #

1
1
3335&&(77:/	# # **#!## # #r  c                 *    ~~| j                  d       y r  r  r  s      r:   r  zOnReadPolicy.scatter_sub      f!!-0r<   c                 *    ~~| j                  d       y r  r  r  s      r:   r  zOnReadPolicy.scatter_add  r  r<   c                 *    ~~| j                  d       y r  r  r  s      r:   r  zOnReadPolicy.scatter_mul  r  r<   c                 *    ~~| j                  d       y r  r  r  s      r:   r  zOnReadPolicy.scatter_div  r  r<   c                 *    ~~| j                  d       y r  r  r  s      r:   r  zOnReadPolicy.scatter_min  r  r<   c                 *    ~~| j                  d       y r  r  r  s      r:   r  zOnReadPolicy.scatter_max  r  r<   c                 *    ~~| j                  d       y r  r  r  s      r:   r  zOnReadPolicy.scatter_update  s    f!!"23r<   c                 0    t        j                  |||      S )z0Create a saveable object for the given variable.)r	   re  r[   r9   primary_varr   s       r:   r<  zOnReadPolicy.get_saveable  s    ++CdCCr<   c                 D    t        j                  ||| j                        S r@  )r	   rh  rZ  r[   r9   r   s      r:   rA  zOnReadPolicy.get_restore_ops  s    ..sFD<M<MNNr<   r0  )r   r   r   r   r$  r6   r  ra   r  r  r  r  r  r  r  r  r  r  r  r  r<  rA  r}   r<   r:   r  r    sz    7*	G: # #0 # #*#1111114DOr<   r  c                       e Zd ZdZd Zd Zd Zd Zd ZddZ		 	 	 dd	Z
	 	 	 dd
ZddZddZddZddZddZddZddZd Zd Zy)OnWritePolicyaa  Policy defined for `tf.VariableSynchronization.ON_WRITE` synchronization.

  This policy is created when the following `synchronization` and `aggregation`
  parameters are specified when creating a `tf.Variable` in `tf.distribute`
  scope and `synchronization` is equal to `tf.VariableSynchronization.ON_WRITE`
  or `tf.VariableSynchronization.AUTO`.
  c                      yr#  r}   ri   s    r:   r$  zOnWritePolicy._is_mirrored  r%  r<   c                 >    |j                         j                         S rt   )r>   r6   r  s     r:   r6   zOnWritePolicy.value  s    ((*0022r<   c                 >    |j                         j                         S rt   r]  r  s     r:   r  zOnWritePolicy._as_graph_element  s    ((*<<>>r<   c                 H    t        j                  |j                               S rt   )r   r  r>   r  s     r:   ra   z OnWritePolicy._get_cross_replica  s     c;;=>>r<   c                     |j                   t        j                  j                  k(  r ||j	                         |fi |S t        |||fi |S rt   )r)   rX  r+   r=   r>   rI   r  s        r:   r  zOnWritePolicy._update_replica  sI    
-;;@@@s446HHH#CEDVDDr<   Nc                 6    t        j                  |||||      S r  )r	   r  r  s         r:   r  zOnWritePolicy.assign  s"    &&U$:O Or<   c                 6    t        j                  |||||      S r  )r	   r  r  s         r:   r  zOnWritePolicy.assign_add  $     **U$:O Or<   c                 6    t        j                  |||||      S r  )r	   r  r  s         r:   r  zOnWritePolicy.assign_sub  r  r<   c                 4    t        j                  ||||      S r  )r	   r  r[   r9   r  r  r   s        r:   r  zOnWritePolicy.scatter_sub      ""\{? ?r<   c                 4    t        j                  ||||      S r  )r	   r  r  s        r:   r  zOnWritePolicy.scatter_add  r  r<   c                 4    t        j                  ||||      S r  )r	   r  r  s        r:   r  zOnWritePolicy.scatter_mul  r  r<   c                 4    t        j                  ||||      S r  )r	   r  r  s        r:   r  zOnWritePolicy.scatter_div  r  r<   c                 8   | j                   t        j                  j                  k7  r[| j                   t        j                  j                  k7  r4t        t        j                  j                  d| j                               t        j                  ||||      S )Nr  rT  r  )
rZ  r*   r+   rQ   r=   rg   r	   rV  r   r  r  s        r:   r  zOnWritePolicy.scatter_min      R33FFFR33888

'
'
.
.#1B1B / DE E ""\{? ?r<   c                 8   | j                   t        j                  j                  k7  r[| j                   t        j                  j                  k7  r4t        t        j                  j                  d| j                               t        j                  ||||      S )Nr  rT  r  )
rZ  r*   r+   rQ   r=   rg   r	   rV  r   r  r  s        r:   r  zOnWritePolicy.scatter_max  r  r<   c                 8   | j                   t        j                  j                  k7  r[| j                   t        j                  j                  k7  r4t        t        j                  j                  d| j                               t        j                  ||||      S )Nr  rT  r  )
rZ  r*   r+   rQ   r=   rg   r	   rV  r   r  r  s        r:   r  zOnWritePolicy.scatter_update  s    R33FFFR33888

'
'
.
.&D4E4E / GH H %%\{? ?r<   c                 0    t        j                  |||      S )z Saveable ops for AUTO variables.)r	   rK  r  s       r:   r<  zOnWritePolicy.get_saveable  s    ,,S+tDDr<   c                 .    t        j                  ||      S rt   )r	   rN  r  s      r:   rA  zOnWritePolicy.get_restore_ops
  s    //V<<r<   r0  r1  )r   r   r   r   r$  r6   r  ra   r  r  r  r  r  r  r  r  r  r  r  r<  rA  r}   r<   r:   r  r    sw    3??
E
O # O # O???????E=r<   r  c                   8     e Zd ZdZd Z fdZ fdZd Z xZS )PerWorkerResourcea  A per-worker CapturableResource class for non-ParameterServer strategy.

  Resources that populate `host_to_resources` should be instances of classes
  subclassing CapturableResource, although currently it's only used and tested
  for StaticHashTable with TPUStrategy.
  c                 ~    t         j                  j                  dd      j                  d       || _        || _        y )Nr  TPUDistributedLookupTabler  )r   'distribution_strategy_input_api_counterget_cellincrease_by	_strategy_host_to_resources)r[   r5   host_to_resourcess      r:   r]   zPerWorkerResource.__init__  s4    ::CC8::E+a.DN/Dr<   c                 f    |dvrt        | j                         |      S t        t        |   |      S )N)r]   __getattribute__r  r  local_resource)r   r  r   r  r  r   s     r:   r  z"PerWorkerResource.__getattribute__  s9     3 3T((*D11"D:4@@r<   c                 j    |dvrt        | j                         ||      S t        t        |   ||      S )N)r  r  )setattrr  r   r  __setattr__)r[   r   r6   r   s      r:   r  zPerWorkerResource.__setattr__"  s9    66T((*D%88"D5dEBBr<   c           	      *   t        j                  t        j                               }t        j                  t        j                  |            }| j                  j                  || j                  t        t        | j                                       S )z)Returns the resource on the local worker.)r   rl   rm   get_host_for_devicer  getnextiter)r[   ro   host_devices      r:   r  z PerWorkerResource.local_resource'  su     --k.A.A.CDN**''79K""&&T$*A*A%B CDF Fr<   )	r   r   r   r   r]   r  r  r  r   r   s   @r:   r  r    s    0AC
Fr<   r  r2  )Vr   rn  typingr   r_  tensorflow.core.protobufr   tensorflow.python.distributer   r   r   rb  r   r	   tensorflow.python.eagerr
   r   tensorflow.python.frameworkr   r   r  r   r   r   r7  r   r   r   r   tensorflow.python.opsr   r   r   r   r   r*   r   rX  tensorflow.python.saved_modelr   tensorflow.python.trackabler   r  !tensorflow.python.training.savingr   tensorflow.python.typesr   r    ds_typesr!   rI   rC   rO   r   r]  r0   r  #register_tensor_conversion_functionTypeSpecr   register_codecBuiltInTypeSpecCodecTypeSpecProtoPER_REPLICA_SPECr  objectr'  	TraceTyper2  Variabler8  rP  SaveableObjectr  rH  rP  rb  rj  r  r  r  r  r  r  r  r  r}   r<   r:   <module>r     s   7    / 4 7 N 4 4 + * 8 7 . + < B 4 3 1 + 2 * 7 6 < @ 9 = ( : )40n44A22 4A@Z+ Z~"$4$D$D$$", ?  > >&(#Y'' #: &  % %///
00AA"H$5$5 "Kv K(/5?? /BC
-}/E/E++C
Z,?#A#A ,,Q66 QM<*H M<`1/88 1$v>, v>z .2,0.3I ?  > >;=
I ?  > >13
;
 ?  > >-/
I ?  > >79(V (VJO> JOZb=N b=J F  Fr<   