
    BVh{[                        d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm	Z	 ddlm
Z
 dd	lmZ dd
lmZ ddlmZ ddlmZ dZ G d de      Z G d deej(                        Z G d deej,                        Z G d de      Z G d deej2                        Zd dZd dZd dZ G d dej<                        Z G d dej@                        Z!y)!zcVarious classes representing TPU distributed values.

Note that the tests are in values_test.py .

    )packed_distributed_variable)tpu_replicated_variable)tpu_util)values)values_util)context)tape)ops)gen_resource_variable_ops)math_ops)variable_scopez{op_name} is only supported for distributed variable (variable created within certain `tf.distribute.Strategy` scope) with NONE  aggregation, got: {aggregation}.c                        e Zd ZdZ fdZ fdZ fdZd Zed        Z	ed        Z
d Z fd	Z fd
Z fdZed        Zd fd	Z xZS )TPUVariableMixinzMixin for TPU variables.c                     t        t        | 
  |i | t        j                         r4| j
                  dz   t        t        | j                              z   | _	        y | j
                  | _	        y )N_)
superr   __init__r
   #executing_eagerly_outside_functions_common_namestrid_primary
_handle_id)selfargskwargs	__class__s      W/home/dcms/DCMS/lib/python3.12/site-packages/tensorflow/python/distribute/tpu_values.pyr   zTPUVariableMixin.__init__+   sX    	
D*D;F; ..0))C/#b6G2HHdo))do    c                 p    t        j                         t        t        |   |      S t        d| d      )Nz`TPUVariableMixin.z&` not accessible within a TPU context.)r   enclosing_tpu_contextr   r   __getattr__AttributeError)r   namer   s     r   r"   zTPUVariableMixin.__getattr__5   sA    %%'/#T6t<<tf$J
KM Mr   c                 f    t        j                         t        t        |          S t        d      )Nz?`TPUVariableMixin.get()` is not supported within a TPU context.)r   r!   r   r   getNotImplementedErrorr   r   s    r   r&   zTPUVariableMixin.get<   s5    %%'/#T.00
KM Mr   c                 "    | j                         S N)
read_valuer   s    r   _get_as_operandz TPUVariableMixin._get_as_operandC   s    ??r   c                    t        j                         }|t        j                         rF| j	                         }t        |t        j                        r|j                         S |j                  S | j                  du}| j                  }|r| j                  g}|j                  | j                  | j                  || j                         |      S )z2The handle by which this variable can be accessed.N)r   r!   r   executing_eagerly_get_on_device_or_primary
isinstancepackedPackedVarAndDeviceon_device_handlehandle_packed_var_valuesget_replicated_var_handler   r   _is_mirrored)r   tpu_contextvar	is_packedvals        r   r5   zTPUVariableMixin.handleF   s     002Kg779**,c	C22	3##%%zz""$.iLLc	 2243D3D37??C373D3D3F3<> >r   c                 .    | j                   j                  S r*   )r5   devicer,   s    r   r?   zTPUVariableMixin.device\   s    ;;r   c                 |   | j                   rt        j                  |        | j                  }t	        |dd      rWt        j                  | j                         j                        5  t        j                  || j                        cddd       S t        j                  || j                        S # 1 sw Y   yxY w)z!Reads the value of this variable.r<   FN)	trainabler	   variable_accessedr5   getattrr
   r?   r0   r   read_variable_opdtype)r   r5   s     r   _read_variable_opz"TPUVariableMixin._read_variable_op`   s    ~~
T"[[Fv{E*::d446==> N(99&$**MN N '77

KKN Ns   ( B22B;c                 p    t        j                         t        t        |          S | j                         S r*   )r   r!   r   r   r+   rF   r(   s    r   r+   zTPUVariableMixin.read_valuem   s2    %%'/#T577##%%r   c                 p    t        j                         t        t        |          S | j                         S r*   )r   r!   r   r   valuerF   r(   s    r   rI   zTPUVariableMixin.values   s2    %%'/#T022##%%r   c                 R    t        j                         t        t        |          S y r*   )r   r!   r   r   _as_graph_elementr(   s    r   rK   z"TPUVariableMixin._as_graph_elementy   s&    %%'/#T<>>r   c                 v   t        j                         r| j                  j                  S t	        j
                  | j                  j                  j                  | j                  j                  j                  | j                  j                  j                  | j                  j                  j                        S r*   )
r   is_saving_non_distributedr   opr   DistributedVarOpr$   graph	tracebacktyper,   s    r   rN   zTPUVariableMixin.op   s|    ,,.]]""4==#3#3#8#8#'==#3#3#9#9#'==#3#3#=#=#'==#3#3#8#8: :r   c                     t        j                         t        t        |   |||      S |3|| j
                  k7  r$t        j                  | j                         |      S |r| j                  S | j                         S )z Converts a variable to a tensor.)rE   r$   as_ref)
r   r!   r   r   _dense_var_to_tensorrE   r   castr+   r5   )r   rE   r$   rT   r   s       r   rU   z%TPUVariableMixin._dense_var_to_tensor   sy     %%'/#T?D @ 1 1 
	u

2]]4??,e44"T[[9(99r   )NNF)__name__
__module____qualname____doc__r   r"   r&   r-   propertyr5   r?   rF   r+   rI   rK   rN   rU   __classcell__r   s   @r   r   r   (   sw     *MM > >*  L&& : :
: 
:r   r   c                   `    e Zd ZdZddZddZddZddZddZddZ	dd	Z
dd
ZddZddZy)TPUDistributedVariablez-DistributedVariable subclass for TPUStrategy.Nc                     t        j                         r| j                  j                  ||||      S | j                  j                  | ||||      S Nuse_lockingr$   r+   )r   rM   r   
assign_sub_policyr   rI   rc   r$   r+   s        r   rd   z!TPUDistributedVariable.assign_sub   R    ,,.]]%%e[$
KK<<""e4J # P Pr   c                     t        j                         r| j                  j                  ||||      S | j                  j                  | ||||      S ra   )r   rM   r   
assign_addre   rf   s        r   ri   z!TPUDistributedVariable.assign_add   rg   r   c                     t        j                         r| j                  j                  ||||      S | j                  j                  | ||||      S ra   )r   rM   r   assignre   rf   s        r   rk   zTPUDistributedVariable.assign   sR    ,,.]]!!%dJGG<<e4J  P Pr   c                     t        j                         r| j                  j                  |||      S | j                  j                  | |||      S N)rc   r$   )r   rM   r   scatter_subre   r   sparse_deltarc   r$   s       r   rn   z"TPUDistributedVariable.scatter_sub   N    ,,.]]&&|[$GG<<##l$ $ @ @r   c                     t        j                         r| j                  j                  |||      S | j                  j                  | |||      S rm   )r   rM   r   scatter_addre   ro   s       r   rs   z"TPUDistributedVariable.scatter_add   rq   r   c                     t        j                         r| j                  j                  |||      S | j                  j                  | |||      S rm   )r   rM   r   scatter_mulre   ro   s       r   ru   z"TPUDistributedVariable.scatter_mul   rq   r   c                     t        j                         r| j                  j                  |||      S | j                  j                  | |||      S rm   )r   rM   r   scatter_divre   ro   s       r   rw   z"TPUDistributedVariable.scatter_div   rq   r   c                     t        j                         r| j                  j                  |||      S | j                  j                  | |||      S rm   )r   rM   r   scatter_minre   ro   s       r   ry   z"TPUDistributedVariable.scatter_min   rq   r   c                     t        j                         r| j                  j                  |||      S | j                  j                  | |||      S rm   )r   rM   r   scatter_maxre   ro   s       r   r{   z"TPUDistributedVariable.scatter_max   rq   r   c                     t        j                         r| j                  j                  |||      S | j                  j                  | |||      S rm   )r   rM   r   scatter_updatere   ro   s       r   r}   z%TPUDistributedVariable.scatter_update   sN    ,,.]])),TJJ<<&&l$ ' @ @r   FNTFN)rW   rX   rY   rZ   rd   ri   rk   rn   rs   ru   rw   ry   r{   r}    r   r   r_   r_      sA    5PPP@@@@@@@r   r_   c                   t     e Zd ZdZd Ze fd       ZddZddZddZ	d Z
d Zd	 Zd
 Zd Zd Zd Z xZS )TPUMirroredVariablezHHolds a map from replica to TPU variables whose values are kept in sync.c                 J    t        | j                  t        j                        S )zReturns whether each of the underlying variables is replicated or sharded to logical cores.

    If True, the handles of the underlying variables are not available outside a
    TPU context.
    )r1   r   r   TPUReplicatedVariabler,   s    r   *_is_replicated_or_sharded_to_logical_coresz>TPUMirroredVariable._is_replicated_or_sharded_to_logical_cores   s"     dmm-CCE Er   c                     | j                         r*t        j                         | j                  j                  S t
        t        |   S r*   )r   r   r!   r   r?   r   r   r(   s    r   r?   zTPUMirroredVariable.device   s<    779&&(0]]!!!$d22r   c                 N   t        j                         }| j                         r|d }| j                  |||||      S |rU| j                  t
        j                  j                  k(  r. t        j                  t        j                        | ||||      S t        | ||||      S )Nc                 &     | j                   |i |S r*   )rd   vakas      r   <lambda>z0TPUMirroredVariable.assign_sub.<locals>.<lambda>       ,!,,*@R*@ r   	update_fnrI   rc   r$   r+   rI   rc   r$   r+   rb   )r   r!   r   _updateaggregationr   VariableAggregationNONEmake_raw_assign_fnr   assign_sub_variable_oprd   )r   rI   rc   r$   r+   r:   assign_sub_fns          r   rd   zTPUMirroredVariable.assign_sub       002K779@m\\!!  ! ! 	N>>CCC<X((
#
:
:<%#% % e4JP Pr   c                 N   t        j                         }| j                         r|d }| j                  |||||      S |rU| j                  t
        j                  j                  k(  r. t        j                  t        j                        | ||||      S t        | ||||      S )Nc                 &     | j                   |i |S r*   )ri   r   s      r   r   z0TPUMirroredVariable.assign_add.<locals>.<lambda>  r   r   r   r   rb   )r   r!   r   r   r   r   r   r   r   r   assign_add_variable_opri   )r   rI   rc   r$   r+   r:   assign_add_fns          r   ri   zTPUMirroredVariable.assign_add   r   r   c                 r   t        j                         }| j                         r|d }| j                  |||||      S t        j                         rU| j                  t
        j                  j                  k(  r. t        j                  t        j                        | ||||      S t        | ||||      S )Nc                 &     | j                   |i |S r*   )rk   r   s      r   r   z,TPUMirroredVariable.assign.<locals>.<lambda>  s    hahh&8R&8 r   r   r   rb   )r   r!   r   r   r   r   r   r   r   r   assign_variable_oprk   )r   rI   rc   r$   r+   r:   	assign_fns          r   rk   zTPUMirroredVariable.assign  s    002K7798i\\!  ! ! 	&&(N>>CCC8X((
#
6
68%#% % e4JP Pr   c                 n    t        j                         r | j                  j                  |i |S t        r*   )r   rM   r   rn   r'   r   r   r   s      r   rn   zTPUMirroredVariable.scatter_sub0  0    ,,.&T]]&&777
r   c                 n    t        j                         r | j                  j                  |i |S t        r*   )r   rM   r   rs   r'   r   s      r   rs   zTPUMirroredVariable.scatter_add5  r   r   c                 n    t        j                         r | j                  j                  |i |S t        r*   )r   rM   r   r{   r'   r   s      r   r{   zTPUMirroredVariable.scatter_max:  r   r   c                 n    t        j                         r | j                  j                  |i |S t        r*   )r   rM   r   ry   r'   r   s      r   ry   zTPUMirroredVariable.scatter_min?  r   r   c                 n    t        j                         r | j                  j                  |i |S t        r*   )r   rM   r   ru   r'   r   s      r   ru   zTPUMirroredVariable.scatter_mulD  r   r   c                 n    t        j                         r | j                  j                  |i |S t        r*   )r   rM   r   rw   r'   r   s      r   rw   zTPUMirroredVariable.scatter_divI  r   r   c                 n    t        j                         r | j                  j                  |i |S t        r*   )r   rM   r   r}   r'   r   s      r   r}   z"TPUMirroredVariable.scatter_updateN  s0    ,,.)T]]))4:6::
r   r~   )rW   rX   rY   rZ   r   r[   r?   rd   ri   rk   rn   rs   r{   ry   ru   rw   r}   r\   r]   s   @r   r   r      sU    PE 3 3P0P0P0





r   r   c                   L     e Zd ZdZd Zd fd	Zd fd	Zd fd	Z fdZ xZ	S )TPULazyDistributedVariablez:TPU Mirrored variable to be initialized lazily in a batch.c                 b    t        | dd      ry | j                  j                          d| _        y )N_is_lazily_initializedFT)rC   _lazy_scopeinitialize_allr   r,   s    r   _initialize_if_uninitializedz7TPULazyDistributedVariable._initialize_if_uninitializedW  s,    t-u5##%"&Dr   c                 H    | j                          t        | 	  ||||      S r*   )r   r   rd   r   rI   rc   r$   r+   r   s        r   rd   z%TPULazyDistributedVariable.assign_sub^  *    %%'7{D* r   c                 H    | j                          t        | 	  ||||      S r*   )r   r   ri   r   s        r   ri   z%TPULazyDistributedVariable.assign_addd  r   r   c                 H    | j                          t        | 	  ||||      S r*   )r   r   rk   r   s        r   rk   z!TPULazyDistributedVariable.assignj  s)    %%'7>{D* r   c                 @    | j                          t        | 	         S r*   )r   r   r+   r(   s    r   r+   z%TPULazyDistributedVariable.read_valueq  s    %%'7r   r~   )
rW   rX   rY   rZ   r   rd   ri   rk   r+   r\   r]   s   @r   r   r   T  s$    B'   r   r   c                   "    e Zd ZdZd Zd Zd Zy)TPUSyncOnReadVariablezGHolds a map from replica to variables whose values are reduced on save.c                     t        j                         #t        j                  j                  | g|i |S  t        j
                  t        j                        | g|i |S r*   )r   r!   r   SyncOnReadVariablerd   r   r   r   r   s      r   rd   z TPUSyncOnReadVariable.assign_suby  n    %%'/&&11$HHHH<X((
#
:
:<<@FCGF>DF Fr   c                     t        j                         #t        j                  j                  | g|i |S  t        j
                  t        j                        | g|i |S r*   )r   r!   r   r   ri   r   r   r   r   s      r   ri   z TPUSyncOnReadVariable.assign_add  r   r   c                     t        j                         #t        j                  j                  | g|i |S  t        j
                  t        j                        | g|i |S r*   )r   r!   r   r   rk   r   r   r   r   s      r   rk   zTPUSyncOnReadVariable.assign  sn    %%'/&&--dDTDVDD8X((
#
6
688<O?COGMO Or   N)rW   rX   rY   rZ   rd   ri   rk   r   r   r   r   r   v  s    OFFOr   r   Nc                 t    t        j                  t        j                        }| j	                  |||||      S Nr   )r   r   r   r   r   )r;   rI   rc   r$   r+   r   s         r   rd   rd     @    --668-	 
 
 r   c                 t    t        j                  t        j                        }| j	                  |||||      S r   )r   r   r   r   r   )r;   rI   rc   r$   r+   r   s         r   ri   ri     r   r   c                 t    t        j                  t        j                        }| j	                  |||||      S r   )r   r   r   r   r   )r;   rI   rc   r$   r+   r   s         r   rk   rk     s@    ))224)	 
 
 r   c                   x    e Zd ZdZ	 	 	 ddZ	 	 	 ddZddZ	 	 ddZddZddZ	dd	Z
dd
ZddZddZddZy)TPUOnWritePolicyzPolicy defined for `tf.VariableSynchronization.ON_WRITE` synchronization.

  This policy is created when `synchronization` is set to
  `tf.VariableSynchronization.AUTO` or `tf.VariableSynchronization.ON_WRITE`.
  Nc                     t        j                         rU|j                  t        j                  j
                  k(  r. t        j                  t        j                        |||||      S t        |||||      S Nr   rb   )
r   r!   r   r   r   r   r   r   r   rd   r   r;   rI   rc   r$   r+   s         r   rd   zTPUOnWritePolicy.assign_sub  x     	&&(>==BBB<X((
#
:
:<%#% % U$:O Or   c                     t        j                         rU|j                  t        j                  j
                  k(  r. t        j                  t        j                        |||||      S t        |||||      S r   )
r   r!   r   r   r   r   r   r   r   ri   r   s         r   ri   zTPUOnWritePolicy.assign_add  r   r   c                     t        j                         rU|j                  t        j                  j
                  k(  r. t        j                  t        j                        |||||      S t        |||||      S r   )
r   r!   r   r   r   r   r   r   r   rk   r   s         r   rk   zTPUOnWritePolicy.assign  sv    &&(>==BBB8X((
#
6
68%#% % U$:O Or   c                 8   t        j                  |      }t        j                         r]| j                  t        j
                  j                  k7  r*t        t        j                  || j                               |||||      S |j                  ||||      S )N)op_namer   )rp   rc   r$   )r   rI   rc   r$   )r   make_raw_scatter_xxx_fnr!   _aggregationr   r   r   r'   _scatter_error_msgformatr   )r   raw_scater_xxx_fnr   r;   rp   rc   r$   scater_xxx_fns           r   _scatter_xxxzTPUOnWritePolicy._scatter_xxx  s     445FGM%%'			n@@EE	E!%%T->-> & @A 	A 
LkN N [[!!	   r   c                 J    | j                  t        j                  d||||      S )Nrn   )r   r   resource_scatter_subr   r;   rp   rc   r$   s        r   rn   zTPUOnWritePolicy.scatter_sub  *    6KK*C{!# #r   c                 J    | j                  t        j                  d||||      S )Nrs   )r   r   resource_scatter_addr   s        r   rs   zTPUOnWritePolicy.scatter_add  r   r   c                 J    | j                  t        j                  d||||      S )Nr{   )r   r   resource_scatter_maxr   s        r   r{   zTPUOnWritePolicy.scatter_max  r   r   c                 J    | j                  t        j                  d||||      S )Nry   )r   r   resource_scatter_minr   s        r   ry   zTPUOnWritePolicy.scatter_min  r   r   c                 J    | j                  t        j                  d||||      S )Nru   )r   r   resource_scatter_mulr   s        r   ru   zTPUOnWritePolicy.scatter_mul  r   r   c                 J    | j                  t        j                  d||||      S )Nrw   )r   r   resource_scatter_divr   s        r   rw   zTPUOnWritePolicy.scatter_div  r   r   c                 J    | j                  t        j                  d||||      S )Nr}   )r   r   resource_scatter_updater   s        r   r}   zTPUOnWritePolicy.scatter_update  s*    6NN-sL+!# #r   r~   r   )rW   rX   rY   rZ   rd   ri   rk   r   rn   rs   r{   ry   ru   rw   r}   r   r   r   r   r     s_     # O* # O$O$  %,#
#
#
#
#
#
#r   r   c                   `     e Zd ZdZ fdZ fdZ fdZd Zd Zd Z	d Z
d	 Zd
 Zd Z xZS )TPUOnReadPolicya{  Policy defined for `tf.VariableSynchronization.ON_READ` synchronization.

  This policy is created when `synchronization` is set to
  `tf.VariableSynchronization.ON_READ` and `aggregation` is set to any of the
  values allowed by the `tf.VariableAggregation` enum such as `NONE`, `SUM`,
  `MEAN` or `ONLY_FIRST_REPLICA`when creating a `tf.Variable` in `tf.distribute`
  scope.
  c                     t        j                         t        t        |   |g|i |S  t        j
                  t        j                        |g|i |S r*   )r   r!   r   r   rd   r   r   r   r   r;   r   r   r   s       r   rd   zTPUOnReadPolicy.assign_sub/  j    %%'/?D4SJ4J6JJ<X((
#
:
:<<?FBFF>DF Fr   c                     t        j                         t        t        |   |g|i |S  t        j
                  t        j                        |g|i |S r*   )r   r!   r   r   ri   r   r   r   r   s       r   ri   zTPUOnReadPolicy.assign_add7  r   r   c                     t        j                         t        t        |   |g|i |S  t        j
                  t        j                        |g|i |S r*   )r   r!   r   r   rk   r   r   r   r   s       r   rk   zTPUOnReadPolicy.assign?  sj    %%'/?D0FtFvFF8X((
#
6
688;N>BNFLN Nr   c                     t         r*   r'   r   s      r   rn   zTPUOnReadPolicy.scatter_subF      
r   c                     t         r*   r   r   s      r   rs   zTPUOnReadPolicy.scatter_addI  r   r   c                     t         r*   r   r   s      r   r{   zTPUOnReadPolicy.scatter_maxL  r   r   c                     t         r*   r   r   s      r   ry   zTPUOnReadPolicy.scatter_minO  r   r   c                     t         r*   r   r   s      r   ru   zTPUOnReadPolicy.scatter_mulR  r   r   c                     t         r*   r   r   s      r   rw   zTPUOnReadPolicy.scatter_divU  r   r   c                     t         r*   r   r   s      r   r}   zTPUOnReadPolicy.scatter_updateX  r   r   )rW   rX   rY   rZ   rd   ri   rk   rn   rs   r{   ry   ru   rw   r}   r\   r]   s   @r   r   r   %  s=    FFNr   r   r~   )"rZ   tensorflow.python.distributer   r2   r   r   r   r   tensorflow.python.eagerr   r	   tensorflow.python.frameworkr
   tensorflow.python.opsr   r   r   r   objectr   DistributedVariabler_   MirroredVariabler   r   r   r   rd   ri   rk   OnWritePolicyr   OnReadPolicyr   r   r   r   <module>r      s    O @ 1 / 4 + ( + ; * 0: j:v j:Z=@-v/I/I =@@|*F,C,C |~ !7  DO,f.G.G O8o#v++ o#d4f)) 4r   