
    AVh.                     @   d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlm	Z	 ddl
mZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ d Z G d dej@                        Z!	 	 	 ddZ" ejF                  e!e"       y)zKA Variable class that is replicated to logical cores for model parallelism.    )absolute_import)division)print_function)abcN)xla_sharding)tpu_util)context)config)ops)tensor_conversion_registry)control_flow_ops)gen_resource_variable_ops)gen_tpu_partition_ops)variable_scope)	variables)save_context)basec                     t        j                  |j                        5   | ||fi |cd d d        S # 1 sw Y   y xY wN)r   device)	update_fnvarvaluekwargss       d/home/dcms/DCMS/lib/python3.12/site-packages/tensorflow/python/distribute/tpu_replicated_variable.py_on_device_updater   &   s6    
zz#** +S%*6*+ + +s   
4=c                      e Zd ZdZd#dZd Zed        Zed        Zed        Z	ed        Z
ed        Zej                  d	        Zej                  d
        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zd Zd Zd Zd Zed        Zed        Zd Zd$dZd Z d Z!d%dZ"d%dZ#d%d Z$d! Z%d" Z&y)&TPUReplicatedVariablea  Container for replicated `Variables` that are treated as a single variable.

  This class maintains a list of replicated variables that are stored on
  separate logic TPU devices. TF2XLA bridge accesses these variables as
  if they were a single variable.
  c                    t        t        j                        rrt        d D              rt	        d       t        fdD              r&t        dD cg c]  }|j                   c}       t        fdD              r&t        dD cg c]  }|j                   c}       | _        || _	        | j                  j                  d      d   | _        d	| _        y	c c}w c c}w )
a  Treats `variables` as a replicated list of `tf.Variable`s.

    Example:

    ```
    variables = [
      tf.Variable(..., shape=(10, 100), dtype=tf.float32),
      tf.Variable(..., shape=(10, 100), dtype=tf.float32),
      tf.Variable(..., shape=(10, 100), dtype=tf.float32),
      tf.Variable(..., shape=(10, 100), dtype=tf.float32),
    ]
    replicated_variable = TPUReplicatedVariable(variables)
    assert replicated_variable.shape.as_list() == [10, 100]
    ```

    Args:
      variables: A list of `ResourceVariable`s that comprise this replicated
        variable. Variables should not be shared between different
        `TPUReplicatedVariable` objects.
      name: String. Name of this container. Defaults to "TPUReplicatedVariable".
    c              3   R   K   | ]  }t        |t        j                          ! y wr   )
isinstancevariables_libVariable).0vs     r   	<genexpr>z1TPUReplicatedVariable.__init__.<locals>.<genexpr>I   s)      GF67Jq-0011GFs   %'zSArgument `variables` should be a non-empty list of `variables.Variable`s. Received c              3   V   K   | ]   }|j                   d    j                   k7   " ywr   N)dtyper$   r%   r   s     r   r&   z1TPUReplicatedVariable.__init__.<locals>.<genexpr>N   $     
<Q177il(((
<   &)zPAll elements in argument `variables` must have the same dtype. Received dtypes: c              3   V   K   | ]   }|j                   d    j                   k7   " ywr(   )shaper*   s     r   r&   z1TPUReplicatedVariable.__init__.<locals>.<genexpr>S   r+   r,   zPAll elements in argument `variables` must have the same shape. Received shapes: :r   N)r!   r   Sequenceany	TypeError
ValueErrorr)   r.   _vars_namesplit_common_name_cached_value)selfr   namer%   s    `  r   __init__zTPUReplicatedVariable.__init__3   s   , i.i3 GF;DGF DF 99BE F F 
<)
<<09:1qww:;=> > 
<)
<<09:1qww:;=> > DJDJ

((-a0DD ;
 ;s   C+
C0
c                 ,    t        | j                        S )zBReturn an iterable for accessing the underlying sharded variables.)iterr4   r9   s    r   __iter__zTPUReplicatedVariable.__iter__]   s    

    c                     | j                   S )z0The name of this object. Used for checkpointing.)r5   r>   s    r   r:   zTPUReplicatedVariable.namea   s     ::r@   c                 4    | j                   d   j                  S )z,The dtype of all `Variable`s in this object.r   )r4   r)   r>   s    r   r)   zTPUReplicatedVariable.dtypef   s     ::a=r@   c                 4    | j                   d   j                  S Nr   )r4   is_initializedr>   s    r   rE   z$TPUReplicatedVariable.is_initializedk       ::a='''r@   c                 4    | j                   d   j                  S rD   )r4   	trainabler>   s    r   rH   zTPUReplicatedVariable.trainableo   s    ::a="""r@   c                 4    | j                   d   j                  S )zThe device this variable is on.r   )r4   r   r>   s    r   r   zTPUReplicatedVariable.devices   s     ::a=r@   c              #      K   | j                   j                  j                         5  d  d d d        y # 1 sw Y   y xY wwr   )handlegraph
as_defaultr>   s    r   _handle_graphz#TPUReplicatedVariable._handle_graphx   s3     				%	%	'   s   %A5	A>Ac              #      K   | j                   .t        j                  | j                   g      5  d  d d d        y d  y # 1 sw Y   y xY wwr   )r8   r   control_dependenciesr>   s    r   _assign_dependenciesz*TPUReplicatedVariable._assign_dependencies}   sJ     %##T%7%7$89    s   -AAAAAc                 4    | j                   d   j                  S rD   )r4   
constraintr>   s    r   rS   z TPUReplicatedVariable.constraint       ::a=###r@   c                 4    | j                   d   j                  S rD   )r4   _in_graph_moder>   s    r   rV   z$TPUReplicatedVariable._in_graph_mode   rF   r@   c                 4    | j                   d   j                  S rD   )r4   
_unique_idr>   s    r   rX   z TPUReplicatedVariable._unique_id   rT   r@   c                 4    | j                   d   j                  S rD   )r4   rL   r>   s    r   rL   zTPUReplicatedVariable.graph       ::a=r@   c                     | j                   S r   )r7   r>   s    r   _shared_namez"TPUReplicatedVariable._shared_name   s    r@   c                 6    t         j                  j                  S r   )r   VariableSynchronizationNONEr>   s    r   synchronizationz%TPUReplicatedVariable.synchronization   s    11666r@   c                 6    t         j                  j                  S r   )r   VariableAggregationr_   r>   s    r   aggregationz!TPUReplicatedVariable.aggregation   s    --222r@   c                 b    t        j                         r| j                  d   gS | j                  S )zThe list of `Variables`.r   )r   in_save_contextr4   r>   s    r   r   zTPUReplicatedVariable.variables   s*     ##%jjm_::r@   c                 J   | j                   d   } |j                  |||fi |}| j                   dd D ]A  }||   ||<   ||j                     ||j                  <   |j                  |j                         C ||   || <   ||j                     || <   |j                  |        |S )For implementing `Trackable`.r      N)r4   _export_to_saved_model_graphrK   append)r9   
object_map
tensor_mapoptionsr   	first_varresource_listr%   s           r   ri   z2TPUReplicatedVariable._export_to_saved_model_graph   s     

1I:I::J3+13MZZ^ % +jm'	(8(89j188$% "),Jt!)"2"23Jtr@   c                 @    t         j                  | j                  d   iS rD   )	trackableVARIABLE_VALUE_KEYr4   r>   s    r   _serialize_to_tensorsz+TPUReplicatedVariable._serialize_to_tensors   s    (($**Q-88r@   c                 J    |t         j                     }| j                  |      S r   )rq   rr   assign)r9   restored_tensorsrestored_tensors      r   _restore_from_tensorsz+TPUReplicatedVariable._restore_from_tensors   s!    &y'C'CDO;;''r@   c                     | |v r#| j                   D ]  }|j                  |        yg }| j                   D ]'  }|j                  |       |j                  ||          ) t        || j                        }||| <   y)rg   )r:   N)r4   _copy_trackable_to_cpurj   r   r:   )r9   rk   r%   copied_varsnew_vars        r   rz   z,TPUReplicatedVariable._copy_trackable_to_cpu   s    zzz -!	  ,- kzz *!	  ,:a=)* &k		Bg jr@   c                 4    | j                   d   j                  S rD   )r4   r.   r>   s    r   r.   zTPUReplicatedVariable.shape   rZ   r@   c                 J   t        j                         st        j                         r| j                  d   j
                  S t        j                         t        d      t        j                         5  t        | dd       }|t        j                         r;t        j                  | j                  D cg c]  }|j
                   c}g d      }n#t        j                  |j                  gg d      }d d d        t!        j"                        S c c}w # 1 sw Y   #xY w)Nr   zQTPUReplicatedVariable.handle is not available outside tpu context or save context_packed_varF)partition_dims	is_packedT)r   re   r	   executing_eagerlyr4   rK   r   enclosing_tpu_contextNotImplementedErroroutside_or_skip_tpu_contextgetattrr
   get_soft_device_placementtpu_partition_opstpu_partitioned_input_v2packed_handler   	replicate)r9   
packed_varr%   tensors       r   rK   zTPUReplicatedVariable.handle   s    ##%)B)B)DZZ]!!!%%'/ !F G G //1 
MT=$7
 !A!A!C$==!%,Aqxx,52& %==''(tM&
M ##F++ -
M 
Ms   5AD6D
	.DDD"c                 V    t        j                  | j                  | j                        S r   )r   read_variable_oprK   r)   r>   s    r   _read_variable_opz'TPUReplicatedVariable._read_variable_op   s    $55dkk4::NNr@   Nc                 j    t        j                         | j                         S | j                         S )z Converts a variable to a tensor.)r   r   
read_valuer   )r9   r)   r:   as_refs       r   _dense_var_to_tensorz*TPUReplicatedVariable._dense_var_to_tensor   s/     %%'/__##%%r@   c                 <    | j                   d   j                         S rD   )r4   r   r>   s    r   r   z TPUReplicatedVariable.read_value   s    ::a=##%%r@   c                     t        j                  |d| j                        t        j                  t        fd| j                  D               S )z;Converts the value to tensor and updates the variable list.value_in_tensor)r:   r)   c              3   :   K   | ]  }t        |fi   y wr   )r   )r$   r%   input_tensorr   r   s     r   r&   z0TPUReplicatedVariable._update.<locals>.<genexpr>   s'      % iLCFC%s   )r   convert_to_tensorr)   r   grouptupler   )r9   r   r   r   r   s    ` `@r   _updatezTPUReplicatedVariable._update   sM    ((%TZZ9L !!	 %^^% 
%& &r@   c                     t        j                         t        j                         rd }| j	                  |||||      S  t        j
                  t        j                        | ||||      S )Nc                 &     | j                   |i |S r   )ru   r   akas      r   <lambda>z.TPUReplicatedVariable.assign.<locals>.<lambda>  s    


A(<(< r@   r   use_lockingr:   r   )r   r   r	   r   r   make_raw_assign_fnr   assign_variable_op)r9   r   r   r:   r   	assign_fns         r   ru   zTPUReplicatedVariable.assign   sz    %%'/73L3L3N<i\\
!  ! !8X((
#
6
68%#% %r@   c                     t        j                         t        j                         rd }| j	                  |||||      S  t        j
                  t        j                        | ||||      S )Nc                 &     | j                   |i |S r   )
assign_subr   s      r   r   z2TPUReplicatedVariable.assign_sub.<locals>.<lambda>      NCNNA,D,D r@   r   )r   r   r	   r   r   r   r   assign_sub_variable_op)r9   r   r   r:   r   assign_sub_fns         r   r   z TPUReplicatedVariable.assign_sub  z    %%'/73L3L3NDm\\
!  ! !<X((
#
:
:<%#% %r@   c                     t        j                         t        j                         rd }| j	                  |||||      S  t        j
                  t        j                        | ||||      S )Nc                 &     | j                   |i |S r   )
assign_addr   s      r   r   z2TPUReplicatedVariable.assign_add.<locals>.<lambda>&  r   r@   r   )r   r   r	   r   r   r   r   assign_add_variable_op)r9   r   r   r:   r   assign_add_fns         r   r   z TPUReplicatedVariable.assign_add$  r   r@   c                     dj                  d t        | j                        D              }| j                  j                  d|dS )N,
c              3   0   K   | ]  \  }}d ||fz    yw)z  %d: %sN r$   ir%   s      r   r&   z0TPUReplicatedVariable.__str__.<locals>.<genexpr>7  s"      ? $1
aV?   :{

}join	enumerater4   	__class____name__)r9   	debug_strs     r   __str__zTPUReplicatedVariable.__str__6  s;    

 ?(1$**(=? ?I NN33Y??r@   c                     dj                  d t        | j                        D              }| j                  j                  d|dS )Nr   c              3   0   K   | ]  \  }}d ||fz    yw)z  %d: %rNr   r   s      r   r&   z1TPUReplicatedVariable.__repr__.<locals>.<genexpr><  s"      ? $1
aV?r   r   r   r   )r9   
debug_reprs     r   __repr__zTPUReplicatedVariable.__repr__;  s;     ?(1$**(=? ?J NN33Z@@r@   )r   NNF)FNT)'r   
__module____qualname____doc__r;   r?   propertyr:   r)   rE   rH   r   
contextlibcontextmanagerrN   rQ   rS   rV   rX   rL   r\   r`   rc   r   ri   rs   rx   rz   r.   rK   r   r   r   r   ru   r   r   r   r   r   r@   r   r   r   +   s   (T     ( ( # #         $ $ ( ( $ $     7 7 3 3  9(!   , ,,O&&&%$%$%$@
Ar@   r   c                 *    | j                  |||      S )N)r)   r:   r   )r   )r   r)   r:   r   s       r   %_tensor_conversion_tpu_replicated_varr   C  s     
	!	!D	!	HHr@   r   )$r   
__future__r   r   r   collectionsr   r   +tensorflow.python.compiler.xla.experimentalr   tensorflow.python.distributer   tensorflow.python.eagerr	   tensorflow.python.frameworkr
   r   r   tensorflow.python.opsr   r   r   r   r   r   r"   tensorflow.python.saved_modelr   tensorflow.python.trackabler   rq   r   r#   r   r   #register_tensor_conversion_functionr   r@   r   <module>r      s    R &  %   D 1 + . + B 2 ; L 0 < 6 9+
SAM22 SAr 15/316I ?  > >@Br@   