
    BVh1                     t   d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ	 ddlm
Z
 ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ dZ G d dej4                        Z	 ddZ ej:                  ee        G d dej<                        Zd Z d Z!d Z"d Z#d Z$d Z%y) z4Utilities for strategies that are backed by DTensor.    )accelerator_util)api)
input_util)layout)cross_device_ops)device_util)distribute_lib)reduce_util)values)context)ops)tensor_conversion_registry)tensor_util)	array_ops)math_ops)summary_ops_v2batchc                   8     e Zd ZdZ fdZd Zed        Z xZS )DTensorDistributedValuea  DistributedValue backed by a DTensor instance.

  This class is useful to align the interface between DTensor and tf.distribute.
  Most of the tf.distribute API will accept/return DistributedValue, whereas
  DTensor low level API will only accept DTensor instance. In order to avoid
  the conversion back and forth between DistributedValue and DTensor, we
  introduce this class so that it can work with both side.
  c                    t        j                         rVt        j                  |      st	        dt        |             t        |   t        j                  |             || _	        y t        |   |g       || _	        y )NzIThe DTensorDistributedValue can only be built with DTensor instance, got )
r   executing_eagerlyd_api
is_dtensor
ValueErrortypesuper__init__unpack_dtensor)selfdtensor	__class__s     f/home/dcms/DCMS/lib/python3.12/site-packages/tensorflow/python/distribute/experimental/dtensor_util.pyr   z DTensorDistributedValue.__init__2   sw      "g& 226w-B C 	Cgu||G,- DM gz"DM    c                     | j                   S N)r   r    s    r#   get_dtensorz#DTensorDistributedValue.get_dtensor@   s    ==r$   c                     | j                   S r&   )_valuesr'   s    r#   r   zDTensorDistributedValue.valuesC   s    
 <<r$   )	__name__
__module____qualname____doc__r   r(   propertyr   __classcell__r"   s   @r#   r   r   (   s&      r$   r   Nc                     ~| j                         }|T|j                  |j                        s9t        dj	                  |j
                  |j                  j
                              |rt        d      |S )NzMIncompatible type conversion requested to type {!r} for variable of type {!r}z5PerReplica doesn't support being used as a reference.)r(   is_compatible_withdtyper   formatnameNotImplementedError)varr4   r6   as_refr!   s        r#   $_dtensor_distributed_value_to_tensorr:   K   sp    
OO'
u77F
	vejj'--*<*<=? ? 
?A A	.r$   c                   l     e Zd ZdZdZ fdZd Zed        Zed        Z	ddZ
ddZdd	Zdd
Z xZS )DTensorReplicaContextaq  ReplicaContext for strategy that is backed by DTensor.

  Since the DTensor is operated in the global context, most of the methods from
  existing strategy ReplicaContext is not applicable since they need to access
  local values. For now most of the methods in this class will raise explicit
  error to user, and we will add more support for local values in future.
  zStrategy that is backed by DTensor is run with a global context, and doesn't support operations for local context, like any call to merge/gather/reduce or local replica ID. Please use any strategy that is not backed by DTensorc                 (    t         |   |d        y )N)replica_id_in_sync_group)r   r   )r    strategyr"   s     r#   r   zDTensorReplicaContext.__init__m   s     
GX=r$   c                     t        j                  | j                         t        j                  }|j
                  | _        d|_        y )NT)r	   _push_per_thread_mode_thread_contextr   _summary_state"is_recording_distribution_strategy(_summary_recording_distribution_strategy)r    summary_states     r#   	__enter__zDTensorReplicaContext.__enter__s   s<     (()=)=>"11M88 	17;M4r$   c                      y)Nr    r'   s    r#   r>   z.DTensorReplicaContext.replica_id_in_sync_group}   s    
 r$   c                 ,    t        | j                        r&   r7   _UNSUPPORTED_ERROR_MSGr'   s    r#   _replica_idz!DTensorReplicaContext._replica_id   s    
d99
::r$   c                 ,    t        | j                        r&   rK   )r    merge_fnargskwargss       r#   
merge_callz DTensorReplicaContext.merge_call       
d99
::r$   c                 ,    t        | j                        r&   rK   )r    	reduce_opvalueoptionss       r#   
all_reducez DTensorReplicaContext.all_reduce   rS   r$   c                 ,    t        | j                        r&   rK   )r    rV   axisrW   s       r#   
all_gatherz DTensorReplicaContext.all_gather   rS   r$   c                 ,    t        | j                        r&   rK   )r    r8   fnrP   rQ   groups         r#   _updatezDTensorReplicaContext._update   rS   r$   )rI   Nr&   )rI   NT)r+   r,   r-   r.   rL   r   rG   r/   r>   rM   rR   rX   r[   r_   r0   r1   s   @r#   r<   r<   _   sX     ><   ; ;;;;;r$   r<   c                 \    t        j                         st        j                  | d       y y )NT)experimental_reset_context)r   is_initializedinitialize_accelerator_system)device_types    r#   "initialize_accelerator_system_oncere      s+     
	(	(	* 22#') 
+r$   c                 z   t        | t              r| j                         S t        | t        j                        rt        | |      S t        | t        j                        r| S t        j                  |       r9t        j                         r#t        j                  |       r| S t        |        y| S t        |        y)z,Convert any input types to DTensor instance.N)
isinstancer   r(   
values_libDistributedValuesconvert_per_replica_to_dtensorr   _DTensorIteratorr   	is_tensorr   r   r   r   #_raise_unsupported_input_type_error)inputsmeshs     r#   convert_inputs_to_dtensorrp      s    /0&*667)&$77&*556MV$  "			&	! 	,F3 m (/r$   c                 0    t        dt        |              )NzUnsupported input types for MirroredStrategy. Please use `strategy.distribute_dataset` or `strategy.distribute_values_from_function` to distribute inputs. Received input type: )r   r   )rn   s    r#   rm   rm      s(     > ?C6l^M 	N Nr$   c                 d    t        | t        j                        xs t        j                  |       S r&   )rg   rh   ri   r   r   )rV   s    r#   is_distributed_valuers      s/    	Z))
+ 
F.3.>.>u.EFr$   c                    | j                   }t        |d   t        t        f      rd}nt	        |d   j
                        }|dk(  r5g }|D ](  }|j                  t        j                  |d             * |dz  }nt        |      }t        j                  j                  |t        |      }t        j                  ||      S )a/  Convert a PerReplica result to a DTensor instance.

  Args:
    per_replica_value: A PerReplica instance whose value will be converted
      to DTensor.
    mesh: The mesh used for layout creation.

  Returns:
    A DTensor instance that packed from per_replica_value with batch sharded
      layout.
  r   rZ      )	batch_dimrank)r   rg   floatintlenshapeappendr   expand_dims_v2listr   Layoutbatch_shardedDEFAULT_BATCH_MESH_DIM_NAMEr   pack)per_replica_valuero   r   rx   resultvbatch_layouts          r#   rj   rj      s     ##&q	E3<(DvayD	QYF  9mmI,,QQ789AID&\F
 ,,
1 - >, 
FL	))r$   c           	         t        j                  | j                         t        |t              r#t        j                  |j                               }t        |      }|s|t        j                         xs | j                  j                  xs d}t        j                  |      }t        j                  |d         5  t!        j"                  t        j$                  |||| j&                              cddd       S t)        || j*                        }|t
        j                  j,                  k(  rt.        j0                  }nt.        j2                  }t5        j6                  |      j9                         r|
 |||      }|S | j&                  dg}t;        |j<                        dkD  r'|j?                  t!        j<                  |      dd        t!        j@                  ||      }| |||dz         } ||d      }|S # 1 sw Y   xY w)z*Implement dtensor based strategy.reduce().Nz/device:CPU:0r   ru   rv   )!r	   2_require_cross_replica_or_default_context_extendedextendedrg   strr
   ReduceOpupperrs   r   current_default_devicecross_device_ops_libget_devices_fromr   devicer   identityreduce_non_distributed_valuenum_replicas_in_syncrp   _meshMEANr   reduce_mean
reduce_sumr   fetch_layoutis_fully_replicatedr{   r|   extendreshape)r?   rU   rV   rZ   distributed_inputdestinationsdevices	new_shapes           r#   dtensor_reducer      s    CC	3$$Y__%67I*51	t|  '') $%%55$#  #33LAG	GAJ	 N

;
;h.K.KMNN N
 $E8>>
:% +&&+++$$I##I 224 D)e  
, ..3I
5;;!yu-ab12eY/E D1H-ee!$E
 
,MN Ns   ?5HH)NNF)&r.   tensorflow.dtensor.pythonr   r   r   r   r   tensorflow.python.distributer   r   r   r	   r
   r   rh   tensorflow.python.eagerr   tensorflow.python.frameworkr   r   r   tensorflow.python.opsr   r   r   r   ri   r   r:   #register_tensor_conversion_functionReplicaContextr<   re   rp   rm   rs   rj   r   rI   r$   r#   <module>r      s    ; 6 2 0 , Q 4 7 4 = + + B 3 + * 0
 &  j::  H (-  ?  > >AC3;N99 3;l	)0<NF
#*L:r$   