
    BVh                     z    d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlm	Z	 ddl
mZ  G d d	ej                        Zy
)zImplement a MirroredStrategy based on the DTensor low level API.

This is an experiment to validate the viability of the DTensor API, and expose
any potential feature gaps between the current API and the need.
    )config)	mesh_util)distribute_lib)dtensor_strategy_extended)dtensor_util)devicec                   `     e Zd ZdZd	dd fdZed        Zed        Zd Ze	d        Z
 xZS )
MirroredStrategya}  Synchronous training across multiple replicas on one machine.

  This strategy is typically used for training on one machine with multiple
  accelerators (GPUs/TPUs).

  For example, a variable created under a `MirroredStrategy` is a distributed
  variable with layout replicated on each dimension. The variables will be
  placed on the `mesh` that is specified in the __init__.
  N)meshc                    | j                  ||       |s| j                  |      }t        j                  | |      }t        |   |       || _        || _        y)a  Synchronous training across multiple replicas on one machine.

    Args:
      devices: a list of device strings, such as ['/gpu:0', '/gpu:1']. If both
        `mesh` and `devices` are None, all the available GPU/TPU will be used.
        If no accelerators are found, CPU is used.
      cross_device_ops: optional, a descendant of `CrossDeviceOps`. The value is
        ignored at the moment, and support will be added later.
      mesh: optional DTensor mesh for the computation. Note that either `mesh`
        or `devices` should be provided, and not both. The mesh should be 1D,
        and will be used to split the input data among that dimension.
    )container_strategyr   N)_validate_init_args_build_mesh_from_device_listr   DTensorStrategyExtendedsuper__init___mesh_devices)selfdevicescross_device_opsr   extended	__class__s        k/home/dcms/DCMS/lib/python3.12/site-packages/tensorflow/python/distribute/experimental/mirrored_strategy.pyr   zMirroredStrategy.__init__(   sX     	T7+..w7d(@@d,H	GXDJDM    c                     |r|rt        d| d|       |rCt        |j                               dk7  r&t        dt        |j                                d      y y )NzGMesh and devices can not be provided at the same time. received mesh = z, devices =    z4The mesh for MirroredStrategy must be 1D, received: D)
ValueErrorlenshape)clsr   r   s      r   r   z$MirroredStrategy._validate_init_args?   sr     **.|G9F G G
 DJJL!Q&Mdjjl+,A/ 0 0 'tr   c                    |rtt         j                  j                  |d         j                  }t	        j
                  |       t        j                  t        j                  t        |      fg|      }|S t        j                         }t        j                  |      }t	        j
                  |       t        j                  t        j                  t        |      fg|      }|S )Nr   )	mesh_dimsr   )r$   device_type)	tf_device
DeviceSpecfrom_stringr%   r   "initialize_accelerator_system_oncer   create_meshDEFAULT_BATCH_MESH_DIM_NAMEr    d_configpreferred_device_typelocal_devices)r"   r   r%   r   s       r   r   z-MirroredStrategy._build_mesh_from_device_listK   s    ((44WQZ@LLk55kB""">>GMNd K 224k&&{3g55kB""">>GMN!#d Kr   c                 2    t        j                  | |||      S )N)r   dtensor_reduce)r   	reduce_opvalueaxiss       r   reducezMirroredStrategy.reduce]   s    &&tYtDDr   c                     | j                   S )z&Returns the mesh used by the strategy.)r   )r   s    r   r   zMirroredStrategy.mesh`   s     ::r   )NN)__name__
__module____qualname____doc__r   classmethodr   r   r4   propertyr   __classcell__)r   s   @r   r
   r
      sV    $ . 	0 	0  "E  r   r
   N)r9   tensorflow.dtensor.pythonr   r,   r   tensorflow.python.distributer   )tensorflow.python.distribute.experimentalr   r   tensorflow.python.frameworkr   r&   Strategyr
    r   r   <module>rC      s2    9 / 7 O B ;F~.. Fr   