
    BVhk              	          d Z ddlZddlZddlmZ ddlmZ ddlmZ ddlm	Z	 ddlm
Z
 ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddlmZ! ddl"m#Z# ddl$m%Z& ddl'm(Z( ddl)m*Z* da+da,e
jZ                  Z-d Z. e.ej^                  ej`                        Z0 e.ejb                  ejd                        Z2 e.ejf                  ejh                        Z4e
jj                  Z5	 	 	 ddZ6d Z7d Z8dd!Z9 ejt                  d"d#$      Z;de;fd%Z<	 dd&Z= e=d d' e9d(      )      Z> e=d d' e9d(      d*      Z? e=d d+ e9d(      )      Z@ ej                  d,ej                  d(      ZC ej                  d-d. d(      Z ej                  d/d0 d'(      ZD ej                  d1d2 d(      ZE ej                  d3d4 d'(      ZF ej                  d5 e6d#6      d 7      Z ej                  d8 e6d#d 9      d 7      ZG ej                  d: e6d#d ;      d 7      ZH ej                  d< e6d'6      d 7      ZI ej                  d= e6d#d >      d 7      ZJ ej                  d? e6d'd >      d 7      ZK ej                  d@ e6d#6      d d A      ZL ej                  dBdC       ZM ej                  dDdE d'(      ZN ej                  dFdG d'(      ZO ej                  dHdI d(      ZP ej                  dJdK d#(      ZQ ej                  dLdM d#N      ZR ej                  dOdP       ZSdQeS_          ej                  dRdS d#(      ZT ej                  dTdU d'(      ZU ej                  dV e9d(      d d'e>d W      ZV ej                  dX e9d'(      d d'd'e>dY      ZW ej                  dZ e9d'(      d d'd'e?dY      ZX ej                  d[ e9d#(      d d'd#e>d \      ZY ej                  d] e9d#d^      d d'd#e>d _      ZZ ej                  d` e9d(      d d+e@d W      Z[de;fdaZ\ e\dbd+d#c      Z] e\ddd'd#c      Z^ e\ded+d#d'f      Z_ e\dgd'd#d'f      Z`dhdigZadj ZbeCeeDeOeQeUgZceeDeOeQgZdeeIeGeLgZeedeez   Zfeceez   ZgeOeQeVeWeeIeUgZheYe[gZieOeQeeIgZjeVeWeYgZkdk Zldl Zmdm Zndn Zodo Zp e*dpg q      j                  erdr        e*dsg q      j                  erdt        e*dug q      j                  erdv        e*dwg q      j                  erdx        e*dyg q      j                  erdz        e*d{g q      j                  erd|        e*d}g q      j                  erd~        e*dg q      j                  erd        e*dg q      j                  erd        e*dg q      j                  erd        e*dg q      j                  erd        e*dg q      j                  erd        e*dg q      j                  erd        e*dg q      j                  erd        e*dg q      j                  erd        e*dg q      j                  erd        e*dg q      j                  erd        e*dg q      j                  erd        e*dg q      j                  erd        e*dg q      j                  erd        e*dg q      j                  erd        e*dg q      j                  erd        e*dg q      j                  erd        e*dg q      j                  erd        e*dg q      j                  erd       y)z1Strategy combinations for combinations.combine().    N)
config_pb2)tf2)central_storage_strategycluster_resolver)collective_all_reduce_strategy)combinations)distribute_lib)mirrored_strategy)multi_process_runner)multi_worker_test_base)one_device_strategy)parameter_server_strategy_v2)sharded_variable)	test_util)tpu_strategy)tpu_cluster_resolver)context)remote)device)errors)flags)device_assignment)
server_lib)	tf_exportFc                       fd}|S )Nc                  L    t        j                         r | i |S  | i |S N)r   enabled)argskwargstf1_clstf2_clss     b/home/dcms/DCMS/lib/python3.12/site-packages/tensorflow/python/distribute/strategy_combinations.pycreatorz!_version_chooser.<locals>.creator5   s,    
{{}d%f%%D#F##     )r"   r#   r%   s   `` r$   _version_chooserr(   4   s    $
 
.r&   c                 "      fd}|S )Nc            	         t         j                  } 	 t        j                         }d}t        s;t        | dd      s|rt        j                  |       da	t        j                  |      ad }	r)t        j                   t        t        j"                        }t%        j&                         rt)        j*                  ||fd	i}nt)        j,                  ||fi }rrt	        d
      |_        |S # t        $ rr d}t        j                  t        | d      xr | j                  xs dt        | d      xr | j                  xs d t        | d      xr | j                  xs d       }Y Fw xY w)NTFtpu zoneproject)r+   r-   r.   )core_assignment"experimental_spmd_xla_partitioningz1Packed Variable is not compatiable with SPMD mode)r   FLAGSr   TPUClusterResolver
ValueErrorhasattrr+   r-   r.   _did_connect_to_clustergetattrr   connect_to_clusterinitialize_tpu_system	_topologydevice_assignment_libDeviceAssignmentSINGLE_CORE_ASSIGNMENTr   r   tpu_libTPUStrategyV2TPUStrategyV1%_enable_packed_variable_in_eager_mode)
r1   resolverdid_automatically_resolver   strategyenable_packed_variableenable_spmd_xla_paritioningr!   steps_per_runuse_single_cores
        r$   _create_tpu_strategyz7_get_tpu_strategy_creator.<locals>._create_tpu_strategyU   su   KKE &88:h"& #	r	"&?!!(+"&&<<XFi/@@
/FF {{}&&

 .I 	h &&
M#48>h "=JKK 2 OS  "' &88eU#1		7Ruf%4%**<%+=Eh	s   C5 5A7E0/E0r'   )rF   rG   rD   rE   r!   rH   s   ````` r$   _get_tpu_strategy_creatorrI   N   s    3 3j 
r&   c                 z   t        d | D              }|t        t        j                         j                  d            kD  r*t        j                          t        j                  d|       t        j                  xj                  dz  c_	        t        j                  xj                  dz  c_	        t        |       S )Nc              3   x   K   | ]2  }t         j                  j                  |      j                  d k(  rd 4 yw)CPU   N)	tf_device
DeviceSpecfrom_stringdevice_type).0ds     r$   	<genexpr>z>_mirrored_strategy_with_collective_key_base.<locals>.<genexpr>   s7      
				)	)!	,	8	8E	A s   8:rL   順 )sumlenr   list_logical_devices_reset_contextr   set_logical_devices_to_at_leastmirrored_libMirroredStrategyV1_collective_key_baseMirroredStrategy)devicesrequired_cpus_numss     r$   +_mirrored_strategy_with_collective_key_basera      s       #goo/DDUKLL--e5GH !!66&@644>4	'	""r&   c                     t         j                  xj                  dz  c_        t         j                  xj                  dz  c_        t        |       }d |j                  _        |S )NrU   c                       yNFr'   r'   r&   r$   <lambda>z7_mirrored_strategy_with_no_merge_call.<locals>.<lambda>       r&   )r[   r\   r]   r^   extended_use_merge_call)r_   outs     r$   %_mirrored_strategy_with_no_merge_callrj      sK    !!66&@644>4!#!.#,,	*r&   Tc                 "      fdfd}|S )Nc            
         t        j                         } | j                         }| j                  r|t	        d| j                  z        d  }t        j
                  | j                         | j                  | j                  || j                  di| j                  xs d      }dt        _        t        j                         j                  d       t        j                         5  t        |      }d d d        sd	 j                   _        	 t%        j&                         j)                          S # 1 sw Y   BxY w# t*        $ r Y S w xY w)
Nz%s://GPUgrpc)cluster_spec	task_typetask_idmasterenvironmentnum_accelerators	rpc_layerFr,   )service_typer   c                       yrd   r'   r'   r&   r$   re   z[_get_multi_worker_mirrored_creator.<locals>._create_multi_worker_mirrored.<locals>.<lambda>   rf   r&   )r   TFConfigClusterResolverrr   ru   rW   SimpleClusterResolverro   rp   rq   rs   CollectiveAllReduceExtended_enable_check_healthr   configure_coordination_service
eager_modeCollectiveAllReduceStrategyrg   rh   r   get_barrierwaitr3   )	tf_configrr   rA   rC   required_gpususe_merge_calls       r$   _create_multi_worker_mirroredzI_get_multi_worker_mirrored_creator.<locals>._create_multi_worker_mirrored   s;    88:IFc'I$7$778:;f55++-%%!!))/%%/H  	  4 OO44"4E 
			 H,hGhH *7h'&&(--/ OH H   Os   (D6"E 6D?	EEc                      	         S # t         j                  $ rZ} d| j                  v rFt        t        j
                        dk\  r*dt        j
                  d   v rt        j                  d       d } ~ ww xY w)NCould not start gRPC serverrM   bazelr   Cannot start std servers.)r   UnknownErrormessagerW   sysargvunittestSkipTest)er   s    r$    skip_if_cannot_start_grpc_serverzL_get_multi_worker_mirrored_creator.<locals>.skip_if_cannot_start_grpc_server   sg    *,, 	&!))	3
chh-1
CHHQK!7 ;<<s   
 A7AA22A7r'   )r   r   r   r   s   `` @r$   "_get_multi_worker_mirrored_creatorr      s    +Z	 
*)r&         )min_shard_bytes
max_shardsc                 (     d  fd}|S )Nc                 0    t        j                  | |      S )N)variable_partitioner)r   ParameterServerStrategyV2)rA   r   s     r$   _create_ps_strategyz5_get_ps_strategy_creator.<locals>._create_ps_strategy   s    'AA'; r&   c            	      J   t        j                         rNt        j                  d      } t	        j
                  t        j                  |       d	id      } |
      S t	        j                         }|j                         j                         } | sy t	        j
                  t        j                  |       d	i|j                  |j                  |j                  |j                  xs d      }|j                  dv rZt        j                          }d|_        	 t        j$                  | |j                  |j                  d|      }|j1                           |
      S # t&        j(                  $ r)}d	|j*                  v rt-        j.                  d
       d }~ww xY w)Nrn   )num_workersnum_psru   rm   )rt   ru   )rt   rp   rq   rs   ru   )workerps   )job_name
task_indexprotocolconfigr   r   )framework_test_utilis_xla_enabledr   create_in_process_clusterr   ry   r   ClusterSpecrx   ro   as_dictrp   rq   rs   ru   r   ConfigProtointer_op_parallelism_threadsServerr   r   r   r   r   join)cluster_defrA   r   worker_configserverr   r   r   r   r   r   s         r$   _create_parameter_serverz:_get_ps_strategy_creator.<locals>._create_parameter_server   s   ))+ +DD!&Fk "77

 
 
-!=1h
 !+?@@"::<i**,446k  "77

 
 
-!=1''##++''16h 
		 0	0"..0562	$$ **"**"& 	 +?@@ "" 	*aii7##$?@@		s   .E& &F"9$FF"r'   )r   r   r   r   r   r   s   ```` @r$   _get_ps_strategy_creatorr      s    
;A ;Az 
"!r&   c                 &     g  fd}|S )a  Returns a callable that returns the pool runner.

  It creates the pool runner only upon first invocation. This avoids creating it
  when this file is imported.

  Args:
    has_chief: whether there should be a chief.
    num_workers: the number of workers excluding the chief.
    initializer: initializer of each process.
    share_gpu: whether to share GPU between the workers.

  Returns:
    A callable that returns the runner.
  c                      sBt        j                  dd      } t        j                  |       }j	                  |       d   S )Nr   F)	has_chiefr   r   has_eval)initializer	share_gpu)r   create_cluster_specr   MultiProcessPoolRunnerappend)ro   runner	containerr   r   r   r   s     r$   get_or_createz,_deferred_pool_runner.<locals>.get_or_createL  sR    +??;q5l $::
K9f vQ<r&   r'   )r   r   r   r   r   r   s   ```` @r$   _deferred_pool_runnerr   8  s    $ )	 	 
r&   rM   )r   )r   r   r   )r   r   r   r      DefaultOneDeviceCPUc                      t        d      S N/cpu:0OneDeviceStrategyr'   r&   r$   re   re   w      -h7 r&   OneDeviceGPUc                      t        d      S N/gpu:0r   r'   r&   r$   re   re   z  r   r&   OneDeviceOnWorker1CPUc                      t        d      S )Nz"/job:worker/replica:0/task:1/cpu:0r   r'   r&   r$   re   re   ~      BC r&   OneDeviceOnWorker1GPUc                      t        d      S )Nz"/job:worker/replica:0/task:1/gpu:0r   r'   r&   r$   re   re     r   r&   TPU)rF   )required_tpuTPUPackedVar)rF   rD   
TPUUseSPMD)rF   rE   
TPUOneStep
TPUOneCore)rF   rG   TPUOneStepOneCoreCloudTPU)r   use_cloud_tpuMirrored1CPUc                      t        dg      S r   ra   r'   r&   r$   re   re         7
C r&   Mirrored1GPUc                      t        dg      S r   r   r'   r&   r$   re   re     r   r&   MirroredCPUAndGPUc                      t        ddg      S Nr   r   r   r'   r&   r$   re   re         788LM r&   Mirrored2CPUsc                      t        ddg      S )Nr   /cpu:1r   r'   r&   r$   re   re     r   r&   Mirrored2GPUsc                      t        ddg      S Nr   z/gpu:1r   r'   r&   r$   re   re     r   r&   Mirrored2GPUsNoMergeCallc                      t        ddg      S r   )rj   r'   r&   r$   re   re     s    18X2FG r&   )required_physical_gpusMirrored2CPUc                      t        ddg      S )Nr   z/cpu:2r   r'   r&   r$   re   re     r   r&   zYMirrored strategy with 2 virtual CPUs.

    Should set up logical devices before use
    CentralStorage2GPUsc                      t        ddg      S r   CentralStorageStrategyr'   r&   r$   re   re         "Hh#78 r&   CentralStorageCPUAndGPUc                      t        ddg      S r   r   r'   r&   r$   re   re     r   r&   MultiWorkerMirrored2x1CPU)r   r   pool_runner_fnno_xlaMultiWorkerMirrored2x1GPU)r   r   r   r   r    MultiWorkerMirrored2x1GPUNoShareMultiWorkerMirrored2x2GPU)r   r   r   r   r   $MultiWorkerMirrored2x2GPUNoMergeCall)r   r   )r   r   r   r   r   MultiWorkerMirrored4x1CPUc           	      R    t        j                  | t        ||||      ||d|      S )N)r   r   r   r   T)r   r   r   r   )r	   NamedDistributionr   )namer   r   r   r   s        r$   parameter_server_strategy_fnr    s<     
	'	'
!%3	 "
 r&   ParameterServer3Worker2PSCPU)r   r   ParameterServer1Worker2PSCPUParameterServer3Worker2PS1GPU)r   r   r   ParameterServer1Worker2PS1GPUgrapheagerc                 0    t        j                  d|        y )NrL   )r   rZ   )num_virtual_cpuss    r$   set_virtual_cpus_to_at_leastr  >  s    ++E3CDr&   c                  <    t        j                  t        ddg      S Nr  r  distributionmode)r	   combinestrategies_minus_tpur'   r&   r$   strategy_minus_tpu_combinationsr  |  s    			'w.@
 r&   c                  :    t        j                  t        dg      S )Nr  r  )r	   r  tpu_strategiesr'   r&   r$   tpu_strategy_combinationsr    s    			>		JJr&   c                  ,    t               t               z   S r   )r  r  r'   r&   r$   all_strategy_combinationsr    s    	(	*-F-H	HHr&   c                  \    t        j                  t        t        t        t
        gddg      S r  )r	   r  r   one_device_strategy_gpu"mirrored_strategy_with_gpu_and_cpumirrored_strategy_with_two_gpusr'   r&   r$   /all_strategy_minus_default_and_tpu_combinationsr    s0    			

!
,
)	 W
 r&   c                  ,    t               t               z   S r   )r  r  r'   r&   r$   'all_strategy_combinations_minus_defaultr    s    57!#$r&   zN__internal__.distribute.combinations.central_storage_strategy_with_gpu_and_cpu)v1)central_storage_strategy_with_gpu_and_cpuzK__internal__.distribute.combinations.central_storage_strategy_with_two_gpus&central_storage_strategy_with_two_gpusz7__internal__.distribute.combinations.cloud_tpu_strategycloud_tpu_strategyz5__internal__.distribute.combinations.default_strategydefault_strategyzG__internal__.distribute.combinations.mirrored_strategy_with_cpu_1_and_2"mirrored_strategy_with_cpu_1_and_2zD__internal__.distribute.combinations.mirrored_strategy_with_two_cpusmirrored_strategy_with_two_cpuszG__internal__.distribute.combinations.mirrored_strategy_with_gpu_and_cpur  zC__internal__.distribute.combinations.mirrored_strategy_with_one_cpumirrored_strategy_with_one_cpuzC__internal__.distribute.combinations.mirrored_strategy_with_one_gpumirrored_strategy_with_one_gpuzD__internal__.distribute.combinations.mirrored_strategy_with_two_gpusr  zR__internal__.distribute.combinations.mirrored_strategy_with_two_gpus_no_merge_call-mirrored_strategy_with_two_gpus_no_merge_callzB__internal__.distribute.combinations.multi_worker_mirrored_2x1_cpumulti_worker_mirrored_2x1_cpuzB__internal__.distribute.combinations.multi_worker_mirrored_2x1_gpumulti_worker_mirrored_2x1_gpuzJ__internal__.distribute.combinations.multi_worker_mirrored_2x1_gpu_noshare%multi_worker_mirrored_2x1_gpu_nosharezB__internal__.distribute.combinations.multi_worker_mirrored_2x2_gpumulti_worker_mirrored_2x2_gpuzP__internal__.distribute.combinations.multi_worker_mirrored_2x2_gpu_no_merge_call+multi_worker_mirrored_2x2_gpu_no_merge_callz8__internal__.distribute.combinations.one_device_strategyr   z<__internal__.distribute.combinations.one_device_strategy_gpur  z1__internal__.distribute.combinations.tpu_strategyr   zN__internal__.distribute.combinations.parameter_server_strategy_3worker_2ps_cpu)parameter_server_strategy_3worker_2ps_cpuzN__internal__.distribute.combinations.parameter_server_strategy_1worker_2ps_cpu)parameter_server_strategy_1worker_2ps_cpuzO__internal__.distribute.combinations.parameter_server_strategy_3worker_2ps_1gpu*parameter_server_strategy_3worker_2ps_1gpuzO__internal__.distribute.combinations.parameter_server_strategy_1worker_2ps_1gpu*parameter_server_strategy_1worker_2ps_1gpuz:__internal__.distribute.combinations.tpu_strategy_one_coretpu_strategy_one_corez<__internal__.distribute.combinations.tpu_strategy_packed_vartpu_strategy_packed_var)FFF)T)NT)s__doc__r   r   tensorflow.core.protobufr   tensorflow.pythonr   tensorflow.python.distributer   r   r   r	   r
   r   r[   r   r   r   one_device_libr   r   r   r   r=   -tensorflow.python.distribute.cluster_resolverr   tensorflow.python.eagerr   r   tensorflow.python.frameworkr   rN   r   r   tensorflow.python.platformr   tensorflow.python.tpur   r:   tensorflow.python.trainingr    tensorflow.python.util.tf_exportr   r5   r9   rz   r(   r\   r^   CentralStorageStrategyV1r   OneDeviceStrategyV1r   r~   rI   ra   rj   r   MinSizePartitionerDEFAULT_PARTITIONERr   r   _two_worker_pool_two_worker_pool_noshare_four_worker_poolr   _get_default_strategyr$  r  one_device_strategy_on_worker_1#one_device_strategy_gpu_on_worker_1r4  tpu_strategy_spmdtpu_strategy_one_stepr3  tpu_strategy_one_step_one_corer#  r'  r(  r  r&  r  r)  r%  r"  r!  r*  r+  r,  r-  r.  multi_worker_mirrored_4x1_cpur  r/  r0  r1  r2  graph_and_eager_modesr  r   strategies_minus_default_and_tpur  all_strategies_minus_defaultall_strategiestwo_replica_strategiesfour_replica_strategiesmultidevice_strategiesmultiworker_strategiesr  r  r  r  r  export_constant__name__r'   r&   r$   <module>rY     s_	   8 
  / ! A 9 G 5 7 J = ? N E 9 2 @ N + * ; . H , L 1 6 	">> 
 $##\%B%B  *5533  %&&(H(H 
 #>>    %	<~#$9*@ :&99!  ,	H"X 9=H )2C  12C	  *2C  2<11(( 
 5l447t  9,887q  #A,"@"@C# 
 'El&D&DC' #
 .|--	$15D 9,88AdK 
 3L22T   766+!<4  766AtD 
 "@!?!?AtD" 
 4\33A.	  "@!?!?C"  "@!?!?C" 
 &D\%C%CM& "
 #A,"@"@M# 
 #A,"@"@M# 
 1O0N0NG1 - &D\%C%CM& "
 # * *H)G)G8* &
 -KL,J,J8- ) !? > >&Q7#!  !? > >&Q7#!  )G(F(F&&Q7+) % !? > >&Q7#!  /Ml.L.L*&QuM#/ + !? > >&Q7$!  ,, -I"!- ) -I"!- ) .J#1A. * .J#1A. * !'* E
 &#-  &#	$   	  @.P %6 '#!!-  "!  '#	  "!! KI	 
T	 /(GH 	Q	 /(DE 	="/(01 	;/(./ 	M	 /(@A 	J	 /(=> 	M	 /(@A 	Ib/(<= 	Ib/(<= 	J	 /(=> 	X	 /(KL 	HR/(;< 	HR/(;< 	P	 /(CD 	HR/(;< 	V	 /(IJ 	>2/(12 	Br/(56 	7B/(N+ 	T	 /(GH 	T	 /(GH 	U	 /(HI 	U	 /(HI 	@R/(34 	Br/(56r&   