
    AVhy                        d Z ddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
mZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddlm!Z! ddl"m#Z# ddl$m%Z% ddl$m&Z& ddl$m'Z' ddl$m(Z( ddl$m)Z) ddl$m*Z+ ddl$m,Z, ddl$m-Z- ddl$m.Z. ddl/m0Z0 dd l1m2Z2 dd!l1m3Z3 dd"l4m5Z5 dd#l4m6Z6  G d$ d%e7      Z8d?d&Z9d@d'Z:d( Z;d) Z<d* Z=d+ Z>d, Z?d- Z@d. ZAd/ ZBd0ej                  d1eDfd2ZEd0ej                  d1eDfd3ZFd0ej                  d1eDfd4ZG G d5 d6ej                        ZI G d7 d8ej                        ZJ G d9 d:ej                        ZK G d; d<eI      ZLd= ZMd> ZNy)Az5Library for testing DistributionStrategy descendants.    N)
config_pb2)	event_pb2session)dataset_ops)collective_all_reduce_strategy)distribute_lib)distribute_utils)mirrored_strategy)reduce_util)tpu_strategy)backprop)context)def_function)test)dtypes)errors)ops)	test_util)	tf_record)	array_ops)gradients_impl)init_ops)init_ops_v2)math_ops)summary_ops_v2)variable_scope)variable_v1)	variables)gfile)	optimizer)training_util)nest)
tf_inspectc                       e Zd Zy)_TestExceptionN)__name__
__module____qualname__     ^/home/dcms/DCMS/lib/python3.12/site-packages/tensorflow/python/distribute/strategy_test_lib.pyr&   r&   :   s    r+   r&   c                 f    |rt        j                         s| S  t        j                         |       S N)r   executing_eagerlyr   function)fnrun_in_functions     r,   _maybe_run_in_functionr3   @   s+    	 9 9 ;I"<  "2&&r+   c                     t               r.   )r&   _s    r,   _raise_exception_fnr7   I   s    r+   c                  R    t        j                         j                  t               y r.   )r	   get_replica_context
merge_callr7   r*   r+   r,   _merge_raises_fnr;   O   s    $$&112EFr+   c                 B    | j                   j                  t               y r.   )extendedcall_for_each_replicar7   dists    r,   _call_raises_fnrA   V   s    --%%&9:r+   c                  R    t        j                         j                  t               y r.   )r	   r9   r:   rA   r*   r+   r,   _merge_call_raises_fnrC   ]   s    $$&11/Br+   c                 B    | j                   j                  t               y r.   )r=   r>   r;   r?   s    r,   _call_merge_raises_fnrE   d   s    --%%&67r+   c                  R    t        j                         j                  t               y r.   )r	   r9   r:   rE   r*   r+   r,   _merge_call_merge_raises_fnrG   l   s    $$&112GHr+   c           	         | j                  t        j                  |             t        j                  |      }| j	                  |d       t        t        j                  t        j                  j                  ||d                     }g }|D ]8  }t        j                         }|j                  |       |j                  |       : |S )z(Reads summary events from log directory.   r   )
assertTruer    ExistsListDirectory	assertLenlistr   tf_record_iteratorospathjoinr   EventParseFromStringappend)	test_caselogdirfilesrecordsresultrevents          r,   _events_from_logdirr]   p   s    u||F+,


f
%%eQ--bggll658.LMN'& aOOE	!
MM% 
-r+   c                     t        j                  t        j                         ||      }t	        j
                  || d      S )zEUtility for create variables that works like variable in keras layer.)dtypeT)initial_valuename	trainable)	functoolspartialr   GlorotUniformr   Variable)ra   shaper_   initializers       r,    create_variable_like_keras_layerri   ~   s<    !!!57+			dd
< <r+   c                     t        j                  | j                        }d|j                  d t	        |j
                          v S )Nvar_list)r$   getfullargspecminimizeargslendefaults)optimizer_objarg_specs     r,   is_optimizer_v2_instancers      s?     &&}'='=>(	x}}%=s8+<+<'=&=>	>>r+   strategyreturnc                 V    t        | t        j                  t        j                  f      S r.   )
isinstancemirrored_libMirroredStrategyMirroredStrategyV1rt   s    r,   is_mirrored_strategyr|      s(    	$$l&E&EF
H Hr+   c                 V    t        | t        j                  t        j                  f      S r.   )rw   mwms_libCollectiveAllReduceStrategyCollectiveAllReduceStrategyV1r{   s    r,   !is_multi_worker_mirrored_strategyr      s*    	HxCC'EEG 
H Hr+   c                 t    t        | t        j                  t        j                  t        j                  f      S r.   )rw   r   TPUStrategyTPUStrategyV1TPUStrategyV2r{   s    r,   is_tpu_strategyr      s0    	H!--|/I/I!//1
2 2r+   c                   f    e Zd ZdZd Z	 	 ddZd Zd Zd Zd Z		 ddZ
	 	 	 dd
Zd ZddZd Zy	)DistributionTestBasez:Some tests that should work with any DistributionStrategy.c                    j                         5  t        ddt        j                        fd}t	        j
                  |      t        j                        d t        j                  dgg      fd}t        d      D ]  } |       \  }}|d	k(  r|\  }|\  } t        j                         d
z
        }	t        j                         d
z
        }
| j                  |
|	       d d d        y # 1 sw Y   y xY w)NkernelrI   rI   ra   rg   r_   c                     t        j                  t        j                  |       g       t        j                  d      z
  }||z  S N      ?r   reshaper   mat_mulidentityxyr   s     r,   lossz<DistributionTestBase._test_minimize_loss_eager.<locals>.loss   B    Q'-/8/A/A"/EF1ur+   c                 *    | j                  d|z        S )N皙?
assign_sub)vgs     r,   updatez>DistributionTestBase._test_minimize_loss_eager.<locals>.update   s    ||C!G$$r+   r   c            
      d   j                   j                  f      } g }g }| D ]  \  }}j                   j                  |      }|j                  |       t	        j
                  |g      5  j                   j                  t        j                  j                  ||      }t	        j
                  j                   j                  |	|fd            5  |j                  j                   j                  |             ddd       ddd        ||fS # 1 sw Y   xY w# 1 sw Y   	xY wzPerform one optimization step.rn   )destinationsF)rn   groupNr=   r>   read_varrU   r   control_dependencies	reduce_tor   ReduceOpSUMr   
g_vbefore_list
after_listr   r   fetcheddgrad_fnoner   s
         r,   stepz<DistributionTestBase._test_minimize_loss_eager.<locals>.step   s"    jj..wcV.D 
 		8DAqJJ''*'


W
%''	2 8

$$$$((!! % =A))

!!!V1$e!DF 8

 3 3A 6788 8			8 J&&8 88 8%   .A*D%+DD%D"D%%D/	
   r   rI   )scoperi   r   float32r   implicit_gradr!   get_filtered_grad_fnr   r   rangeabsnumpy
assertLess)selfr   r   r   ibabeforeaftererror_beforeerror_afterr   r   r   r   s    `         @@@@r,   _test_minimize_loss_eagerz.DistributionTestBase._test_minimize_loss_eager   s    	
 .1/vV^^=f &&t,g..w7g% v&c'( Ry !v16'&	 !+,l)*k
ook<0].1 .1 .1s   CC>>Dc           	         t        j                         }||_        d|j                  _        t        j                         5  t        j                         j                         5  | j                  |      5 }j                         5  t        ddt        j                        fd}t        j                   |      fdt#        j$                  dgg      fd	} |       \  }}	t'        j(                         j+                          t-        d
      D ]%  }
|j+                  ||	f      \  }}|
dk(  r|\  }|\  }' t/        dz
        }t/        dz
        }| j1                  ||       d d d        d d d        d d d        d d d        y # 1 sw Y   "xY w# 1 sw Y   &xY w# 1 sw Y   *xY w# 1 sw Y   y xY w)Ng333333?)configr   r   r   c                     t        j                  t        j                  |       g       t        j                  d      z
  }||z  S r   r   r   s     r,   r   z<DistributionTestBase._test_minimize_loss_graph.<locals>.loss   r   r+   c                 ,    | j                  |z        S r.   r   )r   r   learning_rates     r,   r   z>DistributionTestBase._test_minimize_loss_graph.<locals>.update   s    ||MA-..r+   r   c            
      d   j                   j                  f      } g }g }| D ]  \  }}j                   j                  |      }|j                  |       t	        j
                  |g      5  j                   j                  t        j                  j                  ||      }t	        j
                  j                   j                  |	|fd            5  |j                  j                   j                  |             ddd       ddd        ||fS # 1 sw Y   xY w# 1 sw Y   	xY wr   r   r   s
         r,   r   z<DistributionTestBase._test_minimize_loss_graph.<locals>.step   s"    jj..wcV.D 
 	8DAqJJ''*'


W
%''	2 8

$$$$((!! % =A))

!!!V1$e!DF 8

 3 3A 6788 8	8 J&&8 88 8r   r   r   rI   )r   ConfigProtoallow_soft_placementgpu_optionsper_process_gpu_memory_fractionr   
graph_moder   Graph
as_defaultcached_sessionr   ri   r   r   r   r   r   r   r   global_variables_initializerrunr   r   r   )r   r   soft_placementr   r   sessr   r   
before_out	after_outr   r   r   r   r   r   r   r   r   r   r   s    ` `             @@@@r,   _test_minimize_loss_graphz.DistributionTestBase._test_minimize_loss_graph   s    ##%F"0F9<F6				 11				!11			F		+11/3	
11 0vV^^=f
 &&t,g/ v&c'& #fj),,.224Ry !xxY/016'&	 !_l	Nk
ook<0c11 11 11 11 11 11 11 11 11 11 11 11sU   #F;*F/=F#C(F	6F#>F/F;F F##F,(F//F8	4F;;Gc                 (  	 t        j                         }	fd}| j                         5 }|j                         5  t	        j
                         5  t        j                         }t        j                         s|j                  j                          t	        j                  d       t	        j                  |      	|j                  j                  |      }|j!                  |      }t        j                         sO|j                  	j#                                |j                  |       |j                  	j%                                t'        | |      }| j)                  |d       | j+                  |d   j,                  j.                  d   j0                  d       | j+                  |d   j,                  j.                  d   j2                  d       d d d        d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)Nc                      j                         5  t        j                         j                  } t	        j
                  d|       cddd       S # 1 sw Y   yxY w)z#Function executed for each replica.r   N)r   r	   r9   replica_id_in_sync_groupsummary_opswrite)
replica_idsummary_writers    r,   run_fnzHDistributionTestBase._test_summary_for_replica_zero_only.<locals>.run_fn  sH    $$& 2#779RR
  j12 2 2s   4AAr      rI   r   g        )tempfilemkdtempr   r   r   always_record_summariesr"   get_or_create_global_stepr   r/   rh   r   set_stepcreate_file_writerr=   r>   unwrapinitcloser]   rM   assertEqualsummaryvaluetagsimple_value)
r   r   rW   r   r   global_stepoutput	unwrappedeventsr   s
            @r,   #_test_summary_for_replica_zero_onlyz8DistributionTestBase._test_summary_for_replica_zero_only  s   F2 
			 E$	 E++-E ";;=k&&( 	##%1"55f=nzz//7f((6"i&&($$&'%%'("40f nnVQ
vay((..q155s;
vay((..q1>>D/E E E E E E E E Es;   HG<FG0G<H0G95G<<H	HHc                 D    j                         5  dgt        j                  j                        z   fd}j                  j	                  |        j                  dgt        j                  j                        z         d d d        y # 1 sw Y   y xY w)NFc                      j                  t        j                         j                        } j	                  | t        j                  j                               j                  |           d| <   y )NT)	evaluater	   r9   r   r   ro   r=   worker_devicesassertFalse)r   r   expected_devicesr   s    r,   mark_devices_fnz>DistributionTestBase._test_replica_id.<locals>.mark_devices_fn3  s`    ]]..0IIK

C

(A(A$BC)*56'+$r+   T)r   ro   r=   r   r>   assertAllEqual)r   r   r   r   s   `` @r,   _test_replica_idz%DistributionTestBase._test_replica_id/  s    	
 C3qzz'@'@#AA, jj&&7
*&3qzz'@'@#AACC C Cs   A9BBc                    |j                         5  | j                  t              5  |j                  j	                  t
               d d d        | j                  t              5  |j                  j	                  t               d d d        | j                  t              5  |j                  j	                  t               d d d        | j                  t              5  |j                  j	                  t               d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   gxY w# 1 sw Y   6xY w# 1 sw Y   y xY wr.   )	r   assertRaisesr&   r=   r>   r7   r;   rC   rG   )r   r@   s     r,   _test_call_and_merge_exceptionsz4DistributionTestBase._test_call_and_merge_exceptions>  s   	 I^, A++,?@A^, >++,<=>^, C++,ABC^, I++,GHII IA A> >C CI II Isj   D? DD?$ DD?! D'D? D3>D?D	D?D$	 D?'D0	,D?3D<	8D??Ec                 ,     dg fd}|S )Nr   c                 F   j                  |        j                  | j                         j                  | j                         #j                  | j                                 S j                  d   | j                         dxx   dz  cc<           S )zInput fn for testing.r   rI   )assertIsNotNoner   num_replicas_in_syncnum_input_pipelinesinput_pipeline_id)input_contextdataset_or_callable_fnexpected_input_pipeline_idexpected_num_input_pipelinesexpected_num_replicas_in_syncr   worker_id_counters    r,   	_input_fnzGDistributionTestBase._input_fn_to_test_input_context.<locals>._input_fnS  s    
=)
4$99;
3$88:	#	/3&88	: $%% 	*1-}/N/NO!!#%%r+   r*   )r   r  r  r  r  r  r  s   ````` @r,   _input_fn_to_test_input_contextz4DistributionTestBase._input_fn_to_test_input_contextI  s     & &  r+   c           
         |r| j                   n| j                  }|j                  |      }t        j                         rt        |      }|D ]=  }| j                  t        |j                  t        |                        }	 |||	       ? | j                  t              5  | j                  |j                  t        |                   d d d        t        |      }|D ]=  }| j                  t        |j                  t        |                        }	 |||	       ? y t        j                  |      }| j                  ||j                  j                   |d|       y # 1 sw Y   xY w)NTtest_reinitializeignore_order)assertCountEqualr   !distribute_datasets_from_functionr   r/   iterr   rN   experimental_local_resultsnextr  StopIterationr   make_initializable_iterator_test_input_fn_iteratorr=   r   )
r   rt   input_fnexpected_valuesr  assert_sameiterableiteratorexpected_valuecomputed_values
             r,   _test_input_fn_iterablez,DistributionTestBase._test_input_fn_iterablee  sF   +7$''T=M=MK99(CH  "hh+ 4.44T(^DEGNN34
 ]+ Kh99$x.IJK hh+ 4.44T(^DEGNN34
 88Bh
""8X->->-M-M#2d0< # >K Ks   "*E""E+Nc           
      h     fd} ||j                          |D ]u  }|j                         }	 |t        t        |            D 
cg c]  }
t	        j
                  |
|	       c}
      }|r j                  ||       d j                  ||       w  j                  t        j                        5  |j                         }	 |t        t        |            D 
cg c]  }
t	        j
                  |
|	       c}
       d d d        |r ||j                          |D ]u  }|j                         }	 |t        t        |            D 
cg c]  }
t	        j
                  |
|	       c}
      }|r j                  ||       d j                  ||       w y y c c}
w c c}
w # 1 sw Y   xY wc c}
w )Nc                 L    rj                  |       S j                  |       S r.   )r   r   )r   r   r   s    r,   <lambda>z>DistributionTestBase._test_input_fn_iterator.<locals>.<lambda>  s    ! $--2B r+   )rh   get_nextr   ro   r
   select_replicar  r   r  r   OutOfRangeError)r   r"  devicesr  r   r  r  r   r#  next_elementr[   r$  s   `   `       r,   r  z,DistributionTestBase._test_input_fn_iterator  s    CHX!!") 9&&(lW **1l;  !n 
nn=89 
		611	2 !&&(lW **1l;  !! x##$+ 	;.((*!#g,#
AB++A|<#
  



?


>>
:	;   ! !#
s*   F
4)F#F
:F#
F/
F##F,c           	      B   |j                         5  t        j                  dg t        j                  t        j                         dt        j                  j                        | j                  t        j                                fd}|j                  j                  |      \  }}| j                  |j                  |             |j                  |      }| j                  |      }| j!                  dt#        |      z  |       d d d        y # 1 sw Y   y xY w)Nr   F)rg   r_   rh   rb   aggregationc                  N    j                  d      } j                         }| |fS NrI   )
assign_add
read_value)train_opr   r   s     r,   model_fnz?DistributionTestBase._test_global_step_update.<locals>.model_fn  s+    ))!,&&(r+   )rI   )r   r   get_variabler   int64r   zeros_initializerr   VariableAggregationONLY_FIRST_REPLICAr   r   r=   r>   r   r  r   ro   )r   rt   r5  	train_opsr   global_step_tensorsglobal_step_valuesr   s          @r,   _test_global_step_updatez-DistributionTestBase._test_global_step_update  s    		 L"//
00233FFHk mmI::<=
 "**@@Ji
mmHNN9-.$??F==)<=
tc"5668JK'L L Ls   C:DDc                 <    t        |t        j                        s j                  d       |xs  j	                         }|j                         5  |5 }t        j                  ddgddgddgddgd	dgdd
gg      }t        j                  g d      }d}|j                  j                  s||j                  z  }|j                  j                  ||f|xs  j	                               }	|	j                  d      }	|j                  j                  }
|	j                  ||
      }	|j                  |	      } j!                  |j"                          fd} |||      \  }} j%                  ||        j%                  ||        |||      \  }} j%                  ||        j%                  ||        j'                  t(        j*                        5   |||       d d d        d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)Nzn/a: V1 onlyrI   r               r         )rC  rB  rD  r   rI   r   r   )drop_remainderc                     | j                  t        d       |      \  }}j                  | j                  |      | j                  |      f      \  }}t	        j
                  |      t	        j
                  |      fS )Nc                     | S r.   r*   )zs    r,   r(  zWDistributionTestBase._test_numpy_dataset.<locals>.run_and_concatenate.<locals>.<lambda>  s    Q r+   )experimental_runr3   r   r  npconcatenate)rt   r   r   r   r2   r   s       r,   run_and_concatenatezEDistributionTestBase._test_numpy_dataset.<locals>.run_and_concatenate  su    ((";@!E1}}hAA!D&AA!DF G1~~a ".."333r+   )rw   r	   
StrategyV1skipTestr   r   rK  asarrayr=   _global_batch_sizer  experimental_make_numpy_datasetrepeat"experimental_require_static_shapesbatchmake_dataset_iteratorr   rh   r   r  r   r+  )r   rt   r   r2   r   r   r   r   
batch_sizedsrF  r   rM  x_1y_1x_2y_2s   `  `             r,   _test_numpy_datasetz(DistributionTestBase._test_numpy_dataset  s   h 9 9:
mmN#5 3 3 5N		  )>  )T
**q!fq"g1v2wAAG
Ha
**'
(aj118#@#@@
<<a&$7$"5"5"7 = 9b99Q<b  ((KKn88J~8>b

(
(
,a
mmAMM"4 %Xq1hc3
!S!
!S!$Xq1hc3
!S!
!S!V334 )Ha()? )  )  )>) )? )  )  )  )s=   HFH
G:!H)H:H?HH	HHc                 Z   t         j                  t        j                  fD ]  }|j	                         5   |d      }| j                  d|j                          |dt        j                  j                        }| j                  d|j                          |dt        j                  j                  d      }| j                  d|j                          |dt        j                  j                  d      }| j                  d|j                         d d d         y # 1 sw Y   	xY w)Nr   T)synchronizationF)r_  rb   )	r   
VariableV1r   rf   r   r   rb   VariableSynchronizationON_READ)r   rt   clsv1v2v3v4s          r,   _test_trainable_variablez-DistributionTestBase._test_trainable_variable  s    &&	(:(:; .>> .Xr||,i&G&G&O&OP-i&G&G&O&O!r||,i&G&G&O&O "-. ... .s   C!D  D*	)Fr   F)NTF)NF)r'   r(   r)   __doc__r   r   r   r   r  r  r%  r  r>  r]  rh  r*   r+   r,   r   r      s_    B/1f 05.181t EDC	I: ?D>B $(04+0';RL,$)L.r+   r   c                   X    e Zd ZdZd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zd Zy)OneDeviceDistributionTestBasezESome tests that should work with any one-device DistributionStrategy.c                    |j                  d       }| j                  dg| j                  |j                  |                   |j                  d |f      }| j                  t	        j
                  |j                  |            }| j                  dg|d          | j                  dg|d          |j                  d	 |
      }| j                  dg| j                  |j                  |                   y )Nc                  ,    t        j                  d      S )N      @)r   r   r*   r+   r,   r(  z9OneDeviceDistributionTestBase._test_run.<locals>.<lambda>  s    	 2 22 6 r+   ro  c                     | dz  | | z  dS Nr   )r   r   r*   r   s    r,   r(  z9OneDeviceDistributionTestBase._test_run.<locals>.<lambda>   s    AAE": r+   r   g       @r   g      0@r   c                     |d| z  z   dz   S Nr   r*   r   r   s     r,   r(  z9OneDeviceDistributionTestBase._test_run.<locals>.<lambda>  s    QQY] r+   kwargs      E@)r   r   r   r   r#   map_structure)r   rt   out1out2	out2_valsout3s         r,   	_test_runz'OneDeviceDistributionTestBase._test_run  s    <<67DdmmHOOD,ABC<<:$<IDd00$GHIin-y~.<<24<@Dt}}X__T-BCDr+   c                 F    | j                  |t        dddgfdddgf       y )Nro  rx       E@inputsexpected_test_collective_comms_all_sumr   rt   s     r,   _test_all_reduce_sumz2OneDeviceDistributionTestBase._test_all_reduce_sum  s1    (Bc
#3rC:>N   Pr+   c                 :    | j                  |t        dgdg       y Nro  r  expected_grads _test_collective_comms_gradientsr  r  s     r,   _test_all_reduce_sum_gradientsz<OneDeviceDistributionTestBase._test_all_reduce_sum_gradients  s"    ))(B4 * >r+   c                 :    | j                  |t        dgdg       y r  $_test_collective_comms_gradient_taper  r  s     r,   "_test_all_reduce_sum_gradient_tapez@OneDeviceDistributionTestBase._test_all_reduce_sum_gradient_tape  s"    --(B4 . >r+   c                 F    | j                  |t        dddgfdddgf       y )N       @      5@g      6@r  r  	_all_meanr  s     r,   _test_all_reduce_meanz3OneDeviceDistributionTestBase._test_all_reduce_mean  s1    )R#s$4S#J?O   Qr+   c                 :    | j                  |t        dgdg       y Ng      @r  r  r  r  s     r,   _test_all_reduce_mean_gradientsz=OneDeviceDistributionTestBase._test_all_reduce_mean_gradients  s"    )))RD" * ?r+   c                 :    | j                  |t        dgdg       y r  r  r  r  s     r,   #_test_all_reduce_mean_gradient_tapezAOneDeviceDistributionTestBase._test_all_reduce_mean_gradient_tape  s"    --)RD" . ?r+   c                 L   |j                  fd      | j                  j                                | j                  t        t	        |j
                  |j                  |                        }| j                  |d   g|d          | j                  |d   g|d          y )Nc                 B    t         j                  j                        S r.   r   Datasetfrom_tensorsr6   r  s    r,   r(  zFOneDeviceDistributionTestBase._test_collective_comms.<locals>.<lambda>"      +%%226: r+   r   rI   )make_input_fn_iteratorr   
initializerN   mapr  rJ  r   )r   rt   comm_fnr  r  outputss      `  r,   r  z4OneDeviceDistributionTestBase._test_collective_comms   s    ,,:<F 	MM&##%&mm33))'6:<	=>G 	!wqz2!wqz2r+   c                 B   t        j                         r| j                  d       fd}|j                  fd      | j	                  j                                | j                  || j	                  |j                  |j                  |                         y )Nz5`tf.gradients` is not supported with eager execution.c                 x    t        j                  d      } |      | z  }t        j                  ||g      d   S Nrx  r   r   r   r   	gradientscr   r   r  s      r,   r   zLOneDeviceDistributionTestBase._test_collective_comms_gradients.<locals>.step1  :    


S
!a
!*q.a%%a!-a00r+   c                 B    t         j                  j                        S r.   r  r  s    r,   r(  zPOneDeviceDistributionTestBase._test_collective_comms_gradients.<locals>.<lambda>7  r  r+   )	r   r/   rO  r  r   r  r   r  rJ  r   rt   r  r  r  r   s     ``  r,   r  z>OneDeviceDistributionTestBase._test_collective_comms_gradients,  s      "
mmKL1
 ,,:<F 	MM&##%&//))$79	:;r+   c                     fd}|j                  fd      | j                  j                                | j                  || j                  |j	                  |j                  |                         y )Nc                     t        j                  d      }t        j                         5 }|j	                  |        |      | z  }d d d        j                  |      S # 1 sw Y   xY wNrx  r   r   r   GradientTapewatchgradientr  r   taper   r  s       r,   r   zPOneDeviceDistributionTestBase._test_collective_comms_gradient_tape.<locals>.stepC  a    


S
!a  " d

1AJN ]]1a      A""A+c                 B    t         j                  j                        S r.   r  r  s    r,   r(  zTOneDeviceDistributionTestBase._test_collective_comms_gradient_tape.<locals>.<lambda>K  r  r+   )r  r   r  r   r  rJ  r  s     ``  r,   r  zBOneDeviceDistributionTestBase._test_collective_comms_gradient_tape@  so    ! ,,:<F 	MM&##%&//))$79	:;r+   c                    t        j                         r| j                  d       t        j                  dd      \  }}|j                  d       }d }|j                  ||      }t        j                  |d   j                        5 }|j                  |j                                |j                  |       d d d        y # 1 sw Y   y xY w)N:cross-device tests are not supported with eager execution.r   r   c                 @    t         j                  j                  d      S NrC  r   r  r   r5   s    r,   r(  z[OneDeviceDistributionTestBase._test_device_and_input_device_are_colocated.<locals>.<lambda>Z      +%%++A. r+   c                     | dz   S r1  r*   rr  s    r,   r(  z[OneDeviceDistributionTestBase._test_device_and_input_device_are_colocated.<locals>.<lambda>[  
    A r+   rI   target)r   r/   rO  r   create_local_clusterr  rJ  session_libSessionr  r   r  )r   rt   workersr6   r  r  run_opr   s           r,   +_test_device_and_input_device_are_colocatedzIOneDeviceDistributionTestBase._test_device_and_input_device_are_colocatedT  s      "
mm
FH//15JGQ,,.0FG&&w7F			GAJ$5$5	6 $
hhv  "#
hhv  s   	1CCc                 (   t        j                         r| j                  d       t        j                  dd      \  }}|j                  d       }d } t        j                         |j                        }t        j                  d      5   |||      }d d d        t        j                  |d   j                        5 }|j                  |j                                |j                         d d d        y # 1 sw Y   fxY w# 1 sw Y   y xY w)	Nr  r   r   c                 @    t         j                  j                  d      S r  r  r5   s    r,   r(  ziOneDeviceDistributionTestBase._test_device_and_input_device_are_colocated_with_function.<locals>.<lambda>g  r  r+   c                     | dz   S r1  r*   rr  s    r,   r(  ziOneDeviceDistributionTestBase._test_device_and_input_device_are_colocated_with_function.<locals>.<lambda>h  r  r+   z)/job:worker/replica:0/task:1/device:CPU:0rI   r  )r   r/   rO  r   r  r  r   r0   rJ  r   devicer  r  r  r   r  )	r   rt   r  r6   r  r  rJ  r  r   s	            r,   9_test_device_and_input_device_are_colocated_with_functionzWOneDeviceDistributionTestBase._test_device_and_input_device_are_colocated_with_functiona  s      "
mm
FH//15JGQ,,.0FG.|,,.x/H/HI	?	@ 10f1 
		GAJ$5$5	6 $
hhv  "#
hhv 1 1 s   
C<1D<DDN)r'   r(   r)   rj  r~  r  r  r  r  r  r  r  r  r  r  r  r*   r+   r,   rl  rl    sD    M
EP>>Q??
3;(;(r+   rl  c                   h    e Zd ZdZddZddZddZddZddZddZ		 ddZ
	 dd	Z	 dd
Z	 ddZy)TwoDeviceDistributionTestBasezESome tests that should work with any two-device DistributionStrategy.c                 8   |j                  t        d |            }| j                  ddg| j                  |j	                  |                   |j                  t        d |      |f      }| j                  t        j                  |j                  |            }| j                  ddg|d          | j                  ddg|d          |j                  t        d	 |      |
      }| j                  ddg| j                  |j	                  |                   y )Nc                  D    t        j                         j                  dz   S r1  )r	   r9   r   r*   r+   r,   r(  z9TwoDeviceDistributionTestBase._test_run.<locals>.<lambda>w  s    224MMPQQ r+   rI   r   c                     | dz  | | z  dS rq  r*   rr  s    r,   r(  z9TwoDeviceDistributionTestBase._test_run.<locals>.<lambda>|  s    AAE* r+   r   rB  r   r   c                     |d| z  z   dz   S rt  r*   ru  s     r,   r(  z9TwoDeviceDistributionTestBase._test_run.<locals>.<lambda>  s    QQY] r+   rv  r@     )r   r3   r   r   r   r#   ry  )r   rt   r2   rz  r{  r|  r}  s          r,   r~  z'TwoDeviceDistributionTestBase._test_runu  s   <<.Q D 	Ahood.C DE<<.*O=DH7  LDd00$GHIA	#/A	#/<<."O5=A  CDBxt/D!EFr+   c                 T    | j                  |t        ddgddgddggfdddgf|	       y )
Nr         @     C@r       D@ro  rx  r  r  r  r2   r  r   rt   r2   s      r,   r  z2TwoDeviceDistributionTestBase._test_all_reduce_sum  sC    RC9r3i01sCj!'   )r+   c                 @    | j                  |t        ddgddg|       y Nr   r  ro  r  r  r2   r  r  s      r,   r  z<TwoDeviceDistributionTestBase._test_all_reduce_sum_gradients  s)    ))(B8RH' * )r+   c                 @    | j                  |t        ddgddg|       y r  r  r  s      r,   r  z@TwoDeviceDistributionTestBase._test_all_reduce_sum_gradient_tape  s)    --(B8RH' . )r+   c                 T    | j                  |t        ddgddgddggfdddgf|       y )	Nr   r  r  r  r  r  g     5@r  r  r  s      r,   r  z3TwoDeviceDistributionTestBase._test_all_reduce_mean  sC    RC9r3i01sDk"'   )r+   c                 @    | j                  |t        ddgddg|       y Nr   r  r  r  r  r  s      r,   r  z=TwoDeviceDistributionTestBase._test_all_reduce_mean_gradients  s)    )))RHb"X' * )r+   c                 @    | j                  |t        ddgddg|       y r  r  r  s      r,   r  zATwoDeviceDistributionTestBase._test_all_reduce_mean_gradient_tape  s)    --)RHb"X' . )r+   c                 p   |j                  fd      | j                  j                                | j                  t        t	        |j
                  |j                  t        ||                              }| j                  |d   |d   g|d          | j                  |d   |d   g|d          y )Nc                 B    t         j                  j                        S r.   r   r  from_tensor_slicesr  s    r,   r(  zFTwoDeviceDistributionTestBase._test_collective_comms.<locals>.<lambda>      +%%88@ r+   r   rI   )	r  r   r  rN   r  r  rJ  r3   r   )r   rt   r  r  r  r2   r  s      `   r,   r  z4TwoDeviceDistributionTestBase._test_collective_comms  s    ,,@BF 	MM&##%&mm33))*7ODfNO	PQG
 	!hqk2GAJ?!hqk2GAJ?r+   c                 Z   t        j                         r|s| j                  d       fd}|j                  fd      | j	                  j                                | j                  || j	                  |j                  |j                  t        ||                               y )NzP`tf.gradients` is not supported with eager execution without using tf.functions.c                 x    t        j                  d      } |      | z  }t        j                  ||g      d   S r  r  r  s      r,   r   zLTwoDeviceDistributionTestBase._test_collective_comms_gradients.<locals>.step  r  r+   c                 B    t         j                  j                        S r.   r  r  s    r,   r(  zPTwoDeviceDistributionTestBase._test_collective_comms_gradients.<locals>.<lambda>  r  r+   )
r   r/   rO  r  r   r  r   r  rJ  r3   r   rt   r  r  r  r2   r   s     ``   r,   r  z>TwoDeviceDistributionTestBase._test_collective_comms_gradients  s      "?
mm 2 31
 ,,@BF 	MM&##%&//))*4A6KL	MNr+   c                    fd}|j                  fd      | j                  j                                | j                  || j                  |j	                  |j                  t        ||                               y )Nc                     t        j                  d      }t        j                         5 }|j	                  |        |      | z  }d d d        j                  |      S # 1 sw Y   xY wr  r  r  s       r,   r   zPTwoDeviceDistributionTestBase._test_collective_comms_gradient_tape.<locals>.step  r  r  c                 B    t         j                  j                        S r.   r  r  s    r,   r(  zTTwoDeviceDistributionTestBase._test_collective_comms_gradient_tape.<locals>.<lambda>  r  r+   )r  r   r  r   r  rJ  r3   r  s     ``   r,   r  zBTwoDeviceDistributionTestBase._test_collective_comms_gradient_tape  s{    ! ,,@BF 	MM&##%&//))*4A	r+   Nri  )r'   r(   r)   rj  r~  r  r  r  r  r  r  r  r  r  r*   r+   r,   r  r  r  sR    MG ))
)
)) ;@) .3@ HMN0 <Ar+   r  c                   4    e Zd ZdZd Zd Zd Zd Zd Zd Z	y)	&RemoteSingleWorkerMirroredStrategyBasez!Tests for a Remote single worker.c                      y r.   r*   )r   s    r,   _get_num_gpusz4RemoteSingleWorkerMirroredStrategyBase._get_num_gpus  s    r+   c                 X    | j                  | j                         |j                         y r.   )r   r  r  r   distributions     r,   _testNumReplicasInSyncz=RemoteSingleWorkerMirroredStrategyBase._testNumReplicasInSync  s!    T'')<+L+LMr+   c                 v    t        j                         r| j                  |       y | j                  |d       y )Ng?)r   )r   r/   r   r   r  s     r,   _testMinimizeLossz8RemoteSingleWorkerMirroredStrategyBase._testMinimizeLoss  s/      "
$$\2
$$\$Fr+   c                    |j                         5  t        j                  d      }t        j                  d      5  t        j                  d      }d d d        t        j                         rd}nd}| j                  |j                  |       | j                  j                  d       d d d        y # 1 sw Y   cxY w# 1 sw Y   y xY w)Nr   z/cpu:0z)/job:worker/replica:0/task:0/device:CPU:0z/job:worker/replica:0/task:0)r   r   r   r   r  r   r/   r   )r   r  r   r   r  s        r,   _testDeviceScopez7RemoteSingleWorkerMirroredStrategyBase._testDeviceScope  s    				 	N


R
 a::h #r"#		"	"	$</
qxx(
qxx!LM	N 	N# #	N 	Ns#   +C B4AC 4B=	9C  C	c           	         d }| j                         }d}t        dd|      D cg c]!  }t        |      D cg c]  }||z   	 c}|z  # }}}| j                         5 }| j                  |||z  |d       }	|j	                  |	      }
| j                  |
|j                  j                  ||       d d d        y c c}w c c}}w # 1 sw Y   y xY w)Nc                  @    t         j                  j                  d      S Nd   r  r*   r+   r,   r(  z\RemoteSingleWorkerMirroredStrategyBase._testMakeInputFnIteratorWithDataset.<locals>.<lambda>  s    ,,2237 r+   rI   r   r  r  r  r  )r  r   r   r  r  r  r=   r   )r   r  
dataset_fnnum_gpusnum_workersr   jr  r   r  r"  s              r,   #_testMakeInputFnIteratorWithDatasetzJRemoteSingleWorkerMirroredStrategyBase._testMakeInputFnIteratorWithDataset  s    7J!!#HK !&ah 79 &+8_5!5C 9O 9 
			 
Q$ 55
(3H(<'2%)	 6 +h
 44X>h
""
L))88/4Q
Q 
Q	 6 9
Q 
Qs#   B?B:B?AC:B?Cc           	         d }| j                         }d}g }t        dd|      D ]0  }|j                  t        |      D cg c]  }||z   	 c}|z         2 | j                         5 }| j	                  |||z  |d       }	|j                  |	      }
| j                  |
|j                  j                  ||dd       d d d        y c c}w # 1 sw Y   y xY w)	Nc                      t         j                  j                  d      } t        j                  |       }|j                  S r  )r   r  r   make_one_shot_iteratorr)  )datasetits     r,   r1   zWRemoteSingleWorkerMirroredStrategyBase._testMakeInputFnIteratorWithCallable.<locals>.fn  s2    ##))#.g--g6b[[r+   rI   r   r  r	  FTr  )	r  r   rU   r   r  r  r  r=   r   )r   r  r1   r  r  r  r   r  r   r  r"  s              r,   $_testMakeInputFnIteratorWithCallablezKRemoteSingleWorkerMirroredStrategyBase._testMakeInputFnIteratorWithCallable  s    
 !!#HKO1c8$ K5?;aac;kIJK 
			 6$ 55
(3H(<'2%)	 6 +h
 44X>h
""
L))88/4! # 66 6 <6 6s    C
(ACCN)
r'   r(   r)   rj  r  r   r  r  r  r  r*   r+   r,   r  r    s'    )	NG
NQ*6r+   r  c                 ~    t        j                         }|j                  t        j                  j
                  |       S r.   )r	   r9   
all_reducer   r   r   r   ctxs     r,   r  r  2  s-    **,#	,,00%	88r+   c                 ~    t        j                         }|j                  t        j                  j
                  |       S r.   )r	   r9   r  r   r   MEANr  s     r,   r  r  7  s-    **,#	,,115	99r+   ri  r.   )Orj  rc   rP   r   r   rK  tensorflow.core.protobufr   tensorflow.core.utilr   tensorflow.python.clientr   r  tensorflow.python.data.opsr   tensorflow.python.distributer   r~   r	   r
   r   rx   r   r   tensorflow.python.eagerr   r   r   r   tensorflow.python.frameworkr   r   r   r   tensorflow.python.lib.ior   tensorflow.python.opsr   r   r   r   r   r   r   r   r   r   tensorflow.python.platformr    tensorflow.python.trainingr!   r"   tensorflow.python.utilr#   r$   	Exceptionr&   r3   r7   r;   rA   rC   rE   rG   r]   ri   rs   Strategyboolr|   r   r   TestCaser   rl  r  r  r  r  r*   r+   r,   <module>r+     s`   <  	   / * ; 2 S 7 9 J 4 5 , + 0 ( . . + 1 . + 0 * - * ? 0 - + , 0 4 ' -Y 'G;C8I<?H>#:#: Ht HH%%H*.H2n55 2$ 2W.4== W.t
vDMM vrqDMM qhI6-A I6X9
:r+   