AVh>M2dZddlmZddlZddlZddlmZddlmZddlm Z ddl m Z ddl mZdd l mZdd lmZdd lmZdd lmZdd lmZddlmZddlmZddlmZddlmZddlmZddlmZedgdZ e jBdfdZ"dZ#dZ$dZ%dZ&dZ'dZ(dZ)d Z*d!Z+d"Z,d#Z-d$Z.d%Z/d&Z0d'Z1Gd(d)ejdZ3e3Z4ejjd*Z6ejnjpe jrejnjte jviZzregroup..M+qad+z3Values to regroup had different lengths: len(v) == z , len(v0) == z, v: z, v0: c3^K|]#ttfdD%yw)c3(K|] }| ywr%r&r's rr+z$regroup...Xr,r-N)regrouptuple)r(r* always_wrapr wrap_classs @rr+zregroup..Ws-!  +F++ZE!s)-_fields_makezv[0]: z v[i]: z v[0].keys: z v[i].keys: c3(K|] }| ywr%r&)r(r)keys rr+zregroup..js2a1S62r-TFzids = z , values = ) isinstancelistlenranger0r1hasattrr5rMappingkeyssettype values_libDistributedVariablevalue_containerr_UnreadVariableMirroredVariableid) rr3r2v0r)r*regrouped_tuplev0keysr7same_iddistributed_containers ``` ` ` rr0r0:smay"D ABZ; 4  Vs2w ;!M"%a&#b'1b!9":; ; s2w   +F++ZE  E ABZ6 5 !! ! Vs2w 6"..1!fX]3r7)L''(ct"56 6 !s2w!!Or9R !! ! XXo && CKK WWYF ABZJ 3;; 'I"a*HI ' ]c&k )J-0[#affh--IJ )J 48  WU2622. .  ' !": a{g   B > >? I [_R%8B%> I R.>> ?b#"j99:E39#:aBqE#:FCE :+B/ ,, , ABZ9 "oa&8 88 89   F ] :R$;s&L2('L7"L<c:fd}tj||S)zBSpecialize a nest of regular & per-replica values for one replica.ct|tjst|tjs|S|jSr%)r8rArBDistributedValuesr)x replica_ids r_getzselect_replica.._gets9 1j445 q*66 7 h XXj !!r!r map_structure)rP structuredrQs` rselect_replicarUs"   D* --r!c0t|t||S)z?Specialize a nest of regular & mirrored values for one replica.)assert_mirroredrU)rPrTs rselect_replica_mirroredrXs*  J //r!c<fd}tj|y)zGRaises if the structured is not composed of mirrored or regular values.cvt|tjrt|st d|ddyy)Nz/Expected value to be mirrored across replicas: z in .)r8rArN is_mirrored TypeError)rOrTs r_assert_mirroredz)assert_mirrored.._assert_mirroreds9!Z112;q>  j  ._make_grouped_mirrored..s9Q{%%a(9s!N) r:rAMirroredrgroupallr devicecontrol_dependenciesappendr identity)rgwith_depr)s r_make_grouped_mirroredz.update_regroup.._make_grouped_mirroreds 6{a   (( v&A 9&9 9 hH / ::ahh /!9!91#!>/ **1-.////   x ((////s$3C" %C/C"C C""C+ )r0rArcrrS_local_results)extendedupdatesrd regroupedrls rupdate_regrouprqsE !4!45I   h55y AA)0 0 11r!c,d}t|tjsst|dr|j }nVt|t j r>@/22777  6&>* + ,,R77??77@@77<<>>  &. * ++22777 % % . .. r!cR|jdtjj}|tjjtjjtjj tjj fvrtd|d|d|S)N aggregationz#Invalid variable aggregation mode: rr)rrVariableAggregationrrrONLY_FIRST_REPLICAr)rrs r_validate_aggregationr3s ="*@*@*E*EF+//44//33//44//BBDD !6&>3 44 r!c |jddrd}nd}|jdd}|tjjg}g|d<t |}||d<t |}t |jdd} |jd dtj5|di|} | D]5} t| d s| jtj| _ 7| r8|j|} | | } |j|}||| || }n|j|}||| |}dddtj stj"}|jd dru|j%tjj&|j)tjj&} D]}t+|D]\}}||us ||= |j-||Stjj.|vr.tj,tjj.S#1swYxYw)zHCreate distributed variables with given synchronization and aggregation.!experimental_batch_initializationNLazyVariableClass VariableClass collectionsr_use_var_policyFcaching_device_initializer_op)r) var_policy trainableTr&)popr GraphKeysGLOBAL_VARIABLESrrr{rnr stop_recordingr<rrno_oprrexecuting_eagerlyrrhTRAINABLE_VARIABLESget_collection_ref enumerateadd_to_collections GLOBAL_STEP)strategyreal_mirrored_creator class_mappingpolicy_mappingrvariable_class_keyvar_collectionsrruse_var_policy value_listr)var_policy_clsrvar_clsresultrjlvaluer*trainable_variables rcreate_mirrored_variabler?sD  ZZ3T:,(JJ}d3/}}556O&-f5/.& %f-+8,,.?G. ** t$ :&00J 5 % &1+<+<+D,2245 %))/:n!k:j!!"45gx[ZPf!!/2gx[9f':0  " " $ A zz+t$S]]>>?  s}}@@ Aa%%.q\  !A! ( (!   &1 - }}  O33==44f= -Y::s$I? I A0IIc(t|ddS)N _is_mirroredcy)NFr&r&r!rzis_mirrored..sr!rzrvs rr\r\s 5'#~} 5 88r!ct| Sr%)r\rs ris_sync_on_readrs   r!c4eZdZdZfdZdZdZdZxZS)CachingScopeLocalz;Class for maintaining thread local state for caching scope.cFtt| d|_d|_y)Nr)superr__init__new_cache_scope_countcache_scope_exited_count)self __class__s rrzCachingScopeLocal.__init__s! T+-!"D$%D!r!c.|xjdz c_yNr#)rrs r enter_scopezCachingScopeLocal.enter_scopes!#r!c.|xjdz c_yr)rrs r exit_scopezCachingScopeLocal.exit_scopes!!Q&!r!c4|j|jkDSr%)rrrs rin_caching_scopez"CachingScopeLocal.in_caching_scopes  % %(E(E EEr!) __name__ __module__ __qualname____doc__rrrr __classcell__)rs@rrrsC& $'Fr!rc#K tjr tdtjdtj y#tj wxYww)aScope for caching variable reads for AggregatingVariable. The variable reads for AggregatingVariable inside this scope are cached. i.e. the first read of variable reads the value from possibly remote handle, but subsequent reads are returned using local cached value. For example: strategy = ParameterServerStrategy... with strategy.scope(): # Variable v is of AggregatingVariable type with actual variable residing # on PS. v = tf.Variable(1.0) with distribute_utils.cache_variable_reads(): v.read_value() # Reads value 1.0 v.assign(constant_op.constant(5.0)) # v changes to 5.0 t1 = v.read_value() t2 = v.read_value() # Both t1 & t2 return cached value 1.0 from local CPU. Notes about cache_variable_reads scope: 1. Nesting of scope cache_variable_reads() is not supported 2. And when caching scope is enabled, the thread enabling the cache and mirrored_run._MirroredReplicaThread threads spawned from it will have caching enabled. Yields: A context for caching variables. z+cache_variable_reads scope cannot be nestedN)caching_scope_localrrrrr&r!rcache_variable_readsrsK>%++- D EE##% ""$""$sA)7AA)A&&A)rr)Irrr contextlib threadingtensorflow.python.distributerrtpu_values_librrA(tensorflow.python.distribute.reduce_utilrtensorflow.python.eagerrr tensorflow.python.frameworkr r r tensorflow.python.opsr rrrrtensorflow.python.ops.lossesrtensorflow.python.utilr tensorflow.python.util.tf_exportrr PerReplicar0rUrXrWrqrCryr~rrrrrrr\rlocalrrcontextmanagerrrr OnWritePolicyr OnReadPolicyVARIABLE_POLICY_MAPPINGrBrESyncOnReadVariableVARIABLE_CLASS_MAPPINGTPUOnWritePolicyTPUOnReadPolicyTPU_VARIABLE_POLICY_MAPPINGTPUDistributedVariableTPULazyDistributedVariableTPUMirroredVariableTPUSyncOnReadVariableTPU_VARIABLE_CLASS_MAPPINGr&r!rrsC7E==+*8+3+2764'6 ./01$ *44%`F . 0 32B566 3  ++( I^9F F$() %%%%`'')A)A&& (?(? Z33'')D)D&& (E(E'')H)H&&(F(F ^::BB'')K)K&&(L(L r!