AVhPdZddlmZmZmZddlZddlmZddlm Z ddlm Z ddlm Z ddlm Z dd lm Z dd lmZdd lmZdd lmZdd lmZddlmZddlmZddlmZddlmZddlmZddlmZddlm Z ddl!m"Z"ddl#m$Z$ddl#m%Z%ddl&m'Z'ddl&m(Z(ddl)m*Z*GddZ+GddejXZ-Gdd ej\Z/e'j`dd!"e*d#g$Gd%d&ejbZ2y)'zDTensor Checkpoint. Note that this module contains deprecated functionality, and the DTensor related checkpoint has been integrated with tf.train.Checkpoint. It can be used out of the box to save and restore dtensors. )DictListOptionalN)trackable_object_graph_pb2)api) d_variable)gen_dtensor_ops)layout) save_restore) checkpoint)checkpoint_options graph_view)restore)context) constant_op) errors_impl)ops) array_ops)base)data_structures)py_checkpoint_reader)saveable_object)saveable_object_util) deprecation)nest) tf_exportc eZdZdZdej deejfdZ d de de e jde ejfd Z d de de e jdee ejffd Zy) _DSaverz isinstancer DSaveSpecr?dtyper global_shaperSaveSpecr:r;r%r8r4r<r0 to_stringas_listziprr6rr5r d_tensor_restore_v2rpack_sequence_asr)r&r*r+ restore_specstensor_structurerDsaveable_tensor_structurerErArC tensor_dtypeslayouts global_shapesrestored_tensorsstructured_restored_tensors restore_opss r'rz_DSaver.restoreks2w==I    M**-"$78 .. -$!((3 dJ00 1    4??DJJ $ T->-> @ Ao66 7   yy$//4::}}'' (<(<(>(, (9(9(>(>@@I {{  ((*, - --*JM JFL- COO% &  )<<#($    #'"7"78H8H#JK&)$*@*@*E'G2""#+#3#3 D$4$2k(-- 2 #   s II#r#)__name__ __module__ __qualname____doc__r MeshrrSaveableObjectr(strrr CheckpointOptionsr OperationrFrrr)r'rr3sD6;;!%o&D&D!EAE.=.=*<<=.=  .=fAE>>*<<=> C  >r)rc eZdZdZdej ffd Z d deee jfde e jdeeeeeej ffdej$dee ej(f d ZxZS) _DCheckpointRestoreCoordinatorz4Holds the status of an object-based checkpoint load.r c 2t|di|||_y)Nrj)superr(r%)r&r kwargs __class__s r'r(z'_DCheckpointRestoreCoordinator.__init__s GvDJr)tensor_saveablespython_positionsregistered_saversreaderr,c~g}|ru|tj|j}|D]O}|jjdj }|j j|j|Q|rtj|}td|D} t|j| k7rtd|jd| t|j|j!|j"|j$} t'j(sQt+| j-D]5\} } |j/| | |j0vsJ| |j0| <7|S)aaRun or build restore operations for SaveableObjects. Args: tensor_saveables: `SaveableObject`s which correspond to Tensors. python_positions: `CheckpointPosition`s which correspond to `PythonState` Trackables bound to the checkpoint. registered_savers: a dict mapping saver names-> object name -> Trackable. This argument is not implemented for DTensorCheckpoint. reader: A CheckpointReader. Creates one lazily if None. Returns: When graph building, a list of restore operations, either cached or newly created, to restore `tensor_saveables`. rc34K|]}|jywr#r>).0rDs r' zC_DCheckpointRestoreCoordinator.restore_saveables..sNhHMMNsz0Saveable keys changed when validating. Got back z, was expecting )rNewCheckpointReadersave_path_string object_proto attributescheckpoint_key trackable deserialize get_tensorrvalidate_and_slice_inputssetkeysAssertionErrorrr%rsave_path_tensorr+rexecuting_eagerlysorteditemsr=restore_ops_by_name) r&rqrrrsrtr`positionkeyvalidated_saveablesvalidated_namesnew_restore_opsr> restore_ops r'restore_saveablesz0_DCheckpointRestoreCoordinator.restore_saveablesso, K %99$:O:OP&?(##..q1@@&&v'8'8'=>? 0JJ N:MNNo  " " $ % 8 0 5 5 7 JK K   ,?@HH   /o  & & ( &'<'<'> ? 6 D*   Z (T555 55+5$ " "4 ( 6 r))NN)rarbrcrdr rer(rrgrrfr restore_libCheckpointPositionrr Trackablerrzrrir __classcell__rps@r'rlrls<6;;6:KO 5*.s/>/M/M0N+O5+/{/M/M*N5,4Dd/?1:152,3 5 !5 H H 5&d3==&9:5r)rlcVeZdZdZdej ffd ZddZ ddZddZ xZ S) DTrackableSaverz7A DTensor trackable saver that uses _SingleDeviceSaver.r c:tt| |||_yr#)rnrr(r%)r&r rrps r'r(zDTrackableSaver.__init__s /4)*5DJr)c|j|\}}}}|j}|7|jdtj}t j |}g}|jD](}|jD]} |j| *|jtj|tj||||fS)N)r4r>) _gather_serialized_tensors_saveables_cachepoprOBJECT_GRAPH_PROTO_KEYr$serialized_tensors_to_saveable_cachevaluesextendr=NoRestoreSaveable) r&object_graph_tensorserialized_tensorsfeed_additionsrs graph_protosaveables_dictnamed_saveable_objectssaveable_by_name saveabless r'_gather_saveablesz!DTrackableSaver._gather_saveabless334GH):**N/22 ++-  C C  " *1131'..01)%%i011!! &,, ./ #K  r)c|j|\}}}}|j|k7s(tjst j rt |j|} | j||} t jd5t j| g5tj||_ dddddd||_|j|fS#1swY&xYw#1swY*xYw)ajCreate or retrieve save ops, overrides parents's private method. Args: file_prefix: The prefix for saved checkpoint files. object_graph_tensor: A `Tensor` to which the current object graph will be fed. options: `CheckpointOptions` object. update_ckpt_state: Optional bool flag. Indiciate whether the internal checkpoint state needs to be updated. This is used for async checkpoint, which DTrackableSaver currently does not support. TODO(chienchunh): Implement async checkpoint for DTrackableSaver. Returns: A two-element tuple with a filename tensor and a feed_dict of tensors to feed when running it (if graph building). The feed dict contains the current object graph and any Python state to be saved in the checkpoint. When executing eagerly only the first argument is meaningful. )r)r+z/cpu:0N)r_last_save_object_graphrrrinside_functionrr%rFr6control_dependenciesridentity_cached_save_operation) r&r*rr+update_ckpt_staterrrunused_registered_saverssaversave_ops r' _save_cached_when_graph_buildingz0DTrackableSaver._save_cached_when_graph_building s0"&!7!70"8"2[. $$ 3  $ $ &#*=*=*? djj"89e ; 8g ::h H  % %wi 0 H(1(:(:;(G$ % HH&1d"  & & 66 H HHHs$ C0$C$?C0$C- )C00C9c |xstj}|2tj|jt j Stj|}tj }|rd}n|j} |jtj}|r|j0t3j4t7j8dg|j:j=zt>j@jC|j:jEd|_|j0} |j0|i} n~t3j4t7j8|g|j:j=zt>j@jC|j:jEd} d} tGjH} | jK|tM|j:| || ||jN|j||jP } tSjT| djW|jjX|jjZr|jjZD]} | j\d k(rd}| j^dj`D])}|jb| j\k(s|jd}n|| jfvritSjT| |jW| jhtjj| |j| | }|S#tj $rtj"||}|s[|jj%D]>}|j'|j(j+||j-|@tj.||jcYSwxYw) z7Restore a training checkpoint with host mesh placement.N) save_path dtype_maprmodelrr/) r object_graph_protorrrtrestore_op_cacherr+saveables_cache)r proto_idroot)r feed_dictr+)6r rhutilInitializationOnlyStatus _graph_viewruidrrzrrget_variable_to_dtype_maprrrr NotFoundError_NameBasedRestoreCoordinator list_objects_maybe_initialize_trackable_name_based_restoresadd_name_based_attribute_restoreNameBasedSaverStatus_file_prefix_placeholderrr7rconstantr%r9r r:r;r8rTrackableObjectGraphParseFromStringrl_restore_op_cacherrrrrattached_dependenciesr>nodeschildren local_namenode_idobject_by_proto_idrefCheckpointLoadStatus)r&rr+rtgraph_buildingrobject_graph_stringrestore_coordinatorexisting_trackablefile_prefix_tensorfile_prefix_feed_dictrr rr proto_ref load_statuss r'rzDTrackableSaver.restore;s?+==?G  * *4+;+;SWWY GG ! 5 5i @F 2244Ni224i<"--d.I.IJ" & & .),  ! !' * +djj.J.J.L L MM $ $TZZ%9%9%;! $ D)F% 88#<:? +7\Yr)rzPlease use tf.train.Checkpoint instead of DTensorCheckpoint. DTensor is integrated with tf.train.Checkpoint and it can be used out of the box to save and restore dtensors.)date instructionsz&experimental.dtensor.DTensorCheckpoint)v1c>eZdZdZddej ffd ZxZS)DTensorCheckpointz?Manages saving/restoring trackable values to disk, for DTensor.r c &tt| dd|i|||_|}d}d|_d|_|rrt j|d|}g}||d<|jtj|jd|_tj||_ t|jdD]\}}t|||t!||}t j|||s8|xsg}|j|} | &|j#t%j&||w| |k7s}t)dj+|t-|t/j0t3j4|||_y) Nr save_counterc |dS)Nrrj)items r'z,DTensorCheckpoint.__init__..s Qr))rzVCannot create a Checkpoint with keyword argument {name} if root.{name} already exists.rw)rrj)rnrr(r% _save_counter_save_assign_opr_assert_trackablerr NoDependency_lookup_dependency_rootrrsetattrgetattrr=rTrackableReferencer2formatrgraph_view_libObjectGraphViewweakrefr_saver) r&r rro saver_rootrkv converted_vchildrps r'r(zDTensorCheckpoint.__init__s T+@@@DJJ DD  T6*j fVn &&(*77  ! !. 13d"//5djv||~+?@<1 dAqD!$k [!, 5 ;''* =  & &t'>'>q+'N O k !,,2FFN< <<(" && KK #"7 9:DKr)r#)rarbrcrdr rer(rrs@r'rrsH/:6;;/:/:r)r)3rdtypingrrrrtensorflow.core.protobufrtensorflow.dtensor.pythonrrr r r tensorflow.python.checkpointr rr rrrrtensorflow.python.eagerrtensorflow.python.frameworkrrrtensorflow.python.opsrtensorflow.python.trackablerrtensorflow.python.trainingr!tensorflow.python.training.savingrrtensorflow.python.utilrr tensorflow.python.util.tf_exportrr_CheckpointRestoreCoordinatorrlTrackableSaverr deprecated Checkpointrrjr)r'rs('?)05,2;;E?+33++,7;=B.'6vvr<T%G%G<~id))iX 89  3;2:2:< 9 2:r)