
    BVhm                        d Z ddlZddlZddlZddlmZmZmZmZm	Z	 ddl
mZ ddlmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m!Z! ddl m"Z" ddl m#Z# ddl m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z* ddl)m+Z+ ddl,m-Z- ddl,m.Z. ddl/m0Z0 ddl1m2Z2 ddl1m3Z3 ee&jh                  ee5e*jl                  f   f   Z7ee0jp                  e	ejr                     gejr                  f   Z:	 d6dejr                  d ejv                  d!ejx                  d"d#d$ejz                  f
d%Z>	 d6dejr                  d&e	ej~                     d"d#d$ejv                  fd'Z@d(ejr                  d eAd)ejr                  d$ejr                  fd*ZBd(ejr                  d+e&jh                  d$ejr                  fd,ZCd-ed.ejr                  f   d/e	e*jl                     d0e:d$eejr                  ge:f   fd1ZDd-ed.ejr                  f   d/e	e*jl                     d0e:d$ed.ejr                  f   fd2ZEd3 ZFeGe5e5f   ZHeee5ejr                  f   gejz                  f   ZI G d4 d5      ZJy)7z8Saves and restore variables inside traced @tf.functions.    N)CallableMappingMutableMappingMutableSequenceSequence)logging)	saver_pb2)checkpoint_options)sharding_policies)sharding_util)context)def_function)constant_op)device)dtypes)ops)tensor)tensor_spec)tensor_util)	array_ops)
gen_io_ops)io_ops)
string_ops)registration)metrics)base)trackable_utils)saveable_object)saveable_object_util)core)nest)object_identityfile_prefixshardtaskoptions+checkpoint_options.CheckpointOptions | Nonereturnc                 (   |xs t        j                         }g }g }g }|j                         D ]  \  }}|j                         D ]o  \  }	}
|
	t        |
d      r|
j                  n|}t        |
d      r|
j
                  n|	}|j                  |       |j                  |
       |j                  |       q  |j                  xs |xr |}t        j                  |xs d      5  t        j                  | |||      cddd       S # 1 sw Y   yxY w)a  Save the saveable objects to a checkpoint with `file_prefix`.

  Args:
    file_prefix: A string or scalar string Tensor containing the prefix to
      save under.
    shard: Dict containing tensors. {checkpoint key: {slice_spec: tensor} }
    task: The device spec task of the tensors in the shard.
    options: Optional `CheckpointOptions` object.

  Returns:
    An `Operation`, or None when executing eagerly.
  N_wrapped_name_wrapped_slice_speczCPU:0)r
   CheckpointOptionsitemshasattrr*   r+   appendexperimental_io_devicer   r   r   save_v2)r#   r$   r%   r&   tensor_namestensorsslice_specscheckpoint_keytensor_slices
slice_specr   namespecsave_devices                 ]/home/dcms/DCMS/lib/python3.12/site-packages/tensorflow/python/checkpoint/functional_saver.py_single_shard_saver<   :   s"   $ =);;=','+',{{} !#nm+113 !
F 
	 6?3 $$# 	 6#89 ** 	 	D!v4 !!!& ..D73Ct+
zz+() K>>+|['JK K Ks   &DDshardable_tensorsc                    |xs t        j                         }g }g }g }|D ]  }|j                  r-|j                  j                  }|j                  j                  }n|j
                  |j                  }}|j                  |       |j                  |       |j                  |j                          |j                  xs d}	t        j                  |	      5  t        j                  | |||      }
ddd       i }|D ]<  }
j                  d      }||j                  |j
                  i       |j                  <   > |S # 1 sw Y   NxY w)az  Restore the saveable objects from a checkpoint with `file_prefix`.

  Args:
    file_prefix: A string or scalar string Tensor containing the prefix for
      files to read from.
    shardable_tensors: A list of ShardableTensors to restore.
    options: Optional `CheckpointOptions` object.

  Returns:
    A restored tensor dict (maps checkpoint_key -> slice_spec -> tensor).
  cpu:0Nr   )r
   r,   _tensor_save_specr8   r7   r5   r/   dtyper0   r   r   r   
restore_v2pop
setdefault)r#   r=   r&   r2   tensor_dtypesr4   shardable_tensorr8   r9   restore_devicerestored_tensorsrestored_tensor_dictrestored_tensors                r;   _single_shard_restorerK   i   sV     =);;=',-++ 1))//44d//::d#224D4O4ODdt)//01 11<W.
zz.! ?((\;?? + C&**1-O 4C j!00" #..0C
 
? ?s   D??Efilename_tensor
num_shardsc                 0    t        j                  | ||      S )zAppend sharding information to a filename.

  Args:
    filename_tensor: A string tensor.
    shard: Integer.  The shard for the filename.
    num_shards: An int Tensor for the number of shards.

  Returns:
    A string tensor.
  )r   sharded_filename)rL   r$   rM   s      r;   rO   rO      s     
	$	$_eZ	HH    
saver_namec                 \    t        j                  | t        j                  d|       g      S )N-)r   string_joinr   constant)rL   rQ   s     r;   registered_saver_filenamerV      s3     
		,,q-=>?
A ArP   fn.
trackablescall_with_mapped_capturesc                 J    dt         j                  dt         j                  f fd}|S t        j                  |d      }|j	                  t        j                  dt        j                              dt         j                  dt         j                  ffd	}|S )
HConverts the function to a python or tf.function with a single file arg.r#   r(   c                      |       S )N)rX   r#    )r#   rW   rX   s    r;   save_fnz/_get_mapped_registered_save_fn.<locals>.save_fn   s    ==rP   F	autographr]   shaperA   )r#   c                      | g      S Nr]   )r#   rY   concretes    r;   save_fn_with_replaced_captureszF_get_mapped_registered_save_fn.<locals>.save_fn_with_replaced_captures   s    &x+??rP   	
tensor_libTensorr   functionget_concrete_functionr   
TensorSpecr   string)rW   rX   rY   r^   tf_fnrf   re   s   ```   @r;   _get_mapped_registered_save_fnro      s    >:,, >1B1B >&N!!'U;E****6==I + KH@&&@+5+<+<@ *)rP   c                 J    dt         j                  dt         j                  f fd}|S t        j                  |d      }|j	                  t        j                  dt        j                              dt         j                  dt         j                  ffd	}|S )
r[   merged_prefixr(   c                      |       S )N)rX   rq   r]   )rq   rW   rX   s    r;   
restore_fnz5_get_mapped_registered_restore_fn.<locals>.restore_fn   s    =AArP   Fr_   r]   ra   )rq   c                      | g      S rd   r]   )rq   rY   re   s    r;   !restore_fn_with_replaced_captureszL_get_mapped_registered_restore_fn.<locals>.restore_fn_with_replaced_captures   s    &x-AArP   rg   )rW   rX   rY   rs   rn   ru   re   s   ```   @r;   !_get_mapped_registered_restore_fnrv      s    B
 1 1 Bj6G6G B&!!*>E**!,,2V]]K + MHB!((B-7->->B -,rP   c                       y rd   r]   )argskwargss     r;   <lambda>rz      s    rP   c            
          e Zd ZdZ	 	 ddeej                  ej                  f   ddddfdZ	e
	 	 dd	eej                     ddddd
d fd       Zd
ej                  fdZ ej"                   ej&                  dej*                        fd      dej.                  d
ej.                  fd       Z ej"                   ej&                  dej*                        fd      dej.                  d
ej.                  fd       Zdej4                  d
eeeeej                     f      fdZ	 ddej.                  ddd
ej>                  fdZ 	 ddej.                  ddd
eeej>                  f   fdZ!y)MultiDeviceSaverzSaves checkpoints directly from multiple devices.

  Note that this is a low-level utility which stores Tensors in the keys
  specified by `SaveableObject`s. Higher-level utilities for object-based
  checkpointing are built on top of it.
  Nserialized_tensorsregistered_saverszRegisteredSaversDict | NonerY   zMappedCapturesCallable | Nonec                 L   i | _         i | _        i | _        t               }|j	                         D ]n  \  }}|t
        n|j                  }|j	                         D ]@  \  }}	t        |	t              sd|	i}	|	j	                         D ]  \  }
}d}t        |t        j                        s0|}t        j                  ||
||j                  |j                        }||
f| j                  v rt        d      || j                  ||
f<   | j                  j                  |g       j                  ||
f       t        |j                  t               rft"        j$                  j'                  |j                        }t"        j$                  j'                  t)        j*                  |j                              }nL|j                  }t"        j$                  j'                  t)        j*                  |j-                                     }| j                   j                  |g       j                  t/        j0                  |||j                  ||j2                  d|
j5                         ||	             |j7                  t)        j*                  |j-                                       C q t9        |      | _        i | _        |rj|j	                         D ]V  \  }}t?        tA        jB                  |      ||      }tE        tA        jF                  |      ||      }||f| j<                  |<   X yy)a  Specify a list of `SaveableObject`s to save and restore.

    Args:
      serialized_tensors: A dictionary mapping `Trackable` to a tensor dict,
        which maps checkpoint_key -> (slice_spec ->) -> Tensor/SaveSpec. The
        `Trackable` key is used to get the `restore_from_tensors` function,
        and may be `None` if the tensor is not meant to be restored.
      registered_savers: A dictionary mapping `registration.RegisteredSaver`
        namedtuples to a dictionary of named Trackables. The keys of the
        Trackable dictionary are string names that uniquely identify the
        Trackable in the checkpoint.
      call_with_mapped_captures: TODO
    N )r   r7   r8   rA   r   zReceived multiple tensors with the same checkpoint key and slice spec. This is invalid because one will overwrite the other in the checkpoint. This indicates a bug in the Checkpoint key-generation.)	r@   r   rA   r   r8   rb   r7   r5   	trackable)$_shardable_tensors_by_task_keys_to_restore_fn_restore_fn_to_keyssetr-   _restore_noop_restore_from_tensors
isinstancedictr   SaveSpecrA   r   
ValueErrorrD   r/   str
device_lib
DeviceSpecfrom_stringr   set_cpu0	to_stringr   ShardableTensorr8   stripaddlen_num_unique_tasks_registered_saversro   r   get_save_functionrv   get_restore_function)selfr}   r~   rY   unique_tasksobjtensor_dictrs   r5   tensor_slice_dictr7   tensor_save_spectensor_valuer   r%   registered_namerX   r^   s                     r;   __init__zMultiDeviceSaver.__init__   s!   ( ;= 	# -/ 	 >@ 	 5L.446 3A[$'K=S5N5Nj 0;/@/@/B /A
+.++T2!#45
,=,C,C,E *	A(J(,,o.F.FG+L.77#%#&,,'.. 0 j)T-E-EE-. .
 DN$
"
"NJ#?
@

"
"
-
-j"
=
D
Dz*, (//5**667G7N7NOF((44$--.>.E.EFHD &,,F((44$--f.>.>.@ACD 
)
)
4
4T2
>
E
E++$4%(..',,'--/!/	!
" 

"++F,<,<,>?AS*	A/A	3Aj !.D D):)@)@)B I
%/:0**?;13 7--o>13
 5<Z3H0I rP   	saveablesr(   c                     t        j                         }|D ]-  }t        j                  ||g      }|j	                         ||<   /  | |||      S )z?Constructs a MultiDeviceSaver from a list of `SaveableObject`s.)r   )r"   ObjectIdentityDictionaryr   SaveableCompatibilityConverter_serialize_to_tensors)clsr   r~   rY   r}   saveabler   s          r;   from_saveableszMultiDeviceSaver.from_saveablesK  sc     )AAC H&EE
xj*i&/&E&E&G#H !#46OPPrP   c                 D   t        j                  g t        j                  d      }| j	                  |      }| j                  |      j                  }t        j                  |j                  |j                  |j                  t        j                  j                        S )z7Serializes to a SaverDef referencing the current graph.saver_filename)rb   rA   r8   )filename_tensor_namesave_tensor_namerestore_op_nameversion)r   placeholderr   rm   _traced_save_traced_restoreopr	   SaverDefr8   V2)r   rL   save_tensor
restore_ops       r;   to_protozMultiDeviceSaver.to_protoZ  s    ++,<>O##O4K%%o699J,11$))"""%%	' 'rP   r]   ra   F)input_signaturer`   r#   c                    | j                  |      }t        j                  d      5  t        j                  |g      5  t	        j
                  |      cd d d        cd d d        S # 1 sw Y   nxY w	 d d d        y # 1 sw Y   y xY wNr?   )saver   r   control_dependenciesr   identity)r   r#   save_ops      r;   r   zMultiDeviceSaver._traced_savef  sz     ii$G	G	 /##WI. /!!+./ // // / // / /s"   A<A&	A<&A/	+A<<Bc                 *   | j                  |      }t        j                  d      5  t        j                  |j	                               5  t        j                  |      cd d d        cd d d        S # 1 sw Y   nxY w	 d d d        y # 1 sw Y   y xY wr   )restorer   r   r   valuesr   r   )r   r#   restore_opss      r;   r   z MultiDeviceSaver._traced_restoreo  s    
 ,,{+K	G	 /##K$6$6$89 /!!+./ // // / // / /s#   $B	A3 	B	3A<	8B		Bsharding_callbackc           
         d }| j                   j                         D ci c]"  \  }}|t        ||      D cg c]  }|| c}$ }}}}|xs t        j                         }t        j                  |j                         t        j                         dz  }g }|j                         D ]  \  }}|j                  | ||      f         t        j                         dz  }	t        j                  |	|z
        }
t        j                  t        d|
             t        j                  d|
       |S c c}w c c}}}w )zCalls the sharding callback with shardable_tensors.

    Args:
      sharding_callback: ShardingCallback. The callback function wrapper that
        splits shardable_tensors into shards.

    Returns:
      A list of (task, shards) tuples.
    c                    | j                   }| j                  }| j                  }t        j                  | j                        5  |j                   }d d d        |y x|}|j                  }t        |j                  t        j                        r|j                  |_	        t        | j                  t        j                        r|j                  |_        t        j                  | ||      S # 1 sw Y   xY w)N)r   rb   )r   rb   r@   r   r   r   r8   rh   ri   r*   r7   r+   dataclassesreplace)rF   
tensor_valtensor_shape	save_specsave_spec_tensors        r;   wrap_tensorz9MultiDeviceSaver._get_shards_by_task.<locals>.wrap_tensor  s    #**j%++l"44i::&--. ,$++, 
	 0 8 '%
'--
 innj&7&78%.^^*
"&11:3D3DE+4+?+?*
(  
 +, ,s   C00C9)descriptiong    .A   )callback_durationz+Sharding callback duration: %s microseconds)r   r-   mapr   ShardByTaskPolicyr   SetShardingCallbackDescriptionr   timer/   mathceilAddShardingCallbackDurationmaxr   info)r   r   r   r%   r=   rF   shardable_tensors_by_taskcallback_start_timeshards_by_taskcallback_end_timer   s              r;   _get_shards_by_taskz$MultiDeviceSaver._get_shards_by_tasky  sW   D (,'F'F'L'L'N	!P !P $D# 	'*;8I'J0#".   0 	0!P !P 	B.@@B **%113 ))++N#<#B#B#D JT#45F#GHIJ		c)		"36I"IJ''a!235LL>"$ -0!Ps   D;D6D;6D;r&   r'   c           	          xs t        j                         t        j                  d      5  t	        j
                  t        j                  d      t        j                  d      t        j                  d            }t        j                  |g       j                  D ci c]  }|t        |       c}ddd       dt        j                  f fdt        j                         r6 j                   dkD  r't#        j$                  d	
      dfd       } |        y        S c c}w # 1 sw Y   yxY w)a%  Save the saveable objects to a checkpoint with `file_prefix`.

    Args:
      file_prefix: A string or scalar string Tensor containing the prefix to
        save under.
      options: Optional `CheckpointOptions` object.
    Returns:
      An `Operation`, or None when executing eagerly.
    CPUz^s3://.*z.partz
_temp/partNr(   c            
         g } j                   j                         D ]]  \  }\  }} ||         }|t        j                  |      }t	        d |D              st        d| d      | j                  |       _ j                  j                        }t        |D cg c]  \  }}t        |       c}}      }t        j                  |       t        j                  |d      }	g }
d}|D ]j  \  }}|D ]`  }t        j                   |      5  t#        ||	      }|dz  }d d d        | j%                         |
j%                  t'        |||             b l t        j(                  |
      5  t+        j,                  j/                               d	   }j0                  xs# t3        j4                  |j7                               }t        j                   |      5  t9        j:                  | d
      cd d d        cd d d        S c c}}w # 1 sw Y   xY w# 1 sw Y   nxY w	 d d d        y # 1 sw Y   y xY w)Nc              3      K   | ]8  }t        j                  |      xr |j                  t        j                  k(   : y wrd   )r   
is_tf_typerA   r   rm   ).0xs     r;   	<genexpr>z9MultiDeviceSaver.save.<locals>.save_fn.<locals>.<genexpr>  s8      1 $$Q'DAGGv}},DD1s   >A zNRegistered saver must return a (maybe empty) list of string type tensors. Got .)rM   rM   )r8   r   r   T)delete_old_dirs)r   r-   r!   flattenallr   extendr   experimental_sharding_callbacksumr   r   AddNumCheckpointShardsWrittenr   rU   r   r   rO   r/   r<   r   listr   keysr0   r   r   r   r   merge_v2_checkpoints)saved_prefixesrQ   r^   _maybe_saved_prefixesflattened_saved_prefixesr   shardsrM   num_shards_tensorsharded_saves	shard_idxr%   r$   shard_prefixtensor_device_specmerge_device_specr#   r&   registered_pathsr   tmp_checkpoint_prefixs                    r;   r^   z&MultiDeviceSaver.save.<locals>.save_fn  sh   n '+&=&=&C&C&E 
:
"*lw&'7
'CD+%)\\2F%G
" 1/1 1 ,,@+ADE E 

 8
9
: //

0
02n^D	6FDEj++zB%..zMmi( F,$ 	FEzz$ +,A9,=?LNI 


-


 udGDF	FF ##M2 A "$"A"A"F"F"HI"M** J ))*<*F*F*HI 	 ZZ)* 	A 00k4A		A 	AA A! E 	A 	A 	AA A As7   #H
H5A-H5"H:	H5HH(	$H55H>r   F)jit_compilec                                y rd   r]   )r^   s   r;   tf_function_savez/MultiDeviceSaver.save.<locals>.tf_function_save+  s    	rP   )r(   N)r
   r,   r   r   r   wherer   regex_full_matchr   rU   rT   r   rV   	Operationr   executing_eagerlyr   r   rj   )	r   r#   r&   sharded_suffixrQ   r  r   r^   r   s	   ```   @@@r;   r   zMultiDeviceSaver.save  s$    ?+==?G< 
E	 
 

%
%k:
>


w
'


|
,.n )44
') !33 /ZH
H
.AS]] .A .Ah   "t'='='A/ 0YA
 
s   A7D:+D5?D:5D::Ec           
      H    xs t        j                         dt        t        t        j
                  f   f fdd} j                  j                         D ]J  }t        |D cg c]/  }t        j                  |j                  j                               1 c}      sHd} n t        j                         rW j                  dkD  s|rFt        j                   dd      dt        t        t        j
                  f   ffd       } |       }|S         }|S c c}w )a  Restore the saveable objects from a checkpoint with `file_prefix`.

    Args:
      file_prefix: A string or scalar string Tensor containing the prefix for
        files to read from.
      options: Optional `CheckpointOptions` object.

    Returns:
      When not run eagerly or when saving on a single device, returns a
      dictionary mapping from SaveableObject names to restore operations;
      otherwise, returns an empty dict.
    r(   c            
         i } j                   j                         D ci c]  \  }}|t        |       }}}i }j                  j                         D ]%  \  }}t	        j
                  |      5  t        |      }|j                         D ]  \  }}	|	j                         D ]  \  }
}j                  ||
f   }|
r&|| j                  |i       j                  |i       |
<   n|| j                  |i       |<   ||xx   dz  cc<   ||   dk(  sji }| |   j                         D ]  \  }}||t        j                  |      <     ||      }t        |t              s|j                  |         	 d d d        ( j                  j                         D ]  \  }\  }} |        |S c c}}w # 1 sw Y   hxY w)Nr   r   )r   r-   r   r   r   r   rK   r   rD   r   extract_local_namer   r   updater   )restore_fn_inputsrW   r   restore_fn_input_countr   r%   r$   rI   ckpt_keyslice_and_tensorr7   r   rs   rH   retr   r#   r&   r   s                   r;   rs   z,MultiDeviceSaver.restore.<locals>.restore_fnE  s   (,(@(@(F(F(H J$B"c$i- J  J k88>>@ (*+$ZZ '	*!65'"+
 -A,F,F,H *(h(&6&<&<&> *"
F33X5?5A Bj 17 #--j"=HHb"",. >D ",,Z-/119;$Z0A50'
3q8#%  ):*(E(K(K(M *$Hf#) #?#E#E$  !* !!12c4($$S)=**'	* '	*(*T !% 7 7 = = ?  
!_a; c J'	* '	*s   F.2BF4AF4F44F>	FTr   )r  r`   c                                i S rd   r]   )rs   s   r;   tf_function_restorez5MultiDeviceSaver.restore.<locals>.tf_function_restore  s    	rP   )r
   r,   r   r   r   r  r   r   anyr   is_custom_devicer   r   r  r   r   rj   )	r   r#   r&   has_custom_device_saverstsstr  r   rs   s	   ```     @r;   r   zMultiDeviceSaver.restore2  s   " ?+==?G3S]] 23 3j $..557 	L"g&&ryy':':'<=L	M"&   "(>(>(B(?%@73+=#>  A ()k  Lk3 Ms   *4D
)NNrd   )"__name__
__module____qualname____doc__r   r   	Trackabler   Shardr   classmethodr   r   r	   r   r   r   rj   r   rl   r   rm   rh   ri   r   r   ShardingCallbacktupler   r   r   r  r   r   r]   rP   r;   r|   r|      s    :>CG	]I!$..-2E2E"EF]I 7]I "A	]I~  :>CG	Q$..)Q 7Q "A	Q
 Q Q
'	** 
' <-{--BfmmLN/j&7&7 /J<M<M // <-{--BfmmLN/#**//9/@/@//B&77B c8M$7$7889:BN @Ds$$s =s }}	sp @Dc$$c =c sCMM!"	crP   r|   rd   )Kr  r   r   r   typingr   r   r   r   r   abslr   tensorflow.core.protobufr	   tensorflow.python.checkpointr
   %tensorflow.python.checkpoint.shardingr   r   tensorflow.python.eagerr   r   tensorflow.python.frameworkr   r   r   r   r   r   rh   r   r   tensorflow.python.opsr   r   r   r   tensorflow.python.saved_modelr   0tensorflow.python.saved_model.pywrap_saved_modelr   tensorflow.python.trackabler   r   !tensorflow.python.training.savingr   r   tensorflow.python.typesr    tensorflow.python.utilr!   r"   RegisteredSaverr   r  RegisteredSaversDictConcreteFunctionri   MappedCapturesCallabler  r   r  r<   r   rK   intrO   rV   ro   rv   r   r!  TensorKeyAndSliceSpec	RestoreFnr|   r]   rP   r;   <module>r7     s   ?    O O  . ; C ? + 0 3 < . + < 3 3 + , ( , 6 D , 7 = B ( ' 2   '#t~~*=">>@ !	HZ%6%678*:K:KKM  >B	,K"",K,K 

,K ;	,K
 	]],Kd >B*""* = =>* ;* 	*ZI&&II !!I 	I$A&&A,,A A*j'''(*(*  6* z  !#99:	*.-j'''(-(-  6- c:$$$%	-. -c3h gc:#4#4456EF	q qrP   