
    BVh:                        d Z ddlZddlZddlmZmZ ddlmZ ddlm	Z	 ddl
mZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ  ej6                  d       G d de	j8                               Zeee   ee   f   Z  ej6                  d       G d de	j8                               Z!y)zECheckpoint policies that determine how tensors are split into shards.    N)MutableSequenceSequence)logging)sharding_util)context)device)dtypes)ops)tensor)tensor_shape)	array_ops)
string_ops)	variables)base)	tf_exportz$train.experimental.ShardByTaskPolicyc                   j    e Zd ZdZedefd       Zdeej                     deej                     fdZy)ShardByTaskPolicyzGPolicy that splits tensors into shards based on their device spec task.returnc                      y)Nz:Split tensors into shards based on their device spec task. selfs    g/home/dcms/DCMS/lib/python3.12/site-packages/tensorflow/python/checkpoint/sharding/sharding_policies.pydescriptionzShardByTaskPolicy.description)   s    G    shardable_tensorsc                     i }|D ];  }|j                   }|j                  }|j                  }||j                  |i       |<   = |gS )zCallback to split tensors into shards based on their device spec task.

    Args:
      shardable_tensors: A list of ShardableTensors.

    Returns:
      List of shard dicts containing tensors.
          [ {checkpoint key: {slice_spec: tensor} } ]
    )r   checkpoint_key
slice_spec
setdefault)r   r   tensors_by_taskshardable_tensorr   r   r   s          r   __call__zShardByTaskPolicy.__call__-   sb     O- =&&f'66n#..j 7= 
>2&z3= r   N)__name__
__module____qualname____doc__propertystrr   r   r   ShardableTensorShardr#   r   r   r   r   r   %   sN    OH3 H H!-"?"?@ ##$r   r   z%train.experimental.MaxShardSizePolicyc                       e Zd ZdZ G d d      ZdefdZedefd       Z	de
ej                     de
ej                     fd	Zy
)MaxShardSizePolicya%  Policy that splits tensors into shards with a max shard size.

  Shards may exceed the max shard size if they contain 1. a single scalar/string
  tensor that could not be sliced and exceeds the max shard size or 2. the
  checkpoint object graph, whose size cannot be calculated when saving.
  c                      e Zd ZU dZeed<   eej                     ed<   eej                     ed<   eed<   e	ed<   e
j                  ed<   ej                  ed<   ej                  ed	<   ej"                  j$                  ed
<   ej(                  ed<   ej(                  ed<   ee   ed<   eed<   ee   ed<   ej(                  ed<   deeef   fdZdedefdZdedeej4                     deej                     fdZy)*MaxShardSizePolicy.MaxShardSizePartitionerz4Partition tensors into shards with a max shard size.max_shard_size_large_scalars_tensors_by_shard_shard_size_remaining_checkpoint_key_dtype_device_root_tensor_slice_spec_full_shape_root_shape_root_offset_dtype_size_working_tensor_offset_working_tensor_shaper   c                      j                   j                  }||dk(  rdt        j                  fS  j                   j	                         dt
        dt        f fd} |d      }d}t        d|      D ]?  } ||      } j                  z  |z  }||k  s#| j                  k  s3|t        |      }}A |t        j                  t         j                   |         |z        fS )zGets tensor partition with size closest to shard_size_remaining.

      Returns:
        A tuple containing the axis and size of the next partition.
      r   axisr   c                     j                   j                  |    j                  }|z  }|j                  z  }j                  |z  }|dk(  rt
        j                  S t        j                  ||z        S )Nr   )r>   dimsvaluer<   r3   mathinfceil)r@   axis_lenslice_elemsbytes_per_sliceslices_per_shard	num_elemsr   s        r   num_partitionszVMaxShardSizePolicy.MaxShardSizePartitioner._get_next_partition.<locals>.num_partitionsr   sp    --2248>>8+%(8(8855Hq /yy$4455r      )r>   rankrD   rE   num_elementsintfloatranger<   r3   rF   )	r   rN   rL   	min_partsmin_axisr@   parts_along_axis	part_sizerK   s	   `       @r   _get_next_partitionz>MaxShardSizePolicy.MaxShardSizePartitioner._get_next_partitionf   s     '',,d	$(({,,99;i6s 6u 6 !#ih4. <$)$/ 0 003CC	y(333 $c*:&;I(< iiD66x@AIMNP Pr   	part_axisrV   c           	      ~   t        t        t        j                  | j                  | j
                              }t        t        t        j                  | j                  |            }|||<   t        t        j                  |            | j                  z  }t        j                  | j                        5  t        j                  | j                   ||      }ddd       t"        j$                  j'                  | j(                  | j*                  | j                  |      j,                  j/                         }|| j0                  kD  rWt3        j4                  d| j(                  ||| j0                         | j6                  j9                  | j(                  |ii       n|| j:                  kD  rt=        d| d| j:                   d      | j>                  d   jA                  | j(                  i       |<   | xj:                  |z  c_        | j:                  d	k(  r,| j>                  j9                  i        | j0                  | _        | j                  |xx   |z  cc<   ||xx   |z  cc<   tC        jD                  t        t        t        j                  | j                  |                  | _#        y# 1 sw Y   xY w)
a  Adds the tensor partition to the shard, if possible.

      Args:
        part_axis: The axis of the partition.
        part_size: The size of the partition.

      Raises:
        RuntimeError: When the slice size is larger than the remaining shard
        size.
      beginsizeN	full_name
full_shape
var_offset	var_shapezTensor %s's minimum slice %s has size %s bytes and cannot be partitioned into a shard of max shard size %s bytes. It will be added as an individual shard that exceeds the max shard size.zSlice size (z1 bytes) is larger than the remaining shard size (zM bytes). This should have been caught in MaxShardSizePolicy._add_partition().r   )$listmapoperatorsubr=   r;   r:   rP   rD   prodr<   r
   r   r6   r   slicer7   r   VariableSaveSliceInfor4   r9   specstripr0   r   warningr1   appendr3   RuntimeErrorr2   r    r   TensorShaper>   )r   rX   rV   relative_offsetslice_shapeslice_size_in_bytestensor_slicer   s           r   _add_partitionz9MaxShardSizePolicy.MaxShardSizePartitioner._add_partition   su    
hllD779J9J
KMoX\\4+;+;_MNk(k)		+ 67$:J:JJ::dll# H _;HH %%33((%%00	 4 ! "&eeg	 
 
t22	2 ; ,,j:M++	- 	""!!J#=>	@!;!;;./ 0%%)%?%?$@ ANNO 	O ?K 
			#
*T))2
.z
;""&99"%%*

 
 
'
'
+'+':':$
$ !!),	9,i I- #/#;#;D
hllD,,o
>=@ $Ad CH Hs   6#J22J<r   c                    || _         i g| _        g | _        d}| j                   | _        |D ]J  }|j                  | _        |j                  | _        |j                  | _	        |j                  | _        |j                  | _        | j                  rt        j                  j                   j#                  | j                        }t%        j&                  |j(                        | _        t%        j&                  |j,                        | _        |j0                  | _        n7|j4                  x| _        | _        dg| j.                  j6                  z  | _        t9        j:                  | j                        j<                  | _        | j.                  jA                         | j>                  z  }| j
                  tB        jD                  k(  rdx}| _        n| j                  t8        jF                  k(  rdx}| _        n| j                  t8        jH                  k(  rMtK        jL                         s9tO        jP                         %dx}| _        |stS        jT                  d       d}n|| j                  t8        jH                  k(  r^tO        j                  | j                        5  tK        jL                         s2tO        jP                         jW                  | j                        | _        | j.                  j6                  | j.                  j6                  dk(  r#tY        jZ                  | j                  d      g}n.| j                  D cg c]  }tY        jZ                  |d       }}tK        jL                         r|D 	cg c]  }	|	j]                          }}	n#tO        jP                         jW                  |      }ddd       t_              }ta        |      | _        | j.                  j6                  | j.                  j6                  dk(  r|| j                   kD  rutS        jT                  d| j
                  | j                  || j                          | j                  jc                  | j
                  | j                  | j                  ii       n|| j                  kD  r,| j                  jc                  i        | j                   | _        | j                  | j                  d	   je                  | j
                  i       | j                  <   | xj                  |z  c_        | j2                  dd | _3        | j.                  | _4        |}
|
| j                  kD  r| jk                         \  }}|dk(  r-| j                  jc                  i        | j                   | _        nG| jm                  ||       to        tq        jr                  | jh                              | j>                  z  }
|
| j                  kD  r| jh                  jA                         dkD  rL| jf                  r| jh                  rtO        j                  | j                        5  tu        jv                  | j                  ty        t{        t|        j~                  | jf                  | j2                              | jh                  j                         
      }ddd       n| j                  }t        j                  j!                  | j
                  | j*                  | jf                  | jh                        j                  j                         }| j                  d	   je                  | j
                  i       |<   | xj                  |
z  c_        M g }| j                  d   r|j                  | j                         |j                  | j                         |S c c}w c c}	w # 1 sw Y   xY w# 1 sw Y   xY w)aB  Callback to split tensors into shards with a max shard size.

      Args:
        max_shard_size: The maximum size of a shard file in bytes.
        shardable_tensors: A list of ShardableTensors.

      Returns:
        List of shard dicts containing tensors.
            [ {checkpoint key: {slice_spec: tensor} } ]
      Fr   Nz~The checkpoint sharding policy is being executed in a tf.function. The size of the string/variant constant cannot be obtained.TBYTE)unitzTensor %s is a %s scalar of size %s bytes and cannot be partitioned into a shard of max shard size %s bytes. It will be added as an individual shard that exceeds the max shard size.rb   rZ   r]   )Dr0   r2   r1   r3   r   r4   dtyper5   r   r6   r   r7   r   r8   r   ri   rj   	from_specr   rp   r_   r9   ra   r:   r`   r;   shaperN   r	   as_dtyper\   r<   rO   r   OBJECT_GRAPH_PROTO_KEYvariantstringr   executing_eagerlyr
   get_default_sessionr   rm   runr   string_lengthnumpysummaxrn   r    r=   r>   rW   ru   rP   rD   rg   r   rh   rc   rd   re   rf   as_listrk   rl   extend)r   r0   r   string_size_warning_printedr"   save_slice_info
total_sizesizeselemr\   working_tensor_sizerX   rV   working_tensorremaining_tensor_slice_specshardss                   r   
get_shardsz5MaxShardSizePolicy.MaxShardSizePartitioner.get_shards   s    +d "tdd$)!#'#6#6d / z:
/>>&,,'..,33+66 %..<<FF /)55((*$
)55o6O6OP$
-88$
0@0F0F
F$
T- cD$4$4$9$99$
!??4;;7<<%%224t7G7GG
 4#>#>> +,
+*t'[[FNN**+
+*t'kkV]]*//1))+3*+
+*t',OO ; < +/'[[FMM)zz$,,' ;,,."%"9"9";"?"?###%d $$,0@0@0E0E0J!//0A0A5;= >e $(#4#46 "//6B 6e 6 ((*056tzz|6e6--/33E:e;" 5z* Z$
  (D,<,<,A,AQ,F$---OO --t{{J##% &&%%(8(8$:K:K'LMO D666$$++B/+/+>+>d( $(#4#4 ##B'j--r2  &&*4&
 '+&7&7&:#%)%5%5"(!D$>$>>#'#;#;#=
 9i!^ ""))"-)-)<)<D&	95 DIId889:T=M=MM   "D$>$>> %%224q8((T-G-GDLL) =(##Sll1143D3DF G 1199; =n= = "..N(1(:(:(H(H,,))4422	 )I )4 59D	 & -; !!"%Jt++R0') 	""&99"uz:x f				"d,,-mmD''(mW6 7; ;x= =s8   !B'__&_ _&_A1_
__	_$	N)r$   r%   r&   r'   rP   __annotations__r   r   r+   r)   r	   DType
device_lib
DeviceSpec
tensor_libTensorr   ri   rj   r   rp   r   rQ   tuplerW   ru   r*   r   r   r   r   MaxShardSizePartitionerr/   S   s   >#M$7$788&}':':;;LL"""#####111))))))3-+E22'333 PU3:%6  PD4A 4A 4AlVV $M$A$ABV 
-%%	&	Vr   r   r0   c                     || _         y N)r0   )r   r0   s     r   __init__zMaxShardSizePolicy.__init__V  s
    (Dr   r   c                      y)Nz0Split tensors into shards with a max shard size.r   r   s    r   r   zMaxShardSizePolicy.descriptionY  s    =r   r   c                 V    | j                         j                  | j                  |      S r   )r   r   r0   )r   r   s     r   r#   zMaxShardSizePolicy.__call__]  s,     '')44.0 0r   N)r$   r%   r&   r'   r   rP   r   r(   r)   r   r   r   r*   r+   r#   r   r   r   r-   r-   J   sf    A AF)S ) >3 > >0'(E(EF0##$0r   r-   )"r'   rD   re   typingr   r   abslr   %tensorflow.python.checkpoint.shardingr   tensorflow.python.eagerr   tensorflow.python.frameworkr   r   r	   r
   r   r   r   tensorflow.python.opsr   r   r   tensorflow.python.trackabler   tensorflow.python.utilr   ShardingCallbackr   r   rP   _OffsetAndShaper-   r   r   r   <module>r      s    L   ,  ? + < . + < 4 + , + , , ;<66  =B x}45 <=V077 V0 >V0r   