
    Vh	                     n    d dl Z d dlmZ d dlmZ d dlmZmZ erd dlm	Z	 dgZ
	 ddee   ded	ee   fd
Zy)    N)defaultdict)TYPE_CHECKING)SavePlan	WriteItem)MetadataIndexdedup_save_plans	all_planssave_to_lowest_rankreturnc                    t        t              }i }t        |       D ]C  \  }}|j                  D ]/  }||j                     j                  |       |||j                  <   1 E t        t        |             D cg c]  }t                }}dgt        |       z  |j                         D ]s  \  }	}
|rt        |
      }nt        |
fd      }||	   }|xx   |j                         xs dz  cc<   |
j                  |       |
D ]  }||   j                  |	        u t        |      D ]L  \  }}| |   j                  D cg c]  }|j                  |vr| }}t        j                  | |   |      | |<   N | S c c}w c c}w )z
    Removes duplicate entries from appearing on multiple SavePlans. For each duplicate across
    a set of SavePlans, only the smallest SavePlan in terms of planned storage keeps the entry.
    r   c                     |    S )N )plan_idxplan_to_sizes    ^/home/dcms/DCMS/lib/python3.12/site-packages/torch/distributed/checkpoint/_dedup_save_plans.py<lambda>z"dedup_save_plans.<locals>.<lambda>(   s    <3I     )key   )items)r   set	enumerater   indexaddrangelenmintensor_storage_sizeremovedataclassesreplace)r	   r
   write_item_to_plan_indiceswrite_item_idx_to_write_itemr   plan
write_item_	to_removewrite_item_idxplan_indicesselect_plan_idx
remove_set	new_itemsr   s                 @r   r   r      s    ALC@PCE #I. H$** 	HJ&z'7'78<<XF=G()9)9:	HH ,1Y+@AaCEAIA3Y'L(B(H(H(J 4$!,/O!"IO 2.A
 	_%)G)G)I)NQN%O,$ 	4Hh##N3	44" !*) 4 X* (177
z1 
	 

 *11)H2EYW	(X 7 B(
s   :E>F)F)r    collectionsr   typingr   $torch.distributed.checkpoint.plannerr   r   %torch.distributed.checkpoint.metadatar   __all__listboolr   r   r   r   <module>r4      sM     #   D C

 !&-H~-- 
(^-r   