
    Vh                         d dl Z d dlZd dlmZ d dlmZ erd dlmZ dgZdej                  fdZ
 e
       Zdee   dee   fd	Zy)
    N)TYPE_CHECKING)SavePlan)MetadataIndexdedup_tensorsreturnc                  @   t        j                  t              } t         j                  }| j	                  |       t        j
                         }t        j                  d      }|j                  |       |j	                  |       | j                  |       d| _	        | S )Nza%(asctime)s %(filename)s:%(lineno)s %(levelname)s p:%(processName)s t:%(threadName)s: %(message)sF)
logging	getLogger__name__INFOsetLevelStreamHandler	FormattersetFormatter
addHandler	propagate)loggerlevelconsole	formatters       [/home/dcms/DCMS/lib/python3.12/site-packages/torch/distributed/checkpoint/_dedup_tensors.pyinit_loggerr      s    x(FLLE
OOE##%G!!kI #U
gFM    	all_plansc                    t        |       } i }t        |       D ]A  \  }}|j                  D ]-  }|j                  |j                  g       j                  |       / C |j                         D ci c]  \  }}t        |      dkD  s|| }}}i }|j                         D ]0  \  }	}
|
dd  D ]#  }|j                  |g       j                  |	       % 2 t        |      dkD  rt        j                  d|       |j                         D ]W  \  }}t        |      }| |   j                  D cg c]  }|j                  |vr| }}t        j                  | |   |      | |<   Y | S c c}}w c c}w )N   r   zDuplicate keys to remove: %s)items)list	enumerater   
setdefaultindexappendlenr   infosetdataclassesreplace)r   key_to_planplan_idxplan
write_itemkvreplicated_itemsplan_to_keyskeyplanskeyskey_set	new_itemss                 r   r   r   "   s   YI24K#I. J$** 	JJ"":#3#3R8??I	JJ *5):):)<KAA
1KK 46L&,,. >
Uab	 	>H##Hb188=	>> <12LA&,,. X$d) (177
w. 
	 

 *11)H2EYW	(X + L
s   0E$E$*E*)r&   r	   typingr   $torch.distributed.checkpoint.plannerr   %torch.distributed.checkpoint.metadatar   __all__Loggerr   r   r   r    r   r   <module>r;      sU        9 C
W^^  
T(^ X r   