
    VhS.                     \	   U d dl Z d dlmZ d dlZd dlmZmZmZmZ d dl	m
Z
 ej                  j                   ej                  j                  Zej                  j                  Zej                  j                   Zej                  j"                  Z e       Zee   ed<   defdZd1dZi d	d
 dd dd dd dd dd dd dd dd dd dd dd  d!d" d#d$ d%d& d'd( Z ej2                         sGej4                  j7                  d)d*      Zej;                         D ]  \  ZZejA                  eed+        ejB                  jD                  ejF                  jD                  ejH                  jD                  ejJ                  jD                  ejL                  jD                  ejN                  jD                  ejP                  jD                  ejR                  jD                  ejT                  jD                  ejV                  jD                  ejX                  jD                  ejZ                  jD                  ej\                  jD                  ej^                  jD                  ej`                  jD                  ejb                  jD                  hZ2eejf                  jh                     ed,<   ejj                  jD                  ejl                  jD                  ejn                  jD                  ejp                  jD                  ejr                  jD                  ejr                  jD                  ejt                  jD                  ejv                  jD                  ejx                  jD                  ejz                  jD                  ej|                  jD                  ej~                  jD                  ejn                  jD                  ej                  jD                  ejp                  jD                  ejB                  jD                  ej                  jD                  hZBeejf                  jh                     ed-<   ej`                  jD                  ejb                  jD                  ejt                  jD                  hZCeejf                  jh                     ed.<   ej                  eBe2      ZE G d/ d0      ZFy)2    N)Any)_resolve_process_groupFakeWorkProcessGroupWork)tree_map_onlyused_idsreturnc                  p    	 t        j                  dd      } | t        vrt        j                  |        | S 6)N   i ʚ;)randomrandintr	   add)new_ids    Y/home/dcms/DCMS/lib/python3.12/site-packages/torch/distributed/_tools/fake_collectives.pygenerate_unique_idr      s3    
5)!LL M	     c                 j    t               }t               |_        |j                         }|r| d   |fS |S )Nr   )r   r   seq_idboxed)argsreturn_first_argworkfakework_script_objs       r   create_fakeworkr       s7    :D$&DK**,-=DG()VCVVr   
broadcast_c                      t        |       S Nr   r   s    r   <lambda>r!   0        5 r   
allreduce_c                      t        |       S r   r   r    s    r   r!   r!   1   r"   r   
allgather_c                      t        |       S r   r   r    s    r   r!   r!   2   r"   r   _allgather_base_c                      t        |       S r   r   r    s    r   r!   r!   3   s    od&; r   reduce_scatter_c                      t        |       S r   r   r    s    r   r!   r!   4   s    _T%: r   _reduce_scatter_base_c                      t        |       S r   r   r    s    r   r!   r!   5   s    ?4+@ r   reduce_c                      t        | d      S NF)r   r   r    s    r   r!   r!   6       _TEJ r   gather_c                      t        | d      S r/   r   r    s    r   r!   r!   7   r0   r   scatter_c                      t        |       S r   r   r    s    r   r!   r!   8   s    od3 r   	alltoall_c                      t        |       S r   r   r    s    r   r!   r!   9   s    t4 r   alltoall_base_c                      t        | d      S r/   r   r    s    r   r!   r!   :   s    OD5$Q r   barrierc                      t        | d      S r/   r   r    s    r   r!   r!   ;   r0   r   monitored_barrier_c                       y r    r    s    r   r!   r!   <   s    r   sendc                      t        | d      S r/   r   r    s    r   r!   r!   =   s    /$G r   recv_c                      t        | d      S r/   r   r    s    r   r!   r!   >   s    ?4%H r   recv_any_source_c                      t        | d      S r/   r   r    s    r   r!   r!   ?   s    odU&S r   c10dIMPLMetanon_functional_collectivesfunctional_collectivessync_opsc            
       h   e Zd Zej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  hZej                  j
                  ej                  j
                  ej                  j
                  ej                   j
                  ej"                  j
                  ej$                  j
                  ej&                  j
                  ej(                  j
                  hZej.                  j
                  ej                  j
                  ej0                  j
                  ej2                  j
                  ej4                  j
                  ej6                  j
                  ej8                  j
                  ej:                  j
                  ej8                  j
                  ej>                  j
                  h
Z ejB                  j
                  ejD                  j
                  ejB                  j
                  ejF                  j
                  ejF                  j
                  e$jJ                  j
                  hZ&ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej                   j
                  ej                  j
                  ej$                  j
                  ej&                  j
                  hZ'ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej"                  j
                  ej(                  j
                  ej                  j
                  hZ(ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej"                  j
                  ej                  j
                  ej                  j
                  ej4                  j
                  ej6                  j
                  h
Z)ej                  j
                  ej$                  j
                  hZ*ej8                  j
                  ej8                  j
                  hZ+ej                  j
                  ej.                  j
                  ej                  j
                  ej0                  j
                  ej2                  j
                  ejB                  j
                  ejB                  j
                  hZ,ejF                  j
                  ejF                  j
                  e$jJ                  j
                  hZ-ej>                  j
                  ejD                  j
                  hZ.e/de0de1fd       Z2e/de3fd       Z4e/de1fd       Z5e/de6fd       Z7y)CollectiveOpargr
   c                 r    ddt         j                  ddffd}t        t         j                  ||        S )z?Calculate total memory consumed by the tensors in the argument.r   tr
   Nc                 H    | j                         j                         z  y r   )untyped_storagenbytes)rN   total_memorys    r   	sum_bytesz+CollectiveOp.sum_tensors.<locals>.sum_bytes   s    A--/6688Lr   )torchTensorr   )rL   rS   rR   s     @r   sum_tensorszCollectiveOp.sum_tensors   s9     	9 	9$ 	9 	ellIs3r   c                 T   | t         j                  v rt        j                  |d         S | t         j                  v rt        j                  |d         S | t         j
                  v rt        |d         S | t         j                  v rt        |d         S t        d|  dt               )zKRetrieve the process group for collective operations, except `wait_tensor`.r         Func  not found in )
rK   PG_ARG_1r   unboxPG_ARG_2PG_ARG_3r   PG_ARG_4	TypeErrorcollective_ops)funcr   s     r   get_process_groupzCollectiveOp.get_process_group   s     <(((%%d1g..<(((%%d1g..<((()$q'22<((()$q'22%v^N3CDEEr   c                    | t         j                  v rt         j                  |d         S | t         j                  v rt         j                  |d         S | t         j                  v r|j                         j                         S | t         j                  v r!|d   j                         j                         S | t        j                  j                  k(  r!|d   j                         j                         S | t        j                  j                  k(  r8t        t         j                  |d         t         j                  |d               S | t        j                  j                  k(  rJt        |d   j                         j                         |d   j                         j                               S | t        j                  j                  k(  r!|d   j                         j                         S | t         j                   v rt         j                  |      S | t         j"                  v r@|d   j                         j                         |j                         j                         z   S t%        d|  dt&               )zdCompute the communication tensor size, except for `wait_tensor`, `barrier`, and `monitored_barrier`.r   r   zUnknown function: z in )rK   COMM_TENSOR_ARG_0rV   COMM_TENSOR_ARG_1COMM_TENSOR_ARG_RESrP   rQ   "COMM_TENSOR_SINGLE_UNTYPED_STORAGErD   r+   defaultr5   maxr7   _c10d_functionalall_gather_into_tensor_outCOMM_TENSOR_RES_SUMCOMM_TENSOR_ARG_0_AND_RESra   rb   )rc   resr   kwargss       r   get_comm_tensor_sizez!CollectiveOp.get_comm_tensor_size  s    <111++DG44<111++DG44<333&&(//11<BBB7**,33554--5557**,33554>>)))((a1<3K3KDQRG3T  4&&...Q'')002DG4K4K4M4T4T4V  #>>FFF8++-4466<333++C00<9997**,3358K8K8M8T8T8VVV,TF$~6FGHHr   c                     | t         j                  v rt        j                  |      S | t         j                  v rt        j                  |d         S t        d|  dt               )Nr   rZ   r[   )rK   WKr   r]   WK_ARG_1ra   rb   )rc   rq   s     r   get_workzCollectiveOp.get_work-  sW    <??">>#&&\***>>#a&))%v^N3CDEEr   N)8__name__
__module____qualname__rD   r   rk   r#   r-   r>   r@   rB   r9   r\   r%   r'   r)   r+   r1   r3   r5   r7   r^   rm   	broadcast
all_reduceall_reduce_all_reduce_coalescedall_reduce_coalesced_all_gather_into_tensorrn   _c10d_functional_autograd all_gather_into_tensor_coalescedr_   reduce_scatter_tensorreduce_scatter_tensor_coalescedall_to_all_single_dtensorshard_dim_alltoallr`   rv   ru   rg   rh   ri   rj   rp   ro   staticmethodr   intrV   r   rd   rs   r   rw   r=   r   r   rK   rK   ~   s    			

%%	H 	%%$$""**##H 	""**##++##++$$,,--55..66//7733;;!88@@99AAH 	..6688@@!77??**22!33;;##++H 	$$""**%%	H 			

%%##
B 			

%%--55..66" 	$$ 	//77!88@@ 	%%""**##++##++$$,,..66!77??*& 	**22!33;;##++! 	99AA88@@
 	 	 	 	 
F 
F 
F I I I> Ft F Fr   rK   )T)Gr   typingr   rT   torch._C._distributed_c10dr   r   r   r   torch.utils._pytreer   distributedbatch_isend_irecvopsrD   rm   r   r   setr	   r   __annotations__r   r   _META_FUNCTIONS_running_with_deploylibraryLibrarylib_implitemsop	meta_funcimplr   rk   r#   r-   r>   r@   rB   r%   r)   r+   r'   r1   r3   r5   r7   r9   r;   rG   _ops
OpOverloadr{   r|   r   r   r   wait_tensorr}   r~   r   rn   r   r   r   rH   rI   unionrb   rK   r=   r   r   <module>r      s8       .    # #yy~~99-- !II?? 99U#c( C W55 5 ;	
 : @ J J 3 4 Q J , G H  S!& "u!!#}}$$VV4H(..0 -Ib)V,- 	OOOOLLIIJJ!!OO  &&!!LLMMNNLL##!: C

 5 56 & &&''++33**22&&..//77  ((  (())11**22//7755==44<<44<<33;;''''#6 EJJ112 * 	LL##  (((#ejj##
$  13MNuF uFr   