
    Vh)                        d dl Z d dlZd dlZd dlmZmZ d dlmZmZm	Z	m
Z
mZ d dlmZ d dlZd dlmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZ d d	lmZ d d
lmZm Z m!Z!m"Z"  e
d      Z# ed      Z$	 d4deejJ                  jL                  e'ejJ                  jL                     f   de	e   deeegef   geegef   f   fdZ(	 d4deee$e#f   gee$e#f   f   fdZ)dee'e*   e*f   dee'e*   ejV                  jX                  jZ                  f   fdZ.de/de/de/fdZ0dee/ee/   f   de/dee/   fdZ1dee/   de/fdZ2dee/   dede3fdZ4dee/   dede3fdZ5dede/de3fdZ6dede3fd Z7d!ejp                  d"ejp                  de'e/   fd#Z9d$e:e d%f   dejp                  d&e'e/   de:e d%f   fd'Z;d(ed)ede'e<   fd*Z=d+d,d-d.ed/ed0e'e   d1e/d2e3defd3Z>y)5    N)IterableSequence)CallablecastOptionalTypeVarUnion)	ParamSpec)DTensor)redistribute_cost)DTensorSpec)OpSchema
OpStrategyOutputShardingPlacementListPlacementStrategyRuntimeSchemaInfo)
DeviceMesh)Partial	Placement	ReplicateShard_T_Popschema_inforeturnc                 h     dt         t        gt        f   dt         t        gt        f   f fd}|S )Nimplr   c                     t        t              rng}|D ]-  }t        j                  j                  j                  ||        / | S N)
isinstancelistr   _op_dispatchersharding_propagatorregister_sharding_prop_rule)r   	overloadsoverloadr   r   s      S/home/dcms/DCMS/lib/python3.12/site-packages/torch/distributed/tensor/_ops/utils.pywrapperz#register_prop_rule.<locals>.wrapper/   sN     %R.BRD	! 	H""66RR$	     )r   r   r   )r   r   r*   s   `` r)   register_prop_ruler,   &   s7    z>12	8*n,	- Nr+   c                       dg fd}|S )Nmemory_formatc                 T   t        t              r}ng}|D ]  }d }S|j                  j                  D cg c]  }|j                  v r|j                   }}t        |      rt        |      }n}t        j                  j                  j                  || |        | S c c}w )N)static_kwargkey)r"   r#   _schema	argumentsnameanyr   r   r$   r%   register_op_strategy)	r   r'   r(   curr_schema_infoaspecialized_args2arg_names_that_require_specializing_cache_strategyr   r   s	         r)   r*   z%register_op_strategy.<locals>.wrapperL   s    b$II! 	H#" &--77$vv!SS FF$  $
 '('8(8($ $/ ""66KK$ 0	" $s   !B% )r   r   r*   r9   s   `` @r)   r5   r5   <   s     	:62 Nr+   xc                     t        |       t        u s.t        | t        j                  j
                  j                        r| S | gS r!   )typer#   r"   torchfximmutable_collectionsimmutable_list)r;   s    r)   as_listrB   h   s4     Aw$*Q(F(F(U(UVs
r+   dimndimc                     | dk\  r| S | |z   S )Nr   r:   )rC   rD   s     r)   normalize_dimrF   u   s    (3*d
*r+   dimsc           	         t        | t              rt        | |      f} | S t        | t              r| D cg c]  }t        ||       } }| S t        | t              r#t	        | D cg c]  }t        ||       c}      } | S c c}w c c}w )zENormalize a dim or a sequence of dims, so that they are all positive.)r"   intrF   r#   tuple)rG   rD   rC   s      r)   normalize_dimsrK   y   s    $dD)+
 K	 
D$	489Sc4(99 K 
D%	 $?3mC.?@K :?s   B $Bxsc                 L    t        j                  t        j                  | d      S )N   )	functoolsreduceoperatormul)rL   s    r)   prodrS      s    HLL"a00r+   shapespecc                 J   dgt        |       z  }t        |j                        D ]V  \  }}|j                         st	        t
        |      j                  }||xx   |j                  j                  |      z  cc<   X t        |       D ]  \  }}||   dkD  s|||   k  s y y)z6Check if the shape is shardable according to the spec.rN   FT	len	enumerate
placementsis_shardr   r   rC   meshsizerT   rU   
shards_mapi	placement	shard_dimdim_sizes          r)   is_tensor_shardablerd      s     s5z!J!$//2 79UI.22Iy!TYY^^A%66!7
 !' 8 a=1JqM!9	 r+   c                 P   dgt        |       z  }t        |j                        D ]V  \  }}|j                         st	        t
        |      j                  }||xx   |j                  j                  |      z  cc<   X t        |       D ]  \  }}||   dkD  s|||   z  dk7  s y y)z=Check if the shape is evenly shardable according to the spec.rN   r   FTrW   r^   s          r)   is_tensor_evenly_shardablerf      s     s5z!J!$//2 79UI.22Iy!TYY^^A%66!7
 !' 8a=1(Z]":a"? r+   c                 @    t        fd| j                  D              S )z%Return True if tensor dim is sharded.c              3   @   K   | ]  }|j                          y wr!   )r[   ).0prC   s     r)   	<genexpr>z(is_tensor_dim_sharded.<locals>.<genexpr>   s     81qzz#8s   r4   rZ   )rU   rC   s    `r)   is_tensor_dim_shardedrm      s    8888r+   c                 :    t        d | j                  D              S )z-Return True if tensor is partial on the mesh.c              3   <   K   | ]  }|j                           y wr!   )
is_partial)ri   rj   s     r)   rk   z$is_tensor_partial.<locals>.<genexpr>   s     7!q||~7s   rl   )rU   s    r)   is_tensor_partialrq      s    7t777r+   common_shapeinput_shapec                     t        |       }t        |      }dg|z  }t        dd|z
  d      D ]  }||   | |   k(  s||z   |||z   <    |S )N)rX   range)rr   rs   common_ndim
input_ndimbroadcast_dims_mapidxs         r)   infer_broadcast_dims_mapr{      sr    
 l#K[!J+Rj"- Es|C004>4D{S01E r+   rZ   .ry   c                 X   g }| D ]  }t        |t        t        f      r|j                  |       +t        |t              sJ t        |j                  t        |            }||   }|dk7  r|j                  t	        |             |j                  t                       t        |      S )z=Map each placement based on the output shape after broadcast.ru   )	r"   r   r   appendr   rF   rC   rX   rJ   )rZ   rT   ry   new_placementsra   rb   new_shard_dims          r)   map_placements_after_broadcastr      s     ')N 3	i)W!56!!),i///%immSZ@I.y9M" %%eM&:; %%ik2'3*   r+   src_strategydst_specc                 j    | j                   D cg c]  }t        |j                  |       }}|S c c}w r!   )
strategiesr   output_spec)r   r   stratredistribute_costss       r)   generate_redistribute_costsr      sB    
 ",,' 	%++X6' '
 's   0rN   F)input_index
inplace_opr\   	op_schemasingle_mesh_dim_strategiesr   r   c          	      0   |g| j                   z  }t        j                  | }g }|D ]W  }g }	t        | D ]4  }
|
d   |	j	                  t        | |
             $|	j	                  d        6 |	|d  D cg c]  }t        |t
              s| }}|j                  }t        |      t        |      k(  sJ |d   j                  d   j                  }|r|j                  |d   j                  k7  rt        d t        ||      D              }|st        ||      D cg c]  \  }}t        ||       }}}|dkD  rt        |	d |       }n|	d   |	d   }nt        d      t!        |||      }|j	                  |       Z t#        |      S c c}w c c}}w )Nr   c              3   N   K   | ]  \  }}t        |j                  |        y wr!   )rd   rT   )ri   inpss      r)   rk   z2expand_to_full_mesh_op_strategy.<locals>.<genexpr>  s'      
Q  		1-
s   #%rN   zoutput spec is None)output_specsinput_specsr   )rD   	itertoolsproductzipr}   r   r"   args_strategyrX   r   r   rZ   allr   rJ   RuntimeErrorr   r   )r\   r   r   r   r   all_mesh_dim_strategiesstrategy_combsall_strategiesstrategy_comb	spec_listspecsr   r   input_args_strategy	self_specinputs_shardableinput_strategy
input_specr   r   strategys                        r)   expand_to_full_mesh_op_strategyr      s     ::TYYF&&(?@NN' -,13	-( 	'EQx#  T5!9:  &		' !.*
*Q2LA*
 *
 (55;3':#;;;;'*55a8DD	)..+a.2K2KK   
1;?
 
  366I;2W!.NJ ,NJG! ! Q$Y|%<=Q<+#,Q<L&'<==()'"3H
 !!(+[-,^ n%%O*
*!s   3F	FFr!   )?rO   r   rQ   collections.abcr   r   typingr   r   r   r   r	   typing_extensionsr
   r>   torch.distributed.tensor._apir   *torch.distributed.tensor._collective_utilsr   &torch.distributed.tensor._dtensor_specr   #torch.distributed.tensor._op_schemar   r   r   r   r   r   $torch.distributed.tensor.device_meshr   (torch.distributed.tensor.placement_typesr   r   r   r   r   r   _ops
OpOverloadr#   r,   r5   objectr?   r@   rA   rB   rI   rF   rK   rS   boolrd   rf   rm   rq   Sizer{   rJ   r   floatr   r   r:   r+   r)   <module>r      s      . ; ; '  1 H >  <  T]t_ 04ejj##T%***?*?%@@A+, xj.()*HhZ5O,PP. )xB (2r6"223)X
T&\6!"
 4<77FFFG
+s +# +# +sHSM12 # (3- 1Xc] 1s 1x} K D $hsm ; 4  9 9# 9$ 9
8K 8D 8
**+0::	#Y!in%!::! S	! 9c>	!>(3	%[  =&
=&=& !%] 3=&
 =& =& =&r+   