
    Vhf                        d dl Z d dlZd dlmZ d dlmZmZmZ d dlZd dl	m
Z
 d dlmZ d dlmZ d dlmZ dd	gZ ej$                  e      xZZ e
d
       G d d             Zdej.                  j0                  dedefdZ e
d
      	 	 	 ddedej.                  j0                  deegef   deeeef      dee   dee   fd       Zy)    N)OrderedDict)AnyCallableOptional)compatibility)lazy_format_graph_code)GraphModule)Node	Partitionsplit_moduleT)is_backward_compatiblec                   $    e Zd ZdefdZdefdZy)r   namec                     || _         d| | _        g | _        i | _        i | _        i | _        i | _        t        j                  j                  j                         | _	        i | _        i | _        y )Nsubmod_)r   submod_name
node_namesinputsoutputsdependencies
dependentstorchfxgraphGraphenvironmenttargets)selfr   s     L/home/dcms/DCMS/lib/python3.12/site-packages/torch/fx/passes/split_module.py__init__zPartition.__init__   sc    	$TF+%'')(*-/+-+088>>+?+?+A
-/')    returnc                     d| j                    d| j                   d| j                   d| j                   d| j                   d| j
                   S )Nzname: z
,
 nodes: z,
 inputs: z,
 outputs: z,
 partitions depended on: z,
 partition dependents: )r   r   r   r   r   r   )r   s    r   __repr__zPartition.__repr__    sb    TYYK  ' (} % '((,(9(9': ;&&*oo%68	
r!   N)__name__
__module____qualname__strr    r$    r!   r   r   r      s    
*S 
*
# 
r!   modqualnamer"   c                     | }|j                  d      D ])  }t        ||      st        d| d      t        ||      }+ |S )N.zNode target z not found!)splithasattrAttributeErrorgetattr)r*   r+   attr_valatoms       r   _get_attr_from_qualnamer4   +   sP    Hs# +x& <z!EFF8T*+ Or!   mroot_msplit_callbackqualname_mapkeep_original_orderkeep_original_node_namec                    1>?@ABCDE t         j                  dt        d d             dt        dt        t
        t        f   dt        t
        t        j                  j                  j                  f   f? fd}d	d
l
}i Ci Ai Edt        dt        t           fCEfdDCfd}t        j                  j                  t        j                  j                  t        j                  j                   g}	t#               }
t#               }i }d
}t%               } j&                  j(                  D ]  11j*                  j-                  d      x}st/        |t        j0                  t        j2                  f      rIt/        |j4                  j6                  x}|j8                        r|Evr1E|j4                  j6                  <   1j:                  dv r |1       1j:                  dk(  r1j<                  |	v r1j<                  t        j                  j                   k(  rRt?        1j@                        dk(  sJ t/        1j@                  d	   tB              sJ 1}t%         1      h      |
|<   n	1j<                  t        j                  j                  k(  rJtE        d 1j@                  D              sJ |jG                  1       t%         1      h      |1<   d
|1<   n1j<                  t        j                  j                  k(  rqt?        1j@                        dk(  sJ |1j@                  d	      jG                   1             |jI                  1j@                  d	          1|1j@                  d	   <   ||
|   jG                   1             |D ]  }||   jG                   1               tE        d |jK                         D              sJ d       |jM                         D ci c]  \  }}|tO        |       }}}|
jM                         D ci c]  \  }}|tO        |       }
}}tP        jS                  tT        jV                        r,tP        j                  d|       tP        j                  d|
       tC        |      xs tC        |
      }d} j&                  j(                  D ]  11A1jX                  <   1j:                  dv r"1j:                  dk(  r;t        j                  j&                  j[                  1j@                  d	   Dfd       l|r 1      }||k  sJ d| d|        |}1j<                  |	vst        j                  j&                  j[                  1j@                  1Dfd       t        j                  j&                  j[                  1j\                  1Dfd        t_        Cja                               }g }CjM                         D ],  \  }Bt?        Bjb                        r|je                  |       . g }|rw|jg                         }|je                  |       C|   jh                  D ]A  }C|   jb                  jg                  |       C|   jb                  r1|je                  |       C |rwt?        |      t?        C      k7  rtk        d       ||
fD ]  }|jM                         D ]  \  1}t?        |      d	kD  sJ 1Ct        |d	            jl                  1<   |dd
 D ]  }Ct        |         BBj&                  jo                  1j:                  1j<                  tq        d! 1j@                  D              i 1jr                  "      } 1j*                  ju                         | _        | Bjl                  1<      |D ]`  }C|   Bi }!Bjv                  D ]?  }"A|"   }#|#j:                  d#k(  rt/        |#j<                  t
              sJ ty         |#j<                        }$t/        |$t        jz                  j|                        r?Bj&                  j                  |#j<                        }%|$Bj                  |#j<                  <   n_Bj&                  j                  |"A|"   jr                  $      }%d
|!|"<   n/Bj&                  j                  |"A|"   jr                  $      }%d
|!|"<   A|"   j*                  ju                         |%_        |%Bjl                  A|"   <   B |!B_;        c  j&                  j(                  D ]  1t        1d%      sC1j                     BBjl                  @t        j                  j&                  j[                  1j@                  @fd&      }&t        j                  j&                  j[                  1j\                  @fd'      }'1j:                  d(vr1j<                  }(ncty         1j<                        })1j<                  j                  d)d*      }(|)Bj                  |(<   | Bj                   d)|( }*1j<                  ||*<   t/        |&tp              sJ t/        |'t              sJ r1jX                  nd
}+Bj&                  jo                  1j:                  |(|&|'1jr                  |++      } 1j*                  ju                         | _        | Bjl                  1<    |fD ]  }t        |      D ]  1|1   }t?        |      d	kD  sJ |d
d D ]  }Ct        |         B|1   },|,J d,       Bj&                  jo                  |,j:                  |,j<                  Bjl                  1   fi |,jr                  "      } |,j*                  ju                         | _           i }-i >t        j                  j&                  j                         ?i }.|s) j&                  j(                  D ]  1 |1>|.      \  >}. n* j&                  j(                  D ]  11|-1jX                  <    |s|n|}/t%               }0 j&                  j(                  D 1cg c]  }1|1j:                  d-k(  s|1 }2}1|/D ]'  }C|   Btq        ABfd.Bj                  D              }3t?        |3      }4|4dk(  rBj&                  j                  |3d	          n<|4dkD  rBj&                  j                  |3       nBj&                  j                  d/       |rtBjv                  D 5cg c]  }5|5|2vr|-|5    }6}5|2D ]%  11|0v r |1>|.      \  >}7|0jG                  1       ' |6D ]%  11|0v r |1>|.      \  >}.|0jG                  1       ' t        j                  j                  j                  Bj                  Bj&                        |.Bj                  <   ?j                  Bj                  tq        >fd0Bjv                  D                    }8t?        Bj                        }9|9dkD  rZt        j                  j                  j                  |8      }:t        Bj                        D ]  \  };}<|:|;   j4                  >|<<     |9dk(  s|8>t        t        Bj                              <   * |r*>s( j&                  j(                  D ]  1 |1>|.      \  >}.  j&                  j(                  D ][  11j:                  dk(  s?j                  t        j                  j&                  j[                  1j@                  d	   >fd1             ] t        j                  j                  j                  |.?      }=t         j                  dt        d2|=d             |=S c c}}w c c}}w c c}1w c c}5w )3a  
    Creates subgraphs out of main graph

    Args:
        m (GraphModule): Graph module to split
        root_m (torch.nn.Module): root nn module. Not currently used. Included
            because the root nn module is usually transformed via
            torch.fx._symbolic_trace.symbolic_trace (see example below)
        split_callback (Callable[[Node], int]): Callable function
            that maps a given Node instance to a numeric partition identifier.
            split_module will use this function as the policy for which operations
            appear in which partitions in the output Module.
        qualname_map: Optional[Dict[str, str]]: optional output parameter that returns a
            mapping from new target names in the module after split to old target
            names in the original module.
        keep_original_order: Optional[bool]: keep the original order of the GraphModule
            or use the Topological order of the new constructed GraphModule


    Returns:
        GraphModule: the module after split.

    Example:

        This is a sample setup:

            import torch
            from torch.fx.symbolic_trace import symbolic_trace
            from torch.fx.graph_module import GraphModule
            from torch.fx.node import Node
            from torch.fx.passes.split_module import split_module

            class MyModule(torch.nn.Module):
                def __init__(self) -> None:
                    super().__init__()
                    self.param = torch.nn.Parameter(torch.rand(3, 4))
                    self.linear = torch.nn.Linear(4, 5)

                def forward(self, x, y):
                    z = self.linear(x + self.param).clamp(min=0.0, max=1.0)
                    w = self.linear(y).clamp(min=0.0, max=1.0)
                    return z + w

            # symbolically trace model
            my_module = MyModule()
            my_module_traced = symbolic_trace(my_module)

            # random mod partitioning
            partition_counter = 0
            NPARTITIONS = 3

            def mod_partition(node: Node):
                global partition_counter
                partition = partition_counter % NPARTITIONS
                partition_counter = (partition_counter + 1) % NPARTITIONS
                return partition

            # split module in module with submodules
            module_with_submodules = split_module(
                my_module_traced, my_module, mod_partition
            )

        Output looks like this. Original graph is broken into partitions

            > print(module_with_submodules)
            GraphModule(
                (submod_0): GraphModule(
                    (linear): Linear(in_features=4, out_features=5, bias=True)
                )
                (submod_1): GraphModule(
                    (linear): Linear(in_features=4, out_features=5, bias=True)
                )
                (submod_2): GraphModule()
            )

            def forward(self, x, y):
                param = self.param
                submod_0 = self.submod_0(x, param, y);  x = param = y = None
                getitem = submod_0[0]
                getitem_1 = submod_0[1];  submod_0 = None
                submod_1 = self.submod_1(getitem, getitem_1);  getitem = getitem_1 = None
                getitem_2 = submod_1[0]
                getitem_3 = submod_1[1];  submod_1 = None
                submod_2 = self.submod_2(getitem_2, getitem_3);  getitem_2 = getitem_3 = None
                return submod_2

        Output of split module is the same as output of input traced module.
        This is an example within a test setting:

            > orig_out = my_module_traced(x, y)
            > submodules_out = module_with_submodules(x, y)
            > self.assertEqual(orig_out, submodules_out)
            True
    z%szpre split_moduleT)colorednodebase_mod_envbase_mod_attrsc                 t   | j                   dk(  r t        | j                        dkD  r| j                  d   nt        j                  j
                  }rX|t        j                  j
                  u rdn|f}j                  d| j                  || j                        || j                  <   n5j                  | j                  | j                  |      || j                  <   | j                  j                         || j                     _        ||fS | j                   dk(  rj                  | j                        || j                  <   | j                  j                         || j                     _        t        | j                  t              sJ t!        | j                        }||| j                  <   ||fS )Nplaceholderr   r)   )args	type_expr)rC   default_valueget_attr)oplenrB   inspect	Signatureemptycreate_noder   typerA   targetmetacopyrE   
isinstancer(   r4   )	r=   r>   r?   rD   rB   r2   base_mod_graphr:   r5   s	         r   construct_graphz%split_module.<locals>.construct_graph   sy   
 77m# #DII 2		!8I8I8O8O  ''7+<+<+B+BBBHX  +9*D*D!II"ii	 +E +TYY' +9*D*DKK"ii"/ +E +TYY'
 ,099>>+;L#( ^++ WW
"&4&=&=dkk&JL#+/99>>+;L#(dkk3///.q$++>H*2N4;;'^++r!   r   Ndef_nodeuse_nodec                 v   ddl m} t        | dd       }t        |dd       }t        j	                  d| j
                  |||j
                  nd|       ||k7  ra|G|   }|j                  j                  | j
                         ||j                  j                  |       ||   }|j                  j                  | j
                         | j                  j                  d      x}t         ||      t              D ]  }|   }	|j                  j                  |	j
                         |   j                  dk7  s@t        |	dd       }
|
P|
   }|j                  j                  |	j
                         |j                  j                  |        ||j                  j                  |       y y y y )	Nr   )free_symbols_fx_partitionz*record_cross_partition_use %s (%s) %s (%s)-example_value)keyrA   )%torch.fx.experimental.symbolic_shapesrV   r1   logdebugr   r   
setdefaultr   r   rN   getsortedr(   rF   r   )rS   rT   rV   defineduseddef_partitionuse_partitiondef_valss_node	s_defineds_def_partition
partitionssymbol_to_nodes               r   record_cross_partition_usez0split_module.<locals>.record_cross_partition_use   s   F(OT:x$7		8MM%1HMMs	
 d?" *7 3%%00?#!,,77= *4 0$$//>  (}}00AAGN#L$9sC L!/!2%,,77D)!,//=@ )0(NI(42<Y2G / 7 7 B B6;; O / : : E Ed K!L" &!..99'B '1   r!   c                    t         |             }t        j                  d| j                  |       j	                  |      }|t        |      x|<   }|j                  j                  | j                         || _        y )Nz*instantiate_node_partition_mapping %s (%s))	r(   r\   r]   r   r_   r   r   appendrW   )r=   partition_name	partitionrj   r7   s      r   "instantiate_node_partition_mappingz8split_module.<locals>.instantiate_node_partition_mapping   ss    ^D12		8$))^	

 NN>2	5>~5NNJ~&##DII.+r!   rY   )rA   rE   outputcall_function   c              3   >   K   | ]  }t        |t                 y wN)rP   r
   .0args     r   	<genexpr>zsplit_module.<locals>.<genexpr>@  s     Jz#t44Js   c              3   $   K   | ]  }|d u 
 y wrv   r)   )rx   vs     r   rz   zsplit_module.<locals>.<genexpr>P  s     >q}>s   zautocast must exitzautocast_regions: %szgrad_regions: %s)rA   rE   rr   c                      | d       S rv   r)   )nrl   s    r   <lambda>zsplit_module.<locals>.<lambda>f  s    (B1d(K r!   zRautocast or set_grad_enabled require monotonically increasing partitions:highest: z, this node's: c                      |       S rv   r)   rS   r=   rl   s    r   r   zsplit_module.<locals>.<lambda>v  s    ,FxQU,V r!   c                      |       S rv   r)   r   s    r   r   zsplit_module.<locals>.<lambda>y  s    .HSW.X r!   z cycle exists between partitions!c              3       K   | ]  }|  y wrv   r)   rw   s     r   rz   zsplit_module.<locals>.<genexpr>  s     8ss8s   )rF   rM   rB   kwargsrC   rE   )rC   rW   c                     |    S rv   r)   r   r   s    r   r   zsplit_module.<locals>.<lambda>  s    TU r!   c                     |    S rv   r)   r   s    r   r   zsplit_module.<locals>.<lambda>  s    {1~ r!   )call_modulerE   r-   _)rF   rM   rB   r   rC   r   zMissing exit noderA   c              3   B   K   | ]  }j                   |        y wrv   )r   )rx   r   
orig_nodesrp   s     r   rz   zsplit_module.<locals>.<genexpr>"  s&      
8<I!!*T"23
s   r)   c              3   (   K   | ]	  }|     y wrv   r)   )rx   r   r>   s     r   rz   zsplit_module.<locals>.<genexpr>P  s     B,t$Bs   c                 "    | j                      S rv   )r   )r   r>   s    r   r   zsplit_module.<locals>.<lambda>j  s    |AFF?S r!   zpost split_module)Pr\   r]   r   r
   dictr(   r   r   graph_moduler	   sympyr   amp_enter_autocast_exit_autocast_C_set_grad_enabledr   setr   nodesrN   r_   rP   SymIntSymFloatr=   exprSymbolrF   rM   rG   rB   boolalladdremovevaluesitemsr`   _LOGGERisEnabledForloggingDEBUGr   map_argr   listkeysr   rn   popr   RuntimeErrorr   rK   tuplerL   rO   r   r4   nnModulerE   r   rA   r/   rW   replacer   reversedr   r   rr   r   proxyProxy	enumeratenextiter)Fr5   r6   r7   r8   r9   r:   rR   r   rq   GLOBAL_STATE_NODESgrad_regionsautocast_regionsautocast_exitsactive_gradactive_autocastsvals0akr|   assert_monotonically_increasinghighest_partitionpidoriginal_partition_orderroot_partitionsro   sorted_partitionsroot_partition	dependentregions_mappingregionsrnew_node
new_inputsinp	orig_node	orig_attrrA   gathered_argsgathered_kwargsrM   target_attrr+   r   	exit_nodeorig_mod_envr?   construct_order_partitionsalready_constructed_attr_nodesr=   original_orderoutput_valsnum_output_valsrZ   orig_mod_attr_nodes_based_mod_attrs
output_valnum_outputsoutput_val_proxyioutput_nameretr>   rQ   r   r   rp   rj   rl   rk   sF   ` `  `                                           `            @@@@@@@@r   r   r   5   sP   P II11dC
 , ,39o , S%(("7"7"C"CCD ,D ')J"$J/1N.CT .CXd^ .C`," 			!!		  "" 1<L 5@M13NKu +: IIMM/22S?3u~~ >?.2=.(,0N388==)77;;*4077o%$++9K*K{{ehh888499~***!$))A,555",/1E0F,G[)		 9 99J		JJJJ $$T*),nT.B-C)D &'+t$		 8 88499~*** 1.22>$3GH ''		!5/3tyy|,"%)).*>?! 	:AQ##N4$89	:U+:Z >n&;&;&=>>T@TT>1A1G1G1IJA6!9JJ-9-?-?-ABTQAvayLBLBGMM*,.>?(,7&*+;&<&R\@R#   $
499 771177hHHNN""		!K * &C$+ -.ocUD+ !$ ;;00HHNN""		V HHNN""X7>  $JOO$56!#O%/%5%5%7 3!	9))*"">23
 $&
(,,.  0#N3>> 	2Iy!..22>Bi(55&&y1	2  Z0=>> -l; 7,224 	7MD'w<!###<@Js71:'33D9QR[ 7&s1v.	$??66ww;;8dii88"ii 7  IINN$  /7	%%d+7	77& , &~.	&(
## 	AC"3I ||z)!)"2"2C8883Ay7G7GH	i9"+//":":9;K;K"LK:CI%%i&6&67"+//"="=",S/"6"6 #> #K '+JsO'oo99(o22 :  #'
3)#3388:K5@I!!*S/23	A4 &	;&@  $34)"4#5#56I $//K!HHNN22499>VWM#hhnn445O ww995aE,,S#6,7	!!&)+ #,"7"7!8&BH-1[[L*mU333ot444 7499TD 2277"&)) 3 H !IINN,HM*2I!!$'I$3N -- _- 	D%d+Gw<!###Sb\ &s1v.	*40	 ,A.AA,$??66 ||$++#//57'nn 7  NN'') 	( %'L$&L+088>>+?+?+ANCENGGMM 	D+:lN,(L.	 GGMM 	+D&*L#	+ "5:R  &)U" ()ww}}Qt=8PdQNQ4 <E~.	  
@I@Q@Q
 

 k*aOO"";q>2q OO"";/ OO""2& %++/n, S!/ / ' 9991@,2.. /22489 , 999/>,0,n /22489 160E0E0Q0Qy1
y,,-
 $//!!B1A1ABB


 )++,?$xx~~33J?"+I,=,="> E;,<Q,?,D,D[)EA:DLd9#4#4567y<ED <GGMM 	D+:lN,(L.	  77h!!&&tyy|5ST ((


+
+NN
KCII2CF JA	 KBR R,/s   6}7"}=~~~)NFF)rH   r   collectionsr   typingr   r   r   r   torch.fx._compatibilityr   torch.fx._utilsr   torch.fx.graph_moduler	   torch.fx.noder
   __all__	getLoggerr%   r\   r   r   r   r   r(   r4   intr   r   r   r)   r!   r   <module>r      s     # * *  1 2 -  
'!!!(+ +g d+
 
 ,
0 C C  d+
 .2*/.3||HHOO| dVS[)| 4S>*	|
 "$| &d^| ,|r!   