
    Vhw)                   	   U d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlZddlmZmZ ddlmZ ddlmZmZmZmZmZ ddlmZ ddlmZmZmZmZm Z m!Z!m"Z" dd	l#m$Z$m%Z% ddl&Z&ddl'Z&ddl(Z&ddl)m*c m+Z, dd
l-m.Z. ddl/m0Z0 ddl1m2Z2 ddl3m4Z4 ddl5m6Z6 ddl7m8Z8 ddl9m:Z: ddl;m<Z<m=Z= ddl>m?Z? ddl@mAZA ddlBmCZD ddlEmFZFmGZG ddlHmIZI ddlJmKZKmLZL ddlMmNZN ddlOmCZC ddlPmQZQ ddlRmSZS  e	j                  eU      ZVe&j                  j                  ZXe&j                  j                  ZYeZZe"eZe&j                  j                  f   Z\ G d de       Z] G d d e       Z^ G d! d"e       Z_ e!d#      Z`e"e&j                  j                  j                  ecf   Zd G d$ d%      Ze ee       Zf	 d	 	 	 	 	 	 	 dd&Zg G d' d(      Zh G d) d*ei      Zje"ehejf   Zkdd+Zl G d, d-      Zm G d. d/e      Zn G d0 d1en      Zo G d2 d3en      Zp G d4 d5en      Zq G d6 d7en      Zr G d8 d9en      Zseted:f   Zu G d; d<es      Zv G d= d>ev      Zw G d? d@ev      Zx G dA dBev      Zy G dC dDes      Zz G dE dFez      Z{ G dG dHez      Z| G dI dJez      Z} G dK dLen      Z~ G dM dNen      Z G dO dPen      Z G dQ dR      Z G dS dTe       Zej                   G dU dV             Zej                   G dW dXe             Zej                   G dY dZe             Zej                   G d[ d\e             Zdd]Zdd^Z	 d	 	 	 	 	 	 	 	 	 dd`Zeddadd_f	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddbZ eAec          Z	 	 	 	 	 	 	 	 	 	 	 	 ddcZ ee      j                  ddz  dez  Zg Zdfedg<   eddad_f	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddhZ eDj(                  d_i      	 	 d	 	 	 	 	 	 	 	 	 	 	 ddj       Z	 	 d	 	 	 	 	 	 	 	 	 	 	 ddkZefd_dl	 	 	 	 	 	 	 	 	 ddmZefd_dl	 	 	 	 	 	 	 	 	 ddnZddoZ ej4                  dp      ZddqZddrZddsZddtZdduZddvZ G dw dx      ZddyZ	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 ddzZ e&jJ                         d{dd|	 	 	 	 	 	 	 	 	 dd}       Z e&jN                         dd~       ZddZddZddZddZddZ eAec          Z	 d	 	 	 	 	 	 	 ddZddZddZy)a4  
# Inductor Pattern Matcher

The pattern matcher enables search/replace within an FX graph.

The main entrypoint to the pattern matcher is register_replacement(). Given a
search function and a replacement function this will register a replacement with
a pass (such as torch._inductor.fx_passes.joint_graph.patterns).

Internally the pattern matcher represents patterns as a graph (a DAG). Creating
new patterns manually as a graph is cumbersome and error-prone so the standard
way to create patterns (using register_replacement()) is to provide a search
function and a replacement function which is traced and converted into a graph.

Because the search functions are built somewhat generic (they tend to ignore
tensor sizes, for example) register_replacement() allows you to specify an
`extra_check` function which performs additional checks to verify that the
matched pattern fully matches before returning it.

## Precompiled Patterns

New patterns are added using register_replacement(). Patterns added in this way
can have a compile-time overhead because they need to be traced before
use. Patterns can be precompiled and added using gen_register_replacement()
instead. To do this you call gen_register_replacement() instead of
register_replacement(). The arguments are the same except for an additional
unique name which is used as a lookup key.

## Internals

The match DAG is represented by a graph of `PatternExpr` nodes. Each PatternExpr
implements a `_match` method which returns either a `Match` object for a
successful match or a `FailedMatch` object for a failure to match.
    )annotationsN)ABCabstractmethod)defaultdict)
Collection	GeneratorIterableMappingSequence)Path)AnyCallableNoReturnOptionalProtocolTypeVarUnion)SelfTypeIs)enable_python_dispatcher)counters)is_integer_dtype)unset_fake_temporarily)make_fxstatically_known_true)	_get_attr)immutable_dictimmutable_list)GraphTransformObserver)
OrderedSet   config)aot_functionmake_boxed_func)default_partition)
FakeTensorFakeTensorMode)Transformer   )select_decomp_table)%fallback_node_due_to_unsupported_typec                       e Zd ZU ded<   ddZy)SearchFnstr__name__c                     y N selfargskwargss      O/home/dcms/DCMS/lib/python3.12/site-packages/torch/_inductor/pattern_matcher.py__call__zSearchFn.__call__]           Nr7   r   r8   r   returnr   )r1   
__module____qualname____annotations__r:   r4   r<   r9   r/   r/   Z   s
    M=r<   r/   c                      e Zd ZddZy)	ReplaceFnc                     y r3   r4   r5   s      r9   r:   zReplaceFn.__call__a   r;   r<   Nr=   r1   r?   r@   r:   r4   r<   r9   rC   rC   `   s    =r<   rC   c                  $    e Zd Z	 	 	 	 	 	 	 	 ddZy)TraceFnc                     y r3   r4   )r6   fnr7   r8   s       r9   r:   zTraceFn.__call__e   s    "r<   N)rI   zUnion[SearchFn, ReplaceFn]r7   r   r8   r   r>   torch.fx.GraphModulerE   r4   r<   r9   rG   rG   d   s'    #,#58#DG#	#r<   rG   Tc                      e Zd ZddZy)Multiplec                2    dt               vs| t        u sJ y y )NMULTIPLE)globalsrO   r6   s    r9   __init__zMultiple.__init__q   s    *dh.>>>.>*r<   Nr>   None)r1   r?   r@   rR   r4   r<   r9   rM   rM   p   s    ?r<   rM   c                   ddl m}m} t        j                  j
                  ry| j                  dg       j                         }|j                   ||||j                               | j                  d |j                  j                         D               || d<   y | j                  d |j                  j                         D               y )Nr   )
NodeSourceNodeSourceAction	from_nodec              3  v   K   | ]1  \  }}|t         j                  j                  j                  v r||f 3 y wr3   torchfxproxy_COPY_META_FIELDS.0kvs      r9   	<genexpr>z!_transfer_meta.<locals>.<genexpr>   6      
1EHHNN444 F
   79c              3  v   K   | ]1  \  }}|t         j                  j                  j                  v r||f 3 y wr3   rZ   r_   s      r9   rc   z!_transfer_meta.<locals>.<genexpr>   rd   re   )torch.fx.tracebackrV   rW   r$   traceenabledgetcopyappendREPLACEupdatemetaitems)new_metaold_node	pass_namerV   rW   new_from_nodes         r9   _transfer_metaru   z   s     @
 || ["5::<Z)=M=U=UVW 
 ++-
 	

 !. 
 ++-
 	
r<   c                       e Zd ZU dZded<   ded<   ded<   ded	<   d
ed<   ded<   ded<   	 	 d	 	 	 	 	 	 	 	 	 d fdZedd       ZddZddZ	ddZ
d dZd!dZd"dZ	 	 	 	 	 	 d#dZ	 	 d$	 	 	 	 	 	 	 	 	 d%dZ xZS )&Matcha  
    Represents a successfully matched pattern.

    The `Match` object is returned to represent a successfully matched
    pattern. Included in the Match are the pattern that was matched, the graph
    nodes matched, and any args that were used during the matching.

    The args and kwargs are specific to the type of pattern that was matched and
    provide hints about what was matched.
    PatternExprpattern	list[Any]r7   dict[str, Any]r8   list[torch.fx.Node]nodesz'dict[_TargetExpr, torch.fx.node.Target]targetsMatchContextctxzOptional[torch.fx.GraphModule]replacement_graphc                    t         |           || _        t        |xs g       | _        |xs i | _        g | _        i | _        || _        d | _	        y r3   )
superrR   ry   listr7   r8   r}   r~   r   r   )r6   r   ry   r7   r8   	__class__s        r9   rR   zMatch.__init__   sO     	$	l
!%r<   c                .    | j                   j                  S r3   )r   graphrQ   s    r9   r   zMatch.graph   s    xx~~r<   c                2   | j                   rwt        | j                   j                               t        |j                   j                               z  D ]-  }| j                   |   |j                   |   k7  s#t        d|       | j                  j                  |j                         | j                  j                  |j                         | j                   j                  |j                          | j                  j                  |j                         y )Nzkwarg mismatch: {})	r8   r!   keysFailedMatchr7   extendr}   rn   r~   )r6   otherkeys      r9   r   zMatch.extend   s    ;;!$++"2"2"45
5<<CTCTCV8WW A;;s#u||C'88%&:C@@A 			$

%++&5<<(EMM*r<   c                f    | j                   rt        | j                         g| _         | S g | _         | S r3   )r7   tuplerQ   s    r9   bundlezMatch.bundle   s.    *.))U499%&	 :<	r<   c                <    d| j                    d| j                   dS )NzMatch(..., , )r7   r8   rQ   s    r9   __repr__zMatch.__repr__   s    TYYKr$++a88r<   c                    | j                   }t        | j                        D ]-  }|j                  r|j                  r|j                  |       / y r3   )r   reversedr}   _erasedusers
erase_node)r6   r   ns      r9   erase_nodeszMatch.erase_nodes   s>    

$**% 	$A99QWW  #	$r<   c                    | j                   j                  D cg c]  }|| j                   j                  |   nd ! c}S c c}w r3   )r   outputspattern_to_node)r6   ps     r9   output_nodeszMatch.output_nodes   sF     XX%%
 -.MTXX%%a(tC
 	
 
s   $A c                B    t        d | j                         D              S )Nc              3  &   K   | ]	  }|s|  y wr3   r4   )r`   r   s     r9   rc   z$Match.output_node.<locals>.<genexpr>   s     8!aA8s   )nextr   rQ   s    r9   output_nodezMatch.output_node   s    8t002888r<   c                \    t         j                  | | j                  j                  ||       y r3   )ReplacementPatternEntryreplace_with_graphr   r   )r6   r   r7   s      r9   r   zMatch.replace_with_graph   s$     	 22$((.."3T	
r<   c                X   ddl m}m} t        |j                  |      r|j                  |j                  nt        j                         }|5  |t        j                  t        |      } ||t        j                  j                  |d             }t        | j                        dk(  r@|j                  j                  D ]'  }	t!        |	j"                  | j                  d   d       ) t$        j'                  | | j(                  j                  ||       ddd       y# 1 sw Y   yxY w)	a	  Replace with a graph generated by tracing the replacement_fn.

        Args:
            run_functional_passes (bool). If we should run passes that
                assume functional IR (like DCE, remove_noop_ops), on the
                replacement graph.

        r   )NullHandlerVN)run_functional_passesc                     | j                   d   S Nvalro   )args    r9   <lambda>z*Match.replace_by_example.<locals>.<lambda>  s    388E? r<   r+   replace_by_examplerq   rr   rs   )torch._inductor.virtualizedr   r   
isinstance	fake_mode
contextlibnullcontext	functoolspartialfwd_onlyr[   r\   map_arglenr}   r   ru   ro   r   r   r   )
r6   replacement_fnr7   trace_fnr   r   r   contextreplacementr   s
             r9   r   zMatch.replace_by_example   s    	? q{{K8Q[[=P KK'') 	  	$,,4I # 0 07R SK 4::!#$**00 A"!"!%A"6 $66	!	 	 	s   C
D  D))NN)
r   r   ry   rx   r7   zOptional[Sequence[Any]]r8   zOptional[dict[str, Any]]r>   rT   )r>   torch.fx.Graph)r   rw   r>   rT   )r>   rw   r>   r0   rS   )r>   zlist[Optional[torch.fx.Node]])r>   torch.fx.Node)r   r   r7   Sequence[Any]r>   rT   NT)
r   rC   r7   r   r   zOptional[TraceFn]r   boolr>   rT   )r1   r?   r@   __doc__rA   rR   propertyr   r   r   r   r   r   r   r   r   __classcell__r   s   @r9   rw   rw      s    	 
O44	55 )-+/&& & &	&
 )& 
&&  +
9$
9
!/
7D
	
 '+&*,!, , $	,
  $, 
,r<   rw   c                  4    e Zd ZU dZded<   ddZd	dZd
dZy)r   z
    Represents a unsuccessful match.

    The `FailedMatch` object is returned to represent a failure to match a
    pattern.
    r0   format_stringc                f    || _         t        |      dkD  rt        d|       || _        || _        y )N   zUFormat string too long - use lazy construction of strings instead. Format string is
 )r   r   RuntimeErrorr7   r8   )r6   r   r7   r8   s       r9   rR   zFailedMatch.__init__   sB    * }#hivhwx  	r<   c                b     | j                   j                  | j                  i | j                  S r3   )r   formatr7   r8   rQ   s    r9   __str__zFailedMatch.__str__+  s(    (t!!(($))Ct{{CCr<   c                     yNFr4   rQ   s    r9   __bool__zFailedMatch.__bool__.      r<   N)r   r0   r7   r   r8   r   r>   rT   r   r>   r   )r1   r?   r@   r   rA   rR   r   r   r4   r<   r9   r   r     s     	Dr<   r   c                    t        |       S )z|
    TypeIs cannot act on `self`. Thus this function exists to let mypy
    recognize FailedMatch.__bool__ as a TypeIs.
    )r   )ms    r9   is_matchr   5  s    
 7Nr<   c                  d    e Zd ZU dZded<   ded<   ded<   ded	<   	 d	 	 	 	 	 	 	 ddZddZddZy
)r   zC
    Internal state needed while running PatternExpr._match().
    list[Optional[PatternExpr]]r   z*dict[PatternExpr, Optional[torch.fx.Node]]r   r   r   zlist[NodeOrConstant]exclusive_node_setNc               V    || _         |i n
t        |      | _        || _        g | _        y r3   )r   dictr   r   r   )r6   r   r   r   s       r9   rR   zMatchContext.__init__G  s.     %4%<r$BW
"$r<   c                    || j                   v r)| j                   |   |k(  rt        | |      S t        d      S |j                  ||       }|| j                   vsJ |r|nd| j                   |<   |S )z)wrapper to check reused nodes in patternszrepeated pattern differsN)r   rw   r   _match)r6   ry   noder   s       r9   matchzMatchContext.matchS  sy    d***##G,4T7++"#=>>NN4&d2222201tW%r<   c                    | j                   j                         D ci c]  \  }}|j                         r||| c}}S c c}}w r3   )r   rp   has_multiple_users)r6   ry   r   s      r9   filter_multi_user_patternsz'MatchContext.filter_multi_user_patterns_  sL     "&!5!5!;!;!=
))+0@ TM
 	
 
s   Ar3   )r   r   r   z*Optional[dict[PatternExpr, torch.fx.Node]]r   r   r>   rT   )ry   rx   r   NodeOrConstantr>   MatchResult)r>   z dict[PatternExpr, torch.fx.Node])r1   r?   r@   r   rA   rR   r   r   r4   r<   r9   r   r   =  s^     )(??,,
 GK
%,
% D
%
 
% 

%

r<   r   c                  V    e Zd ZdZed	d       Zd
dZddZddZ	 	 	 	 	 	 ddZ	ddZ
y)rx   z+
    Base class for types of patterns.
    c                     y r3   r4   r6   r   r   s      r9   r   zPatternExpr._matchl  s    MPr<   c                    	 t        | g|j                        j                  | |      S # t        $ r}|cY d }~S d }~ww xY wNr   )r   r   r   r   r6   r   es      r9   r   zPatternExpr.matcho  s<    	djj9??dKK 	H	s   '* 	?:??c                     yr   r4   rQ   s    r9   r   zPatternExpr.has_multiple_usersu  r   r<   c                4    | j                   j                  dz   S )Nz())r   r1   rQ   s    r9   r   zPatternExpr.__repr__x  s    ~~&&--r<   c              #  L   K   | |j                   v r|j                   |     y y wr3   )r   r6   r   searcheds      r9   find_anchor_nodeszPatternExpr.find_anchor_nodes{  s,      3&&&%%d++ 's   "$c                .    t        || j                        S )z
        Compare two `PatternExpr`s and return true if they are the
        same. Note this is NOT matching a pattern - it is comparing the pattern
        structures (for debugging).
        )r   r   )r6   r   s     r9   
pattern_eqzPatternExpr.pattern_eq  s     %00r<   Nr   r   r   r   r>   r   r   r   r>   r   r   r   r   r   r   OrderedSet[torch.fx.Node]r>   z.Generator[Optional[torch.fx.Node], None, None]r   r   r>   r   )r1   r?   r@   r   r   r   r   r   r   r   r   r4   r<   r9   rx   rx   g  sH     P P.,,+D,	7,1r<   rx   c                      e Zd ZdZddZy)Argzn
    Capture an arg which will become an input to the handler.  Args are
    passed in depth first order.
    c                     t        || |g      S )N)r7   rw   r   s      r9   r   z
Arg._match  s    S$dV,,r<   Nr   r   r   r   r>   r   r1   r?   r@   r   r   r4   r<   r9   r  r    s    
-r<   r  c                  (    e Zd ZdZddZddZddZy)	Ignoredz4
    Match an arg, but don't pass it to handler
    c                    t        ||       S r3   r  r   s      r9   r   zIgnored._match  s    S$r<   c                     y)N*r4   rQ   s    r9   r   zIgnored.__repr__  s    r<   c                     y)Nz	Ignored()r4   )r6   pps     r9   pretty_printzIgnored.pretty_print  s    r<   Nr  r   r  PatternPrettyPrinterr>   r0   )r1   r?   r@   r   r   r   r  r4   r<   r9   r  r    s     r<   r  c                  @     e Zd ZdZd fdZddZddZd	 fdZ xZS )

KeywordArgD
    Capture a kwarg which will become an input to the handler.
    c                0    t         |           || _        y r3   r   rR   namer6   r  r   s     r9   rR   zKeywordArg.__init__      	r<   c                "    d| j                   dS )NzKeywordArg(r   r  rQ   s    r9   r   zKeywordArg.__repr__  s    TYYM++r<   c                6    t        || | j                  |i      S )Nr8   )rw   r  r   s      r9   r   zKeywordArg._match  s    S$		4'899r<   c                    t        j                  t        |      }t        |   |      xr | j
                  |j
                  k(  S r3   typingcastr   r   r   r  r6   r   r   s     r9   r   zKeywordArg.pattern_eq  5    D%(w!%(DTYY%**-DDr<   r  r0   r>   rT   r   r  r   	r1   r?   r@   r   rR   r   r   r   r   r   s   @r9   r  r    s#    ,:E Er<   r  c                  L     e Zd ZU dZded<   d fdZd	dZd
dZd fdZ xZ	S )ExclusiveKeywordArgr  r0   r  c                0    t         |           || _        y r3   r  r  s     r9   rR   zExclusiveKeywordArg.__init__  r  r<   c                "    d| j                   dS )NzExclusiveKeywordArg(r   r  rQ   s    r9   r   zExclusiveKeywordArg.__repr__  s    %dii]!44r<   c                    ||j                   v rt        d      S |j                   j                  |       t        || | j                  |i      S )Nzexclusive arg appears twicer  )r   r   rl   rw   r  r   s      r9   r   zExclusiveKeywordArg._match  sH    3)))<==%%d+S$		4'899r<   c                    t        j                  t        |      }t        |   |      xr | j
                  |j
                  k(  S r3   r  r   s     r9   r   zExclusiveKeywordArg.pattern_eq  r!  r<   r"  r   r  r   )
r1   r?   r@   r   rA   rR   r   r   r   r   r   s   @r9   r%  r%    s*     I5:E Er<   r%  c                       e Zd ZU dZded<   ded<   	 d	 	 	 	 	 d fdZeedd              ZddZ	dd	Z
dd
Z	 	 	 	 	 	 ddZddZddZd fdZ xZS )_TargetExprz7
    Base class for filtering match by node.target
    zlist[FnsType]fnszOrderedSet[FnsType]fns_setc                Z   t         |           t        |      st        |t              r|gn
t        |      }|D ]O  t        t        j                  j                        s(|j                  fdj                         D               Q || _        t        |      | _        || _        y )Nc              3  6   K   | ]  }t        |        y wr3   )getattr)r`   overloadrI   s     r9   rc   z'_TargetExpr.__init__.<locals>.<genexpr>  s     PX72x0Ps   )r   rR   callabler   r0   r   r[   _opsOpOverloadPacketr   	overloadsr,  r!   r-  r   )r6   r,  r   rI   r   s      @r9   rR   z_TargetExpr.__init__  s     	}
3(<se$s) 	QB"ejj99:

PPP	Q !#
r<   c                     y r3   r4   rQ   s    r9   opz_TargetExpr.op  s    r<   c                   | j                   d   }t        |t              s|j                  }t	        | j                         dkD  rd| dS | j                   d   t        t        |d       u rd| S | j                   d   t        t        |d       u rd| S t        | j                   d   t        j                  j                        rt        | j                   d         S |S )Nr   r+   [z, ...]torch.z	operator.)
r,  r   r0   r1   r   r0  r[   operatorr3  
OpOverload)r6   
first_reprs     r9   fns_reprz_TargetExpr.fns_repr  s    XXa[
*c*#,,Jtxx=1zl&))XXa[GE:t<<J<((XXa[GHj$??zl++UZZ%:%:;txx{##r<   c                    | j                   t        u rd}n"| j                   dk7  rd| j                    d}nd}| j                  j                   d| j	                          | dS )Nz
, MULTIPLEr+   r   r    ()r   rO   r   r1   r>  )r6   comma_userss     r9   r   z_TargetExpr.__repr__  s^    ::!&KZZ1_tzzl!,KK..))*!DMMO+<[MKKr<   c                X    t        | j                  t              xs | j                  dkD  S )Nr+   )r   r   rM   rQ   s    r9   r   z_TargetExpr.has_multiple_users  s     $**h/A4::>Ar<   c                    t         r3   NotImplementedErrorr   s      r9   r   z_TargetExpr.find_anchor_nodes  s
     "!r<   c                    t        |t        j                  j                        xr2 |j                  | j                  k(  xr t        |      | j                  v S r3   )r   r[   r\   Noder7  extract_targetr-  )r6   r   s     r9   
_match_fnsz_TargetExpr._match_fns  sD    tUXX]]+ 5477"5t$4	
r<   c                    | |j                   v xs6 | j                  t        u xs" t        |j                        | j                  k(  S r3   )r   r   rO   r   r   s      r9   _match_usersz_TargetExpr._match_users  s=    CKK -zzX%-4::$**,	
r<   c                    t        j                  t        |      }t        |   |      xrO | j
                  |j
                  k(  xr4 | j                  |j                  k(  xr | j                  |j                  k(  S r3   )r  r  r   r   r   r7  r,  r   r   s     r9   r   z_TargetExpr.pattern_eq  sf    D%(Gu% *588#*EII%* 

ekk)		
r<   )r+   )r,  z!Union[FnsType, Sequence[FnsType]]r   zUnion[Multiple, int]r>   rT   r   r   r   r   r   r>   r   )r   r   r   r   r>   r   r   )r1   r?   r@   r   rA   rR   r   r   r7  r>  r   r   r   rJ  rL  r   r   r   s   @r9   r+  r+    s     
   UV4=Q	    LB""+D"	7"



 
r<   r+  .c                       e Zd ZdZdd	 	 	 	 	 	 	 	 	 d fdZe	 	 	 	 	 	 dd       Ze	 	 	 	 	 	 dd       ZddZddZ	dd	Z
	 	 	 	 	 	 dd
Zd fdZ xZS )_TargetArgsExprzE
    Base class for filtering match by node.{target,args,kwargs}
    r+   )_usersc               j   t         |   ||       t        |      | _        t	        |      | _        t        d t        j                  ||j                               D              r| j                  | _        n| j                  | _        | j                  | j                  | j
                        | _        y )Nc              3  R   K   | ]  }t        |t        t        t        f       ! y wr3   )r   r   r   r   r`   xs     r9   rc   z+_TargetArgsExpr.__init__.<locals>.<genexpr>5  s&      
 q4u-.
s   %')r   rR   r   r7   r   r8   any	itertoolschainvaluespytree_flattenflattensimple_flattenflat_args_kwargs)r6   r,  rQ  r7   r8   r   s        r9   rR   z_TargetArgsExpr.__init__+  s     	f%$K	6l 
__T6==?;
 
  ..DL..DL $TYY Dr<   c                n    g | |j                         }t        |       g|j                         }||fS r3   )rY  r   r   )r7   r8   rY  specs       r9   r\  z_TargetArgsExpr.simple_flatten>  s:     +4*&--/*D	*FKKM*t|r<   c                    t         t        t        t        t        t        idfdt        j                  | |ffd      }t        j                  |      \  }}||fS )Nc                    t        |       }j                  |      }|!t        j                   ||       fd      S | S )Nc                    t        |       v S r3   typerU  type_mappings    r9   r   zF_TargetArgsExpr.pytree_flatten.<locals>.convert_type.<locals>.<lambda>W  s    d1g&= r<   is_leaf)rd  rj   pytreetree_map)rU  cls
convert_fnconvert_typerf  s      r9   rm  z4_TargetArgsExpr.pytree_flatten.<locals>.convert_typeP  sG    q'C%))#.J% qM= 
 Hr<   c                    t        |       v S r3   rc  re  s    r9   r   z0_TargetArgsExpr.pytree_flatten.<locals>.<lambda>^  s    d1g5 r<   rg  )rU  r   r>   r   )r   r   r   r   r   ri  rj  tree_flatten)r7   r8   normalized_args_treeflatr_  rm  rf  s        @@r9   rZ  z_TargetArgsExpr.pytree_flattenF  s_    
 E%D*
		  &6N5 

 (()=>
dTzr<   c                   | j                         gt        t        | j                        | j                  j                         D cg c]  \  }}| d|  c}}}| j                  t        u r|j                  d       n-| j                  dk7  r|j                  d| j                          | j                  j                   ddj                  |       dS c c}}w )N=_users=MULTIPLEr+   _users=rA  r   r   )r>  mapreprr7   r8   rp   r   rO   rl   r   r1   join)r6   ra   rb   r7   s       r9   r   z_TargetArgsExpr.__repr__c  s    MMO
tyy!
 &*[[%6%6%89TQ1QCj9

 ::!KK)*ZZ1_KK'$**./..))*!DIIdO+<A>> :s   Cc           
        | j                         gfd| j                  D        | j                  j                         D cg c]  \  }}| dj	                  |        c}}}| j
                  t        u r|j                  d       n-| j
                  dk7  r|j                  d| j
                          d}| j                  j                   d|j                  |       dS c c}}w )	Nc              3  @   K   | ]  }j                  |        y wr3   )r  )r`   rU  r  s     r9   rc   z/_TargetArgsExpr.pretty_print.<locals>.<genexpr>r  s     4Qbooa 4s   rs  rt  r+   ru  r   rA  r   )r>  r7   r8   rp   r  r   rO   rl   r   r1   rx  )r6   r  ra   rb   r7   
joiner_strs    `    r9   r  z_TargetArgsExpr.pretty_printo  s    MMO
4$))4
 7;kk6G6G6IJda1R__Q'()J

 ::!KK)*ZZ1_KK'$**./
..))*!JOOD,A+B!DD Ks    C'c                   | j                  |      r+t        |j                        t        | j                        k7  rt        d||       S | j	                  ||      st        d|       S |j                  }|j
                  }t        |      t        | j
                        k  rddlm} t        |j                        sJ  ||j                  |j                  |j
                        }|t        d||       S |\  }}t        |      t        | j                        k(  rDt        |      t        | j
                        k\  r#|D ci c]  }|| j
                  v s|||    }}n/t        d||       S |D ci c]  }|| j
                  v s|||    }}| j                  ||      \  }}	| j                  \  }
}|	|k7  rt        d|	|      S t        |      t        |
      k(  sJ t        ||       }t        t        j                         |
|      D ]  \  }}}t!        |t"              r3|j%                  ||      }t'        |      s|c S |j)                  |       Jt!        |t*        j,                  j.                        s||k7  stt        d||      c S  |j0                  j3                  |       |j                  |j4                  | <   |S c c}w c c}w )Nz&function_mismatch: node={}, pattern={}zmultiple_users {}r   )normalize_functionzargs_structure {} {}z#constant_args: {} {!r}!={pattern!r})rJ  r   r7   r   rL  r8   torch.fx.operator_schemasr}  r2  targetr[  r]  rw   ziprW  countr   rx   r   r   r   r[   r\   rH  r}   rl   r~   )r6   r   r   _args_kwargsr}  normalized_args_and_kwargsi
node_items	node_spec
self_items	self_specr   ry   
child_nodechild_matchs                   r9   r   z_TargetArgsExpr._match}  ss   t$DII#dii.(HGtTT  s+2D99		++w<#dkk**DDKK((();TYY*& *1"#KTSWXX!;wu:TYY/CLCDT4T6=RdkkAQq'!*}RGR&@$  /6Jdkk9Iq'!*}JGJ $UG <
I $ 5 5
I	!5y)LL:#j/111#t&))//*;Z&T 		"Aw
';/!ii<,&&%J6*:O"94 		 	
t++		$7 S Ks    KK0KKc              #    K   | |j                   v r|j                   |     y| j                  d   D ]  }t        |t              s|j	                  ||      D ]d  }t        |t
        j                  j                        s(|j                  D ].  }||vs| j                  |      s| |j                  |       0 f  yw)a  
        This is used when we are matching a pattern with multiple outputs.
        There is a partial match (stored in ctx) and we want to walk
        this pattern to find a connection to an already-matched node.

        Yields candidate nodes that `self._match` might like.
        Nr   )r   r]  r   rx   r   r[   r\   rH  r   rJ  add)r6   r   r   ry   
other_noder   s         r9   r   z!_TargetArgsExpr.find_anchor_nodes  s      3&&&%%d++,,Q/ 		3G';/")";";C"J 3J%j%((--@  * 0 0 3x/#t4&*
 (T 2	33		3s   ACACC&Cc                   t        j                  t        |      }t        |   |      xrW | j
                  d   |j
                  d   k(  xr6 t        d t        | j
                  d   |j
                  d         D              S )Nr+   c              3  n   K   | ]-  \  }}t        |t              r|j                  |      n||k(   / y wr3   r   rx   r   r`   abs      r9   rc   z-_TargetArgsExpr.pattern_eq.<locals>.<genexpr>  6      Aq $.a#=Q16I   35r   )r  r  r   r   r   r]  allr  r   s     r9   r   z_TargetArgsExpr.pattern_eq  s    D%(Gu% %%a(E,B,B1,EE  5 5a 8%:P:PQR:ST 	
r<   )
r,  z/Union[torch.fx.node.Target, str, Sequence[Any]]r7   r   rQ  zUnion[int, Multiple]r8   r   r>   rT   )r7   r   r8   zMapping[Any, Any]r>   z9tuple[Sequence[Any], Union[_SimpleSpec, pytree.TreeSpec]]r   r  r   r   r   )r1   r?   r@   r   rR   staticmethodr\  rZ  r   r  r   r   r   r   r   s   @r9   rP  rP  &  s     ()	E<E E %	E
 E 
E& %6	B  %6	B 8
?E1f33+D3	732	
 	
r<   rP  c                      e Zd ZdZdZy)CallFunctionzR
    Matches a call_function node in the FX graphs: `fns[i](*args, **kwargs)`
    call_functionNr1   r?   r@   r   r7  r4   r<   r9   r  r    s     
Br<   r  c                      e Zd ZdZdZy)
CallMethodzW
    Matches a call_method node in the FX graphs: `fns[i].method(*args, **kwargs)`
    call_methodNr  r4   r<   r9   r  r         
Br<   r  c                      e Zd ZdZdZy)
CallModulezP
    Matches a call_module node in the FX graphs: `module(*args, **kwargs)`
    call_moduleNr  r4   r<   r9   r  r    r  r<   r  c                      e Zd ZdZddZy)_TargetExprVarArgsz[
    Matches a call_function node with any arguments which are passed into the pattern
    c                   | j                  |      st        d      S | j                  ||      st        d      S t        ||       }|j                  j                  |       |j                  |j                  | <   |j                  j                  |j                         |j                  j                  |j                         |S )Nfunction_mismatchmultiple_users)rJ  r   rL  rw   r}   rl   r  r~   r7   r   r8   rn   )r6   r   r   r   s       r9   r   z_TargetExprVarArgs._match  s    t$233  s+/00#t	t++		$	dii 	$r<   Nr   r  r4   r<   r9   r  r    s    r<   r  c                      e Zd ZdZy)CallFunctionVarArgsr  Nr1   r?   r@   r7  r4   r<   r9   r  r    s    	Br<   r  c                      e Zd ZdZy)CallMethodVarArgsr  Nr  r4   r<   r9   r  r        	Br<   r  c                      e Zd ZdZy)CallModuleVarArgsr  Nr  r4   r<   r9   r  r  	  r  r<   r  c                  B     e Zd ZdZdd fdZddZd	dZd
 fdZ xZS )ListOfz$
    Matches a repeated pattern
    c                b    t         |           t        |t              sJ || _        || _        y r3   )r   rR   r   rx   ry   r   )r6   ry   r   r   s      r9   rR   zListOf.__init__  s,    ';///r<   c                N    | j                   j                   d| j                   dS NrA  r   )r   r1   ry   rQ   s    r9   r   zListOf.__repr__  $    ..))*!DLL>;;r<   c                (   t        |t        t        f      rt        |      dk(  rt	        d      S t        ||       }|j                         }d}t        |      D ]  \  }}t        |j                  ||j                        }|j                  | j                  |      }	|j                         }t        |	      s| j                  st	        d||	      c S {d}|j                  |	j!                                 |st	        d      S |j!                         S )Nr   non_listFr   zlist[{}]: {}Tzlist: no_match)r   r   r   r   r   rw   r   	enumerater   r   r   r   ry   r   r   r   r   )
r6   r   r   r   r   matchedr  r  	child_ctxr  s
             r9   r   zListOf._match  s    $u.#d)q.z**#t 88:&t_ 	+MAz$_J4D4DI $//$,,
CK'BBDOK(||&~q+FFGHH['')*	+ /00xxzr<   c                    t        j                  t        |      }t        |   |      xr@ | j
                  j	                  |j
                        xr | j                  |j                  k(  S r3   )r  r  r   r   r   ry   r   r   s     r9   r   zListOf.pattern_eq3  sU    D%(Gu% .''6.-	
r<   F)ry   rx   r   r   r>   rT   r   )r   r|   r   r   r>   r   r   r#  r   s   @r9   r  r    s!    <0
 
r<   r  c                  ~     e Zd ZU ded<   d fdZedd       ZddZddZddZ		 	 	 	 	 	 ddZ
dd	Zd fd
Z xZS )MultiOutputPatternr   r   c                    t         |           t        |d   t              sJ t	        d |D              sJ |       t        |      | _        |d   j                  | _        y )Nr   c              3  H   K   | ]  }|d u xs t        |t                y wr3   )r   rx   rT  s     r9   rc   z.MultiOutputPattern.__init__.<locals>.<genexpr>B  s#     Lq19:
1k ::Ls    ")r   rR   r   r+  r  r   r   r7  )r6   r   r   s     r9   rR   zMultiOutputPattern.__init__?  sU    '!*k222LGLLUgULG}!*--r<   c                h    t        j                  t        | j                  d         }|j                  S Nr   )r  r  r+  r   r,  )r6   outputs     r9   r,  zMultiOutputPattern.fnsF  s&     [$,,q/:zzr<   c                N    | j                   j                   d| j                   dS r  )r   r1   r   rQ   s    r9   r   zMultiOutputPattern.__repr__L  r  r<   c                    | j                   D cg c]  }|j                  |       }}dd }| j                  j                   d|j	                  |       }| d}|S c c}w )Nz,
z  z([z
]))r   r  r   r1   rx  )r6   r  rU  r7   r{  str_outs         r9   r  zMultiOutputPattern.pretty_printO  sh    ,0LL9q"994&\
^^,,-R
0E/FGIT"	 :s   Ac                $   t        j                  t        | j                  d         }|j	                  ||      }t        |      s|S | j                  dd  D ]7  }|| j                  ||      }t        |      s|c S |j                  |       9 |S )Nr   r+   )r  r  r+  r   r   r   _match_from_anchorsr   )r6   r   r   r  r   ry   r  s          r9   r   zMultiOutputPattern._matchV  s    [$,,q/:IIfd#{H||AB' 	"G227C@KK(""HH[!	" r<   c                    t        |j                        }t        d      }|j                  |t	                     D ]3  }|j                  ||      }t        |      r|c S t        |      |_        5 |S )Nzno anchor found)r   r   r   r   r!   r   r   )r6   ry   r   priorr   r   s         r9   r  z&MultiOutputPattern._match_from_anchorsf  sl     S(()$%67--c:<@ 	.D		'4(A{"&u+C	. r<   c                    	 t        | j                  |j                        j                  | |      S # t        $ r}|cY d }~S d }~ww xY wr   )r   r   r   r   r   r   s      r9   r   zMultiOutputPattern.matchs  s>    	DJJ?EEdDQQ 	H	s   03 	AAAAc                   t        j                  t        |      }t        |   |      xr] t        | j                        t        |j                        k(  xr0 t        d t        | j                  |j                        D              S )Nc              3  n   K   | ]-  \  }}t        |t              r|j                  |      n||k(   / y wr3   r  r  s      r9   rc   z0MultiOutputPattern.pattern_eq.<locals>.<genexpr>~  r  r  )	r  r  r   r   r   r   r   r  r  r   s     r9   r   zMultiOutputPattern.pattern_eqy  sm    D%(Gu% DLL!S%77 emm< 	
r<   )r   zSequence[Optional[PatternExpr]]r>   rT   )r>   z-Union[Callable[..., Any], str, Sequence[Any]]r   r  r   )ry   rx   r   r   r>   r   r   r   )r1   r?   r@   rA   rR   r   r,  r   r  r   r  r   r   r   r   s   @r9   r  r  <  s]    ((   
< ")5		
 	
r<   r  c                  J     e Zd ZdZd fdZedd       ZddZd	 fdZ xZ	S )
RepeatedExprzp
    Checks for a repeated pattern. Useful for repeated operations after a node such as `split` or `unbind`
    c                R    t         |           || _        |j                  | _        y r3   )r   rR   inner_patternr7  )r6   r  r   s     r9   rR   zRepeatedExpr.__init__  s#    *""r<   c                .    | j                   j                  S r3   )r  r,  rQ   s    r9   r,  zRepeatedExpr.fns  s    !!%%%r<   c                   |j                  | j                  |      }t        |      s|S |j                  j	                  | j                         | j                  j                  |t                     D ]T  }t        | g|j                        j                  | j                  |      }t        |      s|c S |j                  |       V |S r   )
r   r  r   r   popr   r!   r   r   r   )r6   r   r   r   anchor_nodeanchor_ms         r9   r   zRepeatedExpr._match  s    IId(($/{H	
  --??Z\R 	K#TF$**=CC""KH H%HHX	 r<   c                    t        j                  t        |      }t        |   |      xr% | j
                  j	                  |j
                        S r3   )r  r  r   r   r   r  r   s     r9   r   zRepeatedExpr.pattern_eq  sF    D%(w!%( 
T-?-?-J-J.
 	
r<   )r  r+  r>   rT   )r>   zSequence[FnsType]r   r   )
r1   r?   r@   r   rR   r   r,  r   r   r   r   s   @r9   r  r    s0    #
 & &"
 
r<   r  c                  f    e Zd ZdZddZe ej                  d      dd	d              Zd
dZ	ddZ
y)r  z
    Serializes Patterns to executable python.
    XXX: currently only used and tested for fuse attention patterns. May not cover
    all patterns.
    c                z    t         j                  j                  j                         | _        i | _        i | _        y r3   )r[   r\   r   
_Namespace	namespacememoized_objs_namesmemoized_objs_pprQ   s    r9   rR   zPatternPrettyPrinter.__init__  s*    224;= 8:r<   Nc                   t               }t        | d      sJ | j                  |      }|j                  D cg c]#  }|j                  |    d|j                  |    % }}|j                  | d|        dj                  |      S c c}w )zU
        Serializes obj to python code with obj written out to `output_name`
        r  )r  z = 
)r  hasattrr  r  r  rl   rx  )objoutput_namer  out_strr   r  s         r9   runzPatternPrettyPrinter.run  s     "#sN+++""b") --
 %%c*+3r/B/B3/G.HI
 

 	S	23yy  
s   (B
c                    t        |t              r0| j                  j                  |      x}r|S | j	                  |      S t        |d      r|j                  |       S t        |      S )Nr  )r   rP  r  rj   memoizer  r  rw  )r6   r  memoized_names      r9   r  z!PatternPrettyPrinter.pretty_print  sb    c?+ $ 8 8 < <S AA}A$$||C((3'##D))Cyr<   c                    |j                  |       }|j                         }dD ]  }|j                  |d      } | j                  j	                  |d       }|| j
                  |<   || j                  |<   |S )N)zaten.r:  zprims.r@  )r  r>  replacer  create_namer  r  )r6   r  obj_strobj_nameprefixtmp_names         r9   r  zPatternPrettyPrinter.memoize  s{    ""4(<<>3 	4F''3H	4 >>--h=(0  %%,c"r<   rS   )r  )r  rx   r  r0   r>   r0   )r  r   r>   r0   )r  rP  r>   r0   )r1   r?   r@   r   rR   r  r   	lru_cacher  r  r  r4   r<   r9   r  r    s?    ;
 Y!  !$		r<   r  c                      e Zd Z	 	 	 	 ddZy)_PassDictsTypec                     y r3   r4   )r6   ra   s     r9   __getitem__z_PassDictsType.__getitem__  s     r<   N)ra    tuple[str, torch.fx.node.Target]r>   list[PatternEntry])r1   r?   r@   r  r4   r<   r9   r  r    s    !1!	!r<   r  c                  F    e Zd ZU ded<   ded<   ddZ	 	 d		 	 	 	 	 	 	 d
dZy)PatternEntryrx   ry   Callable[[Match], bool]extra_checkc                    t         r3   rE  r6   r   r   r   s       r9   applyzPatternEntry.apply  s    !!r<   Nc                   |Ht        | j                  d      sJ | j                  j                  D ]  }| j                  |||        y t	        |t
        t        f      rqt        | j                  d      sJ |r,|| j                  j                  |f   j                  d|        y || j                  j                  |f   j                  |        y t        j                  t        t           |      }|D ]  }| j                  |||        y )Nr,  prependr7  r   )r  ry   r,  registerr   r   PatternMatcherPassr7  insertrl   r  r  r   r  )r6   
pass_dictsr  r   rI   rU  s         r9   r  zPatternEntry.register  s     >4<<///ll&& ?j"g>?
T+=$>?4<<...DLLOOV45<<QEDLLOOV45<<TBXn%=zJJ :a9:r<   r   rw   r   r   r   r   r>   rT   r   )r  /Union[_PassDictsType, Sequence[_PassDictsType]]r  z!Union[torch.fx.node.Target, None]r   r   r>   rT   )r1   r?   r@   rA   r  r  r4   r<   r9   r  r    sF    ((" 59	:C: 2: 	:
 
:r<   r  c                       e Zd ZU ded<   ddZy)LoweringPatternEntryCallable[..., Any]handlerc                    t        j                  | j                        t        j                  | j                  |            }|j	                  |      5  |j                  |t        |j                        |j                        }|j                  j                  |j                         |j                  |       d d d        |j                  d   |u sJ |j                          y # 1 sw Y   -xY w)N)r   wrapsr
  r   inserting_beforer  r   r7   r8   ro   rn   replace_all_uses_withr}   r   )r6   r   r   r   r
  r   s         r9   r  zLoweringPatternEntry.apply
  s    /)//$,,/	0A0A$,,PU0VW##D) 	4--guUZZ7H%,,WK##DII.&&{3	4 {{2$&&&	4 	4s   A'C((C1Nr  )r1   r?   r@   rA   r  r4   r<   r9   r  r    s    r<   r  c                  $    e Zd ZU dZded<   ddZy)GraphPatternEntryz8
    A pattern that runs a function on the FX graph
    r	  r
  c                    |j                  |      5   | j                  |g|j                  i |j                   d d d        y # 1 sw Y   y xY wr3   )r  r
  r7   r8   r  s       r9   r  zGraphPatternEntry.apply  sE    ##D) 	=DLL<<u||<	= 	= 	=s   +AANr  )r1   r?   r@   r   rA   r  r4   r<   r9   r  r    s      =r<   r  c                  F    e Zd ZU ded<   e	 	 	 	 	 	 	 	 	 	 dd       ZddZy)r   zCallable[..., list[Any]]normalize_argsc                    G fddt         j                  j                        }| j                         }t	        |      dk(  r|d   }n|d   sJ t        |d   j                  j                        }|D cg c]9  }t        |t         j                  j                        r|j                  |      |f; }	}t        |	t        j                  d            d   }	 	 	 	 	 	 	 	 	 	 d	dj                  |      5  t        |t         j                  j                        sJ   ||      j                    }
t        |
t         j                  j                        r|
g}
d
d	 	 	 	 	 	 dfdt	        |      t	        |
      k(  rt#        ||
      D ]  \  }} ||        nt	        |      dk(  sJ  |d   |
       d d d        | j%                          y c c}w # 1 sw Y   xY w)Nc                  0     e Zd ZdZdZdZd fdZ xZS )<ReplacementPatternEntry.replace_with_graph.<locals>.ReplacerNc                B   |j                   dv rt        | 	  |      S |j                   dk(  r|j                  }| j	                  |      \  }}t        |      sJ j                  |||      }t        |j                  |d       d|j                  v r}d|j                  vro|j                  d   |j                  d<   t        |j                  d   t        j                        r,d|j                  v sJ |j                  d   |j                  d<   |S t        d|       )N)placeholderr  r  Interpreter_Replacerr   r   tensor_metaz
unhandled )r7  r   run_noder  fetch_args_kwargs_from_envr2  r  ru   ro   r   r[   TensorrF  )r6   r   r  r7   r8   resultr   r   s         r9   r  zEReplacementPatternEntry.replace_with_graph.<locals>.Replacer.run_node1  s   7777 7+D1177o-![[F#'#B#B4#HLD&#F+++"00vFF"!'!%"8
 		)e6;;.F-1YYu-=E*%dii&6E#0DII#==#=9==9QFKK6!M)Jtf*=>>r<   r   r   r>   r   )r1   r?   r@   r  r  get_attrr  r   )r   r   s   @r9   Replacerr  ,  s    KKH? ?r<   r"  r+   r   )r   c                   | g}t        t        j                  j                            }|rc|j	                         }||vrK||vrGt        |d      r;|j                  |       ||j                  |<   |j                  |j                         |rby y )Nro   )
r!   r[   r\   rH  r  r  r  ro   r   all_input_nodes)r   tag_name	tag_valueinput_stopsqueuevisitedr   s          r9   percolate_tagszBReplacementPatternEntry.replace_with_graph.<locals>.percolate_tagsT  sy     FE /1Giikw&;.V,KK$)2CHHX&LL!4!45 r<   c                    | j                   dk7  ry | j                  t        j                  k7  ry t	        | j
                        dk(  sJ | j
                  d   S )Nr  r"   r+   )r7  r  r;  getitemr   r7   r   s    r9   maybe_getitemzAReplacementPatternEntry.replace_with_graph.<locals>.maybe_getitemn  sJ    77o-;;("2"22499~***yy|#r<   c           	        | |J y t        | t        j                  j                        sJ |#| j	                  d        j                  |        y t        |t        j                  j                        rd|j                  vr%|j                  j                  | j                         dD ]2  }|| j                  v s 	||| j                  |   t                     4 | j	                  |       j                  |        y t        | j                  j                               }|D ]#  } |      }|t        d       
|||          % j                  |        y )Nr   )	recomputeac_graph_idzcan't handle)r   r[   r\   rH  r  r   ro   rn   r!   r   r   r   AssertionError)oldnewr%  old_usesuseridxr7   r   r.  r*  r  s         r9   r  z;ReplacementPatternEntry.replace_with_graph.<locals>.replacev  sA    ;;&;!#uxx}}555;--d3$$S)c588==1CHH,1 %A #sxx/* #Xsxx/A:dCS --c2$$S):  		 01$ ,D'-C{,^<<D#c(+	,
   %r<   )
r   r   r%  r0   r&  r0   r'  r   r>   rT   r   )r3  zUnion[torch.fx.Node, None]r4  z3Union[torch.fx.Node, Sequence[torch.fx.Node], None]r>   rT   )r[   r\   Interpreterr   r   r   r   r}   r   rH  indexminr;  
itemgetterr  GraphModuler  r  r   )r   r   r   r7   r"  r   	last_noder}   r   indicesr   r3  r4  r.  r*  r  s    ` `         @@@r9   r   z*ReplacementPatternEntry.replace_with_graph%  s	   	?uxx++ 	?4 ))+|!$QI?"?a..445E &a/ Q#G 
 G)<)<Q)?@CI	6	6	6 	6 3		6
 	6( ##I. X	6/1E1EFFF9(#45994@K+uxx}}5*m$C&/C&HC& C& C&J < C$44 #L+ > &HCC%& <(A---Q5qX	6t 	k6X	6 X	6s   <>G>CG$$G-c           
         |j                   J | j                  |||j                    | j                  |j                  i |j                         y r3   )r   r   r  r7   r8   r  s       r9   r  zReplacementPatternEntry.apply  sN    &&222##D<u||<		
r<   N)
r   rw   r   r   r   z+Union[torch.fx.Graph, torch.fx.GraphModule]r7   zSequence[torch.fx.Node]r>   rT   r  )r1   r?   r@   rA   r  r   r  r4   r<   r9   r   r   !  sV    ,,\\\ G\ &	\
 
\ \|
r<   r   c                     yr   r4   )r   s    r9   _return_truerA    s    r<   c                F    t         j                  d| j                  |       y )Nz@Replacement pattern %s failed to apply due to shape mismatch: %s)loginfor1   )	search_fnr   s     r9   log_trace_failurerF    s    HHJ	r<   Fc                p   t         j                  |       |j                        }|s'|   j                  |rt	        |             yd       y||ryt        j                  dfd       t	        |      }|D ]&  |k(  s	|r yt        j                  dfd       ( |j                  |       y)aB  
    Check if a pattern is a duplicate. Because we ignore certain types in searching, but not
    in matching, use the graph to distinguish equivalent search patterns.

    Returns True if a duplicate is found and `skip_duplicates=True` is passed in. Errors if
    `skip_duplicates` is False and a duplicate is found.
    NFTc                     d  dS )NDuplicate pattern: z with no graphr4   )pattern_reprs   r9   r   z1check_and_add_duplicate_pattern.<locals>.<lambda>  s    ),~F r<   c                     d d  dS )NrI  z with duplicated match graph  r4   )	graph_strrJ  s   r9   r   z1check_and_add_duplicate_pattern.<locals>.<lambda>  s    ),7TU^T__`a r<   )r  r  rj   rl   r0   r[   _check)ry   r   seen_patternsskip_duplicatesequiv_pattern_reprsnew_graph_strrM  rJ  s         @@r9   check_and_add_duplicate_patternrS    s     (++G4L'++L9l#**3u:I EII}F	

 JM( 
		)a	

 }-r<   r4   c
           
         g t        j                         j                  j                         d f	d}
d	fd}t        u rt        j                         ryt        j                  d      5  |D cg c]*  }t        |t
        j                        xr |j                  , c}t         |      \  }}n}d}t        |t              r|n|gD ]D  }t        |t              st        ||r|j                   nd|j"                  |	      s< ddd       y t%        ||
|      }|j'                  |       |j(                  cddd       S c c}w # 1 sw Y   yxY w)
a  
    Create a replacement rule based on example functions that get traced
    to create patterns.  This supports both training and inference when
    run on a joint forward+backward graph.

    Args:
        search_fn: traced to give original pattern
        replace_fn: traced to give replacement graph
        example_inputs: example inputs for initial trace
        trace_fn: fwd_only or joint_fwd_bwd
        pass_dict: dict of passes to register to
        extra_check: additional check to run on match(using real shapes)
    c           
       	 t              }|D ]+  }|| j                  vst        d| d| j                          t        t        j                  j                  |D cg c]  }| j                  |    c}d             g }t        j                  j                  j                        5  t              D ]  \  }}t        |   t        j                        s%|r"t        |   j                        r
 ddd       yt        j                  |   j                         |   j!                         |   j                  |   j"                  |      |<   t%        j&                  |   j(                  |   j!                               D ]C  t        t        j*                        st-        fd|D              s3|j/                         E  }|s3|rdfd}	  ||z         }g }
t3        t5        t7        |      t7              z         |j8                  j:                        D ]  \  }}|t7        |      k  r|
j/                  |j<                         0|j8                  j?                  |      5  |j8                  jA                  ||t7        |      z
           }|jB                  |_        |jE                  |       |j8                  jG                  |       ddd        |
|z   }n
	        }tI        ||	      }| jK                         d
   }|J |jM                  |      }tO        |      r |      rz       | _(        t7        | j:                        dk(  rJ| jP                  j8                  j:                  D ]'  }tS        |jT                  | j:                  d
   d       ) 	 ddd       y	 ddd       yc c}w # t        $ r}	t1        |	       Y d}	~	ddd       yd}	~	ww xY w# 1 sw Y   xY w# t        $ r}	t1        |	       Y d}	~	ddd       yd}	~	ww xY w# 1 sw Y   yxY w)z
        Often shapes get burned into the pattern, so our initial match ran with
        `ignore_types=(int, ...)`.

        Recheck the match with the correct shapes.
        z_Not all inputs to pattern found in match.kwargs. Perhaps one of the inputs is unused? argnames=z, match.kwargs=c                     | j                   d   S r   r   )r   s    r9   r   z8register_replacement.<locals>.check_fn.<locals>.<lambda>0  s    QVVE] r<   NF)dtypedevicerequires_gradc              3  :   K   | ]  }t        |k7          y wr3   r   )r`   r  rb   s     r9   rc   z9register_replacement.<locals>.check_fn.<locals>.<genexpr>C  s       ?>?1!q&9?s   c                 >     | t        |       t              z
  d   S r3   )r   )args_newr7   rE  s    r9   search_fn_newz=register_replacement.<locals>.check_fn.<locals>.search_fn_newV  s#    ((3x=3t93L3N*OPPr<   )argnamesexclusive_arg_namesscalar_workaroundr   r+   r   r   T)r\  r   r>   r   )+r   r8   r   r[   r\   r   _dynamoutilsdetect_fake_moder  r   r  r   rW  empty_stridedsizestriderX  rW  rX  shapeSymIntr  rl   rF  r  ranger   r   r}   r  inserting_afterr  r  r  r   fx_to_patternr   r   r   r   ru   ro   )r   r^  r  sym_argsr  gradspecific_patternr]  specific_graphr   sym_arg_namesr  new_noder   specific_pattern_matchr   r7   rb   argnames_staticr_  r  
replace_fnrY  r`  rE  search_fn_patternr   s                   @@r9   check_fnz&register_replacement.<locals>.check_fn  s,    ( 	D5<<'"99A
/RWR^R^Q_a 	 HH089d#9;R
 (*]]  11$7 Z	$]3 /4d1gu||4 0a ?$	Z	 Z	 $11QQ("1gmm#Aw~~&*DG '__T!W]]DGNN<LM /%a63 ?CK? < %OOA.	//*  1#Q%)1-D)Q %'M*-c(mc$i78&,,22+ I; s8},)001C1CD$+11AA+N I'5';';'G'G (S]): ;(H /7mmHO'==hG*00;;KHI II   -x7H%)1)T)B
 $1"%(;&7	$  %%'*D###%5%;%;D%A"./K@V4W*2:t*D'u{{#q("44::@@ &%&VV%*[[^&3 sZ	 Z	t uZ	 Z	 :V ( %))Q7$QZ	 Z	L%I I ( %))Q7$EZ	 Z	@%AZ	 Z	s   !O/0P7 P7B*P7/P7%P7*O6BP7A(P )P7:	PB;P7P7	O=O8+P78O==P7 P
P7	P4P/"P7/P44P77Q c                     D cg c]  }| j                  |       }}t        dt        |       dz         D ].  }d| | vr n%|j                  | j                  d|              0 | r
J d|        |S c c}w )Nr+   	tangents_zleftover kwargs: )r  ri  r   rl   )r8   r  r7   r  rs  s       r9   r  z,register_replacement.<locals>.normalize_args  s    -<=T

4 ==q#f+/* 	5A1#f,KK

Yqc?34	5 9.vj99z >s   A7Ffunctionalize_rng_opsN)rP  )ry   r  r  r   rw   r>   r   )r8   r   r>   rz   )inspect	signature
parametersr   joint_fwd_bwdr[   is_inference_mode_enabledfunctorch_configpatchr   r  rY  gen_pattern_and_search_gmr   r  rS  r   rO  r   r  ry   )rE  rt  example_inputsr   r  r  r`  r_  ru  rP  rv  r  rU  ry   gmpattern_matcher_passrs  rY  s   `` ` ````       @@r9   register_replacementr    so   2 H)))4??DDFGOp pd =  **, 
		e	< "ES%
@AJq%,,';AOO;%
 $3!#KGR (GB %Z:J
	!  .0BC2 "BHH(66$3	 !5" " 
	! * )

 	$E" "%
" "s1    E%/E 4AE%9'E%!E%++E% E%%E.c                V   dd}t         j                         st        dt                |j                  }ddlm}  |j                  d      5  t        ||||      }d d d        t        j                  |       }	|t        vrd}
t        j                  |       nd	}
 |       }t        t         | d
z  |
      5 }|
dk(  r|j                  |       n|j                  d       |j                  |	       |j                  d       d d d        |S # 1 sw Y   xY w# 1 sw Y   |S xY w)Nc                    t        j                  d      } t        j                  d      j                  |       }g }t        t        j
                  j                        D ]_  }t        t        j
                  j                  |      }t        |t              s8t        |t        t        f      sO|j                  |       a dj                  |      }d| d}| | S )Nz            # This is an auto-generated file. Please do not modify it by hand.
            # To re-generate, run:
            # cd ~/pytorch && python torchgen/fuse/gen_patterns.py
            a               # mypy: ignore-errors

            # noqa: F401, E501
            {msg}
            import torch
            import torch._inductor
            import operator

            aten = torch.ops.aten
            prims = torch.ops.prims

            )msgz,
   z1from torch._inductor.pattern_matcher import (
   z,
)
)textwrapdedentr   dirr[   	_inductorpattern_matcherr0  r   rd  
issubclassrx   r+  rl   rx  )auto_generated_msgfile_templatepattern_matcher_importsr  attrformatted_importss         r9   get_file_templatez-_serialize_pattern.<locals>.get_file_template  s    %__
 !
 &'&
( 	  #%778 	5D5??::DAD$%*TK;U*V'..t4	5
 %MM*ABPQbPccij!2 344r<   z0Could not find serialized patterns directory at r   r#   Fry  )r  wr  z.pyz

r  r   )SERIALIZED_PATTERN_PATHis_dirr   r1   torch._functorchr$   r  gen_patternr  r  _serialized_patternsr  openwrite)unique_namerE  r  r   r`  r  pattern_namer  ry   serialized_pattern
write_moder  fs                r9   _serialize_patternr    s)   !5F #))+>?V>WX
 	
 %%L;				e	< ViCTUV .11'{1S//
  .
%'M	%<.(<<j	I QGGM"GGFO	"#	 N)V V Ns   D=ADDD(	fx_passesserialized_patternszvlist[tuple[Any, Iterable[Any], Callable[[Callable[..., Any], Iterable[Any]], torch.fx.GraphModule], Any, PatternExpr]]_known_precompiled_patternsc
                   t        |      }dt        j                  v rt        | ||||      }
nT|j                  }t        j                  d|       }|rt        ||       st        j                  d|        t        ||       }
t        j                  |      D ]'  }t        |t              s|j                  !d |_        ) t         j#                  |||||
f       t%        |||||||||
|	
       y )NPYTORCH_GEN_PATTERNSz.torch._inductor.fx_passes.serialized_patterns.zDPrecompiled pattern %r not found. Run torchgen/fuse/gen_patterns.py.)ru  rP  )r   osenvironr  r1   	importlibimport_moduler  rC  warningr0  ri  	tree_iterr   r(   constantr  rl   r  )r  rE  rt  r  r   r  r  r`  r_  rP  patr  r   r   s                 r9   gen_register_replacementr  #  s    >*N+ NH>O
 !))##<\NK
 ;/KKV a%/  c:&3<<+C
  CL   &&	NH.?E 'r<   ry  c                r   g t        j                  |       j                  j                         }|i }g }d}|D ]4  }||v r|j	                  ||          |j	                  ||          |dz  }6  || |      }	t        |	t        t        t        t        j                  t        j                  f|||      |	fS )Nr   r+   )ignore_typesr^  r`  r_  )r|  r}  r~  r   rl   rk  intfloatr   r[   rX  rW  )
rE  r  r   r`  r_  r^  flat_inputs	input_idxargname	search_gms
             r9   r  r  [  s     A""9-88==?@H KI ''09:~i89NI K0IudELL%++F/ 3	
 		 	r<   c                &    t        | ||||      d   S r  )r  )rE  r  r   r`  r_  s        r9   r  r  ~  s&     %>8->@S	 	r<   r  c                     d fd}|S )z
    Register an aten to inductor IR replacement pattern.  The decorated
    function is saved and then called a lowering time allowing direct
    pattern to inductor IR conversion.
    c                n    t        |       sJ t        |       j                         d| _        | S )Nry   r  r
  r  T)r2  r  r  _inductor_lowering_functionr
  r  	pass_dictry   r   s    r9   	decoratorz,register_lowering_pattern.<locals>.decorator  s;       g	

(9g(
..2+r<   r
  r	  r>   r	  r4   ry   r  r  r   r  s   ```` r9   register_lowering_patternr    s      r<   c                     d fd}|S )zk
    Register a pattern that runs a function on the FX graph, allowing
    custom transformation code.
    c                `    t        |       sJ t        |       j                         | S )Nr  r  )r2  r  r  r  s    r9   r  z)register_graph_pattern.<locals>.decorator  s3       g	

(9g(
.r<   r  r4   r  s   ```` r9   register_graph_patternr    s      r<   c                B    |t        t        | j                              u S r3   )r   iterr}   )r   r   s     r9   is_start_of_fx_graphr    s    4U[[)***r<   z6(?<!_)(_$|_[.]|(\b|_)(set|enter|exit|seed)(\b|_))(?!_)c                    | j                   dk7  ry| t        j                  j                  j                  j
                  t        j                  j                  j                  j
                  fv S )NinductorF)r  r[   opsr  accumulate_grad_defaultresize_storage_bytes_)r7  s    r9   "fixme_incorrect_inductor_schema_opr    sW    	||z! 		++33		0088  r<   c                   t        | j                  t        j                  j                        r5t        | j                        s | j                  j                  j                  S t        | j                  t        j                  j                  j                        ry| j                  dk(  rAt        | j                        sJ t        j                  | j                  j                        rLy| j                  dk(  r<t        | j                  t               sJ t        j                  | j                        ry| j"                  j%                  d      d uS )NFr  Tr  out)r   r  r[   r3  r<  r  _schema
is_mutable_higher_order_opsauto_functionalizeAutoFunctionalizedr7  r2  _mutation_op_researchr1   r0   r8   rj   r-  s    r9   is_mutation_opr    s    UZZ**0={{""---	U,,??RR
 ww/!$$$!!$++"6"67	M	!$++s+++!!$++.;;??5!--r<   c                    d| j                   v sJ d|j                   v sJ | j                   d   |j                   d   k(  S Nmutation_region_idr   )r  r  s     r9   same_mutation_regionsr    sE    166)))166)))66&'1662F+GGGr<   c                ,   |}d|j                   vr3t        | |      s'|j                  }d|j                   vrt        | |      s'|j                   j                  dd      }||ur0|j                  }t        |      r|dz  }||j                   d<   ||ur0|S )Nr  r   r+   )ro   r  prevrj   r   r  )r   r   r   r  s       r9   get_mutation_region_idr    s    A
aff
,5I%QR5SFF aff
,5I%QR5S$8!<
4-FF!!#'9#$	 4-
 r<   c                V    dt        t        | j                              j                  vS r  )r   r  r}   ro   r   s    r9   "should_compute_mutation_region_idsr    s!    tD,='>'C'CCCr<   c                h    d}| j                   D ]!  }t        |      r|dz  }||j                  d<   # y )Nr   r+   r  )r}   r  ro   )r   r  nds      r9   compute_mutation_region_idsr    s=    kk ;"!#(:$%;r<   c                  B     e Zd Z	 d	 	 	 d fdZddZddZd	dZ xZS )
r  c                    t         |           t        t              | _        || _        t        t              | _        y r3   )r   rR   r   r   patternsrs   rO  )r6   rs   r   s     r9   rR   zPatternMatcherPass.__init__  s:     	  	 # >I=Nr<   c                     | j                   |   S r3   )r  )r6   items     r9   r  zPatternMatcherPass.__getitem__  s    }}T""r<   c                   | j                   syt        |t        j                  j                        r|j
                  }nJt        |t        j                  j                        r|}|j                  }nt        dt        |             t        |      rt        |       t        j                  t        |      }d}g }d}| j                   D ]0  \  }}|dk(  rd}|j                  |j!                  ||d             2 |r"|j                  |j!                  dd             | j"                  | j"                  nd}	t        |t        j                  j                        sJ t%        ||	      5  t'        t(        j*                  j-                  |      d	      D ]  }
t/        |
      }|
j0                  dk(  r|
j0                  |f| j                   vr9t3        |
d
      rG| j                   |
j0                  |f   D ]-  }|
j4                  r t|j6                  j9                  |
      }t;        |      r,t=        t?        tA        ||jB                                    dk7  rdtD        jF                  jI                  d      |
jJ                  k(  r-tL        jO                  d|
|
jP                  ||j6                         t;        |      s|jS                  |      s|dz  }|jU                  |||
       tV        d   dxx   dz  cc<   tV        d   dxx   t=        |jB                        z  cc<   0  	 d d d        |S # 1 sw Y   |S xY w)Nr   zJThe input to PatternMatcherPass must be a GraphModule or a Graph, but got Fr  T)r7  r  sort)r7  r  r  )reverse)allow_cpu_inputsr+   !TORCHINDUCTOR_PATTERN_MATCH_DEBUGz
%s%s %s %sr  pattern_matcher_countpattern_matcher_nodes),r  r   r[   r\   r<  r   Graphowning_moduler   rd  r  r  r   r   r  rl   
find_nodesrs   r    sortedrW  rX  from_iterablerI  r7  r-   r   ry   r   r   r   r!   rv  r}   r  r  rj   r  rC  r  r7   r  r  r   )r6   r  r   get_mutation_region_id_partialr  r}   has_call_moduler7  r  rs   r   entryr   s                r9   r  zPatternMatcherPass.apply  s   }}b%((../HHEEHHNN+E$$B\]abd]e\fg  .e4'.)2):):"E*
& -- 	QJB]""&U--F-OP		Q
 LL))])GH&*nn&@DNNFW	"ehh22333#B	2  	Vy<<UCTR V'-77m+(= 
 9PUV!]]DGGV+<= VE||++D1A !&s+I177'ST 
 !zz~~&IJdiiWL$		1emmT{u'8'8';
Aud3 ,-DEJE ,-DEQWWUE'VV 	VB C 	VB s   .EL:L:AL::Mc                8    | j                   j                          y r3   )r  clearrQ   s    r9   r  zPatternMatcherPass.clearU  s    r<   r3   )rs   Optional[str]r>   rT   )r  r  r>   r  )r  +Union[torch.fx.GraphModule, torch.fx.Graph]r>   r  rS   )r1   r?   r@   rR   r  r  r  r   r   s   @r9   r  r    s2     $(O O 
O #>@r<   r  c                     t         r3   rE  r   s     r9   _not_implementedr  Y  s    
r<   c                  	
 |xs i }|j                         D ci c]  \  }}||
 c}}
t        
      t        |      k(  sJ 	 d	 	 	 	 	 d
fdt        j                         	 G 	fddt        j
                  j                        }t        | t        j
                  j                        sJ  ||       j                         }t        |t              st        t        j                  |            S |S c c}}w )z
    Convert an FX graph into a PatternExpr.  This is useful for simple
    patterns that can only match single functions and fixed-length lists.
    c                    ||n}t        | t        t        f      r| v rt        |          S t	        |       |v r
t               S t        | t              rt        d | D              r| r
t               S | S )Nc              3  <   K   | ]  }t        |t                y wr3   )r   r  )r`   ys     r9   rc   z5fx_to_pattern.<locals>.process_arg.<locals>.<genexpr>x  s     &I!z!W'=&Is   )r   r  r  r  rd  r  r   r  )rU  ignore_types_overridecurrent_ignore_typesr  inv_scalar_workarounds      r9   process_argz"fx_to_pattern.<locals>.process_argn  sy     &;%F!L 	 a%&10E+E3A6777**9a3&Iq&I#Ia9r<   c                  l     e Zd ZeZeZeZ	 	 	 	 	 	 	 	 dfdZ	 	 	 	 	 	 	 	 dfdZd fdZ	 xZ
S ) fx_to_pattern.<locals>.Converterc                    t              }|t              k  r|   }n1r|j                  d      sJ |}nt        j                  dd|      }|}|v rt        |      S t        |      S )Ntangentz_\d+$r@  )r   r   
startswithresubr%  r  )	r6   r  r7   r8   r   r  r^  argnumr_  s	         r9   r  z,fx_to_pattern.<locals>.Converter.placeholder  su     VA3x= {((333"f5***400!$''r<   c                b   	}|t         j                  k(  rt        d D              f	 	 	 	 	 d	fd}|}t        j                  |||f      \  }}t
        v r>|D cg c]
  } ||       }}|j                         D ci c]  \  }}| ||       }}}t        |g|i |S c c}w c c}}w )Nc              3  2   K   | ]  }|t         us|  y wr3   )r  )r`   ts     r9   rc   zAfx_to_pattern.<locals>.Converter.call_function.<locals>.<genexpr>  s      Q1C<Qs   c                     | |      S r3   r4   )rU  r  r
  s     r9   process_arg_fn_implzKfx_to_pattern.<locals>.Converter.call_function.<locals>.process_arg_fn_impl  s     'q*?@@r<   rU  rK   r  zOptional[Sequence[type[Any]]]r>   zUnion[T, KeywordArg, Ignored])r;  r,  r   ri  rj  r   rp   r  )
r6   r  r7   r8   process_arg_fnr  r  ra   r  r
  s
           r9   r  z.fx_to_pattern.<locals>.Converter.call_function  s     )N))) LQ Q#/Q LAA+HA
 3A "5!??>D&>JLD&|#378aq)88;A<<>J41a!^A..JJ8888 9Js   B&B+c                l   t         |   |      }|j                  dk(  rzt        |t              rj|j
                  d   }t        |t              sJ t        |      t        |      k(  sJ t        ||      D ]  \  }}t        |j                        |_	        ! |S t        |j                        |_	        |S )Nr  r   )
r   r  r7  r   r   r7   r   r   r  r   )r6   r   rvr7   rr   r   s         r9   r  z)fx_to_pattern.<locals>.Converter.run_node  s    !!$BttxJr5$9vvay!$
3332w#d)+++!"dm -FAs!#))nAG- I qww<Ir<   )r  r0   r7   r   r8   Mapping[str, Any]r>   z&Union[ExclusiveKeywordArg, KeywordArg])r  r0   r7   r   r8   r  r>   rx   )r   r   r>   r   )r1   r?   r@   r  r  r  r!  r  r  r  r   )r   r^  r  r_  r  r
  s   @r9   	Converterr  ~  sr    &&#	(	(  	( &		(
 4	((	9	9  	9 &		9
 	96
	 
	r<   r  r3   r  )rp   r   rW  r  r[   r\   r8  r   r<  r  rx   r  ri  tree_leaves)r  r  r^  r`  r_  ra   rb   r  ry   r  r	  r
  s    `` `    @@@r9   rk  rk  ]  s     */R.?.E.E.GHdaQTH$%->)???? FJ%B	& __F> >EHH(( >@ b%((..///m!Gg{+!&"4"4W"=>>No Is   DT)r   get_decomp_fnc                  t               5  | |       n	t               } t        | |d      | }ddd       ddlm} |r, |j
                         |j
                  j                          j                          |S # 1 sw Y   OxY w)z>Build a normalized inference graph, for use with fx_to_patternNreal)tracing_moder+   remove_noop_ops)r   r,   r   fx_passes.post_gradr%  r   eliminate_dead_code	recompile)rI   r7   r   r   decompositionsr  r%  s          r9   r   r     s     
"	# E,8MO>Q>S 	 >WRf=tD	E 5!
$$&LLNIE Es   %A>>Bc           
        d	 	 	 	 	 	 	 	 dfd}t         j                  j                  d      5   t        | d |t	               dd      |  ddd       sJ ddlm}  |j                         dd	lm	} t               }t        t         j                  j                  j                  j                  t!        d
      t!        d            }t#        ||t$              j'                  |j(                         |j+                  j                         t         j,                  j                  j/                         j                  _        j                  j3                          j5                          S # 1 sw Y   *xY w)z=Build a normalized training graph, for use with fx_to_patternNc                <    rJ t        |       t        | |fi |S r3   )clone_graphr'   )joint_graphinputsr8   r  s      r9   record_joint_graphz)joint_fwd_bwd.<locals>.record_joint_graph  s(     v% f???r<   c                    t        |       S r3   )r&   )gr  s     r9   r   zjoint_fwd_bwd.<locals>.<lambda>  s    + r<   TF)partition_fnr)  keep_inference_input_mutations
enable_logr+   r$  )pointless_viewr   re  )ry   r
  r  )r-  rJ   r.  r   r8   r   r>   z1tuple[torch.fx.GraphModule, torch.fx.GraphModule])r[   _guardstracingr%   r,   r&  r%  r   fx_passes.joint_graphr5  r  r  r  atenviewr  r  r  rA  r  r  r  r\   CodeGen_codegenr'  r(  )rI   r7   r/  r%  r5  matcher_passry   r  s          @r9   r  r    sJ    *.B@)@3@@LO@	:@ 
		t	$ 	
++.0+/	
 	 I24BHH5%'L		##Z%6
68JG \h|$$%rxx  ..0BHHHH  "LLNI? s   E77Fc                    g }t         j                  j                  | j                  | j                  f|j
                         |S r3   )r[   r\   r   r7   r8   rl   )r   r7   s     r9   r  r    s1    )+D	HHaffahh'5Kr<   c                T   t        t        | j                              }t        t        j
                  j                            }t        t               }d }|r|j                         }t        |      D cg c]	  }||vs| }}|r||d      j                  |       n^|j                  |       |r|j                  |ur|j                  |       |}|j                  t        |j                  |d                   |r|s!t        |      t        | j                        k(  sJ y c c}w )Nr  r4   )r   r   r}   r!   r[   r\   rH  r   r  r  rl   r  r   r   r   )r   pendingreadywaitingcursorr   rU  waiting_fors           r9   stable_topological_sortrE    s     8EKK()G uxx}}%'E $G F
{{}"'+@Q%q@@ KO$++D1IIdO&++T1d#F NN8GKKb$9:;   3u:U[[)99999 As   4	D%>D%c                t     t        j                  d      t        j                         d fd              }|S )z0Wrapper around lazy init functions in fx_passes/Nc                 :   t         d   j                         } t        j                  j	                  d       5  t               5  t               5          }d d d        d d d        d d d        | t         d<   S # 1 sw Y   $xY w# 1 sw Y   (xY w# 1 sw Y   ,xY w)Nr  )r   rk   r[   r6  r7  r   r)   )counters_refr  rI   s     r9   	lazy_initz%init_once_fakemode.<locals>.lazy_init;  s      
+002]]""4( 	*@*B 	NDT 	TF	 	 	  ,	 	 	 	 	 	s;   BBA9BB9B>BB	
BB)r>   r   )r   r  r  )rI   rI  s   ` r9   init_once_fakemoderJ  8  s;     __R	  	 r<   c                     d fd}|S )z2Function for extra_check to put pass behind a flagc                $    t        t              S r3   )r0  r$   )r   r  s    r9   
flag_checkzconfig_flag.<locals>.flag_checkN  s    vt$$r<   )r   rw   r>   r   r4   )r  rM  s   ` r9   config_flagrN  K  s    % r<   c                L     G d dt               } ||       j                         S )Nc                        e Zd Zd fdZ xZS )clone_graph.<locals>.CopyGraphc                J   t         |   |      }t        |t        j                  j
                        rn|j                  j                  j                  |j                         | j                  j                  j                  |j                  d       |j                  _        |S r3   )r   r  r   r[   r\   Proxyr   ro   rn   	new_graph_graph_namespacer  r  )r6   rr   rq  r   s      r9   r  z'clone_graph.<locals>.CopyGraph.run_nodeV  sq    w'1H(EHHNN3""))(--8%)^^%D%D%P%PMM4&" Or<   )rr   r   r>   r   )r1   r?   r@   r  r   r   s   @r9   	CopyGraphrQ  U  s    	 	r<   rV  )r*   	transform)input_graphrV  s     r9   r,  r,  T  s$    K  [!++--r<   c                    t        | j                        |kD  r| j                  |   S |y | j                  j                  |      S r3   )r   r7   r8   rj   )r   
arg_number
kwarg_names      r9   get_arg_valuer\  f  s?     499~
"yy$$		{{z**r<   c           	        |g}t        |t        j                  j                        r7|j	                  |j                         D cg c]  }t        ||       c}       | D cg c]  }|j                  |v s| c}S c c}w c c}w r3   )r   r[   r3  r4  r   r5  r0  r  )r}   rI   r,  r1  r   s        r9   filter_nodesr^  q  sf    $C"ejj112

",,.IhGB)IJ"9TdkkS&8D99 J9s   A>#B7Bc                    | j                   dk(  rPt        | j                  t              sJ t	        | j
                  j                  | j                        j                  S | j                  S )zFor call_function and call_method, we directly use the target function;
    For call_module, the target is string, and we treat the module class
     as a function.
    r  )r7  r   r  r0   r   r   r  r   r-  s    r9   rI  rI  y  sO    
 ww-$++s+++114;;?III;;r<   )r@  )rq   r{   rr   r   rs   r0   r>   rT   )r   r   r>   zTypeIs[Match]r{  )rE  r	  r   r   r>   rT   r  )
ry   rx   r   zOptional[torch.fx.Graph]rO  zdict[str, list[Optional[str]]]rP  r   r>   r   )rE  r/   rt  rC   r  Iterable[Any]r   rG   r  r  r  r  r`  )Union[dict[str, Union[float, int]], None]r_  Sequence[str]ru  zUnion[PatternExpr, None]rP  r   r>   r   )r  r0   rE  r/   r  r   r   rG   r`  ra  r>   rx   )r  r0   rE  r/   rt  rC   r  r`  r   rG   r  r  r  r  r`  ra  r_  rb  rP  r   r>   rT   )Nr4   )rE  r/   r  r   r   rG   r`  ra  r_  rb  r>   z(tuple[PatternExpr, torch.fx.GraphModule])rE  r/   r  r   r   rG   r`  ra  r_  rb  r>   rx   )
ry   rx   r  r  r  r  r   r   r>   z2Callable[[Callable[..., Any]], Callable[..., Any]])r   r   r   r   r>   r   )r7  ztorch._ops.OpOverloadr>   r   rN  )r  r   r  r   r>   r   )r   r   r   r   r>   r  )r   r   r>   r   )r   r   r>   rT   )r7   r   r8   r   r>   r   )r4   r4   Nr4   )r  r   r  zSequence[type[Any]]r^  rb  r`  ra  r_  rb  r>   rx   )
rI   r	  r7   r   r   r   r   zOptional[Callable[..., Any]]r>   rJ   )rI   r	  r7   r   r>   rJ   )r   r   r>   zlist[torch.fx.node.Argument])rI   r	  r>   zCallable[[], Any])r  r0   r>   zCallable[[Match], Any])rX  rJ   r>   rJ   r3   )r   r   rZ  r  r[  r  r>   r   )r}   zIterable[torch.fx.Node]rI   r   r>   r|   )r   r   r>   ztorch.fx.node.Target)r   
__future__r   r   dataclassesr   r  r|  rW  loggingr;  r  r  r  r  abcr   r   collectionsr   collections.abcr   r   r	   r
   r   pathlibr   r   r   r   r   r   r   r   typing_extensionsr   r   r[   torch._guardstorch.fxtorch.utils._pytreerb  _pytreeri  torch._dispatch.pythonr   torch._dynamo.utilsr   torch._prims_commonr   torch._subclasses.fake_tensorr   "torch.fx.experimental.proxy_tensorr   %torch.fx.experimental.symbolic_shapesr   torch.fx.graph_moduler   torch.fx.immutable_collectionsr   r   (torch.fx.passes.graph_transform_observerr    torch.utils._ordered_setr!   
_functorchr$   r  _functorch.aot_autogradr%   r&   _functorch.partitionersr'   _subclassesr(   r)   r\   r*   r@  decompositionr,   loweringr-   	getLoggerr1   rC  r  r9  primsConstantrH  r   r/   rC   rG   rK   r   Targetr0   FnsTyperM   rO   ru   rw   r   r   r   r   r   rx   r  r  r  r%  r+  r   _SimpleSpecrP  r  r  r  r  r  r  r  r  r  r  r  r  	dataclassr  r  r  r   rA  rF  rS  r  r  r  __file__parentr  r  rA   r  r  r  r  r  r  r  compiler  r  r  r  r  r  r  r  r  rk  no_gradr   enable_gradr  r  rE  rJ  rN  r,  _seen_patternsr\  r^  rI  r4   r<   r9   <module>r     s  !F #         	 	   # # N N  N N N *    $ $ ; ( 0 @ 6 G + I K / 3 C 7 4   . ; g!yy~~		x./>x >> >#h # CL $$c)
*? ? : IK

(5
BE
	
4 D, 8 E;&''
 '
T 1#  1F-+ -k E E(E+ E4O
+ O
d CHol
k l
^?    (, * * ,
[ ,
^F
 F
R#
; #
L4 4n!X ! : : :: 
< 
 
 	= 	= 	= i
l i
 i
X "	''#' 2' 	'
 
'` ,8CG)+26! " 	
 @ ) A ' 0  
D 'z#( GGG "G 	G
 AG GT x.//+=@UU     $ ,8CG)+!555 5 "	5
 5 @5 )5 A5 '5 5 
5p e4
 DH)+!  A	
 ' . 5L DH)+				!		 		 A			
 '		 		 ,8
 ( 	
  84 ,8
 ( 	
  8,+ "**VW	.(H
D;U Up )+ CG)+e3e%e e A	e
 'e eP 
 #'26
  	
 0  2 + +\!:H&. !C" GK+
+%(+6C++:r<   