
    Vh                        d Z ddlZddlZddlZddlZddlmZmZ ddlm	Z	 ddl
ZddlmZmZmZ ddlmZmZmZmZmZ ddlmZmZ dd	lmZ dd
lmZmZmZmZm Z m!Z!m"Z"m#Z# ddl$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1 ddl2m3Z3m4Z4m5Z5 ddl6m7Z7 ddl8m9Z9 ddl:m;Z; ddl<m=Z= e	rddl>m?Z? d dZ@edeAdej                  j                  fd       ZDd ZE G d de5      ZF G d de=      ZG G d deG      ZH G d deG      ZIy)!aQ  
This module implements variable tracking for PyTorch nn.Module instances during Dynamo tracing.

It provides specialized handling for different types of nn.Module instances through several key classes:

- NNModuleVariable: Handles instance-specific module tracing, specializing on module id() and placing
  parameters directly on the torch.fx.GraphModule. This creates one graph per module instance.

- UnspecializedNNModuleVariable: Provides class-level module tracing, treating nn.Modules like other
  user-defined objects and passing parameters as inputs to the FX graph. This creates one graph per
  module class.

- UnspecializedBuiltinNNModuleVariable: Specifically handles built-in PyTorch modules (e.g. nn.Linear)
  with appropriate optimizations.

- FSDPManagedNNModuleVariable: Special handling for FSDP-wrapped modules with modified guarding behavior
  and parameter handling.

The module integrates with Dynamo's broader tracing functionality to handle module method calls,
parameter access, hooks, and other nn.Module behaviors while maintaining proper scoping and guarding
of module state.
    N)contextmanagernullcontext)TYPE_CHECKING   )graph_break_hintstrace_rules	variables)raise_observed_exceptionunimplementedunimplemented_v2UnspecializeRestartAnalysisUnsupported)GuardBuilderinstall_guard)GenerationTracker)
AttrSourceConstDictKeySourceDictGetItemSourceFSDPNNModuleSourceGetItemSourceNNModuleSource"UnspecializedBuiltinNNModuleSourceUnspecializedNNModuleSource)get_custom_getattrget_fake_valueis_lazy_moduleis_namedtupleis_safe_constantistensoristypennmodule_has_hooksobject_has_getattributeproxy_args_kwargsset_example_valueunpatched_nn_module_callunpatched_nn_module_call_impl   )typestrValueMutationNewVariableTracker)invoke_and_store_as_constant)LazyVariableTracker)SliceVariable)UserDefinedObjectVariable)InstructionTranslatorc                     t        |d      rg fdt        ||      \  }}|D cg c]
  } |       }}|j                         D 	ci c]  \  }}	| |	       }
}}	|j                  |||
       yyc c}w c c}	}w )a~  
    Fairly coupled helper used by NNModuleVariable and UnspecializedNNModuleVariable.

    Used to cause lazy module to be initialized (and delete its init hook) before tracing. Especially
    useful now that 'allowed' modules graph-break on hooks, calling this first ensures there is no hook
    by the time we trace __call__ and thus no graph-break for lazy allowed modules.
    _initialize_hookc                    t        |       r t        |       fd| D         S t        | t              r*| j	                         D ci c]  \  }}| |       c}}S t        | t
        t        t        f      r t        |       fd| D              S t        | t        j                  j                        rt        | j                        S | S c c}}w )Nc              3   .   K   | ]  } |        y wN .0elemconvert_to_fakes     Q/home/dcms/DCMS/lib/python3.12/site-packages/torch/_dynamo/variables/nn_module.py	<genexpr>zBinitialize_lazy_module.<locals>.convert_to_fake.<locals>.<genexpr>]   s      E4!6 E   c              3   .   K   | ]  } |        y wr4   r5   r6   s     r:   r;   zBinitialize_lazy_module.<locals>.convert_to_fake.<locals>.<genexpr>a   s     Ct4Cr<   )r   type
isinstancedictitemslisttuplesettorchfxProxyr   node)xkvr9   txs      r:   r9   z/initialize_lazy_module.<locals>.convert_to_fake[   s    QtAw E1 EFFAt$:;'')D$!Q?1--DDAeS12tAwCCCCAuxx~~.%affb11 Es   CN)hasattrr#   rA   _infer_parameters)rL   modargskwargs
proxy_argsproxy_kwargsarg	fake_argsrJ   rK   fake_kwargsr9   s   `          @r:   initialize_lazy_modulerW   Q   s     s&'
	 $5T6#B 
L5?@c_S)@	@9E9K9K9MNAq/!,,NNc9k:# ( ANs   A7A<
module_keyrO   c              #   "  K   |j                         }|j                  j                  |d      }|dkD  r|  d| n| } 	 ||j                  f|j                  | <   |dz   |j                  |<   d  |j                  | = y # |j                  | = w xY ww)Nr   @r'   )name	num_callsget	__class__nn_module_stack)rX   sourcerL   rO   fully_qualified_namer\   s         r:   record_nn_module_stackrb   m   s     !;;=  !5q9I09AJ<q,:J+*>)N:&-6])*z*Bz*s   ;B1A= /B=BBc                    | rd|j                   v rt        |j                   d         rk|j                   d   }t        | d      }t        |      t        j
                  u rt        |d      }t        |j                  t        j                               y t        | j                  t        j                  t        j                  d                   y y )Nforward__func__)attr)__dict__callabler   r>   types
MethodTyper   
make_guardr   CLOSURE_MATCH	functoolspartialNOT_PRESENT_IN_GENERIC_DICT)r`   rO   fwdforward_sources       r:   &guard_to_detect_forward_monkeypatchingrr   z   s     $#,,y2I)J,,y)C'	:NCyE,,,!+NJ!G.33L4N4NOP !!%%$@@y     c                       e Zd Zddddhej                  Zdededej                  j                  ddf fdZ
d Zd	 Zd
 Z	 	 ddZd ZdddeddfdZd Zd ZddZd ZddZ	 	 	 	 	 	 ddZ	 d	 	 	 	 	 d fdZ xZS )NNModuleVariablemodule_typerX   valuenn_module_stack_sourcereturnNc                     t        |   di | || _        || _        || _        | j
                  sJ | j
                  | _        y )Nr5   )super__init__rv   rX   rw   r`   rx   )selfrv   rX   rw   rQ   r^   s        r:   r|   zNNModuleVariable.__init__   sC     	"6"&$
{{{&*kk#rs   c                 6    | j                   xs | j                  S r4   rx   r`   r}   s    r:   get_nn_module_stack_sourcez+NNModuleVariable.get_nn_module_stack_source       **9dkk9rs   c                     || _         y r4   rx   r}   r`   s     r:   set_nn_module_stack_sourcez+NNModuleVariable.set_nn_module_stack_source   
    &,#rs   c                     | j                   S r4   )rv   r   s    r:   python_typezNNModuleVariable.python_type   s    rs   rL   r/   c                      y r4   r5   )r}   rL   r`   submod	key_extraoptionss         r:   _wrap_submodulez NNModuleVariable._wrap_submodule   s     	rs   c                 z   |j                   j                  | j                        }t        |t        j
                  j                        rg }|j                         D ]{  \  }}t        j                  j                  |      }|j                   j                  || j                  |t        t        | j                  |                   |j                  |       } |S t        |t        j
                  j                   t        j
                  j"                  t        j
                  j$                  f      sJ t'        |             | j                  sJ g }t)        |      D ]Z  \  }}|j                  |j                   j                  || j                  |t        t        | j                  |                         \ |S Nr`   )outputget_submodulerX   r?   rE   nn
ModuleDictrA   r	   ConstantVariablecreateregister_attr_or_moduler   r   r`   append
ModuleListParameterList
Sequentialr(   	enumerate)r}   rL   baseresultr[   r   name_varidxs           r:   unpack_var_sequencez$NNModuleVariable.unpack_var_sequence   sj   yy&&t7dEHH//0F $

 (f$55<<TB		11OO)-T*JK	 2  h'( M588&&(>(>@S@ST
 	4=	 
 {{{$T? 	KCMM		11OO)-S*IJ	 2 	 rs   r[   r*   c                 ,   |j                   j                  | j                        }t        ||      }t	        t        t        | j                  |            j                  t        j                               t        j                  j                  |      S r4   )r   r   rX   rM   r   r   r   r`   rk   r   HASATTRr	   r   r   )r}   rL   r[   rO   r   s        r:   call_obj_hasattrz!NNModuleVariable.call_obj_hasattr   sp     ii%%doo6d#:dkk489DD$$	

 ))0088rs   c                 f    |j                   j                  | j                        }t        |dd      S )NtrainingF)r   r   rX   getattrr}   rL   rO   s      r:   is_trainingzNNModuleVariable.is_training   s)    ii%%doo6sJ..rs   c                     |j                   j                  | j                        }t        j                  |       |j
                  j                  dk7  rt        j                  t        |             t        )zIRestart analysis treating this module as an UnspecializedNNModuleVariabler|   )
r   r   rX   r   tagf_codeco_namemark_class_dynamicr>   r   r   s      r:   convert_to_unspecializedz)NNModuleVariable.convert_to_unspecialized   sT    ii%%doo6c" 99
*00c;))rs   c                 ~   |j                   j                  | j                        }t        |      rt	        d       |j                   j
                  j                  | |      rC|j                   j
                  j                  | |d      }t        |t        j                         S t        j                  |d      }||v S )Nz-NNModuleVariable with custom __getattribute__T)
deleted_okrg   )r   r   rX   r"   r   side_effectshas_pending_mutation_of_attr	load_attrr?   r	   DeletedVariableobject__getattribute__)r}   rL   keyr   mutated_attr	base_dicts         r:   has_key_in_generic_dictz(NNModuleVariable.has_key_in_generic_dict   s    yy&&t7"4(IJ99!!>>tSI9911;;D#RV;WL!,	0I0IJJJ++D*=	irs   c                 @   t        |      rt        d       t        |d      }|yt        |t        j
                        st        d       dt        |d      i}t        j                  || fi |j                  |t        j                  j                  |      gi       S )zDCheck for a __getattr__ and handle it specially if it is implementedz6torch.nn.Module with a custom __getattribute__ definedT)ignore_nn_module_getattrNz6torch.nn.Module with a non-function custom __getattr__r`   __getattr__)r"   r   r   r?   ri   FunctionTyper   r	   UserMethodVariablecall_functionr   r   )r}   r   rL   r[   
obj_source
getattr_fnr   s          r:   _custom_getattr_fallbackz)NNModuleVariable._custom_getattr_fallback  s    "4(RS'tL
*e&8&89RSZ
MBC++JHHVV++224892
 	
rs   c                 >   | j                   xr t        | j                   |      }|j                  j                  | j                        }t
        j                  |d      }d}t               }t        j                  |j                        D ]+  }|j                  |j                  j                                - | j                   st        d       |dk(  rt        j                   | ||      S ||v r||   }	nYd|v r||d   v r||vr	|d   |   }	nAd|v r||d   v r	|d   |   }	n-d|v r||d   v r	|d   |   }	n	 t        j"                  ||      }	d}|d
k(  rt+        | j                   |       |dk(  r#|s!t        j,                  |j                  |      S |rat/        j0                  |	t3        |            }t5        |t6        t8        f      r)|j;                  t        | j=                         |             |S t?        	t@              rk| j                   r,t        t        | j                   d      |      }t        |d      }t        jB                  |	jD                  |      jG                  || gi       S t?        |	tH              r>t        jJ                  |	jL                  t        jN                  tQ        |            |      S t?        |	tR              r&t        jB                  |	jU                  |      |      S t?        |	tV        jX                        rt        jJ                  |	| |      S t[        |	      st]        |	      r t/        j0                  ||	t3        |            S t_        ddta        |       d| dta        |	       dta        |	       dd| dta        |	       dta        |       ddgtb        jd                         t        j                   | ||      S # t$        $ r9 | j'                  |||| j                   	      }
|
|
cY S t)        t$        |       Y w xY w)Nrg   TzGETATTR with no sourcer   _modules_parameters_buffersF)r   rL   r[   r   rd   r^   fgetz$Unsupported nn.Module attribute typeznn.Module subclass: z, name: z, attribute type: z>Dynamo does not support tracing nn.Module attributes of type ``zRefactor your code so that `z	` (type `z`) is not an attribute of `zqCurrently supported attribute types are methods, classmethods, staticmethods, properties, constants, and tensors.)gb_typecontextexplanationhints)3r`   r   r   r   rX   r   r   rD   inspectgetmror^   updaterg   keysr   r	   GetAttrVariablegetattr_staticAttributeErrorr   r
   rr   UserDefinedClassVariabler*   buildr   r?   ru   UnspecializedNNModuleVariabler   r   r    propertyUserFunctionVariabler   r   classmethodr   re   r.   r>   staticmethod__get__ri   r   r   r   r   r(   r   SUPPORTABLE)r}   rL   r[   r`   r   r   object_memberall_class_attribute_namesrI   subobjr   outs               r:   var_getattrzNNModuleVariable.var_getattr  s   >DKK!>yy&&t7++D*=	$'E!/ 	@A%,,QZZ__->?	@ {{23:,,T4GG9t_F)#	*--55z*40Fi'DIm4L,L}-d3F9$:1F)Fz*40F //d; % 924;;E;}55dnnVTT!''FN64JKC# 02OPQ ..t>>@$G J fh';;'
4;;(LdSF'7F 55KK!  -dHb12 , 33OO77T
C! 
 - 55NN4(   2 23 33FDPP!&)Xf-=&,,R9OPP B274=/$Oabijpbqars"`ahioap`qqr s6tfIgfoEVVqryz~r  rA  AB  C> +66		
 ((tFCCM " 66"4DKK 7  %!M("s   8O ,PPPc                    |j                   j                  | j                        }t        | j                  | j	                         ||      5  t        |      }t        |t        j                  j                        r|j                  j                  t        j                  j                  j                  u rt        |      r| j                  |       |rJ d       |rJ |\  }|j                  j                         D ]m  \  }}|j!                  |j                   j#                  || j                  |t%        t'        | j(                  |                  |gi        |j+                         }o |cd d d        S |r+|j,                  |j,                  | _        t1        ||||       |j                   j3                         r|j4                  j7                  d      rw|j4                  dk7  rht        |dd      r| j                  |       ddlm}	  |	| |j                   j<                  d	| j                  gt?        ||       
      cd d d        S | j(                  sJ d       t        |t        j@                  jB                        r#|j                  }
t'        | j(                  d      }n"|jD                  }
t'        | j(                  d      }tG        |
tH        jJ                        r|
jL                  }
t'        |d      }| g|z   }ntG        |
tH        jN                        sJ |jQ                  tS        jT                  |
|      ||      cd d d        S # 1 sw Y   y xY w)Nz3Expected lazy sequential isn't a valid combination?r   )z	torch.nn.z	torch.ao.ztorch.nn.utils.parametrizeT)check_forward_hookscheck_backward_hooksr'   wrap_fx_proxycall_modulerL   proxyzsMust provide a valid source in order to inline, since inlined function may have default args which must be guarded.rd   
_call_implre   )+r   r   rX   rb   r   r   r?   rE   r   r   r^   rd   r!   r   r   rA   r   r   r   r   r`   popcls_to_becomerv   rW   is_root_tracer
__module__
startswithbuilderr   create_proxyr#   rF   GraphModuler   r    ri   rj   re   r   inline_user_function_returnr	   r   )r}   rL   rP   rQ   rO   is_lazyrT   
child_namer   r   fn	fn_sources               r:   r   zNNModuleVariable.call_function|  s    ii%%doo6#OOT<<>C
 a	 %S)G3 3 34MM))UXX-@-@-H-HH%c* 11"5 # I{ "!z*-,,*<*<*> #&J$$		99" OO&#1*T[[*2U#V	 :  	 &&(C# Ca	 a	F $$0'*'8'8D$
 'r3f= 		((*NN--.HINN&BB%T 11"52$0"))00% +48Ca	 a	T {{ Z{ c588#7#78 B *4;;	 BIB *4;; EI"e../B *9j AI 6D=D!"e&8&899955222iH{a	 a	 a	s   D"M8CMC8MMc                     ddl mmm  j                  j
                  j                         fd}dv r j                        S dk(  r8t         j                   j                               5   |      cd d d        S dk(  rMt        j                  t        j                  j                  j                              rj!                  d      S dk(  rd   j#                         sJ t%        d	         sJ d	   j&                  d   j(                     }t%        |t*              r|S |j                  j
                  j                        }j
                  j-                  |t/        t1         j2                              
      S |r7t5              }	j                  j6                   d dt9        |	      S fdfd}
 fd} fd}d dk(  rj
                  j:                  j=                  t?         j2                  d      jA                                srJ g }jC                         D ]  \  }|jE                   ||               |tG                     S dk(  rj
                  j:                  j=                  t?         j2                  d      jA                                g } jH                  d0i  |
dd      D ]  \  }|jE                   ||               |tG                     S dk(  rj
                  j:                  j=                  t?         j2                  d      jA                                g } jJ                  d0i  |
ddd      D ]  \  }|jE                   ||               |tG                     S dk(  rj
                  j:                  j=                  t?         j2                  d      jA                                g } jL                  d0i  |
ddd      D ]  \  }|jE                   ||               |tG                     S dk(  rcj
                  j:                  j=                  t?         j2                  d      jA                                srJ  |jC                               S dk(  r]j
                  j:                  j=                  t?         j2                  d      jA                                 |jM                               S d k(  rej
                  j:                  j=                  t?         j2                  d      jA                                 | jH                  d0i  |
d            S d!k(  rej
                  j:                  j=                  t?         j2                  d      jA                                 | jJ                  d0i  |
d            S d"k(  rOsrJ g }jO                         D ]"  |jE                  j!                               $  |tG                     S d#k(  rsrJ  |j'                               S d$k(  rJsrJ g }j'                         D ]  \  }|jE                   ||               |tG                     S d%k(  r srJ j!                  tQ                    S d&k(  rt%        tR        jT                  jV                  tR        jT                  jX                  f      rCrAd	   j#                         r.j!                  d	   j[                         j\                  v       S d'k(  rωstQ              dk(  sJ tR        jT                  jV                  j^                  tR        jT                  j`                  j^                  tR        jT                  jX                  j^                  tR        jT                  jb                  j^                  tR        jT                  jd                  j^                  f}tg              j^                  |vrt%        d	   th        j                        sJ tk        d	                d	   j[                         t%        tl        tn        f      sJ t5              jp                  }	t%        |	tr        jt                        sJ t?        t?         j2                        d(      }jw                  ti        jx                  |	|
       gt{              z         S  j2                  sJ t%        d	   t|              r:j
                  j~                  rg }t{        t        tQ                          d	   j[                            }t        d	   j[                                  D ]V  \  }}||   t/        t1         j2                              }|jE                  j
                  j-                  ||
             X d	   j[                            }j
                  j-                  |  d)t/        t1         j2                  d	   j[                                     
      }|S  j                         dd*lCmD} t%        d	   |      rd	   j                  j
                        n8d	   j#                         rd	   j[                         nt        d+d	              }j
                  j-                  | j                  t/        t1         j2                              
      S d,k(  szt%        tR        jT                  j                  j                  j                        rd-k(  s=t%        tR        jT                  j                  j                  j                        rgd.k(  rbt5              jp                  }	t?        t?         j2                        d(      }jw                  ti        jx                  |	|
       gz         S j                  j                  v r^t        j                  j                           r<t        d/ t        j                  j                               D              r |      S t         I        S # 1 sw Y   )xY w)1Nr'   )r   ListIteratorVariableTupleVariablec           	          	j                   j                  dj                  di       }t        |j                         t              \  }}ddlm}  |		j                   j                  d| |g||            S )Nget_attrr5   r'   r   call_method)rP   rQ   r   )r   r   rX   r$   rH   r#   r   r   )
r[   	mod_proxyrR   rS   r   rP   rQ   moduler}   rL   s
        r:   generic_call_method_helperz@NNModuleVariable.call_method.<locals>.generic_call_method_helper  s     		..	I innf5'8v'F$J. ii,,!#1j1'	 -  rs   r   _wrapped_call_implrd   _check_input_dimT_get_item_by_idxr   r   __resultc                      t        d t        j                   j                               D              st	        d        y y )Nc              3   <   K   | ]  }|j                           y wr4   )is_python_constantr7   rI   s     r:   r;   zUNNModuleVariable.call_method.<locals>.assert_all_args_kwargs_const.<locals>.<genexpr>5  s      +,$$&s   znon-const NNModule method )all	itertoolschainvaluesr   )rP   rQ   r[   s   r:   assert_all_args_kwargs_constzBNNModuleVariable.call_method.<locals>.assert_all_args_kwargs_const4  s@     09fmmo0V   :4&ABrs   c                              t        	
      } t        j                  |      j                  D cg c]  }|j	                          c}i j                         D ci c]  \  }}||j	                          c}}}|j                          |j                  }| D ci c]  }|||   
 c}S c c}w c c}}w c c}w r4   )r   r   	signaturebindas_python_constantrA   apply_defaults	arguments)namesr   rI   rJ   rK   
bound_argsrP   r  rQ   r  r[   s         r:   
get_kwargsz0NNModuleVariable.call_method.<locals>.get_kwargs:  s    (*&B3**2.33378a1'')89?HA1a**,,HJ %%'#--J.34Az!}$44 9H 5s   B6%B;&Cc                     g }| D ]M  \  }}|j                  j                  j                  ||t         j                  |                         O  |t                     S )Nr   mutation_type)r   r   r   r   r`   r)   )	rA   r   r[   r   r   
gen_sourcer   r}   rL   s	       r:   wrap_valuesz1NNModuleVariable.call_method.<locals>.wrap_valuesE  sm    F % fII55-jd.KL	 6  (>N>PQQrs   c                      j                  |       j                  j                  || t         j                  |                   g      S r   )r   r   r   r   r`   )r[   objr   r   r  r   r}   rL   s     r:   named_embedz1NNModuleVariable.call_method.<locals>.named_embedR  sV     $++D1II55-jd.KL	 6 
 
rs   c                     |j                  d      }|d   dk(  r| S t        |      dkD  r,|j                  d      }t        | |      } t        |      dkD  r,| S )N.r    )splitlenr   r   )r`   r[   
name_splitrI   s       r:   r  z0NNModuleVariable.call_method.<locals>.gen_source_  s[    CJ!}"j/A%NN1%#FA. j/A% Mrs   named_childrenr   r  named_parametersr   prefixrecursenamed_buffersr   remove_duplicatenamed_modulesmemochildrenmodules
parametersbuffersr   r  rA   __len____contains____getitem__re   z.__getitem__(slice))SymNodeVariablez%getitem on NNModuleVariable with key _get_abs_string_index_conv_forward_output_paddingc              3   P   K   | ]  }t        |t        j                           y wr4   )r?   r	   TensorVariabler  s     r:   r;   z/NNModuleVariable.call_method.<locals>.<genexpr>  s%       1i667s   $&r5   )Sr&  r   r   r   rX   r   r   r   rb   r   r   is_torch_inline_allowedr   getfiler^   r  r   r  r?   rA   rw   r   r   r   r   r`   r   __name__r+   guard_on_key_orderaddr   r[   r*  r   r)   r+  r.  r0  r   r(  rE   r   r   ParameterDictr  r   r8  r   r   r   r>   r	   r(   strintre   ri   r   r   r   rB   r-   exportranger   r   tensorr9  evaluate_exprr   r3  conv_ConvNd_ConvTransposeNdrg   rh   r  r  r  r  r{   r   ) r}   rL   r[   rP   rQ   constantr  mod_varr   r   r  r   r#  r   parambufferbuiltin_supportedsrcr   r   
new_modulenew_module_variabler9  r   r   r   r   r  r  r   r  r^   s    `````                   @@@@@@@r:   r   zNNModuleVariable.call_method  sm    	LKoo((-	 	2 77 %%b$77Y (!@!@!BB 8 2$78 8
 %%+*M*MOOF,,==>+
 $**400%%7--///d1g}5551gmmDGMM2G'#@A$$CYY,,S1F9944%mDKK&EF	 5   &B&&//0$w?D/BdFKK	C		5 		5	R 	R	 		 ##II((,,ZZ-P-U-U-WX''F & 5 5 7 9fk$789'>N>PQQ''II((,,4;;6;;= F6v66  Xy1  8e k$678 (>N>PQQ_$II((,,ZZ-P-U-U-WXF 4 4 4 !Xy2DE! 9f k$789 (>N>PQQ_$II((,,ZZ-P-U-U-WXF 4 4 4 !VX/AB! 9f k$789 (>N>PQQZII((,,ZZ-P-U-U-WX''v44677YII((,,ZZ-P-U-U-WXv33566\!II((,,4;;6;;= 6v66OI9NOPPYII((,,ZZ-P-U-U-WX3v33Lj6KLMMV^''F =.55d;<='>N>PQQX''v||~..W_''F & 9fk$789'>N>PQQY''#**3v;77N"6EHH$7$79O9O#PQQ**,#**Q**,?  ]"#d)q.00##//##//&&22&&22##//! F|''/@@!$q'9+E+EFXPTUVPWHXXF1g002!#Sz222VT*33!"e&8&8999 DKK!>
K55222cBFT$Z'  ;;;$q'=1 99##F  c&k 23DG4N4N4PQD'0Q8R8R8T1U'V 	V"3i,]4;;-LMII== & #'* > 	 "(Q(B(B(D!EJ*,))*K*K"& 34-)$++tAw7Q7Q7ST  +L +' /. 11"5/$q'?31g++BII6a++-1g002 Ed1gYOPC[F9944%mDKK&EF	 5   ++6588#3#3#8#8#@#@AO+ 6588#3#3#8#8#I#IJ-- &//B":dkk4#@*MI11..r)D  F$$---))22489 "v}}? 
 .d337&r4v>>S8 8s   
r;;srL   r/   rP   list[VariableTracker]rQ   dict[str, VariableTracker]ry   r*   )F)rA  r   __qualname__r*   _nonvar_fieldsr>   rE  rE   r   Moduler|   r   r   r   r   r   r   r   r   r   r   r   r   r   __classcell__r^   s   @r:   ru   ru      s    	
 
	'	'N22-029>2	2:- )
@
9)
914
9	
9/* 
"fDPi &i -	i
 
ib x? &	x?
 -x? 
x? x?rs   ru   c                        e Zd Zdddhej                  Z	 d fdZd Zd Zd Ze	 e
j                  d      d	               Z fd
Z	 	 	 	 	 	 	 	 ddZ	 	 	 	 	 	 d fdZddZd fdZddZ xZS )r   
value_typeis_state_mutatedrx   Nc                 *   t        |      t        j                  j                  j                  u rt        d      d|v r+t        |d   dd       }t        |      |u rt        |      |d<   t        |    dd|i| d| _	        | j                  | _        y )NzScriptModules aren't supported in UnspecializedNNModuleVariable becuase their .forward function isn't a static member of their typer`  r   rw   Fr5   )r>   rE   jit_scriptRecursiveScriptModuler   r   r{   r|   ra  r`   rx   )r}   rw   rQ   lazy_value_to_becomer^   s       r:   r|   z&UnspecializedNNModuleVariable.__init__2  s    ;%))++AAAW  6!#*6,+?RV#W E{22
 (,E{|$/u// % '+kk#rs   c                 <    t        |t              st        |      S |S r4   )r?   r   r}   attr_sources     r:   _wrap_sourcez*UnspecializedNNModuleVariable._wrap_sourceK  s    +'BC.{;;rs   c                 6    | j                   xs | j                  S r4   r   r   s    r:   r   z8UnspecializedNNModuleVariable.get_nn_module_stack_sourceP  r   rs   c                     || _         y r4   r   r   s     r:   r   z8UnspecializedNNModuleVariable.set_nn_module_stack_sourceS  r   rs   c                  b   t         j                  j                  j                  t         j                  j                  j                  h} t         j                  j                  j
                  j                         D ch c]'  }t        |d      r|| vrt        |j                        ) c}S c c}w )N__code__)
rE   r   r\  __setattr__r|   rg   r  rM   idrn  )	supportedrI   s     r:   _nn_module_method_idsz3UnspecializedNNModuleVariable._nn_module_method_idsV  s|     XX__00%((//2J2JK	 XX__--446
q*%!9*< qzzN
 	
 
s   =,B,c                    	 t        j                  | j                  d      }|t
        j                  j                  j                  t
        j                  j                  j                  t
        j                  j                  j                  fv r6|j                  t        j                  |      | gi       j                  |      S t        | 9  |      S # t        $ r}t        |d }~ww xY w)N__iter__)r   r   r`  r   NotImplementedErrorrE   r   r   rt  r   r   r   r	   r   r   r{   )r}   rL   r   er^   s       r:   r   z1UnspecializedNNModuleVariable.unpack_var_sequencea  s    	-''DB HH((HH""++HH((
 
 11..r2  "!"%& w*2..'  	-%1,	-s    C 	C*C%%C*c                 V   | j                   }t        |      r+|j                  |j                  | _        t	        ||||       t        |t        j                  j                        s5|j                  j                  t        urd}t        | j                  |      }nd}t        | j                  |      }t        |j                  t        j                        rt        |j                   t        j                        r|j                  j                  t        u r|j                   j                  t"        u rd|j$                  vrt'        j(                  |d      }t        |t        j*                        r]|j,                  }| j/                  |d      j1                         j3                         s"| j/                  |d      j1                         j3                         s| j/                  |d      j1                         j3                         s| j/                  |d      j1                         j3                         s|j/                  |d      j3                         sx|j/                  |d	      j3                         sX|j/                  |d
      j3                         s8|j/                  |d      j3                         sd}| j                  j4                  }| j6                  r!t9        t9        | j6                  d      |      }	nd }	t;        | j6                  |       | j6                  r.t=        t?        tA        |            | jC                         ||      n	tE               }
|
5  tG        jH                  ||	      jK                  || gtM        |      z   |      cd d d        S # 1 sw Y   y xY w)N__call__r   rd   _backward_hooks_backward_pre_hooks_forward_hooks_forward_pre_hooks_global_backward_pre_hooks_global_backward_hooks_global_forward_hooks_global_forward_pre_hooksr^   r   )'rw   r   r   r`  rW   r?   rE   rF   r   rx  re   r%   r   r    ri   rj   r   r&   rg   r   r   r   nn_modules_globals_vtr   realizer(  rd   r`   r   rr   rb   rE  rp  r   r   r	   r   r   rB   )r}   rL   rP   rQ   rO   r[   r   forward_method
globals_vtr`   ctxs              r:   r   z+UnspecializedNNModuleVariable.call_functiony  s    jj#  ,"%"3"3"2sD&9 3 4 45%%-EED$/BD$/B
 3<<!1!12s~~u'7'78%%)AA''+HH-$33CCN.%*<*<=55
$$R):;CCEIIK'',ABJJLPPR'',<=EEGKKM'',@AIIKOOQ!--b2NOSSU!--b2JKOOQ!--b2IJNNP!--b2MNRRT$D00B;;
4;; DdKFF.t{{C@ {{ #BsGd==?S  	  	11"VDRRTFT$Z'	 	 	s    5NN(c                 ^   |dv rzt        | j                  |      }| j                  r!t        t        | j                  d      |      }nd }t	        j
                  ||      j                  || gt        |      z   |      S |t        | j                  di       vrq	 t        j                  t        | j                        |      }t        |t              r\t        t        t        | j                  d      |      d      }|j                  t	        j
                  |j                   |      ||      S t#        |d      r3t%        |j&                        | j)                         v rt+        d|        |dk(  rd	| j                  j,                  v r|d
   j/                         }|d   }	t        |	t        j0                        r*|	j3                         t4        j6                  j8                  u s|| j                  j,                  d	   v rd| _        no|| j                  j,                  d   v rd| _        nLt        |	t        j<                  t        j>                  f      s|| j                  j,                  d   v rd| _        |t4        j6                  j@                  jB                  u rjt        |d   t        jD                        rM|j                  t	        j
                  t4        j6                  j@                  jF                        | |d
   g|      S tH        
|   ||||      S # t        $ r d }Y gw xY w)Nr  r^   r   rg   re   rn  z&UnspecializedNNModuleVariable missing ro  r   r   r'   Tr   r   )&r   r`  r`   r   r	   r   r   rB   rw   r   r   r>   r   r?   r   r   re   rM   rp  rn  rr  r   rg   r  r>  r   rE   r   	Parameterra  ru   r   r\  ro  r   __delattr__r{   r   )r}   rL   r[   rP   rQ   r   r`   method	attr_namerw   r^   s             r:   r   z)UnspecializedNNModuleVariable.call_method  s    77$/B{{#Jt{{K$H$O11"VDRRTFT$Z'  wtzz:r:: //TZZ0@$G &,/#z$++{CTJJ 55226??6R  
+v'4+E+E+GG FtfMN }$$**:M:M)M !G668	Q
 ui&>&>?))+uxx/A/AA$**"5"5m"DD,0D)$**"5"5j"AA,0D)%66%CC !DJJ$7$7
$CC -1D)444Q22:
 5522588??3N3NO47O  w"2tT6::I " s   )L L,+L,c                 ~    | j                  ||      }t        |t        j                        r|j	                  |      S y r4   )r   r?   r	   ConstDictVariablemaybe_getitem_const)r}   rL   fieldname_vtdict_vts        r:   getattr_helperz,UnspecializedNNModuleVariable.getattr_helper  s8    ""2u-gy::;..w77rs   c                    |dv r|j                   j                  j                  | |      st        | j                  |      }t        |t              rmt        |      dk(  r_| j                  r>t        | j                  |      }t        |j                  t        j                               t        j                  i       S | j                  r|dv r|j                   j                  j                  | |      st        | j                  |      }t        | j                  |      t        j                  t        j                                |j                   j"                  j%                  j'                                fdt        fdt)        |j+                               D              }t        j,                  |t/        |            S t0        | e  ||      S )N)ry  rz  r{  r|  r   )r|  r{  c                     t         j                  j                  |      }t        |       }t	        |      }t        j                  ||      }||fS r4   )r	   r   r   r   r   r,   )irJ   rK   r   
source_keysource_valuerw   hooks_dict_sources          r:   build_key_valuezBUnspecializedNNModuleVariable.var_getattr.<locals>.build_key_valueF  sQ    0077: 00A1E
01BJO+221lCEz!rs   c              3   >   K   | ]  \  }\  }} |||        y wr4   r5   )r7   r  rJ   rK   r  s       r:   r;   z<UnspecializedNNModuleVariable.var_getattr.<locals>.<genexpr>Q  s'      -6QA1a(s   r   )r   r   r   r   rw   r?   r@   r(  r`   r   r   rk   r   EMPTY_NN_MODULE_HOOKS_DICTr	   r  SEQUENCE_LENGTHrB  rC  r[   r   rA   NNModuleHooksDictVariabler>   r{   r   )	r}   rL   r[   
hooks_dicthooks_sourcer   r  r  r^   s	         @@r:   r   z)UnspecializedNNModuleVariable.var_getattr  s    
 
 99))FFtTR$TZZ6
j$/C
Oq4H{{'1$++t'D%(33 , G G
 %66r:: KK
 II**GGdS T2J *4;; =+66|7S7STUII((,,->-C-C-EF	"  :CJDTDTDV:W F 66Z(1B  w"2t,,rs   c                     t        j                  |      }| j                  |d|      }|| j                  |d|      }|| j                  |d|      }|t        t        |       |S )a  
        Dynamo tracing of nn.Module __getattr__ can be expensive if the model
        has deep submodule hierarchy. Since the __getattr__ is stable, we can
        directly look into the underlying datastructures. This saves a lot of
        compilation time.
        r   r   r   )r	   r   r  r
   r   )r}   rL   r[   r  r   s        r:    manually_trace_nn_module_getattrz>UnspecializedNNModuleVariable.manually_trace_nn_module_getattrZ  sn     ,,T2!!"mW=;%%b*g>C;%%b*g>C;$^R8
rs   ry   N)rL   r/   rP   rX  rQ   rY  ry   r*   rW  rV  )rA  r   rZ  r.   r[  r|   rj  r   r   r   rm   	lru_cacherr  r   r   r   r  r   r  r]  r^  s   @r:   r   r   "  s      
#	1	1	N22
:- Y
  
/0A#A &A -	A
 
AFY; &	Y;
 -Y; 
Y;v;-zrs   r   c                       e Zd ZdZd Zy)$UnspecializedBuiltinNNModuleVariablezg
    Differentiates between builtin nn modules (e.g. torch.nn.Linear) and user defined nn modules.
    c                 <    t        |t              st        |      S |S r4   )r?   r   rh  s     r:   rj  z1UnspecializedBuiltinNNModuleVariable._wrap_sourceq  s    +'IJ5kBBrs   N)rA  r   rZ  __doc__rj  r5   rs   r:   r  r  l  s    rs   r  c                   *     e Zd ZdZd fdZd Z xZS )FSDPManagedNNModuleVariablea  
    Tracing behavior: trace into submodules and treat them as Unspecialized, do not
    register parameters to the top-level, treat them as function inputs.

    Guards behavior: if 'skip_fsdp_guards', many guards that would be installed
    by a vanilla UnspecializedNNModuleVariable are simply dropped, on the basis
    that a user wrapping their model in FSDP(model) is already opting into a
    requirement to not modify internal model state, which would already break FSDP without
    compilation.
    c                 l    |j                  dd       }|J d       t        |   dd|i| || _        y )Nr`   zMFSDPManagedNNModule depends on having an accurate source to control guarding.rw   r5   )r]   r{   r|   r`   )r}   rw   rQ   r`   r^   s       r:   r|   z$FSDPManagedNNModuleVariable.__init__  sG    Hd+! 	
[	
! 	/u//rs   c                     t        |t        t        f      s:t        j                  j
                  j                  rt        |      S t        |      S |S r4   )r?   r   r   rE   _dynamoconfigskip_fsdp_guardsrh  s     r:   rj  z(FSDPManagedNNModuleVariable._wrap_source  sG    ,.IJ
 }}##44)+662;??rs   r  )rA  r   rZ  r  r|   rj  r]  r^  s   @r:   r  r  w  s    	rs   r  rV  )Jr  rm   r   r  ri   
contextlibr   r   typingr   torch.nnrE   r&  r   r   r	   excr
   r   r   r   r   guardsr   r   mutation_guardr   r`   r   r   r   r   r   r   r   r   utilsr   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r   r(   r)   r*   	functionsr+   lazyr,   listsr-   user_definedr.   torch._dynamo.symbolic_convertr/   rW   rE  r   r\  rb   rr   ru   r   r  r  r5   rs   r:   <module>r     s   .     2    8 8  1 .	 	 	    = < 3 %   3 D;8 	+s 	+UXX__ 	+ 	+BD
? D
?NG$= GT
+H "? rs   