
    Vhy                       U d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlZddlZddlZddlZddlZddlZddlmZ ddlmZmZmZmZmZmZ ddlZddlZddlmZ ddl m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z' dd	l(m)Z) dd
l*m+Z+m,Z,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2 ddl3m4Z4 ddl5m6Z6 ddl7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z> ddl?m@Z@mAZA ddlBmCZC ddlDmEZE ddlFmGZG ddlHmIZImJZJmKZKmLZLmMZM ddlNmOZO ddlPmQZQmRZRmSZS ddlTmUZUmVZVmWZW ddlXmYZYmZZZm[Z[m\Z\m]Z] ddl^m_Z_ ddl`maZambZbmcZcmdZdmeZemfZfmgZgmhZhmiZimjZjmkZkmlZlmmZmmnZnmoZompZpmqZqmrZrmsZsmtZt ddlumvZvmwZwmxZxmyZymzZzm{Z{m|Z|m}Z}m~Z~mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ ddlmZmZmZmZmZ ddlmZmZ dd lmZmZmZmZmZmZ dd!lmZmZmZmZmZmZ dd"lmZmZmZmZmZ dd#lmZmZmZmZmZmZmZmZmZmZmZmZ dd$lmZ dd%lmZ dd&lmZ dd'lmZmZmZmZmZmZmZmZmZ dd(lmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ dd)lmZmZmZ dd*lmZ dd+lmZ dd,lmZ dd-lmZmZmZmZmZmZ dd.lmZmZmZmZ dd/lmZmZmZmZ dd0l mZmZmZmZmZmZmZmZm	Z	m
Z
 	 ddlZer	dd1lmZ  e
j                   e      Zej&                  j)                  ed2      ZeZd3 Z G d4 d5      Zej4                   G d6 d7             Z G d8 d9e      Z ed:  ee	      j?                         D              Z ee!   e"d;<    e#       Z$e#e!   e"d<<    G d= d>      Z%d? Z&d@ Z'	 dVdAefdBZ(dC Z)	 dVdDZ*	 dWdEZ+	 dVdFZ,dG Z-dH Z.da/ee#e0      e"dI<   dAe#e0   fdJZ1ej4                   G dK dL             Z2	 dXdAe>fdMZ3ddNdOeer   dPe4fdQZ5 G dR dS      Z6e6jo                         e6_8         G dT dU      Z9y# e$ r dZY w xY w)Yaq  
This module contains classes and utilities for building variable trackers in Dynamo.
Variable trackers are used to convert Python values into symbolic representations
that can be traced and transformed during graph capture.

The key classes are:

- VariableBuilder: Handles source-tracked objects that need guards and proper
  reconstruction in the output graph. Used for inputs, module attributes, etc.

- SourcelessBuilder: Handles ephemeral objects created during tracing that don't
  need source tracking or guards. Used for temporary lists, intermediate values, etc.

Variable trackers enable Dynamo to track the flow of values through the program,
maintain guards for dynamic properties, and reconstruct values in the output graph.
The builders in this module handle converting Python values into appropriate
VariableTracker instances based on their type and usage context.
    N)MutableMapping)AnyCallable
NamedTupleOptionalTYPE_CHECKINGUnion)SymInt)get_metrics_contextis_int_specialization_caseis_torch_sym)TracingContext)call_torchbind)HigherOrderOperator)
FakeTensoris_fakemaybe_get_fake_mode)is_sparse_any	safe_grad)justknobs_check)BackwardState)normalize_source_name)_constrain_range_for_size_nested_int_aware_sort
DimDynamicRelaxedUnspecConstraintStatefulSymbolicContextSubclassSymbolicContextSymbolicContext)immutable_dictimmutable_list)is_traceable_wrapper_subclass)ValueRanges)TensorWeakRef   )configgraph_break_hintsmutation_guardreplay_recordtrace_rules) get_registered_device_interfaces)InternalTorchDynamoErrorunimplementedunimplemented_v2)GuardBuilderinstall_guardmake_dupe_guard)auto_dynamic
auto_unsetFrameStateSizeEntryInferStrideprocess_automatic_dynamic)SideEffects)AttrProxySource
AttrSourceCallMethodItemSourceChainedSourceConstDictKeySourceConvertIntSourceDictGetItemSourceFloatTensorSourceGetItemSource
GradSourceis_constant_sourceis_from_optimizer_sourceListGetItemSourceLocalSourceNumpyTensorSourceOptimizerSourceRandomValueSourceSourceSubclassAttrListSourceTupleIteratorGetItemSource)_extract_tensor_dictbuild_checkpoint_variablebuild_invoke_subgraph_variableclone_inputcommon_constant_types	dict_keysget_fake_valueget_items_from_dictget_locals_to_stealget_static_address_typeis_frozen_dataclassis_function_or_wrapperis_invoke_subgraphis_lru_cache_wrapped_functionis_namedtupleis_parameter_freezing	is_typingis_utils_checkpointis_wrapper_or_member_descriptoristypenamedtuple_fieldsodict_valuesproxy_args_kwargsrange_iteratorset_example_valuetensor_always_has_static_shapetuple_iteratortuple_iterator_getitemtuple_iterator_len unwrap_with_attr_name_if_wrapperwrap_fake_exception   )typestrValueMutationExistingValueMutationNewVariableTrackerVariableTrackerMeta)ConstantVariableEnumVariable)AutocastModeVariableEventVariableNullContextVariablePreserveVersionContextVariableStreamContextVariableStreamVariable)ConstDictVariableDefaultDictVariableDictKeySetVariableFrozensetVariableMappingProxyVariableSetVariable)DeviceMeshVariablePlacementClassVariablePlacementVariableProcessGroupVariableWorldMetaClassVariable)BuiltinMethodVariableCollectionsNamedTupleFunction!CollectiveFunctionRewriteVariableCreateTMADescriptorVariableFunctoolsPartialVariableFunctoolsWrapsVariableSysFunctionVariableTracebackVariableTritonKernelVariableUserFunctionVariableUserMethodVariableWrapperUserFunctionVariable) TorchHigherOrderOperatorVariable)ItertoolsVariable)LazyVariableTracker)	BaseListVariableListIteratorVariableListVariableNamedTupleVariableRangeVariableSizeVariableSliceVariableTupleIteratorVariableTupleVariable)AutogradEngineVariableAutogradFunctionContextVariableAutogradFunctionVariableComptimeVariableDebuggingVariableDelayGraphBreakVariableGetAttrVariableGetSetDescriptorVariableLambdaVariableLoggingLoggerVariableMethodWrapperVariableNumpyDTypeVariableNumpyTypeInfoVariableNumpyVariablePythonModuleVariableRandomClassVariableRandomVariableRegexPatternVariableSavedTensorBoxTorchVersionVariableTypingVariableWeakRefVariable)FSDPManagedNNModuleVariable$UnspecializedBuiltinNNModuleVariableUnspecializedNNModuleVariable)OptimizerVariable)TorchScriptObjectVariableSDPAParamsVariable)NumpyNdarrayVariable$supported_const_comparison_op_valuesSymNodeVariableTensorSubclassVariableTensorVariableUnspecializedPythonVariable)DispatchKeySetVariableFuncTorchInterpreterVariableTorchCtxManagerClassVariableTorchInGraphFunctionVariable)build_torch_function_fnTensorWithTFOverrideVariable#torch_function_mode_stack_state_mgrTorchFunctionModeVariable)
FrozenDataClassVariableKeyedJaggedTensorVariableMutableMappingVariableSourcelessGraphModuleVariableUserDefinedClassVariableUserDefinedDictVariable!UserDefinedExceptionClassVariableUserDefinedListVariableUserDefinedObjectVariableUserDefinedTupleVariable)InstructionTranslatorcudagraph_static_inputsc                     t        j                         5  t        j                  dd       t        | d      cd d d        S # 1 sw Y   y xY w)NignorezThe .grad attribute of a Tensorgrad)warningscatch_warningsfilterwarningshasattr)ts    O/home/dcms/DCMS/lib/python3.12/site-packages/torch/_dynamo/variables/builder.pysafe_has_gradr     s>    		 	 	" "*KLq&!" " "s   "AA
c                       e Zd Zy)_missingN__name__
__module____qualname__     r   r   r   $  s    r   r   c                       e Zd ZU eed<   eeej                  f   ed<   e	ed<   e
ej                  j                  j                     ed<   dZe	ed<   dZe
ej                      ed<   ed	        Zd
 Zd Zd Zd Zy)GraphArgsource_examplepass_arg_as_tensorfake_tensorT	is_tensorNexample_strong_refc                 z    t        | j                  t              r| j                         }|J |S | j                  S N)
isinstancer   r$   )selfrs     r   examplezGraphArg.exampleL  s4    dmm]3A= =H== r   c                     t        | j                  t        j                        r2t	        | j                        | _        t        | j                        sJ y y r   )r   r   torchTensorr$   r   r   r   s    r   __post_init__zGraphArg.__post_init__U  s=    dmmU\\2)$--8DM4++,,, 3r   c                 (     || j                          y r   r   r   codegens     r   reconstructzGraphArg.reconstructZ  s    r   c                      d | _         d | _        y r   )r   r   r   s    r   erasezGraphArg.erase]  s    "&r   c                 l    | j                   j                         |j                   j                         k(  S r   )r   namer   others     r   __eq__zGraphArg.__eq__a  s'    {{!U\\%6%6%888r   )r   r   r   rI   __annotations__r	   r$   r   r
   boolr   _subclassesr   r   r   r   r   propertyr   r   r   r   r  r   r   r   r   r   (  s    N M5<</00" %++77BBCC It 26.5! !-
'9r   r   c                   &     e Zd Zd fdZd Z xZS )BackwardStateGraphArgc                 >    t         |   d t               dd d       y )NF)r   r   r   r   r   )super__init__r   )r   	__class__s    r   r  zBackwardStateGraphArg.__init__f  s&    "_$ 	 	
r   c                    j                   j                  j                  sJ j                  fd       j	                  dd       j                          j                  j                   j                  j                         y )Nc                  D     j                  t        j                  d      S )Nr   )load_import_fromr   r   )r   s   r   <lambda>z3BackwardStateGraphArg.reconstruct.<locals>.<lambda>r  s    G,,]-E-EW r   r   F)txoutputbackward_state_varadd_push_nullcall_functiondup_topstorer   s    `r   r   z!BackwardStateGraphArg.reconstructo  se    zz  3333W	
 	a'gjj''::;r   returnN)r   r   r   r  r   __classcell__r  s   @r   r	  r	  e  s    
<r   r	  c              #      K   | ]8  \  }}|j                  d       s"t        j                  |      rt        |       : yw)_N)
startswithinspectisclassid).0r   members      r   	<genexpr>r%  {  s7      /f??3GOOF$; vJ/s   >A ITERTOOLS_TYPE_IDSITERTOOLS_POLYFILLED_TYPE_IDSc                   H    e Zd ZdZdeddf fdZd Zd Zd Zd	 Z	e
d
        Ze
 ej                  d      d               Zdej                   fdZdej&                  fdZd Zd Zd Ze
 ej                  d      deeed egef   f   fd              Zd ZdefdZde e!e"e#e$f   fdZ%de&fdZ'de(fdZ)de e*e+f   fdZ,de-j\                  de/fdZ0de-jb                  jd                  fdZ3d Z4de-j\                  fdZ5de-j\                  fdZ6d Z7d  Z8d! Z9d" Z: xZ;S )#VariableBuilderz3Wrap a python value in a VariableTracker() instancer   r  Nc                     |J d       t        j                         J d       t        |           || _        || _        |j                         | _        y )NzRConsider SourcelessBuilder for ephemeral objects, usually objects created locally.zExpected active TracingContext)r   try_getr  r  r  r   r   )r   r  r   r  s      r   r  zVariableBuilder.__init__  s^    
 ! 	
`	
! %%'3U5UU3KKM	r   c                    || j                   j                  j                  v rX| j                   j                  j                  |   }t        | j                  |j                        }|r| j                  |       |S | j                   j                  j                  j                  || j                        }|r|S | j                  |      }| j                  |_        | j                  |      r]|| j                   j                  j                  vr;t        |      s0| j                   j                  j                  j                  ||      }| j                   j                  j                  j                  || j                  |       |S r   )r  r  side_effectsr1   r   install_guardsvariable_tracker_cachelookup_wrap_can_lift_attrs_to_inputsr^   track_object_existingadd)r   valueside_effect_result	dup_guard	cached_vtvts         r   __call__zVariableBuilder.__call__  s   DGGNN///!%!<!<U!C'5G5N5NOI##I.%%GGNN99@@T	ZZKK	**2.TWW^^8883E:,,BB5"MB--11%bI	r   c                 D    t        |      t        t        t        t        hv S r   )typer   r   r   r   )r   r9  s     r   r2  z)VariableBuilder._can_lift_attrs_to_inputs  s$    Bx(% 	
 
 	
r   c                     | j                   S r   r   r   s    r   
get_sourcezVariableBuilder.get_source  s    {{r   c                     | j                         }	 |D cg c]  }|j                  |       }}t        |ddi i S c c}w # t        $ r Y y w xY w)Nskiprk   )r>  
make_guardNotImplementedErrorr0   )r   guardsr   guardtmps        r   r.  zVariableBuilder.install_guards  s]    "	9?@6$$U+@C@ 	s##		 A" 		s   A =A A 	AAc                 @    | j                  t        j                        S r   )_type_dispatch_implr&   trace_numpy)clss    r   _type_dispatchzVariableBuilder._type_dispatch  s    &&v'9'9::r   c                    t         j                  t         j                  j                  t         j                  j
                  t         j                  j                  j                  f| j                  ft        t        t        t        j                  t         j                  f| j                  ft         | j"                  ft$        | j&                  ft(        t*        f| j,                  ft        t.              | j0                  ft2        j4                  | j6                  ft8        j:                  | j<                  ft         j>                  j@                  jB                  | jD                  ft         jF                  jH                  | jJ                  ftL        jN                  | jP                  fg}|r1tR        r+|jU                  tR        jV                  | jX                  f       i }|D ]*  \  }}t[        |t              r|n|fD ]  }||vsJ |||<    , |S r   ).r   r   nn	Parameterr  r   functional_tensorFunctionalTensorwrap_tensortuplelistra   collectionsdequeSizewrap_listlikerf   wrap_tuple_iteratorrc   wrap_range_iteratorslicerangewrap_slice_rangerP   wrap_literalrePatternwrap_regex_patternweakrefReferenceTypewrap_weakrefutilshooksRemovableHandlewrap_removable_handlejitScriptFunctionwrap_jit_functiontypesMappingProxyTypewrap_mapping_proxynpappendndarraywrap_numpy_ndarrayr   )rI  rH  entriesresulttsfnr   s          r   rG  z#VariableBuilder._type_dispatch_impl  s    LLHH&&%%00%%77HH	  lK,=,=uzzJ!! S445S445U^S112()3+;+;<ZZ//0""C$4$45[[..0I0IJYY%%s'<'<=##S%;%;<-
2 2NNBJJ(>(>?@ 	FB%b%0Rre &q		
 r   r5  c                 V    | j                  t        j                         t        |      S r   )r.  r/   ID_MATCHr   r   r5  s     r   r_  z"VariableBuilder.wrap_regex_pattern  s!    L112#E**r   c                     | j                  t        j                         t        j                  | j
                  || j                        S Nr   )r.  r/   
TYPE_MATCHr   buildr  r   rw  s     r   rb  zVariableBuilder.wrap_weakref  s3    L334$$TWWeDKKHHr   c                     t        d       y )Nz"unregistered hook removable handle)r-   rw  s     r   rf  z%VariableBuilder.wrap_removable_handle  s    
 	:;r   c                 p    | j                  t        j                         t        |d| j                        S )N_torchdynamo_inliner   )r.  r/   rz  r   r   rw  s     r   ri  z!VariableBuilder.wrap_jit_function  s/    L334*(
 	
r   c                      j                  t        j                          j                  t        j                         t	        d |j                         D              }|st        d        fdt        fd|j                         D              }t        |d       }t        | j                        } j                  j                  j                  j                  ||      S )Nc              3   F   K   | ]  }t        j                  |        y wr   rq   
is_literalr#  ks     r   r%  z5VariableBuilder.wrap_mapping_proxy.<locals>.<genexpr>  s     M1(33A6M   !z+mapping proxy type supports only const keysc                     t        j                  |       }| }t        j                         |      }t	        j                  ||      }||fS r   )rq   creater@   r>  r   )r  vkey
source_keysource_valuer5  r   s         r   build_key_valuez;VariableBuilder.wrap_mapping_proxy.<locals>.build_key_value  sG    "))!,CJ():JGL'..q,?E:r   c              3   6   K   | ]  \  }} ||        y wr   r   )r#  r  r  r  s      r   r%  z5VariableBuilder.wrap_mapping_proxy.<locals>.<genexpr>  s     Etq!_Q*Es   r   )r.  r/   rz  MAPPING_KEYS_CHECKallkeysr-   dictitemsry   r}   r   r  r  r-  track_mutable)r   r5  	all_constr  dict_vtrr  r  s   `     @r   rl  z"VariableBuilder.wrap_mapping_proxy  s    L334 	L;;<M

MM	GH	 Eu{{}EE $E$7%gdkkBww~~**88GGr   c                     ddl m } |d ft        j                  d ft        j                  d fg}i }|D ]9  \  }}t        |t        t        f      r|n|fD ]  }||vsJ ||t        |      <    ; |S )Nr%   )comptimec                     t               S r   )r   rw  s     r   r  z.VariableBuilder._id_dispatch.<locals>.<lambda>%  s
    +;+= r   c                 t    t        t        fd| j                  i| j                  t        j
                        S )Nr   )r   _dataclasses_fields_lambdar   r.  r/   FUNCTION_MATCHrw  s     r   r  z.VariableBuilder._id_dispatch.<locals>.<lambda>(  s5    N.%;;% )),*E*EF% r   c                     t               S r   )r   rw  s     r   r  z.VariableBuilder._id_dispatch.<locals>.<lambda>.  s
    4H4J r   )	r  dataclassesfieldsr   __version__r   rQ  rR  r"  )rI  r  rq  rr  rs  rt  r   s          r   _id_dispatchzVariableBuilder._id_dispatch  s    
 	( =>""  JK
  	#FB%b5$-8Rre #& "r!u#	#
 r   c                 R   12 ddl m}m}  |       rddlm} ddlm} n G d d      } G d d      } |       r	dd	lm}m	} nd
 }d } j                         j                  t        |            }|	 | |      S  j                         j                  t        |            }	|		 |	 |      S t        |      st!        |t"        j$                        r j'                  |      S t)        |      r j+                  t,        j.                         t1        t        |            D 
cg c]7  }
t3        j4                  t7        ||
      t9         j:                  |
            9 }}
t=        |t        |       j:                        }|S t!        |t>        t@        jB                  t@        jD                  f      r| j+                  t,        jF                         tI        d |jK                         D              11sG jL                  jN                  jP                  jS                   j:                  jU                                1 fd2t?        2fdtW        tY        |            D              }t!        |t@        jB                        r]t9         j:                  d      }t[        |t        |       t]         jL                  |      |j^                         j:                        }n!ta        |t        |       j:                        } jL                  jN                  jb                  je                  ||      S tg        |th        jj                  jl                        r jo                  |      S tq        jr                  |      r ju                  |      S tg        |th        jv                  jx                        rIt{        | j:                        } jL                  jN                  jb                  j}                  ||       |S t!        |t~              rutI        d |D              rc|D cg c]"  }t        j5                   jL                  |      $ }} j+                  t,        j                         t        | j:                        S tg        |t        j                  th        j                  th        j                  j                  j                  f      r6 j+                  t,        j                         t        | j:                        S t        j                  |      r6 j+                  t,        j                         t        | j:                        S tg        |t        j                        r6 j+                  t,        j                         t        | j:                        S t        |      rt         j:                        S t        |      rt         j:                        S tg        |t        j                        rt9         j                         d      } t]         jL                  |      |j                        }g }t9         j                         d      }tW        |j                        D ]:  \  }}|j                   t]         jL                  t        ||            |             < i }t9         j                         d      }|j                  j                         D ]N  \  }}tq        jr                  |      st        d        t]         jL                  t        ||            |      ||<   P t         j                         j                  t,        jF                        |j                  t,        j                        |j                  t,        j.                               t        |||      S t        |      r6 j+                  t,        j                         t        | j:                        S t        >tg        |t        j                        r$ j                  t        j                  |            S t        j                  |      rYt        sJ  j+                  t        |      rt,        j                  nt,        jF                         t        | j:                        S t        j                  |      r6 j+                  t,        j                         t        | j:                        S t        j                  |      rtg        |t        j                        r^ j+                  t,        jF                         t9         j:                  d      }t        |j                  t,        j                               n j+                  t,        j                         t        | j:                        S t        j                  |      rK j+                  t,        j                         t        j4                   jL                  | j:                        S t!        |th        j                  j                  j                        r6 j+                  t,        j                         t        | j:                        S tg        |th        j                  j                  j                        r.d }	 |j                  }g } j:                  j                  t,        jF                        g}tg        |t              rt9         j:                  d      }|j                  |j                  t,        j.                               tW        |      D ]:  \  }}|j                   t]         jL                  t        ||            |             < t        |   jL                  jN                  jb                  j}                  |t        | j:                  t        |                  S tg        |t        j                         rt!        t7        |dd       th        j                  j                  j                        rt7        |dd      d k(  rr|t7        |j                  d d       k(  rW j+                  t,        j                         t        t        |j                  t9         j:                  d!            d       S tg        |th        j                  j                        r7 j+                  t,        j                         t	        | j:                        S |th        j
                  j                  j                  j                  u r. j+                  t,        j                         t         fd"      S t        |      ryt        j                  |      ct        j                  |      rd# jL                  jN                  _        t        j                  |      j                  | j:                        S t        r-tg        |t        j                        r j                  |      S tg        |t               r{|th        j"                  j$                  u rt        d$        j+                  t,        jF                  t,        j&                         t)        j*                  | j:                        S tg        |th        j,                  j.                        r} j+                  t,        j                         t9         j:                  d%      } t]         jL                  |      |j0                        } t3        j4                   jL                  |       S tg        |th        j4                        r j+                  t,        j                          jL                  jN                  j7                  d&t        |      d'|j8                  |j:                  |j<                  d(      }!t?        |!j@                  |       tC        |!||jD                   j:                        S tg        |th        j                  jF                        rK j+                  t,        jF                         tI        j4                   jL                  | j:                        S tg        |th        j                  jJ                  jL                        r+ j+                  t,        j                         tO        |      S tg        |th        jP                        rԉ j+                  t,        j                         th        j
                  jR                  jU                  |        jL                  jN                  j7                  d&th        j
                  jR                  jV                  t        |      fi       }"t?        |"j@                  |       tY        |"| j:                        S tg        |th        j                  jZ                        r*|t"        j$                  v rt]        | j:                        S t!        |t^        j`                        rOtc        jd                  |d)d       6 j+                  t,        jF                         tg         j:                        S ti        jj                  |      rg j+                  t,        jF                         ti        | j:                        } jL                  jN                  jb                  j}                  ||      S tg        |th        jl                  jn                        rR j+                  t,        j                         tq         j:                         _        ts        | j:                        S tg        |th        jt                        r, j+                  t,        jv                         ty        |      S t{        j|                  |      rt{        | j:                        S t        j                  |      r7 j+                  t,        j                         t        | j:                        S t        j                  |      r8 j+                  t,        j                         t        | j:                        S t        j                  |      r7 j+                  t,        j                         t        | j:                        S t        j                  |      r8 j+                  t,        j                         t        | j:                        S t        |      t        v rIt        |      t        vr7 j+                  t,        j                         t        | j:                        S t        |      rHtg        |th        j                        r j:                  nt         j:                        }#|j@                  j                         re jL                  jN                  j                  j                  t        |j@                  j                        |#t        j                  *      }$nWtg        |th        j                        r1 jL                  jN                  j                  j                         }$nt        d+        jL                  jN                  j                  j                  t        j                  d,d- jT                        t        $      |$|#      }%t        |#|$d.d d.|$/      |%j@                  j                  d0<   |$j@                  j                  }&tg        |&t        j                        s
J |& d1        jL                  jN                  j                  j                  t        |$|#d              tg        |th        j                        r|$n|$d2k(  }'t        |%|'      S tg        |||f      r9 j+                  t,        j                         t        |d d  j:                        S ||u rt        d23      S ||u rt        d43      S tg        |th        j                  j                  j                        rg j+                  t,        j                         t        |jD                  |j                  |j                  |j                  g j:                  5      S t        j                  |      r7 j+                  t,        j                         t        | j:                        S tc        jd                  |d6d.      r8 j+                  t,        jF                         t        |d7 j:                        S t        |      r8 j+                  t,        jF                         t        |d8 j:                        S |t        j                  u rt         j:                        S |t        j                  u s)t        j                  d9k\  r,|t        j                  u rt        | j:                        S t        |      rQtc        jd                  |d:d.      r8 j+                  t,        jF                         t        |d: j:                        S |t        j                  u r7 j+                  t,        j                         t        | j:                        S |t@        j                  u r7 j+                  t,        j                         t        | j:                        S tg        |t        j                        rNt        j                  |      r7 j+                  t,        j                         t        | j:                        S t        |      r^t        |      \  }}(|(t9         j:                  |(       _        t        j                  |      j                  | j:                        S |t        j                  u r6 j+                  t,        j                         t	         j:                        S t!        |t        j                        rt        j                  |      ri j+                  t,        jF                         t        | j:                        } jL                  jN                  jb                  je                  ||       |S tg        |t        j                  t        j                  f      ri j+                  t,        j                         t        | j:                        } jL                  jN                  jb                  j}                  ||       |S tg        |t        j                         rtg        |j                  th        jj                  jl                  th        jR                  j                  j                  f      r t]         jL                  t9         j:                  d            |j                        })|)rtg        |)t              sJ d;        j+                  t,        j                         t        |j                  |) j:                        S tg        |t        j                         rt#        |      S tg        |t        j$                        rt'        |      S t)        t        |      t              r=t)        |t*              r+ j+                  t,        j                         t-        |      S t)        t        |      t              r$|th        jR                  j.                  j0                  th        jj                  j2                  th        jj                  j4                  fv r2t        j                  |      j                  | j:                        S |th        j                  j6                  u r@ j+                  t,        j                         t9        j:                   jL                        S  j+                  t,        j                         t=        | j:                        S t?        j                  t        |            rd4d<lm }*m!}+ th        jD                  jF                  jI                  |      r jL                  jN                  j                  j                  t        j                  d,d- jT                        t        |      | j:                        },t         j:                  |d.d d.|      |,j@                  j                  d0<   t?        j4                  |,| j:                        S tK        |d=      s jM                  |      S t3        jN                   t]         jL                   |+ j:                              |jQ                         jS                                      t3        jN                   t]         jL                   |* j:                              |jU                                      th        jD                  jF                  jW                   jL                  jN                  jX                  |      }- jL                  jN                  j                  j                  t        j                  d,d- jT                        t        |      |- j:                        },t         j:                  |d.d d.|-      |,j@                  j                  d0<   t?        j4                  |,|- j:                        S tg        |t>        t@        jD                  f      ryt        |      jZ                  t>        jZ                  u rQ j+                  t,        jF                          j+                  t,        j.                          jL                  jN                  jP                  jS                   j:                  jU                                 fd>2t?        2fd?tW        tY        |            D              }ta        |tg        |t@        jD                        rt@        jD                  nt>        t]                j:                  @      }.d#|._/        ta        ||. j:                  A      } jL                  jN                  jb                  j}                  ||      S tg        |t              rJt        |      jZ                  t        jZ                  u r" j+                  t,        jF                          j+                  t,        j.                         tc        t        je                  |            D cg c]F  }t3        j4                  t        jg                  ||      t         j                         |            H }}ti        | j:                  t]               B      }/tk        j4                  ||/ j:                  C      } jL                  jN                  jb                  j}                  ||      S tg        |tl              r j+                  t,        jF                          j+                  t,        j.                         tc        tl        je                  |            D cg c]H  }t3        j4                  tl        jg                  ||      to         j                         |            J }}tq        | j:                  t]               B      }0ts        ||0 j:                  D      } jL                  jN                  jb                  j}                  ||      S t)        t        |      tt              r7 j+                  t,        jF                         tw        | j:                        S ty        |      r| j+                  t,        jF                         t{        j4                   jL                  | j:                        } jL                  jN                  jb                  j}                  ||      S tg        |t|              rtI        dE |D              r|D cg c]"  }t        j5                   jL                  |      $ }}t         j                         j                  t,        j.                         j                         j                  t,        j                               t        | j:                        S t        dF       y  jM                  |      S c c}
w c c}w # t        $ r Y w xY wc c}w c c}w c c}w )GNr   )
has_tritonhas_triton_tma)	Autotuner)JITFunctionc                       e Zd Zy)*VariableBuilder._wrap.<locals>.JITFunctionNr   r   r   r   r  r  B      r   r  c                       e Zd Zy)(VariableBuilder._wrap.<locals>.AutotunerNr   r   r   r   r  r  E  r  r   r  )create_1d_tma_descriptorcreate_2d_tma_descriptorc                       y r   r   r   r   r   r  z7VariableBuilder._wrap.<locals>.create_1d_tma_descriptorO      r   c                       y r   r   r   r   r   r  z7VariableBuilder._wrap.<locals>.create_2d_tma_descriptorR  r  r   r   )	tuple_clsr   c              3   F   K   | ]  }t        j                  |        y wr   r  r  s     r   r%  z(VariableBuilder._wrap.<locals>.<genexpr>s  s     Qq,77:Qr  c                     rt        j                  |      }|}n0t        j                         |       }t	        j                  ||      }t        j                         |      }t	        j                  ||      }||fS r   )rq   r  r<   r>  r   r>   )	ir  r  r  r  r  r5  r  r   s	          r   r  z.VariableBuilder._wrap.<locals>.build_key_value  so    *11!4C!"J!3DOO4Eq!IJ-44Q
CC01BJO+221lCEz!r   c              3   >   K   | ]  \  }\  }} |||        y wr   r   r#  r  r  r  r  s       r   r%  z(VariableBuilder._wrap.<locals>.<genexpr>  *      Av1  1a(   default_factory)r  r   )user_clsr   c              3      K   | ]X  }t        |      t        j                  u xr |j                  d k(  xs& |t        j
                  j                  j                  v  Z yw)r   N)r<  rj  BuiltinMethodTyper   r   rc  _pytreeBUILTIN_TYPESr#  xs     r   r%  z(VariableBuilder._wrap.<locals>.<genexpr>  s[      	.
  aE333O8O 7 U[[((6667	.
s   AA r5  r   funcargskeywordsz*functools.partial with non-literal keyworddtypesaved_tensors)r   r  __self__r    apply)r$  c                     t        t        j                  j                  j                  j
                        j                   j                   j                  j                  j                  j                         fi       S r   )r   r   _dynamoexternal_utilsFakeCompiledAutogradEngineexec_final_callbacksr  r  r  r-  get_ca_final_callbacks_varr   s   r   r  z'VariableBuilder._wrap.<locals>.<lambda>Z  sZ    ,MM00KK``-GGWW^^00KKMO r   TzHDirectly using invoke_subgraph is not supported. Use mark_compile_regionstreamr  r   )	stream_iddevice_indexdevice_typeenter_result)dynamic_dimzaunbacked symint input is not supported yet. If you need this feature, please file a github issue.[^a-zA-Z0-9]+r  F)r   r   grapharg is not a basic Symbol.rk   )rankr%   )target_valuesr   __script_if_tracing_wrapper__original_fn__wrapped__)      r~  z"Failed to produce a valid self obj)FlattenScriptObjectSourceScriptObjectQualifiedNameSource__obj_flatten__c                     t        j                         |       }t        j                  ||      }t	        j                         |      }t        j                  ||      }||fS r   )r<   r>  r   r  r>   )r  r  r  r  r  r  r5  r   s          r   r  z.VariableBuilder._wrap.<locals>.build_key_value  sW    /0A1E
)00J?01BJO+221lCEz!r   c              3   >   K   | ]  \  }\  }} |||        y wr   r   r  s       r   r%  z(VariableBuilder._wrap.<locals>.<genexpr>  r  r  )r  mutation_typer   )r  r   )r   r  )tuple_vtr   )list_vtr   c              3   F   K   | ]  }t        j                  |        y wr   r  r  s     r   r%  z(VariableBuilder._wrap.<locals>.<genexpr>J  s     Aa#..q1Ar  z2dict_keys with non-constant keys are not supported(@  torch.utils._tritonr  r  triton.runtime.autotunerr  triton.runtime.jitr  $triton.tools.experimental_descriptorr  r  rJ  getr<  r  r"  r"   r_   r&   traceable_tensor_subclassesrP  rZ   r.  r/   SEQUENCE_LENGTHr`   r   r  getattrr9   r   r   r  rS  defaultdictOrderedDictrz  r  r  r  r  guard_on_key_orderr4  r   	enumeraterS   rz   r)  r  ry   r-  r  r   r   rL  Modulewrap_modulerq   r  r\  	overridesTorchFunctionModer   r3  	frozensetSourcelessBuilderrv  r|   enumEnumDispatchKey_C
_functorchTransformTyperr   r   is_reorderable_logging_functionBUILTIN_MATCHloggingLoggerr  r   r]   rM   rX   rN   	functoolspartialr>  r  r  rn  r@   r  r  r-   r>   r0   rA  DICT_KEYS_MATCHr   r\   r   rm  genericrp  asarrayr*   is_numpycallabler   is_numpy_dtyper   is_numpy_type_infoiinfor   r   can_rewriteautogradfunctionFunctionMetar   FunctionCtxr  RuntimeErrorrQ  r   r   rj  
MethodTyper  r   _ImperativeEnginer   r  r  r  _exec_final_callbacks_stubr   lookup_callableis_callable_allowed!has_user_defined_allowed_in_graphcreate_with_sourcenumberwrap_unspecialized_primitiver   _higher_order_opsinvoke_subgraph
NAME_MATCHr   makecudaStreamContextr  rw   Streamcreate_proxyr  r  r  rd   noderx   device_SDPAParamsr   pyfunctorchFuncTorchInterpreterr   Eventrc  store_user_object_weakrefget_user_object_from_idrt   _TensorMetar   
contextlibnullcontextr   getattr_staticru   r   is_matching_objectoptim	OptimizerrG   r   DispatchKeySetDISPATCH_KEY_SET_MATCHr   r   is_group_member_typer   is_process_groupr   is_device_meshEQUALS_MATCHr   is_placement_typer   is_placementr&  r'  r   r   r
   r=   has_hint	shape_env$create_unspecified_symint_and_symbolinthintr   DYNAMICSymBoolcreate_unbacked_symintroot_tracercreate_graph_inputr]  subr   metaexprsympySymboltracked_fakesTrackedFaker   r   r   ampautocast_modeautocastrs   
fast_dtype_enabled_cache_enabledr   is_matching_clsr   rY   	tracebackclear_framesr   sysexc_infoversion_info	exceptionr   rW   wrapsr   
namedtupler   r  r   is_supported_builtin_methodri   r0  randomRandomr   r   is_supported_random_obj
ModuleTyper)   DummyModuler   r  TreeSpecro   r   __func__GetSetDescriptorTyper   MethodWrapperTyper   
issubclassBaseExceptionr   rd  BackwardHookrM  Buffer _unsafe_preserve_version_counterrv   constructorr   r   r  r  _libraryfake_class_registrytracing_with_realr   wrap_user_definedrealize_all_typequalified_namer  maybe_to_fake_obj	fake_mode__new__rm   should_reconstruct_allr   rZ  __len____getitem__r   r   rR  rD   r   r   r   r   rV   r   rQ   r{   )3r   r5  r  r  r  r  r  r  type_dispatchid_dispatchr   r  rr  factory_sourcevarr  r  func_srcfunc_objr  args_sourcer  argr  keywords_sourcer  	dt_sourceactual_saved_tensorsr  rC  saved_tensors_sourcestream_source
stream_varstream_proxyevent_proxyr   
new_symintsym_node_proxysym_exprtracing_symint	attr_nameself_objr  r  proxyfake_script_objr  r  r  r  r  s3   `                                                @@r   r1  zVariableBuilder._wrap9  sC   B<:6     ++-11$u+>$ u-- '')--bi8"tU++ )/66554
 ##E**5! < <= .d5k:
 	 $**E4(%dkk48F  ($u+dkkF MED+"9"9;;R;RST 7 78QEJJLQQI " 1155dkk6F6F6HI""  !*+>u+E!F F
 e[445!+DKK9J!K,K$LODGG^$L--%  ;; +T%[ 77>>..<<UFKKuxx/##E**((/$$U++u@@A+E$++FCGGNN''==eSIJE9%# 	.
 	.
 	+
 DIIa&--dggq9IEI 5 56$U4;;??DIIu00%((2E2E2S2ST
  5 56eDKK@@>>uE  : :;$U4;;??w~~. ; ;<(t{{CC ',DKK@@&1EEy001!$//"3V<H9tww9%**EHD$T__%6?K#EJJ/ 3KODGG];-JKCP
 H():JGO,,. 1'2215!"NOoGG.B !,,\-D-DE**<+G+GH&&|'C'CD
 ,HdHEEu 5 56!{{  ^
5"** =**2::e+<==!!%(I2E? ++!,,
 !t{{;;''. 5 56%eDKK@@++E2%*##L$;$;<&t{{G<	i22<3H3HIJ##L$9$9:(t{{CC.::5A ; ;<4;;{{ 
 E5>>22??@ ; ;<+{{  u~~66BBC#' ',':':$ Mkk,,\-D-DEFF.6'1$++'O$(33L4P4PQ &&:; DAq!(( GG]3G%K 6"77>>..DD/;;"0"?  ue../z40%..2I2I2V2V z2.'9$??  ; ;<"(NN:dkk*+U 	  uxx99: 5 56)%DD}}++FFaab  ; ;<!  e_!<!<U!C!O..u5CG@..u5HHdkk I   Jubii044U;;23//???^  7 79P9PQ388t{{SSuzz778 5 56&t{{H=M@-@NJ(//DDu||, 5 5677>>66U!&$)$6$6#(#4#4		L l//7!{{	   4 46 7 78%,,TWWeT[[IIu//;;PPQ 5 56/66u{{+ 5 56MM99%@''..55##;;E	K k..6 {{  uehh223;;;)%DD5*001&&undCK 7 78&dkk::&99%@ 7 78.uT[[IF77>>..DDUFSSu{{445 5 56)$++6DK$U4;;??u334 C CD)%00#88?)%DD!2259 5 56'dkkBB..u5 9 9:%eDKK@@#55e< 5 56)%DD++E2 9 9:${{ 
 uI++5	!>> ; ;<$U4;;??%  eU\\2 %dkk2 
 zz""$GGNN,,QQEJJOO,$.$6$6 R   eU]]3!%!9!9!P!P!RJ "{ "WW^^77JJ'dii8Z 	 K N 4<#-4N$$Z0 "++Hh5 *345 GGNN((//JPT0UV )=
:QR?  #>>BBY78 5 56'{{	  ...A66...A66uyy66??@ 5 56'LL$$NN((	 {{  *99%@ ; ;</dkkJJ##E+H%P 7 78.t{{  +51 7 78.umDKKXXi,,,$DKK88cll"'ES]],B&uT[[AA#E*w/E/E(%0
  7 78.,T[[  ioo% 5 56)%DDk,,, 5 560t{{KK5**
#??F 5 56(t{{CC#E*?FE9 $(i@%%e,??dkk @   fmm# 5 56&dkk::E6==)n.T.T/
  7 78#E$++>FGGNN''55eVDM  0 0-2K2KLM ; ;<){{F GGNN''==eVLMu//0ZNNUXX__ekk.A.A.J.JK6

4;;
 CnnH 
8_ E 4E  ; ;<%{{ 
 u99: ,E22u667
 )//UT*z%/O 5 564U;;UT*!!..""  #))%0CC$++ D   GGG##L$?$?@5AA$''JJ  5 56+{{  '66tE{C 
 ~~11CCEJ22EEFF+S$))<K;;	 F  /7KKtUE/


+ 177;;  5"34--e44  ++V)H)UVKKM002  ++P)B4;;)OP))+ $nn@@RR((%O GGNN..AA'dii8U{{	 B E +3UE4+EJJOOJ' -33{{  ut[%<%<=>U##t||3  7 78 < <= GGNN--11$++2B2B2DE"  !*+>u+E!F F
 ( "%)@)@A  ++35{{	G .2G*,UGDKKXF77>>..DDUFSSu%$u+*=*=*N 7 78 < <= u}}U34
 	 $**%%eQ/():A>F  %t{{:O:QH .44F 77>>..DDUFSSt$ 7 78 < <= t||E23
 	 $**$$UA.,T__->BF  #t{{:O:QG -UGDKKXF77>>..DDUFSSU^4 7 78)%DD ' 7 78,33DGGU4;;WF77>>..DDUFSSy)A5AA HMM!*11$''1=MMOO%001M1MNOO%001J1JK *%DDRS))%00gH Jz   t*4 Ns?   &<Be	!'Be&Be VABe#[ABe(b'Be-e	Be eBe c                    | j                  t        j                         t        || j                        }t        j                  t        |            s|S | j                  j                  j                  j                  ||      S ry  )r.  r/   rz  r   r   r7   "cls_supports_mutation_side_effectsr<  r  r  r-  r3  r   r5  rr  s      r   ro  z!VariableBuilder.wrap_user_defined[  s^    L334*5E==d5kJMww~~**@@OOr   c                    t         j                  rPt        |      t        j                  u r5| j                  t        j                         t        j                  |      S | j                  t        j                         t        |t              rt        d |D              rs| j                  j                         j!                         rK| j                  t        j                         t#        |D cg c]  }t        j                  |       c}      S t%        |      D cg c]4  \  }}t'        j                  |t)        | j+                         |            6 }}}| j,                  j.                  j0                  j3                  d      }t5        | j                  t6              r| j                  j8                  t;        |      v r| j                  }t5        |t<              sJ | j,                  j.                  j>                  jA                  tC        jD                  dd| jF                        t        |      ||      }d|jH                  jJ                  d<   tM        tN        | j,                  ||d |	      }g }	t%        |jP                        D ]  \  }}
t)        ||d
      }|
| j,                  j.                  jR                  |<   tU        ||         |
jV                  jH                  jJ                  d<   tY        jZ                  t        j\                  t_        ||               }|	ja                  |jc                  |              te        |	ddi tg        ||d
d d
      }||jH                  jJ                  d<    ti        jj                  |      || j                        }t        |t<        tl        jn                  f      r0| j,                  j.                  jp                  js                  ||      S |S c c}w c c}}w )Nr5  c              3   F   K   | ]  }t        j                  |        y wr   r  )r#  items     r   r%  z0VariableBuilder.wrap_listlike.<locals>.<genexpr>q  s     H$$//5Hr  r   r   r  r  T	steal_arg)
target_clsr  r  example_valuesubclass_typer   Fbaseindexindex_is_slicetensor_dictr@  rk   )r   r   r   r  ):r&   specialize_intr<  r   rU  r.  r/   CONSTANT_MATCHrq   r  r  r_   rQ  r  r   guard_sourceis_unspecialized_nn_moduler   r  r   r@   r>  r  r  local_scoper  r   rE   
local_namerT   rR  rD  rE  r]  rF  r   r%  rG  wrap_fx_proxy_clsr   r  input_source_to_varrL   r  r  r  TENSOR_MATCHr$   rn  rA  r0   r   r   cls_for_instancerS  rT  r-  r  )r   r5  r  r  r  maybe_gmr   tensor_list_proxylist_variablerC  tensor_variablesource_irD  r  rr  s                  r   rV  zVariableBuilder.wrap_listlikec  sE     T%[EJJ%> ; ;<#**77 	L889 5% H%HH((*EEG ; ;< E!RD"2"9"9$"?!RSS %U+

 4	  &&$T__%6:
 
 77>>--11&9KK
kk$$(;H(EE [[FeT*** $ : : M M'dii8U	 !N ! 8<""''4-)77'#"M F&/0C0C&D :"?(fAeT?N228<AU!HB%%**//> ")) --]585L h11%89: 6**#( H 7?""''
39!2259&U%$ 1 12377>>..<<UFKKK "S
s   2O9#9O>c                    | j                  t        j                         t        t	        |            D cg c]@  } t        | j                  t        | j                         |            t        ||            B }}t        || j                        }| j                  j                  j                  j                  ||      S c c}w ry  )r.  r/   TUPLE_ITERATOR_LENrZ  rh   r)  r  rK   r>  rg   r   r   r  r-  r  )r   r5  r  r  rr  s        r   rW  z#VariableBuilder.wrap_tuple_iterator  s    L;;<
 -e45	
  WODGG%?@QST%UV&ua0
 
 'vdkkBww~~**88GG
s   ACc                 B   | j                  t        j                         t        j                  |      D cg c]  }t        j                  |       }}t        || j                        }| j                  j                  j                  j                  ||      S c c}w ry  )r.  r/   RANGE_ITERATOR_MATCHcopydeepcopyrq   r  r   r   r  r  r-  r  )r   r5  r  r  rr  s        r   rX  z#VariableBuilder.wrap_range_iterator  sw    L==> 6:]]55IJ!((+JJ%eDKK@ww~~**88GG Ks   Bc                 \   dD cg c]@  } t        | j                  t        | j                         |            t	        ||            B }}| j                  t        j                         t        |t              rt        || j                        S t        || j                        S c c}w )N)startstopstepr   )r)  r  r9   r>  r  r.  r/   rz  r   rY  r   r   r   )r   r5  r  r  s       r   r[  z VariableBuilder.wrap_slice_range  s    
 /	
  GODGGZ0A1%EFq!
 
 	L334eU# t{{;; t{{;;
s   AB)rD  c                    ddl m} t        j                  d| j                  j                         t        |              |||       || j                  j                  j                  v rT| j                  j                  j                  |   }|j                  |j                  j                  j                  d   d<   y y )Nr%   )mark_static_addressz Marking static input %s, id: %s)rD  r  _dynamo_static_input_type)
decoratorsr  static_inputs_logdebugr   r   r"  r  r  r-  r  r  r%  rG  )r   r5  rD  r  r|  s        r   mark_static_inputz!VariableBuilder.mark_static_input  s    4.0@0@0BBuI	
 	E/ DGGNN///''..--e4C// IINN./JK 0r   c                 	   ddl m} t        |j                        dk(  r7t	        dt        |      dt        |       dg t        j                  d       t        ||      rt        j                  |j                  d	d
      rt        | j                        S | j                  t        j                          t#        | j                  d      | _        | j%                  |j&                        S t)        |t*        j,                  j.                  t*        j,                  j0                  t*        j,                  j2                  f      rt4        j6                  st9        d       t;        |dd
      rt;        |dd
      st9        d       | j                  t        j                          t=        || j?                               }tA        jB                  tE        |            s|S | jF                  jH                  jJ                  jM                  ||      S tO        jP                  || jF                  jR                        rt)        |t*        jT                  jV                  jX                  jZ                        r*|j]                         }t_        | j                        | _        | j                  t        j                          t*        j`                  j4                  jb                  rte               }|jg                         D ]  }| ji                  ||        |jk                         D ]  }| ji                  ||        |r-|| jF                  jH                  jl                  | jn                  <   |jp                  js                  d      st;        |jt                  dd
      rtw        || j                        }nty        || j                        }tA        jB                  tE        |            s|S | jF                  jH                  jJ                  jM                  ||      S t{        |jt                  t*        j,                  j|                  j~                  j                        r:| j                  t        j                          ty        || j?                               S | jF                  jH                  j                  || jn                  | j?                               S )Nr%   )OptimizedModuler   zUninitialized nn.Modulez6Attempted to trace an uninitialized nn.Module of type .z?Ensure your nn.Module instance has called `super().__init__()`.)gb_typecontextexplanationhints_torchdynamo_disableFr   	_orig_modz5TorchDynamo purposely graph breaks on RNN, GRU, LSTMs_is_fsdp_managed_module_fsdp_use_orig_paramsz3Dynamo only supports FSDP with use_orig_params=Truer  )z	torch.nn.z	torch.ao._dynamo_marked_static)B
eval_framer  len__dict__r.   rl   r'   
USER_ERRORr_   r   r0  forwardr   r   r.  r/   rz  r9   r  r  r   r   rL  RNNGRULSTMr&   	allow_rnnr-   r  r   r>  r7   r  r<  r  r  r-  r3  r(   is_dynamic_nn_moduleexportfxexperimentalproxy_tensor
_AttrProxyget_baser8   r  inline_inbuilt_nn_modulesr[   
parametersr  buffers
nn_modulesr   r   r  r  r   r   rf  paralleldistributedDistributedDataParallelregister_attr_or_module)r   r5  r  rr  freezingpbs          r   r  zVariableBuilder.wrap_module  s   0u~~!#1TU\]bUcTddef&11U	 %)%%emm5KUS
 /dkkBB 7 78$T[[+>DK##EOO44 uuxx||UXX\\588==IJ$$QR53U; 5"95AST  7 780t?PQFAA$u+N77>>..DDUFSS00G
 %!6!6!C!C!N!NO(-dkk: 7 78}}##==02))+ >A**1H*=>  >A**1H*=>  <ADGGNN--dii8**+EF'!8%K >eDKKX6uT[[QAA$u+N77>>..DDUFSSOOUXX..::RR
  7 780t?PQQ77>>99		( :  r   c                    t         j                  sxt        |      t        u rgt	        || j
                        r@| j                  t        j                         t        j                  || j
                        S | j                  |      S t         j                  s"t        |      t        u r| j                  |      S | j                  t        j                         t        j                  || j
                        }t        |t         t"        f      r0| j$                  j&                  j(                  j+                  ||      S |S )Nr  )r&   r  r<  r?  r   r   r.  r/   r  rq   r  wrap_symintspecialize_floatfloatwrap_symfloatr   rR  setr  r  r-  r  r  s      r   r\  zVariableBuilder.wrap_literalU  s    $$e); *%=##L$?$?@'..U4;;OO''..((T%[E-A%%e,, ; ;<%,,5MF%$-ww~~22@@OOMr   c                 v    t        |      r.t        |      | j                  j                  u rt	        dd      y y )Nz*Cannot wrap a Tensor that has already beenz"wrapped by this instance of Dynamo)r   r   r  rt  r,   rw  s     r    assert_not_wrapped_by_this_graphz0VariableBuilder.assert_not_wrapped_by_this_graphg  s;    5>1%8DGG<M<MM*<4  N>r   c           	      "   | j                         }|| j                  j                  j                  vsJ t	        |      d u}t
        j                  rc|sat        |t        j                  j                        s |r;|j                         j                         r| j                  |t                      d}|xrC t
        j                   xs0 t               xs$ t        j                  j
                  j                   }|j                         j#                         s|ra|j                         j%                         sC| j'                  |       | j                  j                  j)                  || j*                  |      S t	        |      dk(  rb| j-                  t.        j0                         | j'                  |       | j                  j                  j)                  || j*                  |      S t3        |      rX| j'                  |       | j                  j                  j)                  |t5        j6                  dd| j*                        |      S t9        |      t
        j:                  v rt9        |      }nt9        |      t        j<                  t        j                  j                  t        j>                  j@                  jB                  t        j>                  jD                  jF                  fv stI        |      sJ t9        |             d }|| j                  j                  jJ                  v }|r#| j                  j                  jJ                  |   S t	        |      dk(  r| j-                  t.        j0                         | j'                  |       i }t9        |      t
        j:                  v rCtM        | j                  || jN                        |d<   | j-                  t.        jP                         t        |t        j<                        rO|jR                  rCt        |t        jT                  jV                  jX                  jZ                        st]        d       t        |t        j<                        r<t_        |      r1| j                  j`                  rt
        jb                  st]        d	       te        |      r8tg        |      -|jh                  tg        |      jh                  k7  rt]        d
       tk        || j                  d|      }| j                  j                  jl                  jo                  t5        j6                  dd| j*                        t9        |      ||      }	tq        | j                  |	|       ts        d| j                  |	|||d|}
|ju                         r-tk        |jv                  | j                  ty        |d      d       t.        jz                  }t        |t|              rt        |      rt.        j                  }| j-                  t        j                  |t        |t              r|n
t        |                   tI        |      r| j-                  t.        j                         | j-                  t.        jP                         t        t        |      j                  t.        j                               |j                         \  }}|D ]S  }t        ||      }ty        | jN                  |      }t        j                   t        | j                  |      |             U |
| j                  j                  jJ                  |<   d|	j                  j                  vsJ t        |      |	j                  j                  d<   |
j                  j                  j                  d   }t        |      | j                  j                  urt        d      t        ||d|      }||	j                  j                  d<   |
S )Nr  Tr   guardedr  r  torch_function_fn3torch.compile does not support strided NestedTensorz-torch.compile does not support sparse TensorszInconsistent dtype between tensor and its gradient. This can happen in FSDP and crashes meta tensor creation. This is potentially a workaround. Fixing it correctly requires some design around FSDP + torch.compile.r  r   r   )r  r  r  r  r   _base)r  r   r   r  r  r  z(Wrapped Tensor must be this graph's fakeFr  r   )Wr>  r  r  r-  rU   r&   r  r   r   rL  rM  r  r  r  r[   r  prepare_freezingis_specialized_nn_moduleis_fsdp_moduler  r  r   r.  r/   rv  rB   r]  rF  r<  r  r   r  r   r   rN  rO  r"   r  r   r   rz  	is_nestednested	_internalnested_tensorNestedTensorr-   r   r  capture_sparse_computer   r   r  wrap_to_fake_tensor_and_recordrD  rE  cache_real_value_when_exportwrap_fx_proxy_is_viewr  r9   r  rA   rC   NOT_NONE_MATCHr  r  rF   r$   TENSOR_SUBCLASS_METADATA_MATCHr0   rJ   rA  r9  __tensor_flatten__r  r   rp  r)  r%  rG  rL   r  r   rt  r,   r   )r   r5  r   is_static_inputmake_graph_attributer  is_duplicate_tensoroptionsr  tensor_proxyr  
guard_typeattrsr  attrinner_valueinner_sourcefake_tensor_valuer  s                      r   rP  zVariableBuilder.wrap_tensorn  s   " DGGNN777771%8D ,,#5%(("4"45 v224OOQ ""50E0G"H"O.  
000 5$&5}}##44 	 !::<@T%%'66811%877>>99tyy :   #5)Y6  5 5611%877>>99tyy :   f%11%877>>99'dii8 :   ;&<<<$ !KM;""!!--88!!33DD	# 
 /u5C
 8<E{C 6 !M %(J(JJ77>>55f=="5)Y6 5 56 	--e4;&<<<+B,G'(  7 78 uell+uell&<&<&J&J&W&WXOP uell+e$WW^^6+H+H
 IJ % % ,y/555D  7dggf
 ww~~11DDFF#S$))4K	 E 
 	%TWWlEB' 
ww''
 
 >> +77!&'2	 "..
fj).Fv.N%44J "&*;< &u-		
 )/ K KL 7 78&v.99,:S:ST //1HE1 %eT2)$++t<#//:ODGG\:;G 6E**62L$5$5$:$::::0DU0K}- ,1166;;OL019J9JJ*+UVVFE52CD-5z*r   c                    t         J t        |t         j                        sJ t        | j	                               }ddlm} |j                  j                   }|r	 d|j                  _        t        j                         5  	 |j                  |      }|rddlm}  ||      }d d d        t)        j*                   t-        | j.                  |                   t1        || j.                  d|      }| j.                  j2                  j4                  j7                  t9        j:                  dd| j<                        t?        |      ||	      }	tA        | j.                  |	|       d
|i}
tC        dtD        | j.                  |	|d|
}|| j.                  j2                  jF                  |<   |jH                  jJ                  jL                  d   }tO        ||d|d|      }||	jJ                  jL                  d<   |S # t        $ r* t        |j                  t         j                        sJ Y w xY w# t"        $ r}t%        t'        |             Y d }~d }~ww xY w# 1 sw Y   xY w)Nr   )_utilT)clone_preserve_stridesFr  r  r  r   r   r  r  r  r  r  r   r   r   r   r  r   )(rm  r   ro  rF   r>  torch._numpyr  flags	writeable
ValueErrorr  nditerr   temp_restore_stack_try_convert_to_tensortorch._prims_commonr  rB  r-   strr   rp  r)  r  r   r  rD  rE  r]  rF  r   r<  r  r  r   r  r  r%  rG  r   )r   r5  r   r  readonlytensor_valuer  er  r  r
  numpy_ndarray_variabler  s                r   rp  z"VariableBuilder.wrap_numpy_ndarrayP  s(   ~~%,,,"4??#45&{{,,,9(,% 1CCE 		&&$;;EBJ#9,#GL		& 	''(H(H(VW6ww	
 **==FF#S$))4	 > 
 	%TWWe\BV$!2 "
+ww'	"

 "
 6L**62.4499>>O #%+
 '/


#%%u  9 "%**bii8889 ' &c!f%%&		& 		&sB   G< I!H2</H/.H/2	I;IIIII'c                    t        |      t        u sJ | j                  | j                  j                  j
                  v r-| j                  j                  j
                  | j                     S | j                  j                  j                  }t        j                         j                  ra|j                         }t        |       | j                  j                  j                  j                  t        || j                  d              n#t!        | j#                               st$        j&                  j(                  j*                  r@| j-                  t.        j0                         t3        j4                  || j                        S | j                  j                         }t7        | j                  |t9        j:                  |      | j                  j=                         j?                               }tA        | j                  j                               }| j                  }tC        |tD              r|jG                         }| j                  j                         tI               v r?tJ        jM                  d| j                  j                                tN        jP                  }nt(        jR                  r|jT                  tV        u rtY               }ntC        |tZ              r6|j\                  *t_        |j\                        j                  |ddi      d   st(        j`                  stN        jP                  }n5| j-                  t.        j0                         t3        j4                  |      S |jc                  || j                  |      }| j                  j                  j                  j                  t        || j                  d              n[t!        | j#                               sJ | j-                  t.        j0                         t3        j4                  || j                        S tC        | j#                         td              rJ tg        | j#                         ji                  t.        jj                               d| j#                         i}	| j                  j                  jl                  jo                  tq        jr                  d	d
| j                        t        |      || j#                               }
|jt                  jv                  }tC        |tx        jz                        s
J | d       |
| j                  j                  jl                  j|                  |<   t        |
|fi |	}|| j                  j                  j
                  | j                  <   t!        | j#                               s| j                  j                  r=tC        | j#                         tZ              st        d| d| j#                                t        | j#                         |dd d|      |
jt                  j                  d<   |S )Nr  r  &%s marked dynamic via source whitelistr   Fr  )r   r  r   r  r  r   r  =Dynamo attempts to add additional input during export: value=	, source=r  r  )Dr<  r?  r   r  r  unspec_variable_mapr=  r   r  #force_unspec_int_unbacked_size_likerC  r   rK  rn  rL  r   rB   r>  r   r  r&   r  r.  r/   r  rq   r  r6   r4   make_scalarr  r  r   r   r;   r  get_dynamic_sourceslogr  r   rA  automatic_dynamic_shapesscalarr2   $get_automatic_dynamic_shapes_mark_asrE   dynamismr  assume_static_by_defaultr>  rH   r0   rA  rz  rD  rE  r]  rF  r%  rH  rI  rJ  bound_symbolsr   r  AssertionErrorr   rG  )r   r5  r=  wrapped_valuer   frame_state_entrynormalized_source_namebase_sourcer  r
  r  r  
unspec_vars                r   r  zVariableBuilder.wrap_symint  s   E{c!!!99:::77>>55dii@@GGNN,,	CC%<<>M%m4GGNN((//M4;;= $DOO$56}}##22 ##L$?$?@'..U4;;OO;;##%D 9#//6+/;;+C+C+E+`+`+b	! &;4;;;K;K;M%N"++K+}5)224{{!%8%::		BDKKDTDTDVW(00//%,,<BD;4((4--.223IAu:V 44(00 ##L$?$?@'..U;;%JJ{{' K M GGNN((//M4;;= &doo&7888 ; ;<#**t{{KKdoo/1BCCCdoo'22<3J3JKLT__./**==FF#S$))4??$	 > 
 !%%**(ELL1WhZ?V3WW1=B""00:$UMEWE
8B**4995!$//"34ww~~j1BK&P$STaSbbklpl{l{l}k~  +3!#( #0+EJJOOJ' r   c           
      
   | j                   | j                  j                  j                  v r-| j                  j                  j                  | j                      S t	        | j                  | j
                  j                         t        j                  |      | j
                  j                         j                               }t        j                  j                  j                  st        | j                               svt!        j"                  |      sat!        j$                  |      sLt        j&                  j                  j(                  j*                  st-        dd      s|j.                  t0        ur@| j3                  t4        j6                         t9        j:                  || j
                        S t        j<                  |t        j>                        }t        j@                  jB                  jE                  |      r@| j3                  t4        j6                         t9        j:                  || j
                        S tG        | j                         tH              rJ tK        | j                         jM                  t4        jN                               tQ        | j                               }||d}tS        || j                  d|      }| j                  j                  jT                  jW                  tY        jZ                  dd	| j                         t]        |      ||
      }t_        | j                  ||       ta        tb        f| j                  ||d|}tG        |tb              sJ || j                  j                  j                  | j                   <   | j                  jd                  r=tG        | j                         tf              sti        d| d| j                                d }	|jj                  jl                  jn                  d   }tq        |      sJ |}	|	jr                  | j                  jr                  u sJ d|	jr                   d       tu        | j                         |d|	d|      |jl                  jn                  d<   tw        | j                   | j                  j                  jx                  ddgt{        |gi              }
| j                  j                  j|                  j                  t        |
j                  | j
                  d              t               j                  ddd       |
S )Nr%  z.pytorch/compiler:unspecialize_float_killswitchFr  )r  )r   	raw_valuer  r  r  r   r  r  r  r'  r(  r  fake mode (^) from fake tensor metadata doesn't match mode({self.tx.fake_mode}) from InstructionTranslatorTr  r  call_methodr  tensorify_float_attempt)	overwrite)Dr   r  r  r)  r6   r   r4   r+  r  r  r   r  r&   r  rB   r>  mathisnanisinf	_inductortriton
cudagraphsr   r/  r2   r.  r/   r  rq   r  tensorfloat64r  r  is_gradtrackingtensorr   rH   r0   rA  rz  r?   r   rD  rE  r]  rF  r<  r  r  r   r  rE   r4  r  r%  rG  r   rt  r   r  r$  rb   rK  rn  rL  sym_numr   r  )r   r5  r6  r5  r   r
  r  r  r9  r  r   s              r   r  zVariableBuilder.wrap_symfloat  s    99:::77>>55dii@@5GGKK++E2'+{{'?'?'A'\'\'^	
 MM  11!$//"34zz% zz%  %%,,77OQVW ''|; ; ;<#**t{{KK U%--@
 8844]C ; ;<#**t{{KK doo/1BCCCdoo'22<3J3JKL #4??#45#%8 7dggv
 **==FF#S$))4	 > 
 	%TWWe]C&'
ww'	

 

 *&ABBB8B**499577>>*T__->"L OP]^ghlhwhwhygz{  !"((--22?C}%%%) **dgg.?.?? 	
+556 7? ?	
?  '/OO#),'



# GG'DGGNN'' #J<4
 	$$++K		4;;PT,UV!!";TT!Rr   c                    | j                   | j                  j                  j                  v r-| j                  j                  j                  | j                      S t	        j
                  |      }t        | j                         t              s6t        | j                         j                  t        j                               d| j                         i}|j                  d|i       t        || j                  d| j                               }| j                  j                  j                  j!                  t#        j$                  dd| j                         t'        |      || j                               }t)        | j                  ||       t+        t,        f| j                  ||d|}|| j                  j                  j                  | j                   <   t/        | j                               s| j                  j0                  r=t        | j                         t2              st5        d	| d
| j                                d }t        |t6              r|j8                  }n#|j:                  j<                  j>                  d   }tA        |      sJ |}|jB                  | j                  jB                  u sJ d|jB                   d       tE        | j                         |d|d|      |j<                  j>                  d<   |S )Nr   r;  Fr  r  r  r   r<  r'  r(  r  r=  r>  Tr  r  )#r   r  r  r)  r   rH  r   r>  rH   r0   rA  r/   rz  updater   rD  rE  r]  rF  r<  r  r  r   rB   r  rE   r4  rq   r5  r  r%  rG  r   rt  r   )r   r5  r5  r
  r  r  r9  r  s           r   r  z,VariableBuilder.wrap_unspecialized_primitive  s   99:::77>>55dii@@U+$//+->?$//+66|7N7NOPT__./U+,6dggt?P
 **==FF#S$))4??$	 > 
 	%TWWe]C&'
ww'	

 

 9C**4995!$//"34ww~~j1BK&P$STaSbbklpl{l{l}k~  !%*&67 * 0 0 * 0 0 5 5 : :? K=))) -$..$''2C2CC /99: ;C CC
 +3!#'-#0+EJJOOJ' r   )<r   r   r   __doc__rI   r  r:  r2  r>  r.  classmethodrJ  r  	lru_cacherG  r]  r^  r_  r`  ra  rb  rf  ri  rl  r  r?  r   r   ro   r  r1  ro  r	   rQ  rR  ra   r   rV  rf   rW  rc   rX  rY  rZ  r[  r   r   r  r  rL  r  r  r\  r  rP  rp  r  r  r  r  r  s   @r   r)  r)    s   =" " 
	"0
 ; ; Y$  $L+

 +
I'"7"7 I<
H4 Y	c8.4oEFF	G  4`1DPs PW5lJ)N#O Wr	H 	HH H<eE5L&9 <u|| D  g gR$ell ` `DF&PrhGR9r   r)  c                 N   t        | t              r| j                  }nt        d|         g }t	        j
                        D ]V  }d }| j                  r*t        t        | j                  d      |j                        }|j                  t        ||             X t        |      S )Nz)Dataclass fields handling fails for type __dataclass_fields__r   )r   r   r5  r-   r  r  r   r>   r9   r   rn  r   )objr5  r  fieldr   s        r   r  r    s    #01		A#GHE##E* F::&3::'=>

F 	.uVDEF r   c                     t        | t        j                        rIt        | t              s9t        j                  |       rt        |       |u s| j                  st        |       } | S r   )r   r   r   r   _is_functional_tensorr   r  rO   )r5  rt  s     r   _clone_inputrW    sQ    %& uj) ++E2'.);  &ELr   r  c                     | |||d|}|t        ddt        i|S t        ddt        i|}|j                  |        |S )N)r  r  r  r  r  r   )r  r   r   install_global)r  r  r  r  r
  kwargsrr  s          r   r  r    sb     &&	
 F  ENEfEE"U.JUfUb!r   c                     | j                   r_t        j                  j                         5  t	        || j
                        |j                  j                  |j                  <   d d d        y y # 1 sw Y   y xY wr   )	r  r   r  DisableTorchFunctionSubclassrW  rt  tracerreal_value_cacher%  r<  s      r   r  r  	  s`    	yy XX224 	8Dr||9ELL))%**5	 		 	 	s   8A--A6c                     |t        | ||||fi |S t        |t        j                        rt	        | ||||fi |S t        ||||||       S r   )_wrap_fx_proxyr   r   r   _wrap_fx_preexisting_tensorhandle_traced_output)r  r  r  r  r  r
  s         r   r  r  <	  sv     E=-
CJ
 	
 
M5<<	0*E=-
CJ
 	
 $2ug}j
 	
r   c                 z   ddl m} t        |t        j                        sJ dt        |              t        ||      sJ d|v r-|d   (|j                  j                  j                  |d          |j                  j                  dk(  r#d|j                  j                  v sBJ d| d       d|j                  j                  vsJ |j                  j                  d           t        j                  j                  j                         5  t        |      |j                   u rnt#        |||       |j$                  r^t        j&                  j)                         5  t+        ||j                         |j,                  j.                  |j                  <   d d d        d	| t0        t2        fv i}d
|v r|d
   J |d
   |d
<   t5        |fd|i|}|j6                  j
                  dk7  r%t        |      |j                   urt9        d|       d d d        t;        ||||||       S # 1 sw Y   xY w# 1 sw Y   %xY w)Nr%   InstructionTranslatorBasez1_wrap_fx_preexisting_tensor expected tensor, got rC  placeholderr  zplaceholder z* doesn't have 'example_value' in node.metar   r   r  rG  zN`tensor` needs to be a `FakeTensor`wrapped by this instance of Dynamo. Found: )symbolic_convertre  r   r   r   r<  r  rC  rM  r%  oprG  r  rc  +_disable_saved_tensors_hooks_during_tracingr   rt  r  r  r  r\  rW  r]  r^  r   r   r   r&  r,   rb  )r  r  r  rH  r  r
  re  rZ  s           r   ra  ra  Q	  s4    =fell+ 
;DL>J+ b34447wx0<
		 12 zz}}%%**//1 	
5'!KL	
1 ejjoo5 	
zz/0	
5
 
			H	H	J  v&",,6(UF;yy XX::< @LAELL11%**= Z"$@ABF w&78+<+HHH&x0F83FLrLVLF==''r||;*>>DXG ; D  E7M:VV1    s&   AH18H%A7H1%H.	*H11H:c                    ddl m} t        ||      sJ d|v r-|d   (|j                  j                  j                  |d          d|j                  j                  vsJ |j                  j                  d           t        j                  j                  j                         5  t        |j                  |d      }d d d        t        ||||||       S # 1 sw Y   xY w)Nr%   rd  rC  r  T)allow_non_graph_fake)rg  re  r   r  rC  rM  r%  rG  r   r  rc  ri  rR   rb  )r  r  r  r  r  r
  re  s          r   r`  r`  	  s     =b34447wx0<
		 12%**//1Xejjooo6V5WX1 
			H	H	J R 'uzz2DQ	R  r5'=* R Rs   %CCc                    dd l }dd l}dd l}t        | |j                        rDt        | |j
                  j                        }t        | |j
                  j                        }t        | |j                        } t        |j                  |        |j                  j                  j                  | |       |j                  |       }	t        | |j                   j"                  j$                        r]| j                  |j                  u rE|r|n|j                  }
|r|j
                  j                  n|r|j
                  j                  n|
|	d<   |j'                  |	        ||fi |S t)        |j                  j*                  d      ra|j                  j*                  j,                  dk(  r>t        |j                  j*                  j.                  |j0                  j2                        s-|j                  j*                  |j4                  j6                  k(  rt9        |j                  j*                        S |j                  j*                  |j0                  j:                  k(  s-|j                  j*                  |j<                  j>                  k(  rtA        |       S tC        | |jD                        r@tG        d | D              r.| D cg c]  }tI        jJ                  |       }}tM        |fi |S t        | tN        tP        f      rt        |j                  |        g }tS        |       D ]  \  }}|%|jU                  tI        jJ                  di |       -|jV                  jY                  dtZ        j\                  ||fi       }d|v r9t        | tP              sJ |d   }|j_                         }ta        ||d	
      |d<   n|}|jU                  tc        d||||d|        t        | |jD                        rtM        ||fi |S tC        | tN              rte        |fi |S tC        | tP        tf        f      rti        |fi |S | jj                  jl                  dk(  s6t)        | d      s*J d| jj                  jl                   dto        |               tq        || jj                  fi |S | "|j                  j*                  |jr                  u rtI        jJ                  di |S t        | |jt                  |jv                  |jx                  f      rI|j                  j                  j                  | |       t        |j                  |        t{        || fi |S t}        j~                  |j                  j*                        r*t        |j                  j*                  |j                        s=|j                  j*                  t               D cg c]  \  }}|j                   c}}v r.t        |j                  |        t        || | j                  fi |S t}        j~                  |j                  j*                        r*t        |j                  j*                  |j                        s=|j                  j*                  t               D cg c]  \  }}|j                   c}}v r#t        |j                  |        t        || fi |S |j                  j*                  dk(  r;|j                  j                  dk(  r"t        |j                  |        tI        | fi |S | kt        | |j                        rU|j                  j*                  dk(  r<|j                  j                  dk(  r#t        |j                  |        t        || fi |S t        | t              r!|j                  j*                  |j                  t        tZ        j\                  |j                  j                  |j                  tZ        j                  |j                  j                  j                  t        |j                  dt              t        |j                  dt              |j                  f
v s1|j                  j                  dk(  rD|j                  j*                  dv r,t        |j                  |        tI        jJ                  | fi |S t        | |j                  j<                  j                        r%ddlXmY} t        |j                  |         ||fi |S t        | t              r|j                  j*                  |j0                  j                  |j                  j<                  j                  |j                  j<                  j                  |j                  j<                  j                  dgtQ        t        j                               z   v r,t        |j                  |        tI        jJ                  | fi |S t        | t        t        t        f      rH|j                  j*                  t        u r,t        |j                  |        tI        jJ                  | fi |S t        | t              s|j                  j*                  dv r,t        |j                  |        tI        jJ                  | fi |S t        dt        |        d|j                  j                   d|j                  j*                   z   d       y c c}w c c}}w c c}}w ) Nr   
class_typer   	set_statec              3   <   K   | ]  }t        |t                y wr   )r   r?  r  s     r   r%  z'handle_traced_output.<locals>.<genexpr>	  s      3 
1c3s   r  )kindtargetr  rZ  r   Fr  r  ztorch.return_types_fieldsz	expected z. == torch.return_types or named tuple but got queryr?  record_eventget_rankget_world_size)
bit_lengthrk   r   
is_integer)hex	__round__ztorch.* op returned non-Tensor  unsupported_operator)	case_namer   r   )etorch._functorch.vmaptorch._subclasses.fake_tensortorch._utilsr   r   rL  rM  ri  rW  rt  rd   r%  r  current_tracertrack_unbacked_symbols
specializer  r   r   rM  r   rq  r   r  r  	Generatorr]  set_rng_stater   _DisableFuncTorchr!  _is_in_bad_forkr   r_   rU  r  rq   r  r   rQ  rR  r  rn  r]  r$  operatorgetitemr  r@   r  r   r!   r   r  r   r<  r   manual_seedr
   SymFloatrB  r   r   r!  rf  r#  r+   current_streamrx   r&  r*  rt   rh  r?  sym_intr  _utils_element_sizeseedmodr  vmap_validate_and_get_batch_sizer  r   _constrain_as_sizebackends
SDPAParamssdpar   r   _are_functorch_transforms_activeis_flash_attention_availablecan_use_flash_attentioncan_use_efficient_attentionr   r  r  r   r-   rl   )r  r  r  r
  r  r  r   is_parameter	is_bufferspecialized_propstensor_typer  sizesunpackedr  valproxy_ir   	options_ir  device_interfacer   s                         r   rb  rb  	  s    (-.!-1C1CD}ehhoo>	
 %]BLLA%**m4
 			  77uM&11-@ }e&7&7&C&C&N&NO''2<<7+8-ellK   ""  XX__  l+ 	()%+7++

!!:.JJ&&+5uzz((115883E3EF:: : ::+EJJ,=,=>>

UXX777::

 : ::(77	uzz	*s 3$13 0 6CC!((+CCE-W--	ME4=	1%**m4. $	FAs{$++<G<  ,,33(#++	 4  w& &mT:::$X.F 'I*7#1U+Ih'
 !(I % #-%&)	
 $9$	J mUZZ0  %;7;;M5) 5W55MD.#9:4G44 ''226JJ=)4 M33>>??mnr  tA  oB  nC  D	5
 &h0G0GS7SS		%**"3"3u7H7H"H&&7w77	MELL%..%--#P	Q
		  77uM%**m4um?w??

))*uzz((%,,7			#C#EA 	'' 
 	%**m4e]M4H4HTGTT

))*uzz((%++6			#C#EA 	 
 	%**m4UM=W==			g	%%**--=*H%**m4999!}ekk2JJ/JJMM]*%**m4UM=W==	M3	'

MMLL&&JJLL!!>>E%%z8<E%%'7B $$
	
$ JJMM]*uzz/@/@N/R 	%**m4&&}@@@	M5>>#6#6#A#A	B,%**m4!%3733	M4	(

HH55NN<<NN77NN;;
 388:
;<	< 	%**m4&&}@@@=3t"45JJ/%**m4&&}@@@	M5	)UZZ->->BV-V%**m4&&}@@@-'(%**--%**:K:K9LMN,	
c DFs   k5k:l c                     t         j                  dk(  rt        j                  S t         j                  dk(  rt        j                  S t         j                  dk(  rt        j
                  S t        dt         j                         )Ndynamicunbacked	obliviousz+invalid automatic_dynamic_shapes_mark_as = )r&    automatic_dynamic_shapes_mark_asr   rA  SIZE_LIKE_UNBACKEDOBLIVIOUS_SIZEr  r   r   r   r0  r0  s
  sn    ..);!!!		0	0J	>,,,		0	0K	?(((9&:a:a9bc
 	
r   _DYNAMIC_SOURCESc                      t         t         S t        t        j                  j                  j
                  j                  dd      j                  d            a t         S )Nr{  r  ,)r  r  r   compilerr&   dynamic_sourcesreplacesplitr   r   r   r,  r,  
  sL    #--55c2>DDSI r   c                   X    e Zd ZU eeef   ed<   eed<   ee	   ed<   de
fdZdedefdZy)	rL  faker   symbolic_contextr  c                 `    t        | j                  | j                  j                         f      S r   )hashr  r   r   r   s    r   __hash__zTrackedFake.__hash__
  s#    TYY 0 0 2344r   r  c                     t        |t              rO| j                  |j                  u xr5 | j                  j	                         |j                  j	                         k(  S y)NF)r   rL  r  r   r   r  s     r   r  zTrackedFake.__eq__
  sF    e[)99

*Xt{{/?/?/AU\\EVEVEX/XXr   N)r   r   r   r	   r   r
   r  rI   r   r   r?  r  objectr  r  r   r   r   rL  rL  
  sA    

F"
##N//5# 5F t r   rL  c           
        89 | j                   rCt        | t        j                  j                  j
                  j                        st        d       |j                         }t               }|j                  j                  j                  j                  | d       }|r|j                  nd }d }	| j                         r$t!        |d      }
t#        | j$                  ||
|      }	t'        |       r|st#        | |||d      }i }t)        |       j+                  |       \  }}|D ]+  }t-        | |      }t!        ||      }t#        ||||      ||<   - t/        |j0                  |j2                  |j4                  |j6                  |	|j8                  |j                  |      S |rz||vrvt;        t<        j>                  g| jA                         z  t<        jB                  g| jA                         z  d g| jA                         z  d g| jA                         z  |	||      S ddl"m#9 tI        9fd	| jK                         D              rt;        | jK                         D cg c]2  }t        |tL              rt<        jN                  nt<        j>                  4 c}t<        jB                  g| jA                         z  d g| jA                         z  d g| jA                         z  |	||      S | jK                         }tQ        |       s| jS                         }| jA                         }d g|z  }tU        |      D cg c]
  }||   | f }}|jW                  tX        
       i }|D ]>  \  }}| }|j                  ||      ||<   |j[                  |||   z  t]        |             @ ng }t_        ||ta        jb                  te        |      te        |                  }tg        |       }i 88fd}ddl4m5} |j                  jl                  r`|j                  jl                  D ]G  } t        | |      r| jn                  |k(  s  || j@                  | jp                  | j                         I g }!g }"g }#g }$tU        | jA                               D ]  }|t-        | dts                     v }%|t-        | dts                     v }&|t-        | dts                     v }'|t-        | dts                     v }(|t-        | dts                     v })tu        |j                               }*|}
t        |
tv              r|
jy                         }
|'sFt        |
tz              r|
j|                  zt        |
j|                        j                  |*|di      |   rPt        j                  d|       t        g| jA                         z  }+t        |+|<   |ta        j                  |+      z  }t        j                  xr |j                  |      },t        j                  xr |j                  |      }-||v rt        j                  d|       d},d}-|,xs |-}.8j                  |      } | d }/d }0|'rt        j                  sd }0t        | d      r| j                  D 1cg c]  }1|1j@                  |k(  s|1 c}1j                         }2|2j                  |2j                  t        d      }/nddl"mP}3  |3t        |2j                  |2j                        d      }/ntt        d      }/ng|%rt        d      }/nX|)s|.r|,rt        d      }/|-rDt        d      }0n7d }/d }0n2| \  }/}4d }0| d| d}5|4|j                  j                  j                  |5<   |#j                  |/       |$j                  |0       |&rt<        j                  }6nr|/|'s|(s 9| jK                         |         r|.rt               }6nFt<        jN                  }6n5|st        j                  s|)rt<        j>                  }6nt<        j                  }6|0t<        jN                  }7nt<        jB                  }7|!j                  |6       |"j                  |7        t;        |!|"|#|$|	||      S c c}w c c}w c c}1w )Nr  r  T)
outer_only)dynamic_sizesdynamic_stridesconstraint_sizesconstraint_stridesview_base_contexttensor_source#shape_env_to_source_to_symbol_cacheinner_contexts)r  r  r  r  r  r  r  r   )is_nested_intc              3   T   K   | ]  }t        |t              xr	  |        ! y wr   )r   r
   )r#  sr  s     r   r%  z%_automatic_dynamic.<locals>.<genexpr>
  s(     
La:a 9q)9%99
Ls   %()r  c                     | v r=ddl m} |    \  }} ||j                  |j                  z  d      }|xs |}||f| <   y ||f| <   y )Nr   StrictMinMaxConstraintFvr	warn_only)%torch.fx.experimental.symbolic_shapesr  r  )	dimconstraint_ranger   r  old_constraint_rangeold_namenew_constraint_rangenew_namedim2constraints	           r   update_dim2constraintz1_automatic_dynamic.<locals>.update_dim2constraint  sk    . T-;C-@* (#9#&&)=)@)@@$   '4H"6"@N3"2D"8N3r   )_RelaxedConstraint_dynamo_strict_unbacked_indices_dynamo_unbacked_indices_dynamo_dynamic_indices_dynamo_weak_dynamic_indices_dynamo_static_indicesFz#automatic dynamic %s marked dynamic)sizer&  _dynamo_dynamic_range)r  r  )lowerupperr  z.size()[])Yr  r   r   r  r  r  r  r-   r   r,  r  tracing_contexttensor_to_contextr  r  r  r9   _automatic_dynamicr  r"   r<  r  r  r   r  r  r  r  r  r   r   STATICr  INFER_STRIDEr  r  anyr  r
   rA  r   striderZ  sortr   
setdefaultr5   r6   r4   make_tensorrQ  r"  torch.export.dynamic_shapesr  export_constraintst_idr  r  r   r;   r  rE   r1  r  r-  r  r3   r2   	make_sizer&   r.  is_size_dynamicis_stride_dynamicallow_ignore_mark_dynamicr   r  popminmaxr   r  r#   r=  source_name_to_debug_namern  r  r0  r2  DUCK):r"  r  r   static_shapesr  r   r  prior_policyr  r  r8  outer_contextr  r  r  r  inner_tensorr  r  ex_size	ex_strider  r  r  pending
candidatesi_strideneg_ir6  r  r  r  
constraintr  r  r  r  marked_strict_unbackedmarked_unbackedmarked_dynamicmarked_weak_dynamicmarked_staticr7  	mark_sizeautomatic_dynamic_sizeautomatic_dynamic_strideautomatic_dynamicconstraint_sizeconstraint_stridedr	dim_ranger  name_dim_namedynamic_sizedynamic_strider  r  s:                                                           @@r   r  r  
  s    	{{:	5<<!!//<< 	KL;;=D)+O99,,>>BB1dKL<H88d (
 48zz| 1.qwwKW$Q'
*r6=T

 7--a0q 	D"1d+L%fd3L#5b,$N4 	 ''55)99*;;,??/'550=0a0a)	
 		
 _4&%,,-7'445?"Vaeeg- $v// 0S
 	
 D

L1668
LL&  '1F&;
""ARARR (445?"Vaeeg- $v// 0S
 	
  ffhGHHJ	eeg#/4Sz:!IaL1"%::/0
& 	IOHeA"x:F1I!!(WQZ"7QH	I
 1
D%11%.%-P a5DN9  ?	yy##))66 	J*&89$&%NNJ$?$?		 MO1557^ z/!"g0#%'
 "
 wq*DceLLga)BCEJJ71.Lce#TTWQ(@#%HH
 "7v{{}!Ek=1%..0K{K0$$0[))*../E5zRSTU II;TB#quuw.I'IaL!4!>!>I!NN ++T0A0Q0QRS0T 	 ++V0A0S0STU0V 	! ?"II>E%)"'+$2N6N
 $''*
"O $f&F&F$(!156%&%<%<!!!!ce  !}},1F*AE*R +A*immT&++
 '>&NO'"9E"J"'8)&=&MO+(?$(O%"&$(!%/"OU $xs!,HFKBII99(C0!!"34%88L'"QVVXa[)
 !CE)11f==%,,L &??L('//N'44N\*~.uz/x ##')-+,O G* ;^!s   7a-/a29a7a7)parent_contextr   r   c          
          t               t        j                  t        j                  j                  t
        fv s&t         t        j                        st               rۉJ t         |      \  }}|st         |      n-t        t              sJ j                  }|j                  |   t        j                  dj                         t!         j"                        t                      t%         fd      }Wt        |t
              rG|j&                  x}	9j(                  j*                  j-                  t/        |	t1                           t        |      rf|j3                         \  }
}|
D ]N  }t5        ||      }t5         |      }t        |      }t7        ||t        |t        j                               P j(                  j8                  j:                   <   t=        |      r|j>                  r|jA                         n|jC                         }|jE                         d|jF                  z  |jE                         |jI                         dj(                  jJ                  <   n8|jE                         |jI                         dj(                  jJ                  <   |r~|rjM                         sltO              saj(                  j*                  j-                  t/        |             j(                  jP                  tS                  j-                         |S  S )N)r  zwrap_to_fake %s %s %s %sc                  @    j                   j                         S )N)r   r  )rt  from_tensor)r"  r   r  r  s   r   r  z0wrap_to_fake_tensor_and_record.<locals>.<lambda>  s$    BLL,,!1 -  r   )r   r   r  )rk   )r  r  values_sizevalues_stride)r  r  )*r<  r   r   rL  rM  r   r   r"   re   r  r9   r$  r  r-  r  r   rQ  shaperj   	item_memor  rK  rn  rL  r:   r  r  r   r  r  r   	is_sparse_valuesvaluesr  ndimr  input_source_to_sizes_stridesr  rB   tracked_fakes_id_to_sourcer"  )r"  r  r   r   r  r  _reasoninner_context_namefake_esym_valr  r  r  
fake_innerinnerr  r  r  s   ```              @r   r   r     s    	QELL%(("4"4jAAa&(+!!!!? "
w 1!RO fj111!'-<<=OP		&KKM!''NG	
 %
 6:.",,,9II##**G%9&%ACST )0002HE1 
$VT2
4()&$7.'(U\\B#3	
 :J		!!33A6  *0)9)9V^^%v}}F ,%{{}!'?BII33F;  --/?BII33F; "v'F'F'H&v.II##**FF,<= II00A7>>vFr   c                   R    e Zd ZdZd
dZedddefd       Zed        Zed	        Z	y)r  a  
    Like builder, but stateless and does not require a source. Useful for simple type->VT objects, or objects
    that are being created/evaporated during inlining (ex: consider a locally made list of tensors we then iterate over
    .), such a list should not show up as an artifact from inputs, nor in reconstruction, nor in the graph. However,
    there may be reasons to represent it as a ListVariable internally.

    NOTE - Objects produced here are born UNGUARDED due to the nature of sources!

    NOTE - This class is very new! It will have some rough edges, but it was created to stem the bleeding of giant
    if/else type->VariableTracker trees that were cropping up all over dynamo.
    r  Nc                     t        d      )NzUse SourcelessBuilder.create()r4  r   s    r   r  zSourcelessBuilder.__init__%  s    =>>r   r  r   c                 P	   t        |      }t        j                  j                  |      }|r	 || |      S t	        |t
              r|S t	        |t        j                        rt        |      S t        j                  |      rt        j                  |      S t        |      rVt        j                  |      At        j                  |      rd| j                   _         t        j                  |      |      S t%        |      r t        j&                  |      |      S t	        |t(        j*                  t,        j.                  t,        j0                  j2                  j4                  f      rt7        |      S t	        |t         t8        j:                  f      rt=        |      S t	        |t>        j@                        rtC        |      S t	        |t>        jD                        rt	        |jF                  t         t8        j:                  f      rvtI        |jF                  |jJ                  jL                        |k(  sJ t        j                  | |jF                        }	 |jO                  | |jJ                  jL                        S t	        |t,        jR                  jT                  jV                        rtY        |      S t	        |t,        jZ                  j\                  j^                  t,        jZ                  j\                  j`                  f      rt        |      S tc        jd                  |      rtc        |      S tg        jh                  |      rtg        |      S |tj        jl                  u rto        |      S t	        |tp        jr                        rtu        |      S t	        |t,        jv                  jx                  jz                  j|                        rt        j                  t        |            S t	        |t        t,        j                  j                              r3t,        jv                  jx                  j                  j                  |      S t	        |t>        j                        rt        |      S t        d|j                   d|j                          y # tP        $ r Y 1w xY w)NTz&Unexpected type in sourceless builder r  )Ir<  r  _type_handlersr  r   ro   r  _HAS_DEFAULT_FACTORY_CLASSr   rq   r  r  r
  r*   r  r  r  r  rW   r0  r  r  r   r  r  r  r  rr   abcABCMetar   rj  re  r   r  r  r  rc  r   var_getattrrB  r  graph_moduleGraphModuler   rc  r  rb  LeafSpecr   r;  r   r8  r  rZ  r   r]  r^  r   r  	variableslazyLazySymNodeFormatStringr  r  flex_attention_backwardhigher_order_ops&FlexAttentionBackwardHighOrderVariableGenericAliasr   r-   r   r   )r  r5  
value_typefast_handler
cls_obj_vts        r   r  zSourcelessBuilder.create(  sr   %[
(77;;JGE**e_-L{EEF,U33((/#**511e_!<!<U!C!O..u5>B		;5;..u5e<<#E*,;%%e,U33DIIu00%((2E2E2S2ST
  &&ckk23+E22u667(//ue../ 5>>D#+++>? 5>>5>>+B+BCuLLL*11"ennEJ!--b%..2I2IJJ uxx44@@A077EKK''00%++2E2E2N2NO
 -U33++E2$U++..u5%e,,ioo%)%00rzz*'..u}}66;;SST#**3u:66tE$;$;$S$STU==**;;bb  u112!%((4Z5J5J4K1ZMdMdLef	
1 ' s   %R 	R%$R%c                 \    t        j                  |       sJ t        j                  |       S )Nr  )rq   r  r  r  s    r   wrap_constant_literalz'SourcelessBuilder.wrap_constant_literalk  s'    **5111&&U33r   c                  B   t         j                  i } t        D ]  }d | |<   
 fd| t        <   fd| t        <   fd| t
        <   fd| t        <   fd| t        j                  <   | t           | t        j                  <   | t           | t        <   | t
           | t        <   d | t        j                  <   d | t        j                   <   d	 | t        j"                  <   d
 | t        j$                  j&                  j(                  <   d | t        j*                  j,                  j.                  <   d | t        j*                  j,                  j0                  <   d | t        j*                  j,                  j2                  <   dd}t4        j6                  D ]  }|| |<   	 | S )Nc                     t        |      S r   )rq   r  r5  s     r   r  z6SourcelessBuilder.make_type_handlers.<locals>.<lambda>u  s    ,<U,C r   c           	      b    t        |D cg c]  } | |       c}t                     S c c}w N)r  )r~   rn   r  r5  r  r  s      r   r  z6SourcelessBuilder.make_type_handlers.<locals>.<lambda>v  s)    +$)*qVB]*:J:L+
 *   ,c                     t        |j                         D ci c]  \  }} | |       | |       c}}t        |      t                     S c c}}w rE  )ry   r  r<  rn   )r  r5  r  r  r  s       r   r  z6SourcelessBuilder.make_type_handlers.<locals>.<lambda>y  sF    +<6;kkmDdaVB]F2qM)DK*,,
 Ds   A
c           	      b    t        |D cg c]  } | |       c}t                     S c c}w rE  )r   rn   rF  s      r   r  z6SourcelessBuilder.make_type_handlers.<locals>.<lambda>~  s)    <$)*qVB]*:J:L,
 *rG  c           	      N    t        |D cg c]  } | |       c}      S c c}w r   )r   rF  s      r   r  z6SourcelessBuilder.make_type_handlers.<locals>.<lambda>  s#    M$)*qVB]*-
 *   "c           	      N    t        |D cg c]  } | |       c}      S c c}w r   )r   rF  s      r   r  z6SourcelessBuilder.make_type_handlers.<locals>.<lambda>  s#    $)*qVB]*2
 *rK  c                     t               S r   )r   rC  s     r   r  z6SourcelessBuilder.make_type_handlers.<locals>.<lambda>  s
    4G4I r   c                     t        |      S r   )r   rC  s     r   r  z6SourcelessBuilder.make_type_handlers.<locals>.<lambda>  s    7KE7R r   c                 ,    t        |t                     S rE  )r   rn   rC  s     r   r  z6SourcelessBuilder.make_type_handlers.<locals>.<lambda>  s    ;Q!1!3<
 r   c                 ,    t        |t                     S rE  )r   rn   rC  s     r   r  z6SourcelessBuilder.make_type_handlers.<locals>.<lambda>  s    :%5%7 r   c                 ,    t        |t                     S rE  r   rn   rC  s     r   r  z6SourcelessBuilder.make_type_handlers.<locals>.<lambda>      7%5%7 r   c                 ,    t        |t                     S rE  rR  rC  s     r   r  z6SourcelessBuilder.make_type_handlers.<locals>.<lambda>  rS  r   c                 ,    t        |t                     S rE  rR  rC  s     r   r  z6SourcelessBuilder.make_type_handlers.<locals>.<lambda>  rS  r   c                     |S r   r   rC  s     r   passthroughz9SourcelessBuilder.make_type_handlers.<locals>.passthrough  s    Lr   )r  r   )r  r  rP   r  r  rR  rQ  r   rU  rS  r  r    r!   r]  r^  rj  r`  r4  r  r(  r)  distributionsconstraints_Real	_Interval
Constraintrp   all_subclasses)handlersr   rW  rI  r  s       @r   make_type_handlersz$SourcelessBuilder.make_type_handlersp  sy   "))& 	DACHQK	D




 
 -5TN(()#+D> #+D> "I%R!!"*
%%& 	!!--BBC 	$$00667 	$$00::; 	$$00;;<	 '55 	(C'HSM	(r   r  )
r   r   r   rN  r  staticmethodro   r  r@  r_  r   r   r   r  r    s\    
? @
* @
o @
 @
D 4 4 : :r   r  c                   2    e Zd ZdZddZedddefd       Zy)	"SourcelessUserDefinedObjectBuilderz
    SourceLessBuilder does not return a UserDefinedObjectVariable, but in some
    cases it might be ok to return UserDefinedObjects. In such case, use this
    builder.
    r  Nc                     t        d      )Nz/Use SourcelessUserDefinedObjectBuilder.create()r+  r   s    r   r  z+SourcelessUserDefinedObjectBuilder.__init__  s    NOOr   r  r   c                     t        |      }t        |t              rt        |t	                     S t        |t        j                  j                        rt        |t	                     S t        |t	                     S rE  )r<  rf  r   r   rn   r   r   rL  r  r   r   )r  r5  r<  s      r   r  z)SourcelessUserDefinedObjectBuilder.create  sa    %[
j.1)%?O?QRRuxx/0%5%7  -UBRBTUUr   r  )r   r   r   rN  r  r`  ro   r  r   r   r   rb  rb    s4    P 	V* 	Vo 	V 	Vr   rb  )NNr   )F(:  rN  r/  rS  r.  r  r  r  r  r   	itertoolsr  rB  r  r]  r]  rV  rT  rj  r   r`  collections.abcr   typingr   r   r   r   r   r	   rI  r   r
   torch._dynamo.utilsr   r   r   torch._guardsr   !torch._higher_order_ops.torchbindr   
torch._opsr   r  r   r   r   torch._subclasses.meta_utilsr   r   torch._utils_internalr   %torch.fx.experimental._backward_stater   torch.fx.experimental._dynamismr   r  r   r   r   r   r   r   r   torch.fx.immutable_collectionsr    r!   torch.utils._python_dispatchr"   torch.utils._sympy.value_rangesr#   torch.utils.weakr$   r  r&   r'   r(   r)   r*   r  r+   excr,   r-   r.   rC  r/   r0   r1   pgor2   r3   r4   r5   r6   r-  r7   r   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rc  rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   rg   rh   ri   rj   r  rl   rm   rn   ro   rp   constantrq   rr   ctx_managerrs   rt   ru   rv   rw   rx   dictsry   rz   r{   r|   r}   r~   r  r   r   r   r   r   	functionsr   r   r   r   r   r   r   r   r   r   r   r   r9  r   iterr   r6  r   listsr   r   r   r   r   r   r   r   r   miscr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   	nn_moduler   r   r   	optimizerr   script_objectr   r  r   rH  r   r   r   r   r   r   r   r   r   r   torch_functionr   r   r   r   user_definedr   r   r   r   r   r   r   r   r   r   numpyrm  ModuleNotFoundErrortorch._dynamo.symbolic_convertr   	getLoggerr   r-  _logginggetArtifactLoggerr  rR  DimListr   r   	dataclassr   r	  r  varsr  r&  r?  r  r  r'  r)  r  rW  r  r  r  ra  r`  rb  r0  r  r  r,  rL  r  r  r   r  r_  r-  rb  r   r   r   <module>r     s  &              	 
     * L L    
 ) < * R R A 1 ? A   J F 7 * T T ? K K A A  '     ,                 B  5       ? # %
 
 
     0 
 ) 4 $        
 DD g!NN44' 
 "	 	 99 99 99x<H <, &/ /Y--// & IcN  +.% s3x /J JZ:  & 26$p >B
, 26;W@ >B0H
V

 (, (3s8$ +	SX 	   $ .3MMd IMav&a37aHS Sl $5#G#G#I   V VI]  	Bs   4P6 6	QQ