
    VhWA                     2   d Z ddlZddlZddlmZ ddlZddlmZ ddlm	Z	 ddl
mZmZ ddlmZmZmZmZmZmZ dd	lmZ d
dlmZ d
dlmZ d
dlmZ d
dlmZ d
dlmZ d
dl m!Z! erddl"m#Z#  G d de$      Z% G d de$      Z& ee'd      Z(d Z) G d de!      Z*y)a
  
This module implements variable tracking for PyTorch optimizers during Dynamo tracing.

The OptimizerVariable class provides specialized handling for optimizer instances by:
- Optimizing the tracing of expensive optimizer initialization
- Managing optimizer state and parameter group tracking
- Handling tensor sources and guards for optimizer state tensors
- Supporting CUDA graph execution through static tensor address management
- Providing special handling for parameter gradients and optimizer state tensors

Key features include:
- Efficient initialization tracing via _init_group optimization
- Automatic marking of optimizer state tensors as static for CUDA graphs
- Proper source tracking for parameter groups, gradients, and state tensors
- Guard installation for optimizer state structure
- Support for both CPU and GPU tensor handling
- Cleanup of static tensor references via finalizers

The module integrates with Dynamo's broader tracing system while providing
optimizer-specific optimizations and safety guarantees.
    N)TYPE_CHECKING)getArtifactLogger)tree_map_only   )GuardBuilderinstall_guard)
AttrSourceConstDictKeySourceDictGetItemSourceGetItemSourceGlobalWeakRefSource
GradSource)GLOBAL_KEY_PREFIX   )VariableTracker)ConstantVariable)ConstDictVariable)ListVariable)GetAttrVariable)UserDefinedObjectVariable)InstructionTranslatorc                       e Zd Zy)ArgMappingExceptionN__name__
__module____qualname__     Q/home/dcms/DCMS/lib/python3.12/site-packages/torch/_dynamo/variables/optimizer.pyr   r   7       r   r   c                       e Zd Zy)GuardInstallExceptionNr   r   r   r    r#   r#   ;   r!   r   r#   
perf_hintsc                     ddl m} | j                  rk || j                  j                  d      }t
        j                  j                  j                  |       d u}|r|xs |j                  j                  |       S |S y)Nr   )get_managerFT)torch._inductor.cudagraph_treesr&   is_cudadeviceindextorch_dynamoutilsget_static_address_typecurrent_node_is_cuda_graph_recorded_tensor)xr&   manageris_static_addresss       r    _is_static_for_cudagraphsr4   B   ss    ;yyahhnne4!MM//GGJRVV! J''FFqI
 %$ r   c                        e Zd Zdddhej                  Z	 	 	 d	 d fdZ	 	 	 	 	 	 d fdZd fdZd Zd Z	d	 Z
d
 Zd ZddZ	 	 ddZd Z xZS )OptimizerVariablegrad_to_sourcetensor_to_sourcestatic_tensor_namesc                 x    t        |   |fi | |xs i | _        |xs i | _        |xs
 t	               | _        y N)super__init__r7   r8   setr9   )selfvaluer7   r9   r8   kwargs	__class__s         r    r=   zOptimizerVariable.__init__\   s@     	)&),2 0 6B#6#?#% r   c                    |dk(  r	 | j                  |       | j                           | j                  |i |\  }} | j                  j                  |i |}| j                  |       | j                  |||||       dt        | j                         }|j                  || j                         | j                  |       t        j                  |      S t        
| =  ||||      S # t        t        f$ r
}	Y d}	~	&d}	~	ww xY w)zVThis is an optimization to avoid tracing the very slow initialization of the optimizer_init_group__optimizer_N)graph_break_if_pending_mutationmove_step_if_cpuget_python_argsr@   rD   map_sources_and_install_guardsupdate_list_argsidstore_global_weakref_by_idcreate_finalizerr   creater   r#   r<   call_method)r?   txnameargsrA   py_args	py_kwargsret_valmangled_name_rB   s             r    rO   zOptimizerVariable.call_methodi   s    = 44R8%%'%9T%9%94%J6%J"0$**00'GYG33B7%%b$K ".bn-=>--lDJJG%%b) (..w77
 w"2tT6::	 ()>? s   CC, ,D Dc                    |dv r"t        | |t        | j                  |            S |dk(  rDddlm} | j
                  j                  D ]  }|d   D ]
  } ||         | j                  |       t        | %  ||      S )N)rD   step)sourceparam_groupsr   mark_static_addressparams)
r   r	   rZ   
decoratorsr]   r@   r[   _set_capturabler<   var_getattr)r?   rP   rQ   r]   groupprB   s         r    ra   zOptimizerVariable.var_getattr   s     **"4jd6STT>!800 +x +A'*++   $w"2t,,r   c                    | j                   j                  D ]i  }|d   D ]_  }|j                  j                  }|j                  j                  t        |      d       }|sA|j                  |      sSddlm	}  |d       k y )Nr^   r   )UnsupportedzPending mutation on parameter)
r@   r[   outputside_effectsid_to_variablegetrK   has_pending_mutationexcre   )r?   rP   grc   rg   variablere   s          r    rF   z1OptimizerVariable.graph_break_if_pending_mutation   s~    
 (( 	GAx[ G!yy55'66::2a5$G A A( K1%&EFFG	Gr   c                     ddl m}  fd} j                  j                  D ]  } ||      sd|d<     j                  xr t         j                  d      }|j                  t        j                  | j                  j                  |            }|j                  D ]L  }t        j                  t        j                  d            }t        j                  d      |j                  |<   N y )Nr   LazyVariableTrackerc                     d}d}| j                  dg       D ]:  }||j                  xs |j                  z  }||j                  j                  vz  }< d| v xr |xr |S )NTr^   
capturable)ri   r(   is_xpur@   state)rb   all_uninitializedall_gpurc   r?   s       r    safe_to_set_capturablezAOptimizerVariable._set_capturable.<locals>.safe_to_set_capturable   sq     $GYYx, ?19900!Qdjj.>.>%>>!?  5(J->J7Jr   Trr   r[   ) rp   r@   r[   rZ   r	   realize_allr   builditemsr   _HashableTrackerr   rN   )	r?   rP   rp   rw   rb   rZ   param_groups_vtparam_group_vtkeys	   `        r    r`   z!OptimizerVariable._set_capturable   s    )	K ZZ,, 	+E%e,&*l#	+ HDKK!H-99!!"djj&=&=vF
 .33 	FN#44 ''5C )9(?(?(EN  %		Fr   c                       fd}|D cg c]
  } ||       }}|j                         D ci c]  \  }}| ||       }}}||fS c c}w c c}}w )z9Get python values equivalent to the variable tracker argsc                    t        | t              r| j                         S t        | t              r| j                  sg S t        | t
              rt        | j                  t              rtt        | j                  j                  t              rP| j                  j                  j                  dk(  r-j                  j                  | j                  j                     S t        )Nr[   )
isinstancer   as_python_constantr   r{   r   rZ   r   baser	   memberr@   r[   r*   r   )argr?   s    r    map_argz2OptimizerVariable.get_python_args.<locals>.map_arg   s    #/0--//C.syy	3 12szz=9szz
;JJOO**n<zz..szz/?/?@@%%r   )r{   )	r?   rR   rA   r   r   new_argskv
new_kwargss	   `        r    rH   z!OptimizerVariable.get_python_args   sY    	& -11SGCL1106?1am?
?## 2?s
   A	Ac                     | j                   j                  j                         D ];  \  }}d|v s|d   j                  s|d   j	                  |j
                        |d<   = y )NrY   )r@   rt   r{   is_cputor)   )r?   rc   rt   s      r    rG   z"OptimizerVariable.move_step_if_cpu   sW    

((..0 	;HAu5=#7#7 %f 0 0 :f	;r   c                 	   ddl m ddlm} i | _        i | _        fd}t        t        j                  || j                  j                         | j                  xr t        | j                  d      }|j                  t        j                  || j                  j                   |            }| j                  xr t        | j                  d      }t        j                  || j                  j                  |      }|j#                          |j$                  j&                  j)                  |j+                                t-        | j                  j                   |j.                        D ]  \  }}	t1        |d         d	kD  r|d   D ]  }
|
j2                  d }t5        | j                  j                  j7                               D ]  \  }}||
u s|} n |sR|j                  t        j                  || j                  j                  |
   t9        |t;        ||                          n |	j=                  |t?        j@                  d            }d
}g }t5        t-        |d   |jC                  |                  D ]  \  }\  }}|j                  }|| j
                  |<   tE        |d      }|j2                  C|| j                  |j2                  <   tG        |j2                        rkd}|jI                  |       tK        |jM                  tN        jP                                |rtR        jU                  tV        jX                        s|D cg c]  }|j+                          }}tR        j[                  d|        t5        | j                  j                  j/                               D ]  \  }\  }}t9        |t;        ||            }|j$                  j&                  j)                  |j+                                t5        |j/                               D ]d  \  }\  }}t]        |t        j                        s$|| j                  vs3|| j
                  vsBt9        |t;        ||            | j
                  |<   f  y c c}w )Nr   r\   r   ro   c                      |        y r;   r   )r1   r]   s    r    mark_staticzEOptimizerVariable.map_sources_and_install_guards.<locals>.mark_static   s    "r   r[   rt   r^   r   TgradF)zGrad tensors %s will be copied during cudagraphs execution.If using cudagraphs and the grad tensor addresses will be the same across runs, use torch._dynamo.decorators.mark_static_address to elide this copy.)/r_   r]   lazyrp   r7   r8   r   r+   Tensorr@   rt   rZ   r	   ry   r   rz   r[   realizerf   guard_on_key_orderaddrQ   zipr{   lenr   	enumeratekeysr   r
   getitem_constr   rN   unpack_var_sequencer   r4   appendr   
make_guardr   CONSTANT_MATCHperf_hint_logisEnabledForloggingDEBUGwarningr   )r?   rP   rp   r   params_groups_sourcer}   state_sourcestate_vtrb   group_vtparam	key_indexir   	params_vt
all_staticnon_static_gradsp_indrc   p_vtparam_sourcegrad_sourcesrcidxr@   p_state_source	inner_idxr   r]   s                               @r    rI   z0OptimizerVariable.map_sources_and_install_guards   s   4-  "	# 	ellK1A1AB  ${{Vz$++~/V-99!!"djj&=&=?ST
 {{Gz$++w'G"((TZZ-=-=|L 	
		$$(():):)<=  #4::#:#:O<Q<QR 8	OE8 5?#a'"8_ "Ezz-$(	$-djj.>.>.C.C.E$F &DAq Ez,-	 %& %/;; / 5 5$&$(JJ$4$4U$;$5(4(:<(S%&!"	 "%"( !..r3C3J3J83TUIJ!$-E(OY%B%B2%FG% W y4  ${{+7%%a(( 
 66%2=D''/4QVV<%*
(//<!+"8"89T9T"UV!W( -"<"<W]]"K:J#K3CHHJ#K #K%%
 %c8	x  ))9)9)?)?)AB 	OC!U.0sCN II((,,^-@-@-BC%.u{{}%= !	6Aqq%,,/!4!44!6!66/@&(:>9(U0D))!,	 $Ls   Rc                    ddl m} || j                  v rZ ||       | j                  |   }| j                  j	                  |j
                  j                  |j                                      n|| j                  v r| j                  |   }nk ||       |j                  t        |      }t        |      }| j                  j	                  |j
                  j                  |j                                      t        j                  |||      S )z%Wrap state tensor in a TensorVariabler   r\   )r_   r]   r8   r9   r   rf   module_key_namerQ   r7   rL   r   r   r   rz   )r?   rP   tensor_valuer]   rZ   global_names         r    wrap_tensorzOptimizerVariable.wrap_tensor\  s    4 4000-**<8F$$(()B)B6;;=)QRT000((6F  -778I<XK(5F$$(()B)B6;;=)QR$$Rv>>r   c           	      $   t        ||      D ]   \  }}t        |t              st        |t              sJ d       t	        |      D ]  \  }}	|j
                  j                  j                  |       t        |	t        j                        r,|j                  j                  | j                  ||	             q|j                  xr t        |j                  |      }
|j                  j                  t        j                   ||	|
               y)z7Update the args and kwargs to the traced optimizer callz-py_arg should be a list in optimizer variableN)r   r   r   listr   rf   rg   mutationr+   r   r{   r   r   rZ   r   r   rz   )r?   rP   rR   rA   rS   rT   r   py_argr   valrZ   s              r    rJ   z"OptimizerVariable.update_list_argsv  s     tW- 	QKC#|,!&$/ C/ (/ QFAsII**33C8!#u||4		(()9)9"c)BC!$!Lcjj!0L		(()>)>r3)OPQ	Qr   c                     | j                   | j                  |j                  j                  fd}|j                  j	                  |       y )Nc                 B      fd}t        j                  |       y )Nc                     D ]  } j                   j                  | d        j                  j                  | d        j                  rj                  j	                          j
                  snj
                  j	                           y r;   )_bufferspop_parametersparams_flatclearparams_flat_unwrap_subclasses)rQ   gmnames_to_deletetcs    r    clear_static_tensor_refsz\OptimizerVariable.create_finalizer.<locals>.init_finalizer.<locals>.clear_static_tensor_refs  sl    + ADKKOOD$/NN&&tT2~~,,.7788>>@Ar   )weakreffinalize)r   r   r   r   r@   s   ` r    init_finalizerz:OptimizerVariable.create_finalizer.<locals>.init_finalizer  s    A U$<=r   )r9   r@   rf   tracing_contextadd_graph_finalizer)r?   rP   r   r   r   r@   s      @@@r    rM   z"OptimizerVariable.create_finalizer  s?    22

YY&&
	> 			%%n5r   )NNN)returnN)rR   zlist[VariableTracker]rA   zdict[str, VariableTracker]r   r   )rP   r   )r   r   r   r   _nonvar_fieldsr=   rO   ra   rF   r`   rH   rG   rI   r   rJ   rM   __classcell__)rB   s   @r    r6   r6   T   s     
#	1	1	N  @ 
@; &	;
 -; 
;B-$GF@$8;
ob?4Q)Q"6r   r6   )+__doc__r   r   typingr   r+   torch._loggingr   torch.utils._pytreer   guardsr   r   rZ   r	   r
   r   r   r   r   r-   r   r   r   constantr   dictsr   listsr   miscr   user_definedr   torch._dynamo.symbolic_convertr   	Exceptionr   r#   r   r   r4   r6   r   r   r    <module>r      s   ,      , - 0  & ! & $  ! 3 D	) 		I 	 "(L9$D61 D6r   