
    %Vji?                     L   d Z ddlZddlZddlmZ ddlZddlmZ ddlm	Z	 ddl
mZmZ ddlmZmZmZmZmZmZ dd	lmZ d
dlmZ d
dlmZ d
dlmZ d
dlmZ d
dlmZ d
dl m!Z! erddl"m#Z#  G d de$          Z% G d de$          Z& ee'd          Z(d Z) G d de!          Z*dS )a
  
This module implements variable tracking for PyTorch optimizers during Dynamo tracing.

The OptimizerVariable class provides specialized handling for optimizer instances by:
- Optimizing the tracing of expensive optimizer initialization
- Managing optimizer state and parameter group tracking
- Handling tensor sources and guards for optimizer state tensors
- Supporting CUDA graph execution through static tensor address management
- Providing special handling for parameter gradients and optimizer state tensors

Key features include:
- Efficient initialization tracing via _init_group optimization
- Automatic marking of optimizer state tensors as static for CUDA graphs
- Proper source tracking for parameter groups, gradients, and state tensors
- Guard installation for optimizer state structure
- Support for both CPU and GPU tensor handling
- Cleanup of static tensor references via finalizers

The module integrates with Dynamo's broader tracing system while providing
optimizer-specific optimizations and safety guarantees.
    N)TYPE_CHECKING)getArtifactLogger)tree_map_only   )GuardBuilderinstall_guard)
AttrSourceConstDictKeySourceDictGetItemSourceGetItemSourceGlobalWeakRefSource
GradSource)GLOBAL_KEY_PREFIX   )VariableTracker)ConstantVariable)ConstDictVariable)ListVariable)GetAttrVariable)UserDefinedObjectVariable)InstructionTranslatorc                       e Zd ZdS )ArgMappingExceptionN__name__
__module____qualname__     [/root/voice-cloning/.venv/lib/python3.11/site-packages/torch/_dynamo/variables/optimizer.pyr   r   7           Dr   r   c                       e Zd ZdS )GuardInstallExceptionNr   r   r   r    r#   r#   ;   r!   r   r#   
perf_hintsc                     ddl m} | j        r\ || j        j        d          }t
          j        j                            |           d u}|r|p|j	        
                    |           S |S dS )Nr   )get_managerFT)torch._inductor.cudagraph_treesr&   is_cudadeviceindextorch_dynamoutilsget_static_address_typecurrent_node_is_cuda_graph_recorded_tensor)xr&   manageris_static_addresss       r    _is_static_for_cudagraphsr4   B   s    ;;;;;;y +ahne44!M/GGJJRVV 	%! J'FFqII
 %$ tr   c                        e Zd Zdddhej        Z	 	 	 d	 d fdZ	 	 	 	 	 	 d fdZd fdZd Zd Z	d Z
d Zd ZddZ	 	 ddZd Z xZS )OptimizerVariablegrad_to_sourcetensor_to_sourcestatic_tensor_namesNreturnc                      t                      j        |fi | |pi | _        |pi | _        |pt	                      | _        d S N)super__init__r7   r8   setr9   )selfvaluer7   r9   r8   kwargs	__class__s         r    r>   zOptimizerVariable.__init__\   sU     	))&))),2 0 6B#6#?#%%   r   argslist[VariableTracker]rB   dict[str, VariableTracker]r   c                 >   |dk    r	 |                      |           |                                   | j        |i |\  }} | j        j        |i |}|                     |           |                     |||||           dt          | j                   }|                    || j                   | 	                    |           t          j        |          S # t          t          f$ r
}	Y d}	~	nd}	~	ww xY wt                                          ||||          S )zVThis is an optimization to avoid tracing the very slow initialization of the optimizer_init_group__optimizer_N)graph_break_if_pending_mutationmove_step_if_cpuget_python_argsrA   rH   map_sources_and_install_guardsupdate_list_argsidstore_global_weakref_by_idcreate_finalizerr   creater   r#   r=   call_method)r@   txnamerD   rB   py_args	py_kwargsret_valmangled_name_rC   s             r    rS   zOptimizerVariable.call_methodi   sF    =  44R888%%'''%9T%94%J6%J%J"0$*0'GYGG33B777%%b$KKK  ?bnn>>--lDJGGG%%b))) (.w777')>?    ww""2tT6:::s   CC C83C8rT   r   c                 &   |dv r%t          | |t          | j        |                    S |dk    r@ddlm} | j        j        D ]}|d         D ]} ||           |                     |           t                      	                    ||          S )N)rH   step)sourceparam_groupsr   mark_static_addressparams)
r   r	   r]   
decoratorsr`   rA   r^   _set_capturabler=   var_getattr)r@   rT   rU   r`   groupprC   s         r    rd   zOptimizerVariable.var_getattr   s     ***"4jd6S6STTTT>!!8888880 + +x + +A''****+   $$$ww""2t,,,r   c                     | j         j        D ]i}|d         D ]^}|j        j        }|j                            t          |          d           }|r&|                    |          rddlm	}  |d          _jd S )Nra   r   )UnsupportedzPending mutation on parameter)
rA   r^   outputside_effectsid_to_variablegetrO   has_pending_mutationexcrh   )r@   rT   grf   rj   variablerh   s          r    rJ   z1OptimizerVariable.graph_break_if_pending_mutation   s    
 ( 	G 	GAx[ G G!y5'6::2a55$GG G A A( K K G111111%+&EFFFG	G 	Gr   c                     ddl m}  fd} j        j        D ]} ||          rd|d<    j        ot           j        d          }|                    t          j        | j        j        |                    }|j	        D ]D}t          j        t          j        d                    }t          j        d          |j	        |<   Ed S )Nr   LazyVariableTrackerc                     d}d}|                      dg           D ]$}||j        p|j        z  }||j        j        vz  }%d| v o|o|S )NTra   
capturable)rl   r(   is_xpurA   state)re   all_uninitializedall_gpurf   r@   s       r    safe_to_set_capturablezAOptimizerVariable._set_capturable.<locals>.safe_to_set_capturable   sl     $GYYx,, ? ?1900!Qdj.>%>>!!5(J->J7Jr   Tru   r^   ) rs   rA   r^   r]   r	   realize_allr   builditemsr   _HashableTrackerr   rR   )	r@   rT   rs   rz   re   r]   param_groups_vtparam_group_vtkeys	   `        r    rc   z!OptimizerVariable._set_capturable   s
   ))))))	K 	K 	K 	K 	K Z, 	+ 	+E%%e,, +&*l#HDK!H!H-99!"dj&=vFF
 
 .3 	F 	FN#4 '55 C )9(?(E(EN %%		F 	Fr   c                 t      fdfd|D             }fd|                                 D             }||fS )z9Get python values equivalent to the variable tracker argsc                    t          | t                    r|                                 S t          | t                    r	| j        sg S t          | t
                    rjt          | j        t                    rPt          | j        j        t                    r1| j        j        j
        dk    rj        j        | j        j                 S t          )Nr^   )
isinstancer   as_python_constantr   r~   r   r]   r   baser	   memberrA   r^   r*   r   )argr@   s    r    map_argz2OptimizerVariable.get_python_args.<locals>.map_arg   s    #/00 
A--///C.. Asy A	3 122Asz=99A sz
;;A JO*n<<z.sz/?@@%%r   c                 &    g | ]} |          S r   r   ).0r   r   s     r    
<listcomp>z5OptimizerVariable.get_python_args.<locals>.<listcomp>   s!    111SGGCLL111r   c                 .    i | ]\  }}| |          S r   r   )r   kvr   s      r    
<dictcomp>z5OptimizerVariable.get_python_args.<locals>.<dictcomp>   s'    ???1a???r   )r~   )r@   rD   rB   new_args
new_kwargsr   s   `    @r    rL   z!OptimizerVariable.get_python_args   sd    	& 	& 	& 	& 	& 2111D111???????
##r   c                     | j         j                                        D ]9\  }}d|v r0|d         j        r#|d                             |j                  |d<   :d S )Nr\   )rA   rw   r~   is_cputor)   )r@   rf   rw   s      r    rK   z"OptimizerVariable.move_step_if_cpu   sb    
(..00 	; 	;HAu5=#7 %f 0 0 : :f	; 	;r   c                 @   ddl m ddlm} i | _        i | _        fd}t          t          j        || j	        j
                   | j        ot          | j        d          }|                    t          j        || j	        j        |                    }| j        ot          | j        d          }t          j        || j	        j
        |          }|                                 |j        j                            |           t+          | j	        j        |j                  D ]\  }}	t/          |d                   d	k    r|d         D ]}
|
j        d }t3          | j	        j
                                                  D ]\  }}||
u r|} n|rW|                    t          j        || j	        j
        |
         t7          |t9          ||                                          n|	                    |t=          j        d                    }d
}g }t3          t+          |d         |                     |                              D ]\  }\  }}|j        }|| j        |<   tC          |d          }|j        ;|| j        |j        <   tE          |j                  sd}|#                    |           ktI          |%                    tL          j'                             |sKtP          )                    tT          j+                  r'd |D             }tP          ,                    d|           t3          | j	        j
                                                  D ]\  }\  }}t7          |t9          ||                    }|j        j                            |           t3          |                                          D ]Z\  }\  }}t[          |t          j                  r8|| j        vr/|| j        vr&t7          |t9          ||                    | j        |<   [d S )Nr   r_   r   rr   c                      |            d S r<   r   )r1   r`   s    r    mark_staticzEOptimizerVariable.map_sources_and_install_guards.<locals>.mark_static   s    """""r   r^   rw   ra   r   TgradFc                 6    g | ]}|                                 S r   )rU   )r   srcs     r    r   zDOptimizerVariable.map_sources_and_install_guards.<locals>.<listcomp>9  s     #K#K#K3CHHJJ#K#K#Kr   )zGrad tensors %s will be copied during cudagraphs execution.If using cudagraphs and the grad tensor addresses will be the same across runs, use torch._dynamo.decorators.mark_static_address to elide this copy.).rb   r`   lazyrs   r7   r8   r   r+   TensorrA   rw   r]   r	   r|   r   r}   r^   realizeri   guard_on_key_orderaddzipr~   lenr   	enumeratekeysr   r
   getitem_constr   rR   unpack_var_sequencer   r4   appendr   
make_guardr   CONSTANT_MATCHperf_hint_logisEnabledForloggingDEBUGwarningr   )r@   rT   rs   r   params_groups_sourcer   state_sourcestate_vtre   group_vtparam	key_indexir   	params_vt
all_staticnon_static_gradsp_indrf   p_vtparam_sourcegrad_sourceidxrA   p_state_source	inner_idxr   r`   s                              @r    rM   z0OptimizerVariable.map_sources_and_install_guards   ss   444444------  "	# 	# 	# 	# 	# 	elK1ABBB  ${Vz$+~/V/V-99!"dj&=?STT
 
 {Gz$+w'G'G"(TZ-=|LL 	
	$((666  #4:#:O<QRR 8	 8	OE8 5?##a''"8_ " "Ez-$(	$-dj.>.C.C.E.E$F$F & &DAq Ezz,-	 %  * % "/;; / 5$&$(J$4U$;$5(4(:<(S(S%& %&!" !"	 	 	 "E ..r3C3J83T3TUUIJ!$-E(OY%B%B2%F%FGG% % W W y4  ${+7%a((  
 6%2=D'/4QV<< =%*
(//<<<!+"8"89T"U"UVVVV  	-"<"<W]"K"K 	#K#K:J#K#K#K %%
 %    ))9)?)?)A)ABB 	 	OC!U.0sCC N I(,,^<<<%.u{{}}%=%=  !	6Aqq%,//!444!666/@&(:>9(U(U0 0D)!,	 	r   c                 &   ddl m} || j        v r] ||           | j        |         }| j                            |j                            |                                                     n|| j        v r| j        |         }ny ||           |	                    t          |          }t          |          }| j                            |j                            |                                                     t          j        |||          S )z%Wrap state tensor in a TensorVariabler   r_   )rb   r`   r8   r9   r   ri   module_key_namerU   r7   rP   r   r   r   r}   )r@   rT   tensor_valuer`   r]   global_names         r    wrap_tensorzOptimizerVariable.wrap_tensorT  s   444444 4000---*<8F$(()B)B6;;==)Q)QRRRRT000(6FF  ---778I<XXK(55F$(()B)B6;;==)Q)QRRR$Rv>>>r   c           	      ,   t          ||          D ]\  }}t          |t                    rt          |t                    s
J d            t	          |          D ]\  }}	|j        j                            |           t          |	t          j	                  r/|j
                            |                     ||	                     m|j        ot          |j        |          }
|j
                            t          j        ||	|
                     dS )z7Update the args and kwargs to the traced optimizer callz-py_arg should be a list in optimizer variableN)r   r   r   listr   ri   rj   mutationr+   r   r~   r   r   r]   r   r   r}   )r@   rT   rD   rB   rV   rW   r   py_argr   valr]   s              r    rN   z"OptimizerVariable.update_list_argsn  s     tW-- 	Q 	QKC#|,, 
Q!&$//  C / (// Q QFAsI*33C888!#u|44 Q	(()9)9"c)B)BCCCC!$!Lcj!0L0L	(()>r3)O)OPPPP	Q 	Qr   c                     | j         | j        |j        j        fd}|j                            |           d S )Nc                 B      fd}t          j        |           d S )Nc                      D ]x} j                             | d            j                            | d            j        rj                                         j        rj                                         yd S r<   )_bufferspop_parametersparams_flatclearparams_flat_unwrap_subclasses)rU   gmnames_to_deletetcs    r    clear_static_tensor_refsz\OptimizerVariable.create_finalizer.<locals>.init_finalizer.<locals>.clear_static_tensor_refs  s    + A ADKOOD$///N&&tT222~ /,,...7 A8>>@@@A Ar   )weakreffinalize)r   r   r   r   rA   s   ` r    init_finalizerz:OptimizerVariable.create_finalizer.<locals>.init_finalizer  sJ    A A A A A A A U$<=====r   )r9   rA   ri   tracing_contextadd_graph_finalizer)r@   rT   r   r   r   rA   s      @@@r    rQ   z"OptimizerVariable.create_finalizer  s_    2
Y&
	> 
	> 
	> 
	> 
	> 
	> 
	> 		%%n55555r   )NNN)r:   N)rD   rE   rB   rF   r:   r   )rT   r   )r   r   r   r   _nonvar_fieldsr>   rS   rd   rJ   rc   rL   rK   rM   r   rN   rQ   __classcell__)rC   s   @r    r6   r6   T   sk        
#	1	N  @ 
@ @ @ @ @ @; &	;
 -; 
; ; ; ; ; ;B- - - - - -$G G GF F F@$ $ $8; ; ;
g g gR? ? ? ?4Q)Q Q Q Q"6 6 6 6 6 6 6r   r6   )+__doc__r   r   typingr   r+   torch._loggingr   torch.utils._pytreer   guardsr   r   r]   r	   r
   r   r   r   r   r-   r   r   r   constantr   dictsr   listsr   miscr   user_definedr   torch._dynamo.symbolic_convertr   	Exceptionr   r#   r   r   r4   r6   r   r   r    <module>r      s   ,                , , , , , , - - - - - - 0 0 0 0 0 0 0 0                & % % % % % ! ! ! ! ! ! & & & & & & $ $ $ $ $ $       ! ! ! ! ! ! 3 3 3 3 3 3  EDDDDDD	 	 	 	 	) 	 	 		 	 	 	 	I 	 	 	 "!(L99  $|6 |6 |6 |6 |61 |6 |6 |6 |6 |6r   