
    %Vjil                        d Z ddlZddlZddlZddlZddlmZmZ ddlm	Z	 ddl
ZddlmZmZmZ ddlmZmZmZmZ ddlmZmZ dd	lmZ dd
lmZmZmZmZmZm Z m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/ ddl0m1Z1m2Z2m3Z3 ddl4m5Z5 ddl6m7Z7 ddl8m9Z9 ddl:m;Z; e	rddl<m=Z= d"dZ>ede?dej@        jA        fd            ZBd ZC G d de3          ZD G d de;          ZE G d deE          ZF G d  d!eE          ZGdS )#aQ  
This module implements variable tracking for PyTorch nn.Module instances during Dynamo tracing.

It provides specialized handling for different types of nn.Module instances through several key classes:

- NNModuleVariable: Handles instance-specific module tracing, specializing on module id() and placing
  parameters directly on the torch.fx.GraphModule. This creates one graph per module instance.

- UnspecializedNNModuleVariable: Provides class-level module tracing, treating nn.Modules like other
  user-defined objects and passing parameters as inputs to the FX graph. This creates one graph per
  module class.

- UnspecializedBuiltinNNModuleVariable: Specifically handles built-in PyTorch modules (e.g. nn.Linear)
  with appropriate optimizations.

- FSDPManagedNNModuleVariable: Special handling for FSDP-wrapped modules with modified guarding behavior
  and parameter handling.

The module integrates with Dynamo's broader tracing functionality to handle module method calls,
parameter access, hooks, and other nn.Module behaviors while maintaining proper scoping and guarding
of module state.
    N)contextmanagernullcontext)TYPE_CHECKING   )graph_break_hintstrace_rules	variables)raise_observed_exceptionunimplemented_v2UnspecializeRestartAnalysisUnsupported)GuardBuilderinstall_guard)GenerationTracker)
AttrSourceConstDictKeySourceDictGetItemSourceFSDPNNModuleSourceGetItemSourceNNModuleSourceUnspecializedNNModuleSource)get_custom_getattrget_fake_valueis_lazy_moduleis_namedtupleis_safe_constantistensoristypennmodule_has_hooksobject_has_getattributeproxy_args_kwargsset_example_valueunpatched_nn_module_callunpatched_nn_module_call_impl   )typestrValueMutationNewVariableTracker)invoke_and_store_as_constant)LazyVariableTracker)SliceVariable)UserDefinedObjectVariable)InstructionTranslatortxr-   c                 8    t          |d          r fdt          ||          \  }}fd|D             }fd|                                D             }	 |                    |||           dS # t          $ r t          t                      Y dS w xY wdS )a~  
    Fairly coupled helper used by NNModuleVariable and UnspecializedNNModuleVariable.

    Used to cause lazy module to be initialized (and delete its init hook) before tracing. Especially
    useful now that 'allowed' modules graph-break on hooks, calling this first ensures there is no hook
    by the time we trace __call__ and thus no graph-break for lazy allowed modules.
    _initialize_hookc                    t          |           r t          |           fd| D              S t          | t                    r fd|                                 D             S t          | t
          t          t          f          r$ t          |           fd| D                       S t          | t          j	        j
                  rt          | j                  S | S )Nc              3   .   K   | ]} |          V  d S N .0elemconvert_to_fakes     [/root/voice-cloning/.venv/lib/python3.11/site-packages/torch/_dynamo/variables/nn_module.py	<genexpr>zBinitialize_lazy_module.<locals>.convert_to_fake.<locals>.<genexpr>[   s-       E E4!6!6 E E E E E E    c                 .    i | ]\  }}| |          S r4   r4   r6   kvr8   s      r9   
<dictcomp>zCinitialize_lazy_module.<locals>.convert_to_fake.<locals>.<dictcomp>]   s)    DDD$!Q??1--DDDr;   c              3   .   K   | ]} |          V  d S r3   r4   r5   s     r9   r:   zBinitialize_lazy_module.<locals>.convert_to_fake.<locals>.<genexpr>_   s-      CCt44CCCCCCr;   )r   type
isinstancedictitemslisttuplesettorchfxProxyr   node)xr8   r.   s    r9   r8   z/initialize_lazy_module.<locals>.convert_to_fakeY   s    Q 	tAww E E E E1 E E EFFAt$$ DDDD!''))DDDDAeS122 tAwwCCCCCCCCCCAux~.. %afb111r;   c                 &    g | ]} |          S r4   r4   )r6   argr8   s     r9   
<listcomp>z*initialize_lazy_module.<locals>.<listcomp>f   s#    @@@c__S))@@@r;   c                 .    i | ]\  }}| |          S r4   r4   r=   s      r9   r@   z*initialize_lazy_module.<locals>.<dictcomp>g   s)    NNNAq//!,,NNNr;   N)hasattrr!   rE   _infer_parametersAttributeErrorr
   )	r.   modargskwargs
proxy_argsproxy_kwargs	fake_argsfake_kwargsr8   s	   `       @r9   initialize_lazy_moduler\   O   s     s&'' 
	 
	 
	 
	 
	 
	 $5T6#B#B 
L@@@@Z@@@	NNNN9K9K9M9MNNN	!!#y+>>>>> 	 	 	$     	' s   A4 4BB
module_keyrU   c              #      K   |                                 }|j                            |d          }|dk    r|  d| n| } 	 ||j        f|j        | <   |dz   |j        |<   d V  |j        | = d S # |j        | = w xY w)Nr   @r%   )name	num_callsget	__class__nn_module_stack)r]   sourcer.   rU   fully_qualified_namera   s         r9   record_nn_module_stackrg   q   s      !;;==  !5q99I09AJ,,,,,:J+*>)N:&-6])*z***Bz*****s   "A. .
A8c                    | rd|j         v rt          |j         d                   rv|j         d         }t          | d          }t          |          t          j        u rt          |d          }t          |                    t          j	                             d S t          |                     t          j        t          j        d                               d S d S )Nforward__func__)attr)__dict__callabler   rB   types
MethodTyper   
make_guardr   CLOSURE_MATCH	functoolspartialNOT_PRESENT_IN_GENERIC_DICT)re   rU   fwdforward_sources       r9   &guard_to_detect_forward_monkeypatchingrw   ~   s      $$#,y2I)J)J$,y)C'	::NCyyE,,,!+NJ!G!G.33L4NOOPPPPP !!%$@y        r;   c                        e Zd Zddddhej        Zdededej        j	        ddf fdZ
d Zd	 Zd
 Z	 	 ddZd ZdddeddfdZd Zd ZddZd ZddZ	 	 	 	 	 	 ddZ	 d 	 	 	 	 	 d fdZ xZS )!NNModuleVariablemodule_typer]   valuenn_module_stack_sourcereturnNc                      t                      j        di | || _        || _        || _        | j        sJ | j        | _        d S )Nr4   )super__init__rz   r]   r{   re   r|   )selfrz   r]   r{   rW   rc   s        r9   r   zNNModuleVariable.__init__   sU     	""6"""&$
{{&*k###r;   c                     | j         p| j        S r3   r|   re   r   s    r9   get_nn_module_stack_sourcez+NNModuleVariable.get_nn_module_stack_source       *9dk9r;   c                     || _         d S r3   r|   r   re   s     r9   set_nn_module_stack_sourcez+NNModuleVariable.set_nn_module_stack_source       &,###r;   c                     | j         S r3   )rz   r   s    r9   python_typezNNModuleVariable.python_type   s    r;   r.   r-   c                     d S r3   r4   )r   r.   re   submod	key_extraoptionss         r9   _wrap_submodulez NNModuleVariable._wrap_submodule   s	     	r;   c                 F   |j                             | j                  }t          |t          j        j                  rg }|                                D ]|\  }}t          j	        
                    |          }|j                             || j        |t          t          | j        |                               |                    |           }|S t          |t          j        j        t          j        j        t          j        j        f          sJ t'          |                      | j        sJ g }t)          |          D ][\  }}|                    |j                             || j        |t          t          | j        |                                         \|S Nre   )outputget_submoduler]   rC   rI   nn
ModuleDictrE   r	   ConstantVariablecreateregister_attr_or_moduler   r   re   append
ModuleListParameterList
Sequentialr&   	enumerate)r   r.   baseresultr`   r   name_varidxs           r9   unpack_var_sequencez$NNModuleVariable.unpack_var_sequence   s   y&&t77dEH/00 	F $

 ( (f$5<<TBB	11O)-T*J*JKK	 2    h''''M58&(>@ST
 
 	 	4==	 	 
 {{$T?? 	 	KCMM	11O)-S*I*IJJ	 2      r;   r`   r(   c                 6   |j                             | j                  }t          ||          }t	          t          t          | j        |                                        t          j
                             t          j                            |          S r3   )r   r   r]   rR   r   r   r   re   rp   r   HASATTRr	   r   r   )r   r.   r`   rU   r   s        r9   call_obj_hasattrz!NNModuleVariable.call_obj_hasattr   s     i%%do66d##:dk48899DD$ 	
 	
 	

 )00888r;   c                 b    |j                             | j                  }t          |dd          S )NtrainingF)r   r   r]   getattrr   r.   rU   s      r9   is_trainingzNNModuleVariable.is_training   s+    i%%do66sJ...r;   c                     |j                             | j                  }t          j        |           |j        j        dk    r!t          j        t          |                     t          )zIRestart analysis treating this module as an UnspecializedNNModuleVariabler   )
r   r   r]   r   tagf_codeco_namemark_class_dynamicrB   r   r   s      r9   convert_to_unspecializedz)NNModuleVariable.convert_to_unspecialized   sY    i%%do66c""" 9
**0c;;;))r;   c                    |j                             | j                  }t          |          r&t	          dd|  d| ddgt
          j                   |j         j                            | |          r=|j         j        	                    | |d          }t          |t          j                   S t                              |d	          }||v S )
Nz3Custom __getattribute__ in nn.Module dict key checkzhas_key_in_generic_dict  }Dynamo does not support checking key existence on `nn.Module` instances that have a custom `__getattribute__` method defined.1Avoid defining `__getattribute__` in your module.gb_typecontextexplanationhintsT)
deleted_okrl   )r   r   r]   r    r   r   SUPPORTABLEside_effectshas_pending_mutation_of_attr	load_attrrC   r	   DeletedVariableobject__getattribute__)r   r.   keyr   mutated_attr	base_dicts         r9   has_key_in_generic_dictz(NNModuleVariable.has_key_in_generic_dict   s    y&&t77"4(( 	M?4??#??5 H&2
 
 
 
 9!>>tSII 	K91;;D#RV;WWL!,	0IJJJJ++D*==	ir;   c                    t          |          r&t          dd|  d| ddgt          j                   t	          |d          }|d	S t          |t          j                  s&t          d
d|  d| ddgt          j                   dt          |d          i}t          j
        || fi |                    |t          j                            |          gi           S )zDCheck for a __getattr__ and handle it specially if it is implementedz5Custom __getattribute__ in nn.Module attribute accessvar_getattr r   r   r   r   T)ignore_nn_module_getattrNz6torch.nn.Module with a non-function custom __getattr__zDynamo detected a nn.Module object with a custom `__getattr__` method, but this method is not a standard Python function (e.g., it might be implemented in C/C++). Dynamo cannot currently trace into such non-standard `__getattr__` methods.zAvoid using objects with non-standard __getattr__ methods within the compiled region. If possible, implement __getattr__ as a standard Python function.re   __getattr__)r    r   r   r   r   rC   rn   FunctionTyper   r	   UserMethodVariablecall_functionr   r   )r   r   r.   r`   
obj_source
getattr_fnr   s          r9   _custom_getattr_fallbackz)NNModuleVariable._custom_getattr_fallback  s6   "4(( 	O4t44d445 H&2
 
 
 
 (tLLL
4*e&899 	P4t44d44-A '2	   $ Z
MBBC+JHHHHVV+2248892
 
 	
r;   c                 	   | j         ot          | j         |          }|j                            | j                  }t
                              |d          }d}t                      }t          j	        |j
                  D ].}|                    |j                                                   /| j         s%t          dd|  d| dg t          j                   |dk    rt#          j        | ||          S ||v r	||         }	nd	|v r||d	         v r||vr|d	         |         }	nd
|v r||d
         v r|d
         |         }	nd|v r||d         v r|d         |         }	nb	 t          j        ||          }	d}nI# t(          $ r< |                     |||| j                   }
|
|
cY S t-          t(          |           Y nw xY w|dk    rt/          | j         |           |dk    r|st#          j        |j
        |          S |rvt3          j        ||	t7          |                    }t9          |t:          t<          f          r5|                    t          |                                  |                     |S tC          |	tD                    rk| j         r3t          t          | j         d          |          }t          |d          }t#          j#        |	j$        |          %                    || gi           S tC          |	tL                    r;t#          j'        |	j(        t#          j)        tU          |                    |          S tC          |	tV                    r)t#          j#        |	,                    |          |          S tC          |	tZ          j.                  rt#          j'        |	| |          S t_          |	          sta          |	          r#t3          j        ||	t7          |                    S t          ddtc          |           d| dtc          |	           dtc          |	           dd| dtc          |	           dtc          |           ddgt          j2                   t#          j        | ||          S )Nrl   Tzgetattr with no sourcer   r   zDynamo does not know how to access an attribute on an `nn.Module` instance that lacks a source. This is usually an internal error in Dynamo.r   r   _modules_parameters_buffersF)r   r.   r`   r   ri   rc   fgetz$Unsupported nn.Module attribute typeznn.Module subclass: z, name: z, attribute type: z>Dynamo does not support tracing nn.Module attributes of type ``zRefactor your code so that `z	` (type `z`) is not an attribute of `zqCurrently supported attribute types are methods, classmethods, staticmethods, properties, constants, and tensors.)3re   r   r   r   r]   r   r   rH   inspectgetmrorc   updaterl   keysr   r   
DYNAMO_BUGr	   GetAttrVariablegetattr_staticrT   r   r
   rw   UserDefinedClassVariabler(   buildr   rC   ry   UnspecializedNNModuleVariabler   r   r   propertyUserFunctionVariabler   r   classmethodr   rj   r,   rB   staticmethod__get__rn   r   r   r   r&   r   )r   r.   r`   re   r   r   object_memberall_class_attribute_namesrM   subobjr   outs               r9   var_getattrzNNModuleVariable.var_getattr<  s>   >DK!>!>y&&t77++D*==	$'EE!// 	@ 	@A%,,QZ__->->????{ 	04t44d447 6)45    :,T4GGGG9t_FF)##	*---555z*40FFi''DIm4L,L,L}-d3FF9$$:1F)F)Fz*40FF /d;; %!   66"4DK 7   %!MMM("     924;EEE;}5dnVTTTT 1	!'FN64J4JKKC# 02OPQQ  ..t>>@@$GG   J fh'' $; 8'
4;(L(LdSSF'77F 5K!    -dHb112 ,,  3O7T

CC!   
 --  5NN4((     233  3FDPPPP!&)) Xf-=-= &,R9O9OPPP Bs74==ss$ssbijpbqbqss sahioapap s s s Ct  C  Cgfoo  C  Cryz~rr  C  C  C> +6		
 
 
 
 (tFCCCCs   !E9 9,F?'F?>F?rV   list[VariableTracker]rW   dict[str, VariableTracker]c                 z   |j                             | j                  }t          | j        |                                 ||          5  t          |          }t          |t          j        j	                  r|j
        j        t          j        j	        j        u rt          |          r|                     |           |r
J d            |rJ |\  }|j                                        D ]r\  }}|                    |j                             || j        |t%          t'          | j        |                              |gi            |                                }s|cd d d            S |r%|j        |j        | _        t1          ||||           |j                                         r|j                            d          rv|j        dk    rkt          |dd          r|                     |           ddlm}	  |	| |j         j        d	| j        gt?          ||          R  
          cd d d            S | j        s
J d            t          |t          j         j!                  r|j        }
t'          | j        d          }n|j"        }
t'          | j        d          }tG          |
tH          j%                  r|
j&        }
t'          |d          }| g|z   }ntG          |
tH          j'                  sJ |(                    tS          j*        |
|          ||          cd d d            S # 1 swxY w Y   d S )Nz3Expected lazy sequential isn't a valid combination?r   )z	torch.nn.z	torch.ao.ztorch.nn.utils.parametrizeT)check_forward_hookscheck_backward_hooksr%   wrap_fx_proxycall_moduler.   proxyzsMust provide a valid source in order to inline, since inlined function may have default args which must be guarded.ri   
_call_implrj   )+r   r   r]   rg   r   r   rC   rI   r   r   rc   ri   r   r   r   rE   r   r   r   r   re   popcls_to_becomerz   r\   is_root_tracer
__module__
startswithbuilderr   create_proxyr!   rJ   GraphModuler   r   rn   ro   rj   r   inline_user_function_returnr	   r   )r   r.   rV   rW   rU   is_lazyrO   
child_namer   r   fn	fn_sources               r9   r   zNNModuleVariable.call_function  s>    i%%do66#OT<<>>C
 
 a	 a	 %S))G3 344M)UX-@-HHH%c** 6 11"555 #  I { "!!z*-,*<*<*>*> # #&J$$	99" O&#1*T[*2U2U#V#V	 :   	 	 	 &&((CCCa	 a	 a	 a	 a	 a	 a	 a	F  >$0'*'8D$
 'r3f=== 	((**-N--.HII- N&BBB%T   6 11"555222222$}0")0% +488    Ca	 a	 a	 a	 a	 a	 a	 a	T {  Z { c58#788 F B *4;	 B BIIB *4; E EI"e.// :B *9j A AI 6D=DD!"e&8999995522iHHH {a	 a	 a	 a	 a	 a	 a	 a	 a	 a	 a	 a	 a	 a	 a	 a	 a	 a	s!   	DL0,CL0<C'L00L47L4Fc                     ddl mmm  j        j                                       fd}dv r                               S dk    rLt           j         	                                          5   |          cd d d            S # 1 swxY w Y   dk    rEt          j        t          j        j        j                            r                    d          S dk    rȉd                                         sJ t%          d	                   sJ d	         j        d         j                 }t%          |t*                    r|S |j        j                                      }j                            |t/          t1           j                            
          S |r5t5                    }	j        j         d dt9          |	          S  fdfd}
 fd} fd}d dk    rj        j                            t?           j        d                     srJ g }                                 D ]$\  }|!                     ||                     % |tE                                S dk    rj        j                            t?           j        d                     g } j#        d4i  |
dd          D ]$\  }|!                     ||                     % |tE                                S dk    rj        j                            t?           j        d                     g } j$        d4i  |
ddd          D ]$\  }|!                     ||                     % |tE                                S dk    rj        j                            t?           j        d                     g } j%        d4i  |
ddd          D ]$\  }|!                     ||                     % |tE                                S dk    rUj        j                            t?           j        d                     srJ  |                                           S dk    rOj        j                            t?           j        d                      |%                                          S d k    rQj        j                            t?           j        d                      | j#        d4i  |
d                    S d!k    rQj        j                            t?           j        d                      | j$        d4i  |
d                    S d"k    r`srJ g }&                                D ]*|!                                                             + |tE                                S d#k    r#srJ  |                                          S d$k    rZsrJ g }                                D ]$\  }|!                     ||                     % |tE                                S d%k    r(srJ                     tO                              S d&k    rt%          tP          j)        j*        tP          j)        j+        f          rPrNd	                                         r4                    d	         ,                                j-        v           S d'k    rstO                    dk    sJ tP          j)        j*        j.        tP          j)        j/        j.        tP          j)        j+        j.        tP          j)        j0        j.        tP          j)        j1        j.        f}te                    j.        |vrt%          d	         tf          j                  sJ ti          d	                               d	         ,                                t%          tj          tl          f          sJ t5                    j7        }	t%          |	tp          j9                  sJ t?          t?           j                  d(          }:                    tg          j;        |	|
           gty                    z             S  j        sJ t%          d	         tz                    rrj        j>        rPg }ty          t          tO                                        d	         ,                                         }t          d	         ,                                                   D ]_\  }}||         t/          t1           j                            }|!                    j                            ||
                     `d	         ,                                         }j                            |  d)t/          t1           j        d	         ,                                                    
          }|S  A                               dd*lBmC} t%          d	         |          r!d	         D                    j                  nTd	                                         rd	         ,                                nt          d+d,  d- d- d- d.g /                    }j                            | j        t/          t1           j                            
          S d0k    s^t%          tP          j)        jF        jG        jH                  rd1k    s/t%          tP          j)        jF        jG        jI                  rmd2k    rgt5                    j7        }	t?          t?           j                  d(          }:                    tg          j;        |	|
           gz             S j        jJ        v rht          j        jJ                           rIt          d3 t          jN        O                                          D                       r |          S t                      Q                              S )5Nr%   )r   ListIteratorVariableTupleVariablec           	          	j                             dj        di           }t          |j                   t                    \  }}ddlm}  |		j                             d| |g|R |                    S )Nget_attrr4   r%   r   call_method)rV   rW   r   )r   r   r]   r"   rL   r!   r   r   )
r`   	mod_proxyrX   rY   r   rV   rW   moduler   r.   s
        r9   generic_call_method_helperz@NNModuleVariable.call_method.<locals>.generic_call_method_helper#  s     	..	 I inf555'8v'F'F$J...... =i,,!#1j11'	 -     r;   r   _wrapped_call_implri   _check_input_dimT_get_item_by_idxr   r   __resultc                      t          d t          j                                                   D                       s(t	          dd d d  d d d dg            d S d S )	Nc              3   >   K   | ]}|                                 V  d S r3   )is_python_constantr6   rM   s     r9   r:   zUNNModuleVariable.call_method.<locals>.assert_all_args_kwargs_const.<locals>.<genexpr>d  s?        +,$$&&     r;   z&non-const argument in nn.Module methodcall_method: r   z(Dynamo does not support calling method `z` of ``nn.Module`` z with non-constant arguments.r   )all	itertoolschainvaluesr   )rV   rW   r  r`   r   s   r9   assert_all_args_kwargs_constzBNNModuleVariable.call_method.<locals>.assert_all_args_kwargs_constc  s      09fmmoo0V0V     	 !DIDII4II$IIII!^#!^ !^8>!^ !^ !^     	 	r;   c                                   t                    } t          j        |          j        d D             i d                                 D                                              j        fd| D             S )Nc                 6    g | ]}|                                 S r4   as_python_constantr  s     r9   rP   zDNNModuleVariable.call_method.<locals>.get_kwargs.<locals>.<listcomp>s  s$    888a1''))888r;   c                 >    i | ]\  }}||                                 S r4   r!  )r6   r>   r?   s      r9   r@   zDNNModuleVariable.call_method.<locals>.get_kwargs.<locals>.<dictcomp>t  s*    HHHA1a**,,HHHr;   c                 "    i | ]}||         S r4   r4   )r6   r>   
bound_argss     r9   r@   zDNNModuleVariable.call_method.<locals>.get_kwargs.<locals>.<dictcomp>x  s    444Az!}444r;   )r   r   	signaturebindrE   apply_defaults	arguments)namesr  r%  rV   r  rW   r  r`   s     @r9   
get_kwargsz0NNModuleVariable.call_method.<locals>.get_kwargso  s    ((***&&B3*2..3884888HHHHH J %%'''#-J4444e4444r;   c                     g }| D ]R\  }}|                     j                            ||t           j        |                                         S |t                                S )Nr   mutation_type)r   r   r   r   re   r'   )	rE   r   r`   r   r  
gen_sourcer   r   r.   s	       r9   wrap_valuesz1NNModuleVariable.call_method.<locals>.wrap_valuesz  s    F %  fI55-jjd.K.KLL	 6      ('>N>P>PQQQQr;   c                                           |           j                            || t           j        |                               g          S r   )r   r   r   r   re   )r`   objr   r  r/  r   r   r.   s     r9   named_embedz1NNModuleVariable.call_method.<locals>.named_embed  si     =$++D11I55-jjd.K.KLL	 6  
 
 
r;   c                     |                     d          }|d         dk    r| S t          |          dk    r8|                    d          }t          | |          } t          |          dk    8| S )N.r    )splitlenr   r   )re   r`   
name_splitrM   s       r9   r/  z0NNModuleVariable.call_method.<locals>.gen_source  sp    CJ!}""j//A%%NN1%%#FA.. j//A%% Mr;   named_childrenr   r-  named_parametersr   prefixrecursenamed_buffersr   remove_duplicatenamed_modulesmemochildrenmodules
parametersbuffersr   r  rE   __len____contains____getitem__rj   z.__getitem__(slice))SymNodeVariablez.Unsupported key type for nn.Module.__getitem__r  r   zEDynamo does not support getitem on `nn.Module` with non-constant key.r   _get_abs_string_index_conv_forward_output_paddingc              3   J   K   | ]}t          |t          j                  V  d S r3   )rC   r	   TensorVariabler  s     r9   r:   z/NNModuleVariable.call_method.<locals>.<genexpr>O  sC         1i677     r;   r4   )Rr6  r   r  r  r]   r   r   r   rg   r   r   is_torch_inline_allowedr   getfilerc   r  r   r  rC   rE   r{   r   r   r   r   re   r   __name__r)   guard_on_key_orderaddr   r:  r   r'   r;  r>  r@  r   r8  rI   r   r   ParameterDictr"  r   rH  r   r   r   rB   r	   r&   strintrj   rn   r   r  r   rF   r+   exportranger   r   tensorrI  evaluate_exprr   rC  conv_ConvNd_ConvTransposeNdrl   rm   r  r  r  r  r   r  ) r   r.   r`   rV   rW   constantr  mod_varr   r  r+  r0  r3  r   parambufferbuiltin_supportedsrcr   r   
new_modulenew_module_variablerI  r  r   r  r  r  r/  r   r  rc   s    `````                   @@@@@@@r9   r  zNNModuleVariable.call_method  s    	LKKKKKKKKKo((--	 	 	 	 	 	 	 	 	2 777 %%b$777Y (!@!@!B!BB  8 8 21$778 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8
 %%%+*MOF,=>>+
 +
% $**4000%%%7--/////d1g}555551gmDGM2G'#@AA $CY,,S11F944%mDK&E&EFF	 5     	L&&B&/??$???D/BdFKKK
	 
	 
	 
	 
	 
	 
	 
	 
			5 		5 		5 		5 		5 		5 		5 		5 		5	R 	R 	R 	R 	R 	R 	R 	R 	R	 	 	 	 	 	 	 	 	 		 	 	 ###I(,,ZZ-P-PQQQ''''F & 5 5 7 7 9 9fkk$778888''>N>P>PQQQQ'''I(,,Z]-S-STTTF6v6    *Xy11    8 8e kk$667777''>N>P>PQQQQ_$$I(,,ZZ-P-PQQQF 4 4 ! !*Xy2DEE! ! 9 9f kk$778888''>N>P>PQQQQ_$$I(,,ZZ-P-PQQQF 4 4 ! !*VX/ABB! ! 9 9f kk$778888''>N>P>PQQQQZI(,,ZZ-P-PQQQ'''';v4466777YI(,,ZZ-P-PQQQ;v3355666\!!I(,,Z]-S-STTT;6v6OOI9N9NOOPPPYI(,,ZZ-P-PQQQ;3v3LLjj6K6KLLMMMV^^''''F = =.55d;;<<<<''>N>P>PQQQQX'''';v||~~...W__''''F & 9 9fkk$778888''>N>P>PQQQQY''''#**3v;;777N""6EH$79O#PQQ # # Q**,, #
 $**Q**,,?   ]""0#d))q...0#/#/&2&2#/! F||'/@@@!$q'9+EFFXXPTUVPWHXHXXXF1g0022!#Sz22222VT**3!"e&899999 DK!>!>
KK5522cBBBFT$ZZ'   ;;$q'=11  6 9# 6F  c&kk 2 233DG4N4N4P4PQD'0Q8R8R8T8T1U'V'V 	 	V"3i,]4;-L-LMMI== & #'* >      "(Q(B(B(D(D!EJ*,)*K*K"444-)$+tAw7Q7Q7S7STT    +L + +' /. 11"555//////$q'?33 1g++BI66a++-- 	1g0022 LIDII4II$IIII!9    C[F944%mDK&E&EFF	 5    +++658#3#8#@AA , O++ 658#3#8#IJJ , --- &&/B":dk4#@#@*MMI11.r)DDD   F$---)24899 .  "v}}??     . .-d33377&&r4v>>>s   B$$B(+B(r.   r-   rV   r   rW   r   r}   r(   )F)rQ  r   __qualname__r(   _nonvar_fieldsrB   rU  rI   r   Moduler   r   r   r   r   r   r   r   r   r   r   r   r   r  __classcell__rc   s   @r9   ry   ry      s        	
 
	'N22-029>2	2 2 2 2 2 2: : :- - -     )   
  @
9)
914
9	
9 
9 
9 
9/ / /* * *       .)
 )
 )
VmD mD mD mD^i &i -	i
 
i i i ib @? &	@?
 -@? 
@? @? @? @? @? @? @? @? @? @?r;   ry   c                        e Zd Zdddhej        Z	 d fdZd Zd Zd	 Ze	e
j        d
                         Z fdZ	 	 	 	 	 	 	 	 ddZ	 	 	 	 	 	 d fdZddZd fdZddZ xZS )r   
value_typeis_state_mutatedr|   r}   Nc                 H   t          |          t          j        j        j        u rt          d          d|v r:t          |d         dd           }t          |          |u rt          |          |d<    t                      j        dd|i| d| _	        | j
        | _        d S )NzScriptModules aren't supported in UnspecializedNNModuleVariable because their .forward function isn't a static member of their typern  r   r{   Fr4   )rB   rI   jit_scriptRecursiveScriptModuler   r   r   r   ro  re   r|   )r   r{   rW   lazy_value_to_becomerc   s       r9   r   z&UnspecializedNNModuleVariable.__init__i  s    ;;%)+AAAW   6!!#*6,+?RV#W#W E{{222
 (,E{{|$//u//// % '+k###r;   c                     |S r3   r4   r   attr_sources     r9   _wrap_sourcez*UnspecializedNNModuleVariable._wrap_source      r;   c                     | j         p| j        S r3   r   r   s    r9   r   z8UnspecializedNNModuleVariable.get_nn_module_stack_source  r   r;   c                     || _         d S r3   r   r   s     r9   r   z8UnspecializedNNModuleVariable.set_nn_module_stack_source  r   r;   c                       t           j        j        j        t           j        j        j        t           j        j        j        h  fdt           j        j        j                                        D             S )Nc                 `    h | ]*}t          |d           r|vt          |j                  +S )__code__)rR   idr~  )r6   rM   	supporteds     r9   	<setcomp>zFUnspecializedNNModuleVariable._nn_module_method_ids.<locals>.<setcomp>  sJ     
 
 
q*%%
 +,9*<*< qzNN*<*<*<r;   )rI   r   rj  __setattr__r   __delattr__rl   r  )r  s   @r9   _nn_module_method_idsz3UnspecializedNNModuleVariable._nn_module_method_ids  sh    
 HO'HO$HO'
	

 
 
 
X_-4466
 
 
 	
r;   c                    	 t          j        | j        d          }n# t          $ r}t          |d }~ww xY w|t
          j        j        j        t
          j        j	        j        t
          j        j
        j        fv r=|                    t          j        |          | gi                               |          S t                                          |          S )N__iter__)r   r   rn  rT   NotImplementedErrorrI   r   r   r  r   r   r  r	   r   r   r   )r   r.   r  erc   s       r9   r   z1UnspecializedNNModuleVariable.unpack_var_sequence  s    	-'DDBB 	- 	- 	-%1,	- H(H"+H(
 
 
 11.r22   "!"%%& ww**2...s    
505r.   r-   rV   r   rW   r   r(   c                 z   | j         }t          |          r%|j        |j        | _        t	          ||||           t          |t          j        j                  s+|j	        j
        t          urd}t          | j        |          }nd}t          | j        |          }t          |j	        t          j                  r!t          |j        t          j                  r|j	        j
        t          u r|j        j
        t"          u rd|j        vrt'          j        |d          }t          |t          j                  r|j        }|                     |d                                                                          s]|                     |d                                                                          s"|                     |d                                                                          s|                     |d                                                                          s|                    |d                                          s|                    |d	                                          s^|                    |d
                                          s6|                    |d                                          sd}| j        j        }| j        r|                     |          }	nd }	t;          | j        |           | j        r>t=          t?          tA          |                    | !                                ||          ntE                      }
|
5  tG          j$        ||	          %                    || gtM          |          z   |          cd d d            S # 1 swxY w Y   d S )N__call__r   ri   _backward_hooks_backward_pre_hooks_forward_hooks_forward_pre_hooks_global_backward_pre_hooks_global_backward_hooks_global_forward_hooks_global_forward_pre_hooksr   )'r{   r   r   rn  r\   rC   rI   rJ   r   r  rj   r#   r   r   rn   ro   r   r$   rl   r   r   r   nn_modules_globals_vtr   realizer8  ri   re   get_source_by_walking_mrorw   rg   rU  r  r   r   r	   r   r   rF   )r   r.   rV   rW   rU   r`   r  forward_method
globals_vtre   ctxs              r9   r   z+UnspecializedNNModuleVariable.call_function  s    j# 	: ,"%"3"2sD&999 3 455	0%-EEED$//BBD$//B
 3<!122	1s~u'788	1 %)AAA'+HHH--$3CCCN.%*<== 15
$$R):;;CCEEIIKK1'',ABBJJLLPPRR1 '',<==EEGGKKMM1 '',@AAIIKKOOQQ	1
 "--b2NOOSSUU1 "--b2JKKOOQQ1 "--b2IJJNNPP1 "--b2MNNRRTT1 %D0B; 	33D99FFF.t{C@@@ {"BsGGd==??S    	  	 	1"VDDDRRTFT$ZZ' 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   '<N00N47N4c                    |dv rpt          | j        |          }| j        r|                     |          }nd }t	          j        ||                              || gt          |          z   |          S |t          | j        di           vr	 t          j
        t          | j                  |          }n# t          $ r d }Y nw xY wt          |t                    rSt          |                     |          d          }|                    t	          j        |j        |          ||          S t%          |d          r_t'          |j                  |                                 v r7t-          dd|  d| d| d| d	| d
| j         dgt.          j                   |dk    rd| j        j        v r|d                                         }|d         }	t          |	t          j                  r%|	                                t:          j        j        u s|| j        j        d         v rd| _         n]|| j        j        d         v rd| _         nAt          |	t          j!        t          j"        f          s|| j        j        d         v rd| _         |t:          j        j#        j$        u r t          |d         t          j%                  s|t:          j        j#        j&        u rE|                    t	          j        t:          j        j#        j&                  | |d         g|          S tO                      (                    ||||          S )Nr  r   rl   rj   r~  z,UnspecializedNNModuleVariable missing methodr  r   z'Dynamo does not support tracing method z of nn.Module z@Dynamo does not really define unspecialized nn.Module very well.r   r  r   r   r%   Tr   r   ))r   rn  re   r  r	   r   r   rF   r{   r   r   rB   rT   rC   r   r   r  rj   rR   r  r~  r  r   r   	DIFFICULTrl   r"  rN  r   rI   r   	Parameterro  ry   r   rj  r  r   r  r   r  )r   r.   r`   rV   rW   r  re   method	attr_namer{   rc   s             r9   r  z)UnspecializedNNModuleVariable.call_method  sv    777$//B{ 77==1"VDDDRRTFT$ZZ'   wtz:r:::: /TZ0@0@$GG!    &,// #D$B$B4$H$H*UU5526?6RRR   
++v''4+E+E+G+GGG JIDII4II$IIII j$ j j^b^h j jZ*4	    }$$$*:M)M)M !G6688	Q
 ui&>??1))++ux/AAA$*"5m"DDD,0D))$*"5j"AAA,0D))%6%C 1 !DJ$7
$CCC -1D) %(/555tAw	(ABB 658?666 55258?3NOO47O   ww""2tT6:::s   'B8 8CCc                     |                      ||          }t          |t          j                  r|                    |          S d S r3   )r   rC   r	   ConstDictVariablemaybe_getitem_const)r   r.   fieldname_vtdict_vts        r9   getattr_helperz,UnspecializedNNModuleVariable.getattr_helperX  sE    ""2u--gy:;; 	8..w777tr;   c                    |dv r|j         j                            | |          st          | j        |          }t          |t                    rot          |          dk    r\| j        rAt          | j        |          }t          |                    t          j                             t          j        i           S | j        r|dv r|j         j                            | |          st          | j        |          }t          | j        |          t                              t          j                             |j         j                                       fdt          fdt'          |                                          D                       }t          j        |t-          |                    S t/                                          ||          S )N)r  r  r  r  r   )r  r  c                     t           j                            |          }t          |           }t	          |          }t          j        ||          }||fS r3   )r	   r   r   r   r   r*   )ir>   r?   r   
source_keysource_valuer{   hooks_dict_sources          r9   build_key_valuezBUnspecializedNNModuleVariable.var_getattr.<locals>.build_key_value  sW    077:: 00A1EE
01BJOO+21lCCEz!r;   c              3   >   K   | ]\  }\  }} |||          V  d S r3   r4   )r6   r  r>   r?   r  s       r9   r:   z<UnspecializedNNModuleVariable.var_getattr.<locals>.<genexpr>  sJ        -6QA1a((     r;   r   )r   r   r   r   r{   rC   rD   r8  re   r   r   rp   r   EMPTY_NN_MODULE_HOOKS_DICTr	   r  SEQUENCE_LENGTHrR  rS  r   rE   NNModuleHooksDictVariablerB   r   r   )	r   r.   r`   
hooks_dicthooks_sourcer   r  r  rc   s	         @@r9   r   z)UnspecializedNNModuleVariable.var_getattr^  s
    
 
 
 9)FFtTRR 
;$TZ66
j$// ;C
OOq4H4H{ '1$+t'D'D%(33 , G   
 %6r::: K	 
 I*GGdSS !T22J *4; = =+66|7STTUUUI(,,->???	" 	" 	" 	" 	"     :CJDTDTDVDV:W:W    F 6Z((1B    ww""2t,,,r;   c                     t          j        |          }|                     |d|          }||                     |d|          }||                     |d|          }|t          t          |           |S )a  
        Dynamo tracing of nn.Module __getattr__ can be expensive if the model
        has deep submodule hierarchy. Since the __getattr__ is stable, we can
        directly look into the underlying datastructures. This saves a lot of
        compilation time.
        r   Nr   r   )r	   r   r  r
   rT   )r   r.   r`   r  r   s        r9    manually_trace_nn_module_getattrz>UnspecializedNNModuleVariable.manually_trace_nn_module_getattr  s}     ,T22!!"mW==;%%b*g>>C;%%b*g>>C;$^R888
r;   r}   N)r.   r-   rV   r   rW   r   r}   r(   rg  rf  )rQ  r   rh  r,   ri  r   rx  r   r   r   rr   cacher  r   r   r  r  r   r  rk  rl  s   @r9   r   r   Y  s         
#	1	N2 2 2 2 2 22  : : :- - - _
 
 _ \
/ / / / /0A#A &A -	A
 
A A A AF`; &	`;
 -`; 
`; `; `; `; `; `;D   ;- ;- ;- ;- ;- ;-z       r;   r   c                       e Zd ZdZd ZdS )$UnspecializedBuiltinNNModuleVariablezg
    Differentiates between builtin nn modules (e.g. torch.nn.Linear) and user defined nn modules.
    c                     |S r3   r4   rv  s     r9   rx  z1UnspecializedBuiltinNNModuleVariable._wrap_source  ry  r;   N)rQ  r   rh  __doc__rx  r4   r;   r9   r  r    s-             r;   r  c                   *     e Zd ZdZd fdZd Z xZS )FSDPManagedNNModuleVariablea  
    Tracing behavior: trace into submodules and treat them as Unspecialized, do not
    register parameters to the top-level, treat them as function inputs.

    Guards behavior: if 'skip_fsdp_guards', many guards that would be installed
    by a vanilla UnspecializedNNModuleVariable are simply dropped, on the basis
    that a user wrapping their model in FSDP(model) is already opting into a
    requirement to not modify internal model state, which would already break FSDP without
    compilation.
    r}   Nc                     |                     dd           }|
J d             t                      j        dd|i| || _        d S )Nre   zMFSDPManagedNNModule depends on having an accurate source to control guarding.r{   r4   )rb   r   r   re   )r   r{   rW   re   rc   s       r9   r   z$FSDPManagedNNModuleVariable.__init__  s\    Hd++!![ "!! 	//u////r;   c                     t          |t          t          f          s4t          j        j        j        rt          |          S t          |          S |S r3   )rC   r   r   rI   _dynamoconfigskip_fsdp_guardsrv  s     r9   rx  z(FSDPManagedNNModuleVariable._wrap_source  sV    ,.IJ
 
 	@ }#4 @)+6662;???r;   r  )rQ  r   rh  r  r   rx  rk  rl  s   @r9   r  r    sV        	 	           r;   r  rf  )Hr  rr   r   r  rn   
contextlibr   r   typingr   torch.nnrI   r6  r   r   r	   excr
   r   r   r   guardsr   r   mutation_guardr   re   r   r   r   r   r   r   r   utilsr   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r   r&   r'   r(   	functionsr)   lazyr*   listsr+   user_definedr,   torch._dynamo.symbolic_convertr-   r\   rU  r   rj  rg   rw   ry   r   r  r  r4   r;   r9   <module>r     s   .           2 2 2 2 2 2 2 2              8 8 8 8 8 8 8 8 8 8            1 0 0 0 0 0 0 0 . . . . . .                                               = < < < < < < < < < 3 3 3 3 3 3 % % % % % %             3 3 3 3 3 3  EDDDDDD   D 	+s 	+UX_ 	+ 	+ 	+ 	+  Bw
? w
? w
? w
? w
? w
? w
? w
?tQ Q Q Q Q$= Q Q Qh
    +H       "?     r;   