
    %Vji"                        d Z ddlZddlZddlZddlZddlZddlmZmZ ddl	m
Z
mZ ddlmZmZmZmZmZ ddlZddlmc mZ ddlmZ ddlmZmZmZmZmZ ddlm Z m!Z! dd	l"m#Z#m$Z$m%Z%m&Z& dd
l'm(Z(m)Z) ddl*m+Z+m,Z,m-Z-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5 ddl6m7Z7 ddl8m9Z9 ddl:m;Z; ddl<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZC ddlDmEZEmFZF ddlGmHZHmIZI ddlJmKZKmLZL ddlMmNZN ddlOmPZP erddlQmRZR dZS e0eTd          ZU e0eTd          ZVdeWfdZXdeWfdZYdeejZ                 deejZ                 fd Z[d!eded"ef         fd#Z\ G d$ d%          Z] G d& d'          Z^ G d( d)          Z_ e^            Z`g d*Za eNeeejb        eg          Zc ejd                    aed+efdefd,Zg G d- d.          Zhd/aid/ajd/akd/aldamejn        	 	 dEd1ed"ef         d2eWd3eWde
d4         fd5            Zoejn        de
d4         fd6            ZpdFd7Zqd8eejZ                 d9eeL         d:eeL         d;eLd<eeL         d=eeL         d>eLderejZ                 fd?Zsd@eeW         dAejZ        dBeeejZ                          dCejZ        dereejZ                          f
dDZtdS )Ga  
Provides functionality for compiling PyTorch's autograd (automatic differentiation) system.

This module implements compiled autograd, which traces and optimizes backward pass
computations at runtime. The key components are:

- AutogradCompilerInstance: Traces and compiles autograd graphs using FX
- Context managers (_enable/_disable): Control when compiled autograd is active
- Utility functions: Support graph manipulation, tensor operations, and hooks

Compiled autograd can significantly improve backward pass performance by removing
Python overhead and enabling additional optimizations. It works by capturing
backward computations into an FX graph that can be compiled and optimized,
while maintaining the same semantics as eager mode autograd.
    N)Counterdefaultdict)	GeneratorSequence)AnyCallableOptionalTYPE_CHECKINGUnion)enable_python_dispatcher)call_accumulate_gradcall_backward	call_hookFakeCompiledAutogradEngineunwrap_maybe_dynamic_intGetItemSourceLocalSource)countersget_chromium_event_loggerlazy_format_graph_codeset_locals_to_steal)AutogradLazyBackwardCompileInfo%CachedAutogradLazyBackwardCompileInfo)compile_contextCompileContext	CompileIdSource)getArtifactLoggertrace_structuredclone_preserve_strides)FakeTensorMode)
FakeTensor)GraphModule)BackwardState)	decomposedisable_autocast_cachedisable_proxy_modes_tracingfetch_object_proxyProxyTorchDispatchModePythonKeyTracertrack_tensor_tree)
DimDynamicShapeEnv)preserve_node_metaset_stack_trace)FloatLikeTypeIntLikeType)
OrderedSet)CapturedTraceback)Proxya  You can turn off compiled autograd by either:
1. Moving the unsupported autograd call outside of the torch.compile'd region.
2. Wrapping the unsupported autograd call in the torch._dynamo.compiled_autograd._disable() context manager.
3. Setting torch._dynamo.config.compiled_autograd=False for the torch.compile call containing the unsupported autograd call.
4. Setting torch._dynamo.config.compiled_autograd=False at the start of the program.compiled_autogradcompiled_autograd_verbosereturnc                  T    t           j        j        j                            d          S )Nr8   )torch_logging	_internal	log_stateis_artifact_enabled     Y/root/voice-cloning/.venv/lib/python3.11/site-packages/torch/_dynamo/compiled_autograd.py snapshot_verbose_logging_enabledrC   T   s%    >#-AA#  rA   c                  8    t           j        j        j        j        S N)r;   	_inductorconfigtriton
cudagraphsr@   rA   rB   snapshot_cudagraph_enabledrJ   Z   s    ?!(33rA   xc                 (    | t          |           S | S rE   r!   )rK   s    rB   maybe_clonerM   ^   s    }%a(((HrA   CompiledFunction.c                 L   t          | j        t                    r| j        j        S t          | j        t                    rVt
          j        j                                        5  | j        	                                cd d d            S # 1 swxY w Y   d S t          d          )NzEUnexpected Lazy Backward Compilation Info Type. Please file an issue.)
isinstance_lazy_backward_infor   	bw_moduler   r;   _subclassesfake_tensorunset_fake_temporarilybw_module_fnAssertionError)rN   s    rB   extract_bw_modulerX   d   s   ,.M  
  3==	,.S
 
 
 *AACC 	G 	G#7DDFF	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G S
 
 	
s   $B

BBc                       e Zd ZdeddfdZdej        j        ddfdZde	ej
                 ddfdZd	e	ej
                 ddfd
ZdS )
NaNCheckeraccumulate_gradr9   Nc                 >    || _         g | _        i | _        g | _        d S rE   )r[   params_indicesparams_to_checkoutput_names)selfr[   s     rB   __init__zNaNChecker.__init__   s(    .)+8:')rA   graphc                 6   t          t          |j                            }|                    dt                    }|                    d          d         j        d         }| j        t          |          k    r| j        | k    sJ |D ]z}|j        d         }|j        t          j
        k    r/|j        d         |u r t          |j        d         t                    sJ | j                            |j        d                    {d |D             | _        d S )Ncall_functionoptargetoutputrf   r      c                     g | ]	}|j         
S r@   )name).0nodes     rB   
<listcomp>z.NaNChecker.prep_with_graph.<locals>.<listcomp>   s    @@@4TY@@@rA   )nextiternodes
find_nodesr   argsr[   boolrg   operatorgetitemrP   intr]   appendr_   )r`   rb   inputs_nodeacc_grad_nodesoutput_nodesrn   
param_nodes          rB   prep_with_graphzNaNChecker.prep_with_graph   s7   4,,--))'; * 
 
 ''8'44Q7<Q?#t(
 (
 
 
"<'78889 # 
	; 
	;D1J
 !X%555OA&+55zq1377 658 &&zq'9::::@@<@@@rA   inputsc                     | j         sd S | j        D ]Y}||         j        }|4t          j        |                                          rJ d| d            ||         | j        d| d<   Zd S )Nz9Compiled autograd running under anomaly mode with inputs[zD] already having NaN gradient. This is not supported. {TURN_OFF_MSG}zinputs[])r[   r]   gradr;   isnananyr^   )r`   r   idxr   s       rB   prep_with_inputszNaNChecker.prep_with_inputs   s    # 	F & 	A 	AC#;#D ;t,,0022  QPS Q Q Q 2
 6<C[D !13!1!1!122	A 	ArA   outc                 N   | j         r|rJ g }| j                                        D ]N\  }}|j        J t	          j        |j                                                  r|                    |           O|r&t          dd	                    |           d          d S g }t          |          D ]K\  }}t	          j        |                                          r |                    | j        |                    L|r&t          dd	                    |           d          d S )Nz9Compiled Autograd returned NaN gradients for parameters: ,.z;Compiled Autograd returned NaN gradients for output nodes: )r[   r^   itemsr   r;   r   r   ry   RuntimeErrorjoin	enumerater_   )r`   r   
nan_params
inputs_strparam	nan_gradsir   s           rB   checkzNaNChecker.check   s`    	NN7$&J%)%9%?%?%A%A 2 2!
Ez---;uz**..00 2%%j111 "gPSPXPXYcPdPdggg    $&I$S>> ; ;4;t$$((** ;$$T%6q%9::: "hRURZRZ[dReRehhh   rA   )__name__
__module____qualname__ru   ra   r;   fxGraphr~   tupleTensorr   r   r@   rA   rB   rZ   rZ      s        * * * * * *AUX^ A A A A A0AuU\': At A A A A u|,       rA   rZ   c            
       R    e Zd ZddZdededef         dededef
d	Zdedefd
Z	dS )OpNamespacer9   Nc                 ,    t                      | _        d S rE   )r   custom_function_name_counterr`   s    rB   ra   zOpNamespace.__init__   s    :A)))))rA   rl   fn.is_custom_functionis_traceablec                    |r-d|z   }| j         |         }| j         |xx         dz  cc<   | | }t          | |          rJ t          |||          |r/t          | |t          j                                                 nCt          j        j        dt          dt          dt          ffd            }t          | ||           |S )NCppNoderj   rt   kwargsr9   c                       | i |S rE   r@   )rt   r   results     rB   run_non_traceable_cpp_in_eagerz7OpNamespace.add.<locals>.run_non_traceable_cpp_in_eager   s    vt.v...rA   )	r   hasattrOpsetattrr;   _dynamoallow_in_graphdisabler   )r`   rl   r   r   r   countr   r   s          @rB   addzOpNamespace.add   s     	$t#D5d;E-d333q8333#E##D4&&&&&D"011 		@D$ < <V D DEEEE ]"/c /S /S / / / / / #"/ D$ >???rA   c                 "    t          | |          S rE   )getattr)r`   rl   s     rB   getzOpNamespace.get   s    tT"""rA   r9   N)
r   r   r   ra   strr   r   ru   r   r   r@   rA   rB   r   r      s        D D D D S#X !	
  
   6# # # # # # # #rA   r   c                   V    e Zd Zdededef         deddfdZded	edefd
ZdefdZ	dS )r   rl   r   .r   r9   Nc                 >    || _         || _        || _        d| _        d S )Nz#torch._dynamo.compiled_autograd.ops)r   r   r   r   )r`   rl   r   r   s       rB   ra   zOp.__init__   s%     "4?rA   rt   r   c                      | j         |i |S rE   )r   )r`   rt   r   s      rB   __call__zOp.__call__   s    tw''''rA   c                 &    | j         dz   | j        z   S )Nr   )r   r   r   s    rB   __repr__zOp.__repr__   s    $t}44rA   )
r   r   r   r   r   r   ru   ra   r   r   r@   rA   rB   r   r      s        @@%c3h/@EI@	@ @ @ @(c (S (S ( ( ( (5# 5 5 5 5 5 5rA   r   )r   sizesscalarshookspacked_datacompiled_autograd_idc           	      Z    t          t          t          | d d                               S )N)r   frame_idframe_compile_id)r   r   r   )r   s    rB   make_compile_contextr     s<    %9!%  	
 	
  rA   c                   n   e Zd Zdedef         ddfdZdej        dee	         de
fdZed	ed
edefd            Zdeej                 dee         deeeef                  deeeeef                           dededeeeej                 ee         ee         f         fdZdee         ddfdZdee         deej                 deej                 dededee         dee         fdZdee         dee         deej                 dedej        j        j        dee         deeej                 df         fdZdee         dee         d ee         d!ed"ee         d#ee         d$edeej                 fd%Z d&ee         d'ej        d(ee         d)ej        deej                 f
d*Z!dej        fd+Z"d,ed-edef         d.ed/edef
d0Z#d,ed1ee         d2ed3ee         deej                 f
d4Z$d-edef         d2ed3ee         deej                 fd5Z%d6ed7ee         d2ed3ee         deej                 f
d8Z&d9ed:edej        fd;Z'd<ej        d=ej        d>eddfd?Z(d@edef         d2edAedej)        j*        fdBZ+dCedDedej        fdEZ,deej                 dCedFedeej                 fdGZ-deej                 dCedFedeej                 fdHZ.dee         dCedeej                 fdIZ/d7eej                 deej                 dCedeej                 fdJZ0dKej        dCedeej                 fdLZ1dMej)        j2        dee         fdNZ3dOedefdPZ4djdQZ5de6e         fdRZ7dSede8fdTZ9d7edeedef         ef         fdUZ:ed2ee         deej)        j;                 fdV            Z<edOej)        j;        defdW            Z=djdXZ>djdYZ?djdZZ@djd[ZAdjd\ZBdjd]ZCdjd^ZDd_edefd`ZE	 dkdaee         dbedeeeeef                           dee         fdcZFddedeGfdeZHdfedgedheej        jI                 ddfdiZJdS )lAutogradCompilerInstancecompiler_fn.r9   Nc                 ,   || _         t          j                    | _        | j        j        | _        t                      | _        t          dd| j                  | _        t                      | _
        t          | j
        d          | _        d | _        d S )NT)allow_fallback_kernelsallow_non_fake_inputs	shape_envsymbolic)r   
contextlib	ExitStackstackcloser/   r   r#   fake_tensor_moder,   	fx_tracerr+   
proxy_modehooks_proxy)r`   r   s     rB   ra   z!AutogradCompilerInstance.__init__  s    &)++
Z%
! .#'"&n!
 !
 !

 )**0LL,0rA   rK   sourcec                 r    t          |t          j                  sJ | j                            ||          S )N)r   )rP   r;   r   r   from_tensor)r`   rK   r   s      rB   	wrap_fakez"AutogradCompilerInstance.wrap_fake,  s6    !U\*****$0060BBBrA   rl   r   c                 <    t          t          |           |          S rE   r   )rl   r   s     rB   r   zAutogradCompilerInstance.source0  s    [..444rA   r   r   r   originsr[   
check_nansc                 
    t           d         dxx         dz  cc<   t          t                     _        t	          t
                     _        t           j                   _         j        	                                 |rt          |          nd  _        t          j                     _        t                                          d j        d j        id           t"          j                                         j        _        t"          j                            t0                     j        _        i  j        _        i  _         fdt8          D             \  } _         _         _         _          j!        "                    tG                                 |\  }}	}
 j!        "                    tI                                 |d	         }	 tK          |          D ]2\  }} &                    | '                    d
|                    ||<   3n:# tP          $ r-}tS          dtU          |           dtV                     |d }~ww xY w ,                    |||            fdtK          |          D             } fdt[          t]          |                    D             }tK          |          D ]F\  }} j        /                    dt`          ||         fi           ||<   ||          j        |j1        <   G ,                    |||	          }tK          |          D ]\  }} '                    d|          }te          |t
                    r* j3        4                    ||tj          j6                  ||<   Zte          |tn                    rF j3        8                     j3        9                    ||tj          j6                  ||          ||<   tu          dtU          |                     ,                    | j        |
           tK          |          D ]\  }} j        |          j        |j1        <     j!        "                    tw          i                       j!        "                     j<                    j!        "                     j=                    j!        "                    t}                                  j<        j3        J  j<        j3        } j!        "                    t"          j        j?        j@        A                    |                     t          t          jD                              |||fS )Nr7   capturesrj   graph_idTlog_pt2_compile_event)
tracer_clsc              3   R   K   | ]!}j                             d |di           V  "dS )placeholderr@   N)r   create_proxy)rm   rl   r`   s     rB   	<genexpr>z9AutogradCompilerInstance.begin_capture.<locals>.<genexpr>T  sM       
 
 N''tRDD
 
 
 
 
 
rA   r   r   zFound tensor of type z,, which is not supported by FakeTensorMode. c           	          g | ]?\  }}j                             |                    d |          t          j                  @S )r   )r   $create_unspecified_symint_and_symbolr   r.   DYNAMIC)rm   r   valr`   s      rB   ro   z:AutogradCompilerInstance.begin_capture.<locals>.<listcomp>l  s[     
 
 
 S N??GS))" 
 
 
rA   c                 *    g | ]}j         |         S r@   )sizes_proxy)rm   r   r`   s     rB   ro   z:AutogradCompilerInstance.begin_capture.<locals>.<listcomp>x  s!    FFF14#A&FFFrA   rd   r   )r   dynamic_dim)hintr   zUnexpected scalar type: )Er   rp   COMPILE_COUNTERidr   rx   aot_id_counterr   r   	__enter__rZ   nan_checkertimetime_nsstart_time_nsr   log_event_startr;   nnModuler   rootr   r   r,   rb   tensor_attrssymnode_proxy_lookup_graph_placeholdersr   scalars_proxyr   packed_data_proxyr   enter_contextr0   r   r   r   r   	ExceptionNotImplementedErrortypeTURN_OFF_MSGbind_objects_to_proxiesrangelenr   r   rn   rP   r   r   r.   r   floatcreate_symfloatnodecreate_unspecified_symbolrW   r'   r   r   r(   experimentalsymbolic_shapes_suppress_guardsr   r   current_compile_id)r`   r   r   r   r   r[   r   
args_proxyinputs_originssizes_originsscalars_originsrK   r   e	sym_sizesproxiesr   symintr   r   symvalenvs   `                     rB   begin_capturez&AutogradCompilerInstance.begin_capture4  s[    	$%j111Q6111''.9#.>.>3DG<<&&(((:DN:o666$!\^^!##33!"&	 	4 	
 	
 	
 $hoo//$x~~~II&(#$&!
 
 
 
+
 
 
	
" 	
  !3!5!56669@6 	
  !9!;!;<<< 1I	#F++ L LQ"nnQHc0J0JKKsL 	 	 	%kQkk]ikk 	 	$$VZHHH
 
 
 
 &e,,
 
 
	 GFFFc)nn0E0EFFF"9-- 	@ 	@IAv44(	 GAJ 6=QZD%fk22..y'=QQ!'** 	L 	LHC[[C00F#s## L#~RR&   
 C'' L#~AAN<<%$.$6 =  
 !  B     %%?cKKK$$Wd.@/RRR"7++ 	K 	KIAv595G5JD%fk22 	
  2///
  !6777
  111
  !7!9!9:::$.:::#-
  H!1BB3GG	
 	
 	
 13344	
 	
s   AH 
I!(I		Icompile_reasonsc                 <    sJ t          dd fd           d S )Nartifactc                      dddS )N!compiled_autograd_compile_reasonsjsonrl   encodingr@   r@   rA   rB   <lambda>z>AutogradCompilerInstance.log_compile_reasons.<locals>.<lambda>  s    ;"! ! rA   c                       S rE   r@   )r  s   rB   r"  z>AutogradCompilerInstance.log_compile_reasons.<locals>.<lambda>  s     rA   metadata_fn
payload_fn)r    )r`   r  s    `rB   log_compile_reasonsz,AutogradCompilerInstance.log_compile_reasons  sL       /...	
 	
 	
 	
 	
 	
rA   pinputspsaved_tensorssaved_tensorspctxctxmaybe_backward_state_idxc           	           fd                                 D             }j        }t          |          |j        |j        |j        ~t          j                    r j        D ]}	|	j	        rt          d          t          j        j        dt          t          j                 dt          t                   dt          t                    dt           ffd            }
 j                            d|
||g|R i 	          d | j        |         dt(          t          j                 f fd
} |            }dt           dt*          dt          t                    dt          j        f fd}t          j        j        j                            ||          }t5          j         j        |          }|S )Nc                 :    g | ]}                     |          S r@   to_proxyrm   r  r`   s     rB   ro   zDAutogradCompilerInstance.proxy_call_aot_backward.<locals>.<listcomp>  s%    SSSDMM!$$SSSrA   z@torch.compile does not currently support higher order gradients.ctx_saved_tensorsctx_symints	flat_argsr9   c                 P    t          j        j        j        j        | |g|R  }|S rE   )r;   
_functorch_aot_autogradruntime_wrappers_backward_prologue_functional)r3  r4  r5  r   maybe_subclass_metadatametadatas       rB   call_aot_bwd_prologuezOAutogradCompilerInstance.proxy_call_aot_backward.<locals>.call_aot_bwd_prologue  sB     "0A_!'	
   C JrA   rd   kindrg   rt   r   c            	      D   dt           j        j        dt          fd}  | j                  }fdt          |t          d u          z
            D             }                                }t          |          t          j                  k    sJ fd|D             }||d t          |          <   |	                               d}i d }t                    j                 rdj                  z  j        xx         dz  cc<   d	t          dt          ffd
}j        j        D ]*}|j        dk    r-||         j        }	 ||j                  |	_        |	|<   |dz  };|j        dk    r5t          |j                  dk    sJ fd|j        d         D             }{|j        dk    r|j        }
j                            |
          }t)          j        j        |t-          |
                     j                            d|di           } ||j                  |_        ||<   	|j        dk    r|j        t           j        j        j        j        k    r t           j        j        j        j        |_        j        j                            |fd          } ||j                  |_        ||<   |j        dk    r{|j        }
j                            |
          }t)          j        j        |t-          |
                     j        j                            |fd          }||_        ||<   t=          d          |J dt           j        fdfd|D             }                     ||           |S )Nrb   r9   c                 @    d}| j         D ]}|j        dk    r|dz  } |S )Nr   r   rj   )rr   rf   )rb   num_argsrn   s      rB   
num_inputszkAutogradCompilerInstance.proxy_call_aot_backward.<locals>.copy_paste_aot_backward_graph.<locals>.num_inputs   s9    !K  Dw-// A rA   c                      g | ]
}|         S r@   r@   )rm   r   pgradss     rB   ro   zkAutogradCompilerInstance.proxy_call_aot_backward.<locals>.copy_paste_aot_backward_graph.<locals>.<listcomp>  s*       q	  rA   c                 :    g | ]}                     |          S r@   r0  r2  s     rB   ro   zkAutogradCompilerInstance.proxy_call_aot_backward.<locals>.copy_paste_aot_backward_graph.<locals>.<listcomp>  s%    :::Qa((:::rA   r   _rj   	node_namec                     d d|  S )NaotrG  r@   )rH  deduped_aot_ids    rB   make_uniquezlAutogradCompilerInstance.proxy_call_aot_backward.<locals>.copy_paste_aot_backward_graph.<locals>.make_unique&  s    9^99i999rA   r   rh   c                     g | ]N}t          |t          j        j                  r+t          j                            |         j                  n|OS r@   )rP   r;   r   Noder6   r   )rm   nr`   value_remaps     rB   ro   zkAutogradCompilerInstance.proxy_call_aot_backward.<locals>.copy_paste_aot_backward_graph.<locals>.<listcomp>2  s\            &a77{1~t~FFF     rA   get_attrr@   rd   c                     |          S rE   r@   rO  rP  s    rB   r"  ziAutogradCompilerInstance.proxy_call_aot_backward.<locals>.copy_paste_aot_backward_graph.<locals>.<lambda>F      A rA   call_modulec                     |          S rE   r@   rS  s    rB   r"  ziAutogradCompilerInstance.proxy_call_aot_backward.<locals>.copy_paste_aot_backward_graph.<locals>.<lambda>O  rT  rA   zshouldn't get herec                      t                      5  t          j        ddddd          cd d d            S # 1 swxY w Y   d S )Nr   {   r)   r;   zerosr@   rA   rB   dummyzfAutogradCompilerInstance.proxy_call_aot_backward.<locals>.copy_paste_aot_backward_graph.<locals>.dummyZ  s    022 8 8 ;q!Q3778 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8s   488c                 f    g | ]-}t          |t          j        j                  r
             n|.S r@   )rP   r;   r   r6   )rm   or[  s     rB   ro   zkAutogradCompilerInstance.proxy_call_aot_backward.<locals>.copy_paste_aot_backward_graph.<locals>.<listcomp>^  sC       DE:a88?a  rA   )!r;   r   r   rx   rb   r  _get_compiled_autograd_symintsr  symintsry   r   r   rr   rf   rn   rl   rt   rg   r   get_fresh_qualnamer   r   r   create_nodeopsatenviewdefaultreshape	node_copyrW   r   r  )rC  rB  	pall_argsr_  psymintsargs_idxpoutputsrL  rn   phrl   qualnamer   outputsrK  r[  rP  aot_idrR   r,  pbackward_staterE  r`   s                 @@@rB   copy_paste_aot_backward_graphzWAutogradCompilerInstance.proxy_call_aot_backward.<locals>.copy_paste_aot_backward_graph  sV    %(.  S         "z)/22H   #(Ct8S4T4T)T#U#U  I 88::Gw<<3s{#3#33333::::':::H(0InGn%*  111 HK7;H ![[N"6* D"Cd&9&&A"C"CC'''1,''':s :s : : : : : : "- *? *?7m++"8,1B)k$)44BG(*K%MHHW((ty>>Q....          "&1	     HH W
**;D#~@@FFHDN/79d;S;STTT!^77
HbRTUUF"-+di"8"8FK(.K%%W//{ein&9&AAA ',in&<&D!^1;;6666 F #.+di"8"8FK(.K%%W--;D#~@@FFHDN/79d;S;STTT!^1;;6666 F %-FM(.K%%()=>>>'''85< 8 8 8 8   IQ  G (((;;;NrA   subclass_meta
is_runtimeunwrapped_argsc                 F    t           j        j        dt          dt          f fd            }t	          j        j        |          }j                            d|t          |          i           }
                                }                    |g|g           |S )Nrt  r9   c                  2                         |           S )N)rs  )creation_fn)rt  rs  rr  s    rB   make_subclasszkAutogradCompilerInstance.proxy_call_aot_backward.<locals>.proxy_subclass_constructor.<locals>.make_subclassi  s    $00J0WWWrA   rd   r>  )r;   r   r   r   pytreetree_mapr1  r   r   r   allocate_dummyr  )rr  rs  rt  rx  punwrapped_argspoutputrh   r`   s   ``     rB   proxy_subclass_constructorzTAutogradCompilerInstance.proxy_call_aot_backward.<locals>.proxy_subclass_constructorf  s     ])Xs Xs X X X X X X *)X %odm^LLOn11$$?++	 2  G ((**F((&G9===MrA   )make_subclass_override)r^  _forward_clsrX   r<  r;  _aot_idr;   is_grad_enabledoutput_inforequires_gradr   r   r   r   r   r3   r   r   r   r   listru   r7  r8  r9  _backward_epilogue_functionalry  rz  r1  )r`   r(  r)  r*  r+  r,  r-  ri  rN   output_alias_infor=  rq  rn  r~  resultspresultsro  rR   r;  r<  rp  rE  s   `    `          @@@@@@rB   proxy_call_aot_backwardz0AutogradCompilerInstance.proxy_call_aot_backward  si   ( TSSSc.P.P.R.RSSS +%&677	#,"2"J!) "" 	%-%9  !$2 &Z  
 
	%	'5	!+.	 !	 		 	 	 	 	 	 
&	%	 ,, (  
  - 	
 	
 #/"./GHOc	tEL/A c	 c	 c	 c	 c	 c	 c	 c	 c	 c	 c	J 0/11		,0	BJ3-	\	 	 	 	 	 	( "0A__##=	 ` 
 
 ?4='::rA   output_metadatasbackward_idxc           
         | j         J | j         |         }|                     |          }|                     |          }	t          |j        d          r|                     ||	||||          }
n(| j                            dt          ||	g|R i           }
|
J t                      5  g }t          |          D ]W\  }}||
|         |
                    d            %|\  }}}}|
                    t          j        ||||                     X|                     ||
           d d d            n# 1 swxY w Y   t          |          S )Nr  rd   r>  )sizedtypelayoutdevice)r   r1  r   r  r  r   r   r   r)   r   ry   r;   emptyr  r   )r`   r   r  r*  r  r,  r-  r+  r(  r)  r  grad_insr   output_metadatar  r  r  r  s                     rB   proxy_call_backwardz,AutogradCompilerInstance.proxy_call_backward  s    +++---''}553#Y// 	22( GG n11$$"  
  2 	 	G """(** 	< 	<57H(12B(C(C  $_"*gcl.BOOD))).=+tKTvfUUU    ((7;;;	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< Xs   +B D77D;>D;
base_sizesbase_stridesbase_storage_offset
view_sizesview_stridesview_storage_offsetc           	      8   ||                      |          |                      |          |                      |          |                      |          |                      |          |                      |          f}|                     t          |d gdz            S )N   )r1  
proxy_callcopy_slices_prologue)	r`   r   r  r  r  r  r  r  rt   s	            rB   call_copy_slices_prologuez2AutogradCompilerInstance.call_copy_slices_prologue  s     MM*%%MM,''MM-..MM*%%MM,''MM-..
 3TD6A:FFFrA   needs_input_gradr   res
grad_slicec                 d    |                      t          ||||fd gt          |          z            S rE   )r  copy_slices_epiloguer  )r`   r  r   r  r  s        rB   call_copy_slices_epiloguez2AutogradCompilerInstance.call_copy_slices_epilogue  s<      vsJ7FS)***
 
 	
rA   c                 ~    t                      5  t          j        ddg          cd d d            S # 1 swxY w Y   d S )Nr   i[rY  r   s    rB   r{  z'AutogradCompilerInstance.allocate_dummy  s    (** 	/ 	/;9~..	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/s   266fn_namer   r   r   c                 <    t                               ||||          S )zBinds ops.fn_name = fn)rb  r   )r`   r  r   r   r   s        rB   bind_functionz&AutogradCompilerInstance.bind_function  s     www$6EEErA   gradsrt   r  c                 l    t                               |          }|                     ||g|R |          S )z:Proxies a call to ops.fn_name(grads, *args) into the graph)rb  r   r  )r`   r  r  rt   r  rf   s         rB   apply_functionalz)AutogradCompilerInstance.apply_functional  s3     WWWrE>D>>?CCCrA   c           	      8    t          j        |          \  }}t          j         fd|          } j                            d||i            fd|D             }                     |fdt          t          |                    D                        |S )z*Proxies a call to fn(*args) into the graphc                 .                         |           S rE   r0  )r  r`   s    rB   r"  z5AutogradCompilerInstance.proxy_call.<locals>.<lambda>  s    t}}Q/?/? rA   rd   rt   r   c                 8    g | ]}                                 S r@   )r{  )rm   rG  r`   s     rB   ro   z7AutogradCompilerInstance.proxy_call.<locals>.<listcomp>  s%    AAAA$%%''AAArA   c                      g | ]
}|         S r@   r@   )rm   r   	proxy_outs     rB   ro   z7AutogradCompilerInstance.proxy_call.<locals>.<listcomp>  s    -W-W-Wqil-W-W-WrA   )ry  tree_flattenrz  r   r   r  r  r  )	r`   r   rt   r  r5  rG  
proxy_argsr   r  s	   `       @rB   r  z#AutogradCompilerInstance.proxy_call  s     *400	1_%?%?%?%?FF
N//Rj 0 
 
	 BAAAAAA$$V-W-W-W-WE#f++DVDV-W-W-WXXXrA   rG  rn  c                 "   t                               d          }t          j        | j        |g|R           }| j                            d||i           }t          |          t          |          k    sJ |                     ||           |S )zEProxies a call to ops.validate_outputs(outputs, *args) into the graphvalidate_outputsrd   r  )	rb  r   ry  rz  r1  r   r   r  r  )r`   rG  rn  rt   r  rf   r  new_proxy_outputss           rB   r  z)AutogradCompilerInstance.validate_outputs  s     WW'((_T]W4Dt4D4DEE
 N77Rj 8 
 
 ?##s7||3333$$W.?@@@rA   old_varnew_varc                    |                      |          }|                      |          }| j                            dt          j        ||fi           }|                                 }|                     |g|g           |S Nrd   r  )r1  r   r   r;   r   r{  r  )r`   r  r  old_var_proxynew_var_proxyr  r   s          rB   
accumulatez#AutogradCompilerInstance.accumulate  s    g..g..N//UYm]-KTV 0 
 
	 $$&&$$fX	{;;;rA   variabler   has_post_hooksc                     | j                             dt          |                     |          |                     |          |fi            d S r  )r   r   r   r1  )r`   r  r   r  s       rB   r[   z(AutogradCompilerInstance.accumulate_grad  s^     	## h''d##
  	$ 		
 		
 		
 		
 		
rA   hookr   c                 h      j                             dt          |g fd|D             R |          S )Nrd   c                 :    g | ]}                     |          S r@   r0  rm   rK   r`   s     rB   ro   z<AutogradCompilerInstance.proxy_call_hook.<locals>.<listcomp>,  s%    111q$--""111rA   )r   r   r   )r`   r  rt   r   s   `   rB   proxy_call_hookz(AutogradCompilerInstance.proxy_call_hook$  sS     ~**1111D111  
 
 	
rA   hook_iddata_idc                     | j         J | j         |         }| j        |         }|                     ||d          }|                                 }|                     |g|g           |S )Nunpack_hook	hook_type)r   r   r  r{  r  )r`   r  r  r  dataproxyr   s          rB   r  z$AutogradCompilerInstance.unpack_hook1  s    +++(%g.$$# % 
 

 !!##$$cUUG444
rA   r   c                 &   | j         J | j         |         }|                     |||         d          }t                      5  t          ||                   ||<   |                     ||         g|g           d d d            n# 1 swxY w Y   |S )Ntensor_pre_hookr  )r   r  r)   rM   r  )r`   r   r  r   r  r  s         rB   r  z(AutogradCompilerInstance.tensor_pre_hook>  s     +++($$1I' % 
 

 )** 	? 	?#F1I..F1I((&)ug>>>	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? s   7BB
B
c           	      `   | j                             dt          j        j        j        j        ||                     ||                   fi           }t                      5  t          ||                   ||<   | 
                    ||         g|g           d d d            n# 1 swxY w Y   |S Nrd   )r   r   r;   _Cr   r7   call_cpp_tensor_pre_hooksr1  r)   rM   r  )r`   r   r  r   r  s        rB   cpp_tensor_pre_hookz,AutogradCompilerInstance.cpp_tensor_pre_hookM  s     ++H.HdmmF1I../	
 
 )** 	? 	?#F1I..F1I((&)ug>>>	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? s    7B##B'*B'c                     | j         J | j         |         }|                     ||d          }t                      5  d |D             }|                     ||           d d d            n# 1 swxY w Y   |S )Npre_hookr  c                 ,    g | ]}t          |          S r@   rM   rm   rK   s     rB   ro   z5AutogradCompilerInstance.pre_hook.<locals>.<listcomp>d  s    555k!nn555rA   r   r  r)   r  )r`   r   r  r  r  s        rB   r  z!AutogradCompilerInstance.pre_hook[  s    +++(&&  ' 
 

 )** 	: 	:55f555F((999	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: s   #A,,A03A0c                     | j         J | j         |         }|                     |||d          }t                      5  d |D             }|                     ||           d d d            n# 1 swxY w Y   |S )N	post_hookr  c                 ,    g | ]}t          |          S r@   r  r  s     rB   ro   z6AutogradCompilerInstance.post_hook.<locals>.<listcomp>t  s    777!{1~~777rA   r  )r`   rn  r   r  r  r  s         rB   r  z"AutogradCompilerInstance.post_hookh  s     +++(&&!	 ' 
 
 )** 	; 	;77w777G((':::	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; s   #A--A14A1inputc                 4   t          |t          j                  sJ | j        J | j        |         }|                     ||d          }t                      5  t          |          g}|                     ||g           d d d            n# 1 swxY w Y   |S )Npost_acc_grad_hookr  )rP   r;   r   r   r  r)   rM   r  )r`   r  r  r  r  r  s         rB   r  z+AutogradCompilerInstance.post_acc_grad_hookx  s     %.....+++($$* % 
 

 )** 	7 	7u%%&C((ug666	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 
s   (BBBrb   c                    i }d}t          |j                  }|d         j        dk    sJ |d         }t          |j                                                  }t          t                    }||         |d         k    sJ |t          |          z   dz
  }||         |d         k    sJ t          |          D ]\  }	}
|s|
j        d         j	        j
        dk    rd}%|
j        d         j	        j
        d	k    }t          |
j        d                                                   dk    }|rF|rDt          |
j                                                  }t          d
 |D                       r|
||	<   |rz|                                D ]D}
t                              d|
           |
j        d                                         |
j        d<   Et          |                                          S g S )NFr   r   rj   r   cudaTcpuc              3      K   | ]]}t          |j        t          j        j                  o|j        j        d v p&t          |j        t                    o|j        j         V  ^dS ))primsrc  N)rP   rg   r;   _ops
OpOverload	namespacer   r   rm   users     rB   r   zDAutogradCompilerInstance.move_graph_nodes_to_cuda.<locals>.<genexpr>  s       
 
  #4;
0EFF G K15FF
 #4;33 ? $ >>
 
 
 
 
 
rA   zMoving node %s from cpu to cuda)r  rr   rg   userskeysr  r   r   metar  r  r  allvaluesverbose_logdebugr  )r`   rb   to_movehas_cuda_inputsrr   r   inputs_usersfirst_getitem_idxlast_getitem_idxr   rn   is_cpu	is_scalar
node_userss                 rB   move_graph_nodes_to_cudaz1AutogradCompilerInstance.move_graph_nodes_to_cuda  s   ,.U[!!Qx(****qFL--//00 344&'<?::::,s</@/@@1D%&,r*::::: .. 	& 	&GAt" ty'7'>'Cv'M'M"&Yu%,1U:FDIe,1133449I &) &!$*//"3"344
 
 
 !+
 
 
 
 
 & "&GAJ  	((( ; ;!!"CTJJJ#'9U#3#8#8#:#:	%   '''	rA   rn   c                     t          |t          j        j                  oG|j        dk    o<|j        t          j        j        j        j	        t          j        j        j
        j        fv S r  )rP   r;   r   rN  rf   rg   rb  rc  sym_sizerx   	sym_numelre  )r`   rn   s     rB   is_sym_nodez$AutogradCompilerInstance.is_sym_node  sU    tUX]++ O?*O	'+UY^-E-MNO	
rA   c                 J   t                      t          | j        j                            d                    D ]1\  }}                    |j                                                   2|t          t                    dz
  k    sJ dt          j        j        dt          ffd}t          | j        j        j                  }| j        j                            |           t          | j        j        j                  }t                               d||z
             d S )Nr   ri   rj   rn   r9   c                 j    | v s| j         dk    r| j        t          v rdS |                                 S )Nrd   T)rf   rg   _impure_targets	is_impure)rn   unpack_nodess    rB   r  z/AutogradCompilerInstance.dce.<locals>.is_impure  s<    |##?**t{o/M/Mt>>###rA   zDCE removed %d nodes)r4   r   r   rb   rs   updater  r  r  r   r;   r   rN  ru   rr   eliminate_dead_coder  r  )r`   r   rn   r  beforeafterr  s         @rB   dcezAutogradCompilerInstance.dce  s   
 3=,, !5!@!@M!@!R!RSS 	3 	3GAt
 1 12222C+,,q00000	$EHM 	$d 	$ 	$ 	$ 	$ 	$ 	$ T^)/0000;;;DN(.//0&5.AAAAArA   c                     g }g }t          | j        j        j                  }t	          |           t	          |          }|j        dk    sJ |j                                        D ]K}|j        t          j
        k    sJ |j        r|                    |           6|                    |           Lt                      }|D ]}t          |j        t                    sJ |j        d         |k    sJ t          |j        d         t                     sJ t#          |          }|                    |j        d                    |j        d         |f|_        |D ]!}	| j        j                            |	           "|S )Nr   r   rj   )rq   r   rb   rr   rp   rl   r  r  rg   rv   rw   ry   setrP   rt   r   rx   r  r   
erase_node)
r`   
used_sizesunused_sizesit
sizes_nodegetitem_nodeused_sizes_idxusednext_size_idxunuseds
             rB   remove_unused_sizesz,AutogradCompilerInstance.remove_unused_sizes  s   
 $.&,--R"XX
'))))&,1133 	2 	2L&(*:::::! 2!!,//// ##L1111#&55 	6 	6Ddi/////9Q<:----dilC00000//Mty|,,,1}5DII" 	4 	4FN ++F3333rA   r   c                 L    t          | j        j        | j        j        |          S rE   )r%   r   r   rb   )r`   r   s     rB   create_graph_modulez,AutogradCompilerInstance.create_graph_module  s    4>.0DbIIIrA   c                 l     j                             dt          j        di             j                                          j                             dd j                                                  |                    fi            g t                      r 
                     j         j                   j         j        j        D ]}dD ]}||j        v r|j        |= t          dd  fd                                                                                                                                                                                                                                                                                            j        r$ j                             j         j                                                                         d	 j                   t9          d
g           t;          dddd          }t<                              d|           t@          !                    d|           t          dfd           dtD          dtF          f         d
tF          dtF          dtF          dtF          dtF          dtH          tF          tF          f         f fd}tK                      &                    dtO          j(                    d j        i j)        d            j*        +                    d d d            | ,                              fS )Nrd   r@   rh   )tensor_metaexample_valuer   r  c                      dddS )N&compiled_autograd_graph_pre_reorderingstringr   r@   r@   rA   rB   r"  z6AutogradCompilerInstance.end_capture.<locals>.<lambda>  s    @$! ! rA   c                      t           j        j         j        j        d j         d                              d          S )NCompiledAutogradPreReorderingFprint_output)r%   r   r   rb   r   print_readabler   s   rB   r"  z6AutogradCompilerInstance.end_capture.<locals>.<lambda>!  sE    {#$947999    n%n00	 rA   r$  r"  r   zCompiled autograd graphT)include_deviceinclude_stridecoloredz%scompiled_autograd_graphc                  0                          d          S )NFr$  )r&  )rb   s   rB   r"  z6AutogradCompilerInstance.end_capture.<locals>.<lambda>L  s    u333GG rA   )r&  compiled_fn.r   r   r   packed_inputsr9   c           	         	 da j        rj                            |           g }t          |          D ]s\  }}|v rj|dk    rO|                    t          j        d|                     t
          j                            |d         d           ^|                    |           tD ]3}	||	         	                                
                    d          ||	<   4t                      5  t          j                  5   | |||||          }
j        rj                            |
           |
cd d d            cd d d            da S # 1 swxY w Y   d d d            n# 1 swxY w Y   da d S # da w xY w)NTr   r  rj   )non_blockingF)in_compiled_autograd_regionr   r   r   ry   r;   r  r   maybe_mark_dynamic
pin_memoryr  _disabler   r   r   )r,  r   r   r   r   r-  filtered_sizesr   integerr   r   runtime_inputs_to_mover`   r  s              rB   runtime_wrapperz=AutogradCompilerInstance.end_capture.<locals>.runtime_wrapperO  sL   4.2+# >$55f===!#$-e$4$4 ; ;LCn,,"Q;;*11%+a2I2IJJJ!M<<^B=OQRSSSS*11':::/ O OA &q	 4 4 6 6 ; ; ; N NF1IIZZ  !5dg!>!>  %+ C ' 4(..s333              /4++                       /4+++e+3333sZ   C,E7 /E(2E6E(E7 E	E(E	E(E7 (E,,E7 /E,0E7 7E;r7   r   r   )-r   r   r   _exec_final_callbacks_stubr   r   ra  
create_argr1  rJ   r  rb   rr   r  r    delay_unpack_hook_nodesreorder_tensor_pre_hook_nodes'reorder_pre_hook_nodes_to_schedule_asapreorder_accumulate_grad_nodes%reorder_pre_hook_nodes_to_mimic_eager reorder_post_acc_grad_hook_nodesreorder_post_hook_nodesr  r   r~   r  r  r   r   r   compiled_autograd_loginfor  r  r   r   r   r   log_event_endr   r   r   r   __exit__r   )	r`   rn  rn   fieldlazy_graph_coder7  rb   r6  r  s	   `     @@@rB   end_capturez$AutogradCompilerInstance.end_capture  s   ##&A		
 	
 	
 	
""^&&t}}W'='=>>@		
 	
 	
 -/%'' 	Y%)%B%B4>CW%X%X"
 N(. 	) 	)D@ ) )DI%%	%() 	 1 1 1 1	
 	
 	
 	
 	$$&&&**,,,44666**,,,22444--///$$&&& 	


 	C,,T^-ABBB 1133(()EDG)E)EFFEH:...0%
 
 
 	""4999$000%GGGG	
 	
 	
 	

$	4!#s(+$	4$	4 $	4 	$	4
 $	4 $	4 38_$	4 $	4 $	4 $	4 $	4 $	4 $	4 $	4L 	"##11LNN!"& 	2 	
 	
 	
 	%%dD$777 0 0 7 777rA   c                     d | D             }|S )Nc                 R    g | ]$}t          |          t          j        j        u "|%S r@   )r  r;   r   rN  )rm   rO  s     rB   ro   z:AutogradCompilerInstance.get_all_nodes.<locals>.<listcomp>  s,    ===qDGGux}$<$<$<$<$<rA   r@   )rt   rr   s     rB   get_all_nodesz&AutogradCompilerInstance.get_all_nodes  s     >=D===rA   c                     | j         dk    s6| j         dk    r-| j        t          j        k    r| j        d         j         dk    rdS dS )Nr   rd   r   TF)rf   rg   rv   rw   rt   )rn   s    rB   is_placeholderz'AutogradCompilerInstance.is_placeholder  sI    7m##G&&x///	!=004urA   c                    | j         j                            dt                    D ]}|j        d         |j        d         }}d}|j        t          j        k    r|}|j        d         }t          ||g          }||j	        urA| 
                    |          s,|                    |           ||                    |           dS )a  
        Usage of AOTAutograd causes all the accumulate_grad_ nodes to get pushed to the end of
        the graph.  This differs from eager mode, which schedules them as soon as possible. This
        pass attempts to reorder the graph to mimic eager behavior.
        rd   re   r   rj   N)r   rb   rs   r   rt   rg   rv   rw   maxprevrL  ry   )r`   rn   r}   	grad_noder  args         rB   r=  z6AutogradCompilerInstance.reorder_accumulate_grad_nodes  s     N(33'; 4 
 
 	- 	-D %)IaL$)A,	JL8#333((-a0	z9-..C$)##D,?,?,D,D#

4   +JJ|,,,	- 	-rA   c                     | j         j                            dt                    D ]K}|j                            dd          dk    r"t          |j                  }|                    |           LdS )zp
        We can delay unpack hooks until they are needed, even later than in the eager autograd engine.
        rd   re   r  Nr  )	r   rb   rs   r   r   r   minr  prepend)r`   rn   
first_users      rB   r:  z0AutogradCompilerInstance.delay_unpack_hook_nodes  s     N(33y 4 
 
 	% 	%D {{D11]BBTZJt$$$$	% 	%rA   c                 \   | j         j                            dt                    D ]}|j                            dd          dk    r"|j        d         }|j        d         }||j        ur?|                     |          s*|	                    |           |	                    |           dS )a  
        Usage of AOTAutograd causes all the tensor_pre_hook nodes to get pushed
        to the end of the graph. This differs from eager mode, which schedules
        them as soon as possible. This pass attempts to reorder the graph to
        mimic eager behavior.
        rd   re   r  Nr  r   rj   )
r   rb   rs   r   r   r   rt   rO  rL  ry   )r`   rn   r  
input_nodes       rB   r;  z6AutogradCompilerInstance.reorder_tensor_pre_hook_nodes  s     N(33y 4 
 
 	* 	*D {{D115FFF9Q<L1J**43F3Fz3R3R*!!,///##D)))	* 	*rA   c                    | j         j                            dt                    D ]b}|j                            dd          dk    r#|j        d         }|                     |j        d                   }g }g }|g}|D ]l}|j        dk    r_|j	        t          j        k    rJ|                    |j        d                    |                    |           |                    |           mt          ||          D ]/\  }}	|                    |           |                    |	           0t          |          }
|
|j        urD|                     |
          s/|
                    |           |D ]}|                    |           ddS )a  
        In this function, we schedule the pre hooks as soon as possible. This
        does not match eager behavior (schedule pre hook right before its
        registered node), but it can make acc grad be scheduled properly when
        the pre hooks are registered to them. After reordering acc grad node, we
        will reorder the pre hooks again to mimic eager behavior.
        rd   re   r  Nr  r   rj   )r   rb   rs   r   r   r   rt   rJ  rf   rg   rv   rw   ry   zipremoverN  rO  rL  )r`   rn   r  input_nodes	to_remove	to_append
hook_blockrO  abrQ  s              rB   r<  z@AutogradCompilerInstance.reorder_pre_hook_nodes_to_schedule_asap  s    N(33y 4 
 
 	+ 	+D {{D11Z??9Q<L,,TYq\::KIIJ  ) )4?**qx8;K/K/K$$QVAY///$$Q'''%%a(((Iy11 & &1""1%%%""1%%%%k""C$)##D,?,?,D,D#

<(((# + +A ''****5	+ 	+rA   c                    g }| j         j                            dt                    D ]7}|j                            dd          dk    r"|                    |           8t          |          D ]}|j        d         }t          |j
                                                  }t          |          dk    rIt          d |D                       sJ t          t          |d         j
                                                            }||j        urD|                    |           |                    |           |D ]}|                    |           dS )a%  
        Usage of AOTAutograd causes all the pre_hook nodes to get pushed to the
        end of the graph. This differs from eager mode, which schedules them
        right before their registered node execution. This pass attempts to
        reorder the graph to mimic eager behavior.
        rd   re   r  Nr  r   c              3   V   K   | ]$}|j         d k    o|j        t          j        k    V  %dS )rd   N)rf   rg   rv   rw   r  s     rB   r   zQAutogradCompilerInstance.reorder_pre_hook_nodes_to_mimic_eager.<locals>.<genexpr>   sL         ?*Nt{h>N/N     rA   )r   rb   rs   r   r   r   ry   reversedrt   r  r  r  r  r  rp   rq   rT  )r`   	pre_hooksrn   hook_getitem_noder  registered_noderw   s          rB   r>  z>AutogradCompilerInstance.reorder_pre_hook_nodes_to_mimic_eager  s    	N(33y 4 
 
 	# 	#D {{D11Z??T""""Y'' 	5 	5D $	!**++E5zzQ   !        #4a(;(;(=(=#>#>??Odi//''(9:::''---$ 5 5G#++G4444%	5 	5rA   c                    g }| j         j                            dt                    D ]7}|j                            dd          dk    r"|                    |           8t          |          D ]}|j        d         }|j        d         }d}t          |j
                                                  D ]!}|j        dk    r|j        t          k    r|} n"|
J d            |                    |           |                    |           dS )	a  
        Usage of AOTAutograd causes all the post_acc_grad_hook nodes to get
        pushed to the end of the graph. This differs from eager mode, which
        schedules them as soon as possible. This pass attempts to reorder the
        graph to mimic eager behavior.
        rd   re   r  Nr  r   rj   z8post_acc_grad_hook must have corresponding acc grad node)r   rb   rs   r   r   r   ry   rc  rt   r  r  r  rf   rg   r   )r`   post_acc_grad_hooksrn   r  r}   acc_grad_noderO  s          rB   r?  z9AutogradCompilerInstance.reorder_post_acc_grad_hook_nodes  s@    !N(33y 4 
 
 	- 	-D {{D115III&&t,,,, 011 	& 	&D9Q<L1J !M**//1122  4?**qx;O/O/O$%ME ,,J -,,
   ...%%%%#	& 	&rA   c           	      j   g }| j         j                            dt                    D ]7j                            dd          dk    r"|                               8t          |          D ]j        d         }j        d         }j        d         }t          |          dk    r>g }|
                    t          |                     |D ]G}|
                    fd	t          |j                                                  D                        Ht          |          }|j        dk    r|j        t"          k    r|j        d         }d}	t          |j                                                  D ]>}
|
j        dk    r1|
j        t          k    r!|
j                            dd          d
k    r|
}	?|	,|	                    |           |                               x|j        ur?|                     |          s*|                    |           |                               dS )a  
        Usage of AOTAutograd causes all the post_hook nodes to get pushed to the
        end of the graph. This differs from eager mode, which schedules them as
        soon as possible. This pass attempts to reorder the graph to mimic eager
        behavior.
        rd   re   r  Nr  r   rj      c              3      K   | ]@}|j         d k    r/|j        t          k    rj                            dd          dk    <|V  AdS )rd   r  Nr  )rf   rg   r   r   r   )rm   r  rn   s     rB   r   zCAutogradCompilerInstance.reorder_post_hook_nodes.<locals>.<genexpr>J  sg       - -?22 K944 KOOK>>+MM 
 NMMM- -rA   r  )r   rb   rs   r   r   r   ry   rc  rt   r  extendr  r  r  rN  rf   rg   r   rO  rL  )r`   
post_hooksr  r|   r[  input_nodes_and_usersrW  rQ  r}   post_acc_grad_hook_noderO  rn   s              @rB   r@  z0AutogradCompilerInstance.reorder_post_hook_nodes0  si    
N(33y 4 
 
 	$ 	$D {{D11[@@d####Z(( (	* (	*D9Q<L9Q<L)A,K<  1$$$&!!((k):):;;;) 	 	
%,, - - - - $Z%5%:%:%<%< = =- - -     +,,Cv((SZ;O-O-O Xa[
*.'j.335566 4 4A//H	11HLLd;;?SSS23/*6+22<@@@ ''---$)##D,?,?,D,D#

<(((##D)))Q(	* (	*rA   tc                     |d S t          |t                    r fd|D             S t          |t                    rt           fd|D                       S t          |t          j        t          j        f          r j        |j                 S t          |t          j                  s|S t           j
        |          }t          |t          j        j        j        j                  sJ |j        S )Nc                 :    g | ]}                     |          S r@   r0  r  s     rB   ro   z5AutogradCompilerInstance.to_proxy.<locals>.<listcomp>m  s%    000DMM!$$000rA   c              3   B   K   | ]}                     |          V  d S rE   r0  r  s     rB   r   z4AutogradCompilerInstance.to_proxy.<locals>.<genexpr>o  s/      55aq))555555rA   )rP   r  r   r;   SymIntSymFloatr   rn   r   r*   r   r   r  proxy_tensor_ProxyTensorr  )r`   rq  rw  s   `  rB   r1  z!AutogradCompilerInstance.to_proxyi  s    94a 	10000a0000a 	655551555555a%,788 	5,QV44!U\** 	H)$.!<<,(=(J(WXXXXX!!rA   objectsr  c                    t          t          j        j                  r|rt	          |          t	          |          k    sJ g }t          t	          |                    D ]?}||         \  }}|                     ||d            |                    |                    @|n(fdt          t	          |                    D             t	          |          t	                    k    sJ t          |d | j	                   S )Nc                      g | ]
}|         S r@   r@   )rm   r   r  s     rB   ro   zDAutogradCompilerInstance.bind_objects_to_proxies.<locals>.<listcomp>  s    CCC!71:CCCrA   constanttracer)
rP   r;   r   r6   r  r  set_node_originry   r-   r   )r`   ry  r  r   bound_proxiesr   nodecall_indexrH  s     `     rB   r  z0AutogradCompilerInstance.bind_objects_to_proxiesy  s    gux~.. 
	D 	D7||s7||3333 "s7||,, 5 5A07
-NI((NDIII!((4444'CCCCuS\\/B/BCCC7||s7||++++'7T$.QQQQrA   indexc                 ~    | j         J | j         |         }t                      }t          ||d | j                   |S )Nr|  )r   r&   r-   r   )r`   r  r  bw_states       rB   bind_backward_statez,AutogradCompilerInstance.bind_backward_state  sE    +++ ' ??(EDPPPPrA   rH  r  pyobjc                 *   d}|4|j         }t          |d          r|j        t          d          |j        }| | d| d}t          j                                                    d         }|                    d|          }t          |           d S )N r  zThis compiled backward function was saved by AOTAutogradCache, which does not support
                    compiled autograd. Please turn off AOTAutogradCache using `TORCHINDUCTOR_AUTOGRAD_CACHE=0`.z (NodeCall )r  z:raw_stack_trace = CapturedTraceback.extract().format()[-1])
r  r   rQ   r   r  r5   extractformatreplacer1   )	r`   rH  r  r  maybe_aot_idforward_clsnew_coderaw_stack_tracenew_stack_traces	            rB   r  z(AutogradCompilerInstance.set_node_origin  s     ,K{I.. 32:&s    +2KKK.KKK+355<<>>rB)11H(
 
 	(((((rA   r   rE   )Kr   r   r   r   r   ra   r;   r   r	   r   r$   r   staticmethodr   r   r   r  rx   r   r  r   ru   r3   r2   r  r'  r   r  autogradfunctionBackwardCFunctionr  r  r  r{  r  r  r  r  r  r[   r   r6   r  r  r  r  r  r  r  r   r  r  r  r  r  r%   r  rG  rN  rJ  rL  r=  r:  r;  r<  r>  r?  r@  r1  r  r&   r  Functionr  r@   rA   rB   r   r     s	       1HS#X$6 14 1 1 1 1C5< C&1A Cj C C C C 5S 5s 5} 5 5 5 \5w
U\"w
 Cyw
 eCJ'(	w

 d5c?+,w
 w
 w
 
sD&[(94;NN	Ow
 w
 w
 w
r
c
 

 
 
 
F#F !.F  -	F
 F F #+3-F 
#F F F FP11 #3-1  -	1
 1 ^$61 #+3-1 
x%s*	+1 1 1 1fGG SMG sm	G
 !G SMG smG !G 
%,	G G G G*
"4.
 
 c]	

 L
 
%,	
 
 
 
/ / / / /
FF S#XF !	F
 F 
F F F F	D	D }	D 		D
 "#	D 
%,		D 	D 	D 	D38$,/BJ3-	%,	   '}47JRSV-	%,	   #      

,1L
JN
	
 
 
 

S#X&
/2
>A
	
 
 
 
3      5<(36;>	el	   5<(36;>	el	   x} s tEL?Q    EL)3;EL3ITW	el	    \,/	el	   */ehn /c / / / /b
 
 
 
 
 
B B B B, SX        DJc Jk J J J J|83 |85#s(1CS1H+I |8 |8 |8 |8| HSM d58=.A    \
 UX] t    \- - - -*% % % %* * * *("+ "+ "+ "+H!5 !5 !5 !5F"& "& "& "&H7* 7* 7* 7*r"# "# " " " "( 48	 #  $uS#X/0	
 
#   ,     )) ) /0	)
 
) ) ) ) ) )rA   r   FTr   dynamicignore_active_disable_ctx)NNNc              #     K   |st           rd V  d S |rt          |          t          u sJ ddlm} |j        j        dk    rda	 d V  dad S # daw xY wt          j	        
                                rddlm} t          j        j        j                            t#          j        t&          |           |          \  }}t)                      r.t          j        j        j                            t,                     dat0          }t0          dz  a	 t          j                            d          5  d V  d d d            n# 1 swxY w Y   |sdat          j        j        j                            ||           t0          dz  at0          |k    s
J d            d S # |sdat          j        j        j                            ||           t0          dz  at0          |k    s
J d            w xY w)	Nr   )
eval_frameforce_eagerTF)cudagraph_treesrj   zINested Compiled Autograd Contexts must return before their parent context)active_disable_ctxr  ru   torch._dynamor  _stancestance%compiled_autograd_enabled_force_eagerr;   r  is_availabletorch._inductorr  r  r   r7   set_autograd_compiler	functoolspartialr   rC   set_verbose_loggerr  compiled_autograd_enableddepthr  set_multithreading_enabled)r   r  r  r  r  prior_compilerprior_dynamicprior_depths           rB   _enabler    s     4 % 0); 0 	)==D((((,,,,,,$55 591>8=5555==== z&&(( <;;;;;;
  2HH!":KHH'  011 S 2EEkRRR(,%KQJE^>>uEE  EEE               & 605- 2HH"M   
+++_ ,++++ & 605- 2HH"M   
+++_ ,++++++s=   A AF ,D=1F =EF EF AG&c               #   N  K   t           j        j        j                            d d          \  } }dat          sda	 d V  | rdadat           j        j        j                            | |           d S # | rdadat           j        j        j                            | |           w xY w)NFT)r;   r  r   r7   r  r  r  )r  r  s     rB   r3  r3  
  s      
 	*@@uMM !& "!
 	-(,%"*@@M	
 	
 	
 	
 	
  	-(,%"*@@M	
 	
 	
 	
s   A2 22B$c                  8   da t          rJ t          j        j        j                            d d           t          j        j        j                            d            t          j        j        j                                         t          j
                    ad S )NF)r  r0  r;   r  r   r7   r  r  clear_cache	itertoolsr   r   r@   rA   rB   resetr  !  sy     %****	H&<<T5III	H&99$???	H&22444o''OOOrA   r   r  r  r  r  r  r  c                     | d         }|                     ||          }|J |                    |           ||z
  }	|                    |||	          }
||
|
                    t          j                  gS )Nr   )memory_format)new_empty_stridedcopy_
as_stridedcloner;   contiguous_format)r   r  r  r  r  r  r  r   r   offsetr  s              rB   r  r  .  s     !9D##J==F
LL #66F"":|VDDJJ
 0 0u?V 0 W WXXrA   r  r   r  r  c                     d gt          |           z  }t          t          |                     D ]K}| |         rA||         |dk    r'||         }|J |                    |           |||<   @||         ||<   L|S )Nr   )r  r  r  )r  r   r  r  grad_inputsr   to_copys          rB   r  r  B  s     26=M9N9N0NK3'(()) 
( 
(A 		(1v~Avva&***  )))!'A!$QArA   )TTr   )u__doc__r   r  r  rv   r   collectionsr   r   collections.abcr   r   typingr   r   r	   r
   r   r;   torch.utils._pytreeutils_pytreery  torch._dispatch.pythonr   torch._dynamo.external_utilsr   r   r   r   r   torch._dynamo.sourcer   r   torch._dynamo.utilsr   r   r   r   /torch._functorch._aot_autograd.runtime_wrappersr   r   torch._guardsr   r   r   r   torch._loggingr   r    torch._prims_commonr"   torch._subclassesr#   torch._subclasses.fake_tensorr$   torch.fxr%   %torch.fx.experimental._backward_stater&   "torch.fx.experimental.proxy_tensorr'   r(   r)   r*   r+   r,   r-   %torch.fx.experimental.symbolic_shapesr.   r/   torch.fx.tracebackr0   r1   torch.typesr2   r3   torch.utils._ordered_setr4   torch.utils._tracebackr5   torch.fx.proxyr6   r  r   rA  r  ru   rC   rJ   r   rM   rX   rZ   r   r   rb  r   r8  r  r   r   rx   r   r   r  r  r0  r  r  contextmanagerr  r3  r  r  r  r  r@   rA   rB   <module>r     s                   , , , , , , , , / / / / / / / / @ @ @ @ @ @ @ @ @ @ @ @ @ @  $ $ $ $ $ $ $ $ $ ; ; ; ; ; ;              < ; ; ; ; ; ; ;                   M L L L L L L L L L L L > > > > > > > > 6 6 6 6 6 6 , , , , , , 4 4 4 4 4 4             ? ? ? ? ? ?                  G F F F F F F F B B B B B B B B 2 2 2 2 2 2 2 2 / / / / / / 4 4 4 4 4 4  %$$$$$$X *)(4GHH *EFF$    4D 4 4 4 48EL) hu|.D    
 
c0B 
 
 
 
6G G G G G G G G^ #  #  #  #  #  #  #  #F5 5 5 5 5 5 5 5  kmm MLL *"=	  ")/##	s 	s 	 	 	 	O) O) O) O) O) O) O) O)f$ "  ). % $  	  &*I I#s(#II  $I  	I I I IX 
),- 
 
 
 
,( ( ( (YU\"Y%Y ;'Y %	Y
 %Y ;'Y %Y 
%,Y Y Y Y(tnL 
(5<(	) 	
 
(5<
 !     rA   