%Vji&7ddlmZddlZddlmZddlmZmZmZm Z m Z ddl Z ddl m Z mZddlmZmZddlmZdd lmZe rdd lmZmZe jed Ze jed Zeee e e j!fZ"eeege"fZ#ej$d GddZ%ej$d GddZ&ej$d GddZ'dZdZ(d[dZ)d\dZ*d]d Z+d^d#Z,d_d(Z-d`d.Z.dad0Z/dbd3Z0dbd4Z1dcd7Z2ej$Gd8d9Z3dddBZ4dedDZ5GdEdFeZ6dfdKZ7dgdQZ8ej$d GdRdSZ9ej$d GdTdUZ:dhdYZ;dS)i) annotationsN)Enum)AnyCallableOptional TYPE_CHECKINGUnion)countersget_metrics_context)GraphPartitionMap InputType) OrderedSet)is_using_cudagraph_partition)SequenceSet perf_hintscudagraph_static_inputsT)frozenceZdZUdZded<dS) FunctionIDz9Unique counter of a function wrapped in cudagraphify_implintidN__name__ __module__ __qualname____doc____annotations__Y/root/voice-cloning/.venv/lib/python3.11/site-packages/torch/_inductor/cudagraph_utils.pyrrs?? GGGGGr!rc<eZdZUdZded<ded<ded<ded<d S) PlaceholderInfoz A serializable version of torch.fx.Node that contains information pertinent to placeholder stack traces. We use these in logging and error messages related to cudagraphs, and will cache these results. strname Optional[str] stack_tracelist[PlaceholderInfo]usersmutating_use_stack_traceNrr r!r"r$r$%sK III    ++++++r!r$cPeZdZUdZded<ded<ded<ded <d ed <ded <d S)WrappedFunctionz Represents a function that you want to record for CUDA graph replay, with a little more metadata so we can identify if we have an applicable CUDA graph in our CUDA graph tree for it. zCallable[..., Any]model Sequence[int]static_input_idxsrrztuple[torch.Tensor, ...] constantsSequence[PlaceholderInfo] placeholdersmutated_input_idxsNrr r!r"r-r-4sc $$$$NNN''''++++%%%%%%r!r-placeholder_node torch.fx.Nodereturnr'cHt|jdkr:tt|jjddS|jD]G}|jtjj j j kr!|jddx}r|cSHdS)Nrr() lenr*nextitermetagettargettorchopsatencopy_default)r5user(s r"&get_mutating_use_stack_trace_from_noderEDs  !""a''D)/00116::=$OOO%## :-5 5 5!hll=$???{ #"""" 4r!placeholder_infoc|jSN)r+)rFs r"get_mutating_use_stack_tracerISs  44r!c|j}|jdd}g}d}|jdkr d|jD}t |}t ||||S)Nr( placeholderc,g|]}t|Sr )to_placeholder_info).0is r" z'to_placeholder_info..^s!HHHA$Q''HHHr!)r&r<r=opr*rEr$)r5r&r(r*r+s r"rMrMWs|  D"'++M4@@K E#m++HH1A1GHHH#I $ $   4e5M N NNr!graphtorch.fx.Graphr)c$d|jDS)NcBg|]}|jdk t|S)rK)rQrM)rNnodes r"rPz(get_placeholder_info..gs4   &*TW =U=UD!!=U=U=Ur!)nodes)rRs r"get_placeholder_inforXfs%  .3k   r!reasonr%c d|S)Nzskipping cudagraphs due to r )rYs r"format_default_skip_messager[ls 1 1 11r!r3r2mutation_indices&Union[AbstractSet[int], Sequence[int]]cd}|D]}||}t|x}rntdt|d}|r|d|S|S)Nzmutated inputs (z instances). Found from : )rIr[r9)r3r\r(idxrKmsgs r"get_mutation_stack_tracercps"$K"3' 6{CC C;  E  &=3/00===  C766666 Jr!funcinputslist[InputType]is_cuda_graph_recorded_tensorCallable[[torch.Tensor], bool]c tjjjjrfdjD}nj}t djt d||rtj |ndS)NcHg|]}|jv ||Sr )r0)rNrardrergs r"rPz&check_for_mutation..sJ+ + + t---00==. .--r!z'check mutation static input indices: %sz#check mutation mutation indices: %s) r? _inductorconfigtritoncudagraph_treesr4static_inputs_logdebugr0rcr3)rdrergr\s``` r"check_for_mutationrqs  $4 3+ + + + + + .+ + +  2143IACSTTT   !24DEEE r!rVc\|jD]#}|jddx}r|cS$dS)Nr()r*r<r=)rVrDr(s r"_get_use_stack_tracerssFz(,,}d;; ;;      4r!device_node_mapping!dict[torch.device, torch.fx.Node]c|tjddtr(|tjdd|tjdx}r?d|jd}t |x}rt|d|St|St|dkr9tt| j dkrdSd| D}td d |S) Nr<cpuz cpu device ()r`rcudac34K|]}t|VdSrH)repr)rNkeys r" z:check_multiple_devices_or_any_cpu_nodes..s(AAscAAAAAAr!zmultiple devices: z, )popr?devicerr=r&rsr[r9r:r;keystypejoin)rtcpu_noderbr( keys_reprs r"'check_multiple_devices_or_any_cpu_nodesrsTEL00$777$%%; U 3 3T:::&**5<+>+>???x0-X]---.x88 8; X.#/V/V/V/VWW W*3///   A%% )..0011 2 2 76 A AtAA&9&>&>&@&@AAAI &'RDIIi.s#XXXCKAWAWCAWAWAWr!r) r[r?rkrlrmrnrr9rXrRrc) rrr4r0 default_msgr\ has_mutationr3rs @r"3check_for_mutation_ignore_cuda_graph_managed_tensorrs ..>??K $4 9 !233 XXXX+=XXX+,,1  4+BH55 ' 6FGGG>**a/ '8tt[8r!rKcV|jr|jS|jD]}|jr |jcSdS)zM Gets the first non-empty stack trace of a placeholder or its users. N)r(r*)rKusers r"get_placeholder_stack_tracersP'&&!$$   $# # # # $ 4r!c&eZdZdZdZdZdZd dZdS) CheckInvariantStatusrr7r%cr|jdkrdS|jdkrdS|jdkrdS|jd|jS)NCudagraphManagedIdxMismatchz-cudagraph managed tensor data pointer changedStaticInputIdxMismatchz!static input data pointer changed&ExpectedDeadIndicesBeforeGraphMismatchz+expected dead indices before graph are livez: )r&r)rs r"__str__zCheckInvariantStatus.__str__sV 95 5 5BB Y2 2 266 YB B B@@i//4:// /r!Nr7r%)rrrSUCCESSrrrrr r!r"rrsEG#$./*000000r!rrecorded_data_ptrSequence[Optional[int]] target_idxsmismatchc ttkr tt|ks Jdfd|D}fd|D}|d}tt||D]\}\} } t| tjsJ||} | | kr=|| } |d| jd| d| dt| d }|S) z} Logs the mismatch between input data pointers and recorded data pointers. This checks only idxs in target_idxs. zClength mismatch between inputs, recorded_data_ptr, and placeholdersc g|] }| Sr r )rNrOres r"rPz)log_data_ptr_mismatch...s000q000r!c g|] }| Sr r )rNrOrs r"rPz)log_data_ptr_mismatch../s===A$Q'===r!z. z input name: z. data pointer changed from z to z. input stack trace:  ) r9 enumerateziprr?Tensordata_ptrr&r) r3rerrr t_tensors t_data_ptrs error_msgrOtensorrindexrKs `` r"log_data_ptr_mismatchrsj v;;#/00 0 0S[[C DUDU5U5U5UM6V5U U1000K000I=======K   I!*3y++F+F!G!G   FH&%,/////A ??   ( (&u-KSS+*:SS-5SS;A??;L;LSS&A+&N&NSSS  r!fn_cache)dict[tuple[int, ...], Callable[..., Any]] new_int_keyrboolct|dzdfd }tjjjjrCtjjjjkr$t|dSdS)Nrr7r%cddS)NzCUDAGraph supports dynamic shapes by recording a new graph for each distinct input size. Recording too many CUDAGraphs may lead to extra overhead. We have observed a0 distinct sizes. Please consider the following options for better performance: a) padding inputs to a few fixed number of shapes; or b) set torch._inductor.config.triton.cudagraph_skip_dynamic_graphs=True. Set torch._inductor.config.triton.cudagraph_dynamic_shape_warn_limit=None to silence this warning.r )num_cudagraphssr"warn_msgz4maybe_warning_due_to_dynamic_shape..warn_msgDs '0> ' ' ' r!TFr) r9rr?rkrlrm"cudagraph_dynamic_shape_warn_limitrr)rrrrs @r""maybe_warning_due_to_dynamic_shaper>s))A-N        %H  / ' J K K hhjj)))t 5r!c2eZdZUdZded<ded<ded<dS) CudagraphCachedInfoz' Info needed to realign inputs r2r3list[Optional[str]] stack_tracesz list[str]cudagraph_fail_reasonsNrr r!r"rr[sB,+++%%%%%%%%%%r!rcFeZdZUdZded<ded<ded<ded<d ed <d S) CudagraphMetadataz. Metadata for recording a CUDA graph. r2r3rr0r4rrzdict[str, torch.Tensor]r1Nrr r!r"rrfsZ,+++&&&&''''%%%%&&&&&&r!r partition_mapr metadatacFg}t}t}t|jD]\}}|jvr|||jvr|||j|}ntd|jd|dgd}| |g}|j D]:} | !| j | %| d;fd|j D} t||||| S)z Convert the cudagraph metadata at the graph level to the graph partition level, given the graph partition info (i.e., mapping from partition input/output index to graph input/output index). N partition_ _placeholder_)r&r(r*r+c,i|]}|j|Sr )r1)rNr&rs r" z4get_partition_cudagraph_metadata..s0+/h &r!)rrinput_index_mappingr0addr4r3r$rappendoutput_index_mappingrconstant_namesr) rrpartition_placeholderspartition_static_input_idxspartition_mutated_input_idxspartition_input_idxgraph_input_idxrKpartition_stack_tracesgraph_output_idxpartition_constantss ` r" get_partition_cudagraph_metadatarss 3=<<4>LL 09)1133,_ h8 8 8 ' + +,? @ @ @ h9 9 9 ( , ,-@ A A A  &"/@KK*V-"2VVATVV )- K %%k2222)>00  ' " ) )(*?@P*Q R R R R " ) )$ / / / /3@3O #$   r!)r5r6r7r')rFr$r7r')r5r6r7r$)rRrSr7r))rYr%r7r%)r3r2r\r]r7r%)rdr-rerfrgrhr7r')rVr6r7r')rtrur7r')rbr%r7r) rrrrr4rr0r/r7r')rKr$r7r') r3r2rerfrrrr/rrr7r%)rrrrr7r)rr rrr7r)< __future__r dataclassesenumrtypingrrrrr r?torch._dynamo.utilsr r torch._inductor.utilsr r torch.utils._ordered_setrutilsrcollections.abcrr AbstractSet_logginggetArtifactLoggerrrrolistrr OutputType ModelType dataclassrr$r-rErIrMrXr[rcrqrsrrrrrrrrrrrrr r!r"rs]""""""@@@@@@@@@@@@@@ ========>>>>>>>>////////////=<<<<<<<<00<HH N44 ' (5el!234 5 d9o& 2 3 d###       $# d### , , , , , , ,$# ,d### & & & & & & &$# &    5555 O O O O 2222(>TTTT8HHHH J J J J         99990    0000040000>:d###&&&&&&&$#&d### ' ' ' ' ' ' '$# '333333r!