&Vji@e ddlZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddl mZe dZgdZdMdZedZdNd Zd Zd ZdOd Zd ZefdZefdZefdZdZGddZdZdZdZdeedZ dZ!dZ"dZ#dZ$dPdZ%dZ&dOdZ'dOd efd!Z(e)d"krddl*Zej+,ej+-e.Z/e/ej+vrej+0e/ddl1Z1d#Z2d$e2Z3e1j4d%e2&Z5e56d'(Z7d)Z8d*Z9e7:d+e9&Z;e;<d,e3-d.Z9e7:d/e9&Z=e=<d,e3-d0Z9e7:d1e9&Z>e><d,e3-e8e>d2Z9e7:d3e9&Z?e?<d,e3-e8e?d4Z9e7:d5e9&Z@e@<d6e3-e@<d7e3-e8e@d8ZAeAD]\ZBZ9e7:eBe9&ZCeC<d,e3-d9ZDeC<d:d;eEdeD<d=ZDeC<d>d?d@eDAeBdBkrdCZDeC<dDdEdFeDGe5FZGdHZHdIZIeGjJd1kr,eHeGjKZLeIeGjMeeLdSeGjJd3kr,eHeGjKZLeIeGjMeeLdSeGjJd+kr&eHeGjKZLeNe!eLdSeGjJd/kr&eHeGjKZLeNe"eLdSeGjJd5kr=eHeGjOZOeHeGjPZPeIeGjMeeOePdSeGjJdBkr9eHeGjKZLeIeGjMe%eLeGjQeGjJdSeGjJdKkr5eHeGjKZLeIeGjMe(eLeGjQLdSdSdS)QN) lru_cache)groupby)Any)format_flamegraphsegmentsmemorycompareFc|d}|d}|s|dd}|d}|d|d|S)Nlinefilename/name:)split)f full_filenameifnamefuncs P/root/voice-cloning/.venv/lib/python3.11/site-packages/torch/cuda/_memory_viz.py _frame_fmtrsW & A jME % C  $ V9D a $  cFgd}gd}|D] }||vrdS |D] }||vrdS dS)N)zunwind::unwindzCapturedTraceback::gathergather_with_cpp_start__libc_start_mainPyEval_ PyObject_ PyFunction_) z core/boxingz /Registerz /Redispatchz pythonrun.czModules/main.czObjects/call.czObjects/methodobject.czpycore_ceval.hzceval.czcpython/abstract.hFT)rr omit_functionsomit_filenamesofs r _frame_filterr%ss   N   N ::55  >>55  4rcB|rt|}fd|DS)Nchg|].}t|d|dt|/S)rr )r%r).0rrs r z_frames_fmt..BsK    6AjM 2 2 1m$$   r)reversed)framesrreverses ` r _frames_fmtr-?sC"&!!       rcd|vr7|dddg}|ddd}n|d|d}g}||fS)Nhistoryrr+ real_sizerequested_sizesize)get)br+r0s r_block_extra_legacyr5IsdA~~9a$$Xr22iLOK0 EE*AfI66  9 rcHd|vrt|S|d|dfS)Nr+r1)r5)r4s r _block_extrar7Ss0q"1%%% X;*+ ++rc|:tjd}tj|d|d}tj|sddl}ddl}td||dd 5}|j d |j  tj |j d tj |j |n#t$rYnwxYwdddn #1swxYwY|d d g}tj|tjtjd}|jJ|jJ|j||j|j}|j||dksJ|S)Nz ~/.cache/T)exist_okz/flamegraph.plrzDownloading flamegraph.pl to: wbz.pl)modesuffixzNhttps://raw.githubusercontent.com/brendangregg/FlameGraph/master/flamegraph.pliz --countnamebytesutf-8)stdinstdoutencoding)ospath expandusermakedirsexiststempfileurllib.requestprintNamedTemporaryFilerequest urlretrieverchmodrenameOSError subprocessPopenPIPEr?r@writeclosereadwait) flamegraph_linesflamegraph_script cache_dirrGurllibrargspresults rrrZs G&&{33  I----(888 7>>+ , , B/@BBCCC  ( (d5 ( A A Q N & &`    ''' !&"34444                    }g 6D JOJOg   A 7    8   GMM"###GMMOOO X]]__FHNNFFHHH 6688q==== Ms6 !D/4C$#D$ C1.D0C11DDDc d}|D]}d|vrCt|\}}||d|dd||d|dn{d}|dD]p}|d}||z }d |vr9|d }||d|dd||d|dL||d|dd |dq|d |z } | r$||d|dd | ddS) NcR|sdSdt|dS)Nz ;T)r,)joinr-)r+s rframes_fragmentz&_write_blocks..frames_fragments. "!>xx FD999:::rr/r`state  rr0r+z; r2z; )r7rS) rprefixblocksrbr4r+accounted_for_sizehszgapss r _write_blocksrl~s;;; >> A  )5a &F& GGYYAgJYY)@)@YYCUYYY    "# y\ J J{^"b("q==x[FGGvUU' UU__V5L5LUUrUUUVVVVGGvHH' HH"HHHIIIIy--  > GGv<<' <>rctj}|dD]-}d|dd|d}t|||d.||S)Nrstream_stream;seg_addressrgioStringIOrlgetvaluesnapshotrrsegrfs rrrsp A #00?3x=??s9~??aX////  QZZ\\ * **rctj}|dD]$}d|d}t|||d%||S)Nrrnrorgrrrvs rrrsd A #00*3x=**aX////  QZZ\\ * **rcdd}tj}fd|D}fd|D}tdd||z Dtdd||z D|D]2}||vr#t|d |||d 3|D]2}||vr#t|d |||d 3||S) Nc"|d|dfS)Nrq total_sizer!rxs r_seg_keyzcompare.._seg_keysIL 122rc*d|dd|dS)Nrnrorprqr!r}s r _seg_infozcompare.._seg_infos =X==S^===rc&h|] }|Sr!r!r(rxr~s r zcompare..s!333S88C==333rc&h|] }|Sr!r!rs rrzcompare..s!111C((3--111rzonly_before = cg|]\}}|Sr!r!r(a_s rr)zcompare..sEEE$!QAEEErz only_after = cg|]\}}|Sr!r!rs rr)zcompare..sDDDA1DDDrz only_before;rgz only_after;)rsrtrIrlru) beforeafterrrr before_segs after_segsrxr~s @rr r sg333>>> A3333F333K11115111J GEE;+CEEE G GHHH FDD*{*BDDD F FGGGMM 8C== * * !>;;S] K K K  QZZ\\ * **rc\dD]$}t|dkr |d|dcS|dz}%|ddS)N)KiMiGiTiPiEiZig@z3.1fB.1fYiB)abs)numunits r _format_sizersX> s88f  ''''' ' ' ' v ????rc eZdZdZdZdZdS)Bytesc||_dSN)value)selfrs r__init__zBytes.__init__s  rc0t|j|zSr)rr)rrhss r__add__z Bytes.__add__sTZ#%&&&rc*t|jSr)rr)rs r__repr__zBytes.__repr__sDJ'''rN)__name__ __module__ __qualname__rrrr!rrrrsA'''(((((rrc@td|dDS)Nc3>K|]}|ddk|dVdS)rcactive_allocatedr2Nr!)r(r4s r zcalc_active..s5TTQ1W:AS3S3Sqy3S3S3S3STTrrg)sumr}s r calc_activers# TT#h-TTT T TTrc`||z}d}|dkr||z dz}d|dd}t||S)Nrrdz (rz % internal))r) free_external free_internaltotalr<pcts r _report_freersS M )E F zzu$+*c****Ell $F $ $$ri@zC Legend: [a ] - a segment in the allocator ^-- a page aU of memory in the segment a-z: pages filled with a single block's content ' ': page is completely free *: page if completely full with multiple blocks 0-9: page is partially full with tensors of multiple blocks (9 == 90% full) (X% internal) - of the free memory, X% is free because we rounded the size of the allocation. c tj}|dttdd}d}d}d}t |ddD]}||dz }d}d}d} g} d} |dD]d} | d d k} | r>t | \}}| | |d f| |z } || d |z z }n || d z }| | d z } e|| z }||z }||z }|dd z tzd z}dt|D}dt|D}d}t| D]\}\}}} ||z }||z}|tz}|d z tzd z}tt| rdnd|dzz}t||D]}t||tz}t||d ztz}||xx||z tz z cc<||dkr"dt||dz||<~|||<|ddkrdn d|d}d|}||z| z|dksJ|ddkr d|dnd}|dtkrC|d|dt|ddt!||d|d |dt#|dd|d t|d|d!t|d|d"t!||d|t$||z|z|ksJ|S)#aVisually reports how the allocator has filled its segments. This printout can help debug fragmentation issues since free fragments will appear as gaps in this printout. The amount of free space is reported for each segment. We distinguish between internal free memory which occurs because the allocator rounds the allocation size, and external free memory, which are the gaps between allocations in a segment. Args: data: snapshot dictionary created from _snapshot() zSummary of segments >= z in size rrc0|dt|fS)Nr|)r)xs rzsegsum..s<+a..(Irkeyr|rgrcrTr2cg|]}dS)rdr!r(rs rr)zsegsum..s---AC---rcg|]}dS)gr!rs rr)zsegsum.. s))))))rrArdz 0123456789* rorz , stream_z stream_[z] z allocated, z freerez segments: ztotal_reserved: ztotal_allocated: z total_free: )rsrtrSr PAGE_SIZEsortedr7appendrange enumeratechrordmaxminintrarlenlegendru) dataouttotal_reservedtotal_allocatedrrrxseg_free_externalseg_free_internal seg_allocated all_rangesboffsetr4activerallocated_sizensegoccupiedfrac active_sizerstart_r2finish_startfinishmjserobodys rsegsumrs +--CIIDi(8(8DDDEEENOMM ZII66 #l++  X ! !AwZ#55F /$0OO!>!!7ND"ABBB/ !QvY%??!!!QvY.! qy GG=(** ** L!A%)3a7--t---))U4[[))) )2:)>)> $ $ %A%f 4 KtmGi'Eki/!3FCv.3//1r6:;;A5&)) $ $I ..!a%9!455QAEY..A;#%%"/DGbL0A0A"BHQKK"#HQKK $8}))/J3x=/J/Jwwx    1 1M ASEV V V V V/28}/A/A+CM+++r |  ) ) IIWDWWE#l"344WW 13DEEWWLRWWW   II43tJ/00444555II:~!6!6:::;;;II<%"8"8<<<===IIK\-GGKKKLLLIIf = (? :n L L L L <<>>rctjfd}tdD]+\}}|r$d|d||,S)Nc gi}i}gdfd}fd}d}t|dd}dD] }||dz }t|D]\}}|ddkrp|d |d } } |} || \} } | d } | }n| | z }| d | d |dt| d| | |f|| <|| z }|ddkrW|d |d } } || | ddf\}}}d|dt| d|ddkry|d |d } } || z}|| | ddf\}}}d|dt| d||vr|||=j|ddkrd|d |d } } |}|d| dt| d|| | f||| <|ddkrm|d |d } } || | }d|dt| d||vr|||=S|ddkrH|d } |d}d t| d!t|d"|dd#t|dS)$NrcrSdzdz}}dz ttd|z|dkrdn|S)Nrrrrr)poprr)rr free_names next_names r_namez$trace..format.._nameOsd (!~~'''?INqA NI#c((Q,''>qAvv1>> >rcD]\}}}||kr|||zkr||fcStdD]-\}}|d}|d}||kr|||zkr d||fcS.dS)Nrrqrseg_)NN)r)addrrsaddrr2rrxrsegment_intervalss r find_segmentz+trace..format..find_segmentWs%6 ' '!eT5==TEDL%8%8;&&&#D$455 - -3I+,5==TEDL%8%8%!::u,,,,:rz entries rr|actionallocrr2MEMz = rrz] free_requestedzdel z # refree_completedz# free completed for rd segment_allocz = cudaMalloc(z, z) segment_freez cudaFree(z) # oom device_freezraise OutOfMemoryError # z requested, z free in CUDA z TOTAL MEM: )rSrrrr3r)entriessegment_addr_to_nameallocation_addr_to_namerrcountrrxrrr2nseg_nameseg_addroffsetrrfreerrrrrs @@@rformatztrace..formatGs@"$!"$   ? ? ? ? ? ?       S\\---... # 0 0C c,/ /NN!'**- $- $HE1{g%%vY& dEGG%1\$%7%7"(#$H!FF!H_F QGG8GGfGGuT{{GGGHHH12D%0@'- 8 000vY& d488dD?QRR a 999%++999::::8 000vY& d 488dD?QRR a H$HHtHHHIII222%%d+++/58//vY& duww TIIIItIIIJJJ!(($d);<<<-1$T**8..vY& d+//d;; ?d??d ???@@@///%%d+++,T28%%y' ed eeteee Q(((#### .e ../////r device_traceszDevice z ---------------- )rsrtrrSru)rr rdrs` @rtracerDs +--CP0P0P0P0P0P0d$/001   II6666 7 7 7 F1III <<>>ra  c|tjdtdtj|}|ddt |dzz zz }t j|d}tjd|dg}t dt| d |S) Nz;device argument is deprecated, plots now contain all device) stacklevelr>zsnapshot.pickle)rbase64z $VIZ_KINDz $SNAPSHOT) warningswarn FutureWarningpickledumpsrr b64encodedecodejson_memory_viz_templatereplacerepr)rviz_kinddevicebufferencoded_buffer json_formats r _format_vizr&s  I     \$  F gS[[1_,--F%f--44W==N*'8NSSTUUK  ' ' T(^^ D D L L[  rc,t||sdnd|S)aGenerate a visualization over time of the memory usage recorded by the trace as an html file. Args: data: Memory snapshot as generated from torch.cuda.memory._snapshot() device (torch.device, optional): Generate the trace for this device, needed if multiple devices have allocations. plot_segments (bool, optional): Plots memory returned from cudaMalloc, rather than individual allocations. Defaults to False. Returns: str: HTML of visualization Active Memory TimelinezActive Cached Memory Timeliner&)rr" plot_segmentss r trace_plotr+s-   -  ,   rc ddl}ddlm}ddlm}m}|ijjD]}|j |j kro|j }g}|r3|j |j |j fvr|||j }|3||j}|r|jjdkr||<|jdt)dzDdt)dzDdfddfd } fd } i} jD]\} } \}}}t-||s| |jkr| |||| ||f<9| |jkr0| | ||f|jt| |jkrH| | ||f|j| |||dz| ||dzf<| |jkr| |||d | ||f<fd| D}t=t?|tAj!dD]\}}d|}|d}|D]S\}}}}||kr!|d||z dd|d|d||d||z}T||dkr'|d|d|z ddddDd<dD]D}|dxx|dzcc<|ds$|d|dddES)Nr) _EventType)Action TensorKeycg|]}gSr!r!rs rr)z(_profile_to_snapshot..s>>>">>>rrcg|] }|dddgd S)Nr)r"rqr|rorgr!)r(r"s rr)z(_profile_to_snapshot..s@     !       r)rrc,|jdkr|jSS)Ncuda)typeindex)r" device_counts r to_devicez'_profile_to_snapshot..to_devices ;& <  rTc |j}|jj} d|}|d |d|kr||d<t|d||z|d< j||}||jnd} |d}d|D}d||d||d } |r! d || | S) Nrrqr|unknownr!c$g|] }dd|jdS)noner)r r r)r)r(r\s rr)z:_profile_to_snapshot..allocate..s$PPPQfa@@PPPrrr)rrr2ror+categoryr) r"storageptrr _categoriesr3rlowerr)r2 tensor_keyversion during_tracer"rrxr<stackrallocation_stacksmemory_profilerwr7s rallocatez&_profile_to_snapshot..allocates:,--!%z"6* y> !S^d%:%:!C N  td{  L"-11*gFF,4,@8=&&(((i!%%j"55PP%PPP     8 _ %f - 4 4Q 7 7 7rc dD];}d|||d|dd|dd.freesh5  A _ %f - 4 4!&M!&M#Ho       rF)rCchg|].\\}}}|j|d|d|df/S)rr2r+r")r(rArBeventr7s rr)z(_profile_to_snapshot..=sT ( !Z5 :$ % %uV}eFmU8_Urrrrqrginactive)r2rcr)r2rcr1r+r|c"g|] }|d | S)rgr!)r(rxs rr)z(_profile_to_snapshot..Us!QQQC3x=QCQQQr)T)"torchtorch._C._profilerr-torch.profiler._memory_profilerr.r/_memory_profile_op_tree sorted_nodestag AllocationparentPyCallPyCCallrfrom_allocation extra_fields alloc_sizer3r6rtimeline isinstanceCREATEDESTROYrr"INCREMENT_VERSION PREEXISTINGitemsrroperator itemgetter)profilerNr-r.r/rKrVpython_parentsrrGr  kv_to_elem_timerrArBr2 blocks_at_endr"rgrx last_addrrrr+rEr6rFrwr7s @@@@@r_profile_to_snapshotrksLLL------AAAAAAAA,,..N(5 8 8 9 - - -\FN ':*"3Z5G!HHH"))&111 '++E,>??C 8u)4q88)7!#&:**,,L>>eL1,<&=&=>>>    q 011     & &H     4     J7E6M2v, Gd*i00   V] " "08z70S0SJ G, - - v~ % % DW 566 *BS8T8T U U U U v/ / / DW 566 *BS8T8T U U U4,>M"&"7"7X=PQR=S=STTTz"6* N %+ $ $ !AtT64H $$dY.>%T%TUUU M  /&*$    t II s<( ( ( M \*Y6LL   RQ8J+?QQQHZ #SS LS^+8} S M #l*;j!Q!Q R R R OrcBt|}t|d|S)a}Generate a visualization over time of the memory usage recorded by kineto memory profiling as an html file. Args: profile: profile as generated by `torch.profiler.profile(profile_memory=True)` device (torch.device, optional): Generate the trace for this device, needed if multiple devices have allocations. Returns: str: HTML of visualization r()rkr&)rer"rws r profile_plotrm^s$$G,,H x!96 B BBrrc$t|d|S)NzAllocator State Historyr))rr"s r segment_plotrols t6 ? ??r__main__ztorch.cuda.memory._snapshot()zpickled memory statistics from z#Visualize memory dumps produced by ) descriptionr)destc8|dddddS)N-o--outputz output.svgz$flamegraph svg (default: output.svg)defaulthelp) add_argument)r\s r_outputrzs3    7      rzmPrints overall allocation statistics and a visualization of how the allocators segments are currently filled.statsinput)rxz`Prints buffer of the most recent allocation events embedded in the snapshot in a Pythonic style.rzaGenerate a flamegraph that visualizes what memory is stored in each allocator segment (aka block)rzNGenerate a flamegraph the program locations contributing to CUDA memory usage.rzGenerate a flamegraph that shows segments (aka blocks) that have been added or removed between two different memorys snapshots.r rr))r+z]Generate a visualization over time of the memory usage recorded by the trace as an html file.)rozfVisualize how allocations are packed into allocator segments at each point in a trace as an html file.z]visualize trace from this device (default: chooses the only device with trace info or errors)z-dz--device)r4rwrxz4path to save the visualization(default: output.html)rtruz output.htmlrvr+z?visualize change to segments rather than individual allocationsz-sz --segments store_true)rrxc|dkrtjj}nt|d}t j|}t |tr|gd}|S)N-rb)rtraces)sysr?r#openrloadr]list)rrrs r_readrsW 3;;  AAT4  A{1~~ dD ! ! 4 $33D rct|d5}||ddddS#1swxYwYdS)Nw)rrS)rrrs r_writers $__  GGDMMM                  s 488)r"r*rorJ)F)FFr)NF)RrrsrrcrBrrPrr functoolsr itertoolsrtypingrcache__all__rr%r-r5r7rrlrrr rrrrrrrrrr&r+rkrmroros.pathrCrealpathdirname__file__thedirremoveargparsefn_namepickledArgumentParserparseradd_subparsers subparsersrzrq add_parserstats_arytrace_a segments_amemory_a compare_aplotscmd trace_plot_arxr parse_argsr[rrrr|routputrIrrr"r!rrrs     $ @ @ @!!!!@,,,!!!!H>>>6*;++++(9++++.?++++4((((((((UUU%%%   y!!   OOOdYYYx $*EEEP C C C C@@s@@@@ zNNN W  bgooh77 8 8F  OOO-G999G $X $C'CCF&&H&55J   BK##G#EEG w///tK##G#EEG w///uK&&z{&KKJG'222 GJ Y$$X;$GGH '000 GH >%%i[%III 8'222 7111 GI E"  [!,,Sk,JJ !!'!888n!!$ dQU!VVVE!!$ MPT!UUU ,  TD  % %l