
    &Vji>              	          d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlm	Z	 d dl
Z
d dlmZ d dlmc mc mZ d dlmZmZmZ  e j        e          Z G d d          Zd	e
j        j        d
eeeeee         f         f         fdZ dS )    N)defaultdict)Iterator)contextmanager)Enum)_apply_to_modules_get_module_fsdp_stateclean_tensor_namec                       e Zd ZU  G d dee          Z ee          Ze	eef         e
d<    e            Zee         e
d<   edd            Zeededed         fd	                        Zed
eddfd            ZdS )SimpleProfilerc                   &    e Zd ZdZdZdZdZdZdZdS )SimpleProfiler.Typeall
all_gatherall_gather_object
reshardingH2DD2HN)	__name__
__module____qualname__ALL	ALLGATHERALLGATHER_OBJ
RESHARDINGr   r        ]/root/voice-cloning/.venv/lib/python3.11/site-packages/torch/distributed/fsdp/_debug_utils.pyTyper      s,         	+!
r   r   results	profilingreturnNc                 j    | j                                          | j                                         d S )N)r   clearr    )clss    r   resetzSimpleProfiler.reset"   s0    r   profile_typec              #     K   || j         vsJ | d            | j                             |           t          j                    }	 d V  t          j                    }| j        |xx         ||z
  z  cc<   | j                             |           d S # t          j                    }| j        |xx         ||z
  z  cc<   | j                             |           w xY w)Nzk is already being profiled. SimpleProfiler does not support profiling multiple instances at the same time. )r    addtime	monotonicr   remove)r$   r&   beginends       r   profilezSimpleProfiler.profile'   s      3=000    100 	,'''  	/EEE.""CK%%%u4%%%M  ..... .""CK%%%u4%%%M  ....s   B ACmsgc                     t          j                    dk    rGt          j                    t           j        j        k    r!t
                              d|| j                   |                                  d S )Nr   z%s %s)	distget_rankget_debug_level
DebugLevelINFOloggerinfor   r%   )r$   r/   s     r   dump_and_resetzSimpleProfiler.dump_and_reset9   sV     =??aD$8$:$:do>R$R$RKKck222		r   )r!   N)r   r   r   strr   r   r   floatr   dict__annotations__setr    classmethodr%   r   r   r.   r8   r   r   r   r   r      s	            sD    !,E 2 2GT#u*222#%%Is3x   [ /3 /8D> / / / ^ [/       [  r   r   modelr!   c                 x    d }d }dg}i }t          | ||d |                                 D             ||          S )a  
    It is used for composable fully_shard() code path, it returns
      1. sharded module tree info: each line represents a submodule name that contains the
    submodule's FQN and its submodule class name, if the submodule is sharded by `fully_shard`,
    the submodule name will add a postfix with ' FULLY SHARDED'. Each increased tree
    level adds 4 spaces before the printed name. A printed sharded module tree info for a toy model
    is like this:
        [CompositeModel] FULLY SHARDED
            l1[Linear]
            u1[UnitModule] FULLY SHARDED
                u1.l1[Linear]
                u1.seq[Sequential]
                    u1.seq.0[ReLU]
                    u1.seq.1[Linear]
                    u1.seq.2[ReLU]
                u1.l2[Linear]
            u2[UnitModule] FULLY SHARDED
                u2.l1[Linear]
                u2.seq[Sequential]
                    u2.seq.0[ReLU]
                    u2.seq.1[Linear]
                    u2.seq.2[ReLU]
                u2.l2[Linear]
            l2[Linear]
      2. a dict mapping from the concated module FQN and class name to a list of its managed
    original parameters' FQNs. An example of the dict for the above toy sharded model is like this:
            {'[CompositeModel]': ['l1.weight', 'l1.bias', 'l2.weight', 'l2.bias'],
             'u1[UnitModule]': ['u1.l1.weight', 'u1.l1.bias', 'u1.seq.1.weight', 'u1.seq.1.bias', 'u1.l2.weight', 'u1.l2.bias'],
             'u2[UnitModule]': ['u2.l1.weight', 'u2.l1.bias', 'u2.seq.1.weight', 'u2.seq.1.bias', 'u2.l2.weight', 'u2.l2.bias']
            }
    All FQNs are prefixed starting from ``model``.

    Args:
        model (torch.nn.Module): Root module (which may or may not be passed to
                                 composable `fully_shard()`).
    c                 <   |dz  }t                    dk    rd         dk    r
d d         n}|dz   | j        j        z   dz   }d|z  |z   }t          |           }	|	|dxx         |dz   z  cc<   d S |	j                            | d           }
|
r|dxx         |d	z   dz   z  cc<   n|dxx         |dz   z  cc<   |
r^|
j        }t          |t          j	                  sJ fd
|j
        D             }||v r||                             |           d S |||<   d S d S )N   r   .[] 
z FULLY SHARDEDc                 4    g | ]}t          |z             S r   )r	   ).0nameprefixs     r   
<listcomp>zX_get_sharded_module_tree_with_module_name_to_fqns.<locals>.module_fn.<locals>.<listcomp>   s3       59!&4-00  r   )len	__class__r   r   _fully_sharded_module_to_handleget
flat_param
isinstanceflat_param_fileFlatParameter_fqnsextend)modulerL   
tree_levelsharded_tree_infosharded_module_name_to_fqns
num_spacestrimed_prefixprefixed_module_nameprinted_prefixed_module_namestatehandleparamglobal_fqnss    `           r   	module_fnzD_get_sharded_module_tree_with_module_name_to_fqns.<locals>.module_fnj   s     !^
KK!OOr
c0A0AF3B3KK 	  -s2V5E5NNQTT'*Z'7:N'N$&v..=a   $@4$GG   F6::64HH 	Ha   ,/??$F     a   $@4$GG    
	P%Ee_%BCCCCC   =B[  K $'BBB+,@AHHUUUUUDO+,@AAA
	P 
	Pr   c                     | d         |fS )Nr   r   )rZ   r[   s     r   	return_fnzD_get_sharded_module_tree_with_module_name_to_fqns.<locals>.return_fn   s     #%@@@r    c                     g | ]\  }}|S r   r   )rJ   key_s      r   rM   zE_get_sharded_module_tree_with_module_name_to_fqns.<locals>.<listcomp>   s    444a444r   )r   named_parameters)r?   rd   rf   rZ   r[   s        r   1_get_sharded_module_tree_with_module_name_to_fqnsrl   B   s{    P"P "P "PHA A A
 	$ 9;4451133444#  r   )!loggingr)   collectionsr   collections.abcr   
contextlibr   enumr   torchtorch.distributeddistributedr1   "torch.distributed.fsdp._flat_paramfsdp_flat_paramrT   $torch.distributed.fsdp._common_utilsr   r   r	   	getLoggerr   r6   r   nnModuletupler9   r;   listrl   r   r   r   <module>r~      s^     # # # # # # $ $ $ $ $ $ % % % % % %                    < < < < < < < < < < < <          
	8	$	$) ) ) ) ) ) ) )X[8?[
3S$s)^$$%[ [ [ [ [ [r   