
    &Vji\                     r   U d dl mZmZ d dlmZmZ d dlZd dlmZ	 d dl
mZ d dlmZ d dlmZmZmZ d dlmZmZ  G d d	e          Zdaee         ed
<   deddfdZ	 ddej        dee         deej        ee         f         fdZ	 ddej        dedee         dej        fdZ	 ddej        dededede	j        dee         dej        fdZ	 ddej        dededee         dej        f
dZ 	 ddej        dee         deej        e!e         f         fdZ"	 ddedee         dee         dej        fdZ#dS )    )ABCabstractmethod)AnyOptionalN)ShardedTensor)Shard)_all_gather_dtensor_create_chunk_dtensor_create_chunk_sharded_tensor)
DeviceMeshDTensorc                      e Zd ZdZedej        deej        ee	         f         fd            Z
edej        de	dej        fd            Ze	 ddej        ded	ed
edej        deej                 dej        fd            Zedej        dededej        fd            Zedej        deej        ee         f         fd            Zededee         dej        fd            ZdS )FSDPExtensionsz
    This enables some customizable hooks to enable composability with tensor
    parallelism. To activate these hooks, use :func:`_set_fsdp_extensions` to
    set a custom :class:`FSDPExtensions` that implements the hooks.
    tensorreturnc                     dS )z6E.g. converting ``DistributedTensor`` to local tensor.N selfr   s     a/root/voice-cloning/.venv/lib/python3.11/site-packages/torch/distributed/fsdp/_fsdp_extensions.pypre_flatten_transformz$FSDPExtensions.pre_flatten_transform   s	     	    param_extensionc                     dS )z6E.g. converting local tensor to ``DistributedTensor``.Nr   )r   r   r   s      r   post_unflatten_transformz'FSDPExtensions.post_unflatten_transform   s	     	r   Nrank
world_sizenum_devices_per_nodepgdevicec                     dS )z6Shards a tensor to chunks and returns the local chunk.Nr   )r   r   r   r   r   r   r    s          r   chunk_tensorzFSDPExtensions.chunk_tensor(   	     	r   device_meshc                     dS )zAShards a tensor/DTensor to DTensor and returns the local DTensor.Nr   )r   r   r   r$   s       r   chunk_dtensorzFSDPExtensions.chunk_dtensor5   s	     	r   c                     dS )z
        This is to be called before loading a *sharded* model state dict and
        should return the tensor and list of shards from which to load data.
        Nr   r   s     r   pre_load_state_dict_transformz,FSDPExtensions.pre_load_state_dict_transform?   s	     	r   parent_meshc                     dS )z
        This is to be called before loading a *sharded* DTensor state dict.
        This gathers tensor in FSDP dimension and returns local tensor of
        TP DTensor.
        Nr   )r   r   r)   s      r   all_gather_dtensorz!FSDPExtensions.all_gather_dtensorJ   r#   r   N)__name__
__module____qualname____doc__r   torchTensortupler   r   r   r   intdistProcessGroupr    r"   r   r&   listr   r(   r   r+   r   r   r   r   r      s          
u|Xc]*	+   ^   
	   ^  *.
 

 
 	

 "
 
 &
 

 
 
 ^
    	
 
   ^  
u|T%[(	)   ^ 

 j)
 
	
 
 
 ^
 
 
r   r   _extensions	flattenerr   c                 
    | a d S r,   )r8   )r9   s    r   _set_fsdp_extensionsr;   [   s    KKKr   r   fsdp_extensionc                 J    ||                     |           \  }}|||fS | d fS r,   )r   )r   r<   
new_tensorr   s       r   _ext_pre_flatten_transformr?   `   s=     !&4&J&J6&R&R#
O&..4<r   r   c                 :    |||                     | |          S | S r,   )r   )r   r   r<   s      r   _ext_post_unflatten_transformrA   k   s*    
 !o&A66vOOOMr   r   r   r   r   c                 @    ||j         nt          } || ||||          S r,   )r"   r   )r   r   r   r   r   r<   chunk_tensor_fns          r   _ext_chunk_tensorrD   u   sB     % 	##) 
 ?
  r   r$   c                 <    ||j         nt          } || ||          S r,   )r&   r
   )r   r   r$   r<   chunk_dtensor_fns        r   _ext_chunk_dtensorrG      s=     % 	$$" 
   r   c                     ||                     |           S t          |           t          u sJ |                                 }| |fS r,   )r(   typer   local_shards)r   r<   shardss      r   "_ext_pre_load_state_dict_transformrL      sQ     !;;FCCC<<=((((  ""FFr   r)   c                 :    ||j         nt          } || |          S r,   )r+   r	   )r   r)   r<   all_gather_dtensor_fns       r   _ext_all_gather_dtensorrO      s2     % 	))  
 ! 555r   r,   )$abcr   r   typingr   r   r1   torch.distributeddistributedr5   +torch.distributed._shard.sharded_tensor.apir   -torch.distributed._shard.sharded_tensor.shardr   #torch.distributed.fsdp._shard_utilsr	   r
   r   torch.distributed.tensorr   r   r   r8   __annotations__r;   r2   r3   r?   rA   r4   r6   rD   rG   r7   rL   rO   r   r   r   <module>rY      s1   # # # # # # # # #                              E E E E E E ? ? ? ? ? ?         
 9 8 8 8 8 8 8 8E E E E ES E E EP )-Xn% , , ,N t     04 L^, 5<#&'    04 L ^, \	     04 L
  	
 	 ^, \   4 04	 L
  ^,	
 \   ( 04	 	L	^,	 5<e$%	 	 	 	 04
6 
6
6*%
6 ^,
6 \	
6 
6 
6 
6 
6 
6r   