
    QVjiU                        U d dl Z d dlmZ d dlmZ d dlZd dlmZ d dlm	Z	 d dl
mZ  G d de	          Z G d	 d
e	          Z G d de	          Z G d de	          Zdaeeeej                                   ed<   dej        fdZdS )    N)chain)Optional_get_device_index)Function)commc                   :    e Zd Zed             Zed             ZdS )	Broadcastc                    t          d |D                       s
J d            d |D             }|| _        t          |          dk    rdS t          |          | _        |d                                         | _        t          j        || j                  }g }t          | j	        dd                    D ](\  }|s!|
                    fd|D                        ) | j        |  t          t          j        |                    S )Nc              3   6   K   | ]}|j         j        d k    V  dS cpuNdevicetype.0is     V/root/voice-cloning/.venv/lib/python3.11/site-packages/torch/nn/parallel/_functions.py	<genexpr>z$Broadcast.forward.<locals>.<genexpr>   +      ::a18=E)::::::    z2Broadcast function not implemented for CPU tensorsc                 .    g | ]}t          |d           S Tr   r   xs     r   
<listcomp>z%Broadcast.forward.<locals>.<listcomp>   #    GGGa(D11GGGr   r       c              3   (   K   | ]}|         V  d S Nr   )r   outputidxs     r   r   z$Broadcast.forward.<locals>.<genexpr>   s'      *M*M66#;*M*M*M*M*M*Mr   )alltarget_gpuslen
num_inputs
get_deviceinput_devicer   broadcast_coalesced	enumerateneeds_input_gradextendmark_non_differentiabletupler   from_iterable)ctxr&   inputsoutputsnon_differentiablesinput_requires_gradr$   s         @r   forwardzBroadcast.forward   s1   ::6::::: 	
 	
@	
 	
: HG;GGG%v;;!2V!!9//11*63?CC (1#2Fqrr2J(K(K 	N 	N$C$& N#***M*M*M*MW*M*M*MMMM##%899U(11222r   c                 B    dt          j        | j        | j        g|R  z   S )Nr"   )ReduceAddCoalescedapplyr*   r(   r2   grad_outputss     r   backwardzBroadcast.backward   s6    +1cn
/;
 
 
 
 	
r   N__name__
__module____qualname__staticmethodr7   r=   r   r   r   r
   r
      sH        3 3 \3$ 
 
 \
 
 
r   r
   c                   :    e Zd Zed             Zed             ZdS )r9   c                     fdt          dt                              D             | _        fdt          dt                              D             }t          j        ||          S )Nc                 D    g | ]}|                                          S r   r)   )r   r   gradss     r   r   z.ReduceAddCoalesced.forward.<locals>.<listcomp>)   s6     
 
 
&'E!H!!
 
 
r   r   c                 *    g | ]}||z            S r   r   )r   r   rG   r(   s     r   r   z.ReduceAddCoalesced.forward.<locals>.<listcomp>-   s'    VVV%A
N*+VVVr   )ranger'   r&   r   reduce_add_coalesced)r2   destinationr(   rG   grads_s     `` r   r7   zReduceAddCoalesced.forward'   s    
 
 
 
+0CJJ
+K+K
 
 
 WVVVVU1c%jj*5U5UVVV(===r   c                 6    dt          j        | j        g|R  z   S )NNN)r
   r:   r&   r;   s     r   r=   zReduceAddCoalesced.backward0   s*    
 OCO;l;;;< 	<r   Nr>   r   r   r   r9   r9   &   sH        > > \> < < \< < <r   r9   c                   :    e Zd Zed             Zed             ZdS )Gatherc                     t          d |D                       s
J d            |dk    rd _        nt          |d          }| _        | _        t	          d |D                        _        t          d |D                       r;|dk    r5t	          d |D                       }t          j        d	           d _        nd
 _        t	           fd|D                        _	        t          j        | j         j                  S )Nc              3   6   K   | ]}|j         j        d k    V  dS r   r   r   s     r   r   z!Gather.forward.<locals>.<genexpr>;   r   r   z/Gather function not implemented for CPU tensorsr   Tc              3   >   K   | ]}|                                 V  d S r"   rF   r   s     r   r   z!Gather.forward.<locals>.<genexpr>D   s*      >>!q||~~>>>>>>r   c              3   F   K   | ]}|                                 d k    V  dS r   N)dimr   ts     r   r   z!Gather.forward.<locals>.<genexpr>E   s.      ,,quuww!|,,,,,,r   r   c              3   @   K   | ]}|                     d           V  dS )r    N)viewrW   s     r   r   z!Gather.forward.<locals>.<genexpr>F   s,      55166!99555555r   zvWas asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.Fc              3   L   K   | ]}|                     j                  V  d S r"   )sizerV   )r   r   r2   s     r   r   z!Gather.forward.<locals>.<genexpr>O   s/      @@Asw@@@@@@r   )r%   target_devicer   rV   r0   
input_gpuswarningswarnunsqueezed_scalarinput_sizesr   gather)r2   r]   rV   r3   s   `   r   r7   zGather.forward9   s9   ::6::::: 	
 	
=	
 	
: E!! %C-mTBBM -C>>v>>>>>,,V,,,,, 		*55f55555FM'  
 %)C!!$)C!@@@@@@@@@{637C,=>>>r   c                     t                               | j        | j        | j        |          }| j        rt          d |D                       }d|z   S )Nc              3   &   K   | ]}|d          V  dS rU   r   )r   gs     r   r   z"Gather.backward.<locals>.<genexpr>X   s&      #B#BQAaD#B#B#B#B#B#Br   rN   )Scatterr:   r^   rb   rV   ra   r0   )r2   grad_outputscattered_gradss      r   r=   zGather.backwardR   sY    !--NCOSWk
 
   	C##B#B/#B#B#BBBOo--r   Nr>   r   r   r   rP   rP   8   sH        ? ? \?0 . . \. . .r   rP   c                   :    e Zd Zed             Zed             ZdS )rg   c                 n   d |D             }|| _         |j        j        dk    r|                                nd| _        d }t
          j                                        r| j        dk    rd |D             }t          j	        |||| j         |          }|t          |          D ]\  }}t
          j                            ||                   5  t
          j                                        }	|	                    ||                    |                    |	           d d d            n# 1 swxY w Y   |S )Nc                 .    g | ]}t          |d           S r   r   r   s     r   r   z#Scatter.forward.<locals>.<listcomp>_   r   r   r   c                 P    g | ]#}t          t          j        |                    $S r   )_get_streamtorchr   )r   r   s     r   r   z#Scatter.forward.<locals>.<listcomp>e   s*    SSSV{5<#7#788SSSr   )rV   r   r   r)   r*   rp   acceleratoris_availabler   scatterr,   device_indexcurrent_streamwait_streamrecord_stream)
r2   r&   chunk_sizesrV   inputstreamsr4   r   r#   main_streams
             r   r7   zScatter.forward]   st   GG;GGG161Be1K1K5++---QS))++ 	T0@B0F0FSS{SSSG,uk;QQ&w// 6 6	6&33KNCC 6 6"'"3"B"B"D"DK++GAJ777((5556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 s   AD))D-	0D-	c                 D    d d d t          j        | j        | j        g|R  fS r"   )rP   r:   r*   rV   )r2   rh   s     r   r=   zScatter.backwardp   s*    T4c.>!V+!V!V!VVVr   Nr>   r   r   r   rg   rg   \   sM          \$ W W \W W Wr   rg   _streamsr   c                    | j         dk    st          j                                        sdS t          j                                        j         | j         k    sJ t
          "dgt          j                                        z  at
          | j                 &t          j        | j                  t
          | j        <   t
          | j                 S )zBGet a background stream for copying between CPU and target device.r   N)	r   rp   rq   rr   current_acceleratorr}   device_countindexStream)r   s    r   ro   ro   y   s     {e5#4#A#A#C#Ct002276;FFFF6E-::<<<%!&fl!;!;FL!!r   )r_   	itertoolsr   typingr   rp   torch._utilsr   torch.autogradr   torch.nn.parallelr   r
   r9   rP   rg   r}   listr   __annotations__r   ro   r   r   r   <module>r      sn                 * * * * * * # # # # # # " " " " " "
 
 
 
 
 
 
 
6< < < < < < < <$!. !. !. !. !.X !. !. !.HW W W W Wh W W W4 48(4./
0 7 7 7
" 
" 
" 
" 
" 
" 
"r   