
    1;jiS                        d dl Z d dl mZmZ d dlmc mZ d dlmZ d dlm	Z	 d Z
d Zd Z G d d	ej                  Z G d
 dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  ZdS )    N)nneinsum	rearrange)	Rearrangec                 
    | d uS N )vals    M/root/voice-cloning/.venv/lib/python3.11/site-packages/conformer/conformer.pyexistsr   
   s    d?    c                 (    t          |           r| n|S r	   )r   )r   ds     r   defaultr      s    ++$331$r   c                 &    | dz  }||| dz   dz  z
  fS )N      r
   )kernel_sizepads     r   calc_same_paddingr      s%    

Ca1,,--r   c                       e Zd Zd ZdS )Swishc                 0    ||                                 z  S r	   )sigmoidselfxs     r   forwardzSwish.forward   s    199;;r   N)__name__
__module____qualname__r   r
   r   r   r   r      s#            r   r   c                   $     e Zd Z fdZd Z xZS )GLUc                 V    t                                                       || _        d S r	   )super__init__dim)r   r(   	__class__s     r   r'   zGLU.__init__   s$    r   c                 n    |                     d| j                  \  }}||                                z  S )Nr   r(   )chunkr(   r   )r   r   outgates       r   r   zGLU.forward   s0    GGA48G,,	TT\\^^##r   r    r!   r"   r'   r   __classcell__r)   s   @r   r$   r$      sG            $ $ $ $ $ $ $r   r$   c                   $     e Zd Z fdZd Z xZS )DepthWiseConv1dc                     t                                                       || _        t          j        ||||          | _        d S )N)groups)r&   r'   paddingr   Conv1dconv)r   chan_inchan_outr   r6   r)   s        r   r'   zDepthWiseConv1d.__init__$   s>    IgxwOOO			r   c                 `    t          j        || j                  }|                     |          S r	   )Fr   r6   r8   r   s     r   r   zDepthWiseConv1d.forward)   s%    E!T\""yy||r   r/   r1   s   @r   r3   r3   #   sL        P P P P P
      r   r3   c                   $     e Zd Z fdZd Z xZS )Scalec                 d    t                                                       || _        || _        d S r	   )r&   r'   fnscale)r   rA   r@   r)   s      r   r'   zScale.__init__0   s+    


r   c                 .     | j         |fi || j        z  S r	   )r@   rA   r   r   kwargss      r   r   zScale.forward5   s#    twq##F##dj00r   r/   r1   s   @r   r>   r>   /   sG            
1 1 1 1 1 1 1r   r>   c                   $     e Zd Z fdZd Z xZS )PreNormc                     t                                                       || _        t          j        |          | _        d S r	   )r&   r'   r@   r   	LayerNormnorm)r   r(   r@   r)   s      r   r'   zPreNorm.__init__9   s5    L%%			r   c                 H    |                      |          } | j        |fi |S r	   )rI   r@   rC   s      r   r   zPreNorm.forward>   s+    IIaLLtwq##F###r   r/   r1   s   @r   rF   rF   8   sG        & & & & &
$ $ $ $ $ $ $r   rF   c                   6     e Zd Z	 	 	 	 d fd	Z	 	 	 d	dZ xZS )
	Attention   @              c                    t                                                       ||z  }|| _        |dz  | _        t	          j        ||d          | _        t	          j        ||dz  d          | _        t	          j        ||          | _        || _	        t	          j
        d|z  dz   |          | _        t	          j        |          | _        d S )Ng      F)biasr   r   )r&   r'   headsrA   r   Linearto_qto_kvto_outmax_pos_emb	Embeddingrel_pos_embDropoutdropout)r   r(   rS   dim_headr\   rX   	inner_dimr)   s          r   r'   zAttention.__init__C   s     	u$	
%
Ic9U;;;	YsIM%@@@
i	3//&<K!(;XFFz'**r   Nc                    j         d         j        | j        | j        t	                    f\  }}}t                    |                               g|                                                   dd          R \  }}	}
t          fd||	|
f          \  }}	}
t          d||	          | j        z  }t          j        |          }t          |d          t          |d	          z
  }|                    | |          |z   }|                     |                              |          }t          d
||          | j        z  }||z   }t	          |          st	          |          rt          |fd          }|st          ||          nt          |fd          }t          j        |j                  j         }t          |d          t          |d          z  }|                    | |           |                    d          }t          d||
          }t          |d          }|                     |          }|                     |          S )Nr   r+   c                 (    t          | d          S )Nzb n (h d) -> b h n d)hr   )trc   s    r   <lambda>z#Attention.forward.<locals>.<lambda>c   s    	!-C K K K r   zb h i d, b h j d -> b h i j)devicez	i -> i ()z	j -> () jzb h n d, n r d -> b h n rc                  @    t          j        j        d d         d iS Nr   rf   torchonesshape)rf   r   s   r   re   z#Attention.forward.<locals>.<lambda>q   s     QWRaR[)R6)R)R r   c                  @    t          j         j        d d         diS rh   ri   )contextrf   s   r   re   z#Attention.forward.<locals>.<lambda>r   sQ    mrmw  zA  zG  HJ  IJ  HJ  zK  n]  V\  n]  n] r   zb i -> b () i ()zb j -> b () () jzb h i j, b h j d -> b h i dzb h n d -> b n (h d))rl   rf   rS   rX   r   r   rU   rV   r,   mapr   rA   rj   aranger   clamprZ   tofinfodtypemaxmasked_fill_softmaxrW   r\   )r   r   rn   maskcontext_masknrX   has_contextqkvdotsseqdistrZ   pos_attn
mask_valueattnr-   rf   rc   s    ``                @@r   r   zAttention.forwardX   s    23ah
TXTdflmtfufu1u.61k;'1%%99Q<<I$**W"5"5";";AR";"H"HII1aKKKKaQRTUYWW1a3Q::TZG l1v...k**YsK-H-HHzz;,44{B&&t,,//225q+FFSh$<< 	16,// 	14!R!R!R!R!RSSD>I  ^7<666wWc  f]  f]  f]  f]  f]  P^  P^L+dj1155JT#566<Qc9d9ddDteZ000||"|%%2D!<<344kk#||C   r   )rM   rN   rO   rP   )NNNr/   r1   s   @r   rL   rL   B   sj         + + + + + +0 $! $! $! $! $! $! $! $!r   rL   c                   *     e Zd Z	 	 d fd	Zd Z xZS )FeedForward   rO   c           	      :   t                                                       t          j        t          j        |||z            t                      t          j        |          t          j        ||z  |          t          j        |                    | _        d S r	   )r&   r'   r   
SequentialrT   r   r[   net)r   r(   multr\   r)   s       r   r'   zFeedForward.__init__   sy     	=Ic3:&&GGJwIcDj#&&Jw
 
r   c                 ,    |                      |          S r	   r   r   s     r   r   zFeedForward.forward       xx{{r   )r   rO   r/   r1   s   @r   r   r   ~   sT         	
 
 
 
 
 
      r   r   c                   .     e Zd Z	 	 	 	 d fd	Zd Z xZS )ConformerConvModuleFr      rO   c                 <   t                                                       ||z  }|st          |          n|dz
  df}t          j        t          j        |          t          d          t          j        ||dz  d          t          d          t          ||||          |st          j
        |          nt          j                    t                      t          j        ||d          t          d          t          j        |          
  
        | _        d S )Nr   r   zb n c -> b c nr   r+   )r   r6   zb c n -> b n c)r&   r'   r   r   r   rH   r   r7   r$   r3   BatchNorm1dIdentityr   r[   r   )	r   r(   causalexpansion_factorr   r\   r^   r6   r)   s	           r   r'   zConformerConvModule.__init__   s     	**	8>X#K000[ST_VWDX=L&''Ic9q=!,,AJJJIyW^___-3FBN9%%%GGIia((&''Jw
 
r   c                 ,    |                      |          S r	   r   r   s     r   r   zConformerConvModule.forward   r   r   )Fr   r   rO   r/   r1   s   @r   r   r      sZ         
 
 
 
 
 
4      r   r   c            
       <     e Zd Zdddddddddd	 fd	
ZddZ xZS )ConformerBlockrN   rM   r   r   r   rO   F	r]   rS   ff_multconv_expansion_factorconv_kernel_sizeattn_dropout
ff_dropoutconv_dropoutconv_causalc       
            t                                                       t          |||          | _        t	          ||||          | _        t          ||
|||	          | _        t          |||          | _        t          || j                  | _        t          dt          || j                            | _        t          dt          || j                            | _        t          j        |          | _        d S )N)r(   r   r\   )r(   r]   rS   r\   )r(   r   r   r   r\   g      ?)r&   r'   r   ff1rL   r   r   r8   ff2rF   r>   r   rH   	post_norm)r   r(   r]   rS   r   r   r   r   r   r   r   r)   s              r   r'   zConformerBlock.__init__   s     	SJOOOCHeWcddd	'cK\q  BR  ^j  k  k  k	SJOOOC++	gc484455gc484455c**r   Nc                     |                      |          |z   }|                     ||          |z   }|                     |          |z   }|                     |          |z   }|                     |          }|S )N)rx   )r   r   r8   r   r   )r   r   rx   s      r   r   zConformerBlock.forward   sl    HHQKK!OIIaI%%)IIaLL1HHQKK!ONN1r   r	   r/   r1   s   @r   r   r      ss        
  !+ + + + + + +4       r   r   c            
       :     e Zd Zdddddddddd	 fd	
Zd
 Z xZS )	ConformerrN   rM   r   r   r   rO   Fr   c       
            t                                                       || _        t          j        g           | _        t          |          D ]0}| j                            t          |||||||                     1d S )N)r(   r]   rS   r   r   r   r   )	r&   r'   r(   r   
ModuleListlayersrangeappendr   )r   r(   depthr]   rS   r   r   r   r   r   r   r   _r)   s                r   r'   zConformer.__init__   s     	mB''u 
	 
	AK~#!(=#3)	  	  	  	 	 	 	
	 
	r   c                 0    | j         D ]} ||          }|S r	   )r   )r   r   blocks      r   r   zConformer.forward   s'    [ 	 	EaAAr   r/   r1   s   @r   r   r      sn          !      >      r   r   )rj   r   r   torch.nn.functional
functionalr<   einopsr   einops.layers.torchr   r   r   r   Moduler   r$   r3   r>   rF   rL   r   r   r   r   r
   r   r   <module>r      sW                           ) ) ) ) ) )  % % %. . .    BI   $ $ $ $ $") $ $ $    bi   1 1 1 1 1BI 1 1 1$ $ $ $ $bi $ $ $:! :! :! :! :!	 :! :! :!x    ")   &    ")   @! ! ! ! !RY ! ! !J% % % % %	 % % % % %r   