~Vji*ddlZddlmZmZmZddlZdgZdejdejfdZ ddejdejd ejdejd ejd eejdeejfd Z d e dej j fdZ dee dedeeefdZdeedeededejdejf dZGddej j ZGddej j ZGddej j ZGddeZdS)N)ListOptionalTupleEmformerlengthsreturnc|jd}ttj|}tj||j|j||| dk}|S)Nr)devicedtype) shapeinttorchmaxitemaranger r expand unsqueeze)r batch_size max_length padding_masks T/root/voice-cloning/.venv/lib/python3.11/site-packages/torchaudio/models/emformer.py_lengths_to_padding_maskr s~q!JUYw'',,..//J< 7>WWW^^J   1  L  utterance right_contextsummarymemsleft_context_keyc|d|dz|dz}|d}|dkrd}n|tj|z |dz } ||dnd} ||dz| z| z} t | }|S)Nrr )r)sizerrrr) rrrrrrTBrright_context_blocks_lengthleft_context_blocks_lengthklengthss r_gen_padding_maskr's 1 q 1 11GLLOOCA1AAvv &'%)G*<*<*@*@*B*B&BW\\RS__&T#AQA]%5%:%:1%=%=%=cd"TYYq\\),GGJdd/AAA r activationc|dkrtjS|dkrtjS|dkrtjSt d|)NrelugelusiluzUnsupported activation )rnnReLUGELUSiLU ValueError)r(s r_get_activation_moduler2'sjVx}} v  x}} v  x}}?:??@@@rweight_init_scale_strategy num_layersc|dt|DS|dkrdt|DS|dkrdt|DStd|)Ncg|]}dSN).0_s r z*_get_weight_init_gains..4s000000r depthwisecBg|]}dtj|dzz S)?r mathsqrtr9 layer_idxs rr;z*_get_weight_init_gains..6s+RRR9di A ...RRRrconstantc<g|]}dtjdz S)r>r?rBs rr;z*_get_weight_init_gains..8s%FFFydill"FFFrz-Unsupported weight_init_scale_strategy value )ranger1)r3r4s r_get_weight_init_gainsrH2s!)00eJ//0000 #{ 2 2RRj@Q@QRRRR #z 1 1FFE*4E4EFFFFeIceefffr col_widthscol_masknum_rowsr ct|t|krtdfdt||D}tj|dS)Nz0Length of col_widths must match that of col_maskcvg|]5\}}|rtj|ntj|6S)r )roneszeros)r9 col_width is_ones_colr rKs rr;z-_gen_attention_mask_block..Cs^ #I{  = 8Yv6666 [9V < < <rr dim)lenr1ziprcat)rIrJrKr mask_blocks `` r_gen_attention_mask_blockrY=s| :#h--''KLLL'**h&?&? J 9ZQ ' ' ''rceZdZdZ ddededed eed ed ef fd Zd e j de j de e j e j ffdZ de j de j dee j de j fdZ dde j de j de j de j de j de j dee j dee j de e j e j e j e j ffdZde j de j de j de j de j de j de e j e j ffdZe jjde j de j de j de j de j de j de j de e j e j e j e j ffdZxZS) _EmformerAttentiona_Emformer layer attention module. Args: input_dim (int): input dimension. num_heads (int): number of attention heads in each Emformer layer. dropout (float, optional): dropout probability. (Default: 0.0) weight_init_gain (float or None, optional): scale factor to apply when initializing attention module parameters. (Default: ``None``) tanh_on_mem (bool, optional): if ``True``, applies tanh to memory elements. (Default: ``False``) negative_inf (float, optional): value to use for negative infinity in attention weights. (Default: -1e8) NFח input_dim num_headsdropoutweight_init_gain tanh_on_mem negative_infct||zdkrtd|d|d||_||_||_||_||_|j|jzdz|_tj |d|zd|_ tj ||d|_ tj ||d|_|rbtj j|j j| tj j|j j| dSdS) Nrz input_dim (z") is not a multiple of num_heads (z).grFT)bias)gain)super__init__r1r^r_r`rbrcscalingrr-Linearemb_to_key_value emb_to_queryout_projinitxavier_uniform_weight)selfr^r_r`rarbrc __class__s rrhz_EmformerAttention.__init__YsB  y A % %e9eeXaeeeff f"" &($.8TA % 1y=t T T!HOOIytOLL 94HH  [ HM ) )$*?*FM] ) ^ ^ ^ HM ) )$*;*BIY ) Z Z Z Z Z [ [rinputrrc|j\}}}|ddz}|d||z }tj||g}||dd\}} || fS)Nrr rFchunksrT)r r!rrWrkchunk) rqrsrr"r:summary_lengthright_ctx_utterance_blockmems_right_ctx_utterance_blockkeyvalues r_gen_key_valuez!_EmformerAttention._gen_key_valuews+1a1)$)*>A,>*>$?!).D:S3T)U)U&**+IJJPPXY_`Paa UEzrattention_weightsattention_maskrc|}||d|j}|d}|d|jz}||||j|d}||ddtj |j}|||jz|d}tj j |d |}tj j |t|j|jS)Nrr rFrS)ptraining)float masked_fillrrcr!r_viewtorboolr- functionalsoftmaxtype_asr`r)rqr~rrattention_weights_floatr"r#attention_probss r_gen_attention_probsz'_EmformerAttention._gen_attention_probssY #4"9"9";";"9"E"EnF^F^_`FaFacgct"u"u  " "1 % %  " "1 % % 7  #&=&B&B1dnVWY[&\&\ #&=&I&I&&q))33A6699%*EEtGX'' #'>&B&B1t~CUWXZ\&]&] #(-556MSU5VV^^_pqqx"**?eDL>Q>Q\`\i*jjjrrrrrrleft_context_valc L|d|d|dz|dz} tj|||g} tj|||gdd\} } ||| tj|z |dz } tj| d|d| z|| |d| zdg} tj| d|d| z|| |d| zdg} fd| | | fD\}}}tj|j z| dd}t||||||} |||}tj||}|j jz| jjzfkrt!d| dd| j}|}|d}|d| |z }|| |z d}jrtj|}ntj|dd }||| | fS) Nr rrFrucg|]T}|djzjjzddUS)rrr ) contiguousrr_r^ transpose)r9tensorr#rqs rr;z4_EmformerAttention._forward_impl..si8 8 8       $ $RT^);T^t~=] ^ ^ h hijlm n n8 8 8 rz+Computed attention has incorrect dimensionsi )minr)r!rlrrWrkrwrrbmmrirr'rr r_r^AssertionErrorrrrmrbtanhclamp)rqrrrrrrrrr"queryr{r|r$reshaped_query reshaped_keyreshaped_valuer~rr attentionoutput_right_context_memsrxoutput_right_context output_memsr#s` @r _forward_implz _EmformerAttention._forward_implsL NN1     q ! !INN1$5$5 5 Q G!!%)]Iw,O"P"PQQ**59dM95U+V+VWW]]eflm]nn U  ',<,H*+ei.@.@.D.D.F.F*FVW*X ')D$))A,,)DDDE$ ! 'BBDDECIFDIIaLL+FFFG$$))A,,)DDFFGE8 8 8 8 8  #u-8 8 8 4 n"Int|&C\E[E[\]_`EaEabb)M7GUY[kll 334E~WcddIo~>> ?   Ndn ,   !!NOO O''1--88::??1dnUU %)MM)$<$<! a89M1~;M9MN/N0B0D0DE   D*[11KK+ksCCCK#[#u<TYYq\\>)**.*<*<      --+= + + ' S%    ! }11!44466 7 $))A,,!3!3A!6!6688 9   r)r\NFr])NN)__name__ __module__ __qualname____doc__rrrrrhrTensorrr}rrrjitexportr __classcell__rrs@rr[r[Ls   ,0!"[[[[ [ #5/ [  [[[[[[[<EL u|]b]iOiIjk <k ku|, k  kkkk64837G=G=<G=G=| G=  G= l G= G=#5<0G=#5<0G= u|U\5<E FG=G=G=G=R#(<#(#(| #(  #( l #( #( u|U\) *#(#(#(#(J Y; <; ; | ;  ; l ;  ,;  ,;  u|U\5<E F; ; ; ; ; ; ; ; rr[ceZdZdZ d+ded ed ed ed ed edededeededeffd Z dedee j de e j fdZde e j dee j e j e j ffdZde j de j dede j de e j de e j f dZde j d e j d!e j de j fd"Zd e j d!e j dee j e j ffd#Zde j d e j d!e j dee j e j ffd$Zd e j d%e j d!e j de j d&ee j dee j e j ff d'Zd e j d%e j d!e j de j dee e j dee j e j e e j ff d(Zd e j d%e j d!e j de j d&e j dee j e j e j ff d)Ze jjd e j d%e j d!e j dee e j de j dee j e j e e j e j ff d*ZxZS),_EmformerLayera$Emformer layer that constitutes Emformer. Args: input_dim (int): input dimension. num_heads (int): number of attention heads. ffn_dim: (int): hidden layer dimension of feedforward network. segment_length (int): length of each input segment. dropout (float, optional): dropout probability. (Default: 0.0) activation (str, optional): activation function to use in feedforward network. Must be one of ("relu", "gelu", "silu"). (Default: "relu") left_context_length (int, optional): length of left context. (Default: 0) max_memory_size (int, optional): maximum number of memory elements to use. (Default: 0) weight_init_gain (float or None, optional): scale factor to apply when initializing attention module parameters. (Default: ``None``) tanh_on_mem (bool, optional): if ``True``, applies tanh to memory elements. (Default: ``False``) negative_inf (float, optional): value to use for negative infinity in attention weights. (Default: -1e8) r\r*rNFr]r^r_ffn_dimsegment_lengthr`r(left_context_lengthmax_memory_sizerarbrcc tt|||| | | |_tj||_tj||d|_ t|} tj tj |tj ||| tj|tj ||tj||_tj ||_tj ||_||_||_||_||_|dk|_dS)N)r^r_r`rarbrcT kernel_sizestride ceil_moder)rgrhr[rrr-Dropoutr` AvgPool1d memory_opr2 Sequential LayerNormrjpos_fflayer_norm_inputlayer_norm_outputrrrr^use_mem)rqr^r_rrr`r(rrrarbrcactivation_modulerrs rrhz_EmformerLayer.__init__Rse +-#%    x''00 ++~im+nn2:>>h)) H  y ) ) HOOIw / /  H  W % % HOOGY / / H  W % %    !& 2 29 = =!&!3!3I!>!>#6 ,."&* rrr rctj|j||j|}tj|j||j|}tj|j||j|}tjd|tj|}||||gS)NrNr r)rrPrr^rint32)rqrr empty_memoryrr past_lengths r _init_statez_EmformerLayer._init_states{4#7T^\bccc  ;t'?T^djkkk ;t'?T^djkkkk!Zu{6RRR .0@+NNrstatecp|ddd}t|j|}t|jt j||jz }|d|j|z d}|d|j|z d}|d|j|z d}|||fS)Nrr rF)rrrrr@ceilr)rqrrpast_left_context_lengthpast_mem_lengthpre_memslc_keylc_vals r _unpack_statez_EmformerLayer._unpack_statesAhqk!n))++ #&t'?#M#M d2DIkDL_>_4`4`aa8D0?BDDEq$25MMOOPq$25MMOOP''rnext_knext_v update_lengthrcltj|d|g}tj|d|g}tj|d|g|j d|d<||jd|jz d|d<||jd|jz d|d<|d|z|d<|S)Nr rFrr)rrWrr r)rqrrrrrnew_knew_vs r _pack_statez_EmformerLayer._pack_states 58V,-- 58V,--9eAh-..0D/D/F/FGaQ$*BBDDEaQ$*BBDDEa8m+a r rc_outputrrc||tj||gz}|||z}||}|Sr7)r`rrWrr)rqrrrresults r_process_attention_outputz(_EmformerLayer._process_attention_outputsW i((59mY5O+P+PPV$$v-''// rc|tj||g}||dd|d|dfSNr)rrrWr!)rqrrrs r_apply_pre_attention_layer_normz._EmformerLayer._apply_pre_attention_layer_normsi 00M9;U1V1VWW ]//2244 5 4}11!444 5  rc||||}||dd|d|dfSr)rr!)rqrrrs r_apply_post_attention_ffnz(_EmformerLayer._apply_post_attention_ffnsY229iWW ++A..0019=T}?Q?QRS?T?T=T3UUUrrrc\|td|jr@||dddddd}n3t jd|j|j}| ||||||\}}||fS)Nz;attention_mask must be not None when for_inference is Falser rFrr)rrrrrr) r1rrpermuteremptyrr r r) rqrrrrrrrnext_ms r_apply_attention_forwardz'_EmformerLayer._apply_attention_forwards  !Z[[ [ < XnnY%6%6q!Q%?%?@@HHAqQQGGk!nn''ioiFV'WWG NN') +   6&  rc L|/||d|j}||\}}}|jrJ||dddddd} | dd} n3tjd |j |j} |j |||| |||\} } } } | | | |d||}| | |fS)Nr rNrFrr)rrrrrrr)rr!r rrrrrrrr rrr)rqrrrrrrrrrrrrrs r_apply_attention_inferz%_EmformerLayer._apply_attention_infers/ =$$Y^^A%6%6y?O$PPE#'#5#5e#<#< && < XnnY%6%6q!Q%?%?@@HHAqQQGbqbkGGk!nn''ioiFV'WWG,0N,@,@'##-A- - ) 666  1B1BD%PP&%''rc|||\}}||||||\}} ||||\} } | | | fS)a1Forward pass for training. B: batch size; D: feature dimension of each frame; T: number of utterance frames; R: number of right context frames; M: number of memory elements. Args: utterance (torch.Tensor): utterance frames, with shape `(T, B, D)`. lengths (torch.Tensor): with shape `(B,)` and i-th element representing number of valid frames for i-th batch element in ``utterance``. right_context (torch.Tensor): right context frames, with shape `(R, B, D)`. mems (torch.Tensor): memory elements, with shape `(M, B, D)`. attention_mask (torch.Tensor): attention mask for underlying attention module. Returns: (Tensor, Tensor, Tensor): Tensor encoded utterance frames, with shape `(T, B, D)`. Tensor updated right context frames, with shape `(R, B, D)`. Tensor updated memory elements, with shape `(M, B, D)`. )rrr) rqrrrrrlayer_norm_utterancelayer_norm_right_contextrroutput_utterancers rrz_EmformerLayer.forwards}H  0 0M J J $!%!>!>  $   " "  ;261O1OPY[dfs1t1t..!5{BBrc|||\}}||||||\}} } ||||\} } | | | | fS)a2Forward pass for inference. B: batch size; D: feature dimension of each frame; T: number of utterance frames; R: number of right context frames; M: number of memory elements. Args: utterance (torch.Tensor): utterance frames, with shape `(T, B, D)`. lengths (torch.Tensor): with shape `(B,)` and i-th element representing number of valid frames for i-th batch element in ``utterance``. right_context (torch.Tensor): right context frames, with shape `(R, B, D)`. state (List[torch.Tensor] or None): list of tensors representing layer internal state generated in preceding invocation of ``infer``. mems (torch.Tensor): memory elements, with shape `(M, B, D)`. Returns: (Tensor, Tensor, List[torch.Tensor], Tensor): Tensor encoded utterance frames, with shape `(T, B, D)`. Tensor updated right context frames, with shape `(R, B, D)`. List[Tensor] list of tensors representing layer internal state generated in current invocation of ``infer``. Tensor updated memory elements, with shape `(M, B, D)`. )rrr) rqrrrrrrrrr output_staterrs rrz_EmformerLayer.infers~R  0 0M J J $/3/J/J '+CT50 0 , ; 261O1OPY[dfs1t1t..!5|[PPr)r\r*rrNFr])rrrrrrstrrrrhrr rrrrrrrrrrrrrrrrrs@rrr?sy0 #$ ,0!",+,+,+,+ ,+  ,+  ,+,+!,+,+#5/,+,+,+,+,+,+,+,+\OcO8EL3IOdSXS_N`OOOO(4 #5(% el\a\h@h:i((((    l  EL!  el  < < |        6;l u|U\) *    VV27,VOT|V u|U\) *VVVV !<!!| ! l ! !. ! u|U\) *!!!!2(<((| ( l ( U\*+ ( u|U\4 +== >((((8-C<-C-C| -C l -C  -C u|U\5<7 8-C-C-C-C^ Y-Q<-Q-Q| -Q U\*+ -Q l -Q u|U\4 +=u|K L-Q-Q-Q-Q-Q-Q-Q-QrrceZdZ ddejjdedededef fd Zdejd ejfd Z d ed ed e efd Z dejd ejfdZ dejdejd e ejejffdZejj ddejdejdee e ejd e ejeje e ejffdZxZS) _EmformerImplremformer_layersrrright_context_lengthrct|dk|_tj||d|_||_||_||_ ||_ ||_ dS)NrTr) rgrhrrr-rrrrrrr)rqrrrrrrrs rrhz_EmformerImpl.__init__Ps~ &* ++&!,   /#6 $8!,.rrsrcl|jd}tj||jz |jz }g}t |dz D]6}|dz|jz}||jz}||||7||||jz dtj|SNrr ) r r@rrrrGappendrrW)rqrsr"num_segsright_context_blocksseg_idxstartends r_gen_right_contextz _EmformerImpl._gen_right_contextfs KN9a$";;t?RRSS!X\** : :Gq[D$77E$33C ' 'eCi(8 9 9 9 9##E!d.G*G*I*I$JKKKy-...rrutterance_lengthc tj||jz }|j}|j}||z}||z}t ||jz|z d}t |dz|jz|} |j|z} |jr8t ||jz d} |dz } | || z | |z ||| |z || |z || z g } n||| |z || |z || z g} | Sr) r@rrrrrrrr)rqrrrrclcrc_startrc_end seg_startseg_end rc_lengthm_start mem_lengthrIs r_gen_attention_mask_col_widthsz,_EmformerImpl._gen_attention_mask_col_widthsqs9-0CCDD  &  %R<B$"55:A>> w{d&99;KLL-8 < 'D$88!<.sJJJ3cY.JJJrcg|]}|dvS))rrr8rs rr;z5_EmformerImpl._gen_attention_mask..sDDDS3&=DDDrcg|]}|dvS))r rr8rs rr;z5_EmformerImpl._gen_attention_mask..sGGGcVmGGGrr c6g|]}tj|Sr8)rrW)r9masks rr;z5_EmformerImpl._gen_attention_mask..s (U(U(UT4(U(U(Ur)r!r@rrrrGrrYrr rrrrWrr)rqrsrrrc_mask query_mask summary_masknum_colsrc_q_cols_mask s_cols_maskmasks_to_concatrrI rc_mask_blockquery_mask_blocksummary_mask_blockrs r_gen_attention_maskz!_EmformerImpl._gen_attention_masks ::a==9-0CCDD  < 4HJJ%//JJJNDDE(OODDDK& LAOOHGGuXGGGNK& 3OX 8 8G<z;XY[`[g%h%h"##$6777ei(U(U_(U(U(UVVVZZ[`[effrrc@|ddd}||}|d|d|jz }||}|jrG||ddddddddn2tjd |j |j }|}|j D]}||||||\}}}|ddd|fS)aGForward pass for training and non-streaming inference. B: batch size; T: max number of input frames in batch; D: feature dimension of each frame. Args: input (torch.Tensor): utterance frames right-padded with right context frames, with shape `(B, T + right_context_length, D)`. lengths (torch.Tensor): with shape `(B,)` and i-th element representing number of valid utterance frames for i-th batch element in ``input``. Returns: (Tensor, Tensor): Tensor output frames, with shape `(B, T, D)`. Tensor output lengths, with shape `(B,)` and i-th element representing number of valid frames for i-th batch element in output frames. r rrFNrr) rrr!rr'rrrrrr r r) rqrsrrrrrrlayers rrz_EmformerImpl.forwards+* aA&&//66 EEJJqMMD,EEEF 11)<<| KDNN9,,Q155 6 6 > >q!Q G G L LQ""U\"JJ ) f fE*/%PTVd*e*e 'FM44~~aA&&//rNstatescb|d|j|jzkr6td|j|jzd|dd|ddd}|d|jz }||d}|d|}t j||jz d}|jr?||ddddddn2t j d |j |j }|} g} t|jD]B\} } | | |||dn|| |\} }} }| | C| ddd|| fS) aForward pass for streaming inference. B: batch size; D: feature dimension of each frame. Args: input (torch.Tensor): utterance frames right-padded with right context frames, with shape `(B, segment_length + right_context_length, D)`. lengths (torch.Tensor): with shape `(B,)` and i-th element representing number of valid frames for i-th batch element in ``input``. states (List[List[torch.Tensor]] or None, optional): list of lists of tensors representing internal state generated in preceding invocation of ``infer``. (Default: ``None``) Returns: (Tensor, Tensor, List[List[Tensor]]): Tensor output frames, with shape `(B, segment_length, D)`. Tensor output lengths, with shape `(B,)` and i-th element representing number of valid frames for i-th batch element in output frames. List[List[Tensor]] output states; list of lists of tensors representing internal state generated in current invocation of ``infer``. r zIPer configured segment_length and right_context_length, expected size of z# for dimension 1 of input, but got .rrFN)rr)r!rrr1rrrrrrrr r enumeraterrr)rqrsrr*right_context_start_idxrroutput_lengthsrr output_statesrCr)rs rrz_EmformerImpl.infers> ::a==D/$2KK K K.&*&9D >q!Q G G GQ""U\"JJ 24 )$*> ? ? / / Iu8= F9,= 99 5FM<   . . . .~~aA&& EEr)rrrr7)rrrrr- ModuleListrrhrrrrr'rrrrrrrrs@rrrOs $%$% //,//! / " /  //////, / / / / / /"c"S"UYZ]U^""""H..%,....`!0U\!0EL!0U5