0;jiP dZddlZddlmZddlmZmZmZddlZ ddl Z ddl Z ddl m cmZddlZddlmZdefdZd$d ed ed efd Z d$d ed ed efd Zd%dedefdZeddede jfdZ d&deee je jfdededeeee jffdZd'de jdede jfdZ de jde j!de jfd Z"d!ee jfd"Z#d#Z$dS)(aModified from https://github.com/openai/whisper/blob/main/whisper/audio.py Add rename_weights() & onnx2torch() & make_non_pad_mask() & mask_to_bias() Copy merge_tokenized_segments() from https://github.com/Mddct/s3tokenizer-long/blob/main/example.py N) lru_cache)ListOptionalUnion) pad_sequence weights_dictci}|D]}d|vr"|dkr |||d<d|vr ||||<(d|vr ||||<8d|vr ||||<Hd|vsJ|dd d d d d dddd dddddd}|||d|<|S)z Rename onnx weights to pytorch format. Parameters ---------- weight_dict: dict The dict containing weights in onnx format Returns ------- A new weight dict containing the weights in pytorch format. quantizerz,/quantizer/rq/model/layers.0/_codebook/Pow_1zquantizer._codebook.embed project_downpositional_embeddingconvblocksN/.MatMulweightAdd_1biasMulAddzmlp.mlpmlpzfsmn_block.Convfsmn_block.weightencoder.)keysreplace)rnew_weight_dictknew_ks K/root/voice-cloning/.venv/lib/python3.11/site-packages/s3tokenizer/utils.py_rename_weightsr! s\O    BB !  BBB?KA ;<<1$$%1!_" #q ( (!-aOA   q[[!-aOA  q====qrrU]]3,,44($$$+GGV$<$%>%%,WUF%;%;GG!5=*=*+273D3F,H,H  3?q/O.u.. / / F onnx_path torch_pathverbosecdtj|}i}d|jjD}|jjD]}|jD]}||vrd\}} ||} |dvrd} nh|dvrd} na|dvrd} nZ|d vrd } nS|d krd } nJ|d krd} nA|dkrd} n8|jdkr&|jdd} | dz} | dz}n|j} | ||j} | d}| d}||vr7tj || nd}||vr7tj || nd}d|j _ d|j _ tj|}tj|}|||<||| <etj |  }d|j _ tj|}t!|jdks| dvr||| <||| <t'|}|rM|D]&\}}t+|d|jd|j't+d|~~|rtj||dS|S)a Open an onnx file and convert to pytorch format. Parameters ---------- onnx_path: str The onnx file to open, typically `speech_tokenizer_v1.onnx` torch_path: str The path to save the torch-formated checkpoint. verbose: bool Logging info or not. Returns ------- A checkpoint dict containing the weights and their names, if torch_path is None. Otherwise save checkpoint dict to the desired path. ci|] }|j| Sname.0 initializers r zonnx2torch..Z-  +r")NN)zonnx::Conv_1519zencoders.conv1.weightzonnx::Conv_2216encoder.conv1.weight)zonnx::Conv_1520zencoders.conv1.biaszonnx::Conv_2217encoder.conv1.bias)zonnx::Conv_1521zencoders.conv2.weightzonnx::Conv_2218encoder.conv2.weight)zonnx::Conv_1522zencoders.conv2.biaszonnx::Conv_2219encoder.conv2.biaszencoders.positional_embeddingencoder.positional_embeddingquantizer.project_in.bias%quantizer._codebook.project_down.biaszonnx::MatMul_2536'quantizer._codebook.project_down.weightLayerNormalizationz/LayerNormalizationz.weightz.biasNrT)r4 :  PyTorch weights saved to )onnxloadgraphr-nodeinputop_typer*r numpy_helperto_arraycopyflags writeabletorch from_numpylenshapetr!itemsprintdtypesave)r#r$r% onnx_modelrinitializer_maprA input_name ln_bias_nameln_weight_namer- weight_nameln_name ln_inputs scale_name bias_namescaler weight_tensor bias_tensor weight_arraynew_weights_dictrvs r onnx2torchrbDs(9%%JL%+7O %EFEF*D FD FJ_,,/9, n-j9 " #9KK$ #7KK$ #9KK$ #7KK#BBB"@KK#>>>"IKK#666"KKK|';;;"&)"3"34I2"N"N)09)<'.'8 &*i !-,2J $ I!*1J )! I(?::!-66' 35559T666@D '/99 ,55' 24448D555?C-1EK)+/DJ($)$4U$;$;M"'"24"8"8K1|jvr$d?|jvr|j d d !|d@<0dA|D}|rO|D](\}}|!t%|dB|jdC|j)t%dD|~|rt+j||dS|S)Fz, Convert V3 ONNX to PyTorch format. ci|] }|j| Sr(r)r+s r r.z!onnx2torch_v3..r/r"ConstantvaluerFc,|vr8tj|}n!|vr|}ndSt j|}|r|jdkr|}|S)Nr:)r>rDrErFrIrJndimrM)r* transposearrrM constant_maprSs r get_tensorz!onnx2torch_v3..get_tensors ? " "#,,_T-BCCHHJJCC \ ! !t$))++CC4  S ! !  1Ar"c>|jD]}|}||cSdS)zgHelper to find bias tensor for an Add node. Checks both inputs to see which one is a parameter.N)rB)rAinprMrls r get_bias_tensorz&onnx2torch_v3..get_bias_tensors;:  C 3A}tr"z /conv1/Convrr0r:r1z /conv2/Convr2r3z/blocks.rrzencoder.blocks.z attn_ln/Mulrz.attn_ln.weightz attn_ln/AddrNz .attn_ln.biasz mlp_ln/Mulz.mlp_ln.weightz mlp_ln/Addz .mlp_ln.biaszattn/query/MatMulT)riz.attn.query.weightzattn/query/Addz.attn.query.biaszattn/key/MatMulz.attn.key.weightz attn/key/Addz.attn.key.biaszattn/value/MatMulz.attn.value.weightzattn/value/Addz.attn.value.biaszattn/out/MatMulz.attn.out.weightz attn/out/Addz.attn.out.biaszmlp/mlp.0/MatMulz .mlp.0.weightz mlp/mlp.0/Addz .mlp.0.biaszmlp/mlp.2/MatMulz .mlp.2.weightz mlp/mlp.2/Addz .mlp.2.biaszfsmn_block/Convrrr5r6r rr7ci|] \}}||| S)Nr()r,rras r r.z!onnx2torch_v3..GsKKKTQQ]Aq]]]r"r;r<r=)F)r>r?r@r-rArC attributer*rDrErMoutputrBrK startswithsplitnumelrNrOrLrPrIrQ)r#r$r%rRrrAattrror*opinputsparts block_part block_idxprefixrM init_namerrarkrlrSs @@@r onnx2torch_v3r~s 9%%JL%+7O L %   <: % %  9''373D3M3M4 4 LQ0        %TTy \ = 3=:fQi3H3HL/ 06{{Q5?Zq 5J5J 12 ] " "3=:fQi3H3HL/ 06{{Q5?Zq 5J5J 12__Z ( (@ MJJsOOEqJ"((--a0I2y22F$$u;E:1I<< 77788$&&2;;#OD))=QWWYY]]=>LF!9!9!9:%%"++:D*VAY:O:O 66677%%"++#OD))=QWWYY]]<=LF!8!8!89%,,>Hj1I?/?/?/ :::;;!T))Hj1I?/?/?/ :::;;!T))HZJqMT>3>3>3 9 ; LK\%7%7%9%9KKKL8 &&(( 4 4DAq}22qw2222333 6*66777 <,,,,,r">filesrctj|\}}||kr(tj|||}|d}|S)a@ Open an audio file and read as mono waveform, resampling as necessary Parameters ---------- file: str The audio file to open sr: int The sample rate to resample the audio if necessary Returns ------- A torch.Tensor containing the audio waveform, in float32 dtype. r) torchaudior? transformsResample)rraudio sample_rates r load_audiorVsQ $..E;b%..{B??FF !HE Lr")maxsizen_melsreturncj|dvs Jd|tjtjtdd}t j|d5}tj|d| |cdddS#1swxYwYdS) ad load the mel filterbank matrix for projecting STFT into a Mel spectrogram. Allows decoupling librosa dependency; saved using: np.savez_compressed( "mel_filters.npz", mel_80=librosa.filters.mel(sr=16000, n_fft=400, n_mels=80), mel_128=librosa.filters.mel(sr=16000, n_fft=400, n_mels=128), ) >PzUnsupported n_mels: assetszmel_filters.npzF) allow_picklemel_N) ospathjoindirname__file__npr?rIrJto)devicer filters_pathfs r _mel_filtersrms Y    ?v ? ?   7<< 9 98 133L E 2 2 2?a/// 23366v>>??????????????????s+0B((B,/B,rrpaddingrctj|s$t|trt |}|||}|dkrt j|d|f}tjd|j }tj |dd|d}|dddf d z}t|j |}||z}tj |d } tj| | d z } | d zd z } | S)ay Compute the log-Mel spectrogram of Parameters ---------- audio: Union[str, np.ndarray, torch.Tensor], shape = (*) The path to audio or either a NumPy array or Tensor containing the audio waveform in 16 kHz n_mels: int The number of Mel-frequency filters, only 80 is supported padding: int Number of zero samples to pad to the right device: Optional[Union[str, torch.device]] If given, the audio tensor is moved to this device before STFT Returns ------- torch.Tensor, shape = (128, n_frames) A Tensor that contains the Mel spectrogram NriT)windowreturn_complex.r:g|=)ming @g@)rI is_tensor isinstancestrrrFpad hann_windowrstftabsrclamplog10maximummax) rrrrrr magnitudesfiltersmel_speclog_specs r log_mel_spectrogramrs2: ?5 ! !& eS ! ! &u%%E   {{ea\**  s # # & &u| 4 4F :eS#fT J J JDc3B3h##%%q(J5<00G#H{8///5577H}Xx||~~';<>> import torch >>> import s3tokenizer >>> lengths = torch.tensor([5, 3, 2]) >>> masks = s3tokenizer.make_non_pad_mask(lengths) masks = [[1, 1, 1, 1, 1], [1, 1, 1, 0, 0], [1, 1, 0, 0, 0]] r)rPrr) sizeritemrIarangeint64r unsqueezeexpand)rr batch_size seq_rangeseq_range_expandseq_length_expandmasks r make_non_pad_maskrs8aJ 1gg'++--*<*<*>*>G Q$#(;$+N444I!**1--44ZII))"-- 0 0D 5Lr"rrPc|jtjksJ|tjtjtjfvsJ||}d|z dz}|S)a\Convert bool-tensor to float-tensor for flash attention. Parameters ---------- lengths (torch.Tensor): Batch of lengths (B, ?). Returns: ------- torch.Tensor: Mask tensor containing indices of padded part (B, ?). Examples: >>> import torch >>> import s3tokenizer >>> lengths = torch.tensor([5, 3, 2]) >>> masks = s3tokenizer.make_non_pad_mask(lengths) masks = [[1, 1, 1, 1, 1], [1, 1, 1, 0, 0], [1, 1, 0, 0, 0]] >>> new_masks = s3tokenizer.mask_to_bias(masks, torch.float32) new_masks = [[-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00], [-0.0000e+00, -0.0000e+00, -0.0000e+00, -1.0000e+10, -1.0000e+10], [-0.0000e+00, -0.0000e+00, -1.0000e+10, -1.0000e+10, -1.0000e+10]] g?g _)rPrIboolfloat32bfloat16float16r)rrPs r mask_to_biasrs[2 : # # # # U]ENEMB B B B B 775>>D $J( "D Kr"datac|}t|tsJtjd|Dtj}d|D}t |dd}|dd|fS) z Padding the data into batch data Parameters ---------- data: List[Tensor], shape of Tensor (128, T) Returns: ------- feats [B, 128, T_max], feats lengths [B] c8g|]}|dS)r)rr,ss r zpadding.. s"!.s # # #qQSSUU # # #r"Tr) batch_first padding_valuerr:)rlistrItensorint32rri)rsample feats_lengthsfeats padded_featss r rrsF fd # ### #L!rItorch.nn.functionalnn functionalrrtorch.nn.utils.rnnrdictr!rrrbr~intrTensorrndarrayrrrrPrrrr(r"r rs  (((((((((( ++++++!$!!!!Hj j #j 3j j j j j \%)"'bbSb!bbbbbJSc. 4??????*15 // bj%,. // // U3 ,- . ////d%%u|%c%%,%%%%P!u|!EK!EL!!!!H7$u|$7777*r"