Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ This repository undertakes a reverse engineering of the S3Tokenizer, offering:
- [x] Model: [S3Tokenizer V1 50hz](https://modelscope.cn/models/iic/CosyVoice-300M)
- [x] Model: [S3Tokenizer V1 25hz](https://modelscope.cn/models/iic/CosyVoice-300M-25Hz)
- [x] Model: [S3Tokenizer V2 25hz](https://modelscope.cn/models/iic/CosyVoice2-0.5B)
- [x] Model: [S3Tokenizer V3 25hz](https://modelscope.cn/models/FunAudioLLM/Fun-CosyVoice3-0.5B-2512)


# Setup
Expand All @@ -34,7 +35,7 @@ pip install s3tokenizer
```py
import s3tokenizer

tokenizer = s3tokenizer.load_model("speech_tokenizer_v1").cuda() # or "speech_tokenizer_v1_25hz speech_tokenizer_v2_25hz"
tokenizer = s3tokenizer.load_model("speech_tokenizer_v1").cuda() # or "speech_tokenizer_v1_25hz speech_tokenizer_v2_25hz speech_tokenizer_v3_25hz"

mels = []
wav_paths = ["s3tokenizer/assets/BAC009S0764W0121.wav", "s3tokenizer/assets/BAC009S0764W0122.wav"]
Expand All @@ -57,7 +58,7 @@ s3tokenizer --wav_scp xxx.scp \
--device "cpu" \
--output_dir "./" \
--batch_size 32 \
--model "speech_tokenizer_v1" # or "speech_tokenizer_v1_25hz speech_tokenizer_v2_25hz"
--model "speech_tokenizer_v1" # or "speech_tokenizer_v1_25hz speech_tokenizer_v2_25hz speech_tokenizer_v3_25hz"
```


Expand All @@ -75,7 +76,7 @@ torchrun --nproc_per_node=8 --nnodes=1 \
--device "cuda" \
--output_dir "./" \
--batch_size 32 \
--model "speech_tokenizer_v1" # or "speech_tokenizer_v1_25hz speech_tokenizer_v2_25hz"
--model "speech_tokenizer_v1" # or "speech_tokenizer_v1_25hz speech_tokenizer_v2_25hz speech_tokenizer_v3_25hz"
```


Expand Down
15 changes: 12 additions & 3 deletions s3tokenizer/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,16 @@
from tqdm import tqdm

from s3tokenizer.model_v2 import S3TokenizerV2
from s3tokenizer.model_v3 import S3TokenizerV3

from .model import S3Tokenizer
from .utils import (load_audio, log_mel_spectrogram, make_non_pad_mask,
mask_to_bias, onnx2torch, padding, merge_tokenized_segments)
mask_to_bias, merge_tokenized_segments, onnx2torch,
onnx2torch_v3, padding)

__all__ = [
'load_audio', 'log_mel_spectrogram', 'make_non_pad_mask', 'mask_to_bias',
'onnx2torch', 'padding', 'merge_tokenized_segments'
'onnx2torch', 'onnx2torch_v3', 'padding', 'merge_tokenized_segments'
]
_MODELS = {
"speech_tokenizer_v1":
Expand All @@ -44,6 +46,9 @@
"speech_tokenizer_v2_25hz":
"https://www.modelscope.cn/models/iic/CosyVoice2-0.5B/"
"resolve/master/speech_tokenizer_v2.onnx",
"speech_tokenizer_v3_25hz":
"https://www.modelscope.cn/models/FunAudioLLM/Fun-CosyVoice3-0.5B-2512/"
"resolve/master/speech_tokenizer_v3.onnx",
}

_SHA256S = {
Expand All @@ -53,6 +58,8 @@
"56285ddd4a83e883ee0cb9f8d69c1089b53a94b1f78ff7e4a0224a27eb4cb486",
"speech_tokenizer_v2_25hz":
"d43342aa12163a80bf07bffb94c9de2e120a8df2f9917cd2f642e7f4219c6f71",
"speech_tokenizer_v3_25hz":
"23236a74175dbdda47afc66dbadd5bcb41303c467a57c261cb8539ad9db9208d",
}


Expand Down Expand Up @@ -144,7 +151,9 @@ def load_model(
else:
raise RuntimeError(
f"Model {name} not found; available models = {available_models()}")
if 'v2' in name:
if 'v3' in name:
model = S3TokenizerV3(name)
elif 'v2' in name:
model = S3TokenizerV2(name)
else:
model = S3Tokenizer(name)
Expand Down
8 changes: 7 additions & 1 deletion s3tokenizer/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,13 @@ class ModelConfig:
class LayerNorm(nn.LayerNorm):

def forward(self, x: Tensor) -> Tensor:
return super().forward(x.float()).type(x.dtype)
return F.layer_norm(
x.float(),
self.normalized_shape,
self.weight.float() if self.weight is not None else None,
self.bias.float() if self.bias is not None else None,
self.eps,
).type(x.dtype)


class Linear(nn.Linear):
Expand Down
4 changes: 2 additions & 2 deletions s3tokenizer/model_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@ def apply_rotary_emb(
) -> Tuple[torch.Tensor, torch.Tensor]:
real = torch.view_as_real(freqs_cis)
cos, sin = real[:, :, 0], real[:, :, 1]
cos = cos.unsqueeze(0).unsqueeze(2)
sin = sin.unsqueeze(0).unsqueeze(2)
cos = cos.unsqueeze(0).unsqueeze(2).to(xq.dtype)
sin = sin.unsqueeze(0).unsqueeze(2).to(xq.dtype)

D = xq.shape[-1]
half_l, half_r = xq[:, :, :, :D // 2], xq[:, :, :, D // 2:]
Expand Down
Loading
Loading