Fangjun Kuang
Committed by GitHub

Add JavaScript API (node-addon) for KittenTTS (#2470)

... ... @@ -10,6 +10,14 @@ arch=$(node -p "require('os').arch()")
platform=$(node -p "require('os').platform()")
node_version=$(node -p "process.versions.node.split('.')[0]")
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2
tar xf kitten-nano-en-v0_1-fp16.tar.bz2
rm kitten-nano-en-v0_1-fp16.tar.bz2
node ./test_tts_non_streaming_kitten_en.js
rm -rf kitten-nano-en-v0_1-fp16
echo "----------non-streaming ASR NeMo Canary----------"
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2
... ...
... ... @@ -37,9 +37,9 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [macos-latest, ubuntu-latest, ubuntu-latest]
os: [macos-latest, ubuntu-latest]
node-version: ["16", "22"]
python-version: ["3.8"]
python-version: ["3.10"]
steps:
- uses: actions/checkout@v4
... ...
... ... @@ -36,7 +36,8 @@ export { OnlineStream,
OnlineRecognizer,
} from './src/main/ets/components/StreamingAsr';
export { OfflineTtsKokoroModelConfig,
export { OfflineTtsKittenModelConfig,
OfflineTtsKokoroModelConfig,
OfflineTtsMatchaModelConfig,
OfflineTtsVitsModelConfig,
OfflineTtsModelConfig,
... ...
... ... @@ -75,6 +75,25 @@ static SherpaOnnxOfflineTtsKokoroModelConfig GetOfflineTtsKokoroModelConfig(
return c;
}
static SherpaOnnxOfflineTtsKittenModelConfig GetOfflineTtsKittenModelConfig(
Napi::Object obj) {
SherpaOnnxOfflineTtsKittenModelConfig c;
memset(&c, 0, sizeof(c));
if (!obj.Has("kitten") || !obj.Get("kitten").IsObject()) {
return c;
}
Napi::Object o = obj.Get("kitten").As<Napi::Object>();
SHERPA_ONNX_ASSIGN_ATTR_STR(model, model);
SHERPA_ONNX_ASSIGN_ATTR_STR(voices, voices);
SHERPA_ONNX_ASSIGN_ATTR_STR(tokens, tokens);
SHERPA_ONNX_ASSIGN_ATTR_STR(data_dir, dataDir);
SHERPA_ONNX_ASSIGN_ATTR_FLOAT(length_scale, lengthScale);
return c;
}
static SherpaOnnxOfflineTtsModelConfig GetOfflineTtsModelConfig(
Napi::Object obj) {
SherpaOnnxOfflineTtsModelConfig c;
... ... @@ -89,6 +108,7 @@ static SherpaOnnxOfflineTtsModelConfig GetOfflineTtsModelConfig(
c.vits = GetOfflineTtsVitsModelConfig(o);
c.matcha = GetOfflineTtsMatchaModelConfig(o);
c.kokoro = GetOfflineTtsKokoroModelConfig(o);
c.kitten = GetOfflineTtsKittenModelConfig(o);
SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads);
... ... @@ -172,6 +192,11 @@ static Napi::External<SherpaOnnxOfflineTts> CreateOfflineTtsWrapper(
SHERPA_ONNX_DELETE_C_STR(c.model.matcha.data_dir);
SHERPA_ONNX_DELETE_C_STR(c.model.matcha.dict_dir);
SHERPA_ONNX_DELETE_C_STR(c.model.kitten.model);
SHERPA_ONNX_DELETE_C_STR(c.model.kitten.voices);
SHERPA_ONNX_DELETE_C_STR(c.model.kitten.tokens);
SHERPA_ONNX_DELETE_C_STR(c.model.kitten.data_dir);
SHERPA_ONNX_DELETE_C_STR(c.model.kokoro.model);
SHERPA_ONNX_DELETE_C_STR(c.model.kokoro.voices);
SHERPA_ONNX_DELETE_C_STR(c.model.kokoro.tokens);
... ...
... ... @@ -39,10 +39,19 @@ export class OfflineTtsKokoroModelConfig {
public lang: string = '';
}
export class OfflineTtsKittenModelConfig {
public model: string = '';
public voices: string = '';
public tokens: string = '';
public dataDir: string = '';
public lengthScale: number = 1.0;
}
export class OfflineTtsModelConfig {
public vits: OfflineTtsVitsModelConfig = new OfflineTtsVitsModelConfig();
public matcha: OfflineTtsMatchaModelConfig = new OfflineTtsMatchaModelConfig();
public kokoro: OfflineTtsKokoroModelConfig = new OfflineTtsKokoroModelConfig();
public kitten: OfflineTtsKittenModelConfig = new OfflineTtsKittenModelConfig();
public numThreads: number = 1;
public debug: boolean = false;
public provider: string = 'cpu';
... ...
... ... @@ -147,6 +147,7 @@ The following tables list the examples in this folder.
|File| Description|
|---|---|
|[./test_tts_non_streaming_kitten_en.js](./test_tts_non_streaming_kitten_en.js)| Text-to-speech with a KittenTTS English Model|
|[./test_tts_non_streaming_kokoro_en.js](./test_tts_non_streaming_kokoro_en.js)| Text-to-speech with a Kokoro English Model|
|[./test_tts_non_streaming_kokoro_zh_en.js](./test_tts_non_streaming_kokoro_zh_en.js)| Text-to-speech with a Kokoro Model supporting Chinese and English|
|[./test_tts_non_streaming_matcha_icefall_en.js](./test_tts_non_streaming_matcha_icefall_en.js)| Text-to-speech with a [MatchaTTS English Model](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker)|
... ... @@ -458,6 +459,16 @@ npm install naudiodon2
node ./test_vad_asr_non_streaming_sense_voice_microphone.js
```
### Text-to-speech with KittenTTS models (English TTS)
```bash
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2
tar xf kitten-nano-en-v0_1-fp16.tar.bz2
rm kitten-nano-en-v0_1-fp16.tar.bz2
node ./test_tts_non_streaming_kitten_en.js
```
### Text-to-speech with Kokoro TTS models (Chinese + English TTS)
```bash
... ...
// Copyright (c) 2025 Xiaomi Corporation
const sherpa_onnx = require('sherpa-onnx-node');
// please refer to
// https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kitten.html
// to download model files
function createOfflineTts() {
const config = {
model: {
kitten: {
model: './kitten-nano-en-v0_1-fp16/model.fp16.onnx',
voices: './kitten-nano-en-v0_1-fp16/voices.bin',
tokens: './kitten-nano-en-v0_1-fp16/tokens.txt',
dataDir: './kitten-nano-en-v0_1-fp16/espeak-ng-data',
},
debug: true,
numThreads: 1,
provider: 'cpu',
},
maxNumSentences: 1,
};
return new sherpa_onnx.OfflineTts(config);
}
const tts = createOfflineTts();
const text =
'Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.'
let start = Date.now();
const audio = tts.generate({text: text, sid: 6, speed: 1.0});
let stop = Date.now();
const elapsed_seconds = (stop - start) / 1000;
const duration = audio.samples.length / audio.sampleRate;
const real_time_factor = elapsed_seconds / duration;
console.log('Wave duration', duration.toFixed(3), 'seconds')
console.log('Elapsed', elapsed_seconds.toFixed(3), 'seconds')
console.log(
`RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`,
real_time_factor.toFixed(3))
const filename = 'test-kitten-en-6.wav';
sherpa_onnx.writeWave(
filename, {samples: audio.samples, sampleRate: audio.sampleRate});
console.log(`Saved to ${filename}`);
... ...