Add keyword spotting API for node-addon-api (#877)

@@ -18,7 +18,7 @@ fi
 SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt  | cut -d " " -f 2  | cut -d '"' -f 2)
 echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION"
 
-# SHERPA_ONNX_VERSION=1.0.23
+# SHERPA_ONNX_VERSION=1.0.24
 
 if [ -z $owner ]; then
   owner=k2-fsa

@@ -6,6 +6,15 @@ d=nodejs-addon-examples
 echo "dir: $d"
 cd $d
 
+echo "----------keyword spotting----------"
+
+curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/kws-models/sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.tar.bz2
+tar xvf sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.tar.bz2
+rm sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.tar.bz2
+
+node ./test_keyword_spotter_transducer.js
+rm -rf sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01
+
 echo "----------add punctuations----------"
 
 curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2

@@ -55,7 +55,7 @@ jobs:
 
           SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt  | cut -d " " -f 2  | cut -d '"' -f 2)
           echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION"
-          # SHERPA_ONNX_VERSION=1.0.23
+          # SHERPA_ONNX_VERSION=1.0.24
 
           src_dir=.github/scripts/node-addon
           sed -i.bak s/SHERPA_ONNX_VERSION/$SHERPA_ONNX_VERSION/g $src_dir/package.json

@@ -62,6 +62,13 @@ The following tables list the examples in this folder.
 |[./test_audio_tagging_zipformer.js](./test_audio_tagging_zipformer.js)| Audio tagging with a Zipformer model|
 |[./test_audio_tagging_ced.js](./test_audio_tagging_ced.js)| Audio tagging with a [CED](https://github.com/RicherMans/CED) model|
 
+## Keyword spotting
+
+|File| Description|
+|---|---|
+|[./test_keyword_spotter_transducer.js](./test_keyword_spotter_transducer.js)| Keyword spotting from a file using a Zipformer model|
+|[./test_keyword_spotter_transducer_microphone.js](./test_keyword_spotter_transducer_microphone.js)| Keyword spotting from a microphone using a Zipformer model|
+
 ## Streaming speech-to-text from files
 
 |File| Description|
@@ -325,3 +332,17 @@ rm sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2
 
 node ./test_punctuation.js
 ```
+
+## Keyword spotting
+
+```bash
+wget https://github.com/k2-fsa/sherpa-onnx/releases/download/kws-models/sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.tar.bz2
+tar xvf sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.tar.bz2
+rm sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.tar.bz2
+
+node ./test_keyword_spotter_transducer.js
+
+# To run keyword spotting using a microphone
+npm install naudiodon2
+node ./test_keyword_spotter_transducer_microphone.js
+```

@@ -79,11 +79,5 @@ ai.on('data', data => {
   }
 });
 
-ai.on('close', () => {
-  console.log('Free resources');
-  stream.free();
-  recognizer.free();
-});
-
 ai.start();
 console.log('Started! Please speak')

@@ -78,11 +78,6 @@ ai.on('data', data => {
   }
 });
 
-ai.on('close', () => {
-  console.log('Free resources');
-  stream.free();
-  recognizer.free();
-});
 
 ai.start();
 console.log('Started! Please speak')

@@ -94,11 +94,5 @@ ai.on('data', data => {
   }
 });
 
-ai.on('close', () => {
-  console.log('Free resources');
-  stream.free();
-  recognizer.free();
-});
-
 ai.start();
 console.log('Started! Please speak')

@@ -82,11 +82,5 @@ ai.on('data', data => {
   }
 });
 
-ai.on('close', () => {
-  console.log('Free resources');
-  stream.free();
-  recognizer.free();
-});
-
 ai.start();
 console.log('Started! Please speak')

+// Copyright (c)  2024  Xiaomi Corporation
+const sherpa_onnx = require('sherpa-onnx-node');
+const performance = require('perf_hooks').performance;
+
+
+// Please download test files from
+// https://github.com/k2-fsa/sherpa-onnx/releases/tag/kws-models
+const config = {
+  'featConfig': {
+    'sampleRate': 16000,
+    'featureDim': 80,
+  },
+  'modelConfig': {
+    'transducer': {
+      'encoder':
+          './sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/encoder-epoch-12-avg-2-chunk-16-left-64.onnx',
+      'decoder':
+          './sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/decoder-epoch-12-avg-2-chunk-16-left-64.onnx',
+      'joiner':
+          './sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/joiner-epoch-12-avg-2-chunk-16-left-64.onnx',
+    },
+    'tokens':
+        './sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/tokens.txt',
+    'numThreads': 1,
+    'provider': 'cpu',
+    'debug': 1,
+  },
+  'keywordsFile':
+      './sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/test_wavs/test_keywords.txt',
+};
+
+const waveFilename =
+    './sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/test_wavs/3.wav';
+
+const kws = new sherpa_onnx.KeywordSpotter(config);
+console.log('Started')
+let start = performance.now();
+const stream = kws.createStream();
+const wave = sherpa_onnx.readWave(waveFilename);
+stream.acceptWaveform({sampleRate: wave.sampleRate, samples: wave.samples});
+
+const tailPadding = new Float32Array(wave.sampleRate * 0.4);
+stream.acceptWaveform({samples: tailPadding, sampleRate: wave.sampleRate});
+
+const detectedKeywords = [];
+while (kws.isReady(stream)) {
+  const keyword = kws.getResult(stream).keyword;
+  if (keyword != '') {
+    detectedKeywords.push(keyword);
+  }
+  kws.decode(stream);
+}
+let stop = performance.now();
+
+console.log('Done')
+
+const elapsed_seconds = (stop - start) / 1000;
+const duration = wave.samples.length / wave.sampleRate;
+const real_time_factor = elapsed_seconds / duration;
+console.log('Wave duration', duration.toFixed(3), 'secodns')
+console.log('Elapsed', elapsed_seconds.toFixed(3), 'secodns')
+console.log(
+    `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`,
+    real_time_factor.toFixed(3))
+console.log(waveFilename)
+console.log('result\n', detectedKeywords)

+// Copyright (c)  2023-2024  Xiaomi Corporation (authors: Fangjun Kuang)
+//
+const portAudio = require('naudiodon2');
+// console.log(portAudio.getDevices());
+
+const sherpa_onnx = require('sherpa-onnx-node');
+
+function createKeywordSpotter() {
+  const config = {
+    'featConfig': {
+      'sampleRate': 16000,
+      'featureDim': 80,
+    },
+    'modelConfig': {
+      'transducer': {
+        'encoder':
+            './sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/encoder-epoch-12-avg-2-chunk-16-left-64.onnx',
+        'decoder':
+            './sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/decoder-epoch-12-avg-2-chunk-16-left-64.onnx',
+        'joiner':
+            './sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/joiner-epoch-12-avg-2-chunk-16-left-64.onnx',
+      },
+      'tokens':
+          './sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/tokens.txt',
+      'numThreads': 2,
+      'provider': 'cpu',
+      'debug': 1,
+    },
+    'keywordsFile':
+        './sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/keywords.txt',
+  };
+
+  return new sherpa_onnx.KeywordSpotter(config);
+}
+
+const kws = createKeywordSpotter();
+const stream = kws.createStream();
+
+let lastText = '';
+let segmentIndex = 0;
+
+const ai = new portAudio.AudioIO({
+  inOptions: {
+    channelCount: 1,
+    closeOnError: true,  // Close the stream if an audio error is detected, if
+                         // set false then just log the error
+    deviceId: -1,  // Use -1 or omit the deviceId to select the default device
+    sampleFormat: portAudio.SampleFormatFloat32,
+    sampleRate: kws.config.featConfig.sampleRate
+  }
+});
+
+const display = new sherpa_onnx.Display(50);
+
+ai.on('data', data => {
+  const samples = new Float32Array(data.buffer);
+
+  stream.acceptWaveform(
+      {sampleRate: kws.config.featConfig.sampleRate, samples: samples});
+
+  while (kws.isReady(stream)) {
+    kws.decode(stream);
+  }
+
+  const keyword = kws.getResult(stream).keyword
+  if (keyword != '') {
+    display.print(segmentIndex, keyword);
+    segmentIndex += 1;
+  }
+});
+
+ai.start();
+console.log('Started! Please speak.')
+console.log(`Only words from ${kws.config.keywordsFile} can be recognized`)

@@ -19,6 +19,7 @@ include_directories(${CMAKE_JS_INC})
 
 set(srcs
   src/audio-tagging.cc
+  src/keyword-spotting.cc
   src/non-streaming-asr.cc
   src/non-streaming-tts.cc
   src/punctuation.cc

+const addon = require('./addon.js');
+const streaming_asr = require('./streaming-asr.js');
+
+class KeywordSpotter {
+  constructor(config) {
+    this.handle = addon.createKeywordSpotter(config);
+    this.config = config
+  }
+
+  createStream() {
+    const handle = addon.createKeywordStream(this.handle);
+    return new streaming_asr.OnlineStream(handle);
+  }
+
+  isReady(stream) {
+    return addon.isKeywordStreamReady(this.handle, stream.handle);
+  }
+
+  decode(stream) {
+    addon.decodeKeywordStream(this.handle, stream.handle);
+  }
+
+  getResult(stream) {
+    const jsonStr = addon.getKeywordResultAsJson(this.handle, stream.handle);
+
+    return JSON.parse(jsonStr);
+  }
+}
+
+module.exports = {
+  KeywordSpotter,
+}

@@ -7,6 +7,7 @@ const slid = require('./spoken-language-identification.js');
 const sid = require('./speaker-identification.js');
 const at = require('./audio-tagg.js');
 const punct = require('./punctuation.js');
+const kws = require('./keyword-spotter.js');
 
 module.exports = {
   OnlineRecognizer: streaming_asr.OnlineRecognizer,
@@ -22,4 +23,5 @@ module.exports = {
   SpeakerEmbeddingManager: sid.SpeakerEmbeddingManager,
   AudioTagging: at.AudioTagging,
   Punctuation: punct.Punctuation,
+  KeywordSpotter: kws.KeywordSpotter,
 }

+// scripts/node-addon-api/src/keyword-spotting.cc
+//
+// Copyright (c)  2024  Xiaomi Corporation
+#include <sstream>
+
+#include "macros.h"  // NOLINT
+#include "napi.h"    // NOLINT
+#include "sherpa-onnx/c-api/c-api.h"
+
+// defined ./streaming-asr.cc
+SherpaOnnxFeatureConfig GetFeatureConfig(Napi::Object obj);
+
+// defined ./streaming-asr.cc
+SherpaOnnxOnlineModelConfig GetOnlineModelConfig(Napi::Object obj);
+
+static Napi::External<SherpaOnnxKeywordSpotter> CreateKeywordSpotterWrapper(
+    const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  if (info.Length() != 1) {
+    std::ostringstream os;
+    os << "Expect only 1 argument. Given: " << info.Length();
+
+    Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException();
+
+    return {};
+  }
+
+  if (!info[0].IsObject()) {
+    Napi::TypeError::New(env, "Expect an object as the argument")
+        .ThrowAsJavaScriptException();
+
+    return {};
+  }
+
+  Napi::Object o = info[0].As<Napi::Object>();
+  SherpaOnnxKeywordSpotterConfig c;
+  memset(&c, 0, sizeof(c));
+  c.feat_config = GetFeatureConfig(o);
+  c.model_config = GetOnlineModelConfig(o);
+
+  SHERPA_ONNX_ASSIGN_ATTR_INT32(max_active_paths, maxActivePaths);
+  SHERPA_ONNX_ASSIGN_ATTR_INT32(num_trailing_blanks, numTrailingBlanks);
+  SHERPA_ONNX_ASSIGN_ATTR_FLOAT(keywords_score, keywordsScore);
+  SHERPA_ONNX_ASSIGN_ATTR_FLOAT(keywords_threshold, keywordsThreshold);
+  SHERPA_ONNX_ASSIGN_ATTR_STR(keywords_file, keywordsFile);
+
+  SherpaOnnxKeywordSpotter *kws = CreateKeywordSpotter(&c);
+
+  if (c.model_config.transducer.encoder) {
+    delete[] c.model_config.transducer.encoder;
+  }
+
+  if (c.model_config.transducer.decoder) {
+    delete[] c.model_config.transducer.decoder;
+  }
+
+  if (c.model_config.transducer.joiner) {
+    delete[] c.model_config.transducer.joiner;
+  }
+
+  if (c.model_config.paraformer.encoder) {
+    delete[] c.model_config.paraformer.encoder;
+  }
+
+  if (c.model_config.paraformer.decoder) {
+    delete[] c.model_config.paraformer.decoder;
+  }
+
+  if (c.model_config.zipformer2_ctc.model) {
+    delete[] c.model_config.zipformer2_ctc.model;
+  }
+
+  if (c.model_config.tokens) {
+    delete[] c.model_config.tokens;
+  }
+
+  if (c.model_config.provider) {
+    delete[] c.model_config.provider;
+  }
+
+  if (c.model_config.model_type) {
+    delete[] c.model_config.model_type;
+  }
+
+  if (c.keywords_file) {
+    delete[] c.keywords_file;
+  }
+
+  if (!kws) {
+    Napi::TypeError::New(env, "Please check your config!")
+        .ThrowAsJavaScriptException();
+
+    return {};
+  }
+
+  return Napi::External<SherpaOnnxKeywordSpotter>::New(
+      env, kws, [](Napi::Env env, SherpaOnnxKeywordSpotter *kws) {
+        DestroyKeywordSpotter(kws);
+      });
+}
+
+static Napi::External<SherpaOnnxOnlineStream> CreateKeywordStreamWrapper(
+    const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  if (info.Length() != 1) {
+    std::ostringstream os;
+    os << "Expect only 1 argument. Given: " << info.Length();
+
+    Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException();
+
+    return {};
+  }
+
+  if (!info[0].IsExternal()) {
+    Napi::TypeError::New(
+        env, "You should pass a keyword spotter pointer as the only argument")
+        .ThrowAsJavaScriptException();
+
+    return {};
+  }
+
+  SherpaOnnxKeywordSpotter *kws =
+      info[0].As<Napi::External<SherpaOnnxKeywordSpotter>>().Data();
+
+  SherpaOnnxOnlineStream *stream = CreateKeywordStream(kws);
+
+  return Napi::External<SherpaOnnxOnlineStream>::New(
+      env, stream, [](Napi::Env env, SherpaOnnxOnlineStream *stream) {
+        DestroyOnlineStream(stream);
+      });
+}
+
+static Napi::Boolean IsKeywordStreamReadyWrapper(
+    const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  if (info.Length() != 2) {
+    std::ostringstream os;
+    os << "Expect only 2 arguments. Given: " << info.Length();
+
+    Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException();
+
+    return {};
+  }
+
+  if (!info[0].IsExternal()) {
+    Napi::TypeError::New(env, "Argument 0 should be a keyword spotter pointer.")
+        .ThrowAsJavaScriptException();
+
+    return {};
+  }
+
+  if (!info[1].IsExternal()) {
+    Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.")
+        .ThrowAsJavaScriptException();
+
+    return {};
+  }
+
+  SherpaOnnxKeywordSpotter *kws =
+      info[0].As<Napi::External<SherpaOnnxKeywordSpotter>>().Data();
+
+  SherpaOnnxOnlineStream *stream =
+      info[1].As<Napi::External<SherpaOnnxOnlineStream>>().Data();
+
+  int32_t is_ready = IsKeywordStreamReady(kws, stream);
+
+  return Napi::Boolean::New(env, is_ready);
+}
+
+static void DecodeKeywordStreamWrapper(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  if (info.Length() != 2) {
+    std::ostringstream os;
+    os << "Expect only 2 arguments. Given: " << info.Length();
+
+    Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException();
+
+    return;
+  }
+
+  if (!info[0].IsExternal()) {
+    Napi::TypeError::New(env, "Argument 0 should be a keyword spotter pointer.")
+        .ThrowAsJavaScriptException();
+
+    return;
+  }
+
+  if (!info[1].IsExternal()) {
+    Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.")
+        .ThrowAsJavaScriptException();
+
+    return;
+  }
+
+  SherpaOnnxKeywordSpotter *kws =
+      info[0].As<Napi::External<SherpaOnnxKeywordSpotter>>().Data();
+
+  SherpaOnnxOnlineStream *stream =
+      info[1].As<Napi::External<SherpaOnnxOnlineStream>>().Data();
+
+  DecodeKeywordStream(kws, stream);
+}
+
+static Napi::String GetKeywordResultAsJsonWrapper(
+    const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  if (info.Length() != 2) {
+    std::ostringstream os;
+    os << "Expect only 2 arguments. Given: " << info.Length();
+
+    Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException();
+
+    return {};
+  }
+
+  if (!info[0].IsExternal()) {
+    Napi::TypeError::New(env, "Argument 0 should be a keyword spotter pointer.")
+        .ThrowAsJavaScriptException();
+
+    return {};
+  }
+
+  if (!info[1].IsExternal()) {
+    Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.")
+        .ThrowAsJavaScriptException();
+
+    return {};
+  }
+
+  SherpaOnnxKeywordSpotter *kws =
+      info[0].As<Napi::External<SherpaOnnxKeywordSpotter>>().Data();
+
+  SherpaOnnxOnlineStream *stream =
+      info[1].As<Napi::External<SherpaOnnxOnlineStream>>().Data();
+
+  const char *json = GetKeywordResultAsJson(kws, stream);
+
+  Napi::String s = Napi::String::New(env, json);
+
+  FreeKeywordResultJson(json);
+
+  return s;
+}
+
+void InitKeywordSpotting(Napi::Env env, Napi::Object exports) {
+  exports.Set(Napi::String::New(env, "createKeywordSpotter"),
+              Napi::Function::New(env, CreateKeywordSpotterWrapper));
+
+  exports.Set(Napi::String::New(env, "createKeywordStream"),
+              Napi::Function::New(env, CreateKeywordStreamWrapper));
+
+  exports.Set(Napi::String::New(env, "isKeywordStreamReady"),
+              Napi::Function::New(env, IsKeywordStreamReadyWrapper));
+
+  exports.Set(Napi::String::New(env, "decodeKeywordStream"),
+              Napi::Function::New(env, DecodeKeywordStreamWrapper));
+
+  exports.Set(Napi::String::New(env, "getKeywordResultAsJson"),
+              Napi::Function::New(env, GetKeywordResultAsJsonWrapper));
+}

@@ -23,6 +23,8 @@ void InitAudioTagging(Napi::Env env, Napi::Object exports);
 
 void InitPunctuation(Napi::Env env, Napi::Object exports);
 
+void InitKeywordSpotting(Napi::Env env, Napi::Object exports);
+
 Napi::Object Init(Napi::Env env, Napi::Object exports) {
   InitStreamingAsr(env, exports);
   InitNonStreamingAsr(env, exports);
@@ -34,6 +36,7 @@ Napi::Object Init(Napi::Env env, Napi::Object exports) {
   InitSpeakerID(env, exports);
   InitAudioTagging(env, exports);
   InitPunctuation(env, exports);
+  InitKeywordSpotting(env, exports);
 
   return exports;
 }

@@ -90,7 +90,7 @@ static SherpaOnnxOnlineParaformerModelConfig GetOnlineParaformerModelConfig(
   return c;
 }
 
-static SherpaOnnxOnlineModelConfig GetOnlineModelConfig(Napi::Object obj) {
+SherpaOnnxOnlineModelConfig GetOnlineModelConfig(Napi::Object obj) {
   SherpaOnnxOnlineModelConfig c;
   memset(&c, 0, sizeof(c));
 

@@ -653,6 +653,20 @@ void DestroyKeywordResult(const SherpaOnnxKeywordResult *r) {
   }
 }
 
+const char *GetKeywordResultAsJson(SherpaOnnxKeywordSpotter *spotter,
+                                   SherpaOnnxOnlineStream *stream) {
+  const sherpa_onnx::KeywordResult &result =
+      spotter->impl->GetResult(stream->impl.get());
+
+  std::string json = result.AsJsonString();
+  char *pJson = new char[json.size() + 1];
+  std::copy(json.begin(), json.end(), pJson);
+  pJson[json.size()] = 0;
+  return pJson;
+}
+
+void FreeKeywordResultJson(const char *s) { delete[] s; }
+
 // ============================================================
 // For VAD
 // ============================================================

@@ -625,6 +625,13 @@ SHERPA_ONNX_API const SherpaOnnxKeywordResult *GetKeywordResult(
 /// @param r A pointer returned by GetKeywordResult()
 SHERPA_ONNX_API void DestroyKeywordResult(const SherpaOnnxKeywordResult *r);
 
+// the user has to call FreeKeywordResultJson() to free the returned pointer
+// to avoid memory leak
+SHERPA_ONNX_API const char *GetKeywordResultAsJson(
+    SherpaOnnxKeywordSpotter *spotter, SherpaOnnxOnlineStream *stream);
+
+SHERPA_ONNX_API void FreeKeywordResultJson(const char *s);
+
 // ============================================================
 // For VAD
 // ============================================================