xuning / sherpaonnx

转到一个项目
Toggle navigation Toggle navigation pinning
Fangjun Kuang
Committed by GitHub
2ca2985d 1 parent bd4b2239

Add C and C++ API for Moonshine models (#1476)

内嵌 并排对比
正在显示 13 个修改的文件 包含 650 行增加 和 0 行删除
@@ -81,6 +81,80 @@ jobs: @@ -81,6 +81,80 @@ jobs:
81 otool -L ./install/lib/libsherpa-onnx-c-api.dylib 81 otool -L ./install/lib/libsherpa-onnx-c-api.dylib
82 fi 82 fi
83 83
  84 + - name: Test vad + Whisper tiny.en
  85 + shell: bash
  86 + run: |
  87 + gcc -o vad-whisper-c-api ./c-api-examples/vad-whisper-c-api.c \
  88 + -I ./build/install/include \
  89 + -L ./build/install/lib/ \
  90 + -l sherpa-onnx-c-api \
  91 + -l onnxruntime
  92 +
  93 + # Now download models
  94 + #
  95 + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx
  96 + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/Obama.wav
  97 +
  98 + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2
  99 + tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2
  100 + rm sherpa-onnx-whisper-tiny.en.tar.bz2
  101 +
  102 + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH
  103 + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH
  104 +
  105 + ./vad-whisper-c-api
  106 +
  107 + rm -rf sherpa-onnx-*
  108 + rm -rf *.onnx
  109 + rm *.wav
  110 +
  111 + - name: Test vad + Moonshine
  112 + shell: bash
  113 + run: |
  114 + gcc -o vad-moonshine-c-api ./c-api-examples/vad-moonshine-c-api.c \
  115 + -I ./build/install/include \
  116 + -L ./build/install/lib/ \
  117 + -l sherpa-onnx-c-api \
  118 + -l onnxruntime
  119 +
  120 + # Now download models
  121 + #
  122 + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx
  123 + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/Obama.wav
  124 +
  125 + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2
  126 + tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2
  127 + rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2
  128 +
  129 + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH
  130 + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH
  131 +
  132 + ./vad-moonshine-c-api
  133 +
  134 + rm -rf sherpa-onnx-*
  135 + rm -rf *.onnx
  136 + rm *.wav
  137 +
  138 + - name: Test Moonshine
  139 + shell: bash
  140 + run: |
  141 + gcc -o moonshine-c-api ./c-api-examples/moonshine-c-api.c \
  142 + -I ./build/install/include \
  143 + -L ./build/install/lib/ \
  144 + -l sherpa-onnx-c-api \
  145 + -l onnxruntime
  146 +
  147 + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2
  148 + tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2
  149 + rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2
  150 +
  151 + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH
  152 + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH
  153 +
  154 + ./moonshine-c-api
  155 +
  156 + rm -rf sherpa-onnx-*
  157 +
84 - name: Test ffmpeg 158 - name: Test ffmpeg
85 if: matrix.os == 'macos-latest' 159 if: matrix.os == 'macos-latest'
86 shell: bash 160 shell: bash
@@ -83,6 +83,28 @@ jobs: @@ -83,6 +83,28 @@ jobs:
83 otool -L ./install/lib/libsherpa-onnx-cxx-api.dylib 83 otool -L ./install/lib/libsherpa-onnx-cxx-api.dylib
84 fi 84 fi
85 85
  86 + - name: Test Moonshine tiny
  87 + shell: bash
  88 + run: |
  89 + g++ -std=c++17 -o moonshine-cxx-api ./cxx-api-examples/moonshine-cxx-api.cc \
  90 + -I ./build/install/include \
  91 + -L ./build/install/lib/ \
  92 + -l sherpa-onnx-cxx-api \
  93 + -l sherpa-onnx-c-api \
  94 + -l onnxruntime
  95 +
  96 + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2
  97 + tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2
  98 + rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2
  99 +
  100 + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH
  101 + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH
  102 +
  103 + ./moonshine-cxx-api
  104 +
  105 + rm -rf sherpa-onnx-*
  106 + rm ./moonshine-cxx-api
  107 +
86 - name: Test whisper 108 - name: Test whisper
87 shell: bash 109 shell: bash
88 run: | 110 run: |
@@ -35,6 +35,9 @@ target_link_libraries(whisper-c-api sherpa-onnx-c-api) @@ -35,6 +35,9 @@ target_link_libraries(whisper-c-api sherpa-onnx-c-api)
35 add_executable(sense-voice-c-api sense-voice-c-api.c) 35 add_executable(sense-voice-c-api sense-voice-c-api.c)
36 target_link_libraries(sense-voice-c-api sherpa-onnx-c-api) 36 target_link_libraries(sense-voice-c-api sherpa-onnx-c-api)
37 37
  38 +add_executable(moonshine-c-api moonshine-c-api.c)
  39 +target_link_libraries(moonshine-c-api sherpa-onnx-c-api)
  40 +
38 add_executable(zipformer-c-api zipformer-c-api.c) 41 add_executable(zipformer-c-api zipformer-c-api.c)
39 target_link_libraries(zipformer-c-api sherpa-onnx-c-api) 42 target_link_libraries(zipformer-c-api sherpa-onnx-c-api)
40 43
@@ -53,6 +56,12 @@ target_link_libraries(telespeech-c-api sherpa-onnx-c-api) @@ -53,6 +56,12 @@ target_link_libraries(telespeech-c-api sherpa-onnx-c-api)
53 add_executable(vad-sense-voice-c-api vad-sense-voice-c-api.c) 56 add_executable(vad-sense-voice-c-api vad-sense-voice-c-api.c)
54 target_link_libraries(vad-sense-voice-c-api sherpa-onnx-c-api) 57 target_link_libraries(vad-sense-voice-c-api sherpa-onnx-c-api)
55 58
  59 +add_executable(vad-whisper-c-api vad-whisper-c-api.c)
  60 +target_link_libraries(vad-whisper-c-api sherpa-onnx-c-api)
  61 +
  62 +add_executable(vad-moonshine-c-api vad-moonshine-c-api.c)
  63 +target_link_libraries(vad-moonshine-c-api sherpa-onnx-c-api)
  64 +
56 add_executable(streaming-zipformer-buffered-tokens-hotwords-c-api 65 add_executable(streaming-zipformer-buffered-tokens-hotwords-c-api
57 streaming-zipformer-buffered-tokens-hotwords-c-api.c) 66 streaming-zipformer-buffered-tokens-hotwords-c-api.c)
58 target_link_libraries(streaming-zipformer-buffered-tokens-hotwords-c-api sherpa-onnx-c-api) 67 target_link_libraries(streaming-zipformer-buffered-tokens-hotwords-c-api sherpa-onnx-c-api)
  1 +// c-api-examples/moonshine-c-api.c
  2 +//
  3 +// Copyright (c) 2024 Xiaomi Corporation
  4 +
  5 +//
  6 +// This file demonstrates how to use Moonshine tiny with sherpa-onnx's C API.
  7 +// clang-format off
  8 +//
  9 +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2
  10 +// tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2
  11 +// rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2
  12 +//
  13 +// clang-format on
  14 +
  15 +#include <stdio.h>
  16 +#include <stdlib.h>
  17 +#include <string.h>
  18 +
  19 +#include "sherpa-onnx/c-api/c-api.h"
  20 +
  21 +int32_t main() {
  22 + const char *wav_filename =
  23 + "./sherpa-onnx-moonshine-tiny-en-int8/test_wavs/0.wav";
  24 + const char *preprocessor =
  25 + "./sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx";
  26 + const char *encoder = "./sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx";
  27 + const char *uncached_decoder =
  28 + "./sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx";
  29 + const char *cached_decoder =
  30 + "./sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx";
  31 + const char *tokens = "./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt";
  32 +
  33 + const SherpaOnnxWave *wave = SherpaOnnxReadWave(wav_filename);
  34 + if (wave == NULL) {
  35 + fprintf(stderr, "Failed to read %s\n", wav_filename);
  36 + return -1;
  37 + }
  38 +
  39 + // Offline model config
  40 + SherpaOnnxOfflineModelConfig offline_model_config;
  41 + memset(&offline_model_config, 0, sizeof(offline_model_config));
  42 + offline_model_config.debug = 1;
  43 + offline_model_config.num_threads = 1;
  44 + offline_model_config.provider = "cpu";
  45 + offline_model_config.tokens = tokens;
  46 + offline_model_config.moonshine.preprocessor = preprocessor;
  47 + offline_model_config.moonshine.encoder = encoder;
  48 + offline_model_config.moonshine.uncached_decoder = uncached_decoder;
  49 + offline_model_config.moonshine.cached_decoder = cached_decoder;
  50 +
  51 + // Recognizer config
  52 + SherpaOnnxOfflineRecognizerConfig recognizer_config;
  53 + memset(&recognizer_config, 0, sizeof(recognizer_config));
  54 + recognizer_config.decoding_method = "greedy_search";
  55 + recognizer_config.model_config = offline_model_config;
  56 +
  57 + const SherpaOnnxOfflineRecognizer *recognizer =
  58 + SherpaOnnxCreateOfflineRecognizer(&recognizer_config);
  59 +
  60 + if (recognizer == NULL) {
  61 + fprintf(stderr, "Please check your config!\n");
  62 + SherpaOnnxFreeWave(wave);
  63 + return -1;
  64 + }
  65 +
  66 + const SherpaOnnxOfflineStream *stream =
  67 + SherpaOnnxCreateOfflineStream(recognizer);
  68 +
  69 + SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, wave->samples,
  70 + wave->num_samples);
  71 + SherpaOnnxDecodeOfflineStream(recognizer, stream);
  72 + const SherpaOnnxOfflineRecognizerResult *result =
  73 + SherpaOnnxGetOfflineStreamResult(stream);
  74 +
  75 + fprintf(stderr, "Decoded text: %s\n", result->text);
  76 +
  77 + SherpaOnnxDestroyOfflineRecognizerResult(result);
  78 + SherpaOnnxDestroyOfflineStream(stream);
  79 + SherpaOnnxDestroyOfflineRecognizer(recognizer);
  80 + SherpaOnnxFreeWave(wave);
  81 +
  82 + return 0;
  83 +}
  1 +// c-api-examples/vad-moonshine-c-api.c
  2 +//
  3 +// Copyright (c) 2024 Xiaomi Corporation
  4 +
  5 +//
  6 +// This file demonstrates how to use VAD + Moonshine with sherpa-onnx's C API.
  7 +// clang-format off
  8 +//
  9 +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx
  10 +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/Obama.wav
  11 +//
  12 +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2
  13 +// tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2
  14 +// rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2
  15 +//
  16 +// clang-format on
  17 +
  18 +#include <stdio.h>
  19 +#include <stdlib.h>
  20 +#include <string.h>
  21 +
  22 +#include "sherpa-onnx/c-api/c-api.h"
  23 +
  24 +int32_t main() {
  25 + const char *wav_filename = "./Obama.wav";
  26 + const char *vad_filename = "./silero_vad.onnx";
  27 +
  28 + const char *preprocessor =
  29 + "./sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx";
  30 + const char *encoder = "./sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx";
  31 + const char *uncached_decoder =
  32 + "./sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx";
  33 + const char *cached_decoder =
  34 + "./sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx";
  35 + const char *tokens = "./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt";
  36 +
  37 + const SherpaOnnxWave *wave = SherpaOnnxReadWave(wav_filename);
  38 + if (wave == NULL) {
  39 + fprintf(stderr, "Failed to read %s\n", wav_filename);
  40 + return -1;
  41 + }
  42 +
  43 + if (wave->sample_rate != 16000) {
  44 + fprintf(stderr, "Expect the sample rate to be 16000. Given: %d\n",
  45 + wave->sample_rate);
  46 + SherpaOnnxFreeWave(wave);
  47 + return -1;
  48 + }
  49 +
  50 + // Offline model config
  51 + SherpaOnnxOfflineModelConfig offline_model_config;
  52 + memset(&offline_model_config, 0, sizeof(offline_model_config));
  53 + offline_model_config.debug = 0;
  54 + offline_model_config.num_threads = 1;
  55 + offline_model_config.provider = "cpu";
  56 + offline_model_config.tokens = tokens;
  57 + offline_model_config.moonshine.preprocessor = preprocessor;
  58 + offline_model_config.moonshine.encoder = encoder;
  59 + offline_model_config.moonshine.uncached_decoder = uncached_decoder;
  60 + offline_model_config.moonshine.cached_decoder = cached_decoder;
  61 +
  62 + // Recognizer config
  63 + SherpaOnnxOfflineRecognizerConfig recognizer_config;
  64 + memset(&recognizer_config, 0, sizeof(recognizer_config));
  65 + recognizer_config.decoding_method = "greedy_search";
  66 + recognizer_config.model_config = offline_model_config;
  67 +
  68 + const SherpaOnnxOfflineRecognizer *recognizer =
  69 + SherpaOnnxCreateOfflineRecognizer(&recognizer_config);
  70 +
  71 + if (recognizer == NULL) {
  72 + fprintf(stderr, "Please check your recognizer config!\n");
  73 + SherpaOnnxFreeWave(wave);
  74 + return -1;
  75 + }
  76 +
  77 + SherpaOnnxVadModelConfig vadConfig;
  78 + memset(&vadConfig, 0, sizeof(vadConfig));
  79 + vadConfig.silero_vad.model = vad_filename;
  80 + vadConfig.silero_vad.threshold = 0.5;
  81 + vadConfig.silero_vad.min_silence_duration = 0.5;
  82 + vadConfig.silero_vad.min_speech_duration = 0.5;
  83 + vadConfig.silero_vad.max_speech_duration = 10;
  84 + vadConfig.silero_vad.window_size = 512;
  85 + vadConfig.sample_rate = 16000;
  86 + vadConfig.num_threads = 1;
  87 + vadConfig.debug = 1;
  88 +
  89 + SherpaOnnxVoiceActivityDetector *vad =
  90 + SherpaOnnxCreateVoiceActivityDetector(&vadConfig, 30);
  91 +
  92 + if (vad == NULL) {
  93 + fprintf(stderr, "Please check your recognizer config!\n");
  94 + SherpaOnnxFreeWave(wave);
  95 + SherpaOnnxDestroyOfflineRecognizer(recognizer);
  96 + return -1;
  97 + }
  98 +
  99 + int32_t window_size = vadConfig.silero_vad.window_size;
  100 + int32_t i = 0;
  101 +
  102 + while (i + window_size < wave->num_samples) {
  103 + SherpaOnnxVoiceActivityDetectorAcceptWaveform(vad, wave->samples + i,
  104 + window_size);
  105 + i += window_size;
  106 +
  107 + while (!SherpaOnnxVoiceActivityDetectorEmpty(vad)) {
  108 + const SherpaOnnxSpeechSegment *segment =
  109 + SherpaOnnxVoiceActivityDetectorFront(vad);
  110 +
  111 + const SherpaOnnxOfflineStream *stream =
  112 + SherpaOnnxCreateOfflineStream(recognizer);
  113 +
  114 + SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate,
  115 + segment->samples, segment->n);
  116 +
  117 + SherpaOnnxDecodeOfflineStream(recognizer, stream);
  118 +
  119 + const SherpaOnnxOfflineRecognizerResult *result =
  120 + SherpaOnnxGetOfflineStreamResult(stream);
  121 +
  122 + float start = segment->start / 16000.0f;
  123 + float duration = segment->n / 16000.0f;
  124 + float stop = start + duration;
  125 +
  126 + fprintf(stderr, "%.3f -- %.3f: %s\n", start, stop, result->text);
  127 +
  128 + SherpaOnnxDestroyOfflineRecognizerResult(result);
  129 + SherpaOnnxDestroyOfflineStream(stream);
  130 +
  131 + SherpaOnnxDestroySpeechSegment(segment);
  132 + SherpaOnnxVoiceActivityDetectorPop(vad);
  133 + }
  134 + }
  135 +
  136 + SherpaOnnxVoiceActivityDetectorFlush(vad);
  137 +
  138 + while (!SherpaOnnxVoiceActivityDetectorEmpty(vad)) {
  139 + const SherpaOnnxSpeechSegment *segment =
  140 + SherpaOnnxVoiceActivityDetectorFront(vad);
  141 +
  142 + const SherpaOnnxOfflineStream *stream =
  143 + SherpaOnnxCreateOfflineStream(recognizer);
  144 +
  145 + SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, segment->samples,
  146 + segment->n);
  147 +
  148 + SherpaOnnxDecodeOfflineStream(recognizer, stream);
  149 +
  150 + const SherpaOnnxOfflineRecognizerResult *result =
  151 + SherpaOnnxGetOfflineStreamResult(stream);
  152 +
  153 + float start = segment->start / 16000.0f;
  154 + float duration = segment->n / 16000.0f;
  155 + float stop = start + duration;
  156 +
  157 + fprintf(stderr, "%.3f -- %.3f: %s\n", start, stop, result->text);
  158 +
  159 + SherpaOnnxDestroyOfflineRecognizerResult(result);
  160 + SherpaOnnxDestroyOfflineStream(stream);
  161 +
  162 + SherpaOnnxDestroySpeechSegment(segment);
  163 + SherpaOnnxVoiceActivityDetectorPop(vad);
  164 + }
  165 +
  166 + SherpaOnnxDestroyOfflineRecognizer(recognizer);
  167 + SherpaOnnxDestroyVoiceActivityDetector(vad);
  168 + SherpaOnnxFreeWave(wave);
  169 +
  170 + return 0;
  171 +}
@@ -81,6 +81,7 @@ int32_t main() { @@ -81,6 +81,7 @@ int32_t main() {
81 vadConfig.silero_vad.threshold = 0.5; 81 vadConfig.silero_vad.threshold = 0.5;
82 vadConfig.silero_vad.min_silence_duration = 0.5; 82 vadConfig.silero_vad.min_silence_duration = 0.5;
83 vadConfig.silero_vad.min_speech_duration = 0.5; 83 vadConfig.silero_vad.min_speech_duration = 0.5;
  84 + vadConfig.silero_vad.max_speech_duration = 5;
84 vadConfig.silero_vad.window_size = 512; 85 vadConfig.silero_vad.window_size = 512;
85 vadConfig.sample_rate = 16000; 86 vadConfig.sample_rate = 16000;
86 vadConfig.num_threads = 1; 87 vadConfig.num_threads = 1;
  1 +// c-api-examples/vad-whisper-c-api.c
  2 +//
  3 +// Copyright (c) 2024 Xiaomi Corporation
  4 +
  5 +//
  6 +// This file demonstrates how to use VAD + Whisper tiny.en with
  7 +// sherpa-onnx's C API.
  8 +//
  9 +// clang-format off
  10 +//
  11 +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx
  12 +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/Obama.wav
  13 +//
  14 +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2
  15 +// tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2
  16 +// rm sherpa-onnx-whisper-tiny.en.tar.bz2
  17 +//
  18 +// clang-format on
  19 +
  20 +#include <stdio.h>
  21 +#include <stdlib.h>
  22 +#include <string.h>
  23 +
  24 +#include "sherpa-onnx/c-api/c-api.h"
  25 +
  26 +int32_t main() {
  27 + const char *wav_filename = "./Obama.wav";
  28 + const char *vad_filename = "./silero_vad.onnx";
  29 +
  30 + const char *encoder = "sherpa-onnx-whisper-tiny.en/tiny.en-encoder.int8.onnx";
  31 + const char *decoder = "sherpa-onnx-whisper-tiny.en/tiny.en-decoder.int8.onnx";
  32 + const char *tokens = "sherpa-onnx-whisper-tiny.en/tiny.en-tokens.txt";
  33 +
  34 + const SherpaOnnxWave *wave = SherpaOnnxReadWave(wav_filename);
  35 + if (wave == NULL) {
  36 + fprintf(stderr, "Failed to read %s\n", wav_filename);
  37 + return -1;
  38 + }
  39 +
  40 + if (wave->sample_rate != 16000) {
  41 + fprintf(stderr, "Expect the sample rate to be 16000. Given: %d\n",
  42 + wave->sample_rate);
  43 + SherpaOnnxFreeWave(wave);
  44 + return -1;
  45 + }
  46 +
  47 + // Offline model config
  48 + SherpaOnnxOfflineModelConfig offline_model_config;
  49 + memset(&offline_model_config, 0, sizeof(offline_model_config));
  50 + offline_model_config.debug = 0;
  51 + offline_model_config.num_threads = 1;
  52 + offline_model_config.provider = "cpu";
  53 + offline_model_config.tokens = tokens;
  54 + offline_model_config.whisper.encoder = encoder;
  55 + offline_model_config.whisper.decoder = decoder;
  56 + offline_model_config.whisper.language = "en";
  57 + offline_model_config.whisper.tail_paddings = 0;
  58 + offline_model_config.whisper.task = "transcribe";
  59 +
  60 + // Recognizer config
  61 + SherpaOnnxOfflineRecognizerConfig recognizer_config;
  62 + memset(&recognizer_config, 0, sizeof(recognizer_config));
  63 + recognizer_config.decoding_method = "greedy_search";
  64 + recognizer_config.model_config = offline_model_config;
  65 +
  66 + const SherpaOnnxOfflineRecognizer *recognizer =
  67 + SherpaOnnxCreateOfflineRecognizer(&recognizer_config);
  68 +
  69 + if (recognizer == NULL) {
  70 + fprintf(stderr, "Please check your recognizer config!\n");
  71 + SherpaOnnxFreeWave(wave);
  72 + return -1;
  73 + }
  74 +
  75 + SherpaOnnxVadModelConfig vadConfig;
  76 + memset(&vadConfig, 0, sizeof(vadConfig));
  77 + vadConfig.silero_vad.model = vad_filename;
  78 + vadConfig.silero_vad.threshold = 0.5;
  79 + vadConfig.silero_vad.min_silence_duration = 0.5;
  80 + vadConfig.silero_vad.min_speech_duration = 0.5;
  81 + vadConfig.silero_vad.max_speech_duration = 10;
  82 + vadConfig.silero_vad.window_size = 512;
  83 + vadConfig.sample_rate = 16000;
  84 + vadConfig.num_threads = 1;
  85 + vadConfig.debug = 1;
  86 +
  87 + SherpaOnnxVoiceActivityDetector *vad =
  88 + SherpaOnnxCreateVoiceActivityDetector(&vadConfig, 30);
  89 +
  90 + if (vad == NULL) {
  91 + fprintf(stderr, "Please check your recognizer config!\n");
  92 + SherpaOnnxFreeWave(wave);
  93 + SherpaOnnxDestroyOfflineRecognizer(recognizer);
  94 + return -1;
  95 + }
  96 +
  97 + int32_t window_size = vadConfig.silero_vad.window_size;
  98 + int32_t i = 0;
  99 +
  100 + while (i + window_size < wave->num_samples) {
  101 + SherpaOnnxVoiceActivityDetectorAcceptWaveform(vad, wave->samples + i,
  102 + window_size);
  103 + i += window_size;
  104 +
  105 + while (!SherpaOnnxVoiceActivityDetectorEmpty(vad)) {
  106 + const SherpaOnnxSpeechSegment *segment =
  107 + SherpaOnnxVoiceActivityDetectorFront(vad);
  108 +
  109 + const SherpaOnnxOfflineStream *stream =
  110 + SherpaOnnxCreateOfflineStream(recognizer);
  111 +
  112 + SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate,
  113 + segment->samples, segment->n);
  114 +
  115 + SherpaOnnxDecodeOfflineStream(recognizer, stream);
  116 +
  117 + const SherpaOnnxOfflineRecognizerResult *result =
  118 + SherpaOnnxGetOfflineStreamResult(stream);
  119 +
  120 + float start = segment->start / 16000.0f;
  121 + float duration = segment->n / 16000.0f;
  122 + float stop = start + duration;
  123 +
  124 + fprintf(stderr, "%.3f -- %.3f: %s\n", start, stop, result->text);
  125 +
  126 + SherpaOnnxDestroyOfflineRecognizerResult(result);
  127 + SherpaOnnxDestroyOfflineStream(stream);
  128 +
  129 + SherpaOnnxDestroySpeechSegment(segment);
  130 + SherpaOnnxVoiceActivityDetectorPop(vad);
  131 + }
  132 + }
  133 +
  134 + SherpaOnnxVoiceActivityDetectorFlush(vad);
  135 +
  136 + while (!SherpaOnnxVoiceActivityDetectorEmpty(vad)) {
  137 + const SherpaOnnxSpeechSegment *segment =
  138 + SherpaOnnxVoiceActivityDetectorFront(vad);
  139 +
  140 + const SherpaOnnxOfflineStream *stream =
  141 + SherpaOnnxCreateOfflineStream(recognizer);
  142 +
  143 + SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, segment->samples,
  144 + segment->n);
  145 +
  146 + SherpaOnnxDecodeOfflineStream(recognizer, stream);
  147 +
  148 + const SherpaOnnxOfflineRecognizerResult *result =
  149 + SherpaOnnxGetOfflineStreamResult(stream);
  150 +
  151 + float start = segment->start / 16000.0f;
  152 + float duration = segment->n / 16000.0f;
  153 + float stop = start + duration;
  154 +
  155 + fprintf(stderr, "%.3f -- %.3f: %s\n", start, stop, result->text);
  156 +
  157 + SherpaOnnxDestroyOfflineRecognizerResult(result);
  158 + SherpaOnnxDestroyOfflineStream(stream);
  159 +
  160 + SherpaOnnxDestroySpeechSegment(segment);
  161 + SherpaOnnxVoiceActivityDetectorPop(vad);
  162 + }
  163 +
  164 + SherpaOnnxDestroyOfflineRecognizer(recognizer);
  165 + SherpaOnnxDestroyVoiceActivityDetector(vad);
  166 + SherpaOnnxFreeWave(wave);
  167 +
  168 + return 0;
  169 +}
@@ -6,5 +6,8 @@ target_link_libraries(streaming-zipformer-cxx-api sherpa-onnx-cxx-api) @@ -6,5 +6,8 @@ target_link_libraries(streaming-zipformer-cxx-api sherpa-onnx-cxx-api)
6 add_executable(whisper-cxx-api ./whisper-cxx-api.cc) 6 add_executable(whisper-cxx-api ./whisper-cxx-api.cc)
7 target_link_libraries(whisper-cxx-api sherpa-onnx-cxx-api) 7 target_link_libraries(whisper-cxx-api sherpa-onnx-cxx-api)
8 8
  9 +add_executable(moonshine-cxx-api ./moonshine-cxx-api.cc)
  10 +target_link_libraries(moonshine-cxx-api sherpa-onnx-cxx-api)
  11 +
9 add_executable(sense-voice-cxx-api ./sense-voice-cxx-api.cc) 12 add_executable(sense-voice-cxx-api ./sense-voice-cxx-api.cc)
10 target_link_libraries(sense-voice-cxx-api sherpa-onnx-cxx-api) 13 target_link_libraries(sense-voice-cxx-api sherpa-onnx-cxx-api)
  1 +// cxx-api-examples/moonshine-cxx-api.cc
  2 +// Copyright (c) 2024 Xiaomi Corporation
  3 +
  4 +//
  5 +// This file demonstrates how to use Moonshine with sherpa-onnx's C++ API.
  6 +//
  7 +// clang-format off
  8 +//
  9 +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2
  10 +// tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2
  11 +// rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2
  12 +//
  13 +// clang-format on
  14 +
  15 +#include <chrono> // NOLINT
  16 +#include <iostream>
  17 +#include <string>
  18 +
  19 +#include "sherpa-onnx/c-api/cxx-api.h"
  20 +
  21 +int32_t main() {
  22 + using namespace sherpa_onnx::cxx; // NOLINT
  23 + OfflineRecognizerConfig config;
  24 +
  25 + config.model_config.moonshine.preprocessor =
  26 + "./sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx";
  27 + config.model_config.moonshine.encoder =
  28 + "./sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx";
  29 + config.model_config.moonshine.uncached_decoder =
  30 + "./sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx";
  31 + config.model_config.moonshine.cached_decoder =
  32 + "./sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx";
  33 + config.model_config.tokens =
  34 + "./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt";
  35 +
  36 + config.model_config.num_threads = 1;
  37 +
  38 + std::cout << "Loading model\n";
  39 + OfflineRecognizer recongizer = OfflineRecognizer::Create(config);
  40 + if (!recongizer.Get()) {
  41 + std::cerr << "Please check your config\n";
  42 + return -1;
  43 + }
  44 + std::cout << "Loading model done\n";
  45 +
  46 + std::string wave_filename =
  47 + "./sherpa-onnx-moonshine-tiny-en-int8/test_wavs/0.wav";
  48 + Wave wave = ReadWave(wave_filename);
  49 + if (wave.samples.empty()) {
  50 + std::cerr << "Failed to read: '" << wave_filename << "'\n";
  51 + return -1;
  52 + }
  53 +
  54 + std::cout << "Start recognition\n";
  55 + const auto begin = std::chrono::steady_clock::now();
  56 +
  57 + OfflineStream stream = recongizer.CreateStream();
  58 + stream.AcceptWaveform(wave.sample_rate, wave.samples.data(),
  59 + wave.samples.size());
  60 +
  61 + recongizer.Decode(&stream);
  62 +
  63 + OfflineRecognizerResult result = recongizer.GetResult(&stream);
  64 +
  65 + const auto end = std::chrono::steady_clock::now();
  66 + const float elapsed_seconds =
  67 + std::chrono::duration_cast<std::chrono::milliseconds>(end - begin)
  68 + .count() /
  69 + 1000.;
  70 + float duration = wave.samples.size() / static_cast<float>(wave.sample_rate);
  71 + float rtf = elapsed_seconds / duration;
  72 +
  73 + std::cout << "text: " << result.text << "\n";
  74 + printf("Number of threads: %d\n", config.model_config.num_threads);
  75 + printf("Duration: %.3fs\n", duration);
  76 + printf("Elapsed seconds: %.3fs\n", elapsed_seconds);
  77 + printf("(Real time factor) RTF = %.3f / %.3f = %.3f\n", elapsed_seconds,
  78 + duration, rtf);
  79 +
  80 + return 0;
  81 +}
@@ -450,6 +450,18 @@ sherpa_onnx::OfflineRecognizerConfig convertConfig( @@ -450,6 +450,18 @@ sherpa_onnx::OfflineRecognizerConfig convertConfig(
450 recognizer_config.model_config.sense_voice.use_itn = 450 recognizer_config.model_config.sense_voice.use_itn =
451 config->model_config.sense_voice.use_itn; 451 config->model_config.sense_voice.use_itn;
452 452
  453 + recognizer_config.model_config.moonshine.preprocessor =
  454 + SHERPA_ONNX_OR(config->model_config.moonshine.preprocessor, "");
  455 +
  456 + recognizer_config.model_config.moonshine.encoder =
  457 + SHERPA_ONNX_OR(config->model_config.moonshine.encoder, "");
  458 +
  459 + recognizer_config.model_config.moonshine.uncached_decoder =
  460 + SHERPA_ONNX_OR(config->model_config.moonshine.uncached_decoder, "");
  461 +
  462 + recognizer_config.model_config.moonshine.cached_decoder =
  463 + SHERPA_ONNX_OR(config->model_config.moonshine.cached_decoder, "");
  464 +
453 recognizer_config.lm_config.model = 465 recognizer_config.lm_config.model =
454 SHERPA_ONNX_OR(config->lm_config.model, ""); 466 SHERPA_ONNX_OR(config->lm_config.model, "");
455 recognizer_config.lm_config.scale = 467 recognizer_config.lm_config.scale =
@@ -389,6 +389,13 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOfflineWhisperModelConfig { @@ -389,6 +389,13 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOfflineWhisperModelConfig {
389 int32_t tail_paddings; 389 int32_t tail_paddings;
390 } SherpaOnnxOfflineWhisperModelConfig; 390 } SherpaOnnxOfflineWhisperModelConfig;
391 391
  392 +SHERPA_ONNX_API typedef struct SherpaOnnxOfflineMoonshineModelConfig {
  393 + const char *preprocessor;
  394 + const char *encoder;
  395 + const char *uncached_decoder;
  396 + const char *cached_decoder;
  397 +} SherpaOnnxOfflineMoonshineModelConfig;
  398 +
392 SHERPA_ONNX_API typedef struct SherpaOnnxOfflineTdnnModelConfig { 399 SHERPA_ONNX_API typedef struct SherpaOnnxOfflineTdnnModelConfig {
393 const char *model; 400 const char *model;
394 } SherpaOnnxOfflineTdnnModelConfig; 401 } SherpaOnnxOfflineTdnnModelConfig;
@@ -424,6 +431,7 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOfflineModelConfig { @@ -424,6 +431,7 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOfflineModelConfig {
424 const char *bpe_vocab; 431 const char *bpe_vocab;
425 const char *telespeech_ctc; 432 const char *telespeech_ctc;
426 SherpaOnnxOfflineSenseVoiceModelConfig sense_voice; 433 SherpaOnnxOfflineSenseVoiceModelConfig sense_voice;
  434 + SherpaOnnxOfflineMoonshineModelConfig moonshine;
427 } SherpaOnnxOfflineModelConfig; 435 } SherpaOnnxOfflineModelConfig;
428 436
429 SHERPA_ONNX_API typedef struct SherpaOnnxOfflineRecognizerConfig { 437 SHERPA_ONNX_API typedef struct SherpaOnnxOfflineRecognizerConfig {
@@ -227,6 +227,15 @@ OfflineRecognizer OfflineRecognizer::Create( @@ -227,6 +227,15 @@ OfflineRecognizer OfflineRecognizer::Create(
227 config.model_config.sense_voice.language.c_str(); 227 config.model_config.sense_voice.language.c_str();
228 c.model_config.sense_voice.use_itn = config.model_config.sense_voice.use_itn; 228 c.model_config.sense_voice.use_itn = config.model_config.sense_voice.use_itn;
229 229
  230 + c.model_config.moonshine.preprocessor =
  231 + config.model_config.moonshine.preprocessor.c_str();
  232 + c.model_config.moonshine.encoder =
  233 + config.model_config.moonshine.encoder.c_str();
  234 + c.model_config.moonshine.uncached_decoder =
  235 + config.model_config.moonshine.uncached_decoder.c_str();
  236 + c.model_config.moonshine.cached_decoder =
  237 + config.model_config.moonshine.cached_decoder.c_str();
  238 +
230 c.lm_config.model = config.lm_config.model.c_str(); 239 c.lm_config.model = config.lm_config.model.c_str();
231 c.lm_config.scale = config.lm_config.scale; 240 c.lm_config.scale = config.lm_config.scale;
232 241
@@ -225,6 +225,13 @@ struct SHERPA_ONNX_API OfflineSenseVoiceModelConfig { @@ -225,6 +225,13 @@ struct SHERPA_ONNX_API OfflineSenseVoiceModelConfig {
225 bool use_itn = false; 225 bool use_itn = false;
226 }; 226 };
227 227
  228 +struct SHERPA_ONNX_API OfflineMoonshineModelConfig {
  229 + std::string preprocessor;
  230 + std::string encoder;
  231 + std::string uncached_decoder;
  232 + std::string cached_decoder;
  233 +};
  234 +
228 struct SHERPA_ONNX_API OfflineModelConfig { 235 struct SHERPA_ONNX_API OfflineModelConfig {
229 OfflineTransducerModelConfig transducer; 236 OfflineTransducerModelConfig transducer;
230 OfflineParaformerModelConfig paraformer; 237 OfflineParaformerModelConfig paraformer;
@@ -241,6 +248,7 @@ struct SHERPA_ONNX_API OfflineModelConfig { @@ -241,6 +248,7 @@ struct SHERPA_ONNX_API OfflineModelConfig {
241 std::string bpe_vocab; 248 std::string bpe_vocab;
242 std::string telespeech_ctc; 249 std::string telespeech_ctc;
243 OfflineSenseVoiceModelConfig sense_voice; 250 OfflineSenseVoiceModelConfig sense_voice;
  251 + OfflineMoonshineModelConfig moonshine;
244 }; 252 };
245 253
246 struct SHERPA_ONNX_API OfflineLMConfig { 254 struct SHERPA_ONNX_API OfflineLMConfig {