Fangjun Kuang
Committed by GitHub

Support decoding multiple streams in Java API. (#2149)

@@ -166,6 +166,8 @@ jobs: @@ -166,6 +166,8 @@ jobs:
166 rm -rf sherpa-onnx-fire-red-* 166 rm -rf sherpa-onnx-fire-red-*
167 167
168 ./run-non-streaming-decode-file-whisper.sh 168 ./run-non-streaming-decode-file-whisper.sh
  169 +
  170 + ./run-non-streaming-decode-file-whisper-multiple.sh
169 rm -rf sherpa-onnx-whisper-* 171 rm -rf sherpa-onnx-whisper-*
170 172
171 ./run-non-streaming-decode-file-nemo.sh 173 ./run-non-streaming-decode-file-nemo.sh
  1 +// Copyright 2025 Xiaomi Corporation
  2 +
  3 +// This file shows how to use an offline whisper, i.e., non-streaming whisper,
  4 +// to decode files.
  5 +import com.k2fsa.sherpa.onnx.*;
  6 +
  7 +public class NonStreamingDecodeFileWhisperMultiple {
  8 + public static void main(String[] args) {
  9 + // please refer to
  10 + // https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/tiny.en.html
  11 + // to download model files
  12 + String encoder = "./sherpa-onnx-whisper-tiny.en/tiny.en-encoder.int8.onnx";
  13 + String decoder = "./sherpa-onnx-whisper-tiny.en/tiny.en-decoder.int8.onnx";
  14 + String tokens = "./sherpa-onnx-whisper-tiny.en/tiny.en-tokens.txt";
  15 +
  16 + String waveFilename0 = "./sherpa-onnx-whisper-tiny.en/test_wavs/0.wav";
  17 + String waveFilename1 = "./sherpa-onnx-whisper-tiny.en/test_wavs/1.wav";
  18 +
  19 + WaveReader reader0 = new WaveReader(waveFilename0);
  20 + WaveReader reader1 = new WaveReader(waveFilename1);
  21 +
  22 + OfflineWhisperModelConfig whisper =
  23 + OfflineWhisperModelConfig.builder().setEncoder(encoder).setDecoder(decoder).build();
  24 +
  25 + OfflineModelConfig modelConfig =
  26 + OfflineModelConfig.builder()
  27 + .setWhisper(whisper)
  28 + .setTokens(tokens)
  29 + .setNumThreads(1)
  30 + .setDebug(true)
  31 + .build();
  32 +
  33 + OfflineRecognizerConfig config =
  34 + OfflineRecognizerConfig.builder()
  35 + .setOfflineModelConfig(modelConfig)
  36 + .setDecodingMethod("greedy_search")
  37 + .build();
  38 +
  39 + OfflineRecognizer recognizer = new OfflineRecognizer(config);
  40 + OfflineStream stream0 = recognizer.createStream();
  41 + stream0.acceptWaveform(reader0.getSamples(), reader0.getSampleRate());
  42 +
  43 + OfflineStream stream1 = recognizer.createStream();
  44 + stream1.acceptWaveform(reader1.getSamples(), reader1.getSampleRate());
  45 +
  46 + OfflineStream[] ss = new OfflineStream[] {stream0, stream1};
  47 + recognizer.decode(ss);
  48 +
  49 + String text0 = recognizer.getResult(stream0).getText();
  50 + String text1 = recognizer.getResult(stream1).getText();
  51 +
  52 + System.out.printf("filename0:%s\nresult0:%s\n\n", waveFilename0, text0);
  53 + System.out.printf("filename1:%s\nresult1:%s\n\n", waveFilename1, text1);
  54 +
  55 + stream0.release();
  56 + stream1.release();
  57 + recognizer.release();
  58 + }
  59 +}
  1 +#!/usr/bin/env bash
  2 +
  3 +set -ex
  4 +
  5 +if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then
  6 + mkdir -p ../build
  7 + pushd ../build
  8 + cmake \
  9 + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \
  10 + -DSHERPA_ONNX_ENABLE_TESTS=OFF \
  11 + -DSHERPA_ONNX_ENABLE_CHECK=OFF \
  12 + -DBUILD_SHARED_LIBS=ON \
  13 + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \
  14 + -DSHERPA_ONNX_ENABLE_JNI=ON \
  15 + ..
  16 +
  17 + make -j4
  18 + ls -lh lib
  19 + popd
  20 +fi
  21 +
  22 +if [ ! -f ../sherpa-onnx/java-api/build/sherpa-onnx.jar ]; then
  23 + pushd ../sherpa-onnx/java-api
  24 + make
  25 + popd
  26 +fi
  27 +
  28 +if [ ! -f ./sherpa-onnx-whisper-tiny.en/tiny.en-tokens.txt ]; then
  29 + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2
  30 +
  31 + tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2
  32 + rm sherpa-onnx-whisper-tiny.en.tar.bz2
  33 +fi
  34 +
  35 +java \
  36 + -Djava.library.path=$PWD/../build/lib \
  37 + -cp ../sherpa-onnx/java-api/build/sherpa-onnx.jar \
  38 + NonStreamingDecodeFileWhisperMultiple.java
@@ -17,6 +17,14 @@ public class OfflineRecognizer { @@ -17,6 +17,14 @@ public class OfflineRecognizer {
17 decode(ptr, s.getPtr()); 17 decode(ptr, s.getPtr());
18 } 18 }
19 19
  20 + public void decode(OfflineStream[] ss) {
  21 + long[] streamPtrs = new long[ss.length];
  22 + for (int i = 0; i < ss.length; ++i) {
  23 + streamPtrs[i] = ss[i].getPtr();
  24 + }
  25 + decodeStreams(ptr, streamPtrs);
  26 + }
  27 +
20 public OfflineStream createStream() { 28 public OfflineStream createStream() {
21 long p = createStream(ptr); 29 long p = createStream(ptr);
22 return new OfflineStream(p); 30 return new OfflineStream(p);
@@ -55,5 +63,7 @@ public class OfflineRecognizer { @@ -55,5 +63,7 @@ public class OfflineRecognizer {
55 63
56 private native void decode(long ptr, long streamPtr); 64 private native void decode(long ptr, long streamPtr);
57 65
  66 + private native void decodeStreams(long ptr, long[] streamPtrs);
  67 +
58 private native Object[] getResult(long streamPtr); 68 private native Object[] getResult(long streamPtr);
59 } 69 }
@@ -18,6 +18,14 @@ public class OnlineRecognizer { @@ -18,6 +18,14 @@ public class OnlineRecognizer {
18 decode(ptr, s.getPtr()); 18 decode(ptr, s.getPtr());
19 } 19 }
20 20
  21 + public void decode(OnlineStream[] ss) {
  22 + long[] streamPtrs = new long[ss.length];
  23 + for (int i = 0; i < ss.length; ++i) {
  24 + streamPtrs[i] = ss[i].getPtr();
  25 + }
  26 + decodeStreams(ptr, streamPtrs);
  27 + }
  28 +
21 public boolean isReady(OnlineStream s) { 29 public boolean isReady(OnlineStream s) {
22 return isReady(ptr, s.getPtr()); 30 return isReady(ptr, s.getPtr());
23 } 31 }
@@ -68,6 +76,8 @@ public class OnlineRecognizer { @@ -68,6 +76,8 @@ public class OnlineRecognizer {
68 76
69 private native void decode(long ptr, long streamPtr); 77 private native void decode(long ptr, long streamPtr);
70 78
  79 + private native void decodeStreams(long ptr, long[] streamPtrs);
  80 +
71 private native boolean isEndpoint(long ptr, long streamPtr); 81 private native boolean isEndpoint(long ptr, long streamPtr);
72 82
73 private native boolean isReady(long ptr, long streamPtr); 83 private native boolean isReady(long ptr, long streamPtr);
@@ -366,22 +366,44 @@ Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_createStream(JNIEnv * /*env*/, @@ -366,22 +366,44 @@ Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_createStream(JNIEnv * /*env*/,
366 366
367 SHERPA_ONNX_EXTERN_C 367 SHERPA_ONNX_EXTERN_C
368 JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_decode( 368 JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_decode(
369 - JNIEnv *env, jobject /*obj*/, jlong ptr, jlong streamPtr) { 369 + JNIEnv *env, jobject /*obj*/, jlong ptr, jlong stream_ptr) {
370 SafeJNI(env, "OfflineRecognizer_decode", [&] { 370 SafeJNI(env, "OfflineRecognizer_decode", [&] {
371 if (!ValidatePointer(env, ptr, "OfflineRecognizer_decode", 371 if (!ValidatePointer(env, ptr, "OfflineRecognizer_decode",
372 "OfflineRecognizer pointer is null.") || 372 "OfflineRecognizer pointer is null.") ||
373 - !ValidatePointer(env, streamPtr, "OfflineRecognizer_decode", 373 + !ValidatePointer(env, stream_ptr, "OfflineRecognizer_decode",
374 "OfflineStream pointer is null.")) { 374 "OfflineStream pointer is null.")) {
375 return; 375 return;
376 } 376 }
377 377
378 auto recognizer = reinterpret_cast<sherpa_onnx::OfflineRecognizer *>(ptr); 378 auto recognizer = reinterpret_cast<sherpa_onnx::OfflineRecognizer *>(ptr);
379 - auto stream = reinterpret_cast<sherpa_onnx::OfflineStream *>(streamPtr); 379 + auto stream = reinterpret_cast<sherpa_onnx::OfflineStream *>(stream_ptr);
380 recognizer->DecodeStream(stream); 380 recognizer->DecodeStream(stream);
381 }); 381 });
382 } 382 }
383 383
384 SHERPA_ONNX_EXTERN_C 384 SHERPA_ONNX_EXTERN_C
  385 +JNIEXPORT void JNICALL
  386 +Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_decodeStreams(
  387 + JNIEnv *env, jobject /*obj*/, jlong ptr, jlongArray stream_ptrs) {
  388 + SafeJNI(env, "OfflineRecognizer_decode_streams", [&] {
  389 + if (!ValidatePointer(env, ptr, "OfflineRecognizer_decode_streams",
  390 + "OfflineRecognizer pointer is null.")) {
  391 + return;
  392 + }
  393 +
  394 + auto recognizer = reinterpret_cast<sherpa_onnx::OfflineRecognizer *>(ptr);
  395 +
  396 + jlong *p = env->GetLongArrayElements(stream_ptrs, nullptr);
  397 + jsize n = env->GetArrayLength(stream_ptrs);
  398 +
  399 + auto ss = reinterpret_cast<sherpa_onnx::OfflineStream **>(p);
  400 + recognizer->DecodeStreams(ss, n);
  401 +
  402 + env->ReleaseLongArrayElements(stream_ptrs, p, JNI_ABORT);
  403 + });
  404 +}
  405 +
  406 +SHERPA_ONNX_EXTERN_C
385 JNIEXPORT jobjectArray JNICALL 407 JNIEXPORT jobjectArray JNICALL
386 Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_getResult(JNIEnv *env, 408 Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_getResult(JNIEnv *env,
387 jobject /*obj*/, 409 jobject /*obj*/,
@@ -340,6 +340,22 @@ JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_OnlineRecognizer_decode( @@ -340,6 +340,22 @@ JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_OnlineRecognizer_decode(
340 } 340 }
341 341
342 SHERPA_ONNX_EXTERN_C 342 SHERPA_ONNX_EXTERN_C
  343 +JNIEXPORT void JNICALL
  344 +Java_com_k2fsa_sherpa_onnx_OnlineRecognizer_decodeStreams(
  345 + JNIEnv *env, jobject /*obj*/, jlong ptr, jlongArray stream_ptrs) {
  346 + auto recognizer = reinterpret_cast<sherpa_onnx::OnlineRecognizer *>(ptr);
  347 +
  348 + jlong *p = env->GetLongArrayElements(stream_ptrs, nullptr);
  349 + jsize n = env->GetArrayLength(stream_ptrs);
  350 +
  351 + auto ss = reinterpret_cast<sherpa_onnx::OnlineStream **>(p);
  352 +
  353 + recognizer->DecodeStreams(ss, n);
  354 +
  355 + env->ReleaseLongArrayElements(stream_ptrs, p, JNI_ABORT);
  356 +}
  357 +
  358 +SHERPA_ONNX_EXTERN_C
343 JNIEXPORT jlong JNICALL 359 JNIEXPORT jlong JNICALL
344 Java_com_k2fsa_sherpa_onnx_OnlineRecognizer_createStream(JNIEnv *env, 360 Java_com_k2fsa_sherpa_onnx_OnlineRecognizer_createStream(JNIEnv *env,
345 jobject /*obj*/, 361 jobject /*obj*/,