正在显示
7 个修改的文件
包含
897 行增加
和
905 行删除
| 1 | - | ||
| 2 | -ENTRY_POINT = ./ | ||
| 3 | - | ||
| 4 | -LIB_SRC_DIR := ../sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx | ||
| 5 | - | ||
| 6 | -LIB_FILES = \ | ||
| 7 | - $(LIB_SRC_DIR)/EndpointRule.java \ | ||
| 8 | - $(LIB_SRC_DIR)/EndpointConfig.java \ | ||
| 9 | - $(LIB_SRC_DIR)/FeatureConfig.java \ | ||
| 10 | - $(LIB_SRC_DIR)/OnlineLMConfig.java \ | ||
| 11 | - $(LIB_SRC_DIR)/OnlineTransducerModelConfig.java \ | ||
| 12 | - $(LIB_SRC_DIR)/OnlineParaformerModelConfig.java \ | ||
| 13 | - $(LIB_SRC_DIR)/OnlineModelConfig.java \ | ||
| 14 | - $(LIB_SRC_DIR)/OnlineRecognizerConfig.java \ | ||
| 15 | - $(LIB_SRC_DIR)/OnlineStream.java \ | ||
| 16 | - $(LIB_SRC_DIR)/OnlineRecognizer.java \ | ||
| 17 | - | ||
| 18 | -WEBSOCKET_DIR:= ./src/websocketsrv | ||
| 19 | -WEBSOCKET_FILES = \ | ||
| 20 | - $(WEBSOCKET_DIR)/ConnectionData.java \ | ||
| 21 | - $(WEBSOCKET_DIR)/DecoderThreadHandler.java \ | ||
| 22 | - $(WEBSOCKET_DIR)/StreamThreadHandler.java \ | ||
| 23 | - $(WEBSOCKET_DIR)/AsrWebsocketServer.java \ | ||
| 24 | - $(WEBSOCKET_DIR)/AsrWebsocketClient.java \ | ||
| 25 | - | ||
| 26 | - | ||
| 27 | -LIB_BUILD_DIR = ./lib | ||
| 28 | - | ||
| 29 | - | ||
| 30 | -EXAMPLE_FILE = DecodeFile.java | ||
| 31 | - | ||
| 32 | -EXAMPLE_Mic = DecodeMic.java | ||
| 33 | - | ||
| 34 | -JAVAC = javac | ||
| 35 | - | ||
| 36 | -BUILD_DIR = build | ||
| 37 | - | ||
| 38 | - | ||
| 39 | -RUNJFLAGS = -Dfile.encoding=utf-8 | ||
| 40 | - | ||
| 41 | - | ||
| 42 | -vpath %.class $(BUILD_DIR) | ||
| 43 | -vpath %.java src | ||
| 44 | - | ||
| 45 | - | ||
| 46 | -buildfile: | ||
| 47 | - $(JAVAC) -cp lib/sherpaonnx.jar -d $(BUILD_DIR) -encoding UTF-8 src/$(EXAMPLE_FILE) | ||
| 48 | - | ||
| 49 | -buildmic: | ||
| 50 | - $(JAVAC) -cp lib/sherpaonnx.jar -d $(BUILD_DIR) -encoding UTF-8 src/$(EXAMPLE_Mic) | ||
| 51 | - | ||
| 52 | -rebuild: clean all | ||
| 53 | - | ||
| 54 | -.PHONY: clean run downjar | ||
| 55 | - | ||
| 56 | -downjar: | ||
| 57 | - wget https://repo1.maven.org/maven2/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar -P ./lib/ | ||
| 58 | - wget https://repo1.maven.org/maven2/org/slf4j/slf4j-simple/1.7.25/slf4j-simple-1.7.25.jar -P ./lib/ | ||
| 59 | - wget https://github.com/TooTallNate/Java-WebSocket/releases/download/v1.5.3/Java-WebSocket-1.5.3.jar -P ./lib/ | ||
| 60 | - | ||
| 61 | - | ||
| 62 | -clean: | ||
| 63 | - rm -frv $(BUILD_DIR)/* | ||
| 64 | - rm -frv $(LIB_BUILD_DIR)/* | ||
| 65 | - mkdir -p $(BUILD_DIR) | ||
| 66 | - mkdir -p ./lib | ||
| 67 | - | ||
| 68 | - | ||
| 69 | -runfile: | ||
| 70 | - | ||
| 71 | - java -cp ./lib/sherpaonnx.jar:build $(RUNJFLAGS) DecodeFile | ||
| 72 | - | ||
| 73 | -runmic: | ||
| 74 | - | ||
| 75 | - java -cp ./lib/sherpaonnx.jar:build $(RUNJFLAGS) DecodeMic | ||
| 76 | - | ||
| 77 | -runsrv: | ||
| 78 | - java -cp $(BUILD_DIR):lib/Java-WebSocket-1.5.3.jar:lib/slf4j-simple-1.7.25.jar:lib/slf4j-api-1.7.25.jar:../lib/sherpaonnx.jar $(RUNJFLAGS) websocketsrv.AsrWebsocketServer ../build/lib/libsherpa-onnx-jni.so ./modeltest.cfg | ||
| 79 | - | ||
| 80 | -runclient: | ||
| 81 | - java -cp $(BUILD_DIR):lib/Java-WebSocket-1.5.3.jar:lib/slf4j-simple-1.7.25.jar:lib/slf4j-api-1.7.25.jar:../lib/sherpaonnx.jar $(RUNJFLAGS) websocketsrv.AsrWebsocketClient ../build/lib/libsherpa-onnx-jni.so 127.0.0.1 8890 ./test.wav 32 | ||
| 82 | - | ||
| 83 | -buildlib: $(LIB_FILES:.java=.class) | ||
| 84 | - | ||
| 85 | - | ||
| 86 | -%.class: %.java | ||
| 87 | - | ||
| 88 | - $(JAVAC) -cp $(BUILD_DIR) -d $(BUILD_DIR) -encoding UTF-8 $< | ||
| 89 | - | ||
| 90 | -buildwebsocket: $(WEBSOCKET_FILES:.java=.class) | ||
| 91 | - | ||
| 92 | - | ||
| 93 | -%.class: %.java | ||
| 94 | - | ||
| 95 | - $(JAVAC) -cp $(BUILD_DIR):lib/slf4j-simple-1.7.25.jar:lib/slf4j-api-1.7.25.jar:lib/Java-WebSocket-1.5.3.jar:../lib/sherpaonnx.jar -d $(BUILD_DIR) -encoding UTF-8 $< | ||
| 96 | - | ||
| 97 | -packjar: | ||
| 98 | - jar cvfe lib/sherpaonnx.jar . -C $(BUILD_DIR) . | ||
| 99 | - | ||
| 100 | -all: clean buildlib packjar buildfile buildmic downjar buildwebsocket | ||
| 101 | - | ||
| 102 | - | ||
| 103 | - | ||
| 104 | - | ||
| 105 | - | ||
| 106 | - | 1 | + |
| 2 | +ENTRY_POINT = ./ | ||
| 3 | + | ||
| 4 | +LIB_SRC_DIR := ../sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx | ||
| 5 | + | ||
| 6 | +LIB_FILES = \ | ||
| 7 | + $(LIB_SRC_DIR)/EndpointRule.java \ | ||
| 8 | + $(LIB_SRC_DIR)/EndpointConfig.java \ | ||
| 9 | + $(LIB_SRC_DIR)/FeatureConfig.java \ | ||
| 10 | + $(LIB_SRC_DIR)/OnlineLMConfig.java \ | ||
| 11 | + $(LIB_SRC_DIR)/OnlineTransducerModelConfig.java \ | ||
| 12 | + $(LIB_SRC_DIR)/OnlineParaformerModelConfig.java \ | ||
| 13 | + $(LIB_SRC_DIR)/OnlineModelConfig.java \ | ||
| 14 | + $(LIB_SRC_DIR)/OnlineRecognizerConfig.java \ | ||
| 15 | + $(LIB_SRC_DIR)/OnlineStream.java \ | ||
| 16 | + $(LIB_SRC_DIR)/OnlineRecognizer.java \ | ||
| 17 | + | ||
| 18 | +WEBSOCKET_DIR:= ./src/websocketsrv | ||
| 19 | +WEBSOCKET_FILES = \ | ||
| 20 | + $(WEBSOCKET_DIR)/ConnectionData.java \ | ||
| 21 | + $(WEBSOCKET_DIR)/DecoderThreadHandler.java \ | ||
| 22 | + $(WEBSOCKET_DIR)/StreamThreadHandler.java \ | ||
| 23 | + $(WEBSOCKET_DIR)/AsrWebsocketServer.java \ | ||
| 24 | + $(WEBSOCKET_DIR)/AsrWebsocketClient.java \ | ||
| 25 | + | ||
| 26 | + | ||
| 27 | +LIB_BUILD_DIR = ./lib | ||
| 28 | + | ||
| 29 | + | ||
| 30 | +EXAMPLE_FILE = DecodeFile.java | ||
| 31 | + | ||
| 32 | +EXAMPLE_Mic = DecodeMic.java | ||
| 33 | + | ||
| 34 | +JAVAC = javac | ||
| 35 | + | ||
| 36 | +BUILD_DIR = build | ||
| 37 | + | ||
| 38 | + | ||
| 39 | +RUNJFLAGS = -Dfile.encoding=utf-8 | ||
| 40 | + | ||
| 41 | +vpath %.class $(BUILD_DIR) | ||
| 42 | +vpath %.java src | ||
| 43 | + | ||
| 44 | + | ||
| 45 | +buildfile: | ||
| 46 | + $(JAVAC) -cp lib/sherpaonnx.jar -d $(BUILD_DIR) -encoding UTF-8 src/$(EXAMPLE_FILE) | ||
| 47 | + | ||
| 48 | +buildmic: | ||
| 49 | + $(JAVAC) -cp lib/sherpaonnx.jar -d $(BUILD_DIR) -encoding UTF-8 src/$(EXAMPLE_Mic) | ||
| 50 | + | ||
| 51 | +rebuild: clean all | ||
| 52 | + | ||
| 53 | +.PHONY: clean run downjar | ||
| 54 | + | ||
| 55 | +downjar: | ||
| 56 | + wget https://repo1.maven.org/maven2/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar -P ./lib/ | ||
| 57 | + wget https://repo1.maven.org/maven2/org/slf4j/slf4j-simple/1.7.25/slf4j-simple-1.7.25.jar -P ./lib/ | ||
| 58 | + wget https://github.com/TooTallNate/Java-WebSocket/releases/download/v1.5.3/Java-WebSocket-1.5.3.jar -P ./lib/ | ||
| 59 | + | ||
| 60 | + | ||
| 61 | +clean: | ||
| 62 | + rm -frv $(BUILD_DIR)/* | ||
| 63 | + rm -frv $(LIB_BUILD_DIR)/* | ||
| 64 | + mkdir -p $(BUILD_DIR) | ||
| 65 | + mkdir -p ./lib | ||
| 66 | + | ||
| 67 | +runfile: | ||
| 68 | + | ||
| 69 | + java -cp ./lib/sherpaonnx.jar:build $(RUNJFLAGS) DecodeFile | ||
| 70 | + | ||
| 71 | +runmic: | ||
| 72 | + | ||
| 73 | + java -cp ./lib/sherpaonnx.jar:build $(RUNJFLAGS) DecodeMic | ||
| 74 | + | ||
| 75 | +runsrv: | ||
| 76 | + java -cp $(BUILD_DIR):lib/Java-WebSocket-1.5.3.jar:lib/slf4j-simple-1.7.25.jar:lib/slf4j-api-1.7.25.jar:../lib/sherpaonnx.jar $(RUNJFLAGS) websocketsrv.AsrWebsocketServer ../build/lib/libsherpa-onnx-jni.so ./modeltest.cfg | ||
| 77 | + | ||
| 78 | +runclient: | ||
| 79 | + java -cp $(BUILD_DIR):lib/Java-WebSocket-1.5.3.jar:lib/slf4j-simple-1.7.25.jar:lib/slf4j-api-1.7.25.jar:../lib/sherpaonnx.jar $(RUNJFLAGS) websocketsrv.AsrWebsocketClient ../build/lib/libsherpa-onnx-jni.so 127.0.0.1 8890 ./test.wav 32 | ||
| 80 | + | ||
| 81 | +buildlib: $(LIB_FILES:.java=.class) | ||
| 82 | + | ||
| 83 | + | ||
| 84 | +%.class: %.java | ||
| 85 | + | ||
| 86 | + $(JAVAC) -cp $(BUILD_DIR) -d $(BUILD_DIR) -encoding UTF-8 $< | ||
| 87 | + | ||
| 88 | +buildwebsocket: $(WEBSOCKET_FILES:.java=.class) | ||
| 89 | + | ||
| 90 | + | ||
| 91 | +%.class: %.java | ||
| 92 | + | ||
| 93 | + $(JAVAC) -cp $(BUILD_DIR):lib/slf4j-simple-1.7.25.jar:lib/slf4j-api-1.7.25.jar:lib/Java-WebSocket-1.5.3.jar:../lib/sherpaonnx.jar -d $(BUILD_DIR) -encoding UTF-8 $< | ||
| 94 | + | ||
| 95 | +packjar: | ||
| 96 | + jar cvfe lib/sherpaonnx.jar . -C $(BUILD_DIR) . | ||
| 97 | + | ||
| 98 | +all: clean buildlib packjar buildfile buildmic downjar buildwebsocket |
| 1 | -#model config | ||
| 2 | -sample_rate=16000 | ||
| 3 | -feature_dim=80 | ||
| 4 | -rule1_min_trailing_silence=2.4 | ||
| 5 | -rule2_min_trailing_silence=1.2 | ||
| 6 | -rule3_min_utterance_length=20 | ||
| 7 | -encoder=/sherpa/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/encoder-epoch-99-avg-1.onnx | ||
| 8 | -decoder=/sherpa/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/decoder-epoch-99-avg-1.onnx | ||
| 9 | -joiner=/sherpa/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/joiner-epoch-99-avg-1.onnx | ||
| 10 | -tokens=/sherpa/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt | ||
| 11 | -num_threads=4 | ||
| 12 | -enable_endpoint_detection=true | ||
| 13 | -decoding_method=modified_beam_search | ||
| 14 | -max_active_paths=4 | ||
| 15 | -lm_model= | ||
| 16 | -lm_scale=0.5 | ||
| 17 | -model_type=zipformer | ||
| 18 | - | ||
| 19 | -#websocket server config | ||
| 20 | -port=8890 | ||
| 21 | -connection_thread_num=16 | ||
| 22 | -stream_thread_num=16 | ||
| 23 | -decoder_thread_num=16 | ||
| 24 | -parallel_decoder_num=16 | ||
| 25 | -decoder_time_idle=200 | ||
| 26 | -deocder_time_out=30000 | 1 | +#model config |
| 2 | +sample_rate=16000 | ||
| 3 | +feature_dim=80 | ||
| 4 | +rule1_min_trailing_silence=2.4 | ||
| 5 | +rule2_min_trailing_silence=1.2 | ||
| 6 | +rule3_min_utterance_length=20 | ||
| 7 | +encoder=/sherpa/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/encoder-epoch-99-avg-1.onnx | ||
| 8 | +decoder=/sherpa/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/decoder-epoch-99-avg-1.onnx | ||
| 9 | +joiner=/sherpa/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/joiner-epoch-99-avg-1.onnx | ||
| 10 | +tokens=/sherpa/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt | ||
| 11 | +num_threads=4 | ||
| 12 | +enable_endpoint_detection=true | ||
| 13 | +decoding_method=modified_beam_search | ||
| 14 | +max_active_paths=4 | ||
| 15 | +lm_model= | ||
| 16 | +lm_scale=0.5 | ||
| 17 | +model_type=zipformer | ||
| 18 | + | ||
| 19 | +#websocket server config | ||
| 20 | +port=8890 | ||
| 21 | +connection_thread_num=16 | ||
| 22 | +stream_thread_num=16 | ||
| 23 | +decoder_thread_num=16 | ||
| 24 | +parallel_decoder_num=16 | ||
| 25 | +decoder_time_idle=200 | ||
| 26 | +deocder_time_out=30000 |
| 1 | -/* | ||
| 2 | - * // Copyright 2022-2023 by zhaoming | ||
| 3 | - */ | ||
| 4 | -/* | ||
| 5 | -Config modelconfig.cfg | ||
| 6 | - sample_rate=16000 | ||
| 7 | - feature_dim=80 | ||
| 8 | - rule1_min_trailing_silence=2.4 | ||
| 9 | - rule2_min_trailing_silence=1.2 | ||
| 10 | - rule3_min_utterance_length=20 | ||
| 11 | - encoder=/sherpa-onnx/build/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/encoder-epoch-99-avg-1.onnx | ||
| 12 | - decoder=/sherpa-onnx/build/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/decoder-epoch-99-avg-1.onnx | ||
| 13 | - joiner=/sherpa-onnx/build/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/joiner-epoch-99-avg-1.onnx | ||
| 14 | - tokens=/sherpa-onnx/build/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt | ||
| 15 | - num_threads=4 | ||
| 16 | - enable_endpoint_detection=false | ||
| 17 | - decoding_method=greedy_search | ||
| 18 | - max_active_paths=4 | ||
| 19 | -*/ | ||
| 20 | - | ||
| 21 | -import com.k2fsa.sherpa.onnx.OnlineRecognizer; | ||
| 22 | -import com.k2fsa.sherpa.onnx.OnlineStream; | ||
| 23 | -import java.io.*; | ||
| 24 | -import java.nio.charset.StandardCharsets; | ||
| 25 | - | ||
| 26 | -public class DecodeFile { | ||
| 27 | - OnlineRecognizer rcgOjb; | ||
| 28 | - OnlineStream streamObj; | ||
| 29 | - String wavfilename; | ||
| 30 | - | ||
| 31 | - public DecodeFile(String fileName) { | ||
| 32 | - wavfilename = fileName; | ||
| 33 | - } | ||
| 34 | - | ||
| 35 | - public void initModelWithPara() { | ||
| 36 | - try { | ||
| 37 | - String modelDir = | ||
| 38 | - "/sherpa-onnx/build_old/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"; | ||
| 39 | - String encoder = modelDir + "/encoder-epoch-99-avg-1.onnx"; | ||
| 40 | - String decoder = modelDir + "/decoder-epoch-99-avg-1.onnx"; | ||
| 41 | - String joiner = modelDir + "/joiner-epoch-99-avg-1.onnx"; | ||
| 42 | - String tokens = modelDir + "/tokens.txt"; | ||
| 43 | - int numThreads = 4; | ||
| 44 | - int sampleRate = 16000; | ||
| 45 | - int featureDim = 80; | ||
| 46 | - boolean enableEndpointDetection = false; | ||
| 47 | - float rule1MinTrailingSilence = 2.4F; | ||
| 48 | - float rule2MinTrailingSilence = 1.2F; | ||
| 49 | - float rule3MinUtteranceLength = 20F; | ||
| 50 | - String decodingMethod = "greedy_search"; | ||
| 51 | - int maxActivePaths = 4; | ||
| 52 | - String lm_model = ""; | ||
| 53 | - float lm_scale = 0.5F; | ||
| 54 | - String modelType = "zipformer"; | ||
| 55 | - rcgOjb = | ||
| 56 | - new OnlineRecognizer( | ||
| 57 | - tokens, | ||
| 58 | - encoder, | ||
| 59 | - decoder, | ||
| 60 | - joiner, | ||
| 61 | - numThreads, | ||
| 62 | - sampleRate, | ||
| 63 | - featureDim, | ||
| 64 | - enableEndpointDetection, | ||
| 65 | - rule1MinTrailingSilence, | ||
| 66 | - rule2MinTrailingSilence, | ||
| 67 | - rule3MinUtteranceLength, | ||
| 68 | - decodingMethod, | ||
| 69 | - lm_model, | ||
| 70 | - lm_scale, | ||
| 71 | - maxActivePaths, | ||
| 72 | - modelType); | ||
| 73 | - streamObj = rcgOjb.createStream(); | ||
| 74 | - } catch (Exception e) { | ||
| 75 | - System.err.println(e); | ||
| 76 | - e.printStackTrace(); | ||
| 77 | - } | ||
| 78 | - } | ||
| 79 | - | ||
| 80 | - public void initModelWithCfg(String cfgFile) { | ||
| 81 | - try { | ||
| 82 | - // you should set setCfgPath() before running this | ||
| 83 | - rcgOjb = new OnlineRecognizer(cfgFile); | ||
| 84 | - streamObj = rcgOjb.createStream(); | ||
| 85 | - } catch (Exception e) { | ||
| 86 | - System.err.println(e); | ||
| 87 | - e.printStackTrace(); | ||
| 88 | - } | ||
| 89 | - } | ||
| 90 | - | ||
| 91 | - public void simpleExample() { | ||
| 92 | - try { | ||
| 93 | - float[] buffer = rcgOjb.readWavFile(wavfilename); // read data from file | ||
| 94 | - streamObj.acceptWaveform(buffer); // feed stream with data | ||
| 95 | - streamObj.inputFinished(); // tell engine you done with all data | ||
| 96 | - OnlineStream ssObj[] = new OnlineStream[1]; | ||
| 97 | - while (rcgOjb.isReady(streamObj)) { // engine is ready for unprocessed data | ||
| 98 | - ssObj[0] = streamObj; | ||
| 99 | - rcgOjb.decodeStreams(ssObj); // decode for multiple stream | ||
| 100 | - // rcgOjb.DecodeStream(streamObj); // decode for single stream | ||
| 101 | - } | ||
| 102 | - | ||
| 103 | - String recText = "simple:" + rcgOjb.getResult(streamObj) + "\n"; | ||
| 104 | - byte[] utf8Data = recText.getBytes(StandardCharsets.UTF_8); | ||
| 105 | - System.out.println(new String(utf8Data)); | ||
| 106 | - rcgOjb.reSet(streamObj); | ||
| 107 | - rcgOjb.releaseStream(streamObj); // release stream | ||
| 108 | - rcgOjb.release(); // release recognizer | ||
| 109 | - | ||
| 110 | - } catch (Exception e) { | ||
| 111 | - System.err.println(e); | ||
| 112 | - e.printStackTrace(); | ||
| 113 | - } | ||
| 114 | - } | ||
| 115 | - | ||
| 116 | - public void streamExample() { | ||
| 117 | - try { | ||
| 118 | - float[] buffer = rcgOjb.readWavFile(wavfilename); // read data from file | ||
| 119 | - float[] chunk = new float[1600]; // //each time read 1600(0.1s) data | ||
| 120 | - int chunkIndex = 0; | ||
| 121 | - for (int i = 0; i < buffer.length; i++) // total wav length loop | ||
| 122 | - { | ||
| 123 | - chunk[chunkIndex] = buffer[i]; | ||
| 124 | - chunkIndex++; | ||
| 125 | - if (chunkIndex >= 1600 || i == (buffer.length - 1)) { | ||
| 126 | - chunkIndex = 0; | ||
| 127 | - streamObj.acceptWaveform(chunk); // feed chunk | ||
| 128 | - if (rcgOjb.isReady(streamObj)) { | ||
| 129 | - rcgOjb.decodeStream(streamObj); | ||
| 130 | - } | ||
| 131 | - String testDate = rcgOjb.getResult(streamObj); | ||
| 132 | - byte[] utf8Data = testDate.getBytes(StandardCharsets.UTF_8); | ||
| 133 | - | ||
| 134 | - if (utf8Data.length > 0) { | ||
| 135 | - System.out.println(Float.valueOf((float) i / 16000) + ":" + new String(utf8Data)); | ||
| 136 | - } | ||
| 137 | - } | ||
| 138 | - } | ||
| 139 | - streamObj.inputFinished(); | ||
| 140 | - while (rcgOjb.isReady(streamObj)) { | ||
| 141 | - rcgOjb.decodeStream(streamObj); | ||
| 142 | - } | ||
| 143 | - | ||
| 144 | - String recText = "stream:" + rcgOjb.getResult(streamObj) + "\n"; | ||
| 145 | - byte[] utf8Data = recText.getBytes(StandardCharsets.UTF_8); | ||
| 146 | - System.out.println(new String(utf8Data)); | ||
| 147 | - rcgOjb.reSet(streamObj); | ||
| 148 | - rcgOjb.releaseStream(streamObj); // release stream | ||
| 149 | - rcgOjb.release(); // release recognizer | ||
| 150 | - | ||
| 151 | - } catch (Exception e) { | ||
| 152 | - System.err.println(e); | ||
| 153 | - e.printStackTrace(); | ||
| 154 | - } | ||
| 155 | - } | ||
| 156 | - | ||
| 157 | - public static void main(String[] args) { | ||
| 158 | - try { | ||
| 159 | - String appDir = System.getProperty("user.dir"); | ||
| 160 | - System.out.println("appdir=" + appDir); | ||
| 161 | - String fileName = appDir + "/test.wav"; | ||
| 162 | - String cfgPath = appDir + "/modeltest.cfg"; | ||
| 163 | - String soPath = appDir + "/../build/lib/libsherpa-onnx-jni.so"; | ||
| 164 | - OnlineRecognizer.setSoPath(soPath); | ||
| 165 | - DecodeFile rcgDemo = new DecodeFile(fileName); | ||
| 166 | - | ||
| 167 | - // ***************** */ | ||
| 168 | - rcgDemo.initModelWithCfg(cfgPath); | ||
| 169 | - rcgDemo.streamExample(); | ||
| 170 | - // **************** */ | ||
| 171 | - rcgDemo.initModelWithCfg(cfgPath); | ||
| 172 | - rcgDemo.simpleExample(); | ||
| 173 | - | ||
| 174 | - } catch (Exception e) { | ||
| 175 | - System.err.println(e); | ||
| 176 | - e.printStackTrace(); | ||
| 177 | - } | ||
| 178 | - } | ||
| 179 | -} | 1 | +/* |
| 2 | + * // Copyright 2022-2023 by zhaoming | ||
| 3 | + */ | ||
| 4 | +/* | ||
| 5 | +Config modelconfig.cfg | ||
| 6 | + sample_rate=16000 | ||
| 7 | + feature_dim=80 | ||
| 8 | + rule1_min_trailing_silence=2.4 | ||
| 9 | + rule2_min_trailing_silence=1.2 | ||
| 10 | + rule3_min_utterance_length=20 | ||
| 11 | + encoder=/sherpa-onnx/build/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/encoder-epoch-99-avg-1.onnx | ||
| 12 | + decoder=/sherpa-onnx/build/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/decoder-epoch-99-avg-1.onnx | ||
| 13 | + joiner=/sherpa-onnx/build/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/joiner-epoch-99-avg-1.onnx | ||
| 14 | + tokens=/sherpa-onnx/build/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt | ||
| 15 | + num_threads=4 | ||
| 16 | + enable_endpoint_detection=false | ||
| 17 | + decoding_method=greedy_search | ||
| 18 | + max_active_paths=4 | ||
| 19 | +*/ | ||
| 20 | + | ||
| 21 | +import com.k2fsa.sherpa.onnx.OnlineRecognizer; | ||
| 22 | +import com.k2fsa.sherpa.onnx.OnlineStream; | ||
| 23 | +import java.io.*; | ||
| 24 | +import java.nio.charset.StandardCharsets; | ||
| 25 | + | ||
| 26 | +public class DecodeFile { | ||
| 27 | + OnlineRecognizer rcgOjb; | ||
| 28 | + OnlineStream streamObj; | ||
| 29 | + String wavfilename; | ||
| 30 | + | ||
| 31 | + public DecodeFile(String fileName) { | ||
| 32 | + wavfilename = fileName; | ||
| 33 | + } | ||
| 34 | + | ||
| 35 | + public void initModelWithPara() { | ||
| 36 | + try { | ||
| 37 | + String modelDir = | ||
| 38 | + "/sherpa-onnx/build_old/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"; | ||
| 39 | + String encoder = modelDir + "/encoder-epoch-99-avg-1.onnx"; | ||
| 40 | + String decoder = modelDir + "/decoder-epoch-99-avg-1.onnx"; | ||
| 41 | + String joiner = modelDir + "/joiner-epoch-99-avg-1.onnx"; | ||
| 42 | + String tokens = modelDir + "/tokens.txt"; | ||
| 43 | + int numThreads = 4; | ||
| 44 | + int sampleRate = 16000; | ||
| 45 | + int featureDim = 80; | ||
| 46 | + boolean enableEndpointDetection = false; | ||
| 47 | + float rule1MinTrailingSilence = 2.4F; | ||
| 48 | + float rule2MinTrailingSilence = 1.2F; | ||
| 49 | + float rule3MinUtteranceLength = 20F; | ||
| 50 | + String decodingMethod = "greedy_search"; | ||
| 51 | + int maxActivePaths = 4; | ||
| 52 | + String lm_model = ""; | ||
| 53 | + float lm_scale = 0.5F; | ||
| 54 | + String modelType = "zipformer"; | ||
| 55 | + rcgOjb = | ||
| 56 | + new OnlineRecognizer( | ||
| 57 | + tokens, | ||
| 58 | + encoder, | ||
| 59 | + decoder, | ||
| 60 | + joiner, | ||
| 61 | + numThreads, | ||
| 62 | + sampleRate, | ||
| 63 | + featureDim, | ||
| 64 | + enableEndpointDetection, | ||
| 65 | + rule1MinTrailingSilence, | ||
| 66 | + rule2MinTrailingSilence, | ||
| 67 | + rule3MinUtteranceLength, | ||
| 68 | + decodingMethod, | ||
| 69 | + lm_model, | ||
| 70 | + lm_scale, | ||
| 71 | + maxActivePaths, | ||
| 72 | + modelType); | ||
| 73 | + streamObj = rcgOjb.createStream(); | ||
| 74 | + } catch (Exception e) { | ||
| 75 | + System.err.println(e); | ||
| 76 | + e.printStackTrace(); | ||
| 77 | + } | ||
| 78 | + } | ||
| 79 | + | ||
| 80 | + public void initModelWithCfg(String cfgFile) { | ||
| 81 | + try { | ||
| 82 | + // you should set setCfgPath() before running this | ||
| 83 | + rcgOjb = new OnlineRecognizer(cfgFile); | ||
| 84 | + streamObj = rcgOjb.createStream(); | ||
| 85 | + } catch (Exception e) { | ||
| 86 | + System.err.println(e); | ||
| 87 | + e.printStackTrace(); | ||
| 88 | + } | ||
| 89 | + } | ||
| 90 | + | ||
| 91 | + public void simpleExample() { | ||
| 92 | + try { | ||
| 93 | + float[] buffer = rcgOjb.readWavFile(wavfilename); // read data from file | ||
| 94 | + streamObj.acceptWaveform(buffer); // feed stream with data | ||
| 95 | + streamObj.inputFinished(); // tell engine you done with all data | ||
| 96 | + OnlineStream ssObj[] = new OnlineStream[1]; | ||
| 97 | + while (rcgOjb.isReady(streamObj)) { // engine is ready for unprocessed data | ||
| 98 | + ssObj[0] = streamObj; | ||
| 99 | + rcgOjb.decodeStreams(ssObj); // decode for multiple stream | ||
| 100 | + // rcgOjb.DecodeStream(streamObj); // decode for single stream | ||
| 101 | + } | ||
| 102 | + | ||
| 103 | + String recText = "simple:" + rcgOjb.getResult(streamObj) + "\n"; | ||
| 104 | + byte[] utf8Data = recText.getBytes(StandardCharsets.UTF_8); | ||
| 105 | + System.out.println(new String(utf8Data)); | ||
| 106 | + rcgOjb.reSet(streamObj); | ||
| 107 | + rcgOjb.releaseStream(streamObj); // release stream | ||
| 108 | + rcgOjb.release(); // release recognizer | ||
| 109 | + | ||
| 110 | + } catch (Exception e) { | ||
| 111 | + System.err.println(e); | ||
| 112 | + e.printStackTrace(); | ||
| 113 | + } | ||
| 114 | + } | ||
| 115 | + | ||
| 116 | + public void streamExample() { | ||
| 117 | + try { | ||
| 118 | + float[] buffer = rcgOjb.readWavFile(wavfilename); // read data from file | ||
| 119 | + float[] chunk = new float[1600]; // //each time read 1600(0.1s) data | ||
| 120 | + int chunkIndex = 0; | ||
| 121 | + for (int i = 0; i < buffer.length; i++) // total wav length loop | ||
| 122 | + { | ||
| 123 | + chunk[chunkIndex] = buffer[i]; | ||
| 124 | + chunkIndex++; | ||
| 125 | + if (chunkIndex >= 1600 || i == (buffer.length - 1)) { | ||
| 126 | + chunkIndex = 0; | ||
| 127 | + streamObj.acceptWaveform(chunk); // feed chunk | ||
| 128 | + if (rcgOjb.isReady(streamObj)) { | ||
| 129 | + rcgOjb.decodeStream(streamObj); | ||
| 130 | + } | ||
| 131 | + String testDate = rcgOjb.getResult(streamObj); | ||
| 132 | + byte[] utf8Data = testDate.getBytes(StandardCharsets.UTF_8); | ||
| 133 | + | ||
| 134 | + if (utf8Data.length > 0) { | ||
| 135 | + System.out.println(Float.valueOf((float) i / 16000) + ":" + new String(utf8Data)); | ||
| 136 | + } | ||
| 137 | + } | ||
| 138 | + } | ||
| 139 | + streamObj.inputFinished(); | ||
| 140 | + while (rcgOjb.isReady(streamObj)) { | ||
| 141 | + rcgOjb.decodeStream(streamObj); | ||
| 142 | + } | ||
| 143 | + | ||
| 144 | + String recText = "stream:" + rcgOjb.getResult(streamObj) + "\n"; | ||
| 145 | + byte[] utf8Data = recText.getBytes(StandardCharsets.UTF_8); | ||
| 146 | + System.out.println(new String(utf8Data)); | ||
| 147 | + rcgOjb.reSet(streamObj); | ||
| 148 | + rcgOjb.releaseStream(streamObj); // release stream | ||
| 149 | + rcgOjb.release(); // release recognizer | ||
| 150 | + | ||
| 151 | + } catch (Exception e) { | ||
| 152 | + System.err.println(e); | ||
| 153 | + e.printStackTrace(); | ||
| 154 | + } | ||
| 155 | + } | ||
| 156 | + | ||
| 157 | + public static void main(String[] args) { | ||
| 158 | + try { | ||
| 159 | + String appDir = System.getProperty("user.dir"); | ||
| 160 | + System.out.println("appdir=" + appDir); | ||
| 161 | + String fileName = appDir + "/test.wav"; | ||
| 162 | + String cfgPath = appDir + "/modeltest.cfg"; | ||
| 163 | + String soPath = appDir + "/../build/lib/libsherpa-onnx-jni.so"; | ||
| 164 | + OnlineRecognizer.setSoPath(soPath); | ||
| 165 | + DecodeFile rcgDemo = new DecodeFile(fileName); | ||
| 166 | + | ||
| 167 | + // ***************** */ | ||
| 168 | + rcgDemo.initModelWithCfg(cfgPath); | ||
| 169 | + rcgDemo.streamExample(); | ||
| 170 | + // **************** */ | ||
| 171 | + rcgDemo.initModelWithCfg(cfgPath); | ||
| 172 | + rcgDemo.simpleExample(); | ||
| 173 | + | ||
| 174 | + } catch (Exception e) { | ||
| 175 | + System.err.println(e); | ||
| 176 | + e.printStackTrace(); | ||
| 177 | + } | ||
| 178 | + } | ||
| 179 | +} |
| 1 | -/* | ||
| 2 | - * // Copyright 2022-2023 by zhaoming | ||
| 3 | - */ | ||
| 4 | -/* | ||
| 5 | -Real-time speech recognition from a microphone with com.k2fsa.sherpa.onnx Java API | ||
| 6 | - | ||
| 7 | -example for cfgFile modelconfig.cfg | ||
| 8 | - sample_rate=16000 | ||
| 9 | - feature_dim=80 | ||
| 10 | - rule1_min_trailing_silence=2.4 | ||
| 11 | - rule2_min_trailing_silence=1.2 | ||
| 12 | - rule3_min_utterance_length=20 | ||
| 13 | - encoder=/sherpa-onnx/build/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/encoder-epoch-99-avg-1.onnx | ||
| 14 | - decoder=/sherpa-onnx/build/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/decoder-epoch-99-avg-1.onnx | ||
| 15 | - joiner=/sherpa-onnx/build/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/joiner-epoch-99-avg-1.onnx | ||
| 16 | - tokens=/sherpa-onnx/build/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt | ||
| 17 | - num_threads=4 | ||
| 18 | - enable_endpoint_detection=true | ||
| 19 | - decoding_method=greedy_search | ||
| 20 | - max_active_paths=4 | ||
| 21 | - | ||
| 22 | -*/ | ||
| 23 | -import com.k2fsa.sherpa.onnx.OnlineRecognizer; | ||
| 24 | -import com.k2fsa.sherpa.onnx.OnlineStream; | ||
| 25 | -import java.io.*; | ||
| 26 | -import java.nio.ByteBuffer; | ||
| 27 | -import java.nio.ByteOrder; | ||
| 28 | -import java.nio.ShortBuffer; | ||
| 29 | -import java.nio.charset.StandardCharsets; | ||
| 30 | -import javax.sound.sampled.AudioFormat; | ||
| 31 | -import javax.sound.sampled.AudioSystem; | ||
| 32 | -import javax.sound.sampled.DataLine; | ||
| 33 | -import javax.sound.sampled.TargetDataLine; | ||
| 34 | - | ||
| 35 | -/** Microphone Example */ | ||
| 36 | -public class DecodeMic { | ||
| 37 | - MicRcgThread micRcgThread = null; // thread handle | ||
| 38 | - | ||
| 39 | - OnlineRecognizer rcgOjb; // the recognizer | ||
| 40 | - | ||
| 41 | - OnlineStream streamObj; // the stream | ||
| 42 | - | ||
| 43 | - public DecodeMic() { | ||
| 44 | - | ||
| 45 | - micRcgThread = new MicRcgThread(); // create a new instance for MicRcgThread | ||
| 46 | - } | ||
| 47 | - | ||
| 48 | - public void open() { | ||
| 49 | - micRcgThread.start(); // start to capture microphone data | ||
| 50 | - } | ||
| 51 | - | ||
| 52 | - public void close() { | ||
| 53 | - micRcgThread.stop(); // close capture | ||
| 54 | - } | ||
| 55 | - | ||
| 56 | - /** init asr engine with config file */ | ||
| 57 | - public void initModelWithCfg(String cfgFile) { | ||
| 58 | - try { | ||
| 59 | - | ||
| 60 | - // set setSoPath() before running this | ||
| 61 | - rcgOjb = new OnlineRecognizer(cfgFile); | ||
| 62 | - | ||
| 63 | - streamObj = rcgOjb.createStream(); // create a stream for asr engine to feed data | ||
| 64 | - } catch (Exception e) { | ||
| 65 | - System.err.println(e); | ||
| 66 | - e.printStackTrace(); | ||
| 67 | - } | ||
| 68 | - } | ||
| 69 | - | ||
| 70 | - /** read data from mic and feed to asr engine */ | ||
| 71 | - class MicRcgThread implements Runnable { | ||
| 72 | - | ||
| 73 | - TargetDataLine capline; // line for capture mic data | ||
| 74 | - | ||
| 75 | - Thread thread; // this thread | ||
| 76 | - int segmentId = 0; // record the segment id when detect endpoint | ||
| 77 | - String preText = ""; // decoded text | ||
| 78 | - | ||
| 79 | - public MicRcgThread() {} | ||
| 80 | - | ||
| 81 | - public void start() { | ||
| 82 | - | ||
| 83 | - thread = new Thread(this); | ||
| 84 | - | ||
| 85 | - thread.start(); // start thread | ||
| 86 | - } | ||
| 87 | - | ||
| 88 | - public void stop() { | ||
| 89 | - capline.stop(); | ||
| 90 | - capline.close(); | ||
| 91 | - capline = null; | ||
| 92 | - thread = null; | ||
| 93 | - } | ||
| 94 | - | ||
| 95 | - /** feed captured microphone data to asr */ | ||
| 96 | - public void decodeSample(byte[] samplebytes) { | ||
| 97 | - try { | ||
| 98 | - ByteBuffer byteBuf = ByteBuffer.wrap(samplebytes); // create a bytebuf for samples | ||
| 99 | - byteBuf.order(ByteOrder.LITTLE_ENDIAN); // set bytebuf to little endian | ||
| 100 | - ShortBuffer shortBuf = byteBuf.asShortBuffer(); // covert to short type | ||
| 101 | - short[] arrShort = new short[shortBuf.capacity()]; // array for copy short data | ||
| 102 | - float[] arrFloat = new float[shortBuf.capacity()]; // array for copy float data | ||
| 103 | - shortBuf.get(arrShort); // put date to arrShort | ||
| 104 | - | ||
| 105 | - for (int i = 0; i < arrShort.length; i++) { | ||
| 106 | - arrFloat[i] = arrShort[i] / 32768f; // loop to covert short data to float -1 to 1 | ||
| 107 | - } | ||
| 108 | - streamObj.acceptWaveform(arrFloat); // feed asr engine with float data | ||
| 109 | - while (rcgOjb.isReady(streamObj)) { // if engine is ready for unprocessed data | ||
| 110 | - | ||
| 111 | - rcgOjb.decodeStream(streamObj); // decode for this stream | ||
| 112 | - } | ||
| 113 | - boolean isEndpoint = | ||
| 114 | - rcgOjb.isEndpoint( | ||
| 115 | - streamObj); // endpoint check, make sure enable_endpoint_detection=true in config | ||
| 116 | - // file | ||
| 117 | - String nowText = rcgOjb.getResult(streamObj); // get asr result | ||
| 118 | - String recText = ""; | ||
| 119 | - byte[] utf8Data; // for covert text to utf8 | ||
| 120 | - if (isEndpoint && nowText.length() > 0) { | ||
| 121 | - rcgOjb.reSet(streamObj); // reSet stream when detect endpoint | ||
| 122 | - segmentId++; | ||
| 123 | - preText = nowText; | ||
| 124 | - recText = "text(seg_" + String.valueOf(segmentId) + "):" + nowText + "\n"; | ||
| 125 | - utf8Data = recText.getBytes(StandardCharsets.UTF_8); | ||
| 126 | - System.out.println(new String(utf8Data)); | ||
| 127 | - } | ||
| 128 | - | ||
| 129 | - if (!nowText.equals(preText)) { // if preText not equal nowtext | ||
| 130 | - preText = nowText; | ||
| 131 | - recText = nowText + "\n"; | ||
| 132 | - utf8Data = recText.getBytes(StandardCharsets.UTF_8); | ||
| 133 | - System.out.println(new String(utf8Data)); | ||
| 134 | - } | ||
| 135 | - } catch (Exception e) { | ||
| 136 | - System.err.println(e); | ||
| 137 | - e.printStackTrace(); | ||
| 138 | - } | ||
| 139 | - } | ||
| 140 | - | ||
| 141 | - /** run mic capture thread */ | ||
| 142 | - public void run() { | ||
| 143 | - System.out.println("Started! Please speak..."); | ||
| 144 | - | ||
| 145 | - AudioFormat.Encoding encoding = AudioFormat.Encoding.PCM_SIGNED; // the pcm format | ||
| 146 | - float rate = 16000.0f; // using 16 kHz | ||
| 147 | - int channels = 1; // single channel | ||
| 148 | - int sampleSize = 16; // sampleSize 16bit | ||
| 149 | - boolean isBigEndian = false; // using little endian | ||
| 150 | - | ||
| 151 | - AudioFormat format = | ||
| 152 | - new AudioFormat( | ||
| 153 | - encoding, rate, sampleSize, channels, (sampleSize / 8) * channels, rate, isBigEndian); | ||
| 154 | - | ||
| 155 | - DataLine.Info info = new DataLine.Info(TargetDataLine.class, format); | ||
| 156 | - | ||
| 157 | - // check system support such data format | ||
| 158 | - if (!AudioSystem.isLineSupported(info)) { | ||
| 159 | - System.out.println(info + " not supported."); | ||
| 160 | - return; | ||
| 161 | - } | ||
| 162 | - | ||
| 163 | - // open a line for capture. | ||
| 164 | - | ||
| 165 | - try { | ||
| 166 | - capline = (TargetDataLine) AudioSystem.getLine(info); | ||
| 167 | - capline.open(format, capline.getBufferSize()); | ||
| 168 | - } catch (Exception ex) { | ||
| 169 | - System.out.println(ex); | ||
| 170 | - return; | ||
| 171 | - } | ||
| 172 | - | ||
| 173 | - // the buf size for mic captured each time | ||
| 174 | - int bufferLengthInBytes = capline.getBufferSize() / 8 * format.getFrameSize(); | ||
| 175 | - byte[] micData = new byte[bufferLengthInBytes]; | ||
| 176 | - int numBytesRead; | ||
| 177 | - | ||
| 178 | - capline.start(); // start to capture mic data | ||
| 179 | - | ||
| 180 | - while (thread != null) { | ||
| 181 | - // read data from line | ||
| 182 | - if ((numBytesRead = capline.read(micData, 0, bufferLengthInBytes)) == -1) { | ||
| 183 | - break; | ||
| 184 | - } | ||
| 185 | - | ||
| 186 | - decodeSample(micData); // decode mic data | ||
| 187 | - } | ||
| 188 | - | ||
| 189 | - // stop and close | ||
| 190 | - | ||
| 191 | - try { | ||
| 192 | - if (capline != null) { | ||
| 193 | - capline.stop(); | ||
| 194 | - capline.close(); | ||
| 195 | - capline = null; | ||
| 196 | - } | ||
| 197 | - | ||
| 198 | - } catch (Exception ex) { | ||
| 199 | - System.err.println(ex); | ||
| 200 | - } | ||
| 201 | - } | ||
| 202 | - } // End class DecodeMic | ||
| 203 | - | ||
| 204 | - public static void main(String s[]) { | ||
| 205 | - try { | ||
| 206 | - String appDir = System.getProperty("user.dir"); | ||
| 207 | - System.out.println("appdir=" + appDir); | ||
| 208 | - String cfgPath = appDir + "/modelconfig.cfg"; | ||
| 209 | - String soPath = appDir + "/../build/lib/libsherpa-onnx-jni.so"; | ||
| 210 | - OnlineRecognizer.setSoPath(soPath); // set so. lib for OnlineRecognizer | ||
| 211 | - | ||
| 212 | - DecodeMic decodeEx = new DecodeMic(); | ||
| 213 | - decodeEx.initModelWithCfg(cfgPath); // init asr engine | ||
| 214 | - decodeEx.open(); // open thread for mic | ||
| 215 | - System.out.print("Press Enter to EXIT!\n"); | ||
| 216 | - char i = (char) System.in.read(); | ||
| 217 | - decodeEx.close(); | ||
| 218 | - } catch (Exception e) { | ||
| 219 | - System.err.println(e); | ||
| 220 | - e.printStackTrace(); | ||
| 221 | - } | ||
| 222 | - } | ||
| 223 | -} | 1 | +/* |
| 2 | + * // Copyright 2022-2023 by zhaoming | ||
| 3 | + */ | ||
| 4 | +/* | ||
| 5 | +Real-time speech recognition from a microphone with com.k2fsa.sherpa.onnx Java API | ||
| 6 | + | ||
| 7 | +example for cfgFile modelconfig.cfg | ||
| 8 | + sample_rate=16000 | ||
| 9 | + feature_dim=80 | ||
| 10 | + rule1_min_trailing_silence=2.4 | ||
| 11 | + rule2_min_trailing_silence=1.2 | ||
| 12 | + rule3_min_utterance_length=20 | ||
| 13 | + encoder=/sherpa-onnx/build/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/encoder-epoch-99-avg-1.onnx | ||
| 14 | + decoder=/sherpa-onnx/build/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/decoder-epoch-99-avg-1.onnx | ||
| 15 | + joiner=/sherpa-onnx/build/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/joiner-epoch-99-avg-1.onnx | ||
| 16 | + tokens=/sherpa-onnx/build/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt | ||
| 17 | + num_threads=4 | ||
| 18 | + enable_endpoint_detection=true | ||
| 19 | + decoding_method=greedy_search | ||
| 20 | + max_active_paths=4 | ||
| 21 | + | ||
| 22 | +*/ | ||
| 23 | +import com.k2fsa.sherpa.onnx.OnlineRecognizer; | ||
| 24 | +import com.k2fsa.sherpa.onnx.OnlineStream; | ||
| 25 | +import java.io.*; | ||
| 26 | +import java.nio.ByteBuffer; | ||
| 27 | +import java.nio.ByteOrder; | ||
| 28 | +import java.nio.ShortBuffer; | ||
| 29 | +import java.nio.charset.StandardCharsets; | ||
| 30 | +import javax.sound.sampled.AudioFormat; | ||
| 31 | +import javax.sound.sampled.AudioSystem; | ||
| 32 | +import javax.sound.sampled.DataLine; | ||
| 33 | +import javax.sound.sampled.TargetDataLine; | ||
| 34 | + | ||
| 35 | +/** Microphone Example */ | ||
| 36 | +public class DecodeMic { | ||
| 37 | + MicRcgThread micRcgThread = null; // thread handle | ||
| 38 | + | ||
| 39 | + OnlineRecognizer rcgOjb; // the recognizer | ||
| 40 | + | ||
| 41 | + OnlineStream streamObj; // the stream | ||
| 42 | + | ||
| 43 | + public DecodeMic() { | ||
| 44 | + | ||
| 45 | + micRcgThread = new MicRcgThread(); // create a new instance for MicRcgThread | ||
| 46 | + } | ||
| 47 | + | ||
| 48 | + public void open() { | ||
| 49 | + micRcgThread.start(); // start to capture microphone data | ||
| 50 | + } | ||
| 51 | + | ||
| 52 | + public void close() { | ||
| 53 | + micRcgThread.stop(); // close capture | ||
| 54 | + } | ||
| 55 | + | ||
| 56 | + /** init asr engine with config file */ | ||
| 57 | + public void initModelWithCfg(String cfgFile) { | ||
| 58 | + try { | ||
| 59 | + | ||
| 60 | + // set setSoPath() before running this | ||
| 61 | + rcgOjb = new OnlineRecognizer(cfgFile); | ||
| 62 | + | ||
| 63 | + streamObj = rcgOjb.createStream(); // create a stream for asr engine to feed data | ||
| 64 | + } catch (Exception e) { | ||
| 65 | + System.err.println(e); | ||
| 66 | + e.printStackTrace(); | ||
| 67 | + } | ||
| 68 | + } | ||
| 69 | + | ||
| 70 | + /** read data from mic and feed to asr engine */ | ||
| 71 | + class MicRcgThread implements Runnable { | ||
| 72 | + | ||
| 73 | + TargetDataLine capline; // line for capture mic data | ||
| 74 | + | ||
| 75 | + Thread thread; // this thread | ||
| 76 | + int segmentId = 0; // record the segment id when detect endpoint | ||
| 77 | + String preText = ""; // decoded text | ||
| 78 | + | ||
| 79 | + public MicRcgThread() {} | ||
| 80 | + | ||
| 81 | + public void start() { | ||
| 82 | + | ||
| 83 | + thread = new Thread(this); | ||
| 84 | + | ||
| 85 | + thread.start(); // start thread | ||
| 86 | + } | ||
| 87 | + | ||
| 88 | + public void stop() { | ||
| 89 | + capline.stop(); | ||
| 90 | + capline.close(); | ||
| 91 | + capline = null; | ||
| 92 | + thread = null; | ||
| 93 | + } | ||
| 94 | + | ||
| 95 | + /** feed captured microphone data to asr */ | ||
| 96 | + public void decodeSample(byte[] samplebytes) { | ||
| 97 | + try { | ||
| 98 | + ByteBuffer byteBuf = ByteBuffer.wrap(samplebytes); // create a bytebuf for samples | ||
| 99 | + byteBuf.order(ByteOrder.LITTLE_ENDIAN); // set bytebuf to little endian | ||
| 100 | + ShortBuffer shortBuf = byteBuf.asShortBuffer(); // covert to short type | ||
| 101 | + short[] arrShort = new short[shortBuf.capacity()]; // array for copy short data | ||
| 102 | + float[] arrFloat = new float[shortBuf.capacity()]; // array for copy float data | ||
| 103 | + shortBuf.get(arrShort); // put date to arrShort | ||
| 104 | + | ||
| 105 | + for (int i = 0; i < arrShort.length; i++) { | ||
| 106 | + arrFloat[i] = arrShort[i] / 32768f; // loop to covert short data to float -1 to 1 | ||
| 107 | + } | ||
| 108 | + streamObj.acceptWaveform(arrFloat); // feed asr engine with float data | ||
| 109 | + while (rcgOjb.isReady(streamObj)) { // if engine is ready for unprocessed data | ||
| 110 | + | ||
| 111 | + rcgOjb.decodeStream(streamObj); // decode for this stream | ||
| 112 | + } | ||
| 113 | + boolean isEndpoint = | ||
| 114 | + rcgOjb.isEndpoint( | ||
| 115 | + streamObj); // endpoint check, make sure enable_endpoint_detection=true in config | ||
| 116 | + // file | ||
| 117 | + String nowText = rcgOjb.getResult(streamObj); // get asr result | ||
| 118 | + String recText = ""; | ||
| 119 | + byte[] utf8Data; // for covert text to utf8 | ||
| 120 | + if (isEndpoint && nowText.length() > 0) { | ||
| 121 | + rcgOjb.reSet(streamObj); // reSet stream when detect endpoint | ||
| 122 | + segmentId++; | ||
| 123 | + preText = nowText; | ||
| 124 | + recText = "text(seg_" + String.valueOf(segmentId) + "):" + nowText + "\n"; | ||
| 125 | + utf8Data = recText.getBytes(StandardCharsets.UTF_8); | ||
| 126 | + System.out.println(new String(utf8Data)); | ||
| 127 | + } | ||
| 128 | + | ||
| 129 | + if (!nowText.equals(preText)) { // if preText not equal nowtext | ||
| 130 | + preText = nowText; | ||
| 131 | + recText = nowText + "\n"; | ||
| 132 | + utf8Data = recText.getBytes(StandardCharsets.UTF_8); | ||
| 133 | + System.out.println(new String(utf8Data)); | ||
| 134 | + } | ||
| 135 | + } catch (Exception e) { | ||
| 136 | + System.err.println(e); | ||
| 137 | + e.printStackTrace(); | ||
| 138 | + } | ||
| 139 | + } | ||
| 140 | + | ||
| 141 | + /** run mic capture thread */ | ||
| 142 | + public void run() { | ||
| 143 | + System.out.println("Started! Please speak..."); | ||
| 144 | + | ||
| 145 | + AudioFormat.Encoding encoding = AudioFormat.Encoding.PCM_SIGNED; // the pcm format | ||
| 146 | + float rate = 16000.0f; // using 16 kHz | ||
| 147 | + int channels = 1; // single channel | ||
| 148 | + int sampleSize = 16; // sampleSize 16bit | ||
| 149 | + boolean isBigEndian = false; // using little endian | ||
| 150 | + | ||
| 151 | + AudioFormat format = | ||
| 152 | + new AudioFormat( | ||
| 153 | + encoding, rate, sampleSize, channels, (sampleSize / 8) * channels, rate, isBigEndian); | ||
| 154 | + | ||
| 155 | + DataLine.Info info = new DataLine.Info(TargetDataLine.class, format); | ||
| 156 | + | ||
| 157 | + // check system support such data format | ||
| 158 | + if (!AudioSystem.isLineSupported(info)) { | ||
| 159 | + System.out.println(info + " not supported."); | ||
| 160 | + return; | ||
| 161 | + } | ||
| 162 | + | ||
| 163 | + // open a line for capture. | ||
| 164 | + | ||
| 165 | + try { | ||
| 166 | + capline = (TargetDataLine) AudioSystem.getLine(info); | ||
| 167 | + capline.open(format, capline.getBufferSize()); | ||
| 168 | + } catch (Exception ex) { | ||
| 169 | + System.out.println(ex); | ||
| 170 | + return; | ||
| 171 | + } | ||
| 172 | + | ||
| 173 | + // the buf size for mic captured each time | ||
| 174 | + int bufferLengthInBytes = capline.getBufferSize() / 8 * format.getFrameSize(); | ||
| 175 | + byte[] micData = new byte[bufferLengthInBytes]; | ||
| 176 | + int numBytesRead; | ||
| 177 | + | ||
| 178 | + capline.start(); // start to capture mic data | ||
| 179 | + | ||
| 180 | + while (thread != null) { | ||
| 181 | + // read data from line | ||
| 182 | + if ((numBytesRead = capline.read(micData, 0, bufferLengthInBytes)) == -1) { | ||
| 183 | + break; | ||
| 184 | + } | ||
| 185 | + | ||
| 186 | + decodeSample(micData); // decode mic data | ||
| 187 | + } | ||
| 188 | + | ||
| 189 | + // stop and close | ||
| 190 | + | ||
| 191 | + try { | ||
| 192 | + if (capline != null) { | ||
| 193 | + capline.stop(); | ||
| 194 | + capline.close(); | ||
| 195 | + capline = null; | ||
| 196 | + } | ||
| 197 | + | ||
| 198 | + } catch (Exception ex) { | ||
| 199 | + System.err.println(ex); | ||
| 200 | + } | ||
| 201 | + } | ||
| 202 | + } // End class DecodeMic | ||
| 203 | + | ||
| 204 | + public static void main(String s[]) { | ||
| 205 | + try { | ||
| 206 | + String appDir = System.getProperty("user.dir"); | ||
| 207 | + System.out.println("appdir=" + appDir); | ||
| 208 | + String cfgPath = appDir + "/modelconfig.cfg"; | ||
| 209 | + String soPath = appDir + "/../build/lib/libsherpa-onnx-jni.so"; | ||
| 210 | + OnlineRecognizer.setSoPath(soPath); // set so. lib for OnlineRecognizer | ||
| 211 | + | ||
| 212 | + DecodeMic decodeEx = new DecodeMic(); | ||
| 213 | + decodeEx.initModelWithCfg(cfgPath); // init asr engine | ||
| 214 | + decodeEx.open(); // open thread for mic | ||
| 215 | + System.out.print("Press Enter to EXIT!\n"); | ||
| 216 | + char i = (char) System.in.read(); | ||
| 217 | + decodeEx.close(); | ||
| 218 | + } catch (Exception e) { | ||
| 219 | + System.err.println(e); | ||
| 220 | + e.printStackTrace(); | ||
| 221 | + } | ||
| 222 | + } | ||
| 223 | +} |
| 1 | -/* | ||
| 2 | - * // Copyright 2022-2023 by zhaomingwork | ||
| 3 | - */ | ||
| 4 | -// java AsrWebsocketClient | ||
| 5 | -// usage: AsrWebsocketClient soPath srvIp srvPort wavPath numThreads | ||
| 6 | -package websocketsrv; | ||
| 7 | - | ||
| 8 | -import com.k2fsa.sherpa.onnx.OnlineRecognizer; | ||
| 9 | -import java.net.URI; | ||
| 10 | -import java.net.URISyntaxException; | ||
| 11 | -import java.nio.*; | ||
| 12 | -import java.util.Map; | ||
| 13 | -import org.java_websocket.client.WebSocketClient; | ||
| 14 | -import org.java_websocket.drafts.Draft; | ||
| 15 | -import org.java_websocket.handshake.ServerHandshake; | ||
| 16 | -import org.slf4j.Logger; | ||
| 17 | -import org.slf4j.LoggerFactory; | ||
| 18 | - | ||
| 19 | -/** This example demonstrates how to connect to websocket server. */ | ||
| 20 | -public class AsrWebsocketClient extends WebSocketClient { | ||
| 21 | - private static final Logger logger = LoggerFactory.getLogger(AsrWebsocketClient.class); | ||
| 22 | - | ||
| 23 | - public AsrWebsocketClient(URI serverUri, Draft draft) { | ||
| 24 | - super(serverUri, draft); | ||
| 25 | - } | ||
| 26 | - | ||
| 27 | - public AsrWebsocketClient(URI serverURI) { | ||
| 28 | - super(serverURI); | ||
| 29 | - } | ||
| 30 | - | ||
| 31 | - public AsrWebsocketClient(URI serverUri, Map<String, String> httpHeaders) { | ||
| 32 | - super(serverUri, httpHeaders); | ||
| 33 | - } | ||
| 34 | - | ||
| 35 | - @Override | ||
| 36 | - public void onOpen(ServerHandshake handshakedata) { | ||
| 37 | - | ||
| 38 | - float[] floats = OnlineRecognizer.readWavFile(AsrWebsocketClient.wavPath); | ||
| 39 | - ByteBuffer buffer = | ||
| 40 | - ByteBuffer.allocate(4 * floats.length) | ||
| 41 | - .order(ByteOrder.LITTLE_ENDIAN); // float is sizeof 4. allocate enough buffer | ||
| 42 | - | ||
| 43 | - for (float f : floats) { | ||
| 44 | - buffer.putFloat(f); | ||
| 45 | - } | ||
| 46 | - buffer.rewind(); | ||
| 47 | - buffer.flip(); | ||
| 48 | - buffer.order(ByteOrder.LITTLE_ENDIAN); | ||
| 49 | - | ||
| 50 | - send(buffer.array()); // send buf to server | ||
| 51 | - send("Done"); // send 'Done' means finished | ||
| 52 | - } | ||
| 53 | - | ||
| 54 | - @Override | ||
| 55 | - public void onMessage(String message) { | ||
| 56 | - | ||
| 57 | - logger.info("received: " + message); | ||
| 58 | - } | ||
| 59 | - | ||
| 60 | - @Override | ||
| 61 | - public void onClose(int code, String reason, boolean remote) { | ||
| 62 | - | ||
| 63 | - logger.info( | ||
| 64 | - "Connection closed by " | ||
| 65 | - + (remote ? "remote peer" : "us") | ||
| 66 | - + " Code: " | ||
| 67 | - + code | ||
| 68 | - + " Reason: " | ||
| 69 | - + reason); | ||
| 70 | - } | ||
| 71 | - | ||
| 72 | - @Override | ||
| 73 | - public void onError(Exception ex) { | ||
| 74 | - ex.printStackTrace(); | ||
| 75 | - // if the error is fatal then onClose will be called additionally | ||
| 76 | - } | ||
| 77 | - | ||
| 78 | - public static OnlineRecognizer rcgobj; | ||
| 79 | - public static String wavPath; | ||
| 80 | - | ||
| 81 | - public static void main(String[] args) throws URISyntaxException { | ||
| 82 | - | ||
| 83 | - if (args.length != 5) { | ||
| 84 | - System.out.println("usage: AsrWebsocketClient soPath srvIp srvPort wavPath numThreads"); | ||
| 85 | - return; | ||
| 86 | - } | ||
| 87 | - | ||
| 88 | - String soPath = args[0]; | ||
| 89 | - String srvIp = args[1]; | ||
| 90 | - String srvPort = args[2]; | ||
| 91 | - String wavPath = args[3]; | ||
| 92 | - int numThreads = Integer.parseInt(args[4]); | ||
| 93 | - System.out.println("serIp=" + srvIp + ",srvPort=" + srvPort + ",wavPath=" + wavPath); | ||
| 94 | - | ||
| 95 | - class ClientThread implements Runnable { | ||
| 96 | - | ||
| 97 | - String soPath; | ||
| 98 | - String srvIp; | ||
| 99 | - String srvPort; | ||
| 100 | - String wavPath; | ||
| 101 | - | ||
| 102 | - ClientThread(String soPath, String srvIp, String srvPort, String wavPath) { | ||
| 103 | - this.soPath = soPath; | ||
| 104 | - this.srvIp = srvIp; | ||
| 105 | - this.srvPort = srvPort; | ||
| 106 | - this.wavPath = wavPath; | ||
| 107 | - } | ||
| 108 | - | ||
| 109 | - public void run() { | ||
| 110 | - try { | ||
| 111 | - | ||
| 112 | - OnlineRecognizer.setSoPath(soPath); | ||
| 113 | - | ||
| 114 | - AsrWebsocketClient.wavPath = wavPath; | ||
| 115 | - | ||
| 116 | - String wsAddress = "ws://" + srvIp + ":" + srvPort; | ||
| 117 | - AsrWebsocketClient c = new AsrWebsocketClient(new URI(wsAddress)); | ||
| 118 | - | ||
| 119 | - c.connect(); | ||
| 120 | - } catch (Exception e) { | ||
| 121 | - e.printStackTrace(); | ||
| 122 | - } | ||
| 123 | - } | ||
| 124 | - } | ||
| 125 | - for (int i = 0; i < numThreads; i++) { | ||
| 126 | - System.out.println("Thread1 is running..."); | ||
| 127 | - Thread t = new Thread(new ClientThread(soPath, srvIp, srvPort, wavPath)); | ||
| 128 | - t.start(); | ||
| 129 | - } | ||
| 130 | - } | ||
| 131 | -} | 1 | +/* |
| 2 | + * // Copyright 2022-2023 by zhaomingwork | ||
| 3 | + */ | ||
| 4 | +// java AsrWebsocketClient | ||
| 5 | +// usage: AsrWebsocketClient soPath srvIp srvPort wavPath numThreads | ||
| 6 | +package websocketsrv; | ||
| 7 | + | ||
| 8 | +import com.k2fsa.sherpa.onnx.OnlineRecognizer; | ||
| 9 | +import java.net.URI; | ||
| 10 | +import java.net.URISyntaxException; | ||
| 11 | +import java.nio.*; | ||
| 12 | +import java.util.Map; | ||
| 13 | +import org.java_websocket.client.WebSocketClient; | ||
| 14 | +import org.java_websocket.drafts.Draft; | ||
| 15 | +import org.java_websocket.handshake.ServerHandshake; | ||
| 16 | +import org.slf4j.Logger; | ||
| 17 | +import org.slf4j.LoggerFactory; | ||
| 18 | + | ||
| 19 | +/** This example demonstrates how to connect to websocket server. */ | ||
| 20 | +public class AsrWebsocketClient extends WebSocketClient { | ||
| 21 | + private static final Logger logger = LoggerFactory.getLogger(AsrWebsocketClient.class); | ||
| 22 | + | ||
| 23 | + public AsrWebsocketClient(URI serverUri, Draft draft) { | ||
| 24 | + super(serverUri, draft); | ||
| 25 | + } | ||
| 26 | + | ||
| 27 | + public AsrWebsocketClient(URI serverURI) { | ||
| 28 | + super(serverURI); | ||
| 29 | + } | ||
| 30 | + | ||
| 31 | + public AsrWebsocketClient(URI serverUri, Map<String, String> httpHeaders) { | ||
| 32 | + super(serverUri, httpHeaders); | ||
| 33 | + } | ||
| 34 | + | ||
| 35 | + @Override | ||
| 36 | + public void onOpen(ServerHandshake handshakedata) { | ||
| 37 | + | ||
| 38 | + float[] floats = OnlineRecognizer.readWavFile(AsrWebsocketClient.wavPath); | ||
| 39 | + ByteBuffer buffer = | ||
| 40 | + ByteBuffer.allocate(4 * floats.length) | ||
| 41 | + .order(ByteOrder.LITTLE_ENDIAN); // float is sizeof 4. allocate enough buffer | ||
| 42 | + | ||
| 43 | + for (float f : floats) { | ||
| 44 | + buffer.putFloat(f); | ||
| 45 | + } | ||
| 46 | + buffer.rewind(); | ||
| 47 | + buffer.flip(); | ||
| 48 | + buffer.order(ByteOrder.LITTLE_ENDIAN); | ||
| 49 | + | ||
| 50 | + send(buffer.array()); // send buf to server | ||
| 51 | + send("Done"); // send 'Done' means finished | ||
| 52 | + } | ||
| 53 | + | ||
| 54 | + @Override | ||
| 55 | + public void onMessage(String message) { | ||
| 56 | + | ||
| 57 | + logger.info("received: " + message); | ||
| 58 | + } | ||
| 59 | + | ||
| 60 | + @Override | ||
| 61 | + public void onClose(int code, String reason, boolean remote) { | ||
| 62 | + | ||
| 63 | + logger.info( | ||
| 64 | + "Connection closed by " | ||
| 65 | + + (remote ? "remote peer" : "us") | ||
| 66 | + + " Code: " | ||
| 67 | + + code | ||
| 68 | + + " Reason: " | ||
| 69 | + + reason); | ||
| 70 | + } | ||
| 71 | + | ||
| 72 | + @Override | ||
| 73 | + public void onError(Exception ex) { | ||
| 74 | + ex.printStackTrace(); | ||
| 75 | + // if the error is fatal then onClose will be called additionally | ||
| 76 | + } | ||
| 77 | + | ||
| 78 | + public static OnlineRecognizer rcgobj; | ||
| 79 | + public static String wavPath; | ||
| 80 | + | ||
| 81 | + public static void main(String[] args) throws URISyntaxException { | ||
| 82 | + | ||
| 83 | + if (args.length != 5) { | ||
| 84 | + System.out.println("usage: AsrWebsocketClient soPath srvIp srvPort wavPath numThreads"); | ||
| 85 | + return; | ||
| 86 | + } | ||
| 87 | + | ||
| 88 | + String soPath = args[0]; | ||
| 89 | + String srvIp = args[1]; | ||
| 90 | + String srvPort = args[2]; | ||
| 91 | + String wavPath = args[3]; | ||
| 92 | + int numThreads = Integer.parseInt(args[4]); | ||
| 93 | + System.out.println("serIp=" + srvIp + ",srvPort=" + srvPort + ",wavPath=" + wavPath); | ||
| 94 | + | ||
| 95 | + class ClientThread implements Runnable { | ||
| 96 | + | ||
| 97 | + String soPath; | ||
| 98 | + String srvIp; | ||
| 99 | + String srvPort; | ||
| 100 | + String wavPath; | ||
| 101 | + | ||
| 102 | + ClientThread(String soPath, String srvIp, String srvPort, String wavPath) { | ||
| 103 | + this.soPath = soPath; | ||
| 104 | + this.srvIp = srvIp; | ||
| 105 | + this.srvPort = srvPort; | ||
| 106 | + this.wavPath = wavPath; | ||
| 107 | + } | ||
| 108 | + | ||
| 109 | + public void run() { | ||
| 110 | + try { | ||
| 111 | + | ||
| 112 | + OnlineRecognizer.setSoPath(soPath); | ||
| 113 | + | ||
| 114 | + AsrWebsocketClient.wavPath = wavPath; | ||
| 115 | + | ||
| 116 | + String wsAddress = "ws://" + srvIp + ":" + srvPort; | ||
| 117 | + AsrWebsocketClient c = new AsrWebsocketClient(new URI(wsAddress)); | ||
| 118 | + | ||
| 119 | + c.connect(); | ||
| 120 | + } catch (Exception e) { | ||
| 121 | + e.printStackTrace(); | ||
| 122 | + } | ||
| 123 | + } | ||
| 124 | + } | ||
| 125 | + for (int i = 0; i < numThreads; i++) { | ||
| 126 | + System.out.println("Thread1 is running..."); | ||
| 127 | + Thread t = new Thread(new ClientThread(soPath, srvIp, srvPort, wavPath)); | ||
| 128 | + t.start(); | ||
| 129 | + } | ||
| 130 | + } | ||
| 131 | +} |
| 1 | -/* | ||
| 2 | - * // Copyright 2022-2023 by zhaoming | ||
| 3 | - */ | ||
| 4 | -// java DecoderThreadHandler | ||
| 5 | -package websocketsrv; | ||
| 6 | - | ||
| 7 | -import com.k2fsa.sherpa.onnx.OnlineRecognizer; | ||
| 8 | -import com.k2fsa.sherpa.onnx.OnlineStream; | ||
| 9 | -import java.nio.*; | ||
| 10 | -import java.nio.charset.StandardCharsets; | ||
| 11 | -import java.time.LocalDateTime; | ||
| 12 | -import java.util.*; | ||
| 13 | -import java.util.List; | ||
| 14 | -import java.util.concurrent.*; | ||
| 15 | -import java.util.concurrent.LinkedBlockingQueue; | ||
| 16 | -import org.java_websocket.WebSocket; | ||
| 17 | -import org.java_websocket.drafts.Draft; | ||
| 18 | -import org.java_websocket.framing.Framedata; | ||
| 19 | -import org.slf4j.Logger; | ||
| 20 | -import org.slf4j.LoggerFactory; | ||
| 21 | - | ||
| 22 | -public class DecoderThreadHandler extends Thread { | ||
| 23 | - private static final Logger logger = LoggerFactory.getLogger(DecoderThreadHandler.class); | ||
| 24 | - // Websocket Queue that waiting for decoding | ||
| 25 | - private LinkedBlockingQueue<WebSocket> decoderQueue; | ||
| 26 | - // the mapping between websocket and connection data | ||
| 27 | - private ConcurrentHashMap<WebSocket, ConnectionData> connMap; | ||
| 28 | - | ||
| 29 | - private OnlineRecognizer rcgOjb = null; // recgnizer object | ||
| 30 | - | ||
| 31 | - // connection data list for this thread to decode in parallel | ||
| 32 | - private List<ConnectionData> connDataList = new ArrayList<ConnectionData>(); | ||
| 33 | - | ||
| 34 | - private int parallelDecoderNum = 10; // parallel decoding number | ||
| 35 | - private int deocderTimeIdle = 10; // idle time(ms) when no job | ||
| 36 | - private int deocderTimeOut = 3000; // if it is timeout(ms), the connection data will be removed | ||
| 37 | - | ||
| 38 | - public DecoderThreadHandler( | ||
| 39 | - LinkedBlockingQueue<WebSocket> decoderQueue, | ||
| 40 | - ConcurrentHashMap<WebSocket, ConnectionData> connMap, | ||
| 41 | - OnlineRecognizer rcgOjb, | ||
| 42 | - int deocderTimeIdle, | ||
| 43 | - int parallelDecoderNum, | ||
| 44 | - int deocderTimeOut) { | ||
| 45 | - this.decoderQueue = decoderQueue; | ||
| 46 | - this.connMap = connMap; | ||
| 47 | - this.rcgOjb = rcgOjb; | ||
| 48 | - this.deocderTimeIdle = deocderTimeIdle; | ||
| 49 | - this.parallelDecoderNum = parallelDecoderNum; | ||
| 50 | - this.deocderTimeOut = deocderTimeOut; | ||
| 51 | - } | ||
| 52 | - | ||
| 53 | - public void run() { | ||
| 54 | - while (true) { | ||
| 55 | - try { | ||
| 56 | - // time(ms) idle if there is no job | ||
| 57 | - | ||
| 58 | - Thread.sleep(deocderTimeIdle); | ||
| 59 | - // clear data list for this threads | ||
| 60 | - connDataList.clear(); | ||
| 61 | - if (rcgOjb == null) continue; | ||
| 62 | - | ||
| 63 | - // loop for total decoder Queue | ||
| 64 | - while (!decoderQueue.isEmpty()) { | ||
| 65 | - | ||
| 66 | - // get websocket | ||
| 67 | - WebSocket conn = decoderQueue.take(); | ||
| 68 | - // get connection data according to websocket | ||
| 69 | - ConnectionData connData = connMap.get(conn); | ||
| 70 | - | ||
| 71 | - // if the websocket closed, continue | ||
| 72 | - if (connData == null) continue; | ||
| 73 | - // get the stream | ||
| 74 | - OnlineStream stream = connData.getStream(); | ||
| 75 | - | ||
| 76 | - // put to decoder list if 1) stream is ready; 2) and | ||
| 77 | - // size not > parallelDecoderNum | ||
| 78 | - if ((rcgOjb.isReady(stream) && connDataList.size() < parallelDecoderNum)) { | ||
| 79 | - | ||
| 80 | - // add to this thread's decoder list | ||
| 81 | - connDataList.add(connData); | ||
| 82 | - // change the handled time for this connection data | ||
| 83 | - connData.setLastHandleTime(LocalDateTime.now()); | ||
| 84 | - } | ||
| 85 | - // break when decoder list size >= parallelDecoderNum | ||
| 86 | - if (connDataList.size() >= parallelDecoderNum) { | ||
| 87 | - break; | ||
| 88 | - } | ||
| 89 | - } | ||
| 90 | - | ||
| 91 | - // if decoder data list for this thread >0 | ||
| 92 | - if (connDataList.size() > 0) { | ||
| 93 | - | ||
| 94 | - // create a stream array for parallel decoding | ||
| 95 | - OnlineStream[] arr = new OnlineStream[connDataList.size()]; | ||
| 96 | - for (int i = 0; i < connDataList.size(); i++) { | ||
| 97 | - | ||
| 98 | - arr[i] = connDataList.get(i).getStream(); | ||
| 99 | - } | ||
| 100 | - | ||
| 101 | - // parallel decoding | ||
| 102 | - rcgOjb.decodeStreams(arr); | ||
| 103 | - } | ||
| 104 | - | ||
| 105 | - // get result for each connection | ||
| 106 | - for (ConnectionData connData : connDataList) { | ||
| 107 | - | ||
| 108 | - OnlineStream stream = connData.getStream(); | ||
| 109 | - WebSocket webSocket = connData.getWebSocket(); | ||
| 110 | - | ||
| 111 | - String txtResult = rcgOjb.getResult(stream); | ||
| 112 | - | ||
| 113 | - // decode text in utf-8 | ||
| 114 | - byte[] utf8Data = txtResult.getBytes(StandardCharsets.UTF_8); | ||
| 115 | - | ||
| 116 | - boolean isEof = (connData.getEof() == true && !rcgOjb.isReady(stream)); | ||
| 117 | - // result | ||
| 118 | - if (utf8Data.length > 0) { | ||
| 119 | - | ||
| 120 | - String jsonResult = | ||
| 121 | - "{\"text\":\"" + txtResult + "\",\"eof\":" + String.valueOf(isEof) + "\"}"; | ||
| 122 | - | ||
| 123 | - if (webSocket.isOpen()) { | ||
| 124 | - // create a TEXT Frame for send back json result | ||
| 125 | - Draft draft = webSocket.getDraft(); | ||
| 126 | - List<Framedata> frames = null; | ||
| 127 | - frames = draft.createFrames(jsonResult, false); | ||
| 128 | - // send to client | ||
| 129 | - webSocket.sendFrame(frames); | ||
| 130 | - } | ||
| 131 | - } | ||
| 132 | - } | ||
| 133 | - // loop for each connection data in this thread | ||
| 134 | - for (ConnectionData connData : connDataList) { | ||
| 135 | - OnlineStream stream = connData.getStream(); | ||
| 136 | - WebSocket webSocket = connData.getWebSocket(); | ||
| 137 | - // if the stream is still ready, put it to decoder Queue again for next decoding | ||
| 138 | - if (rcgOjb.isReady(stream)) { | ||
| 139 | - decoderQueue.put(webSocket); | ||
| 140 | - } | ||
| 141 | - // the duration between last handled time and now | ||
| 142 | - java.time.Duration duration = | ||
| 143 | - java.time.Duration.between(connData.getLastHandleTime(), LocalDateTime.now()); | ||
| 144 | - // close the websocket if 1) data is done and stream not ready; 2) or data is time out; | ||
| 145 | - // 3) or | ||
| 146 | - // connection is closed | ||
| 147 | - if ((connData.getEof() == true | ||
| 148 | - && !rcgOjb.isReady(stream) | ||
| 149 | - && connData.getQueueSamples().isEmpty()) | ||
| 150 | - || duration.toMillis() > deocderTimeOut | ||
| 151 | - || !connData.getWebSocket().isOpen()) { | ||
| 152 | - | ||
| 153 | - logger.info("close websocket!!!"); | ||
| 154 | - | ||
| 155 | - // delay close web socket as data may still in processing | ||
| 156 | - Timer timer = new Timer(); | ||
| 157 | - timer.schedule( | ||
| 158 | - new TimerTask() { | ||
| 159 | - public void run() { | ||
| 160 | - | ||
| 161 | - webSocket.close(); | ||
| 162 | - } | ||
| 163 | - }, | ||
| 164 | - 5000); // 5 seconds | ||
| 165 | - } | ||
| 166 | - } | ||
| 167 | - | ||
| 168 | - } catch (Exception e) { | ||
| 169 | - e.printStackTrace(); | ||
| 170 | - } | ||
| 171 | - } | ||
| 172 | - } | ||
| 173 | -} | 1 | +/* |
| 2 | + * // Copyright 2022-2023 by zhaoming | ||
| 3 | + */ | ||
| 4 | +// java DecoderThreadHandler | ||
| 5 | +package websocketsrv; | ||
| 6 | + | ||
| 7 | +import com.k2fsa.sherpa.onnx.OnlineRecognizer; | ||
| 8 | +import com.k2fsa.sherpa.onnx.OnlineStream; | ||
| 9 | +import java.nio.*; | ||
| 10 | +import java.nio.charset.StandardCharsets; | ||
| 11 | +import java.time.LocalDateTime; | ||
| 12 | +import java.util.*; | ||
| 13 | +import java.util.List; | ||
| 14 | +import java.util.concurrent.*; | ||
| 15 | +import java.util.concurrent.LinkedBlockingQueue; | ||
| 16 | +import org.java_websocket.WebSocket; | ||
| 17 | +import org.java_websocket.drafts.Draft; | ||
| 18 | +import org.java_websocket.framing.Framedata; | ||
| 19 | +import org.slf4j.Logger; | ||
| 20 | +import org.slf4j.LoggerFactory; | ||
| 21 | + | ||
| 22 | +public class DecoderThreadHandler extends Thread { | ||
| 23 | + private static final Logger logger = LoggerFactory.getLogger(DecoderThreadHandler.class); | ||
| 24 | + // Websocket Queue that waiting for decoding | ||
| 25 | + private LinkedBlockingQueue<WebSocket> decoderQueue; | ||
| 26 | + // the mapping between websocket and connection data | ||
| 27 | + private ConcurrentHashMap<WebSocket, ConnectionData> connMap; | ||
| 28 | + | ||
| 29 | + private OnlineRecognizer rcgOjb = null; // recgnizer object | ||
| 30 | + | ||
| 31 | + // connection data list for this thread to decode in parallel | ||
| 32 | + private List<ConnectionData> connDataList = new ArrayList<ConnectionData>(); | ||
| 33 | + | ||
| 34 | + private int parallelDecoderNum = 10; // parallel decoding number | ||
| 35 | + private int deocderTimeIdle = 10; // idle time(ms) when no job | ||
| 36 | + private int deocderTimeOut = 3000; // if it is timeout(ms), the connection data will be removed | ||
| 37 | + | ||
| 38 | + public DecoderThreadHandler( | ||
| 39 | + LinkedBlockingQueue<WebSocket> decoderQueue, | ||
| 40 | + ConcurrentHashMap<WebSocket, ConnectionData> connMap, | ||
| 41 | + OnlineRecognizer rcgOjb, | ||
| 42 | + int deocderTimeIdle, | ||
| 43 | + int parallelDecoderNum, | ||
| 44 | + int deocderTimeOut) { | ||
| 45 | + this.decoderQueue = decoderQueue; | ||
| 46 | + this.connMap = connMap; | ||
| 47 | + this.rcgOjb = rcgOjb; | ||
| 48 | + this.deocderTimeIdle = deocderTimeIdle; | ||
| 49 | + this.parallelDecoderNum = parallelDecoderNum; | ||
| 50 | + this.deocderTimeOut = deocderTimeOut; | ||
| 51 | + } | ||
| 52 | + | ||
| 53 | + public void run() { | ||
| 54 | + while (true) { | ||
| 55 | + try { | ||
| 56 | + // time(ms) idle if there is no job | ||
| 57 | + | ||
| 58 | + Thread.sleep(deocderTimeIdle); | ||
| 59 | + // clear data list for this threads | ||
| 60 | + connDataList.clear(); | ||
| 61 | + if (rcgOjb == null) continue; | ||
| 62 | + | ||
| 63 | + // loop for total decoder Queue | ||
| 64 | + while (!decoderQueue.isEmpty()) { | ||
| 65 | + | ||
| 66 | + // get websocket | ||
| 67 | + WebSocket conn = decoderQueue.take(); | ||
| 68 | + // get connection data according to websocket | ||
| 69 | + ConnectionData connData = connMap.get(conn); | ||
| 70 | + | ||
| 71 | + // if the websocket closed, continue | ||
| 72 | + if (connData == null) continue; | ||
| 73 | + // get the stream | ||
| 74 | + OnlineStream stream = connData.getStream(); | ||
| 75 | + | ||
| 76 | + // put to decoder list if 1) stream is ready; 2) and | ||
| 77 | + // size not > parallelDecoderNum | ||
| 78 | + if ((rcgOjb.isReady(stream) && connDataList.size() < parallelDecoderNum)) { | ||
| 79 | + | ||
| 80 | + // add to this thread's decoder list | ||
| 81 | + connDataList.add(connData); | ||
| 82 | + // change the handled time for this connection data | ||
| 83 | + connData.setLastHandleTime(LocalDateTime.now()); | ||
| 84 | + } | ||
| 85 | + // break when decoder list size >= parallelDecoderNum | ||
| 86 | + if (connDataList.size() >= parallelDecoderNum) { | ||
| 87 | + break; | ||
| 88 | + } | ||
| 89 | + } | ||
| 90 | + | ||
| 91 | + // if decoder data list for this thread >0 | ||
| 92 | + if (connDataList.size() > 0) { | ||
| 93 | + | ||
| 94 | + // create a stream array for parallel decoding | ||
| 95 | + OnlineStream[] arr = new OnlineStream[connDataList.size()]; | ||
| 96 | + for (int i = 0; i < connDataList.size(); i++) { | ||
| 97 | + | ||
| 98 | + arr[i] = connDataList.get(i).getStream(); | ||
| 99 | + } | ||
| 100 | + | ||
| 101 | + // parallel decoding | ||
| 102 | + rcgOjb.decodeStreams(arr); | ||
| 103 | + } | ||
| 104 | + | ||
| 105 | + // get result for each connection | ||
| 106 | + for (ConnectionData connData : connDataList) { | ||
| 107 | + | ||
| 108 | + OnlineStream stream = connData.getStream(); | ||
| 109 | + WebSocket webSocket = connData.getWebSocket(); | ||
| 110 | + | ||
| 111 | + String txtResult = rcgOjb.getResult(stream); | ||
| 112 | + | ||
| 113 | + // decode text in utf-8 | ||
| 114 | + byte[] utf8Data = txtResult.getBytes(StandardCharsets.UTF_8); | ||
| 115 | + | ||
| 116 | + boolean isEof = (connData.getEof() == true && !rcgOjb.isReady(stream)); | ||
| 117 | + // result | ||
| 118 | + if (utf8Data.length > 0) { | ||
| 119 | + | ||
| 120 | + String jsonResult = | ||
| 121 | + "{\"text\":\"" + txtResult + "\",\"eof\":" + String.valueOf(isEof) + "\"}"; | ||
| 122 | + | ||
| 123 | + if (webSocket.isOpen()) { | ||
| 124 | + // create a TEXT Frame for send back json result | ||
| 125 | + Draft draft = webSocket.getDraft(); | ||
| 126 | + List<Framedata> frames = null; | ||
| 127 | + frames = draft.createFrames(jsonResult, false); | ||
| 128 | + // send to client | ||
| 129 | + webSocket.sendFrame(frames); | ||
| 130 | + } | ||
| 131 | + } | ||
| 132 | + } | ||
| 133 | + // loop for each connection data in this thread | ||
| 134 | + for (ConnectionData connData : connDataList) { | ||
| 135 | + OnlineStream stream = connData.getStream(); | ||
| 136 | + WebSocket webSocket = connData.getWebSocket(); | ||
| 137 | + // if the stream is still ready, put it to decoder Queue again for next decoding | ||
| 138 | + if (rcgOjb.isReady(stream)) { | ||
| 139 | + decoderQueue.put(webSocket); | ||
| 140 | + } | ||
| 141 | + // the duration between last handled time and now | ||
| 142 | + java.time.Duration duration = | ||
| 143 | + java.time.Duration.between(connData.getLastHandleTime(), LocalDateTime.now()); | ||
| 144 | + // close the websocket if 1) data is done and stream not ready; 2) or data is time out; | ||
| 145 | + // 3) or | ||
| 146 | + // connection is closed | ||
| 147 | + if ((connData.getEof() == true | ||
| 148 | + && !rcgOjb.isReady(stream) | ||
| 149 | + && connData.getQueueSamples().isEmpty()) | ||
| 150 | + || duration.toMillis() > deocderTimeOut | ||
| 151 | + || !connData.getWebSocket().isOpen()) { | ||
| 152 | + | ||
| 153 | + logger.info("close websocket!!!"); | ||
| 154 | + | ||
| 155 | + // delay close web socket as data may still in processing | ||
| 156 | + Timer timer = new Timer(); | ||
| 157 | + timer.schedule( | ||
| 158 | + new TimerTask() { | ||
| 159 | + public void run() { | ||
| 160 | + | ||
| 161 | + webSocket.close(); | ||
| 162 | + } | ||
| 163 | + }, | ||
| 164 | + 5000); // 5 seconds | ||
| 165 | + } | ||
| 166 | + } | ||
| 167 | + | ||
| 168 | + } catch (Exception e) { | ||
| 169 | + e.printStackTrace(); | ||
| 170 | + } | ||
| 171 | + } | ||
| 172 | + } | ||
| 173 | +} |
| 1 | -/* | ||
| 2 | - * // Copyright 2022-2023 by zhaoming | ||
| 3 | - */ | ||
| 4 | -// java StreamThreadHandler | ||
| 5 | -package websocketsrv; | ||
| 6 | - | ||
| 7 | -import com.k2fsa.sherpa.onnx.OnlineStream; | ||
| 8 | -import java.nio.*; | ||
| 9 | -import java.util.*; | ||
| 10 | -import java.util.concurrent.*; | ||
| 11 | -import java.util.concurrent.LinkedBlockingQueue; | ||
| 12 | -import org.java_websocket.WebSocket; | ||
| 13 | -// thread for processing stream | ||
| 14 | - | ||
| 15 | -public class StreamThreadHandler extends Thread { | ||
| 16 | - // Queue between io network io thread pool and stream thread pool, use websocket as the key | ||
| 17 | - private LinkedBlockingQueue<WebSocket> streamQueue; | ||
| 18 | - // Queue waiting for deocdeing, use websocket as the key | ||
| 19 | - private LinkedBlockingQueue<WebSocket> decoderQueue; | ||
| 20 | - // mapping between websocket connection and connection data | ||
| 21 | - private ConcurrentHashMap<WebSocket, ConnectionData> connMap; | ||
| 22 | - | ||
| 23 | - public StreamThreadHandler( | ||
| 24 | - LinkedBlockingQueue<WebSocket> streamQueue, | ||
| 25 | - LinkedBlockingQueue<WebSocket> decoderQueue, | ||
| 26 | - ConcurrentHashMap<WebSocket, ConnectionData> connMap) { | ||
| 27 | - this.streamQueue = streamQueue; | ||
| 28 | - this.decoderQueue = decoderQueue; | ||
| 29 | - this.connMap = connMap; | ||
| 30 | - } | ||
| 31 | - | ||
| 32 | - public void run() { | ||
| 33 | - while (true) { | ||
| 34 | - try { | ||
| 35 | - // fetch one websocket from queue | ||
| 36 | - WebSocket conn = (WebSocket) this.streamQueue.take(); | ||
| 37 | - // get the connection data according to websocket | ||
| 38 | - ConnectionData connData = connMap.get(conn); | ||
| 39 | - OnlineStream stream = connData.getStream(); | ||
| 40 | - | ||
| 41 | - // handle received binary data | ||
| 42 | - if (!connData.getQueueSamples().isEmpty()) { | ||
| 43 | - // loop to put all received binary data to stream | ||
| 44 | - while (!connData.getQueueSamples().isEmpty()) { | ||
| 45 | - | ||
| 46 | - float[] samples = connData.getQueueSamples().poll(); | ||
| 47 | - | ||
| 48 | - stream.acceptWaveform(samples); | ||
| 49 | - } | ||
| 50 | - // if data is finished | ||
| 51 | - if (connData.getEof() == true) { | ||
| 52 | - | ||
| 53 | - stream.inputFinished(); | ||
| 54 | - } | ||
| 55 | - // add this websocket to decoder Queue if not in the Queue | ||
| 56 | - if (!decoderQueue.contains(conn)) { | ||
| 57 | - | ||
| 58 | - decoderQueue.put(conn); | ||
| 59 | - } | ||
| 60 | - } | ||
| 61 | - | ||
| 62 | - } catch (Exception e) { | ||
| 63 | - e.printStackTrace(); | ||
| 64 | - } | ||
| 65 | - } | ||
| 66 | - } | ||
| 67 | -} | 1 | +/* |
| 2 | + * // Copyright 2022-2023 by zhaoming | ||
| 3 | + */ | ||
| 4 | +// java StreamThreadHandler | ||
| 5 | +package websocketsrv; | ||
| 6 | + | ||
| 7 | +import com.k2fsa.sherpa.onnx.OnlineStream; | ||
| 8 | +import java.nio.*; | ||
| 9 | +import java.util.*; | ||
| 10 | +import java.util.concurrent.*; | ||
| 11 | +import java.util.concurrent.LinkedBlockingQueue; | ||
| 12 | +import org.java_websocket.WebSocket; | ||
| 13 | +// thread for processing stream | ||
| 14 | + | ||
| 15 | +public class StreamThreadHandler extends Thread { | ||
| 16 | + // Queue between io network io thread pool and stream thread pool, use websocket as the key | ||
| 17 | + private LinkedBlockingQueue<WebSocket> streamQueue; | ||
| 18 | + // Queue waiting for deocdeing, use websocket as the key | ||
| 19 | + private LinkedBlockingQueue<WebSocket> decoderQueue; | ||
| 20 | + // mapping between websocket connection and connection data | ||
| 21 | + private ConcurrentHashMap<WebSocket, ConnectionData> connMap; | ||
| 22 | + | ||
| 23 | + public StreamThreadHandler( | ||
| 24 | + LinkedBlockingQueue<WebSocket> streamQueue, | ||
| 25 | + LinkedBlockingQueue<WebSocket> decoderQueue, | ||
| 26 | + ConcurrentHashMap<WebSocket, ConnectionData> connMap) { | ||
| 27 | + this.streamQueue = streamQueue; | ||
| 28 | + this.decoderQueue = decoderQueue; | ||
| 29 | + this.connMap = connMap; | ||
| 30 | + } | ||
| 31 | + | ||
| 32 | + public void run() { | ||
| 33 | + while (true) { | ||
| 34 | + try { | ||
| 35 | + // fetch one websocket from queue | ||
| 36 | + WebSocket conn = (WebSocket) this.streamQueue.take(); | ||
| 37 | + // get the connection data according to websocket | ||
| 38 | + ConnectionData connData = connMap.get(conn); | ||
| 39 | + OnlineStream stream = connData.getStream(); | ||
| 40 | + | ||
| 41 | + // handle received binary data | ||
| 42 | + if (!connData.getQueueSamples().isEmpty()) { | ||
| 43 | + // loop to put all received binary data to stream | ||
| 44 | + while (!connData.getQueueSamples().isEmpty()) { | ||
| 45 | + | ||
| 46 | + float[] samples = connData.getQueueSamples().poll(); | ||
| 47 | + | ||
| 48 | + stream.acceptWaveform(samples); | ||
| 49 | + } | ||
| 50 | + // if data is finished | ||
| 51 | + if (connData.getEof() == true) { | ||
| 52 | + | ||
| 53 | + stream.inputFinished(); | ||
| 54 | + } | ||
| 55 | + // add this websocket to decoder Queue if not in the Queue | ||
| 56 | + if (!decoderQueue.contains(conn)) { | ||
| 57 | + | ||
| 58 | + decoderQueue.put(conn); | ||
| 59 | + } | ||
| 60 | + } | ||
| 61 | + | ||
| 62 | + } catch (Exception e) { | ||
| 63 | + e.printStackTrace(); | ||
| 64 | + } | ||
| 65 | + } | ||
| 66 | + } | ||
| 67 | +} |
-
请 注册 或 登录 后发表评论