xuning / sherpaonnx

转到一个项目
Toggle navigation Toggle navigation pinning
Fangjun Kuang
Committed by GitHub
db85b2c1 1 parent 7322f4e0

Add Android APKs for NeMo CTC models. (#866)

内嵌 并排对比
正在显示 9 个修改的文件 包含 210 行增加 和 3 行删除
@@ -23,8 +23,8 @@ jobs: @@ -23,8 +23,8 @@ jobs:
23 fail-fast: false 23 fail-fast: false
24 matrix: 24 matrix:
25 os: [ubuntu-latest] 25 os: [ubuntu-latest]
26 - total: ["3"]  
27 - index: ["0", "1", "2"] 26 + total: ["5"]
  27 + index: ["0", "1", "2", "3", "4"]
28 28
29 steps: 29 steps:
30 - uses: actions/checkout@v4 30 - uses: actions/checkout@v4
@@ -63,6 +63,12 @@ function testOnlineAsr() { @@ -63,6 +63,12 @@ function testOnlineAsr() {
63 git clone https://huggingface.co/csukuangfj/sherpa-onnx-streaming-zipformer-en-2023-02-21 63 git clone https://huggingface.co/csukuangfj/sherpa-onnx-streaming-zipformer-en-2023-02-21
64 fi 64 fi
65 65
  66 + if [ ! -f ./sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms/tokens.txt ]; then
  67 + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms.tar.bz2
  68 + tar xvf sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms.tar.bz2
  69 + rm sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms.tar.bz2
  70 + fi
  71 +
66 if [ ! -d ./sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13 ]; then 72 if [ ! -d ./sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13 ]; then
67 curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2 73 curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2
68 tar xvf sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2 74 tar xvf sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2
@@ -4,6 +4,7 @@ fun main() { @@ -4,6 +4,7 @@ fun main() {
4 testOnlineAsr("transducer") 4 testOnlineAsr("transducer")
5 testOnlineAsr("zipformer2-ctc") 5 testOnlineAsr("zipformer2-ctc")
6 testOnlineAsr("ctc-hlg") 6 testOnlineAsr("ctc-hlg")
  7 + testOnlineAsr("nemo-ctc")
7 } 8 }
8 9
9 fun testOnlineAsr(type: String) { 10 fun testOnlineAsr(type: String) {
@@ -42,6 +43,17 @@ fun testOnlineAsr(type: String) { @@ -42,6 +43,17 @@ fun testOnlineAsr(type: String) {
42 debug = false, 43 debug = false,
43 ) 44 )
44 } 45 }
  46 + "nemo-ctc" -> {
  47 + waveFilename = "./sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms/test_wavs/0.wav"
  48 + OnlineModelConfig(
  49 + neMoCtc = OnlineNeMoCtcModelConfig(
  50 + model = "./sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms/model.onnx",
  51 + ),
  52 + tokens = "./sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms/tokens.txt",
  53 + numThreads = 1,
  54 + debug = false,
  55 + )
  56 + }
45 "ctc-hlg" -> { 57 "ctc-hlg" -> {
46 waveFilename = "./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18/test_wavs/1.wav" 58 waveFilename = "./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18/test_wavs/1.wav"
47 ctcFstDecoderConfig.graph = "./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18/HLG.fst" 59 ctcFstDecoderConfig.graph = "./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18/HLG.fst"
@@ -170,6 +170,48 @@ def get_models(): @@ -170,6 +170,48 @@ def get_models():
170 popd 170 popd
171 """, 171 """,
172 ), 172 ),
  173 + Model(
  174 + model_name="sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms",
  175 + idx=11,
  176 + lang="en",
  177 + short_name="nemo_ctc_80ms",
  178 + cmd="""
  179 + pushd $model_name
  180 + rm -rf test_wavs
  181 +
  182 + ls -lh
  183 +
  184 + popd
  185 + """,
  186 + ),
  187 + Model(
  188 + model_name="sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-480ms",
  189 + idx=12,
  190 + lang="en",
  191 + short_name="nemo_ctc_480ms",
  192 + cmd="""
  193 + pushd $model_name
  194 + rm -rf test_wavs
  195 +
  196 + ls -lh
  197 +
  198 + popd
  199 + """,
  200 + ),
  201 + Model(
  202 + model_name="sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-1040ms",
  203 + idx=13,
  204 + lang="en",
  205 + short_name="nemo_ctc_1040ms",
  206 + cmd="""
  207 + pushd $model_name
  208 + rm -rf test_wavs
  209 +
  210 + ls -lh
  211 +
  212 + popd
  213 + """,
  214 + ),
173 ] 215 ]
174 216
175 return models 217 return models
@@ -106,6 +106,66 @@ def get_models(): @@ -106,6 +106,66 @@ def get_models():
106 popd 106 popd
107 """, 107 """,
108 ), 108 ),
  109 + Model(
  110 + model_name="sherpa-onnx-nemo-fast-conformer-ctc-be-de-en-es-fr-hr-it-pl-ru-uk-20k",
  111 + idx=7,
  112 + lang="be_de_en_es_fr_hr_it_pl_ru_uk",
  113 + short_name="fast_conformer_ctc_20k",
  114 + cmd="""
  115 + pushd $model_name
  116 +
  117 + rm -rfv test_wavs
  118 +
  119 + ls -lh
  120 +
  121 + popd
  122 + """,
  123 + ),
  124 + Model(
  125 + model_name="sherpa-onnx-nemo-fast-conformer-ctc-en-24500",
  126 + idx=8,
  127 + lang="en",
  128 + short_name="fast_conformer_ctc_24500",
  129 + cmd="""
  130 + pushd $model_name
  131 +
  132 + rm -rfv test_wavs
  133 +
  134 + ls -lh
  135 +
  136 + popd
  137 + """,
  138 + ),
  139 + Model(
  140 + model_name="sherpa-onnx-nemo-fast-conformer-ctc-en-de-es-fr-14288",
  141 + idx=9,
  142 + lang="en_des_es_fr",
  143 + short_name="fast_conformer_ctc_14288",
  144 + cmd="""
  145 + pushd $model_name
  146 +
  147 + rm -rfv test_wavs
  148 +
  149 + ls -lh
  150 +
  151 + popd
  152 + """,
  153 + ),
  154 + Model(
  155 + model_name="sherpa-onnx-nemo-fast-conformer-ctc-es-1424",
  156 + idx=10,
  157 + lang="es",
  158 + short_name="fast_conformer_ctc_1424",
  159 + cmd="""
  160 + pushd $model_name
  161 +
  162 + rm -rfv test_wavs
  163 +
  164 + ls -lh
  165 +
  166 + popd
  167 + """,
  168 + ),
109 ] 169 ]
110 return models 170 return models
111 171
@@ -163,7 +163,8 @@ std::unique_ptr<OfflineCtcModel> OfflineCtcModel::Create( @@ -163,7 +163,8 @@ std::unique_ptr<OfflineCtcModel> OfflineCtcModel::Create(
163 return std::make_unique<OfflineNemoEncDecCtcModel>(mgr, config); 163 return std::make_unique<OfflineNemoEncDecCtcModel>(mgr, config);
164 break; 164 break;
165 case ModelType::kEncDecHybridRNNTCTCBPEModel: 165 case ModelType::kEncDecHybridRNNTCTCBPEModel:
166 - return std::make_unique<OfflineNemoEncDecHybridRNNTCTCBPEModel>(config); 166 + return std::make_unique<OfflineNemoEncDecHybridRNNTCTCBPEModel>(mgr,
  167 + config);
167 break; 168 break;
168 case ModelType::kTdnn: 169 case ModelType::kTdnn:
169 return std::make_unique<OfflineTdnnCtcModel>(mgr, config); 170 return std::make_unique<OfflineTdnnCtcModel>(mgr, config);
@@ -159,6 +159,18 @@ static OnlineRecognizerConfig GetConfig(JNIEnv *env, jobject config) { @@ -159,6 +159,18 @@ static OnlineRecognizerConfig GetConfig(JNIEnv *env, jobject config) {
159 ans.model_config.zipformer2_ctc.model = p; 159 ans.model_config.zipformer2_ctc.model = p;
160 env->ReleaseStringUTFChars(s, p); 160 env->ReleaseStringUTFChars(s, p);
161 161
  162 + // streaming NeMo CTC
  163 + fid = env->GetFieldID(model_config_cls, "neMoCtc",
  164 + "Lcom/k2fsa/sherpa/onnx/OnlineNeMoCtcModelConfig;");
  165 + jobject nemo_ctc_config = env->GetObjectField(model_config, fid);
  166 + jclass nemo_ctc_config_cls = env->GetObjectClass(nemo_ctc_config);
  167 +
  168 + fid = env->GetFieldID(nemo_ctc_config_cls, "model", "Ljava/lang/String;");
  169 + s = (jstring)env->GetObjectField(nemo_ctc_config, fid);
  170 + p = env->GetStringUTFChars(s, nullptr);
  171 + ans.model_config.nemo_ctc.model = p;
  172 + env->ReleaseStringUTFChars(s, p);
  173 +
162 fid = env->GetFieldID(model_config_cls, "tokens", "Ljava/lang/String;"); 174 fid = env->GetFieldID(model_config_cls, "tokens", "Ljava/lang/String;");
163 s = (jstring)env->GetObjectField(model_config, fid); 175 s = (jstring)env->GetObjectField(model_config, fid);
164 p = env->GetStringUTFChars(s, nullptr); 176 p = env->GetStringUTFChars(s, nullptr);
@@ -231,6 +231,45 @@ fun getOfflineModelConfig(type: Int): OfflineModelConfig? { @@ -231,6 +231,45 @@ fun getOfflineModelConfig(type: Int): OfflineModelConfig? {
231 ) 231 )
232 } 232 }
233 233
  234 + 7 -> {
  235 + val modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-be-de-en-es-fr-hr-it-pl-ru-uk-20k"
  236 + return OfflineModelConfig(
  237 + nemo = OfflineNemoEncDecCtcModelConfig(
  238 + model = "$modelDir/model.onnx",
  239 + ),
  240 + tokens = "$modelDir/tokens.txt",
  241 + )
  242 + }
  243 +
  244 + 8 -> {
  245 + val modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-en-24500"
  246 + return OfflineModelConfig(
  247 + nemo = OfflineNemoEncDecCtcModelConfig(
  248 + model = "$modelDir/model.onnx",
  249 + ),
  250 + tokens = "$modelDir/tokens.txt",
  251 + )
  252 + }
  253 +
  254 + 9 -> {
  255 + val modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-en-de-es-fr-14288"
  256 + return OfflineModelConfig(
  257 + nemo = OfflineNemoEncDecCtcModelConfig(
  258 + model = "$modelDir/model.onnx",
  259 + ),
  260 + tokens = "$modelDir/tokens.txt",
  261 + )
  262 + }
  263 +
  264 + 10 -> {
  265 + val modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-es-1424"
  266 + return OfflineModelConfig(
  267 + nemo = OfflineNemoEncDecCtcModelConfig(
  268 + model = "$modelDir/model.onnx",
  269 + ),
  270 + tokens = "$modelDir/tokens.txt",
  271 + )
  272 + }
234 } 273 }
235 return null 274 return null
236 } 275 }
@@ -29,10 +29,15 @@ data class OnlineZipformer2CtcModelConfig( @@ -29,10 +29,15 @@ data class OnlineZipformer2CtcModelConfig(
29 var model: String = "", 29 var model: String = "",
30 ) 30 )
31 31
  32 +data class OnlineNeMoCtcModelConfig(
  33 + var model: String = "",
  34 +)
  35 +
32 data class OnlineModelConfig( 36 data class OnlineModelConfig(
33 var transducer: OnlineTransducerModelConfig = OnlineTransducerModelConfig(), 37 var transducer: OnlineTransducerModelConfig = OnlineTransducerModelConfig(),
34 var paraformer: OnlineParaformerModelConfig = OnlineParaformerModelConfig(), 38 var paraformer: OnlineParaformerModelConfig = OnlineParaformerModelConfig(),
35 var zipformer2Ctc: OnlineZipformer2CtcModelConfig = OnlineZipformer2CtcModelConfig(), 39 var zipformer2Ctc: OnlineZipformer2CtcModelConfig = OnlineZipformer2CtcModelConfig(),
  40 + var neMoCtc: OnlineNeMoCtcModelConfig = OnlineNeMoCtcModelConfig(),
36 var tokens: String, 41 var tokens: String,
37 var numThreads: Int = 1, 42 var numThreads: Int = 1,
38 var debug: Boolean = false, 43 var debug: Boolean = false,
@@ -318,6 +323,36 @@ fun getModelConfig(type: Int): OnlineModelConfig? { @@ -318,6 +323,36 @@ fun getModelConfig(type: Int): OnlineModelConfig? {
318 modelType = "zipformer", 323 modelType = "zipformer",
319 ) 324 )
320 } 325 }
  326 +
  327 + 11 -> {
  328 + val modelDir = "sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms"
  329 + return OnlineModelConfig(
  330 + neMoCtc = OnlineNeMoCtcModelConfig(
  331 + model = "$modelDir/model.onnx",
  332 + ),
  333 + tokens = "$modelDir/tokens.txt",
  334 + )
  335 + }
  336 +
  337 + 12 -> {
  338 + val modelDir = "sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-480ms"
  339 + return OnlineModelConfig(
  340 + neMoCtc = OnlineNeMoCtcModelConfig(
  341 + model = "$modelDir/model.onnx",
  342 + ),
  343 + tokens = "$modelDir/tokens.txt",
  344 + )
  345 + }
  346 +
  347 + 13 -> {
  348 + val modelDir = "sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-1040ms"
  349 + return OnlineModelConfig(
  350 + neMoCtc = OnlineNeMoCtcModelConfig(
  351 + model = "$modelDir/model.onnx",
  352 + ),
  353 + tokens = "$modelDir/tokens.txt",
  354 + )
  355 + }
321 } 356 }
322 return null 357 return null
323 } 358 }