Fangjun Kuang
Committed by GitHub

Add two-pass speech recognition Android/iOS demo (#304)

正在显示 97 个修改的文件 包含 3544 行增加55 行删除
@@ -60,3 +60,10 @@ run-offline-decode-files-nemo-ctc.sh @@ -60,3 +60,10 @@ run-offline-decode-files-nemo-ctc.sh
60 *.jar 60 *.jar
61 sherpa-onnx-nemo-ctc-* 61 sherpa-onnx-nemo-ctc-*
62 *.wav 62 *.wav
  63 +sherpa-onnx-zipformer-*
  64 +sherpa-onnx-conformer-*
  65 +sherpa-onnx-whisper-*
  66 +swift-api-examples/k2fsa-*
  67 +run-*.sh
  68 +two-pass-*.sh
  69 +build-*
@@ -21,10 +21,6 @@ private const val REQUEST_RECORD_AUDIO_PERMISSION = 200 @@ -21,10 +21,6 @@ private const val REQUEST_RECORD_AUDIO_PERMISSION = 200
21 class MainActivity : AppCompatActivity() { 21 class MainActivity : AppCompatActivity() {
22 private val permissions: Array<String> = arrayOf(Manifest.permission.RECORD_AUDIO) 22 private val permissions: Array<String> = arrayOf(Manifest.permission.RECORD_AUDIO)
23 23
24 - // If there is a GPU and useGPU is true, we will use GPU  
25 - // If there is no GPU and useGPU is true, we won't use GPU  
26 - private val useGPU: Boolean = true  
27 -  
28 private lateinit var model: SherpaOnnx 24 private lateinit var model: SherpaOnnx
29 private var audioRecord: AudioRecord? = null 25 private var audioRecord: AudioRecord? = null
30 private lateinit var recordButton: Button 26 private lateinit var recordButton: Button
@@ -91,7 +87,7 @@ class MainActivity : AppCompatActivity() { @@ -91,7 +87,7 @@ class MainActivity : AppCompatActivity() {
91 audioRecord!!.startRecording() 87 audioRecord!!.startRecording()
92 recordButton.setText(R.string.stop) 88 recordButton.setText(R.string.stop)
93 isRecording = true 89 isRecording = true
94 - model.reset() 90 + model.reset(true)
95 textView.text = "" 91 textView.text = ""
96 lastText = "" 92 lastText = ""
97 idx = 0 93 idx = 0
@@ -125,26 +121,32 @@ class MainActivity : AppCompatActivity() { @@ -125,26 +121,32 @@ class MainActivity : AppCompatActivity() {
125 while (model.isReady()) { 121 while (model.isReady()) {
126 model.decode() 122 model.decode()
127 } 123 }
128 - runOnUiThread {  
129 - val isEndpoint = model.isEndpoint()  
130 - val text = model.text  
131 -  
132 - if(text.isNotBlank()) {  
133 - if (lastText.isBlank()) {  
134 - textView.text = "${idx}: ${text}"  
135 - } else {  
136 - textView.text = "${lastText}\n${idx}: ${text}"  
137 - } 124 +
  125 + val isEndpoint = model.isEndpoint()
  126 + val text = model.text
  127 +
  128 + var textToDisplay = lastText;
  129 +
  130 + if(text.isNotBlank()) {
  131 + if (lastText.isBlank()) {
  132 + textToDisplay = "${idx}: ${text}"
  133 + } else {
  134 + textToDisplay = "${lastText}\n${idx}: ${text}"
138 } 135 }
  136 + }
139 137
140 - if (isEndpoint) {  
141 - model.reset()  
142 - if (text.isNotBlank()) {  
143 - lastText = "${lastText}\n${idx}: ${text}"  
144 - idx += 1  
145 - } 138 + if (isEndpoint) {
  139 + model.reset()
  140 + if (text.isNotBlank()) {
  141 + lastText = "${lastText}\n${idx}: ${text}"
  142 + textToDisplay = lastText;
  143 + idx += 1
146 } 144 }
147 } 145 }
  146 +
  147 + runOnUiThread {
  148 + textView.text = textToDisplay
  149 + }
148 } 150 }
149 } 151 }
150 } 152 }
@@ -77,7 +77,7 @@ class SherpaOnnx( @@ -77,7 +77,7 @@ class SherpaOnnx(
77 acceptWaveform(ptr, samples, sampleRate) 77 acceptWaveform(ptr, samples, sampleRate)
78 78
79 fun inputFinished() = inputFinished(ptr) 79 fun inputFinished() = inputFinished(ptr)
80 - fun reset() = reset(ptr) 80 + fun reset(recreate: Boolean = false) = reset(ptr, recreate = recreate)
81 fun decode() = decode(ptr) 81 fun decode() = decode(ptr)
82 fun isEndpoint(): Boolean = isEndpoint(ptr) 82 fun isEndpoint(): Boolean = isEndpoint(ptr)
83 fun isReady(): Boolean = isReady(ptr) 83 fun isReady(): Boolean = isReady(ptr)
@@ -99,7 +99,7 @@ class SherpaOnnx( @@ -99,7 +99,7 @@ class SherpaOnnx(
99 private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int) 99 private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int)
100 private external fun inputFinished(ptr: Long) 100 private external fun inputFinished(ptr: Long)
101 private external fun getText(ptr: Long): String 101 private external fun getText(ptr: Long): String
102 - private external fun reset(ptr: Long) 102 + private external fun reset(ptr: Long, recreate: Boolean)
103 private external fun decode(ptr: Long) 103 private external fun decode(ptr: Long)
104 private external fun isEndpoint(ptr: Long): Boolean 104 private external fun isEndpoint(ptr: Long): Boolean
105 private external fun isReady(ptr: Long): Boolean 105 private external fun isReady(ptr: Long): Boolean
  1 +*.iml
  2 +.gradle
  3 +/local.properties
  4 +/.idea/caches
  5 +/.idea/libraries
  6 +/.idea/modules.xml
  7 +/.idea/workspace.xml
  8 +/.idea/navEditor.xml
  9 +/.idea/assetWizardSettings.xml
  10 +.DS_Store
  11 +/build
  12 +/captures
  13 +.externalNativeBuild
  14 +.cxx
  15 +local.properties
  1 +# Default ignored files
  2 +/shelf/
  3 +/workspace.xml
  1 +<?xml version="1.0" encoding="UTF-8"?>
  2 +<project version="4">
  3 + <component name="CompilerConfiguration">
  4 + <bytecodeTargetLevel target="11" />
  5 + </component>
  6 +</project>
  1 +<?xml version="1.0" encoding="UTF-8"?>
  2 +<project version="4">
  3 + <component name="GradleMigrationSettings" migrationVersion="1" />
  4 + <component name="GradleSettings">
  5 + <option name="linkedExternalProjectsSettings">
  6 + <GradleProjectSettings>
  7 + <option name="testRunner" value="GRADLE" />
  8 + <option name="distributionType" value="DEFAULT_WRAPPED" />
  9 + <option name="externalProjectPath" value="$PROJECT_DIR$" />
  10 + <option name="modules">
  11 + <set>
  12 + <option value="$PROJECT_DIR$" />
  13 + <option value="$PROJECT_DIR$/app" />
  14 + </set>
  15 + </option>
  16 + </GradleProjectSettings>
  17 + </option>
  18 + </component>
  19 +</project>
  1 +<?xml version="1.0" encoding="UTF-8"?>
  2 +<project version="4">
  3 + <component name="ExternalStorageConfigurationManager" enabled="true" />
  4 + <component name="ProjectRootManager" version="2" languageLevel="JDK_11" default="true" project-jdk-name="Android Studio default JDK" project-jdk-type="JavaSDK">
  5 + <output url="file://$PROJECT_DIR$/build/classes" />
  6 + </component>
  7 + <component name="ProjectType">
  8 + <option name="id" value="Android" />
  9 + </component>
  10 +</project>
  1 +<?xml version="1.0" encoding="UTF-8"?>
  2 +<project version="4">
  3 + <component name="VcsDirectoryMappings">
  4 + <mapping directory="$PROJECT_DIR$/../.." vcs="Git" />
  5 + </component>
  6 +</project>
  1 +plugins {
  2 + id 'com.android.application'
  3 + id 'org.jetbrains.kotlin.android'
  4 +}
  5 +
  6 +android {
  7 + namespace 'com.k2fsa.sherpa.onnx'
  8 + compileSdk 32
  9 +
  10 + defaultConfig {
  11 + applicationId "com.k2fsa.sherpa.onnx"
  12 + minSdk 21
  13 + targetSdk 32
  14 + versionCode 1
  15 + versionName "1.0"
  16 +
  17 + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
  18 + }
  19 +
  20 + buildTypes {
  21 + release {
  22 + minifyEnabled false
  23 + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
  24 + }
  25 + }
  26 + compileOptions {
  27 + sourceCompatibility JavaVersion.VERSION_1_8
  28 + targetCompatibility JavaVersion.VERSION_1_8
  29 + }
  30 + kotlinOptions {
  31 + jvmTarget = '1.8'
  32 + }
  33 +}
  34 +
  35 +dependencies {
  36 +
  37 + implementation 'androidx.core:core-ktx:1.7.0'
  38 + implementation 'androidx.appcompat:appcompat:1.5.1'
  39 + implementation 'com.google.android.material:material:1.7.0'
  40 + implementation 'androidx.constraintlayout:constraintlayout:2.1.4'
  41 + testImplementation 'junit:junit:4.13.2'
  42 + androidTestImplementation 'androidx.test.ext:junit:1.1.4'
  43 + androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.0'
  44 +}
  1 +# Add project specific ProGuard rules here.
  2 +# You can control the set of applied configuration files using the
  3 +# proguardFiles setting in build.gradle.
  4 +#
  5 +# For more details, see
  6 +# http://developer.android.com/guide/developing/tools/proguard.html
  7 +
  8 +# If your project uses WebView with JS, uncomment the following
  9 +# and specify the fully qualified class name to the JavaScript interface
  10 +# class:
  11 +#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
  12 +# public *;
  13 +#}
  14 +
  15 +# Uncomment this to preserve the line number information for
  16 +# debugging stack traces.
  17 +#-keepattributes SourceFile,LineNumberTable
  18 +
  19 +# If you keep the line number information, uncomment this to
  20 +# hide the original source file name.
  21 +#-renamesourcefileattribute SourceFile
  1 +package com.k2fsa.sherpa.onnx
  2 +
  3 +import androidx.test.platform.app.InstrumentationRegistry
  4 +import androidx.test.ext.junit.runners.AndroidJUnit4
  5 +
  6 +import org.junit.Test
  7 +import org.junit.runner.RunWith
  8 +
  9 +import org.junit.Assert.*
  10 +
  11 +/**
  12 + * Instrumented test, which will execute on an Android device.
  13 + *
  14 + * See [testing documentation](http://d.android.com/tools/testing).
  15 + */
  16 +@RunWith(AndroidJUnit4::class)
  17 +class ExampleInstrumentedTest {
  18 + @Test
  19 + fun useAppContext() {
  20 + // Context of the app under test.
  21 + val appContext = InstrumentationRegistry.getInstrumentation().targetContext
  22 + assertEquals("com.k2fsa.sherpa.onnx", appContext.packageName)
  23 + }
  24 +}
  1 +<?xml version="1.0" encoding="utf-8"?>
  2 +<manifest xmlns:android="http://schemas.android.com/apk/res/android"
  3 + xmlns:tools="http://schemas.android.com/tools">
  4 +
  5 + <uses-permission android:name="android.permission.RECORD_AUDIO" />
  6 +
  7 + <application
  8 + android:allowBackup="true"
  9 + android:dataExtractionRules="@xml/data_extraction_rules"
  10 + android:fullBackupContent="@xml/backup_rules"
  11 + android:icon="@mipmap/ic_launcher"
  12 + android:label="@string/app_name"
  13 + android:roundIcon="@mipmap/ic_launcher_round"
  14 + android:supportsRtl="true"
  15 + android:theme="@style/Theme.SherpaOnnx2Pass"
  16 + tools:targetApi="31">
  17 + <activity
  18 + android:name=".MainActivity"
  19 + android:exported="true">
  20 + <intent-filter>
  21 + <action android:name="android.intent.action.MAIN" />
  22 +
  23 + <category android:name="android.intent.category.LAUNCHER" />
  24 + </intent-filter>
  25 +
  26 + <meta-data
  27 + android:name="android.app.lib_name"
  28 + android:value="" />
  29 + </activity>
  30 + </application>
  31 +
  32 +</manifest>
  1 +package com.k2fsa.sherpa.onnx
  2 +
  3 +import android.Manifest
  4 +import android.content.pm.PackageManager
  5 +import android.media.AudioFormat
  6 +import android.media.AudioRecord
  7 +import android.media.MediaRecorder
  8 +import android.os.Bundle
  9 +import android.text.method.ScrollingMovementMethod
  10 +import android.util.Log
  11 +import android.widget.Button
  12 +import android.widget.TextView
  13 +import androidx.appcompat.app.AppCompatActivity
  14 +import androidx.core.app.ActivityCompat
  15 +import kotlin.concurrent.thread
  16 +
  17 +private const val TAG = "sherpa-onnx"
  18 +private const val REQUEST_RECORD_AUDIO_PERMISSION = 200
  19 +
  20 +class MainActivity : AppCompatActivity() {
  21 + private val permissions: Array<String> = arrayOf(Manifest.permission.RECORD_AUDIO)
  22 +
  23 + private lateinit var onlineRecognizer: SherpaOnnx
  24 + private lateinit var offlineRecognizer: SherpaOnnxOffline
  25 + private var audioRecord: AudioRecord? = null
  26 + private lateinit var recordButton: Button
  27 + private lateinit var textView: TextView
  28 + private var recordingThread: Thread? = null
  29 +
  30 + private val audioSource = MediaRecorder.AudioSource.MIC
  31 + private val sampleRateInHz = 16000
  32 + private val channelConfig = AudioFormat.CHANNEL_IN_MONO
  33 +
  34 + private var samplesBuffer = arrayListOf<FloatArray>()
  35 +
  36 + // Note: We don't use AudioFormat.ENCODING_PCM_FLOAT
  37 + // since the AudioRecord.read(float[]) needs API level >= 23
  38 + // but we are targeting API level >= 21
  39 + private val audioFormat = AudioFormat.ENCODING_PCM_16BIT
  40 + private var idx: Int = 0
  41 + private var lastText: String = ""
  42 +
  43 + @Volatile
  44 + private var isRecording: Boolean = false
  45 +
  46 + override fun onRequestPermissionsResult(
  47 + requestCode: Int, permissions: Array<String>, grantResults: IntArray
  48 + ) {
  49 + super.onRequestPermissionsResult(requestCode, permissions, grantResults)
  50 + val permissionToRecordAccepted = if (requestCode == REQUEST_RECORD_AUDIO_PERMISSION) {
  51 + grantResults[0] == PackageManager.PERMISSION_GRANTED
  52 + } else {
  53 + false
  54 + }
  55 +
  56 + if (!permissionToRecordAccepted) {
  57 + Log.e(TAG, "Audio record is disallowed")
  58 + finish()
  59 + }
  60 +
  61 + Log.i(TAG, "Audio record is permitted")
  62 + }
  63 +
  64 + override fun onCreate(savedInstanceState: Bundle?) {
  65 + super.onCreate(savedInstanceState)
  66 + setContentView(R.layout.activity_main)
  67 +
  68 + ActivityCompat.requestPermissions(this, permissions, REQUEST_RECORD_AUDIO_PERMISSION)
  69 +
  70 + Log.i(TAG, "Start to initialize first-pass recognizer")
  71 + initOnlineRecognizer()
  72 + Log.i(TAG, "Finished initializing first-pass recognizer")
  73 +
  74 + Log.i(TAG, "Start to initialize second-pass recognizer")
  75 + initOfflineRecognizer()
  76 + Log.i(TAG, "Finished initializing second-pass recognizer")
  77 +
  78 + recordButton = findViewById(R.id.record_button)
  79 + recordButton.setOnClickListener { onclick() }
  80 +
  81 + textView = findViewById(R.id.my_text)
  82 + textView.movementMethod = ScrollingMovementMethod()
  83 + }
  84 +
  85 + private fun onclick() {
  86 + if (!isRecording) {
  87 + val ret = initMicrophone()
  88 + if (!ret) {
  89 + Log.e(TAG, "Failed to initialize microphone")
  90 + return
  91 + }
  92 + Log.i(TAG, "state: ${audioRecord?.state}")
  93 + audioRecord!!.startRecording()
  94 + recordButton.setText(R.string.stop)
  95 + isRecording = true
  96 + onlineRecognizer.reset(true)
  97 + samplesBuffer.clear()
  98 + textView.text = ""
  99 + lastText = ""
  100 + idx = 0
  101 +
  102 + recordingThread = thread(true) {
  103 + processSamples()
  104 + }
  105 + Log.i(TAG, "Started recording")
  106 + } else {
  107 + isRecording = false
  108 + audioRecord!!.stop()
  109 + audioRecord!!.release()
  110 + audioRecord = null
  111 + recordButton.setText(R.string.start)
  112 + Log.i(TAG, "Stopped recording")
  113 + }
  114 + }
  115 +
  116 + private fun processSamples() {
  117 + Log.i(TAG, "processing samples")
  118 +
  119 + val interval = 0.1 // i.e., 100 ms
  120 + val bufferSize = (interval * sampleRateInHz).toInt() // in samples
  121 + val buffer = ShortArray(bufferSize)
  122 +
  123 + while (isRecording) {
  124 + val ret = audioRecord?.read(buffer, 0, buffer.size)
  125 + if (ret != null && ret > 0) {
  126 + val samples = FloatArray(ret) { buffer[it] / 32768.0f }
  127 + samplesBuffer.add(samples)
  128 +
  129 + onlineRecognizer.acceptWaveform(samples, sampleRate = sampleRateInHz)
  130 + while (onlineRecognizer.isReady()) {
  131 + onlineRecognizer.decode()
  132 + }
  133 + val isEndpoint = onlineRecognizer.isEndpoint()
  134 + var textToDisplay = lastText
  135 +
  136 + var text = onlineRecognizer.text
  137 + if (text.isNotBlank()) {
  138 + if (lastText.isBlank()) {
  139 + // textView.text = "${idx}: ${text}"
  140 + textToDisplay = "${idx}: ${text}"
  141 + } else {
  142 + textToDisplay = "${lastText}\n${idx}: ${text}"
  143 + }
  144 + }
  145 +
  146 + if (isEndpoint) {
  147 + onlineRecognizer.reset()
  148 +
  149 + if (text.isNotBlank()) {
  150 + text = runSecondPass()
  151 + lastText = "${lastText}\n${idx}: ${text}"
  152 + idx += 1
  153 + } else {
  154 + samplesBuffer.clear()
  155 + }
  156 + }
  157 +
  158 + runOnUiThread {
  159 + textView.text = textToDisplay.lowercase()
  160 + }
  161 + }
  162 + }
  163 + }
  164 +
  165 + private fun initMicrophone(): Boolean {
  166 + if (ActivityCompat.checkSelfPermission(
  167 + this, Manifest.permission.RECORD_AUDIO
  168 + ) != PackageManager.PERMISSION_GRANTED
  169 + ) {
  170 + ActivityCompat.requestPermissions(this, permissions, REQUEST_RECORD_AUDIO_PERMISSION)
  171 + return false
  172 + }
  173 +
  174 + val numBytes = AudioRecord.getMinBufferSize(sampleRateInHz, channelConfig, audioFormat)
  175 + Log.i(
  176 + TAG, "buffer size in milliseconds: ${numBytes * 1000.0f / sampleRateInHz}"
  177 + )
  178 +
  179 + audioRecord = AudioRecord(
  180 + audioSource,
  181 + sampleRateInHz,
  182 + channelConfig,
  183 + audioFormat,
  184 + numBytes * 2 // a sample has two bytes as we are using 16-bit PCM
  185 + )
  186 + return true
  187 + }
  188 +
  189 + private fun initOnlineRecognizer() {
  190 + // Please change getModelConfig() to add new models
  191 + // See https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html
  192 + // for a list of available models
  193 + val firstType = 1
  194 + println("Select model type ${firstType} for the first pass")
  195 + val config = OnlineRecognizerConfig(
  196 + featConfig = getFeatureConfig(sampleRate = sampleRateInHz, featureDim = 80),
  197 + modelConfig = getModelConfig(type = firstType)!!,
  198 + endpointConfig = getEndpointConfig(),
  199 + enableEndpoint = true,
  200 + )
  201 +
  202 + onlineRecognizer = SherpaOnnx(
  203 + assetManager = application.assets,
  204 + config = config,
  205 + )
  206 + }
  207 +
  208 + private fun initOfflineRecognizer() {
  209 + // Please change getOfflineModelConfig() to add new models
  210 + // See https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html
  211 + // for a list of available models
  212 + val secondType = 1
  213 + println("Select model type ${secondType} for the second pass")
  214 +
  215 + val config = OfflineRecognizerConfig(
  216 + featConfig = getFeatureConfig(sampleRate = sampleRateInHz, featureDim = 80),
  217 + modelConfig = getOfflineModelConfig(type = secondType)!!,
  218 + )
  219 +
  220 + offlineRecognizer = SherpaOnnxOffline(
  221 + assetManager = application.assets,
  222 + config = config,
  223 + )
  224 + }
  225 +
  226 + private fun runSecondPass(): String {
  227 + var totalSamples = 0
  228 + for (a in samplesBuffer) {
  229 + totalSamples += a.size
  230 + }
  231 + var i = 0
  232 +
  233 + val samples = FloatArray(totalSamples)
  234 +
  235 + // todo(fangjun): Make it more efficient
  236 + for (a in samplesBuffer) {
  237 + for (s in a) {
  238 + samples[i] = s
  239 + i += 1
  240 + }
  241 + }
  242 +
  243 +
  244 + val n = maxOf(0, samples.size - 8000)
  245 +
  246 + samplesBuffer.clear()
  247 + samplesBuffer.add(samples.sliceArray(n..samples.size-1))
  248 +
  249 + return offlineRecognizer.decode(samples.sliceArray(0..n), sampleRateInHz)
  250 + }
  251 +}
  1 +package com.k2fsa.sherpa.onnx
  2 +
  3 +import android.content.res.AssetManager
  4 +
  5 +data class EndpointRule(
  6 + var mustContainNonSilence: Boolean,
  7 + var minTrailingSilence: Float,
  8 + var minUtteranceLength: Float,
  9 +)
  10 +
  11 +data class EndpointConfig(
  12 + var rule1: EndpointRule = EndpointRule(false, 2.0f, 0.0f),
  13 + var rule2: EndpointRule = EndpointRule(true, 1.2f, 0.0f),
  14 + var rule3: EndpointRule = EndpointRule(false, 0.0f, 20.0f)
  15 +)
  16 +
  17 +data class OnlineTransducerModelConfig(
  18 + var encoder: String = "",
  19 + var decoder: String = "",
  20 + var joiner: String = "",
  21 +)
  22 +
  23 +data class OnlineParaformerModelConfig(
  24 + var encoder: String = "",
  25 + var decoder: String = "",
  26 +)
  27 +
  28 +data class OnlineModelConfig(
  29 + var transducer: OnlineTransducerModelConfig = OnlineTransducerModelConfig(),
  30 + var paraformer: OnlineParaformerModelConfig = OnlineParaformerModelConfig(),
  31 + var tokens: String,
  32 + var numThreads: Int = 1,
  33 + var debug: Boolean = false,
  34 + var provider: String = "cpu",
  35 + var modelType: String = "",
  36 +)
  37 +
  38 +data class OnlineLMConfig(
  39 + var model: String = "",
  40 + var scale: Float = 0.5f,
  41 +)
  42 +
  43 +data class FeatureConfig(
  44 + var sampleRate: Int = 16000,
  45 + var featureDim: Int = 80,
  46 +)
  47 +
  48 +data class OnlineRecognizerConfig(
  49 + var featConfig: FeatureConfig = FeatureConfig(),
  50 + var modelConfig: OnlineModelConfig,
  51 + var lmConfig: OnlineLMConfig = OnlineLMConfig(),
  52 + var endpointConfig: EndpointConfig = EndpointConfig(),
  53 + var enableEndpoint: Boolean = true,
  54 + var decodingMethod: String = "greedy_search",
  55 + var maxActivePaths: Int = 4,
  56 +)
  57 +
  58 +data class OfflineTransducerModelConfig(
  59 + var encoder: String = "",
  60 + var decoder: String = "",
  61 + var joiner: String = "",
  62 +)
  63 +
  64 +data class OfflineParaformerModelConfig(
  65 + var model: String = "",
  66 +)
  67 +
  68 +data class OfflineWhisperModelConfig(
  69 + var encoder: String = "",
  70 + var decoder: String = "",
  71 +)
  72 +
  73 +data class OfflineModelConfig(
  74 + var transducer: OfflineTransducerModelConfig = OfflineTransducerModelConfig(),
  75 + var paraformer: OfflineParaformerModelConfig = OfflineParaformerModelConfig(),
  76 + var whisper: OfflineWhisperModelConfig = OfflineWhisperModelConfig(),
  77 + var numThreads: Int = 1,
  78 + var debug: Boolean = false,
  79 + var provider: String = "cpu",
  80 + var modelType: String = "",
  81 + var tokens: String,
  82 +)
  83 +
  84 +data class OfflineRecognizerConfig(
  85 + var featConfig: FeatureConfig = FeatureConfig(),
  86 + var modelConfig: OfflineModelConfig,
  87 + // var lmConfig: OfflineLMConfig(), // TODO(fangjun): enable it
  88 + var decodingMethod: String = "greedy_search",
  89 + var maxActivePaths: Int = 4,
  90 +)
  91 +
  92 +class SherpaOnnx(
  93 + assetManager: AssetManager? = null,
  94 + var config: OnlineRecognizerConfig,
  95 +) {
  96 + private val ptr: Long
  97 +
  98 + init {
  99 + if (assetManager != null) {
  100 + ptr = new(assetManager, config)
  101 + } else {
  102 + ptr = newFromFile(config)
  103 + }
  104 + }
  105 +
  106 + protected fun finalize() {
  107 + delete(ptr)
  108 + }
  109 +
  110 + fun acceptWaveform(samples: FloatArray, sampleRate: Int) =
  111 + acceptWaveform(ptr, samples, sampleRate)
  112 +
  113 + fun inputFinished() = inputFinished(ptr)
  114 + fun reset(recreate: Boolean = false) = reset(ptr, recreate = recreate)
  115 + fun decode() = decode(ptr)
  116 + fun isEndpoint(): Boolean = isEndpoint(ptr)
  117 + fun isReady(): Boolean = isReady(ptr)
  118 +
  119 + val text: String
  120 + get() = getText(ptr)
  121 +
  122 + private external fun delete(ptr: Long)
  123 +
  124 + private external fun new(
  125 + assetManager: AssetManager,
  126 + config: OnlineRecognizerConfig,
  127 + ): Long
  128 +
  129 + private external fun newFromFile(
  130 + config: OnlineRecognizerConfig,
  131 + ): Long
  132 +
  133 + private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int)
  134 + private external fun inputFinished(ptr: Long)
  135 + private external fun getText(ptr: Long): String
  136 + private external fun reset(ptr: Long, recreate: Boolean)
  137 + private external fun decode(ptr: Long)
  138 + private external fun isEndpoint(ptr: Long): Boolean
  139 + private external fun isReady(ptr: Long): Boolean
  140 +
  141 + companion object {
  142 + init {
  143 + System.loadLibrary("sherpa-onnx-jni")
  144 + }
  145 + }
  146 +}
  147 +
  148 +class SherpaOnnxOffline(
  149 + assetManager: AssetManager? = null,
  150 + var config: OfflineRecognizerConfig,
  151 +) {
  152 + private val ptr: Long
  153 +
  154 + init {
  155 + if (assetManager != null) {
  156 + ptr = new(assetManager, config)
  157 + } else {
  158 + ptr = newFromFile(config)
  159 + }
  160 + }
  161 +
  162 + protected fun finalize() {
  163 + delete(ptr)
  164 + }
  165 +
  166 + fun decode(samples: FloatArray, sampleRate: Int) = decode(ptr, samples, sampleRate)
  167 +
  168 + private external fun delete(ptr: Long)
  169 +
  170 + private external fun new(
  171 + assetManager: AssetManager,
  172 + config: OfflineRecognizerConfig,
  173 + ): Long
  174 +
  175 + private external fun newFromFile(
  176 + config: OfflineRecognizerConfig,
  177 + ): Long
  178 +
  179 + private external fun decode(ptr: Long, samples: FloatArray, sampleRate: Int): String
  180 +
  181 + companion object {
  182 + init {
  183 + System.loadLibrary("sherpa-onnx-jni")
  184 + }
  185 + }
  186 +}
  187 +
  188 +fun getFeatureConfig(sampleRate: Int, featureDim: Int): FeatureConfig {
  189 + return FeatureConfig(sampleRate = sampleRate, featureDim = featureDim)
  190 +}
  191 +
  192 +/*
  193 +Please see
  194 +https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html
  195 +for a list of pre-trained models.
  196 +
  197 +We only add a few here. Please change the following code
  198 +to add your own. (It should be straightforward to add a new model
  199 +by following the code)
  200 +
  201 +@param type
  202 +0 - csukuangfj/sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23 (Chinese)
  203 + https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-zh-14m-2023-02-23
  204 + encoder/joiner int8, decoder float32
  205 +
  206 +1 - csukuangfj/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17 (English)
  207 + https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#csukuangfj-sherpa-onnx-streaming-zipformer-en-20m-2023-02-17-english
  208 + encoder/joiner int8, decoder fp32
  209 +
  210 + */
  211 +fun getModelConfig(type: Int): OnlineModelConfig? {
  212 + when (type) {
  213 + 0 -> {
  214 + val modelDir = "sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23"
  215 + return OnlineModelConfig(
  216 + transducer = OnlineTransducerModelConfig(
  217 + encoder = "$modelDir/encoder-epoch-99-avg-1.int8.onnx",
  218 + decoder = "$modelDir/decoder-epoch-99-avg-1.onnx",
  219 + joiner = "$modelDir/joiner-epoch-99-avg-1.int8.onnx",
  220 + ),
  221 + tokens = "$modelDir/tokens.txt",
  222 + modelType = "zipformer",
  223 + )
  224 + }
  225 +
  226 + 1 -> {
  227 + val modelDir = "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17"
  228 + return OnlineModelConfig(
  229 + transducer = OnlineTransducerModelConfig(
  230 + encoder = "$modelDir/encoder-epoch-99-avg-1.int8.onnx",
  231 + decoder = "$modelDir/decoder-epoch-99-avg-1.onnx",
  232 + joiner = "$modelDir/joiner-epoch-99-avg-1.int8.onnx",
  233 + ),
  234 + tokens = "$modelDir/tokens.txt",
  235 + modelType = "zipformer",
  236 + )
  237 + }
  238 + }
  239 + return null
  240 +}
  241 +
  242 +/*
  243 +Please see
  244 +https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html
  245 +for a list of pre-trained models.
  246 +
  247 +We only add a few here. Please change the following code
  248 +to add your own LM model. (It should be straightforward to train a new NN LM model
  249 +by following the code, https://github.com/k2-fsa/icefall/blob/master/icefall/rnn_lm/train.py)
  250 +
  251 +@param type
  252 +0 - sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 (Bilingual, Chinese + English)
  253 + https://k2-fsa.github.io/sherpa/onnx/pretrained_models/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-bilingual-chinese-english
  254 + */
  255 +fun getOnlineLMConfig(type: Int): OnlineLMConfig {
  256 + when (type) {
  257 + 0 -> {
  258 + val modelDir = "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"
  259 + return OnlineLMConfig(
  260 + model = "$modelDir/with-state-epoch-99-avg-1.int8.onnx",
  261 + scale = 0.5f,
  262 + )
  263 + }
  264 + }
  265 + return OnlineLMConfig()
  266 +}
  267 +
  268 +// for English models, use a small value for rule2.minTrailingSilence, e.g., 0.8
  269 +fun getEndpointConfig(): EndpointConfig {
  270 + return EndpointConfig(
  271 + rule1 = EndpointRule(false, 2.4f, 0.0f),
  272 + rule2 = EndpointRule(true, 0.8f, 0.0f),
  273 + rule3 = EndpointRule(false, 0.0f, 20.0f)
  274 + )
  275 +}
  276 +
  277 +/*
  278 +Please see
  279 +https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html
  280 +for a list of pre-trained models.
  281 +
  282 +We only add a few here. Please change the following code
  283 +to add your own. (It should be straightforward to add a new model
  284 +by following the code)
  285 +
  286 +@param type
  287 +
  288 +0 - csukuangfj/sherpa-onnx-paraformer-zh-2023-03-28 (Chinese)
  289 + https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-paraformer/paraformer-models.html#csukuangfj-sherpa-onnx-paraformer-zh-2023-03-28-chinese
  290 + int8
  291 +
  292 +1 - icefall-asr-multidataset-pruned_transducer_stateless7-2023-05-04 (English)
  293 + https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#icefall-asr-multidataset-pruned-transducer-stateless7-2023-05-04-english
  294 + encoder int8, decoder/joiner float32
  295 +
  296 +2 - sherpa-onnx-whisper-tiny.en
  297 + https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/tiny.en.html#tiny-en
  298 + encoder int8, decoder int8
  299 +
  300 +3 - sherpa-onnx-whisper-base.en
  301 + https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/tiny.en.html#tiny-en
  302 + encoder int8, decoder int8
  303 +
  304 +4 - pkufool/icefall-asr-zipformer-wenetspeech-20230615 (Chinese)
  305 + https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#pkufool-icefall-asr-zipformer-wenetspeech-20230615-chinese
  306 + encoder/joiner int8, decoder fp32
  307 +
  308 + */
  309 +fun getOfflineModelConfig(type: Int): OfflineModelConfig? {
  310 + when (type) {
  311 + 0 -> {
  312 + val modelDir = "sherpa-onnx-paraformer-zh-2023-03-28"
  313 + return OfflineModelConfig(
  314 + paraformer = OfflineParaformerModelConfig(
  315 + model = "$modelDir/model.int8.onnx",
  316 + ),
  317 + tokens = "$modelDir/tokens.txt",
  318 + modelType = "paraformer",
  319 + )
  320 + }
  321 +
  322 + 1 -> {
  323 + val modelDir = "icefall-asr-multidataset-pruned_transducer_stateless7-2023-05-04"
  324 + return OfflineModelConfig(
  325 + transducer = OfflineTransducerModelConfig(
  326 + encoder = "$modelDir/encoder-epoch-30-avg-4.int8.onnx",
  327 + decoder = "$modelDir/decoder-epoch-30-avg-4.onnx",
  328 + joiner = "$modelDir/joiner-epoch-30-avg-4.onnx",
  329 + ),
  330 + tokens = "$modelDir/tokens.txt",
  331 + modelType = "zipformer",
  332 + )
  333 + }
  334 +
  335 + 2 -> {
  336 + val modelDir = "sherpa-onnx-whisper-tiny.en"
  337 + return OfflineModelConfig(
  338 + whisper = OfflineWhisperModelConfig(
  339 + encoder = "$modelDir/tiny.en-encoder.int8.onnx",
  340 + decoder = "$modelDir/tiny.en-decoder.int8.onnx",
  341 + ),
  342 + tokens = "$modelDir/tiny.en-tokens.txt",
  343 + modelType = "whisper",
  344 + )
  345 + }
  346 +
  347 + 3 -> {
  348 + val modelDir = "sherpa-onnx-whisper-base.en"
  349 + return OfflineModelConfig(
  350 + whisper = OfflineWhisperModelConfig(
  351 + encoder = "$modelDir/base.en-encoder.int8.onnx",
  352 + decoder = "$modelDir/base.en-decoder.int8.onnx",
  353 + ),
  354 + tokens = "$modelDir/base.en-tokens.txt",
  355 + modelType = "whisper",
  356 + )
  357 + }
  358 +
  359 +
  360 + 4 -> {
  361 + val modelDir = "icefall-asr-zipformer-wenetspeech-20230615"
  362 + return OfflineModelConfig(
  363 + transducer = OfflineTransducerModelConfig(
  364 + encoder = "$modelDir/encoder-epoch-12-avg-4.int8.onnx",
  365 + decoder = "$modelDir/decoder-epoch-12-avg-4.onnx",
  366 + joiner = "$modelDir/joiner-epoch-12-avg-4.int8.onnx",
  367 + ),
  368 + tokens = "$modelDir/tokens.txt",
  369 + modelType = "zipformer",
  370 + )
  371 + }
  372 +
  373 + }
  374 + return null
  375 +}
  1 +../../../../../../../../../SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/WaveReader.kt
  1 +<vector xmlns:android="http://schemas.android.com/apk/res/android"
  2 + xmlns:aapt="http://schemas.android.com/aapt"
  3 + android:width="108dp"
  4 + android:height="108dp"
  5 + android:viewportWidth="108"
  6 + android:viewportHeight="108">
  7 + <path android:pathData="M31,63.928c0,0 6.4,-11 12.1,-13.1c7.2,-2.6 26,-1.4 26,-1.4l38.1,38.1L107,108.928l-32,-1L31,63.928z">
  8 + <aapt:attr name="android:fillColor">
  9 + <gradient
  10 + android:endX="85.84757"
  11 + android:endY="92.4963"
  12 + android:startX="42.9492"
  13 + android:startY="49.59793"
  14 + android:type="linear">
  15 + <item
  16 + android:color="#44000000"
  17 + android:offset="0.0" />
  18 + <item
  19 + android:color="#00000000"
  20 + android:offset="1.0" />
  21 + </gradient>
  22 + </aapt:attr>
  23 + </path>
  24 + <path
  25 + android:fillColor="#FFFFFF"
  26 + android:fillType="nonZero"
  27 + android:pathData="M65.3,45.828l3.8,-6.6c0.2,-0.4 0.1,-0.9 -0.3,-1.1c-0.4,-0.2 -0.9,-0.1 -1.1,0.3l-3.9,6.7c-6.3,-2.8 -13.4,-2.8 -19.7,0l-3.9,-6.7c-0.2,-0.4 -0.7,-0.5 -1.1,-0.3C38.8,38.328 38.7,38.828 38.9,39.228l3.8,6.6C36.2,49.428 31.7,56.028 31,63.928h46C76.3,56.028 71.8,49.428 65.3,45.828zM43.4,57.328c-0.8,0 -1.5,-0.5 -1.8,-1.2c-0.3,-0.7 -0.1,-1.5 0.4,-2.1c0.5,-0.5 1.4,-0.7 2.1,-0.4c0.7,0.3 1.2,1 1.2,1.8C45.3,56.528 44.5,57.328 43.4,57.328L43.4,57.328zM64.6,57.328c-0.8,0 -1.5,-0.5 -1.8,-1.2s-0.1,-1.5 0.4,-2.1c0.5,-0.5 1.4,-0.7 2.1,-0.4c0.7,0.3 1.2,1 1.2,1.8C66.5,56.528 65.6,57.328 64.6,57.328L64.6,57.328z"
  28 + android:strokeWidth="1"
  29 + android:strokeColor="#00000000" />
  30 +</vector>
  1 +<?xml version="1.0" encoding="utf-8"?>
  2 +<vector xmlns:android="http://schemas.android.com/apk/res/android"
  3 + android:width="108dp"
  4 + android:height="108dp"
  5 + android:viewportWidth="108"
  6 + android:viewportHeight="108">
  7 + <path
  8 + android:fillColor="#3DDC84"
  9 + android:pathData="M0,0h108v108h-108z" />
  10 + <path
  11 + android:fillColor="#00000000"
  12 + android:pathData="M9,0L9,108"
  13 + android:strokeWidth="0.8"
  14 + android:strokeColor="#33FFFFFF" />
  15 + <path
  16 + android:fillColor="#00000000"
  17 + android:pathData="M19,0L19,108"
  18 + android:strokeWidth="0.8"
  19 + android:strokeColor="#33FFFFFF" />
  20 + <path
  21 + android:fillColor="#00000000"
  22 + android:pathData="M29,0L29,108"
  23 + android:strokeWidth="0.8"
  24 + android:strokeColor="#33FFFFFF" />
  25 + <path
  26 + android:fillColor="#00000000"
  27 + android:pathData="M39,0L39,108"
  28 + android:strokeWidth="0.8"
  29 + android:strokeColor="#33FFFFFF" />
  30 + <path
  31 + android:fillColor="#00000000"
  32 + android:pathData="M49,0L49,108"
  33 + android:strokeWidth="0.8"
  34 + android:strokeColor="#33FFFFFF" />
  35 + <path
  36 + android:fillColor="#00000000"
  37 + android:pathData="M59,0L59,108"
  38 + android:strokeWidth="0.8"
  39 + android:strokeColor="#33FFFFFF" />
  40 + <path
  41 + android:fillColor="#00000000"
  42 + android:pathData="M69,0L69,108"
  43 + android:strokeWidth="0.8"
  44 + android:strokeColor="#33FFFFFF" />
  45 + <path
  46 + android:fillColor="#00000000"
  47 + android:pathData="M79,0L79,108"
  48 + android:strokeWidth="0.8"
  49 + android:strokeColor="#33FFFFFF" />
  50 + <path
  51 + android:fillColor="#00000000"
  52 + android:pathData="M89,0L89,108"
  53 + android:strokeWidth="0.8"
  54 + android:strokeColor="#33FFFFFF" />
  55 + <path
  56 + android:fillColor="#00000000"
  57 + android:pathData="M99,0L99,108"
  58 + android:strokeWidth="0.8"
  59 + android:strokeColor="#33FFFFFF" />
  60 + <path
  61 + android:fillColor="#00000000"
  62 + android:pathData="M0,9L108,9"
  63 + android:strokeWidth="0.8"
  64 + android:strokeColor="#33FFFFFF" />
  65 + <path
  66 + android:fillColor="#00000000"
  67 + android:pathData="M0,19L108,19"
  68 + android:strokeWidth="0.8"
  69 + android:strokeColor="#33FFFFFF" />
  70 + <path
  71 + android:fillColor="#00000000"
  72 + android:pathData="M0,29L108,29"
  73 + android:strokeWidth="0.8"
  74 + android:strokeColor="#33FFFFFF" />
  75 + <path
  76 + android:fillColor="#00000000"
  77 + android:pathData="M0,39L108,39"
  78 + android:strokeWidth="0.8"
  79 + android:strokeColor="#33FFFFFF" />
  80 + <path
  81 + android:fillColor="#00000000"
  82 + android:pathData="M0,49L108,49"
  83 + android:strokeWidth="0.8"
  84 + android:strokeColor="#33FFFFFF" />
  85 + <path
  86 + android:fillColor="#00000000"
  87 + android:pathData="M0,59L108,59"
  88 + android:strokeWidth="0.8"
  89 + android:strokeColor="#33FFFFFF" />
  90 + <path
  91 + android:fillColor="#00000000"
  92 + android:pathData="M0,69L108,69"
  93 + android:strokeWidth="0.8"
  94 + android:strokeColor="#33FFFFFF" />
  95 + <path
  96 + android:fillColor="#00000000"
  97 + android:pathData="M0,79L108,79"
  98 + android:strokeWidth="0.8"
  99 + android:strokeColor="#33FFFFFF" />
  100 + <path
  101 + android:fillColor="#00000000"
  102 + android:pathData="M0,89L108,89"
  103 + android:strokeWidth="0.8"
  104 + android:strokeColor="#33FFFFFF" />
  105 + <path
  106 + android:fillColor="#00000000"
  107 + android:pathData="M0,99L108,99"
  108 + android:strokeWidth="0.8"
  109 + android:strokeColor="#33FFFFFF" />
  110 + <path
  111 + android:fillColor="#00000000"
  112 + android:pathData="M19,29L89,29"
  113 + android:strokeWidth="0.8"
  114 + android:strokeColor="#33FFFFFF" />
  115 + <path
  116 + android:fillColor="#00000000"
  117 + android:pathData="M19,39L89,39"
  118 + android:strokeWidth="0.8"
  119 + android:strokeColor="#33FFFFFF" />
  120 + <path
  121 + android:fillColor="#00000000"
  122 + android:pathData="M19,49L89,49"
  123 + android:strokeWidth="0.8"
  124 + android:strokeColor="#33FFFFFF" />
  125 + <path
  126 + android:fillColor="#00000000"
  127 + android:pathData="M19,59L89,59"
  128 + android:strokeWidth="0.8"
  129 + android:strokeColor="#33FFFFFF" />
  130 + <path
  131 + android:fillColor="#00000000"
  132 + android:pathData="M19,69L89,69"
  133 + android:strokeWidth="0.8"
  134 + android:strokeColor="#33FFFFFF" />
  135 + <path
  136 + android:fillColor="#00000000"
  137 + android:pathData="M19,79L89,79"
  138 + android:strokeWidth="0.8"
  139 + android:strokeColor="#33FFFFFF" />
  140 + <path
  141 + android:fillColor="#00000000"
  142 + android:pathData="M29,19L29,89"
  143 + android:strokeWidth="0.8"
  144 + android:strokeColor="#33FFFFFF" />
  145 + <path
  146 + android:fillColor="#00000000"
  147 + android:pathData="M39,19L39,89"
  148 + android:strokeWidth="0.8"
  149 + android:strokeColor="#33FFFFFF" />
  150 + <path
  151 + android:fillColor="#00000000"
  152 + android:pathData="M49,19L49,89"
  153 + android:strokeWidth="0.8"
  154 + android:strokeColor="#33FFFFFF" />
  155 + <path
  156 + android:fillColor="#00000000"
  157 + android:pathData="M59,19L59,89"
  158 + android:strokeWidth="0.8"
  159 + android:strokeColor="#33FFFFFF" />
  160 + <path
  161 + android:fillColor="#00000000"
  162 + android:pathData="M69,19L69,89"
  163 + android:strokeWidth="0.8"
  164 + android:strokeColor="#33FFFFFF" />
  165 + <path
  166 + android:fillColor="#00000000"
  167 + android:pathData="M79,19L79,89"
  168 + android:strokeWidth="0.8"
  169 + android:strokeColor="#33FFFFFF" />
  170 +</vector>
  1 +<?xml version="1.0" encoding="utf-8"?>
  2 +<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
  3 + xmlns:app="http://schemas.android.com/apk/res-auto"
  4 + xmlns:tools="http://schemas.android.com/tools"
  5 + android:layout_width="match_parent"
  6 + android:layout_height="match_parent"
  7 + tools:context=".MainActivity">
  8 +
  9 + <LinearLayout
  10 + android:layout_width="match_parent"
  11 + android:layout_height="match_parent"
  12 + android:gravity="center"
  13 + android:orientation="vertical">
  14 +
  15 + <TextView
  16 + android:id="@+id/my_text"
  17 + android:layout_width="match_parent"
  18 + android:layout_height="match_parent"
  19 + android:layout_weight="2.5"
  20 + android:padding="24dp"
  21 + android:scrollbars="vertical"
  22 + android:singleLine="false"
  23 + android:text="@string/hint"
  24 + app:layout_constraintBottom_toBottomOf="parent"
  25 + app:layout_constraintEnd_toEndOf="parent"
  26 + app:layout_constraintStart_toStartOf="parent"
  27 + android:gravity="bottom"
  28 + app:layout_constraintTop_toTopOf="parent" />
  29 +
  30 + <Button
  31 + android:id="@+id/record_button"
  32 + android:layout_width="wrap_content"
  33 + android:layout_height="wrap_content"
  34 + android:layout_weight="0.5"
  35 + android:text="@string/start" />
  36 + </LinearLayout>
  37 +
  38 +
  39 +</androidx.constraintlayout.widget.ConstraintLayout>
  1 +<?xml version="1.0" encoding="utf-8"?>
  2 +<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
  3 + <background android:drawable="@drawable/ic_launcher_background" />
  4 + <foreground android:drawable="@drawable/ic_launcher_foreground" />
  5 +</adaptive-icon>
  1 +<?xml version="1.0" encoding="utf-8"?>
  2 +<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
  3 + <background android:drawable="@drawable/ic_launcher_background" />
  4 + <foreground android:drawable="@drawable/ic_launcher_foreground" />
  5 +</adaptive-icon>
  1 +<resources xmlns:tools="http://schemas.android.com/tools">
  2 + <!-- Base application theme. -->
  3 + <style name="Theme.SherpaOnnx2Pass" parent="Theme.MaterialComponents.DayNight.DarkActionBar">
  4 + <!-- Primary brand color. -->
  5 + <item name="colorPrimary">@color/purple_200</item>
  6 + <item name="colorPrimaryVariant">@color/purple_700</item>
  7 + <item name="colorOnPrimary">@color/black</item>
  8 + <!-- Secondary brand color. -->
  9 + <item name="colorSecondary">@color/teal_200</item>
  10 + <item name="colorSecondaryVariant">@color/teal_200</item>
  11 + <item name="colorOnSecondary">@color/black</item>
  12 + <!-- Status bar color. -->
  13 + <item name="android:statusBarColor">?attr/colorPrimaryVariant</item>
  14 + <!-- Customize your theme here. -->
  15 + </style>
  16 +</resources>
  1 +<?xml version="1.0" encoding="utf-8"?>
  2 +<resources>
  3 + <color name="purple_200">#FFBB86FC</color>
  4 + <color name="purple_500">#FF6200EE</color>
  5 + <color name="purple_700">#FF3700B3</color>
  6 + <color name="teal_200">#FF03DAC5</color>
  7 + <color name="teal_700">#FF018786</color>
  8 + <color name="black">#FF000000</color>
  9 + <color name="white">#FFFFFFFF</color>
  10 +</resources>
  1 +<resources>
  2 + <string name="app_name">ASR with Next-gen Kaldi</string>
  3 + <string name="hint">Click the Start button to play speech-to-text with Next-gen Kaldi.
  4 + \n
  5 + \n\n\n
  6 + The source code and pre-trained models are publicly available.
  7 + Please see https://github.com/k2-fsa/sherpa-onnx for details.
  8 + \n\n
  9 + Two-pass speech recognition with Next-gen Kaldi.
  10 + </string>
  11 + <string name="start">Start</string>
  12 + <string name="stop">Stop</string>
  13 +</resources>
  1 +<resources xmlns:tools="http://schemas.android.com/tools">
  2 + <!-- Base application theme. -->
  3 + <style name="Theme.SherpaOnnx2Pass" parent="Theme.MaterialComponents.DayNight.DarkActionBar">
  4 + <!-- Primary brand color. -->
  5 + <item name="colorPrimary">@color/purple_500</item>
  6 + <item name="colorPrimaryVariant">@color/purple_700</item>
  7 + <item name="colorOnPrimary">@color/white</item>
  8 + <!-- Secondary brand color. -->
  9 + <item name="colorSecondary">@color/teal_200</item>
  10 + <item name="colorSecondaryVariant">@color/teal_700</item>
  11 + <item name="colorOnSecondary">@color/black</item>
  12 + <!-- Status bar color. -->
  13 + <item name="android:statusBarColor">?attr/colorPrimaryVariant</item>
  14 + <!-- Customize your theme here. -->
  15 + </style>
  16 +</resources>
  1 +<?xml version="1.0" encoding="utf-8"?><!--
  2 + Sample backup rules file; uncomment and customize as necessary.
  3 + See https://developer.android.com/guide/topics/data/autobackup
  4 + for details.
  5 + Note: This file is ignored for devices older that API 31
  6 + See https://developer.android.com/about/versions/12/backup-restore
  7 +-->
  8 +<full-backup-content>
  9 + <!--
  10 + <include domain="sharedpref" path="."/>
  11 + <exclude domain="sharedpref" path="device.xml"/>
  12 +-->
  13 +</full-backup-content>
  1 +<?xml version="1.0" encoding="utf-8"?><!--
  2 + Sample data extraction rules file; uncomment and customize as necessary.
  3 + See https://developer.android.com/about/versions/12/backup-restore#xml-changes
  4 + for details.
  5 +-->
  6 +<data-extraction-rules>
  7 + <cloud-backup>
  8 + <!-- TODO: Use <include> and <exclude> to control what is backed up.
  9 + <include .../>
  10 + <exclude .../>
  11 + -->
  12 + </cloud-backup>
  13 + <!--
  14 + <device-transfer>
  15 + <include .../>
  16 + <exclude .../>
  17 + </device-transfer>
  18 + -->
  19 +</data-extraction-rules>
  1 +package com.k2fsa.sherpa.onnx
  2 +
  3 +import org.junit.Test
  4 +
  5 +import org.junit.Assert.*
  6 +
  7 +/**
  8 + * Example local unit test, which will execute on the development machine (host).
  9 + *
  10 + * See [testing documentation](http://d.android.com/tools/testing).
  11 + */
  12 +class ExampleUnitTest {
  13 + @Test
  14 + fun addition_isCorrect() {
  15 + assertEquals(4, 2 + 2)
  16 + }
  17 +}
  1 +// Top-level build file where you can add configuration options common to all sub-projects/modules.
  2 +plugins {
  3 + id 'com.android.application' version '7.3.1' apply false
  4 + id 'com.android.library' version '7.3.1' apply false
  5 + id 'org.jetbrains.kotlin.android' version '1.7.20' apply false
  6 +}
  1 +# Project-wide Gradle settings.
  2 +# IDE (e.g. Android Studio) users:
  3 +# Gradle settings configured through the IDE *will override*
  4 +# any settings specified in this file.
  5 +# For more details on how to configure your build environment visit
  6 +# http://www.gradle.org/docs/current/userguide/build_environment.html
  7 +# Specifies the JVM arguments used for the daemon process.
  8 +# The setting is particularly useful for tweaking memory settings.
  9 +org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8
  10 +# When configured, Gradle will run in incubating parallel mode.
  11 +# This option should only be used with decoupled projects. More details, visit
  12 +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects
  13 +# org.gradle.parallel=true
  14 +# AndroidX package structure to make it clearer which packages are bundled with the
  15 +# Android operating system, and which are packaged with your app's APK
  16 +# https://developer.android.com/topic/libraries/support-library/androidx-rn
  17 +android.useAndroidX=true
  18 +# Kotlin code style for this project: "official" or "obsolete":
  19 +kotlin.code.style=official
  20 +# Enables namespacing of each library's R class so that its R class includes only the
  21 +# resources declared in the library itself and none from the library's dependencies,
  22 +# thereby reducing the size of the R class for that library
  23 +android.nonTransitiveRClass=true
  1 +#Sun Sep 10 18:03:03 CST 2023
  2 +distributionBase=GRADLE_USER_HOME
  3 +distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip
  4 +distributionPath=wrapper/dists
  5 +zipStorePath=wrapper/dists
  6 +zipStoreBase=GRADLE_USER_HOME
  1 +#!/usr/bin/env sh
  2 +
  3 +#
  4 +# Copyright 2015 the original author or authors.
  5 +#
  6 +# Licensed under the Apache License, Version 2.0 (the "License");
  7 +# you may not use this file except in compliance with the License.
  8 +# You may obtain a copy of the License at
  9 +#
  10 +# https://www.apache.org/licenses/LICENSE-2.0
  11 +#
  12 +# Unless required by applicable law or agreed to in writing, software
  13 +# distributed under the License is distributed on an "AS IS" BASIS,
  14 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 +# See the License for the specific language governing permissions and
  16 +# limitations under the License.
  17 +#
  18 +
  19 +##############################################################################
  20 +##
  21 +## Gradle start up script for UN*X
  22 +##
  23 +##############################################################################
  24 +
  25 +# Attempt to set APP_HOME
  26 +# Resolve links: $0 may be a link
  27 +PRG="$0"
  28 +# Need this for relative symlinks.
  29 +while [ -h "$PRG" ] ; do
  30 + ls=`ls -ld "$PRG"`
  31 + link=`expr "$ls" : '.*-> \(.*\)$'`
  32 + if expr "$link" : '/.*' > /dev/null; then
  33 + PRG="$link"
  34 + else
  35 + PRG=`dirname "$PRG"`"/$link"
  36 + fi
  37 +done
  38 +SAVED="`pwd`"
  39 +cd "`dirname \"$PRG\"`/" >/dev/null
  40 +APP_HOME="`pwd -P`"
  41 +cd "$SAVED" >/dev/null
  42 +
  43 +APP_NAME="Gradle"
  44 +APP_BASE_NAME=`basename "$0"`
  45 +
  46 +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
  47 +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
  48 +
  49 +# Use the maximum available, or set MAX_FD != -1 to use that value.
  50 +MAX_FD="maximum"
  51 +
  52 +warn () {
  53 + echo "$*"
  54 +}
  55 +
  56 +die () {
  57 + echo
  58 + echo "$*"
  59 + echo
  60 + exit 1
  61 +}
  62 +
  63 +# OS specific support (must be 'true' or 'false').
  64 +cygwin=false
  65 +msys=false
  66 +darwin=false
  67 +nonstop=false
  68 +case "`uname`" in
  69 + CYGWIN* )
  70 + cygwin=true
  71 + ;;
  72 + Darwin* )
  73 + darwin=true
  74 + ;;
  75 + MINGW* )
  76 + msys=true
  77 + ;;
  78 + NONSTOP* )
  79 + nonstop=true
  80 + ;;
  81 +esac
  82 +
  83 +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
  84 +
  85 +
  86 +# Determine the Java command to use to start the JVM.
  87 +if [ -n "$JAVA_HOME" ] ; then
  88 + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
  89 + # IBM's JDK on AIX uses strange locations for the executables
  90 + JAVACMD="$JAVA_HOME/jre/sh/java"
  91 + else
  92 + JAVACMD="$JAVA_HOME/bin/java"
  93 + fi
  94 + if [ ! -x "$JAVACMD" ] ; then
  95 + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
  96 +
  97 +Please set the JAVA_HOME variable in your environment to match the
  98 +location of your Java installation."
  99 + fi
  100 +else
  101 + JAVACMD="java"
  102 + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
  103 +
  104 +Please set the JAVA_HOME variable in your environment to match the
  105 +location of your Java installation."
  106 +fi
  107 +
  108 +# Increase the maximum file descriptors if we can.
  109 +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
  110 + MAX_FD_LIMIT=`ulimit -H -n`
  111 + if [ $? -eq 0 ] ; then
  112 + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
  113 + MAX_FD="$MAX_FD_LIMIT"
  114 + fi
  115 + ulimit -n $MAX_FD
  116 + if [ $? -ne 0 ] ; then
  117 + warn "Could not set maximum file descriptor limit: $MAX_FD"
  118 + fi
  119 + else
  120 + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
  121 + fi
  122 +fi
  123 +
  124 +# For Darwin, add options to specify how the application appears in the dock
  125 +if $darwin; then
  126 + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
  127 +fi
  128 +
  129 +# For Cygwin or MSYS, switch paths to Windows format before running java
  130 +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
  131 + APP_HOME=`cygpath --path --mixed "$APP_HOME"`
  132 + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
  133 +
  134 + JAVACMD=`cygpath --unix "$JAVACMD"`
  135 +
  136 + # We build the pattern for arguments to be converted via cygpath
  137 + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
  138 + SEP=""
  139 + for dir in $ROOTDIRSRAW ; do
  140 + ROOTDIRS="$ROOTDIRS$SEP$dir"
  141 + SEP="|"
  142 + done
  143 + OURCYGPATTERN="(^($ROOTDIRS))"
  144 + # Add a user-defined pattern to the cygpath arguments
  145 + if [ "$GRADLE_CYGPATTERN" != "" ] ; then
  146 + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
  147 + fi
  148 + # Now convert the arguments - kludge to limit ourselves to /bin/sh
  149 + i=0
  150 + for arg in "$@" ; do
  151 + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
  152 + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
  153 +
  154 + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
  155 + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
  156 + else
  157 + eval `echo args$i`="\"$arg\""
  158 + fi
  159 + i=`expr $i + 1`
  160 + done
  161 + case $i in
  162 + 0) set -- ;;
  163 + 1) set -- "$args0" ;;
  164 + 2) set -- "$args0" "$args1" ;;
  165 + 3) set -- "$args0" "$args1" "$args2" ;;
  166 + 4) set -- "$args0" "$args1" "$args2" "$args3" ;;
  167 + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
  168 + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
  169 + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
  170 + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
  171 + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
  172 + esac
  173 +fi
  174 +
  175 +# Escape application args
  176 +save () {
  177 + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
  178 + echo " "
  179 +}
  180 +APP_ARGS=`save "$@"`
  181 +
  182 +# Collect all arguments for the java command, following the shell quoting and substitution rules
  183 +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
  184 +
  185 +exec "$JAVACMD" "$@"
  1 +@rem
  2 +@rem Copyright 2015 the original author or authors.
  3 +@rem
  4 +@rem Licensed under the Apache License, Version 2.0 (the "License");
  5 +@rem you may not use this file except in compliance with the License.
  6 +@rem You may obtain a copy of the License at
  7 +@rem
  8 +@rem https://www.apache.org/licenses/LICENSE-2.0
  9 +@rem
  10 +@rem Unless required by applicable law or agreed to in writing, software
  11 +@rem distributed under the License is distributed on an "AS IS" BASIS,
  12 +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 +@rem See the License for the specific language governing permissions and
  14 +@rem limitations under the License.
  15 +@rem
  16 +
  17 +@if "%DEBUG%" == "" @echo off
  18 +@rem ##########################################################################
  19 +@rem
  20 +@rem Gradle startup script for Windows
  21 +@rem
  22 +@rem ##########################################################################
  23 +
  24 +@rem Set local scope for the variables with windows NT shell
  25 +if "%OS%"=="Windows_NT" setlocal
  26 +
  27 +set DIRNAME=%~dp0
  28 +if "%DIRNAME%" == "" set DIRNAME=.
  29 +set APP_BASE_NAME=%~n0
  30 +set APP_HOME=%DIRNAME%
  31 +
  32 +@rem Resolve any "." and ".." in APP_HOME to make it shorter.
  33 +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
  34 +
  35 +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
  36 +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
  37 +
  38 +@rem Find java.exe
  39 +if defined JAVA_HOME goto findJavaFromJavaHome
  40 +
  41 +set JAVA_EXE=java.exe
  42 +%JAVA_EXE% -version >NUL 2>&1
  43 +if "%ERRORLEVEL%" == "0" goto execute
  44 +
  45 +echo.
  46 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
  47 +echo.
  48 +echo Please set the JAVA_HOME variable in your environment to match the
  49 +echo location of your Java installation.
  50 +
  51 +goto fail
  52 +
  53 +:findJavaFromJavaHome
  54 +set JAVA_HOME=%JAVA_HOME:"=%
  55 +set JAVA_EXE=%JAVA_HOME%/bin/java.exe
  56 +
  57 +if exist "%JAVA_EXE%" goto execute
  58 +
  59 +echo.
  60 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
  61 +echo.
  62 +echo Please set the JAVA_HOME variable in your environment to match the
  63 +echo location of your Java installation.
  64 +
  65 +goto fail
  66 +
  67 +:execute
  68 +@rem Setup the command line
  69 +
  70 +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
  71 +
  72 +
  73 +@rem Execute Gradle
  74 +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
  75 +
  76 +:end
  77 +@rem End local scope for the variables with windows NT shell
  78 +if "%ERRORLEVEL%"=="0" goto mainEnd
  79 +
  80 +:fail
  81 +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
  82 +rem the _cmd.exe /c_ return code!
  83 +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
  84 +exit /b 1
  85 +
  86 +:mainEnd
  87 +if "%OS%"=="Windows_NT" endlocal
  88 +
  89 +:omega
  1 +pluginManagement {
  2 + repositories {
  3 + gradlePluginPortal()
  4 + google()
  5 + mavenCentral()
  6 + }
  7 +}
  8 +dependencyResolutionManagement {
  9 + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS)
  10 + repositories {
  11 + google()
  12 + mavenCentral()
  13 + }
  14 +}
  15 +rootProject.name = "SherpaOnnx2Pass"
  16 +include ':app'
  1 +# See https://github.com/github/gitignore/blob/main/Swift.gitignore
  2 +# Xcode
  3 +#
  4 +# gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore
  5 +
  6 +## User settings
  7 +xcuserdata/
  8 +
  9 +## compatibility with Xcode 8 and earlier (ignoring not required starting Xcode 9)
  10 +*.xcscmblueprint
  11 +*.xccheckout
  12 +
  13 +## compatibility with Xcode 3 and earlier (ignoring not required starting Xcode 4)
  14 +build/
  15 +DerivedData/
  16 +*.moved-aside
  17 +*.pbxuser
  18 +!default.pbxuser
  19 +*.mode1v3
  20 +!default.mode1v3
  21 +*.mode2v3
  22 +!default.mode2v3
  23 +*.perspectivev3
  24 +!default.perspectivev3
  25 +
  26 +## Obj-C/Swift specific
  27 +*.hmap
  28 +
  29 +## App packaging
  30 +*.ipa
  31 +*.dSYM.zip
  32 +*.dSYM
  33 +
  34 +## Playgrounds
  35 +timeline.xctimeline
  36 +playground.xcworkspace
  37 +
  38 +# Swift Package Manager
  39 +#
  40 +# Add this line if you want to avoid checking in source code from Swift Package Manager dependencies.
  41 +# Packages/
  42 +# Package.pins
  43 +# Package.resolved
  44 +# *.xcodeproj
  45 +#
  46 +# Xcode automatically generates this directory with a .xcworkspacedata file and xcuserdata
  47 +# hence it is not needed unless you have added a package configuration file to your project
  48 +# .swiftpm
  49 +
  50 +.build/
  51 +
  52 +# CocoaPods
  53 +#
  54 +# We recommend against adding the Pods directory to your .gitignore. However
  55 +# you should judge for yourself, the pros and cons are mentioned at:
  56 +# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
  57 +#
  58 +# Pods/
  59 +#
  60 +# Add this line if you want to avoid checking in source code from the Xcode workspace
  61 +# *.xcworkspace
  62 +
  63 +# Carthage
  64 +#
  65 +# Add this line if you want to avoid checking in source code from Carthage dependencies.
  66 +# Carthage/Checkouts
  67 +
  68 +Carthage/Build/
  69 +
  70 +# Accio dependency management
  71 +Dependencies/
  72 +.accio/
  73 +
  74 +# fastlane
  75 +#
  76 +# It is recommended to not store the screenshots in the git repo.
  77 +# Instead, use fastlane to re-generate the screenshots whenever they are needed.
  78 +# For more information about the recommended setup visit:
  79 +# https://docs.fastlane.tools/best-practices/source-control/#source-control
  80 +
  81 +fastlane/report.xml
  82 +fastlane/Preview.html
  83 +fastlane/screenshots/**/*.png
  84 +fastlane/test_output
  85 +
  86 +# Code Injection
  87 +#
  88 +# After new code Injection tools there's a generated folder /iOSInjectionProject
  89 +# https://github.com/johnno1962/injectionforxcode
  90 +
  91 +iOSInjectionProject/
  1 +# See https://github.com/github/gitignore/blob/main/Swift.gitignore
  2 +# Xcode
  3 +#
  4 +# gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore
  5 +
  6 +## User settings
  7 +xcuserdata/
  8 +
  9 +## compatibility with Xcode 8 and earlier (ignoring not required starting Xcode 9)
  10 +*.xcscmblueprint
  11 +*.xccheckout
  12 +
  13 +## compatibility with Xcode 3 and earlier (ignoring not required starting Xcode 4)
  14 +build/
  15 +DerivedData/
  16 +*.moved-aside
  17 +*.pbxuser
  18 +!default.pbxuser
  19 +*.mode1v3
  20 +!default.mode1v3
  21 +*.mode2v3
  22 +!default.mode2v3
  23 +*.perspectivev3
  24 +!default.perspectivev3
  25 +
  26 +## Obj-C/Swift specific
  27 +*.hmap
  28 +
  29 +## App packaging
  30 +*.ipa
  31 +*.dSYM.zip
  32 +*.dSYM
  33 +
  34 +## Playgrounds
  35 +timeline.xctimeline
  36 +playground.xcworkspace
  37 +
  38 +# Swift Package Manager
  39 +#
  40 +# Add this line if you want to avoid checking in source code from Swift Package Manager dependencies.
  41 +# Packages/
  42 +# Package.pins
  43 +# Package.resolved
  44 +# *.xcodeproj
  45 +#
  46 +# Xcode automatically generates this directory with a .xcworkspacedata file and xcuserdata
  47 +# hence it is not needed unless you have added a package configuration file to your project
  48 +# .swiftpm
  49 +
  50 +.build/
  51 +
  52 +# CocoaPods
  53 +#
  54 +# We recommend against adding the Pods directory to your .gitignore. However
  55 +# you should judge for yourself, the pros and cons are mentioned at:
  56 +# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
  57 +#
  58 +# Pods/
  59 +#
  60 +# Add this line if you want to avoid checking in source code from the Xcode workspace
  61 +# *.xcworkspace
  62 +
  63 +# Carthage
  64 +#
  65 +# Add this line if you want to avoid checking in source code from Carthage dependencies.
  66 +# Carthage/Checkouts
  67 +
  68 +Carthage/Build/
  69 +
  70 +# Accio dependency management
  71 +Dependencies/
  72 +.accio/
  73 +
  74 +# fastlane
  75 +#
  76 +# It is recommended to not store the screenshots in the git repo.
  77 +# Instead, use fastlane to re-generate the screenshots whenever they are needed.
  78 +# For more information about the recommended setup visit:
  79 +# https://docs.fastlane.tools/best-practices/source-control/#source-control
  80 +
  81 +fastlane/report.xml
  82 +fastlane/Preview.html
  83 +fastlane/screenshots/**/*.png
  84 +fastlane/test_output
  85 +
  86 +# Code Injection
  87 +#
  88 +# After new code Injection tools there's a generated folder /iOSInjectionProject
  89 +# https://github.com/johnno1962/injectionforxcode
  90 +
  91 +iOSInjectionProject/
  1 +// !$*UTF8*$!
  2 +{
  3 + archiveVersion = 1;
  4 + classes = {
  5 + };
  6 + objectVersion = 56;
  7 + objects = {
  8 +
  9 +/* Begin PBXBuildFile section */
  10 + C9A2587D2AAEFFF100E555CA /* SherpaOnnx2PassApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = C9A2587C2AAEFFF100E555CA /* SherpaOnnx2PassApp.swift */; };
  11 + C9A2587F2AAEFFF100E555CA /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = C9A2587E2AAEFFF100E555CA /* ContentView.swift */; };
  12 + C9A258812AAEFFF200E555CA /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = C9A258802AAEFFF200E555CA /* Assets.xcassets */; };
  13 + C9A258842AAEFFF200E555CA /* Preview Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = C9A258832AAEFFF200E555CA /* Preview Assets.xcassets */; };
  14 + C9A2588E2AAF039D00E555CA /* Model.swift in Sources */ = {isa = PBXBuildFile; fileRef = C9A2588A2AAF039D00E555CA /* Model.swift */; };
  15 + C9A258902AAF039D00E555CA /* SherpaOnnxViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = C9A2588C2AAF039D00E555CA /* SherpaOnnxViewModel.swift */; };
  16 + C9A258912AAF039D00E555CA /* Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = C9A2588D2AAF039D00E555CA /* Extension.swift */; };
  17 + C9A258932AAF057E00E555CA /* SherpaOnnx.swift in Sources */ = {isa = PBXBuildFile; fileRef = C9A258922AAF057E00E555CA /* SherpaOnnx.swift */; };
  18 + C9A258962AAF05D100E555CA /* sherpa-onnx.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = C9A258952AAF05D100E555CA /* sherpa-onnx.xcframework */; };
  19 + C9A258982AAF05E400E555CA /* onnxruntime.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = C9A258972AAF05E400E555CA /* onnxruntime.xcframework */; };
  20 +/* End PBXBuildFile section */
  21 +
  22 +/* Begin PBXFileReference section */
  23 + C9A258792AAEFFF100E555CA /* SherpaOnnx2Pass.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = SherpaOnnx2Pass.app; sourceTree = BUILT_PRODUCTS_DIR; };
  24 + C9A2587C2AAEFFF100E555CA /* SherpaOnnx2PassApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SherpaOnnx2PassApp.swift; sourceTree = "<group>"; };
  25 + C9A2587E2AAEFFF100E555CA /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = "<group>"; };
  26 + C9A258802AAEFFF200E555CA /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
  27 + C9A258832AAEFFF200E555CA /* Preview Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = "Preview Assets.xcassets"; sourceTree = "<group>"; };
  28 + C9A2588A2AAF039D00E555CA /* Model.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Model.swift; sourceTree = "<group>"; };
  29 + C9A2588C2AAF039D00E555CA /* SherpaOnnxViewModel.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SherpaOnnxViewModel.swift; sourceTree = "<group>"; };
  30 + C9A2588D2AAF039D00E555CA /* Extension.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Extension.swift; sourceTree = "<group>"; };
  31 + C9A258922AAF057E00E555CA /* SherpaOnnx.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = SherpaOnnx.swift; path = "../../../swift-api-examples/SherpaOnnx.swift"; sourceTree = "<group>"; };
  32 + C9A258952AAF05D100E555CA /* sherpa-onnx.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = "sherpa-onnx.xcframework"; path = "../../build-ios/sherpa-onnx.xcframework"; sourceTree = "<group>"; };
  33 + C9A258972AAF05E400E555CA /* onnxruntime.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = onnxruntime.xcframework; path = "../../build-ios/ios-onnxruntime/1.15.1/onnxruntime.xcframework"; sourceTree = "<group>"; };
  34 +/* End PBXFileReference section */
  35 +
  36 +/* Begin PBXFrameworksBuildPhase section */
  37 + C9A258762AAEFFF100E555CA /* Frameworks */ = {
  38 + isa = PBXFrameworksBuildPhase;
  39 + buildActionMask = 2147483647;
  40 + files = (
  41 + C9A258982AAF05E400E555CA /* onnxruntime.xcframework in Frameworks */,
  42 + C9A258962AAF05D100E555CA /* sherpa-onnx.xcframework in Frameworks */,
  43 + );
  44 + runOnlyForDeploymentPostprocessing = 0;
  45 + };
  46 +/* End PBXFrameworksBuildPhase section */
  47 +
  48 +/* Begin PBXGroup section */
  49 + C9A258702AAEFFF100E555CA = {
  50 + isa = PBXGroup;
  51 + children = (
  52 + C9A2587B2AAEFFF100E555CA /* SherpaOnnx2Pass */,
  53 + C9A2587A2AAEFFF100E555CA /* Products */,
  54 + C9A258942AAF05D100E555CA /* Frameworks */,
  55 + );
  56 + sourceTree = "<group>";
  57 + };
  58 + C9A2587A2AAEFFF100E555CA /* Products */ = {
  59 + isa = PBXGroup;
  60 + children = (
  61 + C9A258792AAEFFF100E555CA /* SherpaOnnx2Pass.app */,
  62 + );
  63 + name = Products;
  64 + sourceTree = "<group>";
  65 + };
  66 + C9A2587B2AAEFFF100E555CA /* SherpaOnnx2Pass */ = {
  67 + isa = PBXGroup;
  68 + children = (
  69 + C9A258922AAF057E00E555CA /* SherpaOnnx.swift */,
  70 + C9A2588D2AAF039D00E555CA /* Extension.swift */,
  71 + C9A2588A2AAF039D00E555CA /* Model.swift */,
  72 + C9A2588C2AAF039D00E555CA /* SherpaOnnxViewModel.swift */,
  73 + C9A2587C2AAEFFF100E555CA /* SherpaOnnx2PassApp.swift */,
  74 + C9A2587E2AAEFFF100E555CA /* ContentView.swift */,
  75 + C9A258802AAEFFF200E555CA /* Assets.xcassets */,
  76 + C9A258822AAEFFF200E555CA /* Preview Content */,
  77 + );
  78 + path = SherpaOnnx2Pass;
  79 + sourceTree = "<group>";
  80 + };
  81 + C9A258822AAEFFF200E555CA /* Preview Content */ = {
  82 + isa = PBXGroup;
  83 + children = (
  84 + C9A258832AAEFFF200E555CA /* Preview Assets.xcassets */,
  85 + );
  86 + path = "Preview Content";
  87 + sourceTree = "<group>";
  88 + };
  89 + C9A258942AAF05D100E555CA /* Frameworks */ = {
  90 + isa = PBXGroup;
  91 + children = (
  92 + C9A258972AAF05E400E555CA /* onnxruntime.xcframework */,
  93 + C9A258952AAF05D100E555CA /* sherpa-onnx.xcframework */,
  94 + );
  95 + name = Frameworks;
  96 + sourceTree = "<group>";
  97 + };
  98 +/* End PBXGroup section */
  99 +
  100 +/* Begin PBXNativeTarget section */
  101 + C9A258782AAEFFF100E555CA /* SherpaOnnx2Pass */ = {
  102 + isa = PBXNativeTarget;
  103 + buildConfigurationList = C9A258872AAEFFF200E555CA /* Build configuration list for PBXNativeTarget "SherpaOnnx2Pass" */;
  104 + buildPhases = (
  105 + C9A258752AAEFFF100E555CA /* Sources */,
  106 + C9A258762AAEFFF100E555CA /* Frameworks */,
  107 + C9A258772AAEFFF100E555CA /* Resources */,
  108 + );
  109 + buildRules = (
  110 + );
  111 + dependencies = (
  112 + );
  113 + name = SherpaOnnx2Pass;
  114 + productName = SherpaOnnx2Pass;
  115 + productReference = C9A258792AAEFFF100E555CA /* SherpaOnnx2Pass.app */;
  116 + productType = "com.apple.product-type.application";
  117 + };
  118 +/* End PBXNativeTarget section */
  119 +
  120 +/* Begin PBXProject section */
  121 + C9A258712AAEFFF100E555CA /* Project object */ = {
  122 + isa = PBXProject;
  123 + attributes = {
  124 + BuildIndependentTargetsInParallel = 1;
  125 + LastSwiftUpdateCheck = 1420;
  126 + LastUpgradeCheck = 1420;
  127 + TargetAttributes = {
  128 + C9A258782AAEFFF100E555CA = {
  129 + CreatedOnToolsVersion = 14.2;
  130 + };
  131 + };
  132 + };
  133 + buildConfigurationList = C9A258742AAEFFF100E555CA /* Build configuration list for PBXProject "SherpaOnnx2Pass" */;
  134 + compatibilityVersion = "Xcode 14.0";
  135 + developmentRegion = en;
  136 + hasScannedForEncodings = 0;
  137 + knownRegions = (
  138 + en,
  139 + Base,
  140 + );
  141 + mainGroup = C9A258702AAEFFF100E555CA;
  142 + productRefGroup = C9A2587A2AAEFFF100E555CA /* Products */;
  143 + projectDirPath = "";
  144 + projectRoot = "";
  145 + targets = (
  146 + C9A258782AAEFFF100E555CA /* SherpaOnnx2Pass */,
  147 + );
  148 + };
  149 +/* End PBXProject section */
  150 +
  151 +/* Begin PBXResourcesBuildPhase section */
  152 + C9A258772AAEFFF100E555CA /* Resources */ = {
  153 + isa = PBXResourcesBuildPhase;
  154 + buildActionMask = 2147483647;
  155 + files = (
  156 + C9A258842AAEFFF200E555CA /* Preview Assets.xcassets in Resources */,
  157 + C9A258812AAEFFF200E555CA /* Assets.xcassets in Resources */,
  158 + );
  159 + runOnlyForDeploymentPostprocessing = 0;
  160 + };
  161 +/* End PBXResourcesBuildPhase section */
  162 +
  163 +/* Begin PBXSourcesBuildPhase section */
  164 + C9A258752AAEFFF100E555CA /* Sources */ = {
  165 + isa = PBXSourcesBuildPhase;
  166 + buildActionMask = 2147483647;
  167 + files = (
  168 + C9A2588E2AAF039D00E555CA /* Model.swift in Sources */,
  169 + C9A258902AAF039D00E555CA /* SherpaOnnxViewModel.swift in Sources */,
  170 + C9A258912AAF039D00E555CA /* Extension.swift in Sources */,
  171 + C9A2587F2AAEFFF100E555CA /* ContentView.swift in Sources */,
  172 + C9A258932AAF057E00E555CA /* SherpaOnnx.swift in Sources */,
  173 + C9A2587D2AAEFFF100E555CA /* SherpaOnnx2PassApp.swift in Sources */,
  174 + );
  175 + runOnlyForDeploymentPostprocessing = 0;
  176 + };
  177 +/* End PBXSourcesBuildPhase section */
  178 +
  179 +/* Begin XCBuildConfiguration section */
  180 + C9A258852AAEFFF200E555CA /* Debug */ = {
  181 + isa = XCBuildConfiguration;
  182 + buildSettings = {
  183 + ALWAYS_SEARCH_USER_PATHS = NO;
  184 + CLANG_ANALYZER_NONNULL = YES;
  185 + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
  186 + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20";
  187 + CLANG_ENABLE_MODULES = YES;
  188 + CLANG_ENABLE_OBJC_ARC = YES;
  189 + CLANG_ENABLE_OBJC_WEAK = YES;
  190 + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
  191 + CLANG_WARN_BOOL_CONVERSION = YES;
  192 + CLANG_WARN_COMMA = YES;
  193 + CLANG_WARN_CONSTANT_CONVERSION = YES;
  194 + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
  195 + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
  196 + CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
  197 + CLANG_WARN_EMPTY_BODY = YES;
  198 + CLANG_WARN_ENUM_CONVERSION = YES;
  199 + CLANG_WARN_INFINITE_RECURSION = YES;
  200 + CLANG_WARN_INT_CONVERSION = YES;
  201 + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
  202 + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
  203 + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
  204 + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
  205 + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
  206 + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
  207 + CLANG_WARN_STRICT_PROTOTYPES = YES;
  208 + CLANG_WARN_SUSPICIOUS_MOVE = YES;
  209 + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
  210 + CLANG_WARN_UNREACHABLE_CODE = YES;
  211 + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
  212 + COPY_PHASE_STRIP = NO;
  213 + DEBUG_INFORMATION_FORMAT = dwarf;
  214 + ENABLE_STRICT_OBJC_MSGSEND = YES;
  215 + ENABLE_TESTABILITY = YES;
  216 + GCC_C_LANGUAGE_STANDARD = gnu11;
  217 + GCC_DYNAMIC_NO_PIC = NO;
  218 + GCC_NO_COMMON_BLOCKS = YES;
  219 + GCC_OPTIMIZATION_LEVEL = 0;
  220 + GCC_PREPROCESSOR_DEFINITIONS = (
  221 + "DEBUG=1",
  222 + "$(inherited)",
  223 + );
  224 + GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
  225 + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
  226 + GCC_WARN_UNDECLARED_SELECTOR = YES;
  227 + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
  228 + GCC_WARN_UNUSED_FUNCTION = YES;
  229 + GCC_WARN_UNUSED_VARIABLE = YES;
  230 + IPHONEOS_DEPLOYMENT_TARGET = 16.2;
  231 + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
  232 + MTL_FAST_MATH = YES;
  233 + ONLY_ACTIVE_ARCH = YES;
  234 + SDKROOT = iphoneos;
  235 + SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
  236 + SWIFT_OPTIMIZATION_LEVEL = "-Onone";
  237 + };
  238 + name = Debug;
  239 + };
  240 + C9A258862AAEFFF200E555CA /* Release */ = {
  241 + isa = XCBuildConfiguration;
  242 + buildSettings = {
  243 + ALWAYS_SEARCH_USER_PATHS = NO;
  244 + CLANG_ANALYZER_NONNULL = YES;
  245 + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
  246 + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20";
  247 + CLANG_ENABLE_MODULES = YES;
  248 + CLANG_ENABLE_OBJC_ARC = YES;
  249 + CLANG_ENABLE_OBJC_WEAK = YES;
  250 + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
  251 + CLANG_WARN_BOOL_CONVERSION = YES;
  252 + CLANG_WARN_COMMA = YES;
  253 + CLANG_WARN_CONSTANT_CONVERSION = YES;
  254 + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
  255 + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
  256 + CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
  257 + CLANG_WARN_EMPTY_BODY = YES;
  258 + CLANG_WARN_ENUM_CONVERSION = YES;
  259 + CLANG_WARN_INFINITE_RECURSION = YES;
  260 + CLANG_WARN_INT_CONVERSION = YES;
  261 + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
  262 + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
  263 + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
  264 + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
  265 + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
  266 + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
  267 + CLANG_WARN_STRICT_PROTOTYPES = YES;
  268 + CLANG_WARN_SUSPICIOUS_MOVE = YES;
  269 + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
  270 + CLANG_WARN_UNREACHABLE_CODE = YES;
  271 + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
  272 + COPY_PHASE_STRIP = NO;
  273 + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
  274 + ENABLE_NS_ASSERTIONS = NO;
  275 + ENABLE_STRICT_OBJC_MSGSEND = YES;
  276 + GCC_C_LANGUAGE_STANDARD = gnu11;
  277 + GCC_NO_COMMON_BLOCKS = YES;
  278 + GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
  279 + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
  280 + GCC_WARN_UNDECLARED_SELECTOR = YES;
  281 + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
  282 + GCC_WARN_UNUSED_FUNCTION = YES;
  283 + GCC_WARN_UNUSED_VARIABLE = YES;
  284 + IPHONEOS_DEPLOYMENT_TARGET = 16.2;
  285 + MTL_ENABLE_DEBUG_INFO = NO;
  286 + MTL_FAST_MATH = YES;
  287 + SDKROOT = iphoneos;
  288 + SWIFT_COMPILATION_MODE = wholemodule;
  289 + SWIFT_OPTIMIZATION_LEVEL = "-O";
  290 + VALIDATE_PRODUCT = YES;
  291 + };
  292 + name = Release;
  293 + };
  294 + C9A258882AAEFFF200E555CA /* Debug */ = {
  295 + isa = XCBuildConfiguration;
  296 + buildSettings = {
  297 + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
  298 + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
  299 + CODE_SIGN_STYLE = Automatic;
  300 + CURRENT_PROJECT_VERSION = 1;
  301 + DEVELOPMENT_ASSET_PATHS = "\"SherpaOnnx2Pass/Preview Content\"";
  302 + ENABLE_PREVIEWS = YES;
  303 + GENERATE_INFOPLIST_FILE = YES;
  304 + HEADER_SEARCH_PATHS = "${PROJECT_DIR}/../../build-ios/sherpa-onnx.xcframework/Headers/";
  305 + INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES;
  306 + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES;
  307 + INFOPLIST_KEY_UILaunchScreen_Generation = YES;
  308 + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
  309 + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
  310 + LD_RUNPATH_SEARCH_PATHS = (
  311 + "$(inherited)",
  312 + "@executable_path/Frameworks",
  313 + );
  314 + MARKETING_VERSION = 1.0;
  315 + OTHER_LDFLAGS = "-lc++";
  316 + PRODUCT_BUNDLE_IDENTIFIER = "com.k2-fsa.org.SherpaOnnx2Pass";
  317 + PRODUCT_NAME = "$(TARGET_NAME)";
  318 + SWIFT_EMIT_LOC_STRINGS = YES;
  319 + SWIFT_OBJC_BRIDGING_HEADER = "${PROJECT_DIR}/../../swift-api-examples/SherpaOnnx-Bridging-Header.h";
  320 + SWIFT_VERSION = 5.0;
  321 + TARGETED_DEVICE_FAMILY = "1,2";
  322 + };
  323 + name = Debug;
  324 + };
  325 + C9A258892AAEFFF200E555CA /* Release */ = {
  326 + isa = XCBuildConfiguration;
  327 + buildSettings = {
  328 + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
  329 + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
  330 + CODE_SIGN_STYLE = Automatic;
  331 + CURRENT_PROJECT_VERSION = 1;
  332 + DEVELOPMENT_ASSET_PATHS = "\"SherpaOnnx2Pass/Preview Content\"";
  333 + ENABLE_PREVIEWS = YES;
  334 + GENERATE_INFOPLIST_FILE = YES;
  335 + HEADER_SEARCH_PATHS = "${PROJECT_DIR}/../../build-ios/sherpa-onnx.xcframework/Headers/";
  336 + INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES;
  337 + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES;
  338 + INFOPLIST_KEY_UILaunchScreen_Generation = YES;
  339 + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
  340 + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
  341 + LD_RUNPATH_SEARCH_PATHS = (
  342 + "$(inherited)",
  343 + "@executable_path/Frameworks",
  344 + );
  345 + MARKETING_VERSION = 1.0;
  346 + OTHER_LDFLAGS = "-lc++";
  347 + PRODUCT_BUNDLE_IDENTIFIER = "com.k2-fsa.org.SherpaOnnx2Pass";
  348 + PRODUCT_NAME = "$(TARGET_NAME)";
  349 + SWIFT_EMIT_LOC_STRINGS = YES;
  350 + SWIFT_OBJC_BRIDGING_HEADER = "${PROJECT_DIR}/../../swift-api-examples/SherpaOnnx-Bridging-Header.h";
  351 + SWIFT_VERSION = 5.0;
  352 + TARGETED_DEVICE_FAMILY = "1,2";
  353 + };
  354 + name = Release;
  355 + };
  356 +/* End XCBuildConfiguration section */
  357 +
  358 +/* Begin XCConfigurationList section */
  359 + C9A258742AAEFFF100E555CA /* Build configuration list for PBXProject "SherpaOnnx2Pass" */ = {
  360 + isa = XCConfigurationList;
  361 + buildConfigurations = (
  362 + C9A258852AAEFFF200E555CA /* Debug */,
  363 + C9A258862AAEFFF200E555CA /* Release */,
  364 + );
  365 + defaultConfigurationIsVisible = 0;
  366 + defaultConfigurationName = Release;
  367 + };
  368 + C9A258872AAEFFF200E555CA /* Build configuration list for PBXNativeTarget "SherpaOnnx2Pass" */ = {
  369 + isa = XCConfigurationList;
  370 + buildConfigurations = (
  371 + C9A258882AAEFFF200E555CA /* Debug */,
  372 + C9A258892AAEFFF200E555CA /* Release */,
  373 + );
  374 + defaultConfigurationIsVisible = 0;
  375 + defaultConfigurationName = Release;
  376 + };
  377 +/* End XCConfigurationList section */
  378 + };
  379 + rootObject = C9A258712AAEFFF100E555CA /* Project object */;
  380 +}
  1 +<?xml version="1.0" encoding="UTF-8"?>
  2 +<Workspace
  3 + version = "1.0">
  4 + <FileRef
  5 + location = "self:">
  6 + </FileRef>
  7 +</Workspace>
  1 +<?xml version="1.0" encoding="UTF-8"?>
  2 +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
  3 +<plist version="1.0">
  4 +<dict>
  5 + <key>IDEDidComputeMac32BitWarning</key>
  6 + <true/>
  7 +</dict>
  8 +</plist>
  1 +{
  2 + "colors" : [
  3 + {
  4 + "idiom" : "universal"
  5 + }
  6 + ],
  7 + "info" : {
  8 + "author" : "xcode",
  9 + "version" : 1
  10 + }
  11 +}
  1 +{
  2 + "images" : [
  3 + {
  4 + "filename" : "k2-1024x1024.png",
  5 + "idiom" : "universal",
  6 + "platform" : "ios",
  7 + "size" : "1024x1024"
  8 + }
  9 + ],
  10 + "info" : {
  11 + "author" : "xcode",
  12 + "version" : 1
  13 + }
  14 +}
  1 +{
  2 + "info" : {
  3 + "author" : "xcode",
  4 + "version" : 1
  5 + }
  6 +}
  1 +//
  2 +// ContentView.swift
  3 +// SherpaOnnx2Pass
  4 +//
  5 +// Created by fangjun on 2023/9/11.
  6 +//
  7 +
  8 +import SwiftUI
  9 +
  10 +struct ContentView: View {
  11 + @StateObject var sherpaOnnxVM = SherpaOnnxViewModel()
  12 +
  13 + var body: some View {
  14 + VStack {
  15 + Text("ASR with Next-gen Kaldi")
  16 + .font(.title)
  17 + if sherpaOnnxVM.status == .stop {
  18 + Text("See https://github.com/k2-fsa/sherpa-onnx")
  19 + Text("Press the Start button to run!")
  20 + }
  21 + ScrollView(.vertical, showsIndicators: true) {
  22 + HStack {
  23 + Text(sherpaOnnxVM.subtitles)
  24 + Spacer()
  25 + }
  26 + }
  27 + Spacer()
  28 + Button {
  29 + toggleRecorder()
  30 + } label: {
  31 + Text(sherpaOnnxVM.status == .stop ? "Start" : "Stop")
  32 + }
  33 + }
  34 + .padding()
  35 + }
  36 +
  37 + private func toggleRecorder() {
  38 + sherpaOnnxVM.toggleRecorder()
  39 + }
  40 +}
  41 +
  42 +struct ContentView_Previews: PreviewProvider {
  43 + static var previews: some View {
  44 + ContentView()
  45 + }
  46 +}
  1 +//
  2 +// Extension.swift
  3 +// SherpaOnnx
  4 +//
  5 +// Created by knight on 2023/4/5.
  6 +//
  7 +
  8 +import AVFoundation
  9 +
  10 +extension AudioBuffer {
  11 + func array() -> [Float] {
  12 + return Array(UnsafeBufferPointer(self))
  13 + }
  14 +}
  15 +
  16 +extension AVAudioPCMBuffer {
  17 + func array() -> [Float] {
  18 + return self.audioBufferList.pointee.mBuffers.array()
  19 + }
  20 +}
  1 +import Foundation
  2 +
  3 +func getResource(_ forResource: String, _ ofType: String) -> String {
  4 + let path = Bundle.main.path(forResource: forResource, ofType: ofType)
  5 + precondition(
  6 + path != nil,
  7 + "\(forResource).\(ofType) does not exist!\n" + "Remember to change \n"
  8 + + " Build Phases -> Copy Bundle Resources\n" + "to add it!"
  9 + )
  10 + return path!
  11 +}
  12 +/// Please refer to
  13 +/// https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html
  14 +/// to download pre-trained models
  15 +
  16 +/// sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 (Bilingual, Chinese + English)
  17 +/// https://k2-fsa.github.io/sherpa/onnx/pretrained_models/zipformer-transducer-models.html
  18 +func getBilingualStreamingZhEnZipformer20230220() -> SherpaOnnxOnlineModelConfig {
  19 + let encoder = getResource("encoder-epoch-99-avg-1.int8", "onnx")
  20 + let decoder = getResource("decoder-epoch-99-avg-1", "onnx")
  21 + let joiner = getResource("joiner-epoch-99-avg-1.int8", "onnx")
  22 + let tokens = getResource("tokens", "txt")
  23 +
  24 + return sherpaOnnxOnlineModelConfig(
  25 + tokens: tokens,
  26 + transducer: sherpaOnnxOnlineTransducerModelConfig(
  27 + encoder: encoder,
  28 + decoder: decoder,
  29 + joiner: joiner),
  30 + numThreads: 1,
  31 + modelType: "zipformer"
  32 + )
  33 +}
  34 +
  35 +/// csukuangfj/sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23 (Chinese)
  36 +/// https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#csukuangfj-sherpa-onnx-streaming-zipformer-zh-14m-2023-02-23-chinese
  37 +
  38 +func getStreamingZh14MZipformer20230223() -> SherpaOnnxOnlineModelConfig {
  39 + let encoder = getResource("encoder-epoch-99-avg-1.int8", "onnx")
  40 + let decoder = getResource("decoder-epoch-99-avg-1", "onnx")
  41 + let joiner = getResource("joiner-epoch-99-avg-1.int8", "onnx")
  42 + let tokens = getResource("tokens", "txt")
  43 +
  44 + return sherpaOnnxOnlineModelConfig(
  45 + tokens: tokens,
  46 + transducer: sherpaOnnxOnlineTransducerModelConfig(
  47 + encoder: encoder,
  48 + decoder: decoder,
  49 + joiner: joiner),
  50 + numThreads: 1,
  51 + modelType: "zipformer"
  52 + )
  53 +}
  54 +
  55 +/// csukuangfj/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17 (English)
  56 +/// https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#csukuangfj-sherpa-onnx-streaming-zipformer-en-20m-2023-02-17-english
  57 +
  58 +func getStreamingEn20MZipformer20230217() -> SherpaOnnxOnlineModelConfig {
  59 + let encoder = getResource("encoder-epoch-99-avg-1.int8", "onnx")
  60 + let decoder = getResource("decoder-epoch-99-avg-1", "onnx")
  61 + let joiner = getResource("joiner-epoch-99-avg-1", "onnx")
  62 + let tokens = getResource("tokens", "txt")
  63 +
  64 + return sherpaOnnxOnlineModelConfig(
  65 + tokens: tokens,
  66 + transducer: sherpaOnnxOnlineTransducerModelConfig(
  67 + encoder: encoder,
  68 + decoder: decoder,
  69 + joiner: joiner),
  70 + numThreads: 1,
  71 + modelType: "zipformer"
  72 + )
  73 +}
  74 +
  75 +/// ========================================
  76 +/// Non-streaming models
  77 +/// ========================================
  78 +
  79 +/// csukuangfj/sherpa-onnx-paraformer-zh-2023-03-28 (Chinese)
  80 +/// https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-paraformer/paraformer-models.html#csukuangfj-sherpa-onnx-paraformer-zh-2023-03-28-chinese
  81 +func getNonStreamingZhParaformer20230328() -> SherpaOnnxOfflineModelConfig {
  82 + let model = getResource("model.int8", "onnx")
  83 + let tokens = getResource("paraformer-tokens", "txt")
  84 +
  85 + return sherpaOnnxOfflineModelConfig(
  86 + tokens: tokens,
  87 + paraformer: sherpaOnnxOfflineParaformerModelConfig(
  88 + model: model),
  89 + numThreads: 1,
  90 + modelType: "paraformer"
  91 + )
  92 +}
  93 +
  94 +// https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/tiny.en.html#tiny-en
  95 +// English, int8 encoder and decoder
  96 +func getNonStreamingWhisperTinyEn() -> SherpaOnnxOfflineModelConfig {
  97 + let encoder = getResource("tiny.en-encoder.int8", "onnx")
  98 + let decoder = getResource("tiny.en-decoder.int8", "onnx")
  99 + let tokens = getResource("tiny.en-tokens", "txt")
  100 +
  101 + return sherpaOnnxOfflineModelConfig(
  102 + tokens: tokens,
  103 + whisper: sherpaOnnxOfflineWhisperModelConfig(
  104 + encoder: encoder,
  105 + decoder: decoder
  106 + ),
  107 + numThreads: 1,
  108 + modelType: "whisper"
  109 + )
  110 +}
  111 +
  112 +// icefall-asr-multidataset-pruned_transducer_stateless7-2023-05-04 (English)
  113 +// https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#icefall-asr-multidataset-pruned-transducer-stateless7-2023-05-04-english
  114 +
  115 +func getNonStreamingEnZipformer20230504() -> SherpaOnnxOfflineModelConfig {
  116 + let encoder = getResource("encoder-epoch-30-avg-4.int8", "onnx")
  117 + let decoder = getResource("decoder-epoch-30-avg-4", "onnx")
  118 + let joiner = getResource("joiner-epoch-30-avg-4", "onnx")
  119 + let tokens = getResource("non-streaming-zipformer-tokens", "txt")
  120 +
  121 + return sherpaOnnxOfflineModelConfig(
  122 + tokens: tokens,
  123 + transducer: sherpaOnnxOfflineTransducerModelConfig(
  124 + encoder: encoder,
  125 + decoder: decoder,
  126 + joiner: joiner),
  127 + numThreads: 1,
  128 + modelType: "zipformer"
  129 + )
  130 +}
  131 +
  132 +/// Please refer to
  133 +/// https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html
  134 +/// to add more models if you need
  1 +{
  2 + "info" : {
  3 + "author" : "xcode",
  4 + "version" : 1
  5 + }
  6 +}
  1 +//
  2 +// SherpaOnnx2PassApp.swift
  3 +// SherpaOnnx2Pass
  4 +//
  5 +// Created by fangjun on 2023/9/11.
  6 +//
  7 +
  8 +import SwiftUI
  9 +
  10 +@main
  11 +struct SherpaOnnx2PassApp: App {
  12 + var body: some Scene {
  13 + WindowGroup {
  14 + ContentView()
  15 + }
  16 + }
  17 +}
  1 +//
  2 +// SherpaOnnxViewModel.swift
  3 +// SherpaOnnx
  4 +//
  5 +// Created by knight on 2023/4/5.
  6 +//
  7 +
  8 +import Foundation
  9 +import AVFoundation
  10 +
  11 +enum Status {
  12 + case stop
  13 + case recording
  14 +}
  15 +
  16 +class SherpaOnnxViewModel: ObservableObject {
  17 + @Published var status: Status = .stop
  18 + @Published var subtitles: String = ""
  19 +
  20 + var sentences: [String] = []
  21 + var samplesBuffer = [[Float]] ()
  22 +
  23 + var audioEngine: AVAudioEngine? = nil
  24 + var recognizer: SherpaOnnxRecognizer! = nil
  25 + var offlineRecognizer: SherpaOnnxOfflineRecognizer! = nil
  26 +
  27 + var lastSentence: String = ""
  28 + // let maxSentence: Int = 10 // for Chinese
  29 + let maxSentence: Int = 6 // for English
  30 +
  31 + var results: String {
  32 + if sentences.isEmpty && lastSentence.isEmpty {
  33 + return ""
  34 + }
  35 + if sentences.isEmpty {
  36 + return "0: \(lastSentence.lowercased())"
  37 + }
  38 +
  39 + let start = max(sentences.count - maxSentence, 0)
  40 + if lastSentence.isEmpty {
  41 + return sentences.enumerated().map { (index, s) in "\(index): \(s.lowercased())" }[start...]
  42 + .joined(separator: "\n")
  43 + } else {
  44 + return sentences.enumerated().map { (index, s) in "\(index): \(s.lowercased())" }[start...]
  45 + .joined(separator: "\n") + "\n\(sentences.count): \(lastSentence.lowercased())"
  46 + }
  47 + }
  48 +
  49 + func updateLabel() {
  50 + DispatchQueue.main.async {
  51 + self.subtitles = self.results
  52 + }
  53 + }
  54 +
  55 + init() {
  56 + initRecognizer()
  57 + initOfflineRecognizer()
  58 + initRecorder()
  59 + }
  60 +
  61 + private func initRecognizer() {
  62 + // Please select one model that is best suitable for you.
  63 + //
  64 + // You can also modify Model.swift to add new pre-trained models from
  65 + // https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html
  66 + // let modelConfig = getBilingualStreamingZhEnZipformer20230220()
  67 + /* let modelConfig = getStreamingZh14MZipformer20230223() */
  68 +
  69 + let modelConfig = getStreamingEn20MZipformer20230217()
  70 +
  71 + let featConfig = sherpaOnnxFeatureConfig(
  72 + sampleRate: 16000,
  73 + featureDim: 80)
  74 +
  75 + var config = sherpaOnnxOnlineRecognizerConfig(
  76 + featConfig: featConfig,
  77 + modelConfig: modelConfig,
  78 + enableEndpoint: true,
  79 + rule1MinTrailingSilence: 2.4,
  80 +
  81 + // rule2MinTrailingSilence: 1.2, // for Chinese
  82 +
  83 + rule2MinTrailingSilence: 0.5, // for English
  84 +
  85 + rule3MinUtteranceLength: 30,
  86 + decodingMethod: "greedy_search",
  87 + maxActivePaths: 4
  88 + )
  89 + recognizer = SherpaOnnxRecognizer(config: &config)
  90 + }
  91 +
  92 + private func initOfflineRecognizer() {
  93 + // let modelConfig = getNonStreamingZhParaformer20230328()
  94 + let modelConfig = getNonStreamingWhisperTinyEn()
  95 +
  96 + // let modelConfig = getNonStreamingEnZipformer20230504()
  97 +
  98 + let featConfig = sherpaOnnxFeatureConfig(
  99 + sampleRate: 16000,
  100 + featureDim: 80)
  101 +
  102 + var config = sherpaOnnxOfflineRecognizerConfig(
  103 + featConfig: featConfig,
  104 + modelConfig: modelConfig,
  105 + decodingMethod: "greedy_search",
  106 + maxActivePaths: 4
  107 + )
  108 + offlineRecognizer = SherpaOnnxOfflineRecognizer(config: &config)
  109 + }
  110 +
  111 + private func initRecorder() {
  112 + print("init recorder")
  113 + audioEngine = AVAudioEngine()
  114 + let inputNode = self.audioEngine?.inputNode
  115 + let bus = 0
  116 + let inputFormat = inputNode?.outputFormat(forBus: bus)
  117 + let outputFormat = AVAudioFormat(
  118 + commonFormat: .pcmFormatFloat32,
  119 + sampleRate: 16000, channels: 1,
  120 + interleaved: false)!
  121 +
  122 + let converter = AVAudioConverter(from: inputFormat!, to: outputFormat)!
  123 +
  124 + inputNode!.installTap(
  125 + onBus: bus,
  126 + bufferSize: 1024,
  127 + format: inputFormat
  128 + ) {
  129 + (buffer: AVAudioPCMBuffer, when: AVAudioTime) in
  130 + var newBufferAvailable = true
  131 +
  132 + let inputCallback: AVAudioConverterInputBlock = {
  133 + inNumPackets, outStatus in
  134 + if newBufferAvailable {
  135 + outStatus.pointee = .haveData
  136 + newBufferAvailable = false
  137 +
  138 + return buffer
  139 + } else {
  140 + outStatus.pointee = .noDataNow
  141 + return nil
  142 + }
  143 + }
  144 +
  145 + let convertedBuffer = AVAudioPCMBuffer(
  146 + pcmFormat: outputFormat,
  147 + frameCapacity:
  148 + AVAudioFrameCount(outputFormat.sampleRate)
  149 + * buffer.frameLength
  150 + / AVAudioFrameCount(buffer.format.sampleRate))!
  151 +
  152 + var error: NSError?
  153 + let _ = converter.convert(
  154 + to: convertedBuffer,
  155 + error: &error, withInputFrom: inputCallback)
  156 +
  157 + // TODO(fangjun): Handle status != haveData
  158 +
  159 + let array = convertedBuffer.array()
  160 + if !array.isEmpty {
  161 + self.samplesBuffer.append(array)
  162 +
  163 + self.recognizer.acceptWaveform(samples: array)
  164 + while (self.recognizer.isReady()){
  165 + self.recognizer.decode()
  166 + }
  167 + let isEndpoint = self.recognizer.isEndpoint()
  168 + let text = self.recognizer.getResult().text
  169 +
  170 + if !text.isEmpty && self.lastSentence != text {
  171 + self.lastSentence = text
  172 + self.updateLabel()
  173 + print(text)
  174 + }
  175 +
  176 + if isEndpoint{
  177 + if !text.isEmpty {
  178 + // Invoke offline recognizer
  179 + var numSamples: Int = 0
  180 + for a in self.samplesBuffer {
  181 + numSamples += a.count
  182 + }
  183 +
  184 + var samples: [Float] = Array(repeating: 0, count: numSamples)
  185 + var i = 0
  186 + for a in self.samplesBuffer {
  187 + for s in a {
  188 + samples[i] = s
  189 + i += 1
  190 + }
  191 + }
  192 +
  193 + // let num = 12000 // For Chinese
  194 + let num = 10000 // For English
  195 + self.lastSentence = self.offlineRecognizer.decode(samples: Array(samples[0..<samples.count-num])).text
  196 +
  197 + let tmp = self.lastSentence
  198 + self.lastSentence = ""
  199 + self.sentences.append(tmp)
  200 +
  201 + self.updateLabel()
  202 +
  203 + i = 0
  204 + if samples.count > num {
  205 + i = samples.count - num
  206 + }
  207 + var tail: [Float] = Array(repeating: 0, count: samples.count - i)
  208 +
  209 + for k in 0 ... samples.count - i - 1 {
  210 + tail[k] = samples[i+k];
  211 + }
  212 +
  213 + self.samplesBuffer = [[Float]]()
  214 + self.samplesBuffer.append(tail)
  215 + } else {
  216 + self.samplesBuffer = [[Float]]()
  217 + }
  218 + self.recognizer.reset()
  219 + }
  220 + }
  221 + }
  222 + }
  223 +
  224 + public func toggleRecorder() {
  225 + if status == .stop {
  226 + startRecorder()
  227 + status = .recording
  228 + } else {
  229 + stopRecorder()
  230 + status = .stop
  231 + }
  232 + }
  233 +
  234 + private func startRecorder() {
  235 + lastSentence = ""
  236 + sentences = []
  237 + samplesBuffer = [[Float]] ()
  238 + updateLabel()
  239 +
  240 + do {
  241 + try self.audioEngine?.start()
  242 + } catch let error as NSError {
  243 + print("Got an error starting audioEngine: \(error.domain), \(error)")
  244 + }
  245 + print("started")
  246 + }
  247 +
  248 + private func stopRecorder() {
  249 + audioEngine?.stop()
  250 + print("stopped")
  251 + }
  252 +}
  1 +../../SherpaOnnx/SherpaOnnx/k2-1024x1024.png
@@ -347,6 +347,8 @@ SherpaOnnxOfflineRecognizerResult *GetOfflineStreamResult( @@ -347,6 +347,8 @@ SherpaOnnxOfflineRecognizerResult *GetOfflineStreamResult(
347 const auto &text = result.text; 347 const auto &text = result.text;
348 348
349 auto r = new SherpaOnnxOfflineRecognizerResult; 349 auto r = new SherpaOnnxOfflineRecognizerResult;
  350 + memset(r, 0, sizeof(SherpaOnnxOfflineRecognizerResult));
  351 +
350 r->text = new char[text.size() + 1]; 352 r->text = new char[text.size() + 1];
351 std::copy(text.begin(), text.end(), const_cast<char *>(r->text)); 353 std::copy(text.begin(), text.end(), const_cast<char *>(r->text));
352 const_cast<char *>(r->text)[text.size()] = 0; 354 const_cast<char *>(r->text)[text.size()] = 0;
@@ -100,4 +100,42 @@ std::unique_ptr<OfflineCtcModel> OfflineCtcModel::Create( @@ -100,4 +100,42 @@ std::unique_ptr<OfflineCtcModel> OfflineCtcModel::Create(
100 return nullptr; 100 return nullptr;
101 } 101 }
102 102
  103 +#if __ANDROID_API__ >= 9
  104 +
  105 +std::unique_ptr<OfflineCtcModel> OfflineCtcModel::Create(
  106 + AAssetManager *mgr, const OfflineModelConfig &config) {
  107 + ModelType model_type = ModelType::kUnkown;
  108 +
  109 + std::string filename;
  110 + if (!config.nemo_ctc.model.empty()) {
  111 + filename = config.nemo_ctc.model;
  112 + } else if (!config.tdnn.model.empty()) {
  113 + filename = config.tdnn.model;
  114 + } else {
  115 + SHERPA_ONNX_LOGE("Please specify a CTC model");
  116 + exit(-1);
  117 + }
  118 +
  119 + {
  120 + auto buffer = ReadFile(mgr, filename);
  121 +
  122 + model_type = GetModelType(buffer.data(), buffer.size(), config.debug);
  123 + }
  124 +
  125 + switch (model_type) {
  126 + case ModelType::kEncDecCTCModelBPE:
  127 + return std::make_unique<OfflineNemoEncDecCtcModel>(mgr, config);
  128 + break;
  129 + case ModelType::kTdnn:
  130 + return std::make_unique<OfflineTdnnCtcModel>(mgr, config);
  131 + break;
  132 + case ModelType::kUnkown:
  133 + SHERPA_ONNX_LOGE("Unknown model type in offline CTC!");
  134 + return nullptr;
  135 + }
  136 +
  137 + return nullptr;
  138 +}
  139 +#endif
  140 +
103 } // namespace sherpa_onnx 141 } // namespace sherpa_onnx
@@ -8,6 +8,11 @@ @@ -8,6 +8,11 @@
8 #include <string> 8 #include <string>
9 #include <utility> 9 #include <utility>
10 10
  11 +#if __ANDROID_API__ >= 9
  12 +#include "android/asset_manager.h"
  13 +#include "android/asset_manager_jni.h"
  14 +#endif
  15 +
11 #include "onnxruntime_cxx_api.h" // NOLINT 16 #include "onnxruntime_cxx_api.h" // NOLINT
12 #include "sherpa-onnx/csrc/offline-model-config.h" 17 #include "sherpa-onnx/csrc/offline-model-config.h"
13 18
@@ -16,9 +21,15 @@ namespace sherpa_onnx { @@ -16,9 +21,15 @@ namespace sherpa_onnx {
16 class OfflineCtcModel { 21 class OfflineCtcModel {
17 public: 22 public:
18 virtual ~OfflineCtcModel() = default; 23 virtual ~OfflineCtcModel() = default;
  24 +
19 static std::unique_ptr<OfflineCtcModel> Create( 25 static std::unique_ptr<OfflineCtcModel> Create(
20 const OfflineModelConfig &config); 26 const OfflineModelConfig &config);
21 27
  28 +#if __ANDROID_API__ >= 9
  29 + static std::unique_ptr<OfflineCtcModel> Create(
  30 + AAssetManager *mgr, const OfflineModelConfig &config);
  31 +#endif
  32 +
22 /** Run the forward method of the model. 33 /** Run the forward method of the model.
23 * 34 *
24 * @param features A tensor of shape (N, T, C). It is changed in-place. 35 * @param features A tensor of shape (N, T, C). It is changed in-place.
@@ -16,6 +16,13 @@ std::unique_ptr<OfflineLM> OfflineLM::Create(const OfflineLMConfig &config) { @@ -16,6 +16,13 @@ std::unique_ptr<OfflineLM> OfflineLM::Create(const OfflineLMConfig &config) {
16 return std::make_unique<OfflineRnnLM>(config); 16 return std::make_unique<OfflineRnnLM>(config);
17 } 17 }
18 18
  19 +#if __ANDROID_API__ >= 9
  20 +std::unique_ptr<OfflineLM> OfflineLM::Create(AAssetManager *mgr,
  21 + const OfflineLMConfig &config) {
  22 + return std::make_unique<OfflineRnnLM>(mgr, config);
  23 +}
  24 +#endif
  25 +
19 void OfflineLM::ComputeLMScore(float scale, int32_t context_size, 26 void OfflineLM::ComputeLMScore(float scale, int32_t context_size,
20 std::vector<Hypotheses> *hyps) { 27 std::vector<Hypotheses> *hyps) {
21 // compute the max token seq so that we know how much space to allocate 28 // compute the max token seq so that we know how much space to allocate
@@ -8,6 +8,11 @@ @@ -8,6 +8,11 @@
8 #include <memory> 8 #include <memory>
9 #include <vector> 9 #include <vector>
10 10
  11 +#if __ANDROID_API__ >= 9
  12 +#include "android/asset_manager.h"
  13 +#include "android/asset_manager_jni.h"
  14 +#endif
  15 +
11 #include "onnxruntime_cxx_api.h" // NOLINT 16 #include "onnxruntime_cxx_api.h" // NOLINT
12 #include "sherpa-onnx/csrc/hypothesis.h" 17 #include "sherpa-onnx/csrc/hypothesis.h"
13 #include "sherpa-onnx/csrc/offline-lm-config.h" 18 #include "sherpa-onnx/csrc/offline-lm-config.h"
@@ -20,6 +25,11 @@ class OfflineLM { @@ -20,6 +25,11 @@ class OfflineLM {
20 25
21 static std::unique_ptr<OfflineLM> Create(const OfflineLMConfig &config); 26 static std::unique_ptr<OfflineLM> Create(const OfflineLMConfig &config);
22 27
  28 +#if __ANDROID_API__ >= 9
  29 + static std::unique_ptr<OfflineLM> Create(AAssetManager *mgr,
  30 + const OfflineLMConfig &config);
  31 +#endif
  32 +
23 /** Rescore a batch of sentences. 33 /** Rescore a batch of sentences.
24 * 34 *
25 * @param x A 2-D tensor of shape (N, L) with data type int64. 35 * @param x A 2-D tensor of shape (N, L) with data type int64.
@@ -19,9 +19,21 @@ class OfflineNemoEncDecCtcModel::Impl { @@ -19,9 +19,21 @@ class OfflineNemoEncDecCtcModel::Impl {
19 env_(ORT_LOGGING_LEVEL_ERROR), 19 env_(ORT_LOGGING_LEVEL_ERROR),
20 sess_opts_(GetSessionOptions(config)), 20 sess_opts_(GetSessionOptions(config)),
21 allocator_{} { 21 allocator_{} {
22 - Init(); 22 + auto buf = ReadFile(config_.nemo_ctc.model);
  23 + Init(buf.data(), buf.size());
23 } 24 }
24 25
  26 +#if __ANDROID_API__ >= 9
  27 + Impl(AAssetManager *mgr, const OfflineModelConfig &config)
  28 + : config_(config),
  29 + env_(ORT_LOGGING_LEVEL_ERROR),
  30 + sess_opts_(GetSessionOptions(config)),
  31 + allocator_{} {
  32 + auto buf = ReadFile(mgr, config_.nemo_ctc.model);
  33 + Init(buf.data(), buf.size());
  34 + }
  35 +#endif
  36 +
25 std::pair<Ort::Value, Ort::Value> Forward(Ort::Value features, 37 std::pair<Ort::Value, Ort::Value> Forward(Ort::Value features,
26 Ort::Value features_length) { 38 Ort::Value features_length) {
27 std::vector<int64_t> shape = 39 std::vector<int64_t> shape =
@@ -57,10 +69,8 @@ class OfflineNemoEncDecCtcModel::Impl { @@ -57,10 +69,8 @@ class OfflineNemoEncDecCtcModel::Impl {
57 std::string FeatureNormalizationMethod() const { return normalize_type_; } 69 std::string FeatureNormalizationMethod() const { return normalize_type_; }
58 70
59 private: 71 private:
60 - void Init() {  
61 - auto buf = ReadFile(config_.nemo_ctc.model);  
62 -  
63 - sess_ = std::make_unique<Ort::Session>(env_, buf.data(), buf.size(), 72 + void Init(void *model_data, size_t model_data_length) {
  73 + sess_ = std::make_unique<Ort::Session>(env_, model_data, model_data_length,
64 sess_opts_); 74 sess_opts_);
65 75
66 GetInputNames(sess_.get(), &input_names_, &input_names_ptr_); 76 GetInputNames(sess_.get(), &input_names_, &input_names_ptr_);
@@ -104,6 +114,12 @@ OfflineNemoEncDecCtcModel::OfflineNemoEncDecCtcModel( @@ -104,6 +114,12 @@ OfflineNemoEncDecCtcModel::OfflineNemoEncDecCtcModel(
104 const OfflineModelConfig &config) 114 const OfflineModelConfig &config)
105 : impl_(std::make_unique<Impl>(config)) {} 115 : impl_(std::make_unique<Impl>(config)) {}
106 116
  117 +#if __ANDROID_API__ >= 9
  118 +OfflineNemoEncDecCtcModel::OfflineNemoEncDecCtcModel(
  119 + AAssetManager *mgr, const OfflineModelConfig &config)
  120 + : impl_(std::make_unique<Impl>(mgr, config)) {}
  121 +#endif
  122 +
107 OfflineNemoEncDecCtcModel::~OfflineNemoEncDecCtcModel() = default; 123 OfflineNemoEncDecCtcModel::~OfflineNemoEncDecCtcModel() = default;
108 124
109 std::pair<Ort::Value, Ort::Value> OfflineNemoEncDecCtcModel::Forward( 125 std::pair<Ort::Value, Ort::Value> OfflineNemoEncDecCtcModel::Forward(
@@ -8,6 +8,11 @@ @@ -8,6 +8,11 @@
8 #include <utility> 8 #include <utility>
9 #include <vector> 9 #include <vector>
10 10
  11 +#if __ANDROID_API__ >= 9
  12 +#include "android/asset_manager.h"
  13 +#include "android/asset_manager_jni.h"
  14 +#endif
  15 +
11 #include "onnxruntime_cxx_api.h" // NOLINT 16 #include "onnxruntime_cxx_api.h" // NOLINT
12 #include "sherpa-onnx/csrc/offline-ctc-model.h" 17 #include "sherpa-onnx/csrc/offline-ctc-model.h"
13 #include "sherpa-onnx/csrc/offline-model-config.h" 18 #include "sherpa-onnx/csrc/offline-model-config.h"
@@ -23,6 +28,12 @@ namespace sherpa_onnx { @@ -23,6 +28,12 @@ namespace sherpa_onnx {
23 class OfflineNemoEncDecCtcModel : public OfflineCtcModel { 28 class OfflineNemoEncDecCtcModel : public OfflineCtcModel {
24 public: 29 public:
25 explicit OfflineNemoEncDecCtcModel(const OfflineModelConfig &config); 30 explicit OfflineNemoEncDecCtcModel(const OfflineModelConfig &config);
  31 +
  32 +#if __ANDROID_API__ >= 9
  33 + OfflineNemoEncDecCtcModel(AAssetManager *mgr,
  34 + const OfflineModelConfig &config);
  35 +#endif
  36 +
26 ~OfflineNemoEncDecCtcModel() override; 37 ~OfflineNemoEncDecCtcModel() override;
27 38
28 /** Run the forward method of the model. 39 /** Run the forward method of the model.
@@ -21,8 +21,20 @@ class OfflineParaformerModel::Impl { @@ -21,8 +21,20 @@ class OfflineParaformerModel::Impl {
21 env_(ORT_LOGGING_LEVEL_ERROR), 21 env_(ORT_LOGGING_LEVEL_ERROR),
22 sess_opts_(GetSessionOptions(config)), 22 sess_opts_(GetSessionOptions(config)),
23 allocator_{} { 23 allocator_{} {
24 - Init(); 24 + auto buf = ReadFile(config_.paraformer.model);
  25 + Init(buf.data(), buf.size());
  26 + }
  27 +
  28 +#if __ANDROID_API__ >= 9
  29 + Impl(AAssetManager *mgr, const OfflineModelConfig &config)
  30 + : config_(config),
  31 + env_(ORT_LOGGING_LEVEL_ERROR),
  32 + sess_opts_(GetSessionOptions(config)),
  33 + allocator_{} {
  34 + auto buf = ReadFile(mgr, config_.paraformer.model);
  35 + Init(buf.data(), buf.size());
25 } 36 }
  37 +#endif
26 38
27 std::pair<Ort::Value, Ort::Value> Forward(Ort::Value features, 39 std::pair<Ort::Value, Ort::Value> Forward(Ort::Value features,
28 Ort::Value features_length) { 40 Ort::Value features_length) {
@@ -49,10 +61,8 @@ class OfflineParaformerModel::Impl { @@ -49,10 +61,8 @@ class OfflineParaformerModel::Impl {
49 OrtAllocator *Allocator() const { return allocator_; } 61 OrtAllocator *Allocator() const { return allocator_; }
50 62
51 private: 63 private:
52 - void Init() {  
53 - auto buf = ReadFile(config_.paraformer.model);  
54 -  
55 - sess_ = std::make_unique<Ort::Session>(env_, buf.data(), buf.size(), 64 + void Init(void *model_data, size_t model_data_length) {
  65 + sess_ = std::make_unique<Ort::Session>(env_, model_data, model_data_length,
56 sess_opts_); 66 sess_opts_);
57 67
58 GetInputNames(sess_.get(), &input_names_, &input_names_ptr_); 68 GetInputNames(sess_.get(), &input_names_, &input_names_ptr_);
@@ -101,6 +111,12 @@ class OfflineParaformerModel::Impl { @@ -101,6 +111,12 @@ class OfflineParaformerModel::Impl {
101 OfflineParaformerModel::OfflineParaformerModel(const OfflineModelConfig &config) 111 OfflineParaformerModel::OfflineParaformerModel(const OfflineModelConfig &config)
102 : impl_(std::make_unique<Impl>(config)) {} 112 : impl_(std::make_unique<Impl>(config)) {}
103 113
  114 +#if __ANDROID_API__ >= 9
  115 +OfflineParaformerModel::OfflineParaformerModel(AAssetManager *mgr,
  116 + const OfflineModelConfig &config)
  117 + : impl_(std::make_unique<Impl>(mgr, config)) {}
  118 +#endif
  119 +
104 OfflineParaformerModel::~OfflineParaformerModel() = default; 120 OfflineParaformerModel::~OfflineParaformerModel() = default;
105 121
106 std::pair<Ort::Value, Ort::Value> OfflineParaformerModel::Forward( 122 std::pair<Ort::Value, Ort::Value> OfflineParaformerModel::Forward(
@@ -8,6 +8,11 @@ @@ -8,6 +8,11 @@
8 #include <utility> 8 #include <utility>
9 #include <vector> 9 #include <vector>
10 10
  11 +#if __ANDROID_API__ >= 9
  12 +#include "android/asset_manager.h"
  13 +#include "android/asset_manager_jni.h"
  14 +#endif
  15 +
11 #include "onnxruntime_cxx_api.h" // NOLINT 16 #include "onnxruntime_cxx_api.h" // NOLINT
12 #include "sherpa-onnx/csrc/offline-model-config.h" 17 #include "sherpa-onnx/csrc/offline-model-config.h"
13 18
@@ -16,6 +21,11 @@ namespace sherpa_onnx { @@ -16,6 +21,11 @@ namespace sherpa_onnx {
16 class OfflineParaformerModel { 21 class OfflineParaformerModel {
17 public: 22 public:
18 explicit OfflineParaformerModel(const OfflineModelConfig &config); 23 explicit OfflineParaformerModel(const OfflineModelConfig &config);
  24 +
  25 +#if __ANDROID_API__ >= 9
  26 + OfflineParaformerModel(AAssetManager *mgr, const OfflineModelConfig &config);
  27 +#endif
  28 +
19 ~OfflineParaformerModel(); 29 ~OfflineParaformerModel();
20 30
21 /** Run the forward method of the model. 31 /** Run the forward method of the model.
@@ -10,6 +10,11 @@ @@ -10,6 +10,11 @@
10 #include <utility> 10 #include <utility>
11 #include <vector> 11 #include <vector>
12 12
  13 +#if __ANDROID_API__ >= 9
  14 +#include "android/asset_manager.h"
  15 +#include "android/asset_manager_jni.h"
  16 +#endif
  17 +
13 #include "sherpa-onnx/csrc/offline-ctc-decoder.h" 18 #include "sherpa-onnx/csrc/offline-ctc-decoder.h"
14 #include "sherpa-onnx/csrc/offline-ctc-greedy-search-decoder.h" 19 #include "sherpa-onnx/csrc/offline-ctc-greedy-search-decoder.h"
15 #include "sherpa-onnx/csrc/offline-ctc-model.h" 20 #include "sherpa-onnx/csrc/offline-ctc-model.h"
@@ -46,10 +51,24 @@ class OfflineRecognizerCtcImpl : public OfflineRecognizerImpl { @@ -46,10 +51,24 @@ class OfflineRecognizerCtcImpl : public OfflineRecognizerImpl {
46 : config_(config), 51 : config_(config),
47 symbol_table_(config_.model_config.tokens), 52 symbol_table_(config_.model_config.tokens),
48 model_(OfflineCtcModel::Create(config_.model_config)) { 53 model_(OfflineCtcModel::Create(config_.model_config)) {
  54 + Init();
  55 + }
  56 +
  57 +#if __ANDROID_API__ >= 9
  58 + OfflineRecognizerCtcImpl(AAssetManager *mgr,
  59 + const OfflineRecognizerConfig &config)
  60 + : config_(config),
  61 + symbol_table_(mgr, config_.model_config.tokens),
  62 + model_(OfflineCtcModel::Create(mgr, config_.model_config)) {
  63 + Init();
  64 + }
  65 +#endif
  66 +
  67 + void Init() {
49 config_.feat_config.nemo_normalize_type = 68 config_.feat_config.nemo_normalize_type =
50 model_->FeatureNormalizationMethod(); 69 model_->FeatureNormalizationMethod();
51 70
52 - if (config.decoding_method == "greedy_search") { 71 + if (config_.decoding_method == "greedy_search") {
53 if (!symbol_table_.contains("<blk>") && 72 if (!symbol_table_.contains("<blk>") &&
54 !symbol_table_.contains("<eps>")) { 73 !symbol_table_.contains("<eps>")) {
55 SHERPA_ONNX_LOGE( 74 SHERPA_ONNX_LOGE(
@@ -69,7 +88,7 @@ class OfflineRecognizerCtcImpl : public OfflineRecognizerImpl { @@ -69,7 +88,7 @@ class OfflineRecognizerCtcImpl : public OfflineRecognizerImpl {
69 decoder_ = std::make_unique<OfflineCtcGreedySearchDecoder>(blank_id); 88 decoder_ = std::make_unique<OfflineCtcGreedySearchDecoder>(blank_id);
70 } else { 89 } else {
71 SHERPA_ONNX_LOGE("Only greedy_search is supported at present. Given %s", 90 SHERPA_ONNX_LOGE("Only greedy_search is supported at present. Given %s",
72 - config.decoding_method.c_str()); 91 + config_.decoding_method.c_str());
73 exit(-1); 92 exit(-1);
74 } 93 }
75 } 94 }
@@ -132,4 +132,121 @@ std::unique_ptr<OfflineRecognizerImpl> OfflineRecognizerImpl::Create( @@ -132,4 +132,121 @@ std::unique_ptr<OfflineRecognizerImpl> OfflineRecognizerImpl::Create(
132 exit(-1); 132 exit(-1);
133 } 133 }
134 134
  135 +#if __ANDROID_API__ >= 9
  136 +std::unique_ptr<OfflineRecognizerImpl> OfflineRecognizerImpl::Create(
  137 + AAssetManager *mgr, const OfflineRecognizerConfig &config) {
  138 + if (!config.model_config.model_type.empty()) {
  139 + const auto &model_type = config.model_config.model_type;
  140 + if (model_type == "transducer") {
  141 + return std::make_unique<OfflineRecognizerTransducerImpl>(mgr, config);
  142 + } else if (model_type == "paraformer") {
  143 + return std::make_unique<OfflineRecognizerParaformerImpl>(mgr, config);
  144 + } else if (model_type == "nemo_ctc") {
  145 + return std::make_unique<OfflineRecognizerCtcImpl>(mgr, config);
  146 + } else if (model_type == "tdnn") {
  147 + return std::make_unique<OfflineRecognizerCtcImpl>(mgr, config);
  148 + } else if (model_type == "whisper") {
  149 + return std::make_unique<OfflineRecognizerWhisperImpl>(mgr, config);
  150 + } else {
  151 + SHERPA_ONNX_LOGE(
  152 + "Invalid model_type: %s. Trying to load the model to get its type",
  153 + model_type.c_str());
  154 + }
  155 + }
  156 +
  157 + Ort::Env env(ORT_LOGGING_LEVEL_ERROR);
  158 +
  159 + Ort::SessionOptions sess_opts;
  160 + std::string model_filename;
  161 + if (!config.model_config.transducer.encoder_filename.empty()) {
  162 + model_filename = config.model_config.transducer.encoder_filename;
  163 + } else if (!config.model_config.paraformer.model.empty()) {
  164 + model_filename = config.model_config.paraformer.model;
  165 + } else if (!config.model_config.nemo_ctc.model.empty()) {
  166 + model_filename = config.model_config.nemo_ctc.model;
  167 + } else if (!config.model_config.tdnn.model.empty()) {
  168 + model_filename = config.model_config.tdnn.model;
  169 + } else if (!config.model_config.whisper.encoder.empty()) {
  170 + model_filename = config.model_config.whisper.encoder;
  171 + } else {
  172 + SHERPA_ONNX_LOGE("Please provide a model");
  173 + exit(-1);
  174 + }
  175 +
  176 + auto buf = ReadFile(mgr, model_filename);
  177 +
  178 + auto encoder_sess =
  179 + std::make_unique<Ort::Session>(env, buf.data(), buf.size(), sess_opts);
  180 +
  181 + Ort::ModelMetadata meta_data = encoder_sess->GetModelMetadata();
  182 +
  183 + Ort::AllocatorWithDefaultOptions allocator; // used in the macro below
  184 +
  185 + auto model_type_ptr =
  186 + meta_data.LookupCustomMetadataMapAllocated("model_type", allocator);
  187 + if (!model_type_ptr) {
  188 + SHERPA_ONNX_LOGE(
  189 + "No model_type in the metadata!\n\n"
  190 + "Please refer to the following URLs to add metadata"
  191 + "\n"
  192 + "(0) Transducer models from icefall"
  193 + "\n "
  194 + "https://github.com/k2-fsa/icefall/blob/master/egs/librispeech/ASR/"
  195 + "pruned_transducer_stateless7/export-onnx.py#L303"
  196 + "\n"
  197 + "(1) Nemo CTC models\n "
  198 + "https://huggingface.co/csukuangfj/"
  199 + "sherpa-onnx-nemo-ctc-en-citrinet-512/blob/main/add-model-metadata.py"
  200 + "\n"
  201 + "(2) Paraformer"
  202 + "\n "
  203 + "https://huggingface.co/csukuangfj/"
  204 + "paraformer-onnxruntime-python-example/blob/main/add-model-metadata.py"
  205 + "\n "
  206 + "(3) Whisper"
  207 + "\n "
  208 + "(4) Tdnn models of the yesno recipe from icefall"
  209 + "\n "
  210 + "https://github.com/k2-fsa/icefall/tree/master/egs/yesno/ASR/tdnn"
  211 + "\n"
  212 + "\n");
  213 + exit(-1);
  214 + }
  215 + std::string model_type(model_type_ptr.get());
  216 +
  217 + if (model_type == "conformer" || model_type == "zipformer" ||
  218 + model_type == "zipformer2") {
  219 + return std::make_unique<OfflineRecognizerTransducerImpl>(mgr, config);
  220 + }
  221 +
  222 + if (model_type == "paraformer") {
  223 + return std::make_unique<OfflineRecognizerParaformerImpl>(mgr, config);
  224 + }
  225 +
  226 + if (model_type == "EncDecCTCModelBPE") {
  227 + return std::make_unique<OfflineRecognizerCtcImpl>(mgr, config);
  228 + }
  229 +
  230 + if (model_type == "tdnn") {
  231 + return std::make_unique<OfflineRecognizerCtcImpl>(mgr, config);
  232 + }
  233 +
  234 + if (strncmp(model_type.c_str(), "whisper", 7) == 0) {
  235 + return std::make_unique<OfflineRecognizerWhisperImpl>(mgr, config);
  236 + }
  237 +
  238 + SHERPA_ONNX_LOGE(
  239 + "\nUnsupported model_type: %s\n"
  240 + "We support only the following model types at present: \n"
  241 + " - Non-streaming transducer models from icefall\n"
  242 + " - Non-streaming Paraformer models from FunASR\n"
  243 + " - EncDecCTCModelBPE models from NeMo\n"
  244 + " - Whisper models\n"
  245 + " - Tdnn models\n",
  246 + model_type.c_str());
  247 +
  248 + exit(-1);
  249 +}
  250 +#endif
  251 +
135 } // namespace sherpa_onnx 252 } // namespace sherpa_onnx
@@ -8,6 +8,11 @@ @@ -8,6 +8,11 @@
8 #include <memory> 8 #include <memory>
9 #include <vector> 9 #include <vector>
10 10
  11 +#if __ANDROID_API__ >= 9
  12 +#include "android/asset_manager.h"
  13 +#include "android/asset_manager_jni.h"
  14 +#endif
  15 +
11 #include "sherpa-onnx/csrc/macros.h" 16 #include "sherpa-onnx/csrc/macros.h"
12 #include "sherpa-onnx/csrc/offline-recognizer.h" 17 #include "sherpa-onnx/csrc/offline-recognizer.h"
13 #include "sherpa-onnx/csrc/offline-stream.h" 18 #include "sherpa-onnx/csrc/offline-stream.h"
@@ -19,6 +24,11 @@ class OfflineRecognizerImpl { @@ -19,6 +24,11 @@ class OfflineRecognizerImpl {
19 static std::unique_ptr<OfflineRecognizerImpl> Create( 24 static std::unique_ptr<OfflineRecognizerImpl> Create(
20 const OfflineRecognizerConfig &config); 25 const OfflineRecognizerConfig &config);
21 26
  27 +#if __ANDROID_API__ >= 9
  28 + static std::unique_ptr<OfflineRecognizerImpl> Create(
  29 + AAssetManager *mgr, const OfflineRecognizerConfig &config);
  30 +#endif
  31 +
22 virtual ~OfflineRecognizerImpl() = default; 32 virtual ~OfflineRecognizerImpl() = default;
23 33
24 virtual std::unique_ptr<OfflineStream> CreateStream( 34 virtual std::unique_ptr<OfflineStream> CreateStream(
@@ -11,6 +11,11 @@ @@ -11,6 +11,11 @@
11 #include <utility> 11 #include <utility>
12 #include <vector> 12 #include <vector>
13 13
  14 +#if __ANDROID_API__ >= 9
  15 +#include "android/asset_manager.h"
  16 +#include "android/asset_manager_jni.h"
  17 +#endif
  18 +
14 #include "sherpa-onnx/csrc/offline-model-config.h" 19 #include "sherpa-onnx/csrc/offline-model-config.h"
15 #include "sherpa-onnx/csrc/offline-paraformer-decoder.h" 20 #include "sherpa-onnx/csrc/offline-paraformer-decoder.h"
16 #include "sherpa-onnx/csrc/offline-paraformer-greedy-search-decoder.h" 21 #include "sherpa-onnx/csrc/offline-paraformer-greedy-search-decoder.h"
@@ -100,6 +105,28 @@ class OfflineRecognizerParaformerImpl : public OfflineRecognizerImpl { @@ -100,6 +105,28 @@ class OfflineRecognizerParaformerImpl : public OfflineRecognizerImpl {
100 config_.feat_config.normalize_samples = false; 105 config_.feat_config.normalize_samples = false;
101 } 106 }
102 107
  108 +#if __ANDROID_API__ >= 9
  109 + OfflineRecognizerParaformerImpl(AAssetManager *mgr,
  110 + const OfflineRecognizerConfig &config)
  111 + : config_(config),
  112 + symbol_table_(mgr, config_.model_config.tokens),
  113 + model_(std::make_unique<OfflineParaformerModel>(mgr,
  114 + config.model_config)) {
  115 + if (config.decoding_method == "greedy_search") {
  116 + int32_t eos_id = symbol_table_["</s>"];
  117 + decoder_ = std::make_unique<OfflineParaformerGreedySearchDecoder>(eos_id);
  118 + } else {
  119 + SHERPA_ONNX_LOGE("Only greedy_search is supported at present. Given %s",
  120 + config.decoding_method.c_str());
  121 + exit(-1);
  122 + }
  123 +
  124 + // Paraformer models assume input samples are in the range
  125 + // [-32768, 32767], so we set normalize_samples to false
  126 + config_.feat_config.normalize_samples = false;
  127 + }
  128 +#endif
  129 +
103 std::unique_ptr<OfflineStream> CreateStream() const override { 130 std::unique_ptr<OfflineStream> CreateStream() const override {
104 return std::make_unique<OfflineStream>(config_.feat_config); 131 return std::make_unique<OfflineStream>(config_.feat_config);
105 } 132 }
@@ -10,6 +10,11 @@ @@ -10,6 +10,11 @@
10 #include <utility> 10 #include <utility>
11 #include <vector> 11 #include <vector>
12 12
  13 +#if __ANDROID_API__ >= 9
  14 +#include "android/asset_manager.h"
  15 +#include "android/asset_manager_jni.h"
  16 +#endif
  17 +
13 #include "sherpa-onnx/csrc/context-graph.h" 18 #include "sherpa-onnx/csrc/context-graph.h"
14 #include "sherpa-onnx/csrc/macros.h" 19 #include "sherpa-onnx/csrc/macros.h"
15 #include "sherpa-onnx/csrc/offline-recognizer-impl.h" 20 #include "sherpa-onnx/csrc/offline-recognizer-impl.h"
@@ -73,6 +78,32 @@ class OfflineRecognizerTransducerImpl : public OfflineRecognizerImpl { @@ -73,6 +78,32 @@ class OfflineRecognizerTransducerImpl : public OfflineRecognizerImpl {
73 } 78 }
74 } 79 }
75 80
  81 +#if __ANDROID_API__ >= 9
  82 + explicit OfflineRecognizerTransducerImpl(
  83 + AAssetManager *mgr, const OfflineRecognizerConfig &config)
  84 + : config_(config),
  85 + symbol_table_(mgr, config_.model_config.tokens),
  86 + model_(std::make_unique<OfflineTransducerModel>(mgr,
  87 + config_.model_config)) {
  88 + if (config_.decoding_method == "greedy_search") {
  89 + decoder_ =
  90 + std::make_unique<OfflineTransducerGreedySearchDecoder>(model_.get());
  91 + } else if (config_.decoding_method == "modified_beam_search") {
  92 + if (!config_.lm_config.model.empty()) {
  93 + lm_ = OfflineLM::Create(mgr, config.lm_config);
  94 + }
  95 +
  96 + decoder_ = std::make_unique<OfflineTransducerModifiedBeamSearchDecoder>(
  97 + model_.get(), lm_.get(), config_.max_active_paths,
  98 + config_.lm_config.scale);
  99 + } else {
  100 + SHERPA_ONNX_LOGE("Unsupported decoding method: %s",
  101 + config_.decoding_method.c_str());
  102 + exit(-1);
  103 + }
  104 + }
  105 +#endif
  106 +
76 std::unique_ptr<OfflineStream> CreateStream( 107 std::unique_ptr<OfflineStream> CreateStream(
77 const std::vector<std::vector<int32_t>> &context_list) const override { 108 const std::vector<std::vector<int32_t>> &context_list) const override {
78 // We create context_graph at this level, because we might have default 109 // We create context_graph at this level, because we might have default
@@ -12,6 +12,11 @@ @@ -12,6 +12,11 @@
12 #include <utility> 12 #include <utility>
13 #include <vector> 13 #include <vector>
14 14
  15 +#if __ANDROID_API__ >= 9
  16 +#include "android/asset_manager.h"
  17 +#include "android/asset_manager_jni.h"
  18 +#endif
  19 +
15 #include "sherpa-onnx/csrc/offline-model-config.h" 20 #include "sherpa-onnx/csrc/offline-model-config.h"
16 #include "sherpa-onnx/csrc/offline-recognizer-impl.h" 21 #include "sherpa-onnx/csrc/offline-recognizer-impl.h"
17 #include "sherpa-onnx/csrc/offline-recognizer.h" 22 #include "sherpa-onnx/csrc/offline-recognizer.h"
@@ -253,16 +258,32 @@ class OfflineRecognizerWhisperImpl : public OfflineRecognizerImpl { @@ -253,16 +258,32 @@ class OfflineRecognizerWhisperImpl : public OfflineRecognizerImpl {
253 : config_(config), 258 : config_(config),
254 symbol_table_(config_.model_config.tokens), 259 symbol_table_(config_.model_config.tokens),
255 model_(std::make_unique<OfflineWhisperModel>(config.model_config)) { 260 model_(std::make_unique<OfflineWhisperModel>(config.model_config)) {
  261 + Init();
  262 + }
  263 +
  264 +#if __ANDROID_API__ >= 9
  265 + OfflineRecognizerWhisperImpl(AAssetManager *mgr,
  266 + const OfflineRecognizerConfig &config)
  267 + : config_(config),
  268 + symbol_table_(mgr, config_.model_config.tokens),
  269 + model_(
  270 + std::make_unique<OfflineWhisperModel>(mgr, config.model_config)) {
  271 + Init();
  272 + }
  273 +
  274 +#endif
  275 +
  276 + void Init() {
256 // tokens.txt from whisper is base64 encoded, so we need to decode it 277 // tokens.txt from whisper is base64 encoded, so we need to decode it
257 symbol_table_.ApplyBase64Decode(); 278 symbol_table_.ApplyBase64Decode();
258 279
259 - if (config.decoding_method == "greedy_search") { 280 + if (config_.decoding_method == "greedy_search") {
260 decoder_ = std::make_unique<OfflineWhisperGreedySearchDecoder>( 281 decoder_ = std::make_unique<OfflineWhisperGreedySearchDecoder>(
261 config_.model_config.whisper, model_.get()); 282 config_.model_config.whisper, model_.get());
262 } else { 283 } else {
263 SHERPA_ONNX_LOGE( 284 SHERPA_ONNX_LOGE(
264 "Only greedy_search is supported at present for whisper. Given %s", 285 "Only greedy_search is supported at present for whisper. Given %s",
265 - config.decoding_method.c_str()); 286 + config_.decoding_method.c_str());
266 exit(-1); 287 exit(-1);
267 } 288 }
268 } 289 }
@@ -58,6 +58,12 @@ std::string OfflineRecognizerConfig::ToString() const { @@ -58,6 +58,12 @@ std::string OfflineRecognizerConfig::ToString() const {
58 return os.str(); 58 return os.str();
59 } 59 }
60 60
  61 +#if __ANDROID_API__ >= 9
  62 +OfflineRecognizer::OfflineRecognizer(AAssetManager *mgr,
  63 + const OfflineRecognizerConfig &config)
  64 + : impl_(OfflineRecognizerImpl::Create(mgr, config)) {}
  65 +#endif
  66 +
61 OfflineRecognizer::OfflineRecognizer(const OfflineRecognizerConfig &config) 67 OfflineRecognizer::OfflineRecognizer(const OfflineRecognizerConfig &config)
62 : impl_(OfflineRecognizerImpl::Create(config)) {} 68 : impl_(OfflineRecognizerImpl::Create(config)) {}
63 69
@@ -9,6 +9,11 @@ @@ -9,6 +9,11 @@
9 #include <string> 9 #include <string>
10 #include <vector> 10 #include <vector>
11 11
  12 +#if __ANDROID_API__ >= 9
  13 +#include "android/asset_manager.h"
  14 +#include "android/asset_manager_jni.h"
  15 +#endif
  16 +
12 #include "sherpa-onnx/csrc/offline-lm-config.h" 17 #include "sherpa-onnx/csrc/offline-lm-config.h"
13 #include "sherpa-onnx/csrc/offline-model-config.h" 18 #include "sherpa-onnx/csrc/offline-model-config.h"
14 #include "sherpa-onnx/csrc/offline-stream.h" 19 #include "sherpa-onnx/csrc/offline-stream.h"
@@ -55,6 +60,10 @@ class OfflineRecognizer { @@ -55,6 +60,10 @@ class OfflineRecognizer {
55 public: 60 public:
56 ~OfflineRecognizer(); 61 ~OfflineRecognizer();
57 62
  63 +#if __ANDROID_API__ >= 9
  64 + OfflineRecognizer(AAssetManager *mgr, const OfflineRecognizerConfig &config);
  65 +#endif
  66 +
58 explicit OfflineRecognizer(const OfflineRecognizerConfig &config); 67 explicit OfflineRecognizer(const OfflineRecognizerConfig &config);
59 68
60 /// Create a stream for decoding. 69 /// Create a stream for decoding.
@@ -11,8 +11,8 @@ @@ -11,8 +11,8 @@
11 #include "onnxruntime_cxx_api.h" // NOLINT 11 #include "onnxruntime_cxx_api.h" // NOLINT
12 #include "sherpa-onnx/csrc/macros.h" 12 #include "sherpa-onnx/csrc/macros.h"
13 #include "sherpa-onnx/csrc/onnx-utils.h" 13 #include "sherpa-onnx/csrc/onnx-utils.h"
14 -#include "sherpa-onnx/csrc/text-utils.h"  
15 #include "sherpa-onnx/csrc/session.h" 14 #include "sherpa-onnx/csrc/session.h"
  15 +#include "sherpa-onnx/csrc/text-utils.h"
16 16
17 namespace sherpa_onnx { 17 namespace sherpa_onnx {
18 18
@@ -23,8 +23,20 @@ class OfflineRnnLM::Impl { @@ -23,8 +23,20 @@ class OfflineRnnLM::Impl {
23 env_(ORT_LOGGING_LEVEL_ERROR), 23 env_(ORT_LOGGING_LEVEL_ERROR),
24 sess_opts_{GetSessionOptions(config)}, 24 sess_opts_{GetSessionOptions(config)},
25 allocator_{} { 25 allocator_{} {
26 - Init(config); 26 + auto buf = ReadFile(config_.model);
  27 + Init(buf.data(), buf.size());
  28 + }
  29 +
  30 +#if __ANDROID_API__ >= 9
  31 + Impl(AAssetManager *mgr, const OfflineLMConfig &config)
  32 + : config_(config),
  33 + env_(ORT_LOGGING_LEVEL_ERROR),
  34 + sess_opts_{GetSessionOptions(config)},
  35 + allocator_{} {
  36 + auto buf = ReadFile(mgr, config_.model);
  37 + Init(buf.data(), buf.size());
27 } 38 }
  39 +#endif
28 40
29 Ort::Value Rescore(Ort::Value x, Ort::Value x_lens) { 41 Ort::Value Rescore(Ort::Value x, Ort::Value x_lens) {
30 std::array<Ort::Value, 2> inputs = {std::move(x), std::move(x_lens)}; 42 std::array<Ort::Value, 2> inputs = {std::move(x), std::move(x_lens)};
@@ -37,10 +49,8 @@ class OfflineRnnLM::Impl { @@ -37,10 +49,8 @@ class OfflineRnnLM::Impl {
37 } 49 }
38 50
39 private: 51 private:
40 - void Init(const OfflineLMConfig &config) {  
41 - auto buf = ReadFile(config_.model);  
42 -  
43 - sess_ = std::make_unique<Ort::Session>(env_, buf.data(), buf.size(), 52 + void Init(void *model_data, size_t model_data_length) {
  53 + sess_ = std::make_unique<Ort::Session>(env_, model_data, model_data_length,
44 sess_opts_); 54 sess_opts_);
45 55
46 GetInputNames(sess_.get(), &input_names_, &input_names_ptr_); 56 GetInputNames(sess_.get(), &input_names_, &input_names_ptr_);
@@ -66,6 +76,11 @@ class OfflineRnnLM::Impl { @@ -66,6 +76,11 @@ class OfflineRnnLM::Impl {
66 OfflineRnnLM::OfflineRnnLM(const OfflineLMConfig &config) 76 OfflineRnnLM::OfflineRnnLM(const OfflineLMConfig &config)
67 : impl_(std::make_unique<Impl>(config)) {} 77 : impl_(std::make_unique<Impl>(config)) {}
68 78
  79 +#if __ANDROID_API__ >= 9
  80 +OfflineRnnLM::OfflineRnnLM(AAssetManager *mgr, const OfflineLMConfig &config)
  81 + : impl_(std::make_unique<Impl>(mgr, config)) {}
  82 +#endif
  83 +
69 OfflineRnnLM::~OfflineRnnLM() = default; 84 OfflineRnnLM::~OfflineRnnLM() = default;
70 85
71 Ort::Value OfflineRnnLM::Rescore(Ort::Value x, Ort::Value x_lens) { 86 Ort::Value OfflineRnnLM::Rescore(Ort::Value x, Ort::Value x_lens) {
@@ -7,6 +7,11 @@ @@ -7,6 +7,11 @@
7 7
8 #include <memory> 8 #include <memory>
9 9
  10 +#if __ANDROID_API__ >= 9
  11 +#include "android/asset_manager.h"
  12 +#include "android/asset_manager_jni.h"
  13 +#endif
  14 +
10 #include "onnxruntime_cxx_api.h" // NOLINT 15 #include "onnxruntime_cxx_api.h" // NOLINT
11 #include "sherpa-onnx/csrc/offline-lm-config.h" 16 #include "sherpa-onnx/csrc/offline-lm-config.h"
12 #include "sherpa-onnx/csrc/offline-lm.h" 17 #include "sherpa-onnx/csrc/offline-lm.h"
@@ -19,6 +24,10 @@ class OfflineRnnLM : public OfflineLM { @@ -19,6 +24,10 @@ class OfflineRnnLM : public OfflineLM {
19 24
20 explicit OfflineRnnLM(const OfflineLMConfig &config); 25 explicit OfflineRnnLM(const OfflineLMConfig &config);
21 26
  27 +#if __ANDROID_API__ >= 9
  28 + OfflineRnnLM(AAssetManager *mgr, const OfflineLMConfig &config);
  29 +#endif
  30 +
22 /** Rescore a batch of sentences. 31 /** Rescore a batch of sentences.
23 * 32 *
24 * @param x A 2-D tensor of shape (N, L) with data type int64. 33 * @param x A 2-D tensor of shape (N, L) with data type int64.
@@ -19,8 +19,20 @@ class OfflineTdnnCtcModel::Impl { @@ -19,8 +19,20 @@ class OfflineTdnnCtcModel::Impl {
19 env_(ORT_LOGGING_LEVEL_ERROR), 19 env_(ORT_LOGGING_LEVEL_ERROR),
20 sess_opts_(GetSessionOptions(config)), 20 sess_opts_(GetSessionOptions(config)),
21 allocator_{} { 21 allocator_{} {
22 - Init(); 22 + auto buf = ReadFile(config_.tdnn.model);
  23 + Init(buf.data(), buf.size());
  24 + }
  25 +
  26 +#if __ANDROID_API__ >= 9
  27 + Impl(AAssetManager *mgr, const OfflineModelConfig &config)
  28 + : config_(config),
  29 + env_(ORT_LOGGING_LEVEL_ERROR),
  30 + sess_opts_(GetSessionOptions(config)),
  31 + allocator_{} {
  32 + auto buf = ReadFile(mgr, config_.tdnn.model);
  33 + Init(buf.data(), buf.size());
23 } 34 }
  35 +#endif
24 36
25 std::pair<Ort::Value, Ort::Value> Forward(Ort::Value features) { 37 std::pair<Ort::Value, Ort::Value> Forward(Ort::Value features) {
26 auto nnet_out = 38 auto nnet_out =
@@ -48,10 +60,8 @@ class OfflineTdnnCtcModel::Impl { @@ -48,10 +60,8 @@ class OfflineTdnnCtcModel::Impl {
48 OrtAllocator *Allocator() const { return allocator_; } 60 OrtAllocator *Allocator() const { return allocator_; }
49 61
50 private: 62 private:
51 - void Init() {  
52 - auto buf = ReadFile(config_.tdnn.model);  
53 -  
54 - sess_ = std::make_unique<Ort::Session>(env_, buf.data(), buf.size(), 63 + void Init(void *model_data, size_t model_data_length) {
  64 + sess_ = std::make_unique<Ort::Session>(env_, model_data, model_data_length,
55 sess_opts_); 65 sess_opts_);
56 66
57 GetInputNames(sess_.get(), &input_names_, &input_names_ptr_); 67 GetInputNames(sess_.get(), &input_names_, &input_names_ptr_);
@@ -90,6 +100,12 @@ class OfflineTdnnCtcModel::Impl { @@ -90,6 +100,12 @@ class OfflineTdnnCtcModel::Impl {
90 OfflineTdnnCtcModel::OfflineTdnnCtcModel(const OfflineModelConfig &config) 100 OfflineTdnnCtcModel::OfflineTdnnCtcModel(const OfflineModelConfig &config)
91 : impl_(std::make_unique<Impl>(config)) {} 101 : impl_(std::make_unique<Impl>(config)) {}
92 102
  103 +#if __ANDROID_API__ >= 9
  104 +OfflineTdnnCtcModel::OfflineTdnnCtcModel(AAssetManager *mgr,
  105 + const OfflineModelConfig &config)
  106 + : impl_(std::make_unique<Impl>(mgr, config)) {}
  107 +#endif
  108 +
93 OfflineTdnnCtcModel::~OfflineTdnnCtcModel() = default; 109 OfflineTdnnCtcModel::~OfflineTdnnCtcModel() = default;
94 110
95 std::pair<Ort::Value, Ort::Value> OfflineTdnnCtcModel::Forward( 111 std::pair<Ort::Value, Ort::Value> OfflineTdnnCtcModel::Forward(
@@ -8,6 +8,11 @@ @@ -8,6 +8,11 @@
8 #include <utility> 8 #include <utility>
9 #include <vector> 9 #include <vector>
10 10
  11 +#if __ANDROID_API__ >= 9
  12 +#include "android/asset_manager.h"
  13 +#include "android/asset_manager_jni.h"
  14 +#endif
  15 +
11 #include "onnxruntime_cxx_api.h" // NOLINT 16 #include "onnxruntime_cxx_api.h" // NOLINT
12 #include "sherpa-onnx/csrc/offline-ctc-model.h" 17 #include "sherpa-onnx/csrc/offline-ctc-model.h"
13 #include "sherpa-onnx/csrc/offline-model-config.h" 18 #include "sherpa-onnx/csrc/offline-model-config.h"
@@ -22,6 +27,11 @@ namespace sherpa_onnx { @@ -22,6 +27,11 @@ namespace sherpa_onnx {
22 class OfflineTdnnCtcModel : public OfflineCtcModel { 27 class OfflineTdnnCtcModel : public OfflineCtcModel {
23 public: 28 public:
24 explicit OfflineTdnnCtcModel(const OfflineModelConfig &config); 29 explicit OfflineTdnnCtcModel(const OfflineModelConfig &config);
  30 +
  31 +#if __ANDROID_API__ >= 9
  32 + OfflineTdnnCtcModel(AAssetManager *mgr, const OfflineModelConfig &config);
  33 +#endif
  34 +
25 ~OfflineTdnnCtcModel() override; 35 ~OfflineTdnnCtcModel() override;
26 36
27 /** Run the forward method of the model. 37 /** Run the forward method of the model.
@@ -38,6 +38,29 @@ class OfflineTransducerModel::Impl { @@ -38,6 +38,29 @@ class OfflineTransducerModel::Impl {
38 } 38 }
39 } 39 }
40 40
  41 +#if __ANDROID_API__ >= 9
  42 + Impl(AAssetManager *mgr, const OfflineModelConfig &config)
  43 + : config_(config),
  44 + env_(ORT_LOGGING_LEVEL_WARNING),
  45 + sess_opts_(GetSessionOptions(config)),
  46 + allocator_{} {
  47 + {
  48 + auto buf = ReadFile(mgr, config.transducer.encoder_filename);
  49 + InitEncoder(buf.data(), buf.size());
  50 + }
  51 +
  52 + {
  53 + auto buf = ReadFile(mgr, config.transducer.decoder_filename);
  54 + InitDecoder(buf.data(), buf.size());
  55 + }
  56 +
  57 + {
  58 + auto buf = ReadFile(mgr, config.transducer.joiner_filename);
  59 + InitJoiner(buf.data(), buf.size());
  60 + }
  61 + }
  62 +#endif
  63 +
41 std::pair<Ort::Value, Ort::Value> RunEncoder(Ort::Value features, 64 std::pair<Ort::Value, Ort::Value> RunEncoder(Ort::Value features,
42 Ort::Value features_length) { 65 Ort::Value features_length) {
43 std::array<Ort::Value, 2> encoder_inputs = {std::move(features), 66 std::array<Ort::Value, 2> encoder_inputs = {std::move(features),
@@ -221,6 +244,12 @@ class OfflineTransducerModel::Impl { @@ -221,6 +244,12 @@ class OfflineTransducerModel::Impl {
221 OfflineTransducerModel::OfflineTransducerModel(const OfflineModelConfig &config) 244 OfflineTransducerModel::OfflineTransducerModel(const OfflineModelConfig &config)
222 : impl_(std::make_unique<Impl>(config)) {} 245 : impl_(std::make_unique<Impl>(config)) {}
223 246
  247 +#if __ANDROID_API__ >= 9
  248 +OfflineTransducerModel::OfflineTransducerModel(AAssetManager *mgr,
  249 + const OfflineModelConfig &config)
  250 + : impl_(std::make_unique<Impl>(mgr, config)) {}
  251 +#endif
  252 +
224 OfflineTransducerModel::~OfflineTransducerModel() = default; 253 OfflineTransducerModel::~OfflineTransducerModel() = default;
225 254
226 std::pair<Ort::Value, Ort::Value> OfflineTransducerModel::RunEncoder( 255 std::pair<Ort::Value, Ort::Value> OfflineTransducerModel::RunEncoder(
@@ -8,6 +8,11 @@ @@ -8,6 +8,11 @@
8 #include <utility> 8 #include <utility>
9 #include <vector> 9 #include <vector>
10 10
  11 +#if __ANDROID_API__ >= 9
  12 +#include "android/asset_manager.h"
  13 +#include "android/asset_manager_jni.h"
  14 +#endif
  15 +
11 #include "onnxruntime_cxx_api.h" // NOLINT 16 #include "onnxruntime_cxx_api.h" // NOLINT
12 #include "sherpa-onnx/csrc/hypothesis.h" 17 #include "sherpa-onnx/csrc/hypothesis.h"
13 #include "sherpa-onnx/csrc/offline-model-config.h" 18 #include "sherpa-onnx/csrc/offline-model-config.h"
@@ -19,6 +24,11 @@ struct OfflineTransducerDecoderResult; @@ -19,6 +24,11 @@ struct OfflineTransducerDecoderResult;
19 class OfflineTransducerModel { 24 class OfflineTransducerModel {
20 public: 25 public:
21 explicit OfflineTransducerModel(const OfflineModelConfig &config); 26 explicit OfflineTransducerModel(const OfflineModelConfig &config);
  27 +
  28 +#if __ANDROID_API__ >= 9
  29 + OfflineTransducerModel(AAssetManager *mgr, const OfflineModelConfig &config);
  30 +#endif
  31 +
22 ~OfflineTransducerModel(); 32 ~OfflineTransducerModel();
23 33
24 /** Run the encoder. 34 /** Run the encoder.
@@ -35,6 +35,24 @@ class OfflineWhisperModel::Impl { @@ -35,6 +35,24 @@ class OfflineWhisperModel::Impl {
35 } 35 }
36 } 36 }
37 37
  38 +#if __ANDROID_API__ >= 9
  39 + Impl(AAssetManager *mgr, const OfflineModelConfig &config)
  40 + : config_(config),
  41 + env_(ORT_LOGGING_LEVEL_ERROR),
  42 + sess_opts_(GetSessionOptions(config)),
  43 + allocator_{} {
  44 + {
  45 + auto buf = ReadFile(mgr, config.whisper.encoder);
  46 + InitEncoder(buf.data(), buf.size());
  47 + }
  48 +
  49 + {
  50 + auto buf = ReadFile(mgr, config.whisper.decoder);
  51 + InitDecoder(buf.data(), buf.size());
  52 + }
  53 + }
  54 +#endif
  55 +
38 std::pair<Ort::Value, Ort::Value> ForwardEncoder(Ort::Value features) { 56 std::pair<Ort::Value, Ort::Value> ForwardEncoder(Ort::Value features) {
39 auto encoder_out = encoder_sess_->Run( 57 auto encoder_out = encoder_sess_->Run(
40 {}, encoder_input_names_ptr_.data(), &features, 1, 58 {}, encoder_input_names_ptr_.data(), &features, 1,
@@ -226,6 +244,12 @@ class OfflineWhisperModel::Impl { @@ -226,6 +244,12 @@ class OfflineWhisperModel::Impl {
226 OfflineWhisperModel::OfflineWhisperModel(const OfflineModelConfig &config) 244 OfflineWhisperModel::OfflineWhisperModel(const OfflineModelConfig &config)
227 : impl_(std::make_unique<Impl>(config)) {} 245 : impl_(std::make_unique<Impl>(config)) {}
228 246
  247 +#if __ANDROID_API__ >= 9
  248 +OfflineWhisperModel::OfflineWhisperModel(AAssetManager *mgr,
  249 + const OfflineModelConfig &config)
  250 + : impl_(std::make_unique<Impl>(mgr, config)) {}
  251 +#endif
  252 +
229 OfflineWhisperModel::~OfflineWhisperModel() = default; 253 OfflineWhisperModel::~OfflineWhisperModel() = default;
230 254
231 std::pair<Ort::Value, Ort::Value> OfflineWhisperModel::ForwardEncoder( 255 std::pair<Ort::Value, Ort::Value> OfflineWhisperModel::ForwardEncoder(
@@ -11,6 +11,11 @@ @@ -11,6 +11,11 @@
11 #include <utility> 11 #include <utility>
12 #include <vector> 12 #include <vector>
13 13
  14 +#if __ANDROID_API__ >= 9
  15 +#include "android/asset_manager.h"
  16 +#include "android/asset_manager_jni.h"
  17 +#endif
  18 +
14 #include "onnxruntime_cxx_api.h" // NOLINT 19 #include "onnxruntime_cxx_api.h" // NOLINT
15 #include "sherpa-onnx/csrc/offline-model-config.h" 20 #include "sherpa-onnx/csrc/offline-model-config.h"
16 21
@@ -19,6 +24,11 @@ namespace sherpa_onnx { @@ -19,6 +24,11 @@ namespace sherpa_onnx {
19 class OfflineWhisperModel { 24 class OfflineWhisperModel {
20 public: 25 public:
21 explicit OfflineWhisperModel(const OfflineModelConfig &config); 26 explicit OfflineWhisperModel(const OfflineModelConfig &config);
  27 +
  28 +#if __ANDROID_API__ >= 9
  29 + OfflineWhisperModel(AAssetManager *mgr, const OfflineModelConfig &config);
  30 +#endif
  31 +
22 ~OfflineWhisperModel(); 32 ~OfflineWhisperModel();
23 33
24 /** Run the encoder model. 34 /** Run the encoder model.
@@ -20,6 +20,7 @@ @@ -20,6 +20,7 @@
20 #include <fstream> 20 #include <fstream>
21 21
22 #include "sherpa-onnx/csrc/macros.h" 22 #include "sherpa-onnx/csrc/macros.h"
  23 +#include "sherpa-onnx/csrc/offline-recognizer.h"
23 #include "sherpa-onnx/csrc/online-recognizer.h" 24 #include "sherpa-onnx/csrc/online-recognizer.h"
24 #include "sherpa-onnx/csrc/onnx-utils.h" 25 #include "sherpa-onnx/csrc/onnx-utils.h"
25 #include "sherpa-onnx/csrc/wave-reader.h" 26 #include "sherpa-onnx/csrc/wave-reader.h"
@@ -53,7 +54,7 @@ class SherpaOnnx { @@ -53,7 +54,7 @@ class SherpaOnnx {
53 stream_->InputFinished(); 54 stream_->InputFinished();
54 } 55 }
55 56
56 - const std::string GetText() const { 57 + std::string GetText() const {
57 auto result = recognizer_.GetResult(stream_.get()); 58 auto result = recognizer_.GetResult(stream_.get());
58 return result.text; 59 return result.text;
59 } 60 }
@@ -67,7 +68,13 @@ class SherpaOnnx { @@ -67,7 +68,13 @@ class SherpaOnnx {
67 68
68 bool IsReady() const { return recognizer_.IsReady(stream_.get()); } 69 bool IsReady() const { return recognizer_.IsReady(stream_.get()); }
69 70
70 - void Reset() const { return recognizer_.Reset(stream_.get()); } 71 + void Reset(bool recreate) {
  72 + if (recreate) {
  73 + stream_ = recognizer_.CreateStream();
  74 + } else {
  75 + recognizer_.Reset(stream_.get());
  76 + }
  77 + }
71 78
72 void Decode() const { recognizer_.DecodeStream(stream_.get()); } 79 void Decode() const { recognizer_.DecodeStream(stream_.get()); }
73 80
@@ -77,6 +84,28 @@ class SherpaOnnx { @@ -77,6 +84,28 @@ class SherpaOnnx {
77 int32_t input_sample_rate_ = -1; 84 int32_t input_sample_rate_ = -1;
78 }; 85 };
79 86
  87 +class SherpaOnnxOffline {
  88 + public:
  89 +#if __ANDROID_API__ >= 9
  90 + SherpaOnnxOffline(AAssetManager *mgr, const OfflineRecognizerConfig &config)
  91 + : recognizer_(mgr, config) {}
  92 +#endif
  93 +
  94 + explicit SherpaOnnxOffline(const OfflineRecognizerConfig &config)
  95 + : recognizer_(config) {}
  96 +
  97 + std::string Decode(int32_t sample_rate, const float *samples, int32_t n) {
  98 + auto stream = recognizer_.CreateStream();
  99 + stream->AcceptWaveform(sample_rate, samples, n);
  100 +
  101 + recognizer_.DecodeStream(stream.get());
  102 + return stream->GetResult().text;
  103 + }
  104 +
  105 + private:
  106 + OfflineRecognizer recognizer_;
  107 +};
  108 +
80 static OnlineRecognizerConfig GetConfig(JNIEnv *env, jobject config) { 109 static OnlineRecognizerConfig GetConfig(JNIEnv *env, jobject config) {
81 OnlineRecognizerConfig ans; 110 OnlineRecognizerConfig ans;
82 111
@@ -248,6 +277,122 @@ static OnlineRecognizerConfig GetConfig(JNIEnv *env, jobject config) { @@ -248,6 +277,122 @@ static OnlineRecognizerConfig GetConfig(JNIEnv *env, jobject config) {
248 return ans; 277 return ans;
249 } 278 }
250 279
  280 +static OfflineRecognizerConfig GetOfflineConfig(JNIEnv *env, jobject config) {
  281 + OfflineRecognizerConfig ans;
  282 +
  283 + jclass cls = env->GetObjectClass(config);
  284 + jfieldID fid;
  285 +
  286 + //---------- decoding ----------
  287 + fid = env->GetFieldID(cls, "decodingMethod", "Ljava/lang/String;");
  288 + jstring s = (jstring)env->GetObjectField(config, fid);
  289 + const char *p = env->GetStringUTFChars(s, nullptr);
  290 + ans.decoding_method = p;
  291 + env->ReleaseStringUTFChars(s, p);
  292 +
  293 + fid = env->GetFieldID(cls, "maxActivePaths", "I");
  294 + ans.max_active_paths = env->GetIntField(config, fid);
  295 +
  296 + //---------- feat config ----------
  297 + fid = env->GetFieldID(cls, "featConfig",
  298 + "Lcom/k2fsa/sherpa/onnx/FeatureConfig;");
  299 + jobject feat_config = env->GetObjectField(config, fid);
  300 + jclass feat_config_cls = env->GetObjectClass(feat_config);
  301 +
  302 + fid = env->GetFieldID(feat_config_cls, "sampleRate", "I");
  303 + ans.feat_config.sampling_rate = env->GetIntField(feat_config, fid);
  304 +
  305 + fid = env->GetFieldID(feat_config_cls, "featureDim", "I");
  306 + ans.feat_config.feature_dim = env->GetIntField(feat_config, fid);
  307 +
  308 + //---------- model config ----------
  309 + fid = env->GetFieldID(cls, "modelConfig",
  310 + "Lcom/k2fsa/sherpa/onnx/OfflineModelConfig;");
  311 + jobject model_config = env->GetObjectField(config, fid);
  312 + jclass model_config_cls = env->GetObjectClass(model_config);
  313 +
  314 + fid = env->GetFieldID(model_config_cls, "tokens", "Ljava/lang/String;");
  315 + s = (jstring)env->GetObjectField(model_config, fid);
  316 + p = env->GetStringUTFChars(s, nullptr);
  317 + ans.model_config.tokens = p;
  318 + env->ReleaseStringUTFChars(s, p);
  319 +
  320 + fid = env->GetFieldID(model_config_cls, "numThreads", "I");
  321 + ans.model_config.num_threads = env->GetIntField(model_config, fid);
  322 +
  323 + fid = env->GetFieldID(model_config_cls, "debug", "Z");
  324 + ans.model_config.debug = env->GetBooleanField(model_config, fid);
  325 +
  326 + fid = env->GetFieldID(model_config_cls, "provider", "Ljava/lang/String;");
  327 + s = (jstring)env->GetObjectField(model_config, fid);
  328 + p = env->GetStringUTFChars(s, nullptr);
  329 + ans.model_config.provider = p;
  330 + env->ReleaseStringUTFChars(s, p);
  331 +
  332 + fid = env->GetFieldID(model_config_cls, "modelType", "Ljava/lang/String;");
  333 + s = (jstring)env->GetObjectField(model_config, fid);
  334 + p = env->GetStringUTFChars(s, nullptr);
  335 + ans.model_config.model_type = p;
  336 + env->ReleaseStringUTFChars(s, p);
  337 +
  338 + // transducer
  339 + fid = env->GetFieldID(model_config_cls, "transducer",
  340 + "Lcom/k2fsa/sherpa/onnx/OfflineTransducerModelConfig;");
  341 + jobject transducer_config = env->GetObjectField(model_config, fid);
  342 + jclass transducer_config_cls = env->GetObjectClass(transducer_config);
  343 +
  344 + fid = env->GetFieldID(transducer_config_cls, "encoder", "Ljava/lang/String;");
  345 + s = (jstring)env->GetObjectField(transducer_config, fid);
  346 + p = env->GetStringUTFChars(s, nullptr);
  347 + ans.model_config.transducer.encoder_filename = p;
  348 + env->ReleaseStringUTFChars(s, p);
  349 +
  350 + fid = env->GetFieldID(transducer_config_cls, "decoder", "Ljava/lang/String;");
  351 + s = (jstring)env->GetObjectField(transducer_config, fid);
  352 + p = env->GetStringUTFChars(s, nullptr);
  353 + ans.model_config.transducer.decoder_filename = p;
  354 + env->ReleaseStringUTFChars(s, p);
  355 +
  356 + fid = env->GetFieldID(transducer_config_cls, "joiner", "Ljava/lang/String;");
  357 + s = (jstring)env->GetObjectField(transducer_config, fid);
  358 + p = env->GetStringUTFChars(s, nullptr);
  359 + ans.model_config.transducer.joiner_filename = p;
  360 + env->ReleaseStringUTFChars(s, p);
  361 +
  362 + // paraformer
  363 + fid = env->GetFieldID(model_config_cls, "paraformer",
  364 + "Lcom/k2fsa/sherpa/onnx/OfflineParaformerModelConfig;");
  365 + jobject paraformer_config = env->GetObjectField(model_config, fid);
  366 + jclass paraformer_config_cls = env->GetObjectClass(paraformer_config);
  367 +
  368 + fid = env->GetFieldID(paraformer_config_cls, "model", "Ljava/lang/String;");
  369 +
  370 + s = (jstring)env->GetObjectField(paraformer_config, fid);
  371 + p = env->GetStringUTFChars(s, nullptr);
  372 + ans.model_config.paraformer.model = p;
  373 + env->ReleaseStringUTFChars(s, p);
  374 +
  375 + // whisper
  376 + fid = env->GetFieldID(model_config_cls, "whisper",
  377 + "Lcom/k2fsa/sherpa/onnx/OfflineWhisperModelConfig;");
  378 + jobject whisper_config = env->GetObjectField(model_config, fid);
  379 + jclass whisper_config_cls = env->GetObjectClass(whisper_config);
  380 +
  381 + fid = env->GetFieldID(whisper_config_cls, "encoder", "Ljava/lang/String;");
  382 + s = (jstring)env->GetObjectField(whisper_config, fid);
  383 + p = env->GetStringUTFChars(s, nullptr);
  384 + ans.model_config.whisper.encoder = p;
  385 + env->ReleaseStringUTFChars(s, p);
  386 +
  387 + fid = env->GetFieldID(whisper_config_cls, "decoder", "Ljava/lang/String;");
  388 + s = (jstring)env->GetObjectField(whisper_config, fid);
  389 + p = env->GetStringUTFChars(s, nullptr);
  390 + ans.model_config.whisper.decoder = p;
  391 + env->ReleaseStringUTFChars(s, p);
  392 +
  393 + return ans;
  394 +}
  395 +
251 } // namespace sherpa_onnx 396 } // namespace sherpa_onnx
252 397
253 SHERPA_ONNX_EXTERN_C 398 SHERPA_ONNX_EXTERN_C
@@ -287,10 +432,48 @@ JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_SherpaOnnx_delete( @@ -287,10 +432,48 @@ JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_SherpaOnnx_delete(
287 } 432 }
288 433
289 SHERPA_ONNX_EXTERN_C 434 SHERPA_ONNX_EXTERN_C
290 -JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_SherpaOnnx_reset( 435 +JNIEXPORT jlong JNICALL Java_com_k2fsa_sherpa_onnx_SherpaOnnxOffline_new(
  436 + JNIEnv *env, jobject /*obj*/, jobject asset_manager, jobject _config) {
  437 +#if __ANDROID_API__ >= 9
  438 + AAssetManager *mgr = AAssetManager_fromJava(env, asset_manager);
  439 + if (!mgr) {
  440 + SHERPA_ONNX_LOGE("Failed to get asset manager: %p", mgr);
  441 + }
  442 +#endif
  443 + auto config = sherpa_onnx::GetOfflineConfig(env, _config);
  444 + SHERPA_ONNX_LOGE("config:\n%s", config.ToString().c_str());
  445 + auto model = new sherpa_onnx::SherpaOnnxOffline(
  446 +#if __ANDROID_API__ >= 9
  447 + mgr,
  448 +#endif
  449 + config);
  450 +
  451 + return (jlong)model;
  452 +}
  453 +
  454 +SHERPA_ONNX_EXTERN_C
  455 +JNIEXPORT jlong JNICALL
  456 +Java_com_k2fsa_sherpa_onnx_SherpaOnnxOffline_newFromFile(JNIEnv *env,
  457 + jobject /*obj*/,
  458 + jobject _config) {
  459 + auto config = sherpa_onnx::GetOfflineConfig(env, _config);
  460 + SHERPA_ONNX_LOGE("config:\n%s", config.ToString().c_str());
  461 + auto model = new sherpa_onnx::SherpaOnnxOffline(config);
  462 +
  463 + return (jlong)model;
  464 +}
  465 +
  466 +SHERPA_ONNX_EXTERN_C
  467 +JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_SherpaOnnxOffline_delete(
291 JNIEnv *env, jobject /*obj*/, jlong ptr) { 468 JNIEnv *env, jobject /*obj*/, jlong ptr) {
  469 + delete reinterpret_cast<sherpa_onnx::SherpaOnnxOffline *>(ptr);
  470 +}
  471 +
  472 +SHERPA_ONNX_EXTERN_C
  473 +JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_SherpaOnnx_reset(
  474 + JNIEnv *env, jobject /*obj*/, jlong ptr, jboolean recreate) {
292 auto model = reinterpret_cast<sherpa_onnx::SherpaOnnx *>(ptr); 475 auto model = reinterpret_cast<sherpa_onnx::SherpaOnnx *>(ptr);
293 - model->Reset(); 476 + model->Reset(recreate);
294 } 477 }
295 478
296 SHERPA_ONNX_EXTERN_C 479 SHERPA_ONNX_EXTERN_C
@@ -329,6 +512,22 @@ JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_SherpaOnnx_acceptWaveform( @@ -329,6 +512,22 @@ JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_SherpaOnnx_acceptWaveform(
329 } 512 }
330 513
331 SHERPA_ONNX_EXTERN_C 514 SHERPA_ONNX_EXTERN_C
  515 +JNIEXPORT jstring JNICALL Java_com_k2fsa_sherpa_onnx_SherpaOnnxOffline_decode(
  516 + JNIEnv *env, jobject /*obj*/, jlong ptr, jfloatArray samples,
  517 + jint sample_rate) {
  518 + auto model = reinterpret_cast<sherpa_onnx::SherpaOnnxOffline *>(ptr);
  519 +
  520 + jfloat *p = env->GetFloatArrayElements(samples, nullptr);
  521 + jsize n = env->GetArrayLength(samples);
  522 +
  523 + auto text = model->Decode(sample_rate, p, n);
  524 +
  525 + env->ReleaseFloatArrayElements(samples, p, JNI_ABORT);
  526 +
  527 + return env->NewStringUTF(text.c_str());
  528 +}
  529 +
  530 +SHERPA_ONNX_EXTERN_C
332 JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_SherpaOnnx_inputFinished( 531 JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_SherpaOnnx_inputFinished(
333 JNIEnv *env, jobject /*obj*/, jlong ptr) { 532 JNIEnv *env, jobject /*obj*/, jlong ptr) {
334 reinterpret_cast<sherpa_onnx::SherpaOnnx *>(ptr)->InputFinished(); 533 reinterpret_cast<sherpa_onnx::SherpaOnnx *>(ptr)->InputFinished();
  1 +decode-file
  2 +decode-file-non-streaming
@@ -175,7 +175,7 @@ class SherpaOnnxRecognizer { @@ -175,7 +175,7 @@ class SherpaOnnxRecognizer {
175 let recognizer: OpaquePointer! 175 let recognizer: OpaquePointer!
176 let stream: OpaquePointer! 176 let stream: OpaquePointer!
177 177
178 - /// Constructor taking a model config and a decoder config. 178 + /// Constructor taking a model config
179 init( 179 init(
180 config: UnsafePointer<SherpaOnnxOnlineRecognizerConfig>! 180 config: UnsafePointer<SherpaOnnxOnlineRecognizerConfig>!
181 ) { 181 ) {
@@ -198,8 +198,7 @@ class SherpaOnnxRecognizer { @@ -198,8 +198,7 @@ class SherpaOnnxRecognizer {
198 /// - Parameters: 198 /// - Parameters:
199 /// - samples: Audio samples normalized to the range [-1, 1] 199 /// - samples: Audio samples normalized to the range [-1, 1]
200 /// - sampleRate: Sample rate of the input audio samples. Must match 200 /// - sampleRate: Sample rate of the input audio samples. Must match
201 - /// the one expected by the model. It must be 16000 for  
202 - /// models from icefall. 201 + /// the one expected by the model.
203 func acceptWaveform(samples: [Float], sampleRate: Int = 16000) { 202 func acceptWaveform(samples: [Float], sampleRate: Int = 16000) {
204 AcceptWaveform(stream, Int32(sampleRate), samples, Int32(samples.count)) 203 AcceptWaveform(stream, Int32(sampleRate), samples, Int32(samples.count))
205 } 204 }
@@ -238,3 +237,163 @@ class SherpaOnnxRecognizer { @@ -238,3 +237,163 @@ class SherpaOnnxRecognizer {
238 return IsEndpoint(recognizer, stream) == 1 ? true : false 237 return IsEndpoint(recognizer, stream) == 1 ? true : false
239 } 238 }
240 } 239 }
  240 +
  241 +// For offline APIs
  242 +
  243 +func sherpaOnnxOfflineTransducerModelConfig(
  244 + encoder: String = "",
  245 + decoder: String = "",
  246 + joiner: String = ""
  247 +) -> SherpaOnnxOfflineTransducerModelConfig {
  248 + return SherpaOnnxOfflineTransducerModelConfig(
  249 + encoder: toCPointer(encoder),
  250 + decoder: toCPointer(decoder),
  251 + joiner: toCPointer(joiner)
  252 + )
  253 +}
  254 +
  255 +func sherpaOnnxOfflineParaformerModelConfig(
  256 + model: String = ""
  257 +) -> SherpaOnnxOfflineParaformerModelConfig {
  258 + return SherpaOnnxOfflineParaformerModelConfig(
  259 + model: toCPointer(model)
  260 + )
  261 +}
  262 +
  263 +func sherpaOnnxOfflineNemoEncDecCtcModelConfig(
  264 + model: String = ""
  265 +) -> SherpaOnnxOfflineNemoEncDecCtcModelConfig {
  266 + return SherpaOnnxOfflineNemoEncDecCtcModelConfig(
  267 + model: toCPointer(model)
  268 + )
  269 +}
  270 +
  271 +func sherpaOnnxOfflineWhisperModelConfig(
  272 + encoder: String = "",
  273 + decoder: String = ""
  274 +) -> SherpaOnnxOfflineWhisperModelConfig {
  275 + return SherpaOnnxOfflineWhisperModelConfig(
  276 + encoder: toCPointer(encoder),
  277 + decoder: toCPointer(decoder)
  278 + )
  279 +}
  280 +
  281 +func sherpaOnnxOfflineTdnnModelConfig(
  282 + model: String = ""
  283 +) -> SherpaOnnxOfflineTdnnModelConfig {
  284 + return SherpaOnnxOfflineTdnnModelConfig(
  285 + model: toCPointer(model)
  286 + )
  287 +}
  288 +
  289 +func sherpaOnnxOfflineLMConfig(
  290 + model: String = "",
  291 + scale: Float = 1.0
  292 +) -> SherpaOnnxOfflineLMConfig {
  293 + return SherpaOnnxOfflineLMConfig(
  294 + model: toCPointer(model),
  295 + scale: scale
  296 + )
  297 +}
  298 +
  299 +func sherpaOnnxOfflineModelConfig(
  300 + tokens: String,
  301 + transducer: SherpaOnnxOfflineTransducerModelConfig = sherpaOnnxOfflineTransducerModelConfig(),
  302 + paraformer: SherpaOnnxOfflineParaformerModelConfig = sherpaOnnxOfflineParaformerModelConfig(),
  303 + nemoCtc: SherpaOnnxOfflineNemoEncDecCtcModelConfig = sherpaOnnxOfflineNemoEncDecCtcModelConfig(),
  304 + whisper: SherpaOnnxOfflineWhisperModelConfig = sherpaOnnxOfflineWhisperModelConfig(),
  305 + tdnn: SherpaOnnxOfflineTdnnModelConfig = sherpaOnnxOfflineTdnnModelConfig(),
  306 + numThreads: Int = 1,
  307 + provider: String = "cpu",
  308 + debug: Int = 0,
  309 + modelType: String = ""
  310 +) -> SherpaOnnxOfflineModelConfig {
  311 + return SherpaOnnxOfflineModelConfig(
  312 + transducer: transducer,
  313 + paraformer: paraformer,
  314 + nemo_ctc: nemoCtc,
  315 + whisper: whisper,
  316 + tdnn: tdnn,
  317 + tokens: toCPointer(tokens),
  318 + num_threads: Int32(numThreads),
  319 + debug: Int32(debug),
  320 + provider: toCPointer(provider),
  321 + model_type: toCPointer(modelType)
  322 + )
  323 +}
  324 +
  325 +func sherpaOnnxOfflineRecognizerConfig(
  326 + featConfig: SherpaOnnxFeatureConfig,
  327 + modelConfig: SherpaOnnxOfflineModelConfig,
  328 + lmConfig: SherpaOnnxOfflineLMConfig = sherpaOnnxOfflineLMConfig(),
  329 + decodingMethod: String = "greedy_search",
  330 + maxActivePaths: Int = 4
  331 +) -> SherpaOnnxOfflineRecognizerConfig {
  332 + return SherpaOnnxOfflineRecognizerConfig(
  333 + feat_config: featConfig,
  334 + model_config: modelConfig,
  335 + lm_config: lmConfig,
  336 + decoding_method: toCPointer(decodingMethod),
  337 + max_active_paths: Int32(maxActivePaths)
  338 + )
  339 +}
  340 +
  341 +class SherpaOnnxOfflineRecongitionResult {
  342 + /// A pointer to the underlying counterpart in C
  343 + let result: UnsafePointer<SherpaOnnxOfflineRecognizerResult>!
  344 +
  345 + /// Return the actual recognition result.
  346 + /// For English models, it contains words separated by spaces.
  347 + /// For Chinese models, it contains Chinese words.
  348 + var text: String {
  349 + return String(cString: result.pointee.text)
  350 + }
  351 +
  352 + init(result: UnsafePointer<SherpaOnnxOfflineRecognizerResult>!) {
  353 + self.result = result
  354 + }
  355 +
  356 + deinit {
  357 + if let result {
  358 + DestroyOfflineRecognizerResult(result)
  359 + }
  360 + }
  361 +}
  362 +
  363 +class SherpaOnnxOfflineRecognizer {
  364 + /// A pointer to the underlying counterpart in C
  365 + let recognizer: OpaquePointer!
  366 +
  367 + init(
  368 + config: UnsafePointer<SherpaOnnxOfflineRecognizerConfig>!
  369 + ) {
  370 + recognizer = CreateOfflineRecognizer(config)
  371 + }
  372 +
  373 + deinit {
  374 + if let recognizer {
  375 + DestroyOfflineRecognizer(recognizer)
  376 + }
  377 + }
  378 +
  379 + /// Decode wave samples.
  380 + ///
  381 + /// - Parameters:
  382 + /// - samples: Audio samples normalized to the range [-1, 1]
  383 + /// - sampleRate: Sample rate of the input audio samples. Must match
  384 + /// the one expected by the model.
  385 + func decode(samples: [Float], sampleRate: Int = 16000) -> SherpaOnnxOfflineRecongitionResult {
  386 + let stream: OpaquePointer! = CreateOfflineStream(recognizer)
  387 +
  388 + AcceptWaveformOffline(stream, Int32(sampleRate), samples, Int32(samples.count))
  389 +
  390 + DecodeOfflineStream(recognizer, stream)
  391 +
  392 + let result: UnsafeMutablePointer<SherpaOnnxOfflineRecognizerResult>? = GetOfflineStreamResult(
  393 + stream)
  394 +
  395 + DestroyOfflineStream(stream)
  396 +
  397 + return SherpaOnnxOfflineRecongitionResult(result: result)
  398 + }
  399 +}
  1 +import AVFoundation
  2 +
  3 +extension AudioBuffer {
  4 + func array() -> [Float] {
  5 + return Array(UnsafeBufferPointer(self))
  6 + }
  7 +}
  8 +
  9 +extension AVAudioPCMBuffer {
  10 + func array() -> [Float] {
  11 + return self.audioBufferList.pointee.mBuffers.array()
  12 + }
  13 +}
  14 +
  15 +func run() {
  16 + let encoder = "./sherpa-onnx-whisper-tiny.en/tiny.en-encoder.int8.onnx"
  17 + let decoder = "./sherpa-onnx-whisper-tiny.en/tiny.en-decoder.int8.onnx"
  18 + let tokens = "./sherpa-onnx-whisper-tiny.en/tiny.en-tokens.txt"
  19 +
  20 + let whisperConfig = sherpaOnnxOfflineWhisperModelConfig(
  21 + encoder: encoder,
  22 + decoder: decoder
  23 + )
  24 +
  25 + let modelConfig = sherpaOnnxOfflineModelConfig(
  26 + tokens: tokens,
  27 + whisper: whisperConfig,
  28 + debug: 0,
  29 + modelType: "whisper"
  30 + )
  31 +
  32 + let featConfig = sherpaOnnxFeatureConfig(
  33 + sampleRate: 16000,
  34 + featureDim: 80
  35 + )
  36 + var config = sherpaOnnxOfflineRecognizerConfig(
  37 + featConfig: featConfig,
  38 + modelConfig: modelConfig
  39 + )
  40 +
  41 + let recognizer = SherpaOnnxOfflineRecognizer(config: &config)
  42 +
  43 + let filePath = "./sherpa-onnx-whisper-tiny.en/test_wavs/0.wav"
  44 + let fileURL: NSURL = NSURL(fileURLWithPath: filePath)
  45 + let audioFile = try! AVAudioFile(forReading: fileURL as URL)
  46 +
  47 + let audioFormat = audioFile.processingFormat
  48 + assert(audioFormat.channelCount == 1)
  49 + assert(audioFormat.commonFormat == AVAudioCommonFormat.pcmFormatFloat32)
  50 +
  51 + let audioFrameCount = UInt32(audioFile.length)
  52 + let audioFileBuffer = AVAudioPCMBuffer(pcmFormat: audioFormat, frameCapacity: audioFrameCount)
  53 +
  54 + try! audioFile.read(into: audioFileBuffer!)
  55 + let array: [Float]! = audioFileBuffer?.array()
  56 + let result = recognizer.decode(samples: array, sampleRate: Int(audioFormat.sampleRate))
  57 + print("\nresult is:\n\(result.text)")
  58 +}
  59 +
  60 +@main
  61 +struct App {
  62 + static func main() {
  63 + run()
  64 + }
  65 +}