Support authenticating audio processors (#390)

* Support authenticating audio processors

* spotless

* fix tests

* spotless

@@ -40,3 +40,13 @@ interface AudioProcessingController {
      */
     fun setBypassForRenderPreProcessing(bypass: Boolean)
 }
+
+/**
+ * @suppress
+ */
+interface AuthedAudioProcessingController : AudioProcessingController {
+    /**
+     * @suppress
+     */
+    fun authenticate(url: String, token: String)
+}

@@ -34,16 +34,35 @@ interface AudioProcessorInterface {
 
     /**
      * Initialize the audio processing.
+     *
+     * Note: audio processing methods will be called regardless of whether
+     * [isEnabled] returns true or not.
      */
     fun initializeAudioProcessing(sampleRateHz: Int, numChannels: Int)
 
     /**
      * Called when the sample rate has changed.
+     *
+     * Note: audio processing methods will be called regardless of whether
+     * [isEnabled] returns true or not.
      */
     fun resetAudioProcessing(newRate: Int)
 
     /**
      * Process the audio frame (10ms).
+     *
+     * Note: audio processing methods will be called regardless of whether
+     * [isEnabled] returns true or not.
      */
     fun processAudio(numBands: Int, numFrames: Int, buffer: ByteBuffer)
 }
+
+/**
+ * @suppress
+ */
+interface AuthedAudioProcessorInterface : AudioProcessorInterface {
+    /**
+     * @suppress
+     */
+    fun authenticate(url: String, token: String)
+}

@@ -38,3 +38,12 @@ data class AudioProcessorOptions(
      */
     val renderPreBypass: Boolean = false,
 )
+
+internal fun AudioProcessorOptions.authenticateProcessors(url: String, token: String) {
+    if (capturePostProcessor is AuthedAudioProcessorInterface) {
+        capturePostProcessor.authenticate(url, token)
+    }
+    if (renderPreProcessor is AuthedAudioProcessorInterface) {
+        renderPreProcessor.authenticate(url, token)
+    }
+}

@@ -287,7 +287,7 @@ internal object RTCModule {
         @Named(InjectionNames.OVERRIDE_PEER_CONNECTION_FACTORY_OPTIONS)
         peerConnectionFactoryOptions: PeerConnectionFactory.Options?,
         memoryManager: CloseableManager,
-        audioProcessingFactory: AudioProcessingFactory?,
+        audioProcessingFactory: AudioProcessingFactory,
     ): PeerConnectionFactory {
         return PeerConnectionFactory.builder()
             .setAudioDeviceModule(audioDeviceModule)

@@ -31,7 +31,7 @@ import io.livekit.android.RoomOptions
 import io.livekit.android.Version
 import io.livekit.android.audio.AudioHandler
 import io.livekit.android.audio.AudioProcessingController
-import io.livekit.android.audio.AudioProcessorOptions
+import io.livekit.android.audio.AuthedAudioProcessingController
 import io.livekit.android.audio.CommunicationWorkaround
 import io.livekit.android.dagger.InjectionNames
 import io.livekit.android.e2ee.E2EEManager
@@ -41,6 +41,7 @@ import io.livekit.android.memory.CloseableManager
 import io.livekit.android.renderer.TextureViewRenderer
 import io.livekit.android.room.network.NetworkCallbackManager
 import io.livekit.android.room.participant.*
+import io.livekit.android.room.provisions.LKObjects
 import io.livekit.android.room.track.*
 import io.livekit.android.util.FlowObservable
 import io.livekit.android.util.LKLog
@@ -76,6 +77,7 @@ constructor(
     private val e2EEManagerFactory: E2EEManager.Factory,
     private val communicationWorkaround: CommunicationWorkaround,
     val audioProcessingController: AudioProcessingController,
+    val lkObjects: LKObjects,
 ) : RTCEngine.Listener, ParticipantListener {
 
     private lateinit var coroutineScope: CoroutineScope
@@ -211,11 +213,6 @@ constructor(
     var e2eeOptions: E2EEOptions? = null
 
     /**
-     * @see external audio processing options
-     */
-    var audioProcessorOptions: AudioProcessorOptions? = null
-
-    /**
      * Default options to use when creating an audio track.
      */
     var audioTrackCaptureDefaults: LocalAudioTrackOptions by defaultsManager::audioTrackCaptureDefaults
@@ -370,10 +367,12 @@ constructor(
         val connectJob = coroutineScope.launch(
             ioDispatcher + emptyCoroutineExceptionHandler,
         ) {
+            if (audioProcessingController is AuthedAudioProcessingController) {
+                audioProcessingController.authenticate(url, token)
+            }
             engine.join(url, token, options, roomOptions)
-            networkCallbackManager.registerCallback()
-
             ensureActive()
+            networkCallbackManager.registerCallback()
             if (options.audio) {
                 val audioTrack = localParticipant.createAudioTrack()
                 localParticipant.publishAudioTrack(audioTrack)

+/*
+ * Copyright 2024 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.livekit.android.room.provisions
+
+import livekit.org.webrtc.EglBase
+import javax.inject.Inject
+import javax.inject.Provider
+
+/**
+ * Provides access to objects used internally.
+ */
+// Note, to avoid accidentally instantiating an unneeded object,
+// only store Providers here.
+//
+// Additionally, the provided objects should only be singletons.
+// Otherwise the created objects may not be the one used internally.
+class LKObjects
+@Inject
+constructor(
+    private val eglBaseProvider: Provider<EglBase>,
+) {
+    val eglBase: EglBase
+        get() = eglBaseProvider.get()
+}

@@ -16,21 +16,21 @@
 
 package io.livekit.android.webrtc
 
-import io.livekit.android.audio.AudioProcessingController
 import io.livekit.android.audio.AudioProcessorInterface
 import io.livekit.android.audio.AudioProcessorOptions
+import io.livekit.android.audio.AuthedAudioProcessingController
+import io.livekit.android.audio.authenticateProcessors
 import livekit.org.webrtc.AudioProcessingFactory
 import livekit.org.webrtc.ExternalAudioProcessingFactory
 import java.nio.ByteBuffer
 
-class CustomAudioProcessingFactory(private val audioProcessorOptions: AudioProcessorOptions) : AudioProcessingController {
+class CustomAudioProcessingFactory(private var audioProcessorOptions: AudioProcessorOptions) : AuthedAudioProcessingController {
 
     private val externalAudioProcessor = ExternalAudioProcessingFactory()
 
     init {
         if (audioProcessorOptions.capturePostProcessor != null) {
             setCapturePostProcessing(audioProcessorOptions.capturePostProcessor)
-            setBypassForCapturePostProcessing(audioProcessorOptions.capturePostBypass)
         } else {
             setCapturePostProcessing(null)
             setBypassForCapturePostProcessing(false)
@@ -48,23 +48,31 @@ class CustomAudioProcessingFactory(private val audioProcessorOptions: AudioProce
         return externalAudioProcessor
     }
 
+    override fun authenticate(url: String, token: String) {
+        audioProcessorOptions.authenticateProcessors(url, token)
+    }
+
     override fun setCapturePostProcessing(processing: AudioProcessorInterface?) {
+        audioProcessorOptions = audioProcessorOptions.copy(capturePostProcessor = processing)
         externalAudioProcessor.setCapturePostProcessing(
             processing.toAudioProcessing(),
         )
     }
 
     override fun setBypassForCapturePostProcessing(bypass: Boolean) {
+        audioProcessorOptions = audioProcessorOptions.copy(capturePostBypass = bypass)
         externalAudioProcessor.setBypassFlagForCapturePost(bypass)
     }
 
     override fun setRenderPreProcessing(processing: AudioProcessorInterface?) {
+        audioProcessorOptions = audioProcessorOptions.copy(renderPreProcessor = processing)
         externalAudioProcessor.setRenderPreProcessing(
             processing.toAudioProcessing(),
         )
     }
 
     override fun setBypassForRenderPreProcessing(bypass: Boolean) {
+        audioProcessorOptions = audioProcessorOptions.copy(renderPreBypass = bypass)
         externalAudioProcessor.setBypassFlagForRenderPre(bypass)
     }
 
@@ -76,7 +84,9 @@ class CustomAudioProcessingFactory(private val audioProcessorOptions: AudioProce
         }
 
         override fun reset(newRate: Int) {
-            audioProcessing?.resetAudioProcessing(newRate)
+            // bug in webrtc lib causes newRate to be off by a factor of 10
+            // TODO: remove divide by 10 when updating libwebrtc
+            audioProcessing?.resetAudioProcessing(newRate / 10)
         }
 
         override fun process(numBands: Int, numFrames: Int, buffer: ByteBuffer?) {

+/*
+ * Copyright 2024 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.livekit.android.mock
+
+import io.livekit.android.room.provisions.LKObjects
+
+object MockLKObjects {
+    fun get(): LKObjects {
+        return LKObjects(
+            eglBaseProvider = { MockEglBase() },
+        )
+    }
+}

@@ -102,6 +102,7 @@ class RoomTest {
             e2EEManagerFactory = e2EEManagerFactory,
             communicationWorkaround = NoopCommunicationWorkaround(),
             audioProcessingController = MockAudioProcessingController(),
+            lkObjects = MockLKObjects.get(),
         )
     }
 

@@ -227,7 +227,6 @@ class CallViewModel(
     private suspend fun connectToRoom() {
         try {
             room.e2eeOptions = getE2EEOptions()
-            room.audioProcessorOptions = audioProcessorOptions
             room.connect(
                 url = url,
                 token = token,