CameraX zoom feature and cleanup (#422)

* Clean up and example gesture zoom feature

* spotless

* fix build

+/*
+ * Copyright 2024 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.livekit.android.camerax.ui
+
+import android.content.Context
+import android.view.ScaleGestureDetector
+import android.view.ScaleGestureDetector.SimpleOnScaleGestureListener
+import androidx.camera.core.Camera
+import io.livekit.android.room.track.LocalVideoTrack
+import io.livekit.android.util.LKLog
+import kotlinx.coroutines.flow.StateFlow
+import livekit.org.webrtc.getCameraX
+
+class ScaleZoomHelper(
+    private val cameraFlow: StateFlow<Camera?>?,
+) {
+    constructor(localVideoTrack: LocalVideoTrack) : this(localVideoTrack.capturer.getCameraX())
+
+    init {
+        if (cameraFlow != null) {
+            LKLog.w { "null camera flow passed in to ScaleZoomHelper, zoom is disabled." }
+        }
+    }
+
+    fun zoom(factor: Float) {
+        val camera = cameraFlow?.value ?: return
+        val zoomState = camera.cameraInfo.zoomState.value ?: return
+        val currentZoom = zoomState.zoomRatio
+        val newZoom = (currentZoom * factor).coerceIn(zoomState.minZoomRatio, zoomState.maxZoomRatio)
+
+        if (newZoom != currentZoom) {
+            camera.cameraControl.setZoomRatio(newZoom)
+        }
+    }
+
+    companion object {
+        fun createGestureDetector(context: Context, localVideoTrack: LocalVideoTrack): ScaleGestureDetector {
+            return createGestureDetector(context, localVideoTrack.capturer.getCameraX())
+        }
+
+        fun createGestureDetector(context: Context, cameraFlow: StateFlow<Camera?>?): ScaleGestureDetector {
+            val helper = ScaleZoomHelper(cameraFlow)
+
+            return ScaleGestureDetector(
+                context,
+                object : SimpleOnScaleGestureListener() {
+                    override fun onScale(detector: ScaleGestureDetector): Boolean {
+                        helper.zoom(detector.scaleFactor)
+                        return true
+                    }
+                },
+            ).apply {
+                isQuickScaleEnabled = false
+            }
+        }
+    }
+}

@@ -18,10 +18,16 @@ package livekit.org.webrtc
 
 import android.content.Context
 import android.hardware.camera2.CameraManager
+import androidx.annotation.OptIn
 import androidx.camera.camera2.interop.ExperimentalCamera2Interop
+import androidx.camera.core.Camera
 import androidx.lifecycle.LifecycleOwner
 import io.livekit.android.room.track.video.CameraCapturerWithSize
 import io.livekit.android.room.track.video.CameraEventsDispatchHandler
+import io.livekit.android.util.FlowObservable
+import io.livekit.android.util.flow
+import io.livekit.android.util.flowDelegate
+import kotlinx.coroutines.flow.StateFlow
 
 @ExperimentalCamera2Interop
 internal class CameraXCapturer(
@@ -29,9 +35,11 @@ internal class CameraXCapturer(
     private val lifecycleOwner: LifecycleOwner,
     cameraName: String?,
     eventsHandler: CameraVideoCapturer.CameraEventsHandler?,
-) : CameraCapturer(cameraName, eventsHandler, Camera2Enumerator(context)) {
+) : CameraCapturer(cameraName, eventsHandler, CameraXEnumerator(context, lifecycleOwner)) {
 
-    var cameraControlListener: CameraXSession.CameraControlListener? = null
+    @FlowObservable
+    @get:FlowObservable
+    var currentCamera by flowDelegate<Camera?>(null)
 
     override fun createCameraSession(
         createSessionCallback: CameraSession.CreateSessionCallback,
@@ -44,8 +52,37 @@ internal class CameraXCapturer(
         framerate: Int,
     ) {
         CameraXSession(
-            createSessionCallback,
-            events,
+            object : CameraSession.CreateSessionCallback {
+                override fun onDone(session: CameraSession) {
+                    createSessionCallback.onDone(session)
+                    currentCamera = (session as CameraXSession).camera
+                }
+
+                override fun onFailure(failureType: CameraSession.FailureType, error: String) {
+                    createSessionCallback.onFailure(failureType, error)
+                }
+            },
+            object : CameraSession.Events {
+                override fun onCameraOpening() {
+                    events.onCameraOpening()
+                }
+
+                override fun onCameraError(session: CameraSession, error: String) {
+                    events.onCameraError(session, error)
+                }
+
+                override fun onCameraDisconnected(session: CameraSession) {
+                    events.onCameraDisconnected(session)
+                }
+
+                override fun onCameraClosed(session: CameraSession) {
+                    events.onCameraClosed(session)
+                }
+
+                override fun onFrameCaptured(session: CameraSession, frame: VideoFrame) {
+                    events.onFrameCaptured(session, frame)
+                }
+            },
             applicationContext,
             lifecycleOwner,
             surfaceTextureHelper,
@@ -53,14 +90,13 @@ internal class CameraXCapturer(
             width,
             height,
             framerate,
-            cameraControlListener,
         )
     }
 }
 
 @ExperimentalCamera2Interop
 internal class CameraXCapturerWithSize(
-    private val capturer: CameraXCapturer,
+    internal val capturer: CameraXCapturer,
     private val cameraManager: CameraManager,
     private val deviceName: String?,
     cameraEventsDispatchHandler: CameraEventsDispatchHandler,
@@ -69,3 +105,20 @@ internal class CameraXCapturerWithSize(
         return CameraXHelper.findClosestCaptureFormat(cameraManager, deviceName, width, height)
     }
 }
+
+/**
+ * Gets the [androidx.camera.core.Camera] from the VideoCapturer if it's using CameraX.
+ */
+@OptIn(ExperimentalCamera2Interop::class)
+fun VideoCapturer.getCameraX(): StateFlow<Camera?>? {
+    val actualCapturer = if (this is CameraXCapturerWithSize) {
+        this.capturer
+    } else {
+        this
+    }
+
+    if (actualCapturer is CameraXCapturer) {
+        return actualCapturer::currentCamera.flow
+    }
+    return null
+}

@@ -26,6 +26,9 @@ import androidx.camera.camera2.interop.Camera2CameraInfo
 import androidx.camera.camera2.interop.ExperimentalCamera2Interop
 import androidx.lifecycle.LifecycleOwner
 
+/**
+ * @suppress
+ */
 @ExperimentalCamera2Interop
 class CameraXEnumerator(
     context: Context,

@@ -28,10 +28,14 @@ import io.livekit.android.room.track.video.CameraEventsDispatchHandler
 class CameraXHelper {
     companion object {
 
+        /**
+         * Gets a CameraProvider that uses CameraX for its sessions.
+         *
+         * For use with [CameraCapturerUtils.registerCameraProvider].
+         */
         @ExperimentalCamera2Interop
         fun getCameraProvider(
             lifecycleOwner: LifecycleOwner,
-            controlListener: CameraXSession.CameraControlListener?,
         ) = object : CameraCapturerUtils.CameraProvider {
 
             private var enumerator: CameraXEnumerator? = null
@@ -51,9 +55,7 @@ class CameraXHelper {
                 val enumerator = provideEnumerator(context)
                 val targetDeviceName = enumerator.findCamera(options.deviceId, options.position)
                 val targetVideoCapturer = enumerator.createCapturer(targetDeviceName, eventsHandler) as CameraXCapturer
-                controlListener?.let {
-                    targetVideoCapturer.cameraControlListener = it
-                }
+
                 return CameraXCapturerWithSize(
                     targetVideoCapturer,
                     context.getSystemService(Context.CAMERA_SERVICE) as CameraManager,

@@ -42,6 +42,9 @@ import livekit.org.webrtc.CameraEnumerationAndroid.CaptureFormat
 import java.util.concurrent.Executor
 import java.util.concurrent.TimeUnit
 
+/**
+ * @suppress
+ */
 @androidx.camera.camera2.interop.ExperimentalCamera2Interop
 class CameraXSession
 internal constructor(
@@ -54,14 +57,14 @@ internal constructor(
     private val width: Int,
     private val height: Int,
     private val frameRate: Int,
-    private val cameraControlListener: CameraControlListener? = null,
 ) : CameraSession {
 
     private var state = SessionState.RUNNING
     private var cameraThreadHandler = surfaceTextureHelper.handler
     private lateinit var cameraProvider: ProcessCameraProvider
     private lateinit var surfaceProvider: SurfaceProvider
-    private var camera: Camera? = null
+    var camera: Camera? = null
+        private set
     private var surface: Surface? = null
     private var cameraOrientation: Int = 0
     private var isCameraFrontFacing: Boolean = true
@@ -168,11 +171,12 @@ internal constructor(
                         cameraProvider.unbindAll()
 
                         // Bind use cases to camera
-                        camera = cameraProvider.bindToLifecycle(lifecycleOwner, cameraSelector, imageAnalysis, preview).apply {
-                            cameraControlListener?.onCameraControlAvailable(this.cameraControl)
+                        camera = cameraProvider.bindToLifecycle(lifecycleOwner, cameraSelector, imageAnalysis, preview)
+
+                        cameraThreadHandler.post {
+                            sessionCallback.onDone(this@CameraXSession)
                         }
                     }
-                    sessionCallback.onDone(this@CameraXSession)
                 } catch (e: Exception) {
                     reportError("Failed to open camera: $e")
                 }

@@ -64,7 +64,7 @@ import livekit.LivekitModels.VideoQuality as ProtoVideoQuality
 open class LocalVideoTrack
 @AssistedInject
 constructor(
-    @Assisted private var capturer: VideoCapturer,
+    @Assisted capturer: VideoCapturer,
     @Assisted private var source: VideoSource,
     @Assisted name: String,
     @Assisted options: LocalVideoTrackOptions,
@@ -81,6 +81,9 @@ constructor(
     @Assisted private var dispatchObserver: CaptureDispatchObserver? = null,
 ) : VideoTrack(name, rtcTrack) {
 
+    var capturer = capturer
+        private set
+
     override var rtcTrack: livekit.org.webrtc.VideoTrack = rtcTrack
         internal set
 

@@ -73,8 +73,11 @@ val <T> KProperty0<T>.flow: StateFlow<T>
 @MustBeDocumented
 annotation class FlowObservable
 
+/**
+ * @suppress
+ */
 @FlowObservable
-internal class MutableStateFlowDelegate<T>
+class MutableStateFlowDelegate<T>
 internal constructor(
     private val flow: MutableStateFlow<T>,
     private val onSetValue: ((newValue: T, oldValue: T) -> Unit)? = null,
@@ -94,8 +97,11 @@ internal constructor(
     }
 }
 
+/**
+ * @suppress
+ */
 @FlowObservable
-internal class StateFlowDelegate<T>
+class StateFlowDelegate<T>
 internal constructor(
     private val flow: StateFlow<T>,
 ) : StateFlow<T> by flow {
@@ -108,14 +114,20 @@ internal constructor(
     }
 }
 
-internal fun <T> flowDelegate(
+/**
+ * @suppress
+ */
+fun <T> flowDelegate(
     initialValue: T,
     onSetValue: ((newValue: T, oldValue: T) -> Unit)? = null,
 ): MutableStateFlowDelegate<T> {
     return MutableStateFlowDelegate(MutableStateFlow(initialValue), onSetValue)
 }
 
-internal fun <T> flowDelegate(
+/**
+ * @suppress
+ */
+fun <T> flowDelegate(
     stateFlow: StateFlow<T>,
 ): StateFlowDelegate<T> {
     return StateFlowDelegate(stateFlow)

@@ -21,9 +21,12 @@ import android.app.Application
 import android.content.Intent
 import android.media.projection.MediaProjectionManager
 import android.os.Build
+import androidx.annotation.OptIn
+import androidx.camera.camera2.interop.ExperimentalCamera2Interop
 import androidx.lifecycle.AndroidViewModel
 import androidx.lifecycle.LiveData
 import androidx.lifecycle.MutableLiveData
+import androidx.lifecycle.ProcessLifecycleOwner
 import androidx.lifecycle.viewModelScope
 import com.github.ajalt.timberkt.Timber
 import io.livekit.android.AudioOptions
@@ -43,6 +46,7 @@ import io.livekit.android.room.track.CameraPosition
 import io.livekit.android.room.track.LocalScreencastVideoTrack
 import io.livekit.android.room.track.LocalVideoTrack
 import io.livekit.android.room.track.Track
+import io.livekit.android.room.track.video.CameraCapturerUtils
 import io.livekit.android.sample.model.StressTest
 import io.livekit.android.sample.service.ForegroundService
 import io.livekit.android.util.LKLog
@@ -57,7 +61,9 @@ import kotlinx.coroutines.flow.combine
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.isActive
 import kotlinx.coroutines.launch
+import livekit.org.webrtc.CameraXHelper
 
+@OptIn(ExperimentalCamera2Interop::class)
 class CallViewModel(
     val url: String,
     val token: String,
@@ -93,6 +99,7 @@ class CallViewModel(
         ),
     )
 
+    private var cameraProvider: CameraCapturerUtils.CameraProvider? = null
     val audioHandler = room.audioHandler as AudioSwitchHandler
 
     val participants = room::remoteParticipants.flow
@@ -139,6 +146,14 @@ class CallViewModel(
     val permissionAllowed = mutablePermissionAllowed.hide()
 
     init {
+
+        CameraXHelper.getCameraProvider(ProcessLifecycleOwner.get()).let {
+            if (it.isSupported(application)) {
+                CameraCapturerUtils.registerCameraProvider(it)
+                cameraProvider = it
+            }
+        }
+
         viewModelScope.launch {
             // Collect any errors.
             launch {
@@ -329,6 +344,9 @@ class CallViewModel(
         val application = getApplication<Application>()
         val foregroundServiceIntent = Intent(application, ForegroundService::class.java)
         application.stopService(foregroundServiceIntent)
+        cameraProvider?.let {
+            CameraCapturerUtils.unregisterCameraProvider(it)
+        }
     }
 
     fun setMicEnabled(enabled: Boolean) {

@@ -17,6 +17,7 @@
 package io.livekit.android.composesample.ui
 
 import androidx.compose.foundation.background
+import androidx.compose.foundation.gestures.detectTransformGestures
 import androidx.compose.foundation.layout.Box
 import androidx.compose.runtime.Composable
 import androidx.compose.runtime.DisposableEffect
@@ -27,11 +28,14 @@ import androidx.compose.runtime.remember
 import androidx.compose.runtime.setValue
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.graphics.Color
+import androidx.compose.ui.input.pointer.pointerInput
 import androidx.compose.ui.layout.onGloballyPositioned
 import androidx.compose.ui.platform.LocalView
 import androidx.compose.ui.viewinterop.AndroidView
+import io.livekit.android.camerax.ui.ScaleZoomHelper
 import io.livekit.android.renderer.TextureViewRenderer
 import io.livekit.android.room.Room
+import io.livekit.android.room.track.LocalVideoTrack
 import io.livekit.android.room.track.RemoteVideoTrack
 import io.livekit.android.room.track.VideoTrack
 import livekit.org.webrtc.RendererCommon
@@ -63,6 +67,13 @@ fun VideoRenderer(
         return
     }
 
+    val scaleZoomHelper = remember(room, videoTrack) {
+        if (videoTrack is LocalVideoTrack) {
+            ScaleZoomHelper(videoTrack)
+        } else {
+            null
+        }
+    }
     val videoSinkVisibility = remember(room, videoTrack) { ComposeVisibility() }
     var boundVideoTrack by remember { mutableStateOf<VideoTrack?>(null) }
     var view: TextureViewRenderer? by remember { mutableStateOf(null) }
@@ -127,6 +138,13 @@ fun VideoRenderer(
         },
         update = { v -> setupVideoIfNeeded(videoTrack, v) },
         modifier = modifier
-            .onGloballyPositioned { videoSinkVisibility.onGloballyPositioned(it) },
+            .onGloballyPositioned { videoSinkVisibility.onGloballyPositioned(it) }
+            .pointerInput(Unit) {
+                detectTransformGestures(
+                    onGesture = { _, _, zoom, _ ->
+                        scaleZoomHelper?.zoom(zoom)
+                    },
+                )
+            },
     )
 }

@@ -26,11 +26,9 @@ import android.widget.Toast
 import androidx.activity.result.contract.ActivityResultContracts
 import androidx.appcompat.app.AlertDialog
 import androidx.appcompat.app.AppCompatActivity
-import androidx.camera.core.CameraControl
 import androidx.lifecycle.lifecycleScope
 import androidx.recyclerview.widget.LinearLayoutManager
 import com.xwray.groupie.GroupieAdapter
-import io.livekit.android.room.track.video.CameraCapturerUtils
 import io.livekit.android.sample.common.R
 import io.livekit.android.sample.databinding.CallActivityBinding
 import io.livekit.android.sample.dialog.showAudioProcessorSwitchDialog
@@ -39,14 +37,9 @@ import io.livekit.android.sample.dialog.showSelectAudioDeviceDialog
 import io.livekit.android.sample.model.StressTest
 import kotlinx.coroutines.flow.collectLatest
 import kotlinx.parcelize.Parcelize
-import livekit.org.webrtc.CameraXHelper
-import livekit.org.webrtc.CameraXSession
 
 class CallActivity : AppCompatActivity() {
 
-    private var cameraProvider: CameraCapturerUtils.CameraProvider? = null
-    private var cameraControl: CameraControl? = null
-
     private val viewModel: CallViewModel by viewModelByFactory {
         val args = intent.getParcelableExtra<BundleArgs>(KEY_ARGS)
             ?: throw NullPointerException("args is null!")
@@ -81,22 +74,6 @@ class CallActivity : AppCompatActivity() {
 
         setContentView(binding.root)
 
-        val controlListener = object : CameraXSession.CameraControlListener {
-            override fun onCameraControlAvailable(control: CameraControl) {
-                cameraControl = control
-            }
-        }
-
-        CameraXHelper.getCameraProvider(
-            this,
-            controlListener,
-        ).let {
-            if (it.isSupported(this@CallActivity)) {
-                CameraCapturerUtils.registerCameraProvider(it)
-                cameraProvider = it
-            }
-        }
-
         // Audience row setup
         val audienceAdapter = GroupieAdapter()
         binding.audienceRow.apply {
@@ -240,9 +217,6 @@ class CallActivity : AppCompatActivity() {
     override fun onDestroy() {
         binding.audienceRow.adapter = null
         binding.speakerView.adapter = null
-        cameraProvider?.let {
-            CameraCapturerUtils.unregisterCameraProvider(it)
-        }
         super.onDestroy()
     }