xuning / livekitAndroidXuningTest

转到一个项目
Toggle navigation Toggle navigation pinning
davidliu
Committed by GitHub
69027624 1 parent 2f31a855

VirtualBackgroundVideoProcessor and track-processors package (#660) 

* save

* port js shaders

* save

* fix downsampler

* save - trying to get blur shader workging

* fix wrong height to downsample

* expose specific floats arguments for coordinate matrixes

* constant vertex shader

* properly set sizes for framebuffer to draw into

* working blur shader

* Rename DownSamplerShader to ResamplerShader

* save

* Virtual background and track-processors module

* Delete experimental files

* Clean up example

* Clean up documentation

* Delete unused file

* revert change to module.md

* cleanup documentation

* revert unwanted change

* changeset

* spotless

* spotless
内嵌 并排对比
正在显示 55 个修改的文件 包含 1598 行增加 和 356 行删除
---
"client-sdk-android": minor
---
Add VirtualBackgroundVideoProcessor and track-processors package
... ...
... ... @@ -3,6 +3,7 @@
<component name="CompilerConfiguration">
<bytecodeTargetLevel target="1.8">
<module name="livekit-android.examples.screenshareaudio" target="17" />
<module name="livekit-android.examples.selfie-segmentation" target="17" />
</bytecodeTargetLevel>
</component>
</project>
\ No newline at end of file
... ...
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="AndroidLintVisibleForTests" enabled="true" level="WARNING" enabled_by_default="true">
<scope name="Library Projects" level="WARNING" enabled="false" />
</inspection_tool>
<inspection_tool class="MemberVisibilityCanBePrivate" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<scope name="Library Projects" level="WEAK WARNING" enabled="false" />
</inspection_tool>
... ...
... ... @@ -60,3 +60,39 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
#####################################################################################
Parts of this source code come from the WebRTC project, following a BSD-style license
https://webrtc.googlesource.com/src
Copyright (c) 2011, The WebRTC project authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
* Neither the name of Google nor the names of its contributors may
be used to endorse or promote products derived from this software
without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
... ...
/*
* Copyright 2024 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.livekit.android.selfie
import android.graphics.BitmapFactory
import android.graphics.Color
import android.graphics.ImageFormat
import android.graphics.Matrix
import android.graphics.Paint
import android.graphics.Rect
import android.graphics.YuvImage
import android.os.Build
import android.util.Log
import android.view.Surface
import androidx.core.graphics.set
import com.google.mlkit.vision.common.InputImage
import com.google.mlkit.vision.segmentation.Segmentation
import com.google.mlkit.vision.segmentation.Segmenter
import com.google.mlkit.vision.segmentation.selfie.SelfieSegmenterOptions
import io.livekit.android.room.track.video.NoDropVideoProcessor
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.cancel
import kotlinx.coroutines.channels.BufferOverflow
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex
import livekit.org.webrtc.EglBase
import livekit.org.webrtc.SurfaceTextureHelper
import livekit.org.webrtc.VideoFrame
import livekit.org.webrtc.VideoSink
import livekit.org.webrtc.YuvHelper
import java.io.ByteArrayOutputStream
import java.nio.ByteBuffer
class SelfieBitmapVideoProcessor(eglBase: EglBase, dispatcher: CoroutineDispatcher) : NoDropVideoProcessor() {
private var targetSink: VideoSink? = null
private val segmenter: Segmenter
private var lastRotation = 0
private var lastWidth = 0
private var lastHeight = 0
private val surfaceTextureHelper = SurfaceTextureHelper.create("BitmapToYUV", eglBase.eglBaseContext)
private val surface = Surface(surfaceTextureHelper.surfaceTexture)
private val scope = CoroutineScope(dispatcher)
private val taskFlow = MutableSharedFlow<VideoFrame>(
replay = 0,
extraBufferCapacity = 1,
onBufferOverflow = BufferOverflow.SUSPEND,
)
init {
val options =
SelfieSegmenterOptions.Builder()
.setDetectorMode(SelfieSegmenterOptions.STREAM_MODE)
.build()
segmenter = Segmentation.getClient(options)
// Funnel processing into a single flow that won't buffer,
// since processing will be slower than video capture
scope.launch {
taskFlow.collect { frame ->
processFrame(frame)
}
}
}
override fun onCapturerStarted(started: Boolean) {
if (started) {
surfaceTextureHelper.startListening { frame ->
targetSink?.onFrame(frame)
}
}
}
override fun onCapturerStopped() {
surfaceTextureHelper.stopListening()
}
override fun onFrameCaptured(frame: VideoFrame) {
if (taskFlow.tryEmit(frame)) {
frame.retain()
}
}
suspend fun processFrame(frame: VideoFrame) {
// toI420 causes a retain, so a corresponding frameBuffer.release is needed when done.
val frameBuffer = frame.buffer.toI420() ?: return
val rotationDegrees = frame.rotation
val dataY = frameBuffer.dataY
val dataU = frameBuffer.dataU
val dataV = frameBuffer.dataV
val nv12Buffer = ByteBuffer.allocateDirect(dataY.limit() + dataU.limit() + dataV.limit())
// For some reason, I420ToNV12 actually expects YV12
YuvHelper.I420ToNV12(
frameBuffer.dataY,
frameBuffer.strideY,
frameBuffer.dataV,
frameBuffer.strideV,
frameBuffer.dataU,
frameBuffer.strideU,
nv12Buffer,
frameBuffer.width,
frameBuffer.height,
)
// Use YuvImage to convert to bitmap
val yuvImage = YuvImage(nv12Buffer.array(), ImageFormat.NV21, frameBuffer.width, frameBuffer.height, null)
val stream = ByteArrayOutputStream()
yuvImage.compressToJpeg(Rect(0, 0, frameBuffer.width, frameBuffer.height), 100, stream)
val bitmap = BitmapFactory.decodeByteArray(
stream.toByteArray(),
0,
stream.size(),
BitmapFactory.Options().apply { inMutable = true },
)
// No longer need the original frame buffer any more.
frameBuffer.release()
frame.release()
// Ready for segementation processing.
val inputImage = InputImage.fromBitmap(bitmap, 0)
val task = segmenter.process(inputImage)
val latch = Mutex(true)
task.addOnSuccessListener { segmentationMask ->
val mask = segmentationMask.buffer
// Do some image processing
for (y in 0 until segmentationMask.height) {
for (x in 0 until segmentationMask.width) {
val backgroundConfidence = 1 - mask.float
if (backgroundConfidence > 0.8f) {
bitmap[x, y] = Color.GREEN // Color off the background
}
}
}
// Prepare for creating the processed video frame.
if (lastRotation != rotationDegrees) {
surfaceTextureHelper?.setFrameRotation(rotationDegrees)
lastRotation = rotationDegrees
}
if (lastWidth != bitmap.width || lastHeight != bitmap.height) {
surfaceTextureHelper?.setTextureSize(bitmap.width, bitmap.height)
lastWidth = bitmap.width
lastHeight = bitmap.height
}
surfaceTextureHelper?.handler?.post {
val canvas = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
surface.lockHardwareCanvas()
} else {
surface.lockCanvas(null)
}
if (canvas != null) {
// Create the video frame.
canvas.drawBitmap(bitmap, Matrix(), Paint())
surface.unlockCanvasAndPost(canvas)
}
bitmap.recycle()
latch.unlock()
}
}.addOnFailureListener {
Log.e("SelfieVideoProcessor", "failed to process frame!")
}
latch.lock()
}
override fun setSink(sink: VideoSink?) {
targetSink = sink
}
fun dispose() {
segmenter.close()
surfaceTextureHelper.stopListening()
surfaceTextureHelper.dispose()
scope.cancel()
}
}
/*
* Copyright 2024 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.livekit.android.selfie
import android.util.Log
import com.google.mlkit.vision.common.InputImage
import com.google.mlkit.vision.segmentation.Segmentation
import com.google.mlkit.vision.segmentation.Segmenter
import com.google.mlkit.vision.segmentation.selfie.SelfieSegmenterOptions
import io.livekit.android.room.track.video.NoDropVideoProcessor
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.channels.BufferOverflow
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.launch
import livekit.org.webrtc.VideoFrame
import livekit.org.webrtc.VideoSink
import java.nio.ByteBuffer
class SelfieVideoProcessor(dispatcher: CoroutineDispatcher) : NoDropVideoProcessor() {
private var targetSink: VideoSink? = null
private val segmenter: Segmenter
private val scope = CoroutineScope(dispatcher)
private val taskFlow = MutableSharedFlow<VideoFrame>(
replay = 0,
extraBufferCapacity = 1,
onBufferOverflow = BufferOverflow.SUSPEND,
)
init {
val options =
SelfieSegmenterOptions.Builder()
.setDetectorMode(SelfieSegmenterOptions.STREAM_MODE)
.build()
segmenter = Segmentation.getClient(options)
// Funnel processing into a single flow that won't buffer,
// since processing will be slower than video capture
scope.launch {
taskFlow.collect { frame ->
processFrame(frame)
}
}
}
override fun onCapturerStarted(started: Boolean) {
}
override fun onCapturerStopped() {
}
override fun onFrameCaptured(frame: VideoFrame) {
if (taskFlow.tryEmit(frame)) {
frame.retain()
}
}
fun processFrame(frame: VideoFrame) {
// toI420 causes a retain, so a corresponding frameBuffer.release is needed when done.
val frameBuffer = frame.buffer.toI420() ?: return
val byteBuffer = ByteBuffer.allocateDirect(frameBuffer.dataY.limit() + frameBuffer.dataV.limit() + frameBuffer.dataU.limit())
// YV12 is exactly like I420, but the order of the U and V planes is reversed.
// In the name, "YV" refers to the plane order: Y, then V (then U).
.put(frameBuffer.dataY)
.put(frameBuffer.dataV)
.put(frameBuffer.dataU)
val image = InputImage.fromByteBuffer(
byteBuffer,
frameBuffer.width,
frameBuffer.height,
0,
InputImage.IMAGE_FORMAT_YV12,
)
val task = segmenter.process(image)
task.addOnSuccessListener { segmentationMask ->
val mask = segmentationMask.buffer
val dataY = frameBuffer.dataY
// Do some image processing
for (i in 0 until segmentationMask.height) {
for (j in 0 until segmentationMask.width) {
val backgroundConfidence = 1 - mask.float
if (backgroundConfidence > 0.8f) {
val position = dataY.position()
val yValue = 0x80.toByte()
dataY.position(position)
dataY.put(yValue)
} else {
dataY.position(dataY.position() + 1)
}
}
}
// Send the final frame off to the sink.
targetSink?.onFrame(VideoFrame(frameBuffer, frame.rotation, frame.timestampNs))
// Release any remaining resources
frameBuffer.release()
frame.release()
}.addOnFailureListener {
Log.e("SelfieVideoProcessor", "failed to process frame!")
}
}
override fun setSink(sink: VideoSink?) {
targetSink = sink
}
fun dispose() {
segmenter.close()
}
}
... ... @@ -33,9 +33,9 @@ android {
}
dependencies {
implementation 'com.google.mlkit:segmentation-selfie:16.0.0-beta4'
api project(":livekit-android-sdk")
api project(":livekit-android-track-processors")
api "androidx.core:core-ktx:${libs.versions.androidx.core.get()}"
implementation 'androidx.appcompat:appcompat:1.6.1'
... ... @@ -44,6 +44,8 @@ dependencies {
api "androidx.lifecycle:lifecycle-runtime-ktx:${libs.versions.androidx.lifecycle.get()}"
api "androidx.lifecycle:lifecycle-viewmodel-ktx:${libs.versions.androidx.lifecycle.get()}"
api "androidx.lifecycle:lifecycle-common-java8:${libs.versions.androidx.lifecycle.get()}"
implementation project(':livekit-android-camerax')
implementation libs.lifecycle.process
testImplementation 'junit:junit:4.13.2'
androidTestImplementation 'androidx.test.ext:junit:1.1.5'
androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1'
... ...
/*
* Copyright 2024 LiveKit, Inc.
* Copyright 2024-2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
... ...
/*
* Copyright 2024 LiveKit, Inc.
* Copyright 2024-2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
... ... @@ -19,6 +19,7 @@ package io.livekit.android.selfie
import android.Manifest
import android.content.pm.PackageManager
import android.os.Bundle
import android.widget.Button
import android.widget.Toast
import androidx.activity.ComponentActivity
import androidx.activity.result.contract.ActivityResultContracts
... ... @@ -37,6 +38,13 @@ class MainActivity : AppCompatActivity() {
viewModel = ViewModelProvider(this)[MainViewModel::class.java]
val enableButton = findViewById<Button>(R.id.button)
enableButton.setOnClickListener {
val state = viewModel.toggleProcessor()
enableButton.setText(if (state) "Disable" else "Enable")
}
val renderer = findViewById<TextureViewRenderer>(R.id.renderer)
viewModel.room.initVideoRenderer(renderer)
viewModel.track.observe(this) { track ->
... ...
/*
* Copyright 2024 LiveKit, Inc.
* Copyright 2024-2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
... ... @@ -17,18 +17,34 @@
package io.livekit.android.selfie
import android.app.Application
import android.graphics.drawable.BitmapDrawable
import androidx.annotation.OptIn
import androidx.appcompat.content.res.AppCompatResources
import androidx.camera.camera2.interop.ExperimentalCamera2Interop
import androidx.camera.core.ImageAnalysis
import androidx.lifecycle.AndroidViewModel
import androidx.lifecycle.MutableLiveData
import androidx.lifecycle.ProcessLifecycleOwner
import io.livekit.android.LiveKit
import io.livekit.android.LiveKitOverrides
import io.livekit.android.room.track.CameraPosition
import io.livekit.android.room.track.LocalVideoTrack
import io.livekit.android.room.track.LocalVideoTrackOptions
import io.livekit.android.room.track.video.CameraCapturerUtils
import io.livekit.android.track.processing.video.VirtualBackgroundVideoProcessor
import io.livekit.android.util.LoggingLevel
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.asExecutor
import livekit.org.webrtc.CameraXHelper
import livekit.org.webrtc.EglBase
@OptIn(ExperimentalCamera2Interop::class)
class MainViewModel(application: Application) : AndroidViewModel(application) {
init {
LiveKit.loggingLevel = LoggingLevel.INFO
}
val eglBase = EglBase.create()
val room = LiveKit.create(
application,
... ... @@ -37,20 +53,35 @@ class MainViewModel(application: Application) : AndroidViewModel(application) {
),
)
val track = MutableLiveData<LocalVideoTrack?>(null)
val processor = VirtualBackgroundVideoProcessor(eglBase, Dispatchers.IO).apply {
val drawable = AppCompatResources.getDrawable(application, R.drawable.background) as BitmapDrawable
backgroundImage = drawable.bitmap
}
private var cameraProvider: CameraCapturerUtils.CameraProvider? = null
// For direct I420 processing:
// val processor = SelfieVideoProcessor(Dispatchers.IO)
val processor = SelfieBitmapVideoProcessor(eglBase, Dispatchers.IO)
private var imageAnalysis = ImageAnalysis.Builder().build()
.apply { setAnalyzer(Dispatchers.IO.asExecutor(), processor.imageAnalyzer) }
init {
CameraXHelper.createCameraProvider(ProcessLifecycleOwner.get(), arrayOf(imageAnalysis)).let {
if (it.isSupported(application)) {
CameraCapturerUtils.registerCameraProvider(it)
cameraProvider = it
}
}
}
val track = MutableLiveData<LocalVideoTrack?>(null)
fun startCapture() {
val selfieVideoTrack = room.localParticipant.createVideoTrack(
val videoTrack = room.localParticipant.createVideoTrack(
options = LocalVideoTrackOptions(position = CameraPosition.FRONT),
videoProcessor = processor,
)
selfieVideoTrack.startCapture()
track.postValue(selfieVideoTrack)
videoTrack.startCapture()
track.postValue(videoTrack)
}
override fun onCleared() {
... ... @@ -58,5 +89,14 @@ class MainViewModel(application: Application) : AndroidViewModel(application) {
track.value?.stopCapture()
room.release()
processor.dispose()
cameraProvider?.let {
CameraCapturerUtils.unregisterCameraProvider(it)
}
}
fun toggleProcessor(): Boolean {
val newState = !processor.enabled
processor.enabled = newState
return newState
}
}
... ...
... ... @@ -3,7 +3,6 @@
xmlns:tools="http://schemas.android.com/tools"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:background="#F00"
tools:context=".MainActivity">
<io.livekit.android.renderer.TextureViewRenderer
... ... @@ -11,4 +10,11 @@
android:layout_width="match_parent"
android:layout_height="match_parent" />
<Button
android:id="@+id/button"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_margin="10dp"
android:text="Disable" />
</FrameLayout>
... ...
<resources>
<string name="app_name">selfie-segmentation</string>
</resources>
\ No newline at end of file
<string name="app_name">LK Virtual Background Example</string>
</resources>
... ...
/*
* Copyright 2024 LiveKit, Inc.
* Copyright 2024-2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
... ...
... ... @@ -21,11 +21,13 @@ okhttp = "4.12.0"
preferenceKtx = "1.2.1"
protobuf = "3.22.0"
protobufJavalite = "3.22.0"
segmentationSelfie = "16.0.0-beta6"
semver4j = "3.1.0"
appcompat = "1.6.1"
material = "1.12.0"
viewpager2 = "1.0.0"
noise = "2.0.0"
lifecycleProcess = "2.8.7"
[libraries]
android-jain-sip-ri = { module = "javax.sip:android-jain-sip-ri", version.ref = "androidJainSipRi" }
... ... @@ -56,6 +58,7 @@ leakcanary-android = { module = "com.squareup.leakcanary:leakcanary-android", ve
okhttp-lib = { module = "com.squareup.okhttp3:okhttp", version.ref = "okhttp" }
okhttp-coroutines = { module = "com.squareup.okhttp3:okhttp", version.ref = "okhttp" }
protobuf-javalite = { module = "com.google.protobuf:protobuf-javalite", version.ref = "protobufJavalite" }
segmentation-selfie = { module = "com.google.mlkit:segmentation-selfie", version.ref = "segmentationSelfie" }
semver4j = { module = "com.vdurmont:semver4j", version.ref = "semver4j" }
webrtc = { module = "io.github.webrtc-sdk:android-prefixed", version.ref = "webrtc" }
... ... @@ -103,6 +106,7 @@ androidx-ui-tooling-preview = { group = "androidx.compose.ui", name = "ui-toolin
androidx-ui-test-manifest = { group = "androidx.compose.ui", name = "ui-test-manifest" }
androidx-ui-test-junit4 = { group = "androidx.compose.ui", name = "ui-test-junit4" }
androidx-material3 = { group = "androidx.compose.material3", name = "material3" }
lifecycle-process = { group = "androidx.lifecycle", name = "lifecycle-process", version.ref = "lifecycleProcess" }
[plugins]
... ...
... ... @@ -12,7 +12,8 @@ Utilities and composables for use with Jetpack Compose.
# Package io.livekit.android.room
Room is the primary class that manages the connection to the LiveKit Room. It exposes listeners that lets you hook into room events.
Room is the primary class that manages the connection to the LiveKit Room. It exposes listeners that
lets you hook into room events.
# Package io.livekit.android.room.track
... ...
/build
\ No newline at end of file
... ...
# Track Processors for LiveKit Android SDK
[![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.livekit/livekit-android-track-processors/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.livekit/livekit-android-camerax)
This library provides track processors for use with the Android LiveKit SDK.
## Installation
```groovy title="build.gradle"
implementation "io.livekit:livekit-android-track-processors:<current livekit sdk release>"
```
See our [release page](https://github.com/livekit/client-sdk-android/releases) for details on the
current release version.
## Usage of prebuilt processors
This package exposes `VirtualBackgroundVideoProcessor` as a pre-prepared video processor.
```
val processor = VirtualBackgroundVideoProcessor(eglBase).apply {
// Optionally set a background image.
// Will blur the background of the video if none is set.
val drawable = AppCompatResources.getDrawable(application, R.drawable.background) as BitmapDrawable
backgroundImage = drawable.bitmap
}
```
### Register the image analyzer in the CameraProvider
`VirtualBackgroundVideoProcessor` requires the use of our CameraX provider.
```
val imageAnalysis = ImageAnalysis.Builder().build()
.apply { setAnalyzer(Dispatchers.IO.asExecutor(), processor.imageAnalyzer) }
CameraXHelper.createCameraProvider(ProcessLifecycleOwner.get(), arrayOf(imageAnalysis)).let {
if (it.isSupported(application)) {
CameraCapturerUtils.registerCameraProvider(it)
}
}
```
### Create and publish the video track
```
val videoTrack = room.localParticipant.createVideoTrack(
options = LocalVideoTrackOptions(position = CameraPosition.FRONT),
videoProcessor = processor,
)
videoTrack.startCapture()
room.localParticipant.publishVideoTrack(videoTrack)
```
You can find an offline example of the `VirtualBackgroundVideoProcessor` in
use [here](https://github.com/livekit/client-sdk-android/tree/main/examples/virtual-background).
... ...
plugins {
id "org.jetbrains.dokka"
id 'com.android.library'
id 'kotlin-android'
id 'kotlin-kapt'
}
android {
namespace 'io.livekit.android.track.processing'
compileSdkVersion androidSdk.compileVersion
defaultConfig {
minSdkVersion androidSdk.minVersion
targetSdkVersion androidSdk.targetVersion
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
consumerProguardFiles "consumer-rules.pro"
}
buildTypes {
release {
minifyEnabled false
proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
}
}
compileOptions {
sourceCompatibility java_version
targetCompatibility java_version
}
kotlinOptions {
freeCompilerArgs = ["-Xinline-classes", "-opt-in=kotlin.RequiresOptIn"]
jvmTarget = java_version
}
testOptions {
unitTests {
includeAndroidResources = true
}
}
}
dokkaHtml {
moduleName.set("livekit-android-test")
dokkaSourceSets {
configureEach {
skipEmptyPackages.set(true)
includeNonPublic.set(false)
includes.from("module.md")
displayName.set("LiveKit Track Processors")
sourceLink {
localDirectory.set(file("src/main/java"))
// URL showing where the source code can be accessed through the web browser
remoteUrl.set(new URL(
"https://github.com/livekit/client-sdk-android/tree/master/livekit-android-track-processors/src/main/java"))
// Suffix which is used to append the line number to the URL. Use #L for GitHub
remoteLineSuffix.set("#L")
}
perPackageOption {
matchingRegex.set(".*\\.dagger.*")
suppress.set(true)
}
perPackageOption {
matchingRegex.set(".*\\.util.*")
suppress.set(true)
}
}
}
}
dependencies {
implementation(project(":livekit-android-sdk"))
implementation(project(":livekit-android-camerax"))
implementation libs.timber
implementation libs.coroutines.lib
implementation libs.androidx.annotation
implementation libs.webrtc
implementation libs.segmentation.selfie
testImplementation libs.junit
testImplementation libs.robolectric
androidTestImplementation libs.androidx.test.junit
androidTestImplementation libs.espresso
}
tasks.withType(Test).configureEach {
systemProperty "robolectric.logging.enabled", true
}
apply from: rootProject.file('gradle/gradle-mvn-push.gradle')
apply from: rootProject.file('gradle/dokka-kotlin-dep-fix.gradle')
afterEvaluate {
publishing {
publications {
// Creates a Maven publication called "release".
release(MavenPublication) {
// Applies the component for the release build variant.
from components.release
// You can then customize attributes of the publication as shown below.
groupId = GROUP
artifactId = POM_ARTIFACT_ID
version = VERSION_NAME
}
}
}
}
... ...
POM_NAME=Track Processors for LiveKit Android SDK
POM_ARTIFACT_ID=livekit-android-track-processors
POM_PACKAGING=aar
... ...
# Module livekit-android-track-processors
Track processors for LiveKit Android SDK.
... ...
# Add project specific ProGuard rules here.
# You can control the set of applied configuration files using the
# proguardFiles setting in build.gradle.
#
# For more details, see
# http://developer.android.com/guide/developing/tools/proguard.html
# If your project uses WebView with JS, uncomment the following
# and specify the fully qualified class name to the JavaScript interface
# class:
#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
# public *;
#}
# Uncomment this to preserve the line number information for
# debugging stack traces.
#-keepattributes SourceFile,LineNumberTable
# If you keep the line number information, uncomment this to
# hide the original source file name.
#-renamesourcefileattribute SourceFile
\ No newline at end of file
... ...
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
</manifest>
... ...
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.livekit.android.track.processing.video
import android.graphics.Bitmap
import android.opengl.GLES20
import android.opengl.GLES30
import io.livekit.android.track.processing.video.opengl.LKGlTextureFrameBuffer
import io.livekit.android.track.processing.video.shader.BlurShader
import io.livekit.android.track.processing.video.shader.CompositeShader
import io.livekit.android.track.processing.video.shader.ResamplerShader
import io.livekit.android.track.processing.video.shader.createBlurShader
import io.livekit.android.track.processing.video.shader.createBoxBlurShader
import io.livekit.android.track.processing.video.shader.createCompsiteShader
import io.livekit.android.track.processing.video.shader.createResampler
import io.livekit.android.util.LKLog
import livekit.org.webrtc.GlTextureFrameBuffer
import livekit.org.webrtc.GlUtil
import livekit.org.webrtc.RendererCommon
import java.nio.ByteBuffer
/**
* Blurs the background of the camera video stream.
*/
class VirtualBackgroundTransformer(
val blurRadius: Float = 16f,
val downSampleFactor: Int = 2,
) : RendererCommon.GlDrawer {
data class MaskHolder(val width: Int, val height: Int, val buffer: ByteBuffer)
private lateinit var compositeShader: CompositeShader
private lateinit var blurShader: BlurShader
private lateinit var boxBlurShader: BlurShader
private var bgTexture = 0
private var frameTexture = 0
private lateinit var bgTextureFrameBuffers: Pair<GlTextureFrameBuffer, GlTextureFrameBuffer>
private lateinit var downSampler: ResamplerShader
var backgroundImageStateLock = Any()
var backgroundImage: Bitmap? = null
set(value) {
if (value == field) {
return
}
synchronized(backgroundImageStateLock) {
field = value
backgroundImageNeedsUploading = true
}
}
var backgroundImageNeedsUploading = false
// For double buffering the final mask
private var readMaskIndex = 0 // Index for renderFrame to read from
private var writeMaskIndex = 1 // Index for updateMask to write to
private fun swapMaskIndexes() {
if (readMaskIndex == 0) {
readMaskIndex = 1
writeMaskIndex = 0
} else {
readMaskIndex = 0
writeMaskIndex = 1
}
}
var newMask: MaskHolder? = null
lateinit var anotherTempMaskFrameBuffer: LKGlTextureFrameBuffer
lateinit var tempMaskTextureFrameBuffer: GlTextureFrameBuffer
lateinit var finalMaskFrameBuffers: List<GlTextureFrameBuffer>
var initialized = false
fun initialize() {
LKLog.e { "initialize shaders" }
compositeShader = createCompsiteShader()
blurShader = createBlurShader()
boxBlurShader = createBoxBlurShader()
bgTexture = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D)
frameTexture = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D)
bgTextureFrameBuffers = GlTextureFrameBuffer(GLES20.GL_RGBA) to GlTextureFrameBuffer(GLES20.GL_RGBA)
downSampler = createResampler()
// For double buffering the final mask
anotherTempMaskFrameBuffer = LKGlTextureFrameBuffer(GLES30.GL_R32F, GLES30.GL_RED, GLES30.GL_FLOAT)
tempMaskTextureFrameBuffer = GlTextureFrameBuffer(GLES20.GL_RGBA)
finalMaskFrameBuffers = listOf(GlTextureFrameBuffer(GLES20.GL_RGBA), GlTextureFrameBuffer(GLES20.GL_RGBA))
GlUtil.checkNoGLES2Error("VirtualBackgroundTransformer.initialize")
initialized = true
}
override fun drawOes(
oesTextureId: Int,
texMatrix: FloatArray,
frameWidth: Int,
frameHeight: Int,
viewportX: Int,
viewportY: Int,
viewportWidth: Int,
viewportHeight: Int,
) {
LKLog.e { "drawOes" }
if (!initialized) {
initialize()
}
newMask?.let {
updateMaskFrameBuffer(it)
newMask = null
}
val backgroundTexture: Int
synchronized(backgroundImageStateLock) {
val backgroundImage = this.backgroundImage
if (backgroundImage != null) {
val bgTextureFrameBuffer = bgTextureFrameBuffers.first
if (backgroundImageNeedsUploading || true) {
val byteBuffer = ByteBuffer.allocateDirect(backgroundImage.byteCount)
backgroundImage.copyPixelsToBuffer(byteBuffer)
byteBuffer.rewind()
// Upload the background into a texture
bgTextureFrameBuffer.setSize(backgroundImage.width, backgroundImage.height)
GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, bgTextureFrameBuffer.textureId)
checkNoError("bindBackgroundTexture")
GLES20.glTexSubImage2D(
/*target*/
GLES20.GL_TEXTURE_2D,
0,
0,
0,
backgroundImage.width,
backgroundImage.height,
/*format*/
GLES20.GL_RGBA,
/*type*/
GLES20.GL_UNSIGNED_BYTE,
byteBuffer,
)
checkNoError("updateBackgroundFrameBuffer")
backgroundImageNeedsUploading = false
}
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0)
backgroundTexture = bgTextureFrameBuffer.textureId
} else {
val downSampleWidth = frameWidth / downSampleFactor
val downSampleHeight = frameHeight / downSampleFactor
val downSampledFrameTexture = downSampler.resample(oesTextureId, downSampleWidth, downSampleHeight, IDENTITY)
backgroundTexture =
blurShader.applyBlur(downSampledFrameTexture, blurRadius, downSampleWidth, downSampleHeight, bgTextureFrameBuffers)
}
}
compositeShader.renderComposite(
backgroundTextureId = backgroundTexture,
frameTextureId = oesTextureId,
maskTextureId = finalMaskFrameBuffers[readMaskIndex].textureId,
viewportX = viewportX,
viewportY = viewportY,
viewportWidth = viewportWidth,
viewportHeight = viewportHeight,
texMatrix = texMatrix,
)
}
/**
* Thread-safe method to set the foreground mask.
*/
fun updateMask(segmentationMask: MaskHolder) {
newMask = segmentationMask
}
private fun updateMaskFrameBuffer(segmentationMask: MaskHolder) {
val width = segmentationMask.width
val height = segmentationMask.height
anotherTempMaskFrameBuffer.setSize(segmentationMask.width, segmentationMask.height)
tempMaskTextureFrameBuffer.setSize(segmentationMask.width, segmentationMask.height)
finalMaskFrameBuffers[0].setSize(segmentationMask.width, segmentationMask.height)
finalMaskFrameBuffers[1].setSize(segmentationMask.width, segmentationMask.height)
// Upload the mask into a texture
GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, anotherTempMaskFrameBuffer.textureId)
checkNoError("bindMaskTexture")
GLES20.glTexSubImage2D(
/*target*/
GLES20.GL_TEXTURE_2D,
0,
0,
0,
width,
height,
/*format*/
GLES30.GL_RED,
/*type*/
GLES20.GL_FLOAT,
segmentationMask.buffer,
)
checkNoError("updateMaskFrameBuffer")
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0)
val finalMaskBuffer = finalMaskFrameBuffers[writeMaskIndex]
val frameBuffers = tempMaskTextureFrameBuffer to finalMaskBuffer
boxBlurShader.applyBlur(anotherTempMaskFrameBuffer.textureId, 2f, width, height, frameBuffers)
// Swap indicies for next frame.
swapMaskIndexes()
}
override fun drawRgb(p0: Int, p1: FloatArray?, p2: Int, p3: Int, p4: Int, p5: Int, p6: Int, p7: Int) {
TODO("Not yet implemented")
}
override fun drawYuv(p0: IntArray?, p1: FloatArray?, p2: Int, p3: Int, p4: Int, p5: Int, p6: Int, p7: Int) {
TODO("Not yet implemented")
}
override fun release() {
compositeShader.release()
blurShader.release()
boxBlurShader.release()
bgTextureFrameBuffers.first.release()
bgTextureFrameBuffers.second.release()
downSampler.release()
anotherTempMaskFrameBuffer.release()
tempMaskTextureFrameBuffer.release()
finalMaskFrameBuffers.forEach {
it.release()
}
}
companion object {
val TAG = VirtualBackgroundTransformer::class.java.simpleName
val IDENTITY =
floatArrayOf(
1f, 0f, 0f, 0f,
0f, 1f, 0f, 0f,
0f, 0f, 1f, 0f,
0f, 0f, 0f, 1f,
)
}
private fun checkNoError(message: String) {
GlUtil.checkNoGLES2Error("$TAG.$message")
}
}
... ...
/*
* Copyright 2024-2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.livekit.android.track.processing.video
import android.graphics.Bitmap
import android.graphics.Matrix
import android.view.Surface
import androidx.annotation.OptIn
import androidx.camera.core.ExperimentalGetImage
import androidx.camera.core.ImageAnalysis
import androidx.camera.core.ImageProxy
import com.google.mlkit.vision.common.InputImage
import com.google.mlkit.vision.segmentation.Segmentation
import com.google.mlkit.vision.segmentation.Segmenter
import com.google.mlkit.vision.segmentation.selfie.SelfieSegmenterOptions
import io.livekit.android.room.track.video.NoDropVideoProcessor
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.cancel
import kotlinx.coroutines.channels.BufferOverflow
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.launch
import livekit.org.webrtc.EglBase
import livekit.org.webrtc.EglRenderer
import livekit.org.webrtc.GlUtil
import livekit.org.webrtc.SurfaceTextureHelper
import livekit.org.webrtc.VideoFrame
import livekit.org.webrtc.VideoSink
import java.util.concurrent.Semaphore
/**
* A virtual background video processor for the local camera video stream.
*
* By default, blurs the background of the video stream.
* Setting [backgroundImage] will use the provided image instead.
*/
class VirtualBackgroundVideoProcessor(private val eglBase: EglBase, dispatcher: CoroutineDispatcher = Dispatchers.Default) : NoDropVideoProcessor() {
private var targetSink: VideoSink? = null
private val segmenter: Segmenter
private var lastRotation = 0
private var lastWidth = 0
private var lastHeight = 0
private val surfaceTextureHelper = SurfaceTextureHelper.create("BitmapToYUV", eglBase.eglBaseContext)
private val surface = Surface(surfaceTextureHelper.surfaceTexture)
private val backgroundTransformer = VirtualBackgroundTransformer()
private val eglRenderer = EglRenderer(VirtualBackgroundVideoProcessor::class.java.simpleName)
.apply {
init(eglBase.eglBaseContext, EglBase.CONFIG_PLAIN, backgroundTransformer)
createEglSurface(surface)
}
private val scope = CoroutineScope(dispatcher)
private val taskFlow = MutableSharedFlow<VideoFrame>(
replay = 0,
extraBufferCapacity = 1,
onBufferOverflow = BufferOverflow.SUSPEND,
)
/**
* Enables or disables the virtual background.
*
* Defaults to true.
*/
var enabled: Boolean = true
var backgroundImage: Bitmap? = null
set(value) {
field = value
backgroundImageNeedsUpdating = true
}
private var backgroundImageNeedsUpdating = false
init {
val options =
SelfieSegmenterOptions.Builder()
.setDetectorMode(SelfieSegmenterOptions.STREAM_MODE)
.build()
segmenter = Segmentation.getClient(options)
// Funnel processing into a single flow that won't buffer,
// since processing may be slower than video capture.
scope.launch {
taskFlow.collect { frame ->
processFrame(frame)
frame.release()
}
}
}
private var lastMask: VirtualBackgroundTransformer.MaskHolder? = null
private inner class ImageAnalyser : ImageAnalysis.Analyzer {
val latch = Semaphore(1, true)
@OptIn(ExperimentalGetImage::class)
override fun analyze(imageProxy: ImageProxy) {
val image = imageProxy.image
if (enabled && image != null) {
// Put 0 for rotation degrees
// We'll rotate it together with the original video frame in the shader.
val inputImage = InputImage.fromMediaImage(image, 0)
latch.acquire()
val task = segmenter.process(inputImage)
task.addOnSuccessListener { mask ->
val holder = VirtualBackgroundTransformer.MaskHolder(mask.width, mask.height, mask.buffer)
lastMask = holder
latch.release()
}
latch.acquire()
latch.release()
}
imageProxy.close()
}
}
@Suppress("unused")
val imageAnalyzer: ImageAnalysis.Analyzer = ImageAnalyser()
override fun onCapturerStarted(started: Boolean) {
if (started) {
surfaceTextureHelper.startListening { frame ->
targetSink?.onFrame(frame)
}
}
}
override fun onCapturerStopped() {
surfaceTextureHelper.stopListening()
}
override fun onFrameCaptured(frame: VideoFrame) {
// If disabled, just pass through to the sink.
if (!enabled) {
targetSink?.onFrame(frame)
return
}
try {
frame.retain()
} catch (e: Exception) {
return
}
// If the frame is succesfully emitted, the process flow will own the frame.
if (!taskFlow.tryEmit(frame)) {
frame.release()
}
}
fun processFrame(frame: VideoFrame) {
if (lastRotation != frame.rotation) {
lastRotation = frame.rotation
backgroundImageNeedsUpdating = true
}
if (lastWidth != frame.rotatedWidth || lastHeight != frame.rotatedHeight) {
surfaceTextureHelper.setTextureSize(frame.rotatedWidth, frame.rotatedHeight)
lastWidth = frame.rotatedWidth
lastHeight = frame.rotatedHeight
backgroundImageNeedsUpdating = true
}
frame.retain()
surfaceTextureHelper.handler.post {
val backgroundImage = this.backgroundImage
if (backgroundImageNeedsUpdating && backgroundImage != null) {
val imageAspect = backgroundImage.width / backgroundImage.height.toFloat()
val targetAspect = frame.rotatedWidth / frame.rotatedHeight.toFloat()
var sx = 0
var sy = 0
var sWidth = backgroundImage.width
var sHeight = backgroundImage.height
if (imageAspect > targetAspect) {
sWidth = Math.round(backgroundImage.height * targetAspect)
sx = Math.round((backgroundImage.width - sWidth) / 2f)
} else {
sHeight = Math.round(backgroundImage.width / targetAspect)
sy = Math.round((backgroundImage.height - sHeight) / 2f)
}
val diffAspect = targetAspect / imageAspect
val matrix = Matrix()
matrix.postRotate(-frame.rotation.toFloat())
val resizedImage = Bitmap.createBitmap(
backgroundImage,
sx,
sy,
sWidth,
sHeight,
matrix,
true,
)
backgroundTransformer.backgroundImage = resizedImage
backgroundImageNeedsUpdating = false
}
lastMask?.let {
backgroundTransformer.updateMask(it)
}
lastMask = null
eglRenderer.onFrame(frame)
frame.release()
}
}
override fun setSink(sink: VideoSink?) {
targetSink = sink
}
fun dispose() {
scope.cancel()
segmenter.close()
surfaceTextureHelper.stopListening()
surfaceTextureHelper.dispose()
surface.release()
eglRenderer.release()
backgroundTransformer.release()
GlUtil.checkNoGLES2Error("VirtualBackgroundVideoProcessor.dispose")
}
}
... ...
/*
* Copyright 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
package io.livekit.android.track.processing.video.opengl;
import android.opengl.GLES20;
import android.opengl.GLES30;
import livekit.org.webrtc.GlUtil;
/**
* Helper class for handling OpenGL framebuffer with only color attachment and no depth or stencil
* buffer. Intended for simple tasks such as texture copy, texture downscaling, and texture color
* conversion. This class is not thread safe and must be used by a thread with an active GL context.
*/
// TODO(magjed): Add unittests for this class.
public class LKGlTextureFrameBuffer {
private final int internalFormat;
private final int pixelFormat;
private final int type;
private int frameBufferId;
private int textureId;
private int width;
private int height;
/**
* Generate texture and framebuffer resources. An EGLContext must be bound on the current thread
* when calling this function. The framebuffer is not complete until setSize() is called.
*/
public LKGlTextureFrameBuffer(int internalFormat, int pixelFormat, int type) {
this.internalFormat = internalFormat;
this.pixelFormat = pixelFormat;
this.type = type;
this.width = 0;
this.height = 0;
}
/**
* (Re)allocate texture. Will do nothing if the requested size equals the current size. An
* EGLContext must be bound on the current thread when calling this function. Must be called at
* least once before using the framebuffer. May be called multiple times to change size.
*/
public void setSize(int width, int height) {
if (width <= 0 || height <= 0) {
throw new IllegalArgumentException("Invalid size: " + width + "x" + height);
}
if (width == this.width && height == this.height) {
return;
}
this.width = width;
this.height = height;
// Lazy allocation the first time setSize() is called.
if (textureId == 0) {
textureId = GlUtil.generateTexture(GLES30.GL_TEXTURE_2D);
}
if (frameBufferId == 0) {
final int frameBuffers[] = new int[1];
GLES30.glGenFramebuffers(1, frameBuffers, 0);
frameBufferId = frameBuffers[0];
}
// Allocate texture.
GLES30.glActiveTexture(GLES30.GL_TEXTURE0);
GLES30.glBindTexture(GLES30.GL_TEXTURE_2D, textureId);
GLES30.glTexImage2D(GLES30.GL_TEXTURE_2D, 0, internalFormat, width, height, 0, pixelFormat,
type, null);
GLES30.glBindTexture(GLES30.GL_TEXTURE_2D, 0);
GlUtil.checkNoGLES2Error("GlTextureFrameBuffer setSize");
// Attach the texture to the framebuffer as color attachment.
GLES30.glBindFramebuffer(GLES30.GL_FRAMEBUFFER, frameBufferId);
GLES30.glFramebufferTexture2D(
GLES30.GL_FRAMEBUFFER, GLES30.GL_COLOR_ATTACHMENT0, GLES30.GL_TEXTURE_2D, textureId, 0);
// Check that the framebuffer is in a good state.
final int status = GLES30.glCheckFramebufferStatus(GLES30.GL_FRAMEBUFFER);
if (status != GLES30.GL_FRAMEBUFFER_COMPLETE) {
throw new IllegalStateException("Framebuffer not complete, status: " + status);
}
GLES30.glBindFramebuffer(GLES30.GL_FRAMEBUFFER, 0);
}
public int getWidth() {
return width;
}
public int getHeight() {
return height;
}
/**
* Gets the OpenGL frame buffer id. This value is only valid after setSize() has been called.
*/
public int getFrameBufferId() {
return frameBufferId;
}
/**
* Gets the OpenGL texture id. This value is only valid after setSize() has been called.
*/
public int getTextureId() {
return textureId;
}
/**
* Release texture and framebuffer. An EGLContext must be bound on the current thread when calling
* this function. This object should not be used after this call.
*/
public void release() {
GLES20.glDeleteTextures(1, new int[]{textureId}, 0);
textureId = 0;
GLES20.glDeleteFramebuffers(1, new int[]{frameBufferId}, 0);
frameBufferId = 0;
width = 0;
height = 0;
}
}
... ...
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.livekit.android.track.processing.video.shader
import android.opengl.GLES20
import livekit.org.webrtc.GlShader
import livekit.org.webrtc.GlTextureFrameBuffer
import livekit.org.webrtc.GlUtil
private const val BLUR_FRAGMENT_SHADER = """#version 300 es
precision mediump float;
in vec2 texCoords;
uniform sampler2D u_texture;
uniform vec2 u_texelSize;
uniform vec2 u_direction;
uniform float u_radius;
out vec4 fragColor;
void main() {
float sigma = u_radius;
float twoSigmaSq = 2.0 * sigma * sigma;
float totalWeight = 0.0;
vec3 result = vec3(0.0);
const int MAX_SAMPLES = 16;
int radius = int(min(float(MAX_SAMPLES), ceil(u_radius)));
for (int i = -MAX_SAMPLES; i <= MAX_SAMPLES; ++i) {
float offset = float(i);
if (abs(offset) > float(radius)) continue;
float weight = exp(-(offset * offset) / twoSigmaSq);
vec2 sampleCoord = texCoords + u_direction * u_texelSize * offset;
result += texture(u_texture, sampleCoord).rgb * weight;
totalWeight += weight;
}
fragColor = vec4(result / totalWeight, 1.0);
}
"""
internal fun createBlurShader(): BlurShader {
val shader = GlShader(CONSTANT_VERTEX_SHADER_SOURCE, BLUR_FRAGMENT_SHADER)
return BlurShader(
shader = shader,
texMatrixLocation = 0,
inPosLocation = shader.getAttribLocation(VERTEX_SHADER_POS_COORD_NAME),
inTcLocation = 0,
texture = shader.getUniformLocation("u_texture"),
texelSize = shader.getUniformLocation("u_texelSize"),
direction = shader.getUniformLocation("u_direction"),
radius = shader.getUniformLocation("u_radius"),
)
}
internal data class BlurShader(
val shader: GlShader,
val inPosLocation: Int,
val inTcLocation: Int,
val texMatrixLocation: Int,
val texture: Int,
val texelSize: Int,
val direction: Int,
val radius: Int,
) {
fun release() {
shader.release()
}
fun applyBlur(
inputTextureId: Int,
blurRadius: Float,
viewportWidth: Int,
viewportHeight: Int,
processFrameBuffer: Pair<GlTextureFrameBuffer, GlTextureFrameBuffer>,
texMatrix: FloatArray? = null,
): Int {
shader.useProgram()
// Upload the texture coordinates.
ShaderUtil.loadCoordMatrix(
inPosLocation = inPosLocation,
inPosFloats = FULL_RECTANGLE_BUFFER,
inTcLocation = inTcLocation,
inTcFloats = if (texMatrix != null) FULL_RECTANGLE_TEXTURE_BUFFER else null,
texMatrixLocation = texMatrixLocation,
texMatrix = texMatrix,
)
GlUtil.checkNoGLES2Error("BlurShader.loadCoordMatrix")
processFrameBuffer.first.setSize(viewportWidth, viewportHeight)
processFrameBuffer.second.setSize(viewportWidth, viewportHeight)
GlUtil.checkNoGLES2Error("BlurShader.updateFrameBufferSizes")
val texelWidth = 1.0f / viewportWidth
val texelHeight = 1.0f / viewportHeight
// First pass - horizontal blur
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, processFrameBuffer.first.frameBufferId)
GLES20.glViewport(0, 0, viewportWidth, viewportHeight)
GlUtil.checkNoGLES2Error("BlurShader.glBindFramebuffer")
GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, inputTextureId)
GlUtil.checkNoGLES2Error("BlurShader.bind oes")
GLES20.glUniform1i(texture, 0)
GLES20.glUniform2f(texelSize, texelWidth, texelHeight)
GLES20.glUniform2f(direction, 1.0f, 0.0f) // Horizontal
GLES20.glUniform1f(radius, blurRadius)
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4)
GlUtil.checkNoGLES2Error("BlurShader.GL_TRIANGLE_STRIP")
// cleanup
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0)
// Second pass - vertical blur
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, processFrameBuffer.second.frameBufferId)
GLES20.glViewport(0, 0, viewportWidth, viewportHeight)
GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, processFrameBuffer.first.textureId)
GLES20.glUniform1i(texture, 0)
GLES20.glUniform2f(texelSize, texelWidth, texelHeight)
GLES20.glUniform2f(direction, 0.0f, 1.0f) // Vertical
GLES20.glUniform1f(radius, blurRadius)
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4)
GlUtil.checkNoGLES2Error("BlurShader.GL_TRIANGLE_STRIP2")
// cleanup
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0)
GlUtil.checkNoGLES2Error("BlurShader.applyBlur")
return processFrameBuffer.second.textureId
}
}
... ...
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.livekit.android.track.processing.video.shader
import livekit.org.webrtc.GlShader
private const val BOX_BLUR_SHADER_SOURCE = """#version 300 es
precision mediump float;
in vec2 texCoords;
uniform sampler2D u_texture;
uniform vec2 u_texelSize; // 1.0 / texture size
uniform vec2 u_direction; // (1.0, 0.0) for horizontal, (0.0, 1.0) for vertical
uniform float u_radius; // blur radius in texels
out vec4 fragColor;
void main() {
vec3 sum = vec3(0.0);
float count = 0.0;
// Limit radius to avoid excessive loop cost
const int MAX_RADIUS = 16;
int radius = int(min(float(MAX_RADIUS), u_radius));
for (int i = -MAX_RADIUS; i <= MAX_RADIUS; ++i) {
if (abs(i) > radius) continue;
vec2 offset = u_direction * u_texelSize * float(i);
sum += texture(u_texture, texCoords + offset).rgb;
count += 1.0;
}
fragColor = vec4(sum / count, 1.0);
}
"""
internal fun createBoxBlurShader(): BlurShader {
val shader = GlShader(CONSTANT_VERTEX_SHADER_SOURCE, BOX_BLUR_SHADER_SOURCE)
return BlurShader(
shader = shader,
texMatrixLocation = 0,
inPosLocation = shader.getAttribLocation(VERTEX_SHADER_POS_COORD_NAME),
inTcLocation = 0,
texture = shader.getUniformLocation("u_texture"),
texelSize = shader.getUniformLocation("u_texelSize"),
direction = shader.getUniformLocation("u_direction"),
radius = shader.getUniformLocation("u_radius"),
)
}
... ...
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.livekit.android.track.processing.video.shader
import android.opengl.GLES11Ext
import android.opengl.GLES20
import livekit.org.webrtc.GlShader
import livekit.org.webrtc.GlUtil
private const val COMPOSITE_FRAGMENT_SHADER_SOURCE = """#version 300 es
#extension GL_OES_EGL_image_external_essl3 : require
precision mediump float;
in vec2 texCoords;
uniform sampler2D background;
uniform samplerExternalOES frame;
uniform sampler2D mask;
out vec4 fragColor;
void main() {
vec4 frameTex = texture(frame, texCoords);
vec4 bgTex = texture(background, texCoords);
float maskVal = texture(mask, texCoords).r;
// Compute screen-space gradient to detect edge sharpness
float grad = length(vec2(dFdx(maskVal), dFdy(maskVal)));
float edgeSoftness = 6.0; // higher = softer
// Create a smooth edge around binary transition
float smoothAlpha = smoothstep(0.5 - grad * edgeSoftness, 0.5 + grad * edgeSoftness, maskVal);
// Optional: preserve frame alpha, or override as fully opaque
vec4 blended = mix(bgTex, vec4(frameTex.rgb, 1.0), 0.0 + smoothAlpha);
fragColor = blended;
}
"""
internal fun createCompsiteShader(): CompositeShader {
val shader = GlShader(DEFAULT_VERTEX_SHADER_SOURCE, COMPOSITE_FRAGMENT_SHADER_SOURCE)
return CompositeShader(
shader = shader,
texMatrixLocation = shader.getUniformLocation(VERTEX_SHADER_TEX_MAT_NAME),
inPosLocation = shader.getAttribLocation(VERTEX_SHADER_POS_COORD_NAME),
inTcLocation = shader.getAttribLocation(VERTEX_SHADER_TEX_COORD_NAME),
mask = shader.getUniformLocation("mask"),
frame = shader.getUniformLocation("frame"),
background = shader.getUniformLocation("background"),
)
}
internal data class CompositeShader(
val shader: GlShader,
val inPosLocation: Int,
val inTcLocation: Int,
val texMatrixLocation: Int,
val mask: Int,
val frame: Int,
val background: Int,
) {
fun renderComposite(
backgroundTextureId: Int,
frameTextureId: Int,
maskTextureId: Int,
viewportX: Int,
viewportY: Int,
viewportWidth: Int,
viewportHeight: Int,
texMatrix: FloatArray,
) {
GLES20.glViewport(viewportX, viewportY, viewportWidth, viewportHeight)
GLES20.glClearColor(1f, 1f, 1f, 1f)
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT)
// Set up uniforms for the composite shader
shader.useProgram()
ShaderUtil.loadCoordMatrix(
inPosLocation = inPosLocation,
inPosFloats = FULL_RECTANGLE_BUFFER,
inTcLocation = inTcLocation,
inTcFloats = FULL_RECTANGLE_TEXTURE_BUFFER,
texMatrixLocation = texMatrixLocation,
texMatrix = texMatrix,
)
GlUtil.checkNoGLES2Error("loadCoordMatrix")
GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, backgroundTextureId)
GLES20.glUniform1i(background, 0)
GlUtil.checkNoGLES2Error("GL_TEXTURE0")
GLES20.glActiveTexture(GLES20.GL_TEXTURE1)
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, frameTextureId)
GLES20.glUniform1i(frame, 1)
GlUtil.checkNoGLES2Error("GL_TEXTURE1")
GLES20.glActiveTexture(GLES20.GL_TEXTURE2)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, maskTextureId)
GLES20.glUniform1i(mask, 2)
GlUtil.checkNoGLES2Error("GL_TEXTURE2")
// Draw composite
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4)
GlUtil.checkNoGLES2Error("GL_TRIANGLE_STRIP")
// Cleanup
GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0)
GLES20.glActiveTexture(GLES20.GL_TEXTURE1)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0)
GLES20.glActiveTexture(GLES20.GL_TEXTURE2)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0)
GlUtil.checkNoGLES2Error("renderComposite")
}
fun release() {
shader.release()
}
}
... ...
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.livekit.android.track.processing.video.shader
internal const val DEFAULT_VERTEX_SHADER_SOURCE = """#version 300 es
out vec2 texCoords;
in vec4 in_pos;
in vec4 in_tc;
uniform mat4 tex_mat;
void main() {
gl_Position = in_pos;
texCoords = (tex_mat * in_tc).xy;
}
"""
internal const val CONSTANT_VERTEX_SHADER_SOURCE = """#version 300 es
in vec2 in_pos;
out vec2 texCoords;
void main() {
texCoords = (in_pos + 1.0) / 2.0;
gl_Position = vec4(in_pos, 0, 1.0);
}
"""
internal const val VERTEX_SHADER_TEX_MAT_NAME = "tex_mat"
internal const val VERTEX_SHADER_TEX_COORD_NAME = "in_tc"
internal const val VERTEX_SHADER_POS_COORD_NAME = "in_pos"
... ...
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.livekit.android.track.processing.video.shader
import android.opengl.GLES11Ext
import android.opengl.GLES20
import livekit.org.webrtc.GlShader
import livekit.org.webrtc.GlTextureFrameBuffer
import livekit.org.webrtc.GlUtil
private const val DOWNSAMPLER_VERTEX_SHADER_SOURCE = """
attribute vec4 in_pos;
attribute vec4 in_tc;
uniform mat4 tex_mat;
varying vec2 v_uv;
void main() {
v_uv = (tex_mat * in_tc).xy;
gl_Position = in_pos;
}
"""
private const val DOWNSAMPLER_FRAGMENT_SHADER_SOURCE = """#extension GL_OES_EGL_image_external : require
precision mediump float;
varying vec2 v_uv;
uniform samplerExternalOES u_texture;
void main() {
gl_FragColor = texture2D(u_texture, v_uv);
}
"""
// Vertex coordinates in Normalized Device Coordinates, i.e. (-1, -1) is bottom-left and (1, 1)
// is top-right.
internal val FULL_RECTANGLE_BUFFER = GlUtil.createFloatBuffer(
floatArrayOf(
-1.0f,
-1.0f, // Bottom left.
1.0f,
-1.0f, // Bottom right.
-1.0f,
1.0f, // Top left.
1.0f,
1.0f, // Top right.
),
)
// Texture coordinates - (0, 0) is bottom-left and (1, 1) is top-right.
internal val FULL_RECTANGLE_TEXTURE_BUFFER = GlUtil.createFloatBuffer(
floatArrayOf(
0.0f,
0.0f, // Bottom left.
1.0f,
0.0f, // Bottom right.
0.0f,
1.0f, // Top left.
1.0f,
1.0f, // Top right.
),
)
internal fun createResampler(): ResamplerShader {
val textureFrameBuffer = GlTextureFrameBuffer(GLES20.GL_RGBA)
val shader = GlShader(DOWNSAMPLER_VERTEX_SHADER_SOURCE, DOWNSAMPLER_FRAGMENT_SHADER_SOURCE)
return ResamplerShader(
shader = shader,
textureFrameBuffer = textureFrameBuffer,
texMatrixLocation = shader.getUniformLocation(VERTEX_SHADER_TEX_MAT_NAME),
inPosLocation = shader.getAttribLocation(VERTEX_SHADER_POS_COORD_NAME),
inTcLocation = shader.getAttribLocation(VERTEX_SHADER_TEX_COORD_NAME),
texture = shader.getUniformLocation("u_texture"),
)
}
/**
* A shader that resamples a texture at a new size.
*/
internal data class ResamplerShader(
val shader: GlShader,
val textureFrameBuffer: GlTextureFrameBuffer,
val texMatrixLocation: Int,
val inPosLocation: Int,
val inTcLocation: Int,
val texture: Int,
) {
fun release() {
shader.release()
textureFrameBuffer.release()
}
fun resample(
inputTexture: Int,
newWidth: Int,
newHeight: Int,
texMatrix: FloatArray,
): Int {
textureFrameBuffer.setSize(newWidth, newHeight)
shader.useProgram()
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, textureFrameBuffer.frameBufferId)
GLES20.glViewport(0, 0, newWidth, newHeight)
ShaderUtil.loadCoordMatrix(
inPosLocation = inPosLocation,
inPosFloats = FULL_RECTANGLE_BUFFER,
inTcLocation = inTcLocation,
inTcFloats = FULL_RECTANGLE_TEXTURE_BUFFER,
texMatrixLocation = texMatrixLocation,
texMatrix = texMatrix,
)
GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, inputTexture)
GlUtil.checkNoGLES2Error("ResamplerShader.glBindTexture")
GLES20.glUniform1i(texture, 0)
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4)
GlUtil.checkNoGLES2Error("ResamplerShader.glDrawArrays")
// cleanup
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0)
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0)
GlUtil.checkNoGLES2Error("ResamplerShader.applyDownsampling")
return textureFrameBuffer.textureId
}
}
... ...
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.livekit.android.track.processing.video.shader
import android.opengl.GLES20
import java.nio.FloatBuffer
internal object ShaderUtil {
fun loadCoordMatrix(
inPosLocation: Int,
inPosFloats: FloatBuffer? = null,
inTcLocation: Int,
inTcFloats: FloatBuffer? = null,
texMatrixLocation: Int,
texMatrix: FloatArray? = null,
) {
if (inPosFloats != null) {
// Upload the vertex coordinates.
GLES20.glEnableVertexAttribArray(inPosLocation)
GLES20.glVertexAttribPointer(
inPosLocation,
/* size= */
2,
/* type= */
GLES20.GL_FLOAT,
/* normalized= */
false,
/* stride= */
0,
inPosFloats,
)
}
if (inTcFloats != null) {
// Upload the texture coordinates.
GLES20.glEnableVertexAttribArray(inTcLocation)
GLES20.glVertexAttribPointer(
inTcLocation,
/* size= */
2,
/* type= */
GLES20.GL_FLOAT,
/* normalized= */
false,
/* stride= */
0,
inTcFloats,
)
}
if (texMatrix != null) {
// Upload the texture transformation matrix.
GLES20.glUniformMatrix4fv(
texMatrixLocation,
/* count= */
1,
/* transpose= */
false,
texMatrix,
/* offset= */
0,
)
}
}
}
... ...
... ... @@ -96,7 +96,9 @@ class CallViewModel(
appContext = application,
options = getRoomOptions(),
overrides = LiveKitOverrides(
audioOptions = AudioOptions(audioProcessorOptions = audioProcessorOptions),
audioOptions = AudioOptions(
audioProcessorOptions = audioProcessorOptions,
),
),
)
... ...
... ... @@ -26,7 +26,8 @@ include ':livekit-lint'
include ':video-encode-decode-test'
include ':sample-app-basic'
include ':sample-app-record-local'
include ':examples:selfie-segmentation'
include ':examples:virtual-background'
include ':livekit-android-test'
include ':livekit-android-camerax'
include ':examples:screenshare-audio'
include ':livekit-android-track-processors'
... ...