davidliu
Committed by GitHub

Selfie ML video processing examples (#378)

* Selfie ML video processing examples

* spotless

* cleanup
正在显示 35 个修改的文件 包含 912 行增加4 行删除
  1 +<?xml version="1.0" encoding="UTF-8"?>
  2 +<project version="4">
  3 + <component name="ProjectMigrations">
  4 + <option name="MigrateToGradleLocalJavaHome">
  5 + <set>
  6 + <option value="$PROJECT_DIR$" />
  7 + </set>
  8 + </option>
  9 + </component>
  10 +</project>
@@ -23,6 +23,11 @@ buildscript { @@ -23,6 +23,11 @@ buildscript {
23 apply plugin: 'io.codearte.nexus-staging' 23 apply plugin: 'io.codearte.nexus-staging'
24 24
25 subprojects { 25 subprojects {
  26 + // Ignore examples folder, it's not a module itself.
  27 + if (project.name == "examples") {
  28 + return
  29 + }
  30 +
26 repositories { 31 repositories {
27 google() 32 google()
28 mavenCentral() 33 mavenCentral()
1 ext { 1 ext {
2 - android_build_tools_version = '8.0.2' 2 + android_build_tools_version = '8.2.2'
3 compose_version = '1.2.1' 3 compose_version = '1.2.1'
4 compose_compiler_version = '1.4.5' 4 compose_compiler_version = '1.4.5'
5 kotlin_version = '1.8.20' 5 kotlin_version = '1.8.20'
6 java_version = JavaVersion.VERSION_1_8 6 java_version = JavaVersion.VERSION_1_8
7 dokka_version = '1.5.0' 7 dokka_version = '1.5.0'
8 androidSdk = [ 8 androidSdk = [
9 - compileVersion: 33,  
10 - targetVersion : 33, 9 + compileVersion: 34,
  10 + targetVersion : 34,
11 minVersion : 21, 11 minVersion : 21,
12 ] 12 ]
13 versions = [ 13 versions = [
  1 +plugins {
  2 + id 'com.android.application'
  3 + id 'org.jetbrains.kotlin.android'
  4 +}
  5 +
  6 +android {
  7 + namespace 'io.livekit.android.selfie'
  8 + compileSdk androidSdk.compileVersion
  9 +
  10 + defaultConfig {
  11 + applicationId "io.livekit.android.selfie"
  12 + minSdk androidSdk.minVersion
  13 + targetSdk androidSdk.targetVersion
  14 + versionCode 1
  15 + versionName "1.0"
  16 +
  17 + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
  18 + }
  19 +
  20 + buildTypes {
  21 + release {
  22 + minifyEnabled false
  23 + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
  24 + }
  25 + }
  26 + compileOptions {
  27 + sourceCompatibility java_version
  28 + targetCompatibility java_version
  29 + }
  30 + kotlinOptions {
  31 + jvmTarget = java_version
  32 + }
  33 +}
  34 +
  35 +dependencies {
  36 + implementation 'com.google.mlkit:segmentation-selfie:16.0.0-beta4'
  37 +
  38 + api project(":livekit-android-sdk")
  39 +
  40 + api "androidx.core:core-ktx:${versions.androidx_core}"
  41 + implementation 'androidx.appcompat:appcompat:1.6.1'
  42 + implementation 'com.google.android.material:material:1.11.0'
  43 + api deps.coroutines.lib
  44 + api "androidx.lifecycle:lifecycle-runtime-ktx:${versions.androidx_lifecycle}"
  45 + api "androidx.lifecycle:lifecycle-viewmodel-ktx:${versions.androidx_lifecycle}"
  46 + api "androidx.lifecycle:lifecycle-common-java8:${versions.androidx_lifecycle}"
  47 + testImplementation 'junit:junit:4.13.2'
  48 + androidTestImplementation 'androidx.test.ext:junit:1.1.5'
  49 + androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1'
  50 +}
  1 +# Add project specific ProGuard rules here.
  2 +# You can control the set of applied configuration files using the
  3 +# proguardFiles setting in build.gradle.
  4 +#
  5 +# For more details, see
  6 +# http://developer.android.com/guide/developing/tools/proguard.html
  7 +
  8 +# If your project uses WebView with JS, uncomment the following
  9 +# and specify the fully qualified class name to the JavaScript interface
  10 +# class:
  11 +#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
  12 +# public *;
  13 +#}
  14 +
  15 +# Uncomment this to preserve the line number information for
  16 +# debugging stack traces.
  17 +#-keepattributes SourceFile,LineNumberTable
  18 +
  19 +# If you keep the line number information, uncomment this to
  20 +# hide the original source file name.
  21 +#-renamesourcefileattribute SourceFile
  1 +/*
  2 + * Copyright 2024 LiveKit, Inc.
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +
  17 +package io.livekit.android.selfie
  18 +
  19 +import androidx.test.ext.junit.runners.AndroidJUnit4
  20 +import androidx.test.platform.app.InstrumentationRegistry
  21 +import org.junit.Assert.*
  22 +import org.junit.Test
  23 +import org.junit.runner.RunWith
  24 +
  25 +/**
  26 + * Instrumented test, which will execute on an Android device.
  27 + *
  28 + * See [testing documentation](http://d.android.com/tools/testing).
  29 + */
  30 +@RunWith(AndroidJUnit4::class)
  31 +class ExampleInstrumentedTest {
  32 + @Test
  33 + fun useAppContext() {
  34 + // Context of the app under test.
  35 + val appContext = InstrumentationRegistry.getInstrumentation().targetContext
  36 + assertEquals("io.livekit.android.selfie", appContext.packageName)
  37 + }
  38 +}
  1 +<?xml version="1.0" encoding="utf-8"?>
  2 +<manifest xmlns:android="http://schemas.android.com/apk/res/android">
  3 +
  4 + <application
  5 + android:allowBackup="true"
  6 + android:icon="@mipmap/ic_launcher"
  7 + android:label="@string/app_name"
  8 + android:roundIcon="@mipmap/ic_launcher_round"
  9 + android:supportsRtl="true"
  10 + android:theme="@style/Theme.Livekitandroid">
  11 + <activity
  12 + android:name=".MainActivity"
  13 + android:exported="true">
  14 + <intent-filter>
  15 + <action android:name="android.intent.action.MAIN" />
  16 +
  17 + <category android:name="android.intent.category.LAUNCHER" />
  18 + </intent-filter>
  19 + </activity>
  20 + </application>
  21 +
  22 +</manifest>
  1 +/*
  2 + * Copyright 2024 LiveKit, Inc.
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +
  17 +package io.livekit.android.selfie
  18 +
  19 +import android.Manifest
  20 +import android.content.pm.PackageManager
  21 +import android.os.Bundle
  22 +import android.widget.Toast
  23 +import androidx.activity.ComponentActivity
  24 +import androidx.activity.result.contract.ActivityResultContracts
  25 +import androidx.appcompat.app.AppCompatActivity
  26 +import androidx.core.content.ContextCompat
  27 +import androidx.lifecycle.ViewModelProvider
  28 +import io.livekit.android.renderer.TextureViewRenderer
  29 +
  30 +class MainActivity : AppCompatActivity() {
  31 +
  32 + lateinit var viewModel: MainViewModel
  33 +
  34 + override fun onCreate(savedInstanceState: Bundle?) {
  35 + super.onCreate(savedInstanceState)
  36 + setContentView(R.layout.activity_main)
  37 +
  38 + viewModel = ViewModelProvider(this)[MainViewModel::class.java]
  39 +
  40 + val renderer = findViewById<TextureViewRenderer>(R.id.renderer)
  41 + viewModel.room.initVideoRenderer(renderer)
  42 + viewModel.track.observe(this) { track ->
  43 + track?.addRenderer(renderer)
  44 + }
  45 +
  46 + requestNeededPermissions {
  47 + viewModel.startCapture()
  48 + }
  49 + }
  50 +}
  51 +
  52 +fun ComponentActivity.requestNeededPermissions(onPermissionsGranted: (() -> Unit)? = null) {
  53 + val requestPermissionLauncher =
  54 + registerForActivityResult(
  55 + ActivityResultContracts.RequestMultiplePermissions(),
  56 + ) { grants ->
  57 + // Check if any permissions weren't granted.
  58 + for (grant in grants.entries) {
  59 + if (!grant.value) {
  60 + Toast.makeText(
  61 + this,
  62 + "Missing permission: ${grant.key}",
  63 + Toast.LENGTH_SHORT,
  64 + )
  65 + .show()
  66 + }
  67 + }
  68 +
  69 + // If all granted, notify if needed.
  70 + if (onPermissionsGranted != null && grants.all { it.value }) {
  71 + onPermissionsGranted()
  72 + }
  73 + }
  74 +
  75 + val neededPermissions = listOf(Manifest.permission.RECORD_AUDIO, Manifest.permission.CAMERA)
  76 + .filter { ContextCompat.checkSelfPermission(this, it) == PackageManager.PERMISSION_DENIED }
  77 + .toTypedArray()
  78 +
  79 + if (neededPermissions.isNotEmpty()) {
  80 + requestPermissionLauncher.launch(neededPermissions)
  81 + } else {
  82 + onPermissionsGranted?.invoke()
  83 + }
  84 +}
  1 +/*
  2 + * Copyright 2024 LiveKit, Inc.
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +
  17 +package io.livekit.android.selfie
  18 +
  19 +import android.app.Application
  20 +import androidx.lifecycle.AndroidViewModel
  21 +import androidx.lifecycle.MutableLiveData
  22 +import io.livekit.android.LiveKit
  23 +import io.livekit.android.LiveKitOverrides
  24 +import io.livekit.android.room.track.CameraPosition
  25 +import io.livekit.android.room.track.LocalVideoTrack
  26 +import io.livekit.android.room.track.LocalVideoTrackOptions
  27 +import kotlinx.coroutines.Dispatchers
  28 +import livekit.org.webrtc.EglBase
  29 +
  30 +class MainViewModel(application: Application) : AndroidViewModel(application) {
  31 +
  32 + val eglBase = EglBase.create()
  33 + val room = LiveKit.create(
  34 + application,
  35 + overrides = LiveKitOverrides(
  36 + eglBase = eglBase,
  37 + ),
  38 + )
  39 +
  40 + val track = MutableLiveData<LocalVideoTrack?>(null)
  41 +
  42 + // For direct I420 processing:
  43 + // val processor = SelfieVideoProcessor(Dispatchers.IO)
  44 + val processor = SelfieBitmapVideoProcessor(eglBase, Dispatchers.IO)
  45 +
  46 + fun startCapture() {
  47 + val selfieVideoTrack = room.localParticipant.createVideoTrack(
  48 + options = LocalVideoTrackOptions(position = CameraPosition.FRONT),
  49 + videoProcessor = processor,
  50 + )
  51 +
  52 + selfieVideoTrack.startCapture()
  53 + track.postValue(selfieVideoTrack)
  54 + }
  55 +
  56 + override fun onCleared() {
  57 + super.onCleared()
  58 + track.value?.stopCapture()
  59 + room.release()
  60 + processor.dispose()
  61 + }
  62 +}
  1 +/*
  2 + * Copyright 2024 LiveKit, Inc.
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +
  17 +package io.livekit.android.selfie
  18 +
  19 +import android.graphics.BitmapFactory
  20 +import android.graphics.Color
  21 +import android.graphics.ImageFormat
  22 +import android.graphics.Matrix
  23 +import android.graphics.Paint
  24 +import android.graphics.Rect
  25 +import android.graphics.YuvImage
  26 +import android.os.Build
  27 +import android.util.Log
  28 +import android.view.Surface
  29 +import androidx.core.graphics.set
  30 +import com.google.mlkit.vision.common.InputImage
  31 +import com.google.mlkit.vision.segmentation.Segmentation
  32 +import com.google.mlkit.vision.segmentation.Segmenter
  33 +import com.google.mlkit.vision.segmentation.selfie.SelfieSegmenterOptions
  34 +import kotlinx.coroutines.CoroutineDispatcher
  35 +import kotlinx.coroutines.CoroutineScope
  36 +import kotlinx.coroutines.cancel
  37 +import kotlinx.coroutines.channels.BufferOverflow
  38 +import kotlinx.coroutines.flow.MutableSharedFlow
  39 +import kotlinx.coroutines.launch
  40 +import kotlinx.coroutines.sync.Mutex
  41 +import livekit.org.webrtc.EglBase
  42 +import livekit.org.webrtc.SurfaceTextureHelper
  43 +import livekit.org.webrtc.VideoFrame
  44 +import livekit.org.webrtc.VideoProcessor
  45 +import livekit.org.webrtc.VideoSink
  46 +import livekit.org.webrtc.YuvHelper
  47 +import java.io.ByteArrayOutputStream
  48 +import java.nio.ByteBuffer
  49 +
  50 +class SelfieBitmapVideoProcessor(eglBase: EglBase, dispatcher: CoroutineDispatcher) : VideoProcessor {
  51 +
  52 + private var targetSink: VideoSink? = null
  53 + private val segmenter: Segmenter
  54 +
  55 + private var lastRotation = 0
  56 + private var lastWidth = 0
  57 + private var lastHeight = 0
  58 + private val surfaceTextureHelper = SurfaceTextureHelper.create("BitmapToYUV", eglBase.eglBaseContext)
  59 + private val surface = Surface(surfaceTextureHelper.surfaceTexture)
  60 +
  61 + private val scope = CoroutineScope(dispatcher)
  62 + private val taskFlow = MutableSharedFlow<VideoFrame>(
  63 + replay = 0,
  64 + extraBufferCapacity = 1,
  65 + onBufferOverflow = BufferOverflow.SUSPEND,
  66 + )
  67 +
  68 + init {
  69 + val options =
  70 + SelfieSegmenterOptions.Builder()
  71 + .setDetectorMode(SelfieSegmenterOptions.STREAM_MODE)
  72 + .build()
  73 + segmenter = Segmentation.getClient(options)
  74 +
  75 + // Funnel processing into a single flow that won't buffer,
  76 + // since processing will be slower than video capture
  77 + scope.launch {
  78 + taskFlow.collect { frame ->
  79 + processFrame(frame)
  80 + }
  81 + }
  82 + }
  83 +
  84 + override fun onCapturerStarted(started: Boolean) {
  85 + if (started) {
  86 + surfaceTextureHelper.startListening { frame ->
  87 + targetSink?.onFrame(frame)
  88 + }
  89 + }
  90 + }
  91 +
  92 + override fun onCapturerStopped() {
  93 + surfaceTextureHelper.stopListening()
  94 + }
  95 +
  96 + override fun onFrameCaptured(frame: VideoFrame) {
  97 + if (taskFlow.tryEmit(frame)) {
  98 + frame.retain()
  99 + }
  100 + }
  101 +
  102 + suspend fun processFrame(frame: VideoFrame) {
  103 + // toI420 causes a retain, so a corresponding frameBuffer.release is needed when done.
  104 + val frameBuffer = frame.buffer.toI420() ?: return
  105 + val rotationDegrees = frame.rotation
  106 +
  107 + val dataY = frameBuffer.dataY
  108 + val dataU = frameBuffer.dataU
  109 + val dataV = frameBuffer.dataV
  110 + val nv12Buffer = ByteBuffer.allocateDirect(dataY.limit() + dataU.limit() + dataV.limit())
  111 +
  112 + // For some reason, I420ToNV12 actually expects YV12
  113 + YuvHelper.I420ToNV12(
  114 + frameBuffer.dataY,
  115 + frameBuffer.strideY,
  116 + frameBuffer.dataV,
  117 + frameBuffer.strideV,
  118 + frameBuffer.dataU,
  119 + frameBuffer.strideU,
  120 + nv12Buffer,
  121 + frameBuffer.width,
  122 + frameBuffer.height,
  123 + )
  124 +
  125 + // Use YuvImage to convert to bitmap
  126 + val yuvImage = YuvImage(nv12Buffer.array(), ImageFormat.NV21, frameBuffer.width, frameBuffer.height, null)
  127 + val stream = ByteArrayOutputStream()
  128 + yuvImage.compressToJpeg(Rect(0, 0, frameBuffer.width, frameBuffer.height), 100, stream)
  129 +
  130 + val bitmap = BitmapFactory.decodeByteArray(
  131 + stream.toByteArray(),
  132 + 0,
  133 + stream.size(),
  134 + BitmapFactory.Options().apply { inMutable = true },
  135 + )
  136 +
  137 + // No longer need the original frame buffer any more.
  138 + frameBuffer.release()
  139 + frame.release()
  140 +
  141 + val inputImage = InputImage.fromBitmap(bitmap, 0)
  142 + val task = segmenter.process(inputImage)
  143 +
  144 + val latch = Mutex(true)
  145 + task.addOnSuccessListener { segmentationMask ->
  146 + val mask = segmentationMask.buffer
  147 +
  148 + // Do some image processing
  149 + for (y in 0 until segmentationMask.height) {
  150 + for (x in 0 until segmentationMask.width) {
  151 + val backgroundConfidence = 1 - mask.float
  152 +
  153 + if (backgroundConfidence > 0.8f) {
  154 + bitmap[x, y] = Color.GREEN // Color off the background
  155 + }
  156 + }
  157 + }
  158 +
  159 + if (lastRotation != rotationDegrees) {
  160 + surfaceTextureHelper?.setFrameRotation(rotationDegrees)
  161 + lastRotation = rotationDegrees
  162 + }
  163 +
  164 + if (lastWidth != bitmap.width || lastHeight != bitmap.height) {
  165 + surfaceTextureHelper?.setTextureSize(bitmap.width, bitmap.height)
  166 + lastWidth = bitmap.width
  167 + lastHeight = bitmap.height
  168 + }
  169 +
  170 + surfaceTextureHelper?.handler?.post {
  171 + val canvas = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
  172 + surface.lockHardwareCanvas()
  173 + } else {
  174 + surface.lockCanvas(null)
  175 + }
  176 +
  177 + if (canvas != null) {
  178 + canvas.drawBitmap(bitmap, Matrix(), Paint())
  179 + surface.unlockCanvasAndPost(canvas)
  180 + }
  181 + bitmap.recycle()
  182 + latch.unlock()
  183 + }
  184 + }.addOnFailureListener {
  185 + Log.e("SelfieVideoProcessor", "failed to process frame!")
  186 + }
  187 + latch.lock()
  188 + }
  189 +
  190 + override fun setSink(sink: VideoSink?) {
  191 + targetSink = sink
  192 + }
  193 +
  194 + fun dispose() {
  195 + segmenter.close()
  196 + surfaceTextureHelper.stopListening()
  197 + surfaceTextureHelper.dispose()
  198 + scope.cancel()
  199 + }
  200 +}
  1 +/*
  2 + * Copyright 2024 LiveKit, Inc.
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +
  17 +package io.livekit.android.selfie
  18 +
  19 +import android.util.Log
  20 +import com.google.mlkit.vision.common.InputImage
  21 +import com.google.mlkit.vision.segmentation.Segmentation
  22 +import com.google.mlkit.vision.segmentation.Segmenter
  23 +import com.google.mlkit.vision.segmentation.selfie.SelfieSegmenterOptions
  24 +import kotlinx.coroutines.CoroutineDispatcher
  25 +import kotlinx.coroutines.CoroutineScope
  26 +import kotlinx.coroutines.channels.BufferOverflow
  27 +import kotlinx.coroutines.flow.MutableSharedFlow
  28 +import kotlinx.coroutines.launch
  29 +import livekit.org.webrtc.VideoFrame
  30 +import livekit.org.webrtc.VideoProcessor
  31 +import livekit.org.webrtc.VideoSink
  32 +import java.nio.ByteBuffer
  33 +
  34 +class SelfieVideoProcessor(dispatcher: CoroutineDispatcher) : VideoProcessor {
  35 +
  36 + private var targetSink: VideoSink? = null
  37 + private val segmenter: Segmenter
  38 +
  39 + private val scope = CoroutineScope(dispatcher)
  40 + private val taskFlow = MutableSharedFlow<VideoFrame>(
  41 + replay = 0,
  42 + extraBufferCapacity = 1,
  43 + onBufferOverflow = BufferOverflow.SUSPEND,
  44 + )
  45 +
  46 + init {
  47 + val options =
  48 + SelfieSegmenterOptions.Builder()
  49 + .setDetectorMode(SelfieSegmenterOptions.STREAM_MODE)
  50 + .build()
  51 + segmenter = Segmentation.getClient(options)
  52 +
  53 + // Funnel processing into a single flow that won't buffer,
  54 + // since processing will be slower than video capture
  55 + scope.launch {
  56 + taskFlow.collect { frame ->
  57 + processFrame(frame)
  58 + }
  59 + }
  60 + }
  61 +
  62 + override fun onCapturerStarted(started: Boolean) {
  63 + }
  64 +
  65 + override fun onCapturerStopped() {
  66 + }
  67 +
  68 + override fun onFrameCaptured(frame: VideoFrame) {
  69 + if (taskFlow.tryEmit(frame)) {
  70 + frame.retain()
  71 + }
  72 + }
  73 +
  74 + fun processFrame(frame: VideoFrame) {
  75 + // toI420 causes a retain, so a corresponding frameBuffer.release is needed when done.
  76 + val frameBuffer = frame.buffer.toI420() ?: return
  77 + val byteBuffer = ByteBuffer.allocateDirect(frameBuffer.dataY.limit() + frameBuffer.dataV.limit() + frameBuffer.dataU.limit())
  78 + // YV12 is exactly like I420, but the order of the U and V planes is reversed.
  79 + // In the name, "YV" refers to the plane order: Y, then V (then U).
  80 + .put(frameBuffer.dataY)
  81 + .put(frameBuffer.dataV)
  82 + .put(frameBuffer.dataU)
  83 +
  84 + val image = InputImage.fromByteBuffer(
  85 + byteBuffer,
  86 + frameBuffer.width,
  87 + frameBuffer.height,
  88 + 0,
  89 + InputImage.IMAGE_FORMAT_YV12,
  90 + )
  91 +
  92 + val task = segmenter.process(image)
  93 + task.addOnSuccessListener { segmentationMask ->
  94 + val mask = segmentationMask.buffer
  95 +
  96 + val dataY = frameBuffer.dataY
  97 +
  98 + // Do some image processing
  99 + for (i in 0 until segmentationMask.height) {
  100 + for (j in 0 until segmentationMask.width) {
  101 + val backgroundConfidence = 1 - mask.float
  102 +
  103 + if (backgroundConfidence > 0.8f) {
  104 + val position = dataY.position()
  105 + val yValue = 0x80.toByte()
  106 + dataY.position(position)
  107 + dataY.put(yValue)
  108 + } else {
  109 + dataY.position(dataY.position() + 1)
  110 + }
  111 + }
  112 + }
  113 +
  114 + // Send the final frame off to the sink.
  115 + targetSink?.onFrame(VideoFrame(frameBuffer, frame.rotation, frame.timestampNs))
  116 +
  117 + // Release any remaining resources
  118 + frameBuffer.release()
  119 + frame.release()
  120 + }.addOnFailureListener {
  121 + Log.e("SelfieVideoProcessor", "failed to process frame!")
  122 + }
  123 + }
  124 +
  125 + override fun setSink(sink: VideoSink?) {
  126 + targetSink = sink
  127 + }
  128 +
  129 + fun dispose() {
  130 + segmenter.close()
  131 + }
  132 +}
  1 +<?xml version="1.0" encoding="utf-8"?>
  2 +<vector xmlns:android="http://schemas.android.com/apk/res/android"
  3 + android:width="108dp"
  4 + android:height="108dp"
  5 + android:viewportWidth="108"
  6 + android:viewportHeight="108">
  7 + <path
  8 + android:fillColor="#3DDC84"
  9 + android:pathData="M0,0h108v108h-108z" />
  10 + <path
  11 + android:fillColor="#00000000"
  12 + android:pathData="M9,0L9,108"
  13 + android:strokeWidth="0.8"
  14 + android:strokeColor="#33FFFFFF" />
  15 + <path
  16 + android:fillColor="#00000000"
  17 + android:pathData="M19,0L19,108"
  18 + android:strokeWidth="0.8"
  19 + android:strokeColor="#33FFFFFF" />
  20 + <path
  21 + android:fillColor="#00000000"
  22 + android:pathData="M29,0L29,108"
  23 + android:strokeWidth="0.8"
  24 + android:strokeColor="#33FFFFFF" />
  25 + <path
  26 + android:fillColor="#00000000"
  27 + android:pathData="M39,0L39,108"
  28 + android:strokeWidth="0.8"
  29 + android:strokeColor="#33FFFFFF" />
  30 + <path
  31 + android:fillColor="#00000000"
  32 + android:pathData="M49,0L49,108"
  33 + android:strokeWidth="0.8"
  34 + android:strokeColor="#33FFFFFF" />
  35 + <path
  36 + android:fillColor="#00000000"
  37 + android:pathData="M59,0L59,108"
  38 + android:strokeWidth="0.8"
  39 + android:strokeColor="#33FFFFFF" />
  40 + <path
  41 + android:fillColor="#00000000"
  42 + android:pathData="M69,0L69,108"
  43 + android:strokeWidth="0.8"
  44 + android:strokeColor="#33FFFFFF" />
  45 + <path
  46 + android:fillColor="#00000000"
  47 + android:pathData="M79,0L79,108"
  48 + android:strokeWidth="0.8"
  49 + android:strokeColor="#33FFFFFF" />
  50 + <path
  51 + android:fillColor="#00000000"
  52 + android:pathData="M89,0L89,108"
  53 + android:strokeWidth="0.8"
  54 + android:strokeColor="#33FFFFFF" />
  55 + <path
  56 + android:fillColor="#00000000"
  57 + android:pathData="M99,0L99,108"
  58 + android:strokeWidth="0.8"
  59 + android:strokeColor="#33FFFFFF" />
  60 + <path
  61 + android:fillColor="#00000000"
  62 + android:pathData="M0,9L108,9"
  63 + android:strokeWidth="0.8"
  64 + android:strokeColor="#33FFFFFF" />
  65 + <path
  66 + android:fillColor="#00000000"
  67 + android:pathData="M0,19L108,19"
  68 + android:strokeWidth="0.8"
  69 + android:strokeColor="#33FFFFFF" />
  70 + <path
  71 + android:fillColor="#00000000"
  72 + android:pathData="M0,29L108,29"
  73 + android:strokeWidth="0.8"
  74 + android:strokeColor="#33FFFFFF" />
  75 + <path
  76 + android:fillColor="#00000000"
  77 + android:pathData="M0,39L108,39"
  78 + android:strokeWidth="0.8"
  79 + android:strokeColor="#33FFFFFF" />
  80 + <path
  81 + android:fillColor="#00000000"
  82 + android:pathData="M0,49L108,49"
  83 + android:strokeWidth="0.8"
  84 + android:strokeColor="#33FFFFFF" />
  85 + <path
  86 + android:fillColor="#00000000"
  87 + android:pathData="M0,59L108,59"
  88 + android:strokeWidth="0.8"
  89 + android:strokeColor="#33FFFFFF" />
  90 + <path
  91 + android:fillColor="#00000000"
  92 + android:pathData="M0,69L108,69"
  93 + android:strokeWidth="0.8"
  94 + android:strokeColor="#33FFFFFF" />
  95 + <path
  96 + android:fillColor="#00000000"
  97 + android:pathData="M0,79L108,79"
  98 + android:strokeWidth="0.8"
  99 + android:strokeColor="#33FFFFFF" />
  100 + <path
  101 + android:fillColor="#00000000"
  102 + android:pathData="M0,89L108,89"
  103 + android:strokeWidth="0.8"
  104 + android:strokeColor="#33FFFFFF" />
  105 + <path
  106 + android:fillColor="#00000000"
  107 + android:pathData="M0,99L108,99"
  108 + android:strokeWidth="0.8"
  109 + android:strokeColor="#33FFFFFF" />
  110 + <path
  111 + android:fillColor="#00000000"
  112 + android:pathData="M19,29L89,29"
  113 + android:strokeWidth="0.8"
  114 + android:strokeColor="#33FFFFFF" />
  115 + <path
  116 + android:fillColor="#00000000"
  117 + android:pathData="M19,39L89,39"
  118 + android:strokeWidth="0.8"
  119 + android:strokeColor="#33FFFFFF" />
  120 + <path
  121 + android:fillColor="#00000000"
  122 + android:pathData="M19,49L89,49"
  123 + android:strokeWidth="0.8"
  124 + android:strokeColor="#33FFFFFF" />
  125 + <path
  126 + android:fillColor="#00000000"
  127 + android:pathData="M19,59L89,59"
  128 + android:strokeWidth="0.8"
  129 + android:strokeColor="#33FFFFFF" />
  130 + <path
  131 + android:fillColor="#00000000"
  132 + android:pathData="M19,69L89,69"
  133 + android:strokeWidth="0.8"
  134 + android:strokeColor="#33FFFFFF" />
  135 + <path
  136 + android:fillColor="#00000000"
  137 + android:pathData="M19,79L89,79"
  138 + android:strokeWidth="0.8"
  139 + android:strokeColor="#33FFFFFF" />
  140 + <path
  141 + android:fillColor="#00000000"
  142 + android:pathData="M29,19L29,89"
  143 + android:strokeWidth="0.8"
  144 + android:strokeColor="#33FFFFFF" />
  145 + <path
  146 + android:fillColor="#00000000"
  147 + android:pathData="M39,19L39,89"
  148 + android:strokeWidth="0.8"
  149 + android:strokeColor="#33FFFFFF" />
  150 + <path
  151 + android:fillColor="#00000000"
  152 + android:pathData="M49,19L49,89"
  153 + android:strokeWidth="0.8"
  154 + android:strokeColor="#33FFFFFF" />
  155 + <path
  156 + android:fillColor="#00000000"
  157 + android:pathData="M59,19L59,89"
  158 + android:strokeWidth="0.8"
  159 + android:strokeColor="#33FFFFFF" />
  160 + <path
  161 + android:fillColor="#00000000"
  162 + android:pathData="M69,19L69,89"
  163 + android:strokeWidth="0.8"
  164 + android:strokeColor="#33FFFFFF" />
  165 + <path
  166 + android:fillColor="#00000000"
  167 + android:pathData="M79,19L79,89"
  168 + android:strokeWidth="0.8"
  169 + android:strokeColor="#33FFFFFF" />
  170 +</vector>
  1 +<vector xmlns:android="http://schemas.android.com/apk/res/android"
  2 + xmlns:aapt="http://schemas.android.com/aapt"
  3 + android:width="108dp"
  4 + android:height="108dp"
  5 + android:viewportWidth="108"
  6 + android:viewportHeight="108">
  7 + <path android:pathData="M31,63.928c0,0 6.4,-11 12.1,-13.1c7.2,-2.6 26,-1.4 26,-1.4l38.1,38.1L107,108.928l-32,-1L31,63.928z">
  8 + <aapt:attr name="android:fillColor">
  9 + <gradient
  10 + android:endX="85.84757"
  11 + android:endY="92.4963"
  12 + android:startX="42.9492"
  13 + android:startY="49.59793"
  14 + android:type="linear">
  15 + <item
  16 + android:color="#44000000"
  17 + android:offset="0.0" />
  18 + <item
  19 + android:color="#00000000"
  20 + android:offset="1.0" />
  21 + </gradient>
  22 + </aapt:attr>
  23 + </path>
  24 + <path
  25 + android:fillColor="#FFFFFF"
  26 + android:fillType="nonZero"
  27 + android:pathData="M65.3,45.828l3.8,-6.6c0.2,-0.4 0.1,-0.9 -0.3,-1.1c-0.4,-0.2 -0.9,-0.1 -1.1,0.3l-3.9,6.7c-6.3,-2.8 -13.4,-2.8 -19.7,0l-3.9,-6.7c-0.2,-0.4 -0.7,-0.5 -1.1,-0.3C38.8,38.328 38.7,38.828 38.9,39.228l3.8,6.6C36.2,49.428 31.7,56.028 31,63.928h46C76.3,56.028 71.8,49.428 65.3,45.828zM43.4,57.328c-0.8,0 -1.5,-0.5 -1.8,-1.2c-0.3,-0.7 -0.1,-1.5 0.4,-2.1c0.5,-0.5 1.4,-0.7 2.1,-0.4c0.7,0.3 1.2,1 1.2,1.8C45.3,56.528 44.5,57.328 43.4,57.328L43.4,57.328zM64.6,57.328c-0.8,0 -1.5,-0.5 -1.8,-1.2s-0.1,-1.5 0.4,-2.1c0.5,-0.5 1.4,-0.7 2.1,-0.4c0.7,0.3 1.2,1 1.2,1.8C66.5,56.528 65.6,57.328 64.6,57.328L64.6,57.328z"
  28 + android:strokeWidth="1"
  29 + android:strokeColor="#00000000" />
  30 +</vector>
  1 +<?xml version="1.0" encoding="utf-8"?>
  2 +<FrameLayout xmlns:android="http://schemas.android.com/apk/res/android"
  3 + xmlns:tools="http://schemas.android.com/tools"
  4 + android:layout_width="match_parent"
  5 + android:layout_height="match_parent"
  6 + android:background="#F00"
  7 + tools:context=".MainActivity">
  8 +
  9 + <io.livekit.android.renderer.TextureViewRenderer
  10 + android:id="@+id/renderer"
  11 + android:layout_width="match_parent"
  12 + android:layout_height="match_parent" />
  13 +
  14 +</FrameLayout>
  1 +<?xml version="1.0" encoding="utf-8"?>
  2 +<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
  3 + <background android:drawable="@drawable/ic_launcher_background" />
  4 + <foreground android:drawable="@drawable/ic_launcher_foreground" />
  5 + <monochrome android:drawable="@drawable/ic_launcher_foreground" />
  6 +</adaptive-icon>
  1 +<?xml version="1.0" encoding="utf-8"?>
  2 +<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
  3 + <background android:drawable="@drawable/ic_launcher_background" />
  4 + <foreground android:drawable="@drawable/ic_launcher_foreground" />
  5 + <monochrome android:drawable="@drawable/ic_launcher_foreground" />
  6 +</adaptive-icon>
  1 +<resources xmlns:tools="http://schemas.android.com/tools">
  2 + <!-- Base application theme. -->
  3 + <style name="Base.Theme.Livekitandroid" parent="Theme.Material3.DayNight.NoActionBar">
  4 + <!-- Customize your dark theme here. -->
  5 + <!-- <item name="colorPrimary">@color/my_dark_primary</item> -->
  6 + </style>
  7 +</resources>
  1 +<?xml version="1.0" encoding="utf-8"?>
  2 +<resources>
  3 + <color name="black">#FF000000</color>
  4 + <color name="white">#FFFFFFFF</color>
  5 +</resources>
  1 +<resources>
  2 + <string name="app_name">selfie-segmentation</string>
  3 +</resources>
  1 +<resources xmlns:tools="http://schemas.android.com/tools">
  2 + <!-- Base application theme. -->
  3 + <style name="Base.Theme.Livekitandroid" parent="Theme.Material3.DayNight.NoActionBar">
  4 + <!-- Customize your light theme here. -->
  5 + <!-- <item name="colorPrimary">@color/my_light_primary</item> -->
  6 + </style>
  7 +
  8 + <style name="Theme.Livekitandroid" parent="Base.Theme.Livekitandroid" />
  9 +</resources>
  1 +/*
  2 + * Copyright 2024 LiveKit, Inc.
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +
  17 +package io.livekit.android.selfie
  18 +
  19 +import org.junit.Assert.*
  20 +import org.junit.Test
  21 +
  22 +/**
  23 + * Example local unit test, which will execute on the development machine (host).
  24 + *
  25 + * See [testing documentation](http://d.android.com/tools/testing).
  26 + */
  27 +class ExampleUnitTest {
  28 + @Test
  29 + fun addition_isCorrect() {
  30 + assertEquals(4, 2 + 2)
  31 + }
  32 +}
1 #Mon May 01 22:58:53 JST 2023 1 #Mon May 01 22:58:53 JST 2023
2 distributionBase=GRADLE_USER_HOME 2 distributionBase=GRADLE_USER_HOME
3 distributionPath=wrapper/dists 3 distributionPath=wrapper/dists
4 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.0-bin.zip 4 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip
5 zipStoreBase=GRADLE_USER_HOME 5 zipStoreBase=GRADLE_USER_HOME
6 zipStorePath=wrapper/dists 6 zipStorePath=wrapper/dists
@@ -10,3 +10,4 @@ include ':livekit-lint' @@ -10,3 +10,4 @@ include ':livekit-lint'
10 include ':video-encode-decode-test' 10 include ':video-encode-decode-test'
11 include ':sample-app-basic' 11 include ':sample-app-basic'
12 include ':sample-app-record-local' 12 include ':sample-app-record-local'
  13 +include ':examples:selfie-segmentation'