android screenshare as video track (#16)

* screencast implementation

* update usage comment

* fill in track source parameter

* Make screenCaptureConnection.stop() idempotent

* minor cleanup

* update compose sample application to include screenshare

* fix tests

@@ -2,8 +2,8 @@
 
 buildscript {
     ext {
-        compose_version = '1.0.3'
-        kotlin_version = '1.5.30'
+        compose_version = '1.0.4'
+        kotlin_version = '1.5.31'
         java_version = JavaVersion.VERSION_1_8
         dokka_version = '1.5.0'
     }

@@ -5,5 +5,13 @@
     <uses-permission android:name="android.permission.INTERNET" />
     <uses-permission android:name="android.permission.RECORD_AUDIO" />
     <uses-permission android:name="android.permission.CAMERA" />
+    <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
 
+    <application>
+        <service
+            android:name="io.livekit.android.room.track.screencapture.ScreenCaptureService"
+            android:enabled="true"
+            android:foregroundServiceType="mediaProjection"
+            android:stopWithTask="true" />
+    </application>
 </manifest>

@@ -109,7 +109,7 @@ internal constructor(
         return joinResponse
     }
 
-    private suspend fun configure(joinResponse: LivekitRtc.JoinResponse) {
+    private fun configure(joinResponse: LivekitRtc.JoinResponse) {
         if (this::publisher.isInitialized || this::subscriber.isInitialized) {
             // already configured
             return

@@ -2,6 +2,7 @@ package io.livekit.android.room.participant
 
 import android.Manifest
 import android.content.Context
+import android.content.Intent
 import com.google.protobuf.ByteString
 import dagger.assisted.Assisted
 import dagger.assisted.AssistedFactory
@@ -23,6 +24,7 @@ internal constructor(
     private val peerConnectionFactory: PeerConnectionFactory,
     private val context: Context,
     private val eglBase: EglBase,
+    private val screencastVideoTrackFactory: LocalScreencastVideoTrack.Factory,
 ) :
     Participant(info.sid, info.identity) {
 
@@ -72,6 +74,27 @@ internal constructor(
         )
     }
 
+    /**
+     * Creates a screencast video track.
+     *
+     * @param mediaProjectionPermissionResultData The resultData returned from launching
+     * [MediaProjectionManager.createScreenCaptureIntent()](https://developer.android.com/reference/android/media/projection/MediaProjectionManager#createScreenCaptureIntent()).
+     */
+    fun createScreencastTrack(
+        name: String = "",
+        mediaProjectionPermissionResultData: Intent,
+    ): LocalScreencastVideoTrack {
+        return LocalScreencastVideoTrack.createTrack(
+            mediaProjectionPermissionResultData,
+            peerConnectionFactory,
+            context,
+            name,
+            LocalVideoTrackOptions(isScreencast = true),
+            eglBase,
+            screencastVideoTrackFactory
+        )
+    }
+
     suspend fun publishAudioTrack(
         track: LocalAudioTrack,
         options: AudioTrackPublishOptions = AudioTrackPublishOptions(),
@@ -85,6 +108,7 @@ internal constructor(
         val cid = track.rtcTrack.id()
         val builder = LivekitRtc.AddTrackRequest.newBuilder().apply {
             disableDtx = !options.dtx
+            source = LivekitModels.TrackSource.MICROPHONE
         }
         val trackInfo = engine.addTrack(
             cid = cid,
@@ -124,6 +148,11 @@ internal constructor(
         val builder = LivekitRtc.AddTrackRequest.newBuilder().apply {
             width = track.dimensions.width
             height = track.dimensions.height
+            source = if(track.options.isScreencast){
+                LivekitModels.TrackSource.SCREEN_SHARE
+            } else {
+                LivekitModels.TrackSource.CAMERA
+            }
         }
         val trackInfo = engine.addTrack(
             cid = cid,

+package io.livekit.android.room.track
+
+import android.app.Notification
+import android.content.Context
+import android.content.Intent
+import android.media.projection.MediaProjection
+import dagger.assisted.Assisted
+import dagger.assisted.AssistedFactory
+import dagger.assisted.AssistedInject
+import io.livekit.android.room.track.screencapture.ScreenCaptureConnection
+import org.webrtc.*
+import java.util.*
+
+class LocalScreencastVideoTrack
+@AssistedInject
+constructor(
+    @Assisted capturer: VideoCapturer,
+    @Assisted source: VideoSource,
+    @Assisted name: String,
+    @Assisted options: LocalVideoTrackOptions,
+    @Assisted rtcTrack: org.webrtc.VideoTrack,
+    @Assisted mediaProjectionCallback: MediaProjectionCallback,
+    peerConnectionFactory: PeerConnectionFactory,
+    context: Context,
+    eglBase: EglBase,
+) : LocalVideoTrack(
+    capturer,
+    source,
+    name,
+    options,
+    rtcTrack,
+    peerConnectionFactory,
+    context,
+    eglBase
+) {
+
+    private val serviceConnection = ScreenCaptureConnection(context)
+
+    init {
+        mediaProjectionCallback.onStopCallback = { stop() }
+    }
+
+    /**
+     * A foreground service is generally required prior to [startCapture]. This method starts up
+     * a helper foreground service that only serves to display a notification while capturing. This
+     * foreground service will stop upon the end of screen capture.
+     *
+     * You may choose to use your own foreground service instead of this method, but it must be
+     * started prior to calling [startCapture].
+     *
+     * @see [io.livekit.android.room.track.screencapture.ScreenCaptureService.start]
+     */
+    suspend fun startForegroundService(notificationId: Int?, notification: Notification?) {
+        serviceConnection.connect()
+        serviceConnection.startForeground(notificationId, notification)
+    }
+
+    override fun stop() {
+        super.stop()
+        serviceConnection.stop()
+    }
+
+    @AssistedFactory
+    interface Factory {
+        fun create(
+            capturer: VideoCapturer,
+            source: VideoSource,
+            name: String,
+            options: LocalVideoTrackOptions,
+            rtcTrack: org.webrtc.VideoTrack,
+            mediaProjectionCallback: MediaProjectionCallback,
+        ): LocalScreencastVideoTrack
+    }
+
+    /**
+     * Needed to deal with circular dependency.
+     */
+    class MediaProjectionCallback : MediaProjection.Callback() {
+        var onStopCallback: (() -> Unit)? = null
+
+        override fun onStop() {
+            onStopCallback?.invoke()
+        }
+    }
+
+    companion object {
+        internal fun createTrack(
+            mediaProjectionPermissionResultData: Intent,
+            peerConnectionFactory: PeerConnectionFactory,
+            context: Context,
+            name: String,
+            options: LocalVideoTrackOptions,
+            rootEglBase: EglBase,
+            screencastVideoTrackFactory: Factory
+        ): LocalScreencastVideoTrack {
+            val source = peerConnectionFactory.createVideoSource(options.isScreencast)
+            val callback = MediaProjectionCallback()
+            val capturer = createScreenCapturer(mediaProjectionPermissionResultData, callback)
+            capturer.initialize(
+                SurfaceTextureHelper.create("ScreenVideoCaptureThread", rootEglBase.eglBaseContext),
+                context,
+                source.capturerObserver
+            )
+            val track = peerConnectionFactory.createVideoTrack(UUID.randomUUID().toString(), source)
+
+            return screencastVideoTrackFactory.create(
+                capturer = capturer,
+                source = source,
+                options = options,
+                name = name,
+                rtcTrack = track,
+                mediaProjectionCallback = callback
+            )
+        }
+
+
+        private fun createScreenCapturer(
+            resultData: Intent,
+            callback: MediaProjectionCallback
+        ): ScreenCapturerAndroid {
+            return ScreenCapturerAndroid(resultData, callback)
+        }
+    }
+}
\ No newline at end of file

@@ -8,12 +8,13 @@ import io.livekit.android.util.LKLog
 import org.webrtc.*
 import java.util.*
 
+
 /**
  * A representation of a local video track (generally input coming from camera or screen).
  *
  * [startCapture] should be called before use.
  */
-class LocalVideoTrack(
+open class LocalVideoTrack(
     private var capturer: VideoCapturer,
     private var source: VideoSource,
     name: String,
@@ -40,7 +41,7 @@ class LocalVideoTrack(
     private val sender: RtpSender?
         get() = transceiver?.sender
 
-    fun startCapture() {
+    open fun startCapture() {
         capturer.startCapture(
             options.captureParams.width,
             options.captureParams.height,
@@ -157,5 +158,6 @@ class LocalVideoTrack(
             }
             return null
         }
+
     }
 }
\ No newline at end of file

+package io.livekit.android.room.track.screencapture
+
+import android.app.Notification
+import android.content.ComponentName
+import android.content.Context
+import android.content.Context.BIND_AUTO_CREATE
+import android.content.Intent
+import android.content.ServiceConnection
+import android.os.IBinder
+import io.livekit.android.util.LKLog
+import kotlinx.coroutines.suspendCancellableCoroutine
+import kotlin.coroutines.Continuation
+import kotlin.coroutines.resume
+
+/**
+ * Handles connecting to a [ScreenCaptureService].
+ */
+internal class ScreenCaptureConnection(private val context: Context) {
+    public var isBound = false
+        private set
+    private var service: ScreenCaptureService? = null
+    private val queuedConnects = mutableSetOf<Continuation<Unit>>()
+    private val connection: ServiceConnection = object : ServiceConnection {
+        override fun onServiceDisconnected(name: ComponentName) {
+            LKLog.v { "Screen capture service is disconnected" }
+            isBound = false
+            service = null
+        }
+
+        override fun onServiceConnected(name: ComponentName, binder: IBinder) {
+            LKLog.v { "Screen capture service is connected" }
+            val screenCaptureBinder = binder as ScreenCaptureService.ScreenCaptureBinder
+            service = screenCaptureBinder.service
+            handleConnect()
+        }
+    }
+
+    suspend fun connect() {
+        if (isBound) {
+            return
+        }
+
+        val intent = Intent(context, ScreenCaptureService::class.java)
+        context.bindService(intent, connection, BIND_AUTO_CREATE)
+        return suspendCancellableCoroutine {
+            synchronized(this) {
+                if (isBound) {
+                    it.resume(Unit)
+                } else {
+                    queuedConnects.add(it)
+                }
+            }
+        }
+    }
+
+    fun startForeground(notificationId: Int? = null, notification: Notification? = null) {
+        service?.start(notificationId, notification)
+    }
+
+    private fun handleConnect() {
+        synchronized(this) {
+            isBound = true
+            queuedConnects.forEach { it.resume(Unit) }
+            queuedConnects.clear()
+        }
+    }
+
+    fun stop() {
+        if (isBound) {
+            context.unbindService(connection)
+        }
+        service = null
+        isBound = false
+    }
+}
\ No newline at end of file

+package io.livekit.android.room.track.screencapture
+
+import android.app.Notification
+import android.app.NotificationChannel
+import android.app.NotificationManager
+import android.app.Service
+import android.content.Context
+import android.content.Intent
+import android.os.Binder
+import android.os.Build
+import android.os.IBinder
+import androidx.annotation.RequiresApi
+import androidx.core.app.NotificationCompat
+
+/**
+ * A foreground service is required for screen capture on API level Q (29) and up.
+ * This a simple default foreground service to display a notification while screen
+ * capturing.
+ */
+
+open class ScreenCaptureService : Service() {
+    private var binder: IBinder = ScreenCaptureBinder()
+    private var bindCount = 0
+    override fun onBind(intent: Intent?): IBinder {
+        bindCount++
+        return binder
+    }
+
+    /**
+     * @param notificationId id of the notification to be used, or null for [DEFAULT_CHANNEL_ID]
+     * @param notification notification to be used, or null for a default notification.
+     */
+    fun start(notificationId: Int?, notification: Notification?) {
+        val actualNotification = if (notification != null) {
+            notification
+        } else {
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+                createNotificationChannel()
+            }
+            NotificationCompat.Builder(this, DEFAULT_CHANNEL_ID)
+                .setPriority(NotificationCompat.PRIORITY_DEFAULT)
+                .build()
+        }
+
+        val actualId = notificationId ?: DEFAULT_NOTIFICATION_ID
+        startForeground(actualId, actualNotification)
+    }
+
+    @RequiresApi(Build.VERSION_CODES.O)
+    private fun createNotificationChannel() {
+        val channel = NotificationChannel(
+            DEFAULT_CHANNEL_ID,
+            "Screen Capture",
+            NotificationManager.IMPORTANCE_LOW
+        )
+        val service = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
+        service.createNotificationChannel(channel)
+    }
+
+    override fun onUnbind(intent: Intent?): Boolean {
+        bindCount--
+
+        if (bindCount == 0) {
+            stopSelf()
+        }
+        return false
+    }
+
+
+    inner class ScreenCaptureBinder : Binder() {
+        val service: ScreenCaptureService
+            get() = this@ScreenCaptureService
+    }
+
+    companion object {
+        const val DEFAULT_NOTIFICATION_ID = 2345
+        const val DEFAULT_CHANNEL_ID = "livekit_screen_capture"
+    }
+}
\ No newline at end of file

@@ -14,6 +14,7 @@ import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith
 import org.mockito.Mock
+import org.mockito.Mockito
 import org.mockito.junit.MockitoJUnit
 import org.robolectric.RobolectricTestRunner
 import org.webrtc.EglBase
@@ -31,19 +32,11 @@ class RoomTest {
     @Mock
     lateinit var rtcEngine: RTCEngine
 
-    @Mock
-    lateinit var peerConnectionFactory: PeerConnectionFactory
     var eglBase: EglBase = MockEglBase()
 
     val localParticantFactory = object : LocalParticipant.Factory {
         override fun create(info: LivekitModels.ParticipantInfo): LocalParticipant {
-            return LocalParticipant(
-                info,
-                rtcEngine,
-                peerConnectionFactory,
-                context,
-                eglBase,
-            )
+            return Mockito.mock(LocalParticipant::class.java)
         }
     }
 

 package io.livekit.android.composesample
 
+import android.app.Activity
 import android.media.AudioManager
+import android.media.projection.MediaProjectionManager
 import android.os.Bundle
 import android.os.Parcelable
 import androidx.activity.compose.setContent
+import androidx.activity.result.contract.ActivityResultContracts
 import androidx.appcompat.app.AppCompatActivity
 import androidx.compose.foundation.background
 import androidx.compose.foundation.layout.*
@@ -44,6 +47,19 @@ class CallActivity : AppCompatActivity() {
     private var previousSpeakerphoneOn = true
     private var previousMicrophoneMute = false
 
+    private val screenCaptureIntentLauncher =
+        registerForActivityResult(
+            ActivityResultContracts.StartActivityForResult()
+        ) { result ->
+            val resultCode = result.resultCode
+            val data = result.data
+            if (resultCode != Activity.RESULT_OK || data == null) {
+                return@registerForActivityResult
+            }
+            viewModel.startScreenCapture(data)
+        }
+
+
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
 
@@ -73,17 +89,25 @@ class CallActivity : AppCompatActivity() {
                 val micEnabled by viewModel.micEnabled.observeAsState(true)
                 val videoEnabled by viewModel.videoEnabled.observeAsState(true)
                 val flipButtonEnabled by viewModel.flipButtonVideoEnabled.observeAsState(true)
+                val screencastEnabled by viewModel.screencastEnabled.observeAsState(false)
                 Content(
                     room,
                     participants,
                     micEnabled,
                     videoEnabled,
-                    flipButtonEnabled
+                    flipButtonEnabled,
+                    screencastEnabled,
                 )
             }
         }
     }
 
+    private fun requestMediaProjection() {
+        val mediaProjectionManager =
+            getSystemService(MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
+        screenCaptureIntentLauncher.launch(mediaProjectionManager.createScreenCaptureIntent())
+    }
+
     @Preview(showBackground = true, showSystemUi = true)
     @Composable
     fun Content(
@@ -92,6 +116,7 @@ class CallActivity : AppCompatActivity() {
         micEnabled: Boolean = true,
         videoEnabled: Boolean = true,
         flipButtonEnabled: Boolean = true,
+        screencastEnabled: Boolean = false,
     ) {
         ConstraintLayout(
             modifier = Modifier
@@ -224,6 +249,24 @@ class CallActivity : AppCompatActivity() {
                         tint = Color.White,
                     )
                 }
+                FloatingActionButton(
+                    onClick = {
+                        if (!screencastEnabled) {
+                            requestMediaProjection()
+                        } else {
+                            viewModel.stopScreenCapture()
+                        }
+                    },
+                    backgroundColor = Color.DarkGray,
+                ) {
+                    val resource =
+                        if (screencastEnabled) R.drawable.baseline_cast_connected_24 else R.drawable.baseline_cast_24
+                    Icon(
+                        painterResource(id = resource),
+                        contentDescription = "Flip Camera",
+                        tint = Color.White,
+                    )
+                }
             }
         }
     }

 package io.livekit.android.composesample
 
 import android.app.Application
+import android.content.Intent
 import androidx.lifecycle.AndroidViewModel
 import androidx.lifecycle.LiveData
 import androidx.lifecycle.MutableLiveData
@@ -10,12 +11,11 @@ import io.livekit.android.ConnectOptions
 import io.livekit.android.LiveKit
 import io.livekit.android.room.Room
 import io.livekit.android.room.RoomListener
+import io.livekit.android.room.participant.AudioTrackPublishOptions
 import io.livekit.android.room.participant.Participant
 import io.livekit.android.room.participant.RemoteParticipant
-import io.livekit.android.room.track.CameraPosition
-import io.livekit.android.room.track.LocalAudioTrack
-import io.livekit.android.room.track.LocalVideoTrack
-import io.livekit.android.room.track.LocalVideoTrackOptions
+import io.livekit.android.room.participant.VideoTrackPublishOptions
+import io.livekit.android.room.track.*
 import kotlinx.coroutines.launch
 
 class CallViewModel(
@@ -30,6 +30,7 @@ class CallViewModel(
 
     private var localAudioTrack: LocalAudioTrack? = null
     private var localVideoTrack: LocalVideoTrack? = null
+    private var localScreencastTrack: LocalScreencastVideoTrack? = null
 
     private val mutableMicEnabled = MutableLiveData(true)
     val micEnabled = mutableMicEnabled.hide()
@@ -40,6 +41,9 @@ class CallViewModel(
     private val mutableFlipVideoButtonEnabled = MutableLiveData(true)
     val flipButtonVideoEnabled = mutableFlipVideoButtonEnabled.hide()
 
+    private val mutableScreencastEnabled = MutableLiveData(false)
+    val screencastEnabled = mutableScreencastEnabled.hide()
+
     init {
         viewModelScope.launch {
             val room = LiveKit.connect(
@@ -50,14 +54,18 @@ class CallViewModel(
                 this@CallViewModel
             )
 
+            // Create and publish audio/video tracks
             val localParticipant = room.localParticipant
             val audioTrack = localParticipant.createAudioTrack()
-            localParticipant.publishAudioTrack(audioTrack)
+            localParticipant.publishAudioTrack(audioTrack, AudioTrackPublishOptions(dtx = true))
             this@CallViewModel.localAudioTrack = audioTrack
             mutableMicEnabled.postValue(audioTrack.enabled)
 
             val videoTrack = localParticipant.createVideoTrack()
-            localParticipant.publishVideoTrack(videoTrack)
+            localParticipant.publishVideoTrack(
+                videoTrack,
+                VideoTrackPublishOptions(simulcast = false)
+            )
             videoTrack.startCapture()
             this@CallViewModel.localVideoTrack = videoTrack
             mutableVideoEnabled.postValue(videoTrack.enabled)
@@ -67,6 +75,34 @@ class CallViewModel(
         }
     }
 
+    fun startScreenCapture(mediaProjectionPermissionResultData: Intent) {
+        val localParticipant = room.value?.localParticipant ?: return
+        viewModelScope.launch {
+            val screencastTrack =
+                localParticipant.createScreencastTrack(mediaProjectionPermissionResultData = mediaProjectionPermissionResultData)
+            localParticipant.publishVideoTrack(
+                screencastTrack
+            )
+
+            // Must start the foreground prior to startCapture.
+            screencastTrack.startForegroundService(null, null)
+            screencastTrack.startCapture()
+
+            this@CallViewModel.localScreencastTrack = screencastTrack
+            mutableScreencastEnabled.postValue(screencastTrack.enabled)
+        }
+    }
+
+    fun stopScreenCapture() {
+        viewModelScope.launch {
+            localScreencastTrack?.let { localScreencastVideoTrack ->
+                localScreencastVideoTrack.stop()
+                room.value?.localParticipant?.unpublishTrack(localScreencastVideoTrack)
+                mutableScreencastEnabled.postValue(localScreencastTrack?.enabled ?: false)
+            }
+        }
+    }
+
     private fun updateParticipants(room: Room) {
         mutableRemoteParticipants.postValue(
             room.remoteParticipants

@@ -8,6 +8,6 @@ class SampleApplication : Application() {
 
     override fun onCreate() {
         super.onCreate()
-        LiveKit.loggingLevel = LoggingLevel.OFF
+        LiveKit.loggingLevel = LoggingLevel.VERBOSE
     }
 }
\ No newline at end of file

+<vector xmlns:android="http://schemas.android.com/apk/res/android"
+    android:width="24dp"
+    android:height="24dp"
+    android:viewportWidth="24"
+    android:viewportHeight="24"
+    android:tint="?attr/colorControlNormal">
+  <path
+      android:fillColor="@android:color/white"
+      android:pathData="M21,3L3,3c-1.1,0 -2,0.9 -2,2v3h2L3,5h18v14h-7v2h7c1.1,0 2,-0.9 2,-2L23,5c0,-1.1 -0.9,-2 -2,-2zM1,18v3h3c0,-1.66 -1.34,-3 -3,-3zM1,14v2c2.76,0 5,2.24 5,5h2c0,-3.87 -3.13,-7 -7,-7zM1,10v2c4.97,0 9,4.03 9,9h2c0,-6.08 -4.93,-11 -11,-11z"/>
+</vector>

+<vector xmlns:android="http://schemas.android.com/apk/res/android"
+    android:width="24dp"
+    android:height="24dp"
+    android:viewportWidth="24"
+    android:viewportHeight="24"
+    android:tint="?attr/colorControlNormal">
+  <path
+      android:fillColor="@android:color/white"
+      android:pathData="M1,18v3h3c0,-1.66 -1.34,-3 -3,-3zM1,14v2c2.76,0 5,2.24 5,5h2c0,-3.87 -3.13,-7 -7,-7zM19,7L5,7v1.63c3.96,1.28 7.09,4.41 8.37,8.37L19,17L19,7zM1,10v2c4.97,0 9,4.03 9,9h2c0,-6.08 -4.93,-11 -11,-11zM21,3L3,3c-1.1,0 -2,0.9 -2,2v3h2L3,5h18v14h-7v2h7c1.1,0 2,-0.9 2,-2L23,5c0,-1.1 -0.9,-2 -2,-2z"/>
+</vector>