Adaptive stream fixes (#73)

* tests

* update visibility immediately

* handle visibility change from hidden

* surface view version

* clean up and readme update

@@ -82,7 +82,8 @@ screenCaptureIntentLauncher.launch(mediaProjectionManager.createScreenCaptureInt
 
 
 ### Rendering subscribed tracks
 ### Rendering subscribed tracks
 
 
-LiveKit uses WebRTC-provided `org.webrtc.SurfaceViewRenderer` to render video tracks. A `TextureView` implementation is also provided through `TextureViewRenderer`. Subscribed audio tracks are automatically played.
+LiveKit uses `SurfaceViewRenderer` to render video tracks. A `TextureView` implementation is also
+provided through `TextureViewRenderer`. Subscribed audio tracks are automatically played.
 
 
 ```kt
 ```kt
 class MainActivity : AppCompatActivity(), RoomListener {
 class MainActivity : AppCompatActivity(), RoomListener {
@@ -121,7 +122,7 @@ class MainActivity : AppCompatActivity(), RoomListener {
     }
     }
 
 
     private fun attachVideo(videoTrack: VideoTrack) {
     private fun attachVideo(videoTrack: VideoTrack) {
-        // viewBinding.renderer is a `org.webrtc.SurfaceViewRenderer` in your
+        // viewBinding.renderer is a `io.livekit.android.renderer.SurfaceViewRenderer` in your
         // layout
         // layout
         videoTrack.addRenderer(viewBinding.renderer)
         videoTrack.addRenderer(viewBinding.renderer)
     }
     }

+package io.livekit.android.renderer
+
+import android.content.Context
+import android.util.AttributeSet
+import android.view.View
+import io.livekit.android.room.track.video.ViewVisibility
+import org.webrtc.SurfaceViewRenderer
+
+class SurfaceViewRenderer : SurfaceViewRenderer, ViewVisibility.Notifier {
+    constructor(context: Context) : super(context)
+    constructor(context: Context, attrs: AttributeSet) : super(context, attrs)
+
+    override var viewVisibility: ViewVisibility? = null
+    override fun onVisibilityChanged(changedView: View, visibility: Int) {
+        super.onVisibilityChanged(changedView, visibility)
+        viewVisibility?.recalculate()
+    }
+}

@@ -10,30 +10,26 @@
 package io.livekit.android.renderer
 package io.livekit.android.renderer
 
 
 import android.content.Context
 import android.content.Context
-import android.view.SurfaceView
-import android.view.SurfaceHolder
-import org.webrtc.RendererCommon.RendererEvents
-import org.webrtc.RendererCommon.VideoLayoutMeasure
-import kotlin.jvm.JvmOverloads
-import org.webrtc.RendererCommon.GlDrawer
-import org.webrtc.RendererCommon.ScalingType
 import android.content.res.Resources.NotFoundException
 import android.content.res.Resources.NotFoundException
 import android.graphics.Matrix
 import android.graphics.Matrix
+import android.graphics.SurfaceTexture
 import android.os.Looper
 import android.os.Looper
 import android.util.AttributeSet
 import android.util.AttributeSet
+import android.view.Surface
+import android.view.SurfaceHolder
 import android.view.TextureView
 import android.view.TextureView
+import android.view.View
+import io.livekit.android.room.track.video.ViewVisibility
 import org.webrtc.*
 import org.webrtc.*
-import android.graphics.SurfaceTexture
-import android.view.Surface
-
-import org.webrtc.ThreadUtils
+import org.webrtc.RendererCommon.*
 import java.util.concurrent.CountDownLatch
 import java.util.concurrent.CountDownLatch
 
 
 
 
 /**
 /**
- * Display the video stream on a SurfaceView.
+ * Display the video stream on a TextureView.
  */
  */
-class TextureViewRenderer : TextureView, SurfaceHolder.Callback, TextureView.SurfaceTextureListener, VideoSink, RendererEvents {
+class TextureViewRenderer : TextureView, SurfaceHolder.Callback, TextureView.SurfaceTextureListener, VideoSink,
+    RendererEvents, ViewVisibility.Notifier {
     // Cached resource name.
     // Cached resource name.
     private val resourceName: String
     private val resourceName: String
     private val videoLayoutMeasure = VideoLayoutMeasure()
     private val videoLayoutMeasure = VideoLayoutMeasure()
@@ -359,4 +355,10 @@ class TextureViewRenderer : TextureView, SurfaceHolder.Callback, TextureView.Sur
     companion object {
     companion object {
         private const val TAG = "SurfaceViewRenderer"
         private const val TAG = "SurfaceViewRenderer"
     }
     }
+
+    override var viewVisibility: ViewVisibility? = null
+    override fun onVisibilityChanged(changedView: View, visibility: Int) {
+        super.onVisibilityChanged(changedView, visibility)
+        viewVisibility?.recalculate()
+    }
 }
 }

@@ -33,12 +33,17 @@ class RemoteTrackPublication(
                 trackJob = CoroutineScope(ioDispatcher).launch {
                 trackJob = CoroutineScope(ioDispatcher).launch {
                     value.events.collect {
                     value.events.collect {
                         when (it) {
                         when (it) {
-                            is TrackEvent.VisibilityChanged -> handleVisibilityChanged(it)
-                            is TrackEvent.VideoDimensionsChanged -> handleVideoDimensionsChanged(it)
+                            is TrackEvent.VisibilityChanged -> handleVisibilityChanged(it.isVisible)
+                            is TrackEvent.VideoDimensionsChanged -> handleVideoDimensionsChanged(it.newDimensions)
                             is TrackEvent.StreamStateChanged -> handleStreamStateChanged(it)
                             is TrackEvent.StreamStateChanged -> handleStreamStateChanged(it)
                         }
                         }
                     }
                     }
                 }
                 }
+
+                if (value is RemoteVideoTrack) {
+                    handleVideoDimensionsChanged(value.lastDimensions)
+                    handleVisibilityChanged(value.lastVisibility)
+                }
             }
             }
         }
         }
 
 
@@ -158,13 +163,13 @@ class RemoteTrackPublication(
         sendUpdateTrackSettings.invoke()
         sendUpdateTrackSettings.invoke()
     }
     }
 
 
-    private fun handleVisibilityChanged(trackEvent: TrackEvent.VisibilityChanged) {
-        disabled = !trackEvent.isVisible
+    private fun handleVisibilityChanged(isVisible: Boolean) {
+        disabled = !isVisible
         sendUpdateTrackSettings.invoke()
         sendUpdateTrackSettings.invoke()
     }
     }
 
 
-    private fun handleVideoDimensionsChanged(trackEvent: TrackEvent.VideoDimensionsChanged) {
-        videoDimensions = trackEvent.newDimensions
+    private fun handleVideoDimensionsChanged(newDimensions: Track.Dimensions) {
+        videoDimensions = newDimensions
         sendUpdateTrackSettings.invoke()
         sendUpdateTrackSettings.invoke()
     }
     }
 
 

@@ -23,8 +23,10 @@ class RemoteVideoTrack(
     private val sinkVisibilityMap = mutableMapOf<VideoSink, VideoSinkVisibility>()
     private val sinkVisibilityMap = mutableMapOf<VideoSink, VideoSinkVisibility>()
     private val visibilities = sinkVisibilityMap.values
     private val visibilities = sinkVisibilityMap.values
 
 
-    private var lastVisibility = false
-    private var lastDimensions: Dimensions = Dimensions(0, 0)
+    internal var lastVisibility = false
+        private set
+    internal var lastDimensions: Dimensions = Dimensions(0, 0)
+        private set
 
 
     override fun addRenderer(renderer: VideoSink) {
     override fun addRenderer(renderer: VideoSink) {
         if (autoManageVideo && renderer is View) {
         if (autoManageVideo && renderer is View) {
@@ -49,6 +51,9 @@ class RemoteVideoTrack(
         super.removeRenderer(renderer)
         super.removeRenderer(renderer)
         val visibility = sinkVisibilityMap.remove(renderer)
         val visibility = sinkVisibilityMap.remove(renderer)
         visibility?.close()
         visibility?.close()
+        if (autoManageVideo && visibility != null) {
+            recalculateVisibility()
+        }
     }
     }
 
 
     override fun stop() {
     override fun stop() {

@@ -8,6 +8,7 @@ import android.view.ViewTreeObserver
 import androidx.annotation.CallSuper
 import androidx.annotation.CallSuper
 import androidx.compose.ui.layout.LayoutCoordinates
 import androidx.compose.ui.layout.LayoutCoordinates
 import io.livekit.android.room.track.Track
 import io.livekit.android.room.track.Track
+import io.livekit.android.room.track.video.ViewVisibility.Notifier
 import java.util.*
 import java.util.*
 
 
 abstract class VideoSinkVisibility : Observable() {
 abstract class VideoSinkVisibility : Observable() {
@@ -63,41 +64,55 @@ class ComposeVisibility : VideoSinkVisibility() {
     }
     }
 }
 }
 
 
+/**
+ * A [VideoSinkVisibility] for views. If using a custom view other than the sdk provided renderers,
+ * you must implement [Notifier], override [View.onVisibilityChanged] and call through to [recalculate], or
+ * the visibility may not be calculated correctly.
+ */
 class ViewVisibility(private val view: View) : VideoSinkVisibility() {
 class ViewVisibility(private val view: View) : VideoSinkVisibility() {
 
 
+    private val lastVisibility = false
+    private val lastSize = Track.Dimensions(0, 0)
+
     private val handler = Handler(Looper.getMainLooper())
     private val handler = Handler(Looper.getMainLooper())
-    private val globalLayoutListener = object : ViewTreeObserver.OnGlobalLayoutListener {
-        val lastVisibility = false
-        val lastSize = Track.Dimensions(0, 0)
-
-        override fun onGlobalLayout() {
-            handler.removeCallbacksAndMessages(null)
-            handler.postDelayed({
-                var shouldNotify = false
-                val newVisibility = isVisible()
-                val newSize = size()
-                if (newVisibility != lastVisibility) {
-                    shouldNotify = true
-                }
-                if (newSize != lastSize) {
-                    shouldNotify = true
-                }
-
-                if (shouldNotify) {
-                    notifyChanged()
-                }
-            }, 2000)
-        }
+    private val globalLayoutListener = ViewTreeObserver.OnGlobalLayoutListener {
+        handler.removeCallbacksAndMessages(null)
+        handler.postDelayed({
+            recalculate()
+        }, 2000)
+    }
+
+    interface Notifier {
+        var viewVisibility: ViewVisibility?
     }
     }
 
 
     init {
     init {
         view.viewTreeObserver.addOnGlobalLayoutListener(globalLayoutListener)
         view.viewTreeObserver.addOnGlobalLayoutListener(globalLayoutListener)
+        if (view is Notifier) {
+            view.viewVisibility = this
+        }
     }
     }
 
 
     private val loc = IntArray(2)
     private val loc = IntArray(2)
     private val viewRect = Rect()
     private val viewRect = Rect()
     private val windowRect = Rect()
     private val windowRect = Rect()
 
 
+    fun recalculate() {
+        var shouldNotify = false
+        val newVisibility = isVisible()
+        val newSize = size()
+        if (newVisibility != lastVisibility) {
+            shouldNotify = true
+        }
+        if (newSize != lastSize) {
+            shouldNotify = true
+        }
+
+        if (shouldNotify) {
+            notifyChanged()
+        }
+    }
+
     private fun isViewAncestorsVisible(view: View): Boolean {
     private fun isViewAncestorsVisible(view: View): Boolean {
         if (view.visibility != View.VISIBLE) {
         if (view.visibility != View.VISIBLE) {
             return false
             return false
@@ -132,5 +147,8 @@ class ViewVisibility(private val view: View) : VideoSinkVisibility() {
         super.close()
         super.close()
         handler.removeCallbacksAndMessages(null)
         handler.removeCallbacksAndMessages(null)
         view.viewTreeObserver.removeOnGlobalLayoutListener(globalLayoutListener)
         view.viewTreeObserver.removeOnGlobalLayoutListener(globalLayoutListener)
+        if (view is Notifier && view.viewVisibility == this) {
+            view.viewVisibility = null
+        }
     }
     }
 }
 }

+package io.livekit.android.mock
+
+import org.webrtc.VideoSink
+import org.webrtc.VideoTrack
+
+class MockVideoStreamTrack(
+    val id: String = "id",
+    val kind: String = AUDIO_TRACK_KIND,
+    var enabled: Boolean = true,
+    var state: State = State.LIVE,
+) : VideoTrack(1L) {
+    val sinks = mutableSetOf<VideoSink>()
+    override fun id(): String = id
+
+    override fun kind(): String = kind
+
+    override fun enabled(): Boolean = enabled
+
+    override fun setEnabled(enable: Boolean): Boolean {
+        enabled = enable
+        return true
+    }
+
+    override fun state(): State {
+        return state
+    }
+
+    override fun dispose() {
+    }
+
+    override fun addSink(sink: VideoSink) {
+        sinks.add(sink)
+    }
+
+    override fun removeSink(sink: VideoSink) {
+        sinks.remove(sink)
+    }
+}

+package io.livekit.android.room.track
+
+import io.livekit.android.BaseTest
+import io.livekit.android.events.EventCollector
+import io.livekit.android.events.TrackEvent
+import io.livekit.android.mock.MockVideoStreamTrack
+import io.livekit.android.room.track.video.VideoSinkVisibility
+import kotlinx.coroutines.ExperimentalCoroutinesApi
+import org.junit.Assert
+import org.junit.Before
+import org.junit.Test
+import org.webrtc.VideoFrame
+import org.webrtc.VideoSink
+
+@OptIn(ExperimentalCoroutinesApi::class)
+class RemoteVideoTrackTest : BaseTest() {
+
+    lateinit var track: RemoteVideoTrack
+
+    @Before
+    fun setup() {
+        track = RemoteVideoTrack(
+            name = "track",
+            rtcTrack = MockVideoStreamTrack(),
+            autoManageVideo = true,
+            dispatcher = coroutineRule.dispatcher
+        )
+    }
+
+    @Test
+    fun defaultVisibility() = runTest {
+        Assert.assertFalse(track.lastVisibility)
+        Assert.assertEquals(Track.Dimensions(0, 0), track.lastDimensions)
+    }
+
+    @Test
+    fun addVisibility() = runTest {
+        val visible = true
+        val size = Track.Dimensions(100, 100)
+        val visibility = CustomVisibility(visible = visible, size = size)
+        val eventCollector = EventCollector(track.events, coroutineRule.scope)
+        track.addRenderer(EmptyVideoSink(), visibility)
+        val events = eventCollector.stopCollecting()
+
+        Assert.assertEquals(2, events.size)
+        val visibilityEvent = events.first { it is TrackEvent.VisibilityChanged } as TrackEvent.VisibilityChanged
+        val sizeEvent = events.first { it is TrackEvent.VideoDimensionsChanged } as TrackEvent.VideoDimensionsChanged
+
+        Assert.assertEquals(visible, visibilityEvent.isVisible)
+        Assert.assertEquals(size, sizeEvent.newDimensions)
+    }
+
+    @Test
+    fun removeVisibility() = runTest {
+        val sink = EmptyVideoSink()
+        val visibility = CustomVisibility(visible = true, size = Track.Dimensions(100, 100))
+        track.addRenderer(sink, visibility)
+        val eventCollector = EventCollector(track.events, coroutineRule.scope)
+        track.removeRenderer(sink)
+        val events = eventCollector.stopCollecting()
+
+        Assert.assertEquals(2, events.size)
+        val visibilityEvent = events.first { it is TrackEvent.VisibilityChanged } as TrackEvent.VisibilityChanged
+        val sizeEvent = events.first { it is TrackEvent.VideoDimensionsChanged } as TrackEvent.VideoDimensionsChanged
+
+        Assert.assertEquals(false, visibilityEvent.isVisible)
+        Assert.assertEquals(Track.Dimensions(0, 0), sizeEvent.newDimensions)
+    }
+
+    @Test
+    fun changeVisibility() = runTest {
+        val visibility = CustomVisibility(visible = true, size = Track.Dimensions(100, 100))
+        track.addRenderer(EmptyVideoSink(), visibility)
+
+        val visible = false
+        val size = Track.Dimensions(200, 200)
+
+        val eventCollector = EventCollector(track.events, coroutineRule.scope)
+        visibility.visible = visible
+        visibility.size = size
+        val events = eventCollector.stopCollecting()
+
+        Assert.assertEquals(2, events.size)
+        val visibilityEvent = events.first { it is TrackEvent.VisibilityChanged } as TrackEvent.VisibilityChanged
+        val sizeEvent = events.first { it is TrackEvent.VideoDimensionsChanged } as TrackEvent.VideoDimensionsChanged
+        Assert.assertEquals(visible, visibilityEvent.isVisible)
+        Assert.assertEquals(size, sizeEvent.newDimensions)
+    }
+
+    @Test
+    fun multipleVisibility() = runTest {
+        val visible = true
+        val size = Track.Dimensions(100, 100)
+        val hiddenVisibility = CustomVisibility(visible = false, size = Track.Dimensions(0, 0))
+        val visibility = CustomVisibility(visible = visible, size = size)
+        track.addRenderer(EmptyVideoSink(), hiddenVisibility)
+
+        val eventCollector = EventCollector(track.events, coroutineRule.scope)
+        track.addRenderer(EmptyVideoSink(), visibility)
+        val events = eventCollector.stopCollecting()
+
+        // Make sure we're grabbing the max of visibility and dimensions
+        Assert.assertEquals(2, events.size)
+        val visibilityEvent = events.first { it is TrackEvent.VisibilityChanged } as TrackEvent.VisibilityChanged
+        val sizeEvent = events.first { it is TrackEvent.VideoDimensionsChanged } as TrackEvent.VideoDimensionsChanged
+
+        Assert.assertEquals(visible, visibilityEvent.isVisible)
+        Assert.assertEquals(size, sizeEvent.newDimensions)
+    }
+}
+
+private class EmptyVideoSink : VideoSink {
+    override fun onFrame(p0: VideoFrame?) {
+    }
+}
+
+private class CustomVisibility(
+    visible: Boolean = false,
+    size: Track.Dimensions = Track.Dimensions(0, 0)
+) : VideoSinkVisibility() {
+
+    var visible = visible
+        set(value) {
+            field = value
+            notifyChanged()
+        }
+
+    var size = size
+        set(value) {
+            field = value
+            notifyChanged()
+        }
+
+    override fun isVisible() = visible
+
+    override fun size() = size
+
+}