Custom webrtc and simulcast (#11)

* move to custom webrtc 92.4515.01

* simulcast video encoder factory with wrapper

* set video encodings and simulcast on transceiver

* rtpparameter converter method

* add in comment about dimensions

* update gradle

@@ -26,5 +26,10 @@
       <option name="name" value="MavenRepo" />
       <option name="url" value="https://repo.maven.apache.org/maven2/" />
     </remote-repository>
+    <remote-repository>
+      <option name="id" value="maven" />
+      <option name="name" value="maven" />
+      <option name="url" value="https://jitpack.io" />
+    </remote-repository>
   </component>
 </project>
\ No newline at end of file

@@ -11,10 +11,9 @@ buildscript {
         google()
         mavenCentral()
         jcenter()
-
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:7.0.2'
+        classpath 'com.android.tools.build:gradle:7.0.3'
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
         classpath "org.jetbrains.kotlin:kotlin-serialization:$kotlin_version"
         classpath "org.jetbrains.dokka:dokka-gradle-plugin:$dokka_version"
@@ -31,6 +30,7 @@ subprojects {
         google()
         mavenCentral()
         jcenter()
+        maven { url 'https://jitpack.io' }
     }
 }
 

@@ -99,7 +99,7 @@ dependencies {
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
     implementation deps.kotlinx_coroutines
     implementation 'org.jetbrains.kotlinx:kotlinx-serialization-json:1.1.0'
-    api 'org.webrtc:google-webrtc:1.0.32006'
+    api 'com.github.webrtc-sdk:android:92.4515.01'
     api "com.squareup.okhttp3:okhttp:4.9.0"
     implementation "com.google.protobuf:protobuf-java:${versions.protobuf}"
     implementation "com.google.protobuf:protobuf-java-util:${versions.protobuf}"

@@ -4,6 +4,7 @@ import android.content.Context
 import dagger.Module
 import dagger.Provides
 import io.livekit.android.util.LKLog
+import io.livekit.android.webrtc.SimulcastVideoEncoderFactoryWrapper
 import org.webrtc.*
 import org.webrtc.audio.AudioDeviceModule
 import org.webrtc.audio.JavaAudioDeviceModule
@@ -103,10 +104,10 @@ class RTCModule {
         ): VideoEncoderFactory {
 
             return if (videoHwAccel) {
-                DefaultVideoEncoderFactory(
+                SimulcastVideoEncoderFactoryWrapper(
                     eglContext,
-                    true,
-                    true
+                    enableIntelVp8Encoder = true,
+                    enableH264HighProfile = true,
                 )
             } else {
                 SoftwareVideoEncoderFactory()
@@ -149,6 +150,6 @@ class RTCModule {
 
         @Provides
         @Named(InjectionNames.OPTIONS_VIDEO_HW_ACCEL)
-        fun videoHwAccel() = false
+        fun videoHwAccel() = true
     }
 }
\ No newline at end of file

@@ -72,4 +72,7 @@ class PublisherTransportObserver(
     override fun onAddTrack(p0: RtpReceiver?, p1: Array<out MediaStream>?) {
     }
 
+    override fun onRemoveTrack(p0: RtpReceiver?) {
+    }
+
 }
\ No newline at end of file

@@ -25,6 +25,9 @@ class SubscriberTransportObserver(
         engine.listener?.onAddTrack(track, streams)
     }
 
+    override fun onRemoveTrack(p0: RtpReceiver?) {
+    }
+
     override fun onTrack(transceiver: RtpTransceiver) {
         when (transceiver.mediaType) {
             MediaStreamTrack.MediaType.MEDIA_TYPE_AUDIO -> LKLog.v { "peerconn started receiving audio" }

 package io.livekit.android.room.participant
 
 import android.content.Context
+import android.media.MediaCodecInfo
 import com.google.protobuf.ByteString
 import dagger.assisted.Assisted
 import dagger.assisted.AssistedFactory
@@ -9,9 +10,8 @@ import io.livekit.android.room.RTCEngine
 import io.livekit.android.room.track.*
 import io.livekit.android.util.LKLog
 import livekit.LivekitModels
-import livekit.LivekitRtc
 import org.webrtc.*
-import java.nio.ByteBuffer
+import kotlin.math.abs
 
 class LocalParticipant
 @AssistedInject
@@ -93,6 +93,7 @@ internal constructor(
 
     suspend fun publishVideoTrack(
         track: LocalVideoTrack,
+        options: VideoTrackPublishOptions = VideoTrackPublishOptions(),
         publishListener: PublishListener? = null
     ) {
         if (localTrackPublications.any { it.track == track }) {
@@ -101,13 +102,18 @@ internal constructor(
         }
 
         val cid = track.rtcTrack.id()
-        val trackInfo =
-            engine.addTrack(cid = cid, name = track.name, kind = LivekitModels.TrackType.VIDEO, dimensions = track.dimensions)
+        val trackInfo = engine.addTrack(
+            cid = cid,
+            name = track.name,
+            kind = LivekitModels.TrackType.VIDEO,
+            dimensions = track.dimensions
+        )
+        val encodings = computeVideoEncodings(track.dimensions, options)
         val transInit = RtpTransceiver.RtpTransceiverInit(
             RtpTransceiver.RtpTransceiverDirection.SEND_ONLY,
-            listOf(this.sid)
+            listOf(this.sid),
+            encodings
         )
-        // TODO: video encodings & simulcast
         val transceiver = engine.publisher.peerConnection.addTransceiver(track.rtcTrack, transInit)
         track.transceiver = transceiver
 
@@ -116,11 +122,69 @@ internal constructor(
             return
         }
 
+        // TODO: enable setting preferred codec
+
         val publication = LocalTrackPublication(trackInfo, track, this)
         addTrackPublication(publication)
         publishListener?.onPublishSuccess(publication)
     }
 
+    private fun computeVideoEncodings(
+        dimensions: Track.Dimensions,
+        options: VideoTrackPublishOptions
+    ): List<RtpParameters.Encoding> {
+        val (width, height) = dimensions
+        var encoding = options.videoEncoding
+        val simulcast = options.simulcast
+
+        if ((encoding == null && !simulcast) || width == 0 || height == 0) {
+            return emptyList()
+        }
+
+        if (encoding == null) {
+            encoding = determineAppropriateEncoding(width, height)
+            LKLog.d { "using video encoding: $encoding" }
+        }
+
+        val encodings = mutableListOf<RtpParameters.Encoding>()
+        if (simulcast) {
+            encodings.add(encoding.toRtpEncoding("f"))
+
+            val presets = presetsForResolution(width, height)
+            val midPreset = presets[1]
+            val lowPreset = presets[0]
+
+            // if resolution is high enough, we send both h and q res.
+            // otherwise only send h
+            if (width >= 960) {
+                encodings.add(midPreset.encoding.toRtpEncoding("h", 2.0))
+                encodings.add(lowPreset.encoding.toRtpEncoding("q", 4.0))
+            } else {
+                encodings.add(lowPreset.encoding.toRtpEncoding("h", 2.0))
+            }
+        } else {
+            encodings.add(encoding.toRtpEncoding())
+        }
+        return encodings
+    }
+
+    private fun determineAppropriateEncoding(width: Int, height: Int): VideoEncoding {
+        val presets = presetsForResolution(width, height)
+
+        return presets
+            .last { width >= it.capture.width && height >= it.capture.height }
+            .encoding
+    }
+
+    private fun presetsForResolution(width: Int, height: Int): List<VideoPreset> {
+        val aspectRatio = width.toFloat() / height
+        if (abs(aspectRatio - 16f / 9f) < abs(aspectRatio - 4f / 3f)) {
+            return PRESETS_16_9
+        } else {
+            return PRESETS_4_3
+        }
+    }
+
     fun unpublishTrack(track: Track) {
         val publication = localTrackPublications.firstOrNull { it.track == track }
         if (publication === null) {
@@ -202,4 +266,38 @@ internal constructor(
     interface Factory {
         fun create(info: LivekitModels.ParticipantInfo): LocalParticipant
     }
+
+    companion object {
+        private val PRESETS_16_9 = listOf(
+            VideoPreset169.QVGA,
+            VideoPreset169.VGA,
+            VideoPreset169.QHD,
+            VideoPreset169.HD,
+            VideoPreset169.FHD
+        )
+
+        private val PRESETS_4_3 = listOf(
+            VideoPreset43.QVGA,
+            VideoPreset43.VGA,
+            VideoPreset43.QHD,
+            VideoPreset43.HD,
+            VideoPreset43.FHD
+        )
+    }
 }
+
+interface TrackPublishOptions {
+    val name: String?
+}
+
+data class VideoTrackPublishOptions(
+    override val name: String? = null,
+    val videoEncoding: VideoEncoding? = null,
+    //val videoCodec: VideoCodec? = null,
+    val simulcast: Boolean = false
+) : TrackPublishOptions
+
+data class AudioTrackPublishOptions(
+    override val name: String? = null,
+    val audioBitrate: Int? = null,
+) : TrackPublishOptions
\ No newline at end of file

@@ -24,8 +24,12 @@ class LocalVideoTrack(
     override var rtcTrack: org.webrtc.VideoTrack = rtcTrack
         internal set
 
-    val dimensions: Dimensions =
-        Dimensions(options.captureParams.width, options.captureParams.height)
+    /**
+     * Note: these dimensions are only requested params, and may differ
+     * from the actual capture format used by the camera.
+     */
+    val dimensions: Dimensions
+        get() = Dimensions(options.captureParams.width, options.captureParams.height)
 
     internal var transceiver: RtpTransceiver? = null
     private val sender: RtpSender?

 package io.livekit.android.room.track
 
+import org.webrtc.RtpParameters
+
 class LocalVideoTrackOptions(
     var isScreencast: Boolean = false,
     var position: CameraPosition = CameraPosition.FRONT,
-    var captureParams: VideoCaptureParameter = VideoPreset.QHD.capture
+    var captureParams: VideoCaptureParameter = VideoPreset169.QHD.capture
 )
 
-class VideoCaptureParameter(
+data class VideoCaptureParameter(
     val width: Int,
     val height: Int,
     val maxFps: Int,
 )
 
-class VideoEncoding(
+data class VideoEncoding(
     val maxBitrate: Int,
     val maxFps: Int,
-)
+) {
+    fun toRtpEncoding(
+        rid: String? = null,
+        scaleDownBy: Double = 1.0,
+    ): RtpParameters.Encoding {
+        return RtpParameters.Encoding(rid, true, scaleDownBy).apply {
+            numTemporalLayers = 1
+            maxBitrateBps = maxBitrate
+            maxFramerate = maxFps
+
+            // only set on the full track
+            if (scaleDownBy == 1.0) {
+                networkPriority = 3 // high, from priority.h in webrtc
+                bitratePriority = 4.0
+            } else {
+                networkPriority = 1 // low, from priority.h in webrtc
+                bitratePriority = 1.0
+            }
+
+        }
+    }
+}
+
+enum class VideoCodec(val codecName: String) {
+    VP8("vp8"),
+    H264("h264"),
+}
 
 enum class CameraPosition {
     FRONT,
     BACK
 }
 
+interface VideoPreset {
+    val capture: VideoCaptureParameter
+    val encoding: VideoEncoding
+}
+
 /**
- * Video presets along with suggested bitrates
+ * 16:9 Video presets along with suggested bitrates
  */
-enum class VideoPreset(
-    val capture: VideoCaptureParameter,
-    val encoding: VideoEncoding,
-) {
+enum class VideoPreset169(
+    override val capture: VideoCaptureParameter,
+    override val encoding: VideoEncoding,
+) : VideoPreset {
     QVGA(
-        VideoCaptureParameter(320, 240, 15),
-        VideoEncoding(100_000, 15),
+        VideoCaptureParameter(320, 180, 15),
+        VideoEncoding(125_000, 15),
     ),
     VGA(
         VideoCaptureParameter(640, 360, 30),
@@ -39,15 +72,43 @@ enum class VideoPreset(
     ),
     QHD(
         VideoCaptureParameter(960, 540, 30),
-        VideoEncoding(700_000, 30),
+        VideoEncoding(800_000, 30),
     ),
     HD(
         VideoCaptureParameter(1280, 720, 30),
-        VideoEncoding(2_000_000, 30),
+        VideoEncoding(2_500_000, 30),
     ),
     FHD(
         VideoCaptureParameter(1920, 1080, 30),
         VideoEncoding(4_000_000, 30),
     )
+}
 
+/**
+ * 4:3 Video presets along with suggested bitrates
+ */
+enum class VideoPreset43(
+    override val capture: VideoCaptureParameter,
+    override val encoding: VideoEncoding,
+) : VideoPreset {
+    QVGA(
+        VideoCaptureParameter(240, 180, 15),
+        VideoEncoding(100_000, 15),
+    ),
+    VGA(
+        VideoCaptureParameter(480, 360, 30),
+        VideoEncoding(320_000, 30),
+    ),
+    QHD(
+        VideoCaptureParameter(720, 540, 30),
+        VideoEncoding(640_000, 30),
+    ),
+    HD(
+        VideoCaptureParameter(960, 720, 30),
+        VideoEncoding(2_000_000, 30),
+    ),
+    FHD(
+        VideoCaptureParameter(1440, 1080, 30),
+        VideoEncoding(3_200_000, 30),
+    )
 }
\ No newline at end of file

@@ -44,7 +44,7 @@ open class Track(
         }
     }
 
-    class Dimensions(var width: Int, var height: Int)
+    data class Dimensions(var width: Int, var height: Int)
 
     open fun stop() {
         rtcTrack.setEnabled(false)

+package io.livekit.android.webrtc
+
+import io.livekit.android.util.LKLog
+import org.webrtc.*
+import java.util.concurrent.*
+
+/*
+Copyright 2017, Lyo Kato <lyo.kato at gmail.com> (Original Author)
+Copyright 2017-2021, Shiguredo Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+ */
+internal class SimulcastVideoEncoderFactoryWrapper(
+    sharedContext: EglBase.Context?,
+    enableIntelVp8Encoder: Boolean,
+    enableH264HighProfile: Boolean
+) : VideoEncoderFactory {
+
+    /**
+     * Factory that prioritizes software encoder.
+     *
+     * When the selected codec can't be handled by the software encoder,
+     * it uses the hardware encoder as a fallback. However, this class is
+     * primarily used to address an issue in libwebrtc, and does not have
+     * purposeful usecase itself.
+     *
+     * To use simulcast in libwebrtc, SimulcastEncoderAdapter is used.
+     * SimulcastEncoderAdapter takes in a primary and fallback encoder.
+     * If HardwareVideoEncoderFactory and SoftwareVideoEncoderFactory are
+     * passed in directly as primary and fallback, when H.264 is used,
+     * libwebrtc will crash.
+     *
+     * This is because SoftwareVideoEncoderFactory does not handle H.264,
+     * so [SoftwareVideoEncoderFactory.createEncoder] returns null, and
+     * the libwebrtc side does not handle nulls, regardless of whether the
+     * fallback is actually used or not.
+     *
+     * To avoid nulls, we simply pass responsibility over to the HardwareVideoEncoderFactory.
+     * This results in HardwareVideoEncoderFactory being both the primary and fallback,
+     * but there aren't any specific problems in doing so.
+     */
+    private class Fallback(private val hardwareVideoEncoderFactory: VideoEncoderFactory) :
+        VideoEncoderFactory {
+
+        private val softwareVideoEncoderFactory: VideoEncoderFactory = SoftwareVideoEncoderFactory()
+
+        override fun createEncoder(info: VideoCodecInfo): VideoEncoder? {
+            val softwareEncoder = softwareVideoEncoderFactory.createEncoder(info)
+            val hardwareEncoder = hardwareVideoEncoderFactory.createEncoder(info)
+            return if (hardwareEncoder != null && softwareEncoder != null) {
+                VideoEncoderFallback(hardwareEncoder, softwareEncoder)
+            } else {
+                softwareEncoder ?: hardwareEncoder
+            }
+        }
+
+        override fun getSupportedCodecs(): Array<VideoCodecInfo> {
+            val supportedCodecInfos: MutableList<VideoCodecInfo> = mutableListOf()
+            supportedCodecInfos.addAll(softwareVideoEncoderFactory.supportedCodecs)
+            supportedCodecInfos.addAll(hardwareVideoEncoderFactory.supportedCodecs)
+            return supportedCodecInfos.toTypedArray()
+        }
+
+    }
+
+    /**
+     * Wraps each stream encoder and performs the following:
+     * - Starts up a single thread
+     * - When the width/height from [initEncode] doesn't match the frame buffer's,
+     *   scales the frame prior to encoding.
+     * - Always calls the encoder on the thread.
+     */
+    private class StreamEncoderWrapper(private val encoder: VideoEncoder) : VideoEncoder {
+
+        val executor: ExecutorService = Executors.newSingleThreadExecutor()
+        var streamSettings: VideoEncoder.Settings? = null
+
+        override fun initEncode(
+            settings: VideoEncoder.Settings,
+            callback: VideoEncoder.Callback?
+        ): VideoCodecStatus {
+            streamSettings = settings
+            val future = executor.submit(Callable {
+                LKLog.i {
+                    """initEncode() thread=${Thread.currentThread().name} [${Thread.currentThread().id}]
+                |  streamSettings:
+                |    numberOfCores=${settings.numberOfCores}
+                |    width=${settings.width}
+                |    height=${settings.height}
+                |    startBitrate=${settings.startBitrate}
+                |    maxFramerate=${settings.maxFramerate}
+                |    automaticResizeOn=${settings.automaticResizeOn}
+                |    numberOfSimulcastStreams=${settings.numberOfSimulcastStreams}
+                |    lossNotification=${settings.capabilities.lossNotification}
+            """.trimMargin()
+                }
+                return@Callable encoder.initEncode(settings, callback)
+            })
+            return future.get()
+        }
+
+        override fun release(): VideoCodecStatus {
+            val future = executor.submit(Callable { return@Callable encoder.release() })
+            return future.get()
+        }
+
+        override fun encode(
+            frame: VideoFrame,
+            encodeInfo: VideoEncoder.EncodeInfo?
+        ): VideoCodecStatus {
+            val future = executor.submit(Callable {
+                //LKLog.d { "encode() buffer=${frame.buffer}, thread=${Thread.currentThread().name} " +
+                //        "[${Thread.currentThread().id}]" }
+                if (streamSettings == null) {
+                    return@Callable encoder.encode(frame, encodeInfo)
+                } else if (frame.buffer.width == streamSettings!!.width) {
+                    return@Callable encoder.encode(frame, encodeInfo)
+                } else {
+                    // The incoming buffer is different than the streamSettings received in initEncode()
+                    // Need to scale.
+                    val originalBuffer = frame.buffer
+                    // TODO: Do we need to handle when the scale factor is weird?
+                    val adaptedBuffer = originalBuffer.cropAndScale(
+                        0, 0, originalBuffer.width, originalBuffer.height,
+                        streamSettings!!.width, streamSettings!!.height
+                    )
+                    val adaptedFrame = VideoFrame(adaptedBuffer, frame.rotation, frame.timestampNs)
+                    val result = encoder.encode(adaptedFrame, encodeInfo)
+                    adaptedBuffer.release()
+                    return@Callable result
+                }
+            })
+            return future.get()
+        }
+
+        override fun setRateAllocation(
+            allocation: VideoEncoder.BitrateAllocation?,
+            frameRate: Int
+        ): VideoCodecStatus {
+            val future = executor.submit(Callable {
+                return@Callable encoder.setRateAllocation(
+                    allocation,
+                    frameRate
+                )
+            })
+            return future.get()
+        }
+
+        override fun getScalingSettings(): VideoEncoder.ScalingSettings {
+            val future = executor.submit(Callable { return@Callable encoder.scalingSettings })
+            return future.get()
+        }
+
+        override fun getImplementationName(): String {
+            val future = executor.submit(Callable { return@Callable encoder.implementationName })
+            return future.get()
+        }
+    }
+
+    private class StreamEncoderWrapperFactory(private val factory: VideoEncoderFactory) :
+        VideoEncoderFactory {
+        override fun createEncoder(videoCodecInfo: VideoCodecInfo?): VideoEncoder? {
+            val encoder = factory.createEncoder(videoCodecInfo)
+            if (encoder == null) {
+                return null
+            }
+            return StreamEncoderWrapper(encoder)
+        }
+
+        override fun getSupportedCodecs(): Array<VideoCodecInfo> {
+            return factory.supportedCodecs
+        }
+    }
+
+
+    private val primary: VideoEncoderFactory
+    private val fallback: VideoEncoderFactory
+    private val native: SimulcastVideoEncoderFactory
+
+    init {
+        val hardwareVideoEncoderFactory = HardwareVideoEncoderFactory(
+            sharedContext, enableIntelVp8Encoder, enableH264HighProfile
+        )
+        primary = StreamEncoderWrapperFactory(hardwareVideoEncoderFactory)
+        fallback = StreamEncoderWrapperFactory(Fallback(primary))
+        native = SimulcastVideoEncoderFactory(primary, fallback)
+    }
+
+    override fun createEncoder(info: VideoCodecInfo?): VideoEncoder? {
+        return native.createEncoder(info)
+    }
+
+    override fun getSupportedCodecs(): Array<VideoCodecInfo> {
+        return native.supportedCodecs
+    }
+
+}
\ No newline at end of file