Send video layers (#55)

* signal method * videoLayers param on addTrack request * screen share presets * implemented * cleaner syntax
livekit · Dec 15, 2021 · 28e7091 · 28e7091
1 parent b24fad0
commit 28e7091
Show file tree

Hide file tree

Showing 4 changed files with 160 additions and 57 deletions.
diff --git a/lib/src/participant/local_participant.dart b/lib/src/participant/local_participant.dart
@@ -119,29 +119,35 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
       }
     }
 
+    logger.fine(
+        'Compute encodings with resolution: ${dimensions}, options: ${publishOptions}');
+
+    // Video encodings and simulcasts
+    final encodings = Utils.computeVideoEncodings(
+      isScreenShare: track.source == TrackSource.screenShareVideo,
+      dimensions: dimensions,
+      options: publishOptions,
+    );
+
+    logger.fine('Using encodings: ${encodings?.map((e) => e.toMap())}');
+
+    final layers = Utils.computeVideoLayers(dimensions, encodings);
+
+    logger.fine('Video layers: ${layers.map((e) => e)}');
+
     final trackInfo = await room.engine.addTrack(
       cid: track.getCid(),
       name: track.name,
       kind: track.kind,
       source: track.source.toPBType(),
       dimensions: dimensions,
+      videoLayers: layers,
     );
 
     logger.fine('publishVideoTrack addTrack response: ${trackInfo}');
 
     await track.start();
 
-    logger.fine(
-        'Compute encodings with resolution: ${dimensions}, options: ${publishOptions}');
-
-    // Video encodings and simulcasts
-    final encodings = Utils.computeVideoEncodings(
-      dimensions: dimensions,
-      options: publishOptions,
-    );
-
-    logger.fine('Using encodings: ${encodings?.map((e) => e.toMap())}');
-
     final transceiverInit = rtc.RTCRtpTransceiverInit(
       direction: rtc.TransceiverDirection.SendOnly,
       sendEncodings: encodings,

diff --git a/lib/src/rtc_engine.dart b/lib/src/rtc_engine.dart
@@ -149,6 +149,7 @@ class RTCEngine extends Disposable with EventsEmittable<EngineEvent> {
     required lk_models.TrackSource source,
     VideoDimensions? dimensions,
     bool? dtx,
+    List<lk_models.VideoLayer>? videoLayers,
   }) async {
     // TODO: Check if cid already published
 
@@ -160,6 +161,7 @@ class RTCEngine extends Disposable with EventsEmittable<EngineEvent> {
       source: source,
       dimensions: dimensions,
       dtx: dtx,
+      videoLayers: videoLayers,
     );
 
     // wait for response, or timeout

diff --git a/lib/src/signal_client.dart b/lib/src/signal_client.dart
@@ -150,6 +150,7 @@ class SignalClient extends Disposable with EventsEmittable<SignalEvent> {
     required lk_models.TrackSource source,
     VideoDimensions? dimensions,
     bool? dtx,
+    List<lk_models.VideoLayer>? videoLayers,
   }) {
     final req = lk_rtc.AddTrackRequest(
       cid: cid,
@@ -158,10 +159,17 @@ class SignalClient extends Disposable with EventsEmittable<SignalEvent> {
       source: source,
     );
 
-    if (type == lk_models.TrackType.VIDEO && dimensions != null) {
+    if (type == lk_models.TrackType.VIDEO) {
       // video specific
-      req.width = dimensions.width;
-      req.height = dimensions.height;
+      if (dimensions != null) {
+        req.width = dimensions.width;
+        req.height = dimensions.height;
+      }
+      if (videoLayers != null && videoLayers.isNotEmpty) {
+        req.layers
+          ..clear()
+          ..addAll(videoLayers);
+      }
     }
 
     if (type == lk_models.TrackType.AUDIO && dtx != null) {
@@ -184,6 +192,17 @@ class SignalClient extends Disposable with EventsEmittable<SignalEvent> {
         subscription: subscription,
       ));
 
+  void sendUpdateVideoLayers(
+    String trackSid,
+    List<lk_models.VideoLayer> layers,
+  ) =>
+      _sendRequest(lk_rtc.SignalRequest(
+        updateLayers: lk_rtc.UpdateVideoLayers(
+          trackSid: trackSid,
+          layers: layers,
+        ),
+      ));
+
   void sendLeave() => _sendRequest(lk_rtc.SignalRequest(
         leave: lk_rtc.LeaveRequest(),
       ));

diff --git a/lib/src/utils.dart b/lib/src/utils.dart
@@ -1,9 +1,12 @@
 import 'dart:async';
 
 import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
+import 'package:collection/collection.dart';
 
+import './proto/livekit_models.pb.dart' as lk_models;
 import 'extensions.dart';
 import 'livekit.dart';
+import 'logger.dart';
 import 'options.dart';
 import 'track/options.dart';
 import 'types.dart';
@@ -55,85 +58,158 @@ class Utils {
     );
   }
 
-  static List<VideoParameters> _presetsForDimensions(
-    VideoDimensions dimensions,
-  ) {
-    final double aspect = dimensions.width / dimensions.height;
+  static List<VideoParameters> _presetsForDimensions({
+    required bool isScreenShare,
+    required VideoDimensions dimensions,
+  }) {
+    if (isScreenShare) return VideoParameters.presetsScreenShare;
+
+    final double aspect = dimensions.width > dimensions.height
+        ? dimensions.width / dimensions.height
+        : dimensions.height / dimensions.width;
     if ((aspect - 16.0 / 9.0).abs() < (aspect - 4.0 / 3.0).abs()) {
       return VideoParameters.presets169;
     }
     return VideoParameters.presets43;
   }
 
-  static VideoParameters _findPresetForDimensions(
-    VideoDimensions dimensions, {
+  static VideoEncoding _findAppropriateEncoding({
+    required bool isScreenShare,
+    required VideoDimensions dimensions,
     required List<VideoParameters> presets,
   }) {
     assert(presets.isNotEmpty, 'presets should not be empty');
-    VideoParameters result = presets.first;
+    VideoEncoding result = presets.first.encoding;
+
+    // handle portrait by swapping dimensions
+    final size = dimensions.max();
+
     for (final preset in presets) {
-      if (dimensions.width >= preset.dimensions.width &&
-          dimensions.height >= preset.dimensions.height) result = preset;
+      result = preset.encoding;
+      if (preset.dimensions.width >= size) break;
     }
 
     return result;
   }
 
+  static final videoRids = ['q', 'h', 'f'];
+
+  static List<rtc.RTCRtpEncoding> encodingsFromPresets(
+    VideoDimensions dimensions, {
+    required List<VideoParameters> presets,
+  }) {
+    List<rtc.RTCRtpEncoding> result = [];
+    presets.forEachIndexed((i, e) {
+      if (i >= videoRids.length) {
+        return;
+      }
+      final size = dimensions.max();
+      final rid = videoRids[i];
+      result.add(e.encoding.toRTCRtpEncoding(
+        rid: rid,
+        scaleResolutionDownBy: size / e.dimensions.height,
+      ));
+    });
+    return result;
+  }
+
   static List<rtc.RTCRtpEncoding>? computeVideoEncodings({
+    required bool isScreenShare,
     VideoDimensions? dimensions,
     VideoPublishOptions? options,
   }) {
     options ??= const VideoPublishOptions();
 
     VideoEncoding? videoEncoding = options.videoEncoding;
 
-    if ((videoEncoding == null && !options.simulcast) || dimensions == null) {
+    final useSimulcast = !isScreenShare && options.simulcast;
+
+    if ((videoEncoding == null && !useSimulcast) || dimensions == null) {
       // don't set encoding when we are not simulcasting and user isn't restricting
       // encoding parameters
       return null;
     }
 
-    final presets = _presetsForDimensions(dimensions);
+    final presets = _presetsForDimensions(
+      isScreenShare: isScreenShare,
+      dimensions: dimensions,
+    );
 
     if (videoEncoding == null) {
       // find the right encoding based on width/height
-      final preset = _findPresetForDimensions(dimensions, presets: presets);
-      // print('Using preset: ${preset.id}');
-      videoEncoding = preset.encoding;
-      //   log.debug('using video encoding', videoEncoding);
+      videoEncoding = _findAppropriateEncoding(
+        isScreenShare: isScreenShare,
+        dimensions: dimensions,
+        presets: presets,
+      );
+      logger.fine('using video encoding', videoEncoding);
     }
 
     // Not simulcast
-    if (!options.simulcast) return [videoEncoding.toRTCRtpEncoding()];
-
-    // Compute for simulcast
-    final midPreset = presets[1];
-    final lowPreset = presets[0];
-    return [
-      videoEncoding.toRTCRtpEncoding(
-        rid: 'f',
-      ),
-      // if resolution is high enough, we would send both h and q res..
-      // otherwise only send h
-      if (dimensions.max() >= 960) ...[
-        midPreset.encoding.toRTCRtpEncoding(
-          rid: 'h',
-          // passing decimals to hardware encoder of android devices
-          // often causes issues so we better use integers
-          scaleResolutionDownBy: 2,
-        ),
-        lowPreset.encoding.toRTCRtpEncoding(
-          rid: 'q',
-          scaleResolutionDownBy: 4,
-        ),
-      ] else
-        lowPreset.encoding.toRTCRtpEncoding(
-          rid: 'h',
-          scaleResolutionDownBy: 2,
-        ),
-    ];
+    if (!useSimulcast) return [videoEncoding.toRTCRtpEncoding()];
+
+    final VideoParameters lowPreset = presets.first;
+    VideoParameters? midPreset;
+    if (presets.length > 1) {
+      midPreset = presets[1];
+    }
+    final original = VideoParameters(
+      dimensions: dimensions,
+      encoding: videoEncoding,
+    );
+
+    final size = dimensions.max();
+    List<VideoParameters> computedPresets = [original];
+
+    if (size >= 960 && midPreset != null) {
+      computedPresets = [lowPreset, midPreset, original];
+    } else if (size >= 500) {
+      computedPresets = [lowPreset, original];
+    }
+
+    return encodingsFromPresets(
+      dimensions,
+      presets: computedPresets,
+    );
   }
 
+  static List<lk_models.VideoLayer> computeVideoLayers(
+    VideoDimensions dimensions,
+    List<rtc.RTCRtpEncoding>? encodings,
+  ) {
+    // default to a single layer, HQ
+    if (encodings == null) {
+      return [
+        lk_models.VideoLayer(
+          quality: lk_models.VideoQuality.HIGH,
+          width: dimensions.width,
+          height: dimensions.height,
+          bitrate: 0,
+        )
+      ];
+    }
+
+    return encodings.map((e) {
+      final scale = e.scaleResolutionDownBy ?? 1;
+      var quality = videoQualityForRid(e.rid);
+      if (quality == null && encodings.length == 1) {
+        quality = lk_models.VideoQuality.HIGH;
+      }
+      return lk_models.VideoLayer(
+        quality: quality,
+        width: (dimensions.width.toDouble() / scale).floor(),
+        height: (dimensions.height.toDouble() / scale).floor(),
+        bitrate: e.maxBitrate ?? 0,
+      );
+    }).toList();
+  }
+
+  static lk_models.VideoQuality? videoQualityForRid(String? rid) => {
+        'f': lk_models.VideoQuality.HIGH,
+        'h': lk_models.VideoQuality.MEDIUM,
+        'q': lk_models.VideoQuality.LOW,
+      }[rid];
+
   // makes a debounce func, with 1 param
   static Function(T) createDebounceFunc<T>(
     Function(T) f, {