Simulcast support for iOS/Android.

Simulcast support for iOS SDK (#4)

Support for simulcast in Android SDK (#3)

include simulcast headers for mac also (#10)

Fix simulcast using hardware encoder on Android (#48)

Co-authored-by: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com>
Co-authored-by: Angelika Serwa <angelika.serwa@gmail.com>

sdk/BUILD.gn

@@ -723,6 +723,7 @@ if (is_ios || is_mac) {
       ]
 
       deps = [
+        ":simulcast",
         ":base_objc",
         ":native_video",
         ":videocodec_objc",
@@ -822,6 +823,22 @@ if (is_ios || is_mac) {
       ]
     }
 
+    rtc_library("simulcast") {
+      sources = [
+        "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
+        "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm",
+        "objc/api/video_codec/RTCVideoEncoderSimulcast.h",
+        "objc/api/video_codec/RTCVideoEncoderSimulcast.mm",
+      ]
+
+      deps = [
+        ":base_objc",
+        ":wrapped_native_codec_objc",
+        "../media:rtc_media_base",
+        "../media:rtc_simulcast_encoder_adapter",
+      ]
+    }
+
     rtc_library("mediaconstraints_objc") {
       configs += [ "..:no_global_constructors" ]
       sources = [
@@ -1368,6 +1385,9 @@ if (is_ios || is_mac) {
           "objc/api/video_codec/RTCVideoEncoderAV1.h",
           "objc/api/video_frame_buffer/RTCNativeI420Buffer.h",
           "objc/api/video_frame_buffer/RTCNativeMutableI420Buffer.h",
+          # Added for Simulcast support
+          "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
+          "objc/api/video_codec/RTCVideoEncoderSimulcast.h",
         ]
 
         if (!build_with_chromium) {
@@ -1518,6 +1538,9 @@ if (is_ios || is_mac) {
           "objc/components/video_codec/RTCVideoEncoderH264.h",
           "objc/components/video_frame_buffer/RTCCVPixelBuffer.h",
           "objc/helpers/RTCDispatcher.h",
+          # Added for Simulcast support
+          "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
+          "objc/api/video_codec/RTCVideoEncoderSimulcast.h",
         ]
         if (!build_with_chromium) {
           sources += [

sdk/android/BUILD.gn

@@ -530,6 +530,8 @@ if (is_android) {
     sources = [
       "api/org/webrtc/SoftwareVideoDecoderFactory.java",
       "api/org/webrtc/SoftwareVideoEncoderFactory.java",
+      "api/org/webrtc/SimulcastVideoEncoder.java",
+      "api/org/webrtc/SimulcastVideoEncoderFactory.java",
     ]
 
     deps = [
@@ -908,6 +910,21 @@ if (current_os == "linux" || is_android) {
     ]
   }
 
+  rtc_library("simulcast_jni") {
+    visibility = [ "*" ]
+    allow_poison = [ "software_video_codecs" ]
+    sources = [
+      "src/jni/simulcast_video_encoder.cc",
+      "src/jni/simulcast_video_encoder.h"
+    ]
+    deps = [
+      ":base_jni",
+      ":video_jni",
+      ":native_api_codecs",
+      "../../media:rtc_simulcast_encoder_adapter"
+    ]
+  }
+
   rtc_library("swcodecs_jni") {
     visibility = [ "*" ]
     allow_poison = [ "software_video_codecs" ]
@@ -921,6 +938,7 @@ if (current_os == "linux" || is_android) {
       ":libvpx_vp8_jni",
       ":libvpx_vp9_jni",
       ":native_api_jni",
+      ":simulcast_jni",
       ":video_jni",
       "../../api/environment",
       "../../api/video_codecs:builtin_video_decoder_factory",

sdk/android/api/org/webrtc/SimulcastVideoEncoder.java

@@ -0,0 +1,28 @@
+package org.webrtc;
+
+public class SimulcastVideoEncoder extends WrappedNativeVideoEncoder {
+
+    static native long nativeCreateEncoder(VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info);
+
+    VideoEncoderFactory primary;
+    VideoEncoderFactory fallback;
+    VideoCodecInfo info;
+
+    public SimulcastVideoEncoder(VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info) {
+        this.primary = primary;
+        this.fallback = fallback;
+        this.info = info;
+    }
+
+    @Override
+    public long createNativeVideoEncoder() {
+        return nativeCreateEncoder(primary, fallback, info);
+    }
+
+    @Override
+    public boolean isHardwareEncoder() {
+        return false;
+    }
+
+}
+

sdk/android/api/org/webrtc/SimulcastVideoEncoderFactory.java

@@ -0,0 +1,43 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import androidx.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Arrays;
+
+public class SimulcastVideoEncoderFactory implements VideoEncoderFactory {
+
+    VideoEncoderFactory primary;
+    VideoEncoderFactory fallback;
+
+    public SimulcastVideoEncoderFactory(VideoEncoderFactory primary, VideoEncoderFactory fallback) {
+        this.primary = primary;
+        this.fallback = fallback;
+    }
+
+    @Nullable
+    @Override
+    public VideoEncoder createEncoder(VideoCodecInfo info) {
+        return new SimulcastVideoEncoder(primary, fallback, info);
+    }
+
+    @Override
+    public VideoCodecInfo[] getSupportedCodecs() {
+        List<VideoCodecInfo> codecs = new ArrayList<VideoCodecInfo>();
+        codecs.addAll(Arrays.asList(primary.getSupportedCodecs()));
+        codecs.addAll(Arrays.asList(fallback.getSupportedCodecs()));
+        return codecs.toArray(new VideoCodecInfo[codecs.size()]);
+    }
+
+}

sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java

@@ -45,8 +45,8 @@ class HardwareVideoEncoder implements VideoEncoder {
   private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000;
   private static final int DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US = 100000;
 
-  // Size of the input frames should be multiple of 16 for the H/W encoder.
-  private static final int REQUIRED_RESOLUTION_ALIGNMENT = 16;
+  // Size of the input frames should be multiple of 2 for the H/W encoder.
+  private static final int REQUIRED_RESOLUTION_ALIGNMENT = 2;
 
   /**
    * Keeps track of the number of output buffers that have been passed down the pipeline and not yet
@@ -210,6 +210,11 @@ public VideoCodecStatus initEncode(Settings settings, Callback callback) {
     this.callback = callback;
     automaticResizeOn = settings.automaticResizeOn;
 
+    if (settings.width % REQUIRED_RESOLUTION_ALIGNMENT != 0
+        || settings.height % REQUIRED_RESOLUTION_ALIGNMENT != 0) {
+      Logging.e(TAG, "MediaCodec requires 2x2 alignment.");
+      return VideoCodecStatus.ERR_SIZE;
+    }
     this.width = settings.width;
     this.height = settings.height;
     useSurfaceMode = canUseSurface();
@@ -533,6 +538,12 @@ private VideoCodecStatus resetCodec(int newWidth, int newHeight, boolean newUseS
     if (status != VideoCodecStatus.OK) {
       return status;
     }
+
+    if (newWidth % REQUIRED_RESOLUTION_ALIGNMENT != 0
+        || newHeight % REQUIRED_RESOLUTION_ALIGNMENT != 0) {
+      Logging.e(TAG, "MediaCodec requires 2x2 alignment.");
+      return VideoCodecStatus.ERR_SIZE;
+    }
     width = newWidth;
     height = newHeight;
     useSurfaceMode = newUseSurfaceMode;

sdk/android/src/jni/simulcast_video_encoder.cc

@@ -0,0 +1,34 @@
+#include <jni.h>
+
+#include "sdk/android/src/jni/jni_helpers.h"
+#include "sdk/android/src/jni/video_encoder_factory_wrapper.h"
+#include "sdk/android/src/jni/video_codec_info.h"
+#include "sdk/android/native_api/codecs/wrapper.h"
+#include "media/engine/simulcast_encoder_adapter.h"
+#include "rtc_base/logging.h"
+
+using namespace webrtc;
+using namespace webrtc::jni;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// (VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info)
+JNIEXPORT jlong JNICALL Java_org_webrtc_SimulcastVideoEncoder_nativeCreateEncoder(JNIEnv *env, jclass klass, jobject primary, jobject fallback, jobject info) {
+    RTC_LOG(LS_INFO) << "Create simulcast video encoder";
+    JavaParamRef<jobject> info_ref(info);
+    SdpVideoFormat format = VideoCodecInfoToSdpVideoFormat(env, info_ref);
+
+    // TODO: 影響は軽微だが、リークする可能性があるので将来的に修正したい
+    // https://github.com/shiguredo-webrtc-build/webrtc-build/pull/16#pullrequestreview-600675795
+    return NativeToJavaPointer(std::make_unique<SimulcastEncoderAdapter>(
+			    JavaToNativeVideoEncoderFactory(env, primary).release(),
+			    JavaToNativeVideoEncoderFactory(env, fallback).release(),
+			    format).release());
+}
+
+
+#ifdef __cplusplus
+}
+#endif

sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.h

@@ -0,0 +1,13 @@
+#import "RTCMacros.h"
+#import "RTCVideoEncoder.h"
+#import "RTCVideoEncoderFactory.h"
+#import "RTCVideoCodecInfo.h"
+
+RTC_OBJC_EXPORT
+@interface RTC_OBJC_TYPE (RTCVideoEncoderSimulcast) : NSObject
+
++ (id<RTC_OBJC_TYPE(RTCVideoEncoder)>)simulcastEncoderWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
+                                                         fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback
+                                                   videoCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)videoCodecInfo;
+
+@end

sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.mm

@@ -0,0 +1,26 @@
+#import <Foundation/Foundation.h>
+
+#import "RTCMacros.h"
+#import "RTCVideoEncoderSimulcast.h"
+#import "RTCWrappedNativeVideoEncoder.h"
+#import "api/peerconnection/RTCVideoCodecInfo+Private.h"
+
+#include "native/api/video_encoder_factory.h"
+#include "media/engine/simulcast_encoder_adapter.h"
+
+@implementation RTC_OBJC_TYPE (RTCVideoEncoderSimulcast)
+
++ (id<RTC_OBJC_TYPE(RTCVideoEncoder)>)simulcastEncoderWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
+                                                         fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback
+                                                   videoCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)videoCodecInfo {
+    auto nativePrimary = webrtc::ObjCToNativeVideoEncoderFactory(primary);
+    auto nativeFallback = webrtc::ObjCToNativeVideoEncoderFactory(fallback);
+    auto nativeFormat = [videoCodecInfo nativeSdpVideoFormat];
+    return [[RTC_OBJC_TYPE(RTCWrappedNativeVideoEncoder) alloc]
+        initWithNativeEncoder: std::make_unique<webrtc::SimulcastEncoderAdapter>(
+            nativePrimary.release(),
+            nativeFallback.release(),
+            std::move(nativeFormat))];
+}
+
+@end

sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h

@@ -0,0 +1,16 @@
+#import <Foundation/Foundation.h>
+
+#import "RTCMacros.h"
+#import "RTCVideoEncoderFactory.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+RTC_OBJC_EXPORT
+@interface RTC_OBJC_TYPE (RTCVideoEncoderFactorySimulcast) : NSObject <RTC_OBJC_TYPE(RTCVideoEncoderFactory)>
+
+- (instancetype)initWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
+                       fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback;
+
+@end
+
+NS_ASSUME_NONNULL_END

sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm

@@ -0,0 +1,39 @@
+#import <Foundation/Foundation.h>
+
+#import "RTCMacros.h"
+#import "RTCVideoCodecInfo.h"
+#import "RTCVideoEncoderFactorySimulcast.h"
+#import "api/video_codec/RTCVideoEncoderSimulcast.h"
+
+@interface RTC_OBJC_TYPE (RTCVideoEncoderFactorySimulcast) ()
+
+@property id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> primary;
+@property id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> fallback;
+
+@end
+
+
+@implementation RTC_OBJC_TYPE (RTCVideoEncoderFactorySimulcast)
+
+@synthesize primary = _primary;
+@synthesize fallback = _fallback;
+
+- (instancetype)initWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
+                       fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback {
+    if (self = [super init]) {
+        _primary = primary;
+        _fallback = fallback;
+    }
+    return self;
+}
+
+- (nullable id<RTC_OBJC_TYPE(RTCVideoEncoder)>)createEncoder: (RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info {
+    return [RTCVideoEncoderSimulcast simulcastEncoderWithPrimary: _primary fallback: _fallback videoCodecInfo: info];
+}
+
+- (NSArray<RTC_OBJC_TYPE(RTCVideoCodecInfo) *> *)supportedCodecs {
+    return [[_primary supportedCodecs] arrayByAddingObjectsFromArray: [_fallback supportedCodecs]];
+}
+
+
+@end

sdk/objc/native/src/audio/audio_device_ios.mm

@@ -62,16 +62,6 @@
 const UInt16 kFixedPlayoutDelayEstimate = 30;
 const UInt16 kFixedRecordDelayEstimate = 30;
 
-enum AudioDeviceMessageType : uint32_t {
-  kMessageTypeInterruptionBegin,
-  kMessageTypeInterruptionEnd,
-  kMessageTypeValidRouteChange,
-  kMessageTypeCanPlayOrRecordChange,
-  kMessageTypePlayoutGlitchDetected,
-  kMessageOutputVolumeChange,
-  kMessageTypeAudioWillRecord,
-};
-
 using ios::CheckAndLogError;
 
 #if !defined(NDEBUG)