Update M83 code to work with owt-sdk (webrtc-sdk#68)

* Add HEVC support for iOS/Android

* Some changes for building with OWT

* Enable openssl

* Add create_peerconnection_factory to WebRTC.framework. (webrtc-sdk#46)

* Set kVTCompressionPropertyKey_RealTime to true. (webrtc-sdk#51)

* H265 packetization_mode setting fix (webrtc-sdk#53)

* add H.265 QP parsing logic (webrtc-sdk#47)

* Fix linux build error. (webrtc-sdk#54)

* Add h264 prefix NAL parser implmentation for enabling frame-marking for h.264 (webrtc-sdk#58)

* Make hevc rtp depacketizer/tracker conforming to h.264 design

Co-authored-by: jianjunz <jianjun.zhu@intel.com>
Co-authored-by: Cyril Lashkevich <notorca@gmail.com>
Co-authored-by: Piasy <xz4215@gmail.com>
Co-authored-by: ShiJinCheng <874042641@qq.com>

BUILD.gn

@@ -193,6 +193,25 @@ config("common_inherited_config") {
       target_gen_dir,
     ]
   }
+  if (build_with_owt) {
+    include_dirs = [
+      # The overrides must be included first as that is the mechanism for
+      # selecting the override headers in Chromium.
+      #"../webrtc_overrides",
+
+      # Allow includes to be prefixed with webrtc/ in case it is not an
+      # immediate subdirectory of the top-level.
+      ".",
+
+      # Just like the root WebRTC directory is added to include path, the
+      # corresponding directory tree with generated files needs to be added too.
+      # Note: this path does not change depending on the current target, e.g.
+      # it is always "//gen/third_party/webrtc" when building with Chromium.
+      # See also: http://cs.chromium.org/?q=%5C"default_include_dirs
+      # https://gn.googlesource.com/gn/+/master/docs/reference.md#target_gen_dir
+      target_gen_dir,
+    ]
+  }
   if (is_posix || is_fuchsia) {
     defines += [ "WEBRTC_POSIX" ]
   }
@@ -237,6 +256,10 @@ config("common_inherited_config") {
   if (is_ubsan) {
     cflags += [ "-fsanitize=float-cast-overflow" ]
   }
+
+  if (!rtc_use_h265) {
+    defines += [ "DISABLE_H265" ]
+  }
 }
 
 # TODO(bugs.webrtc.org/9693): Remove the possibility to suppress this warning
@@ -329,7 +352,7 @@ config("common_config") {
     ]
   }
 
-  if (build_with_chromium) {
+  if (build_with_chromium || build_with_owt) {
     defines += [
       # NOTICE: Since common_inherited_config is used in public_configs for our
       # targets, there's no point including the defines in that config here.
@@ -457,10 +480,13 @@ if (!build_with_chromium) {
   rtc_static_library("webrtc") {
     # Only the root target and the test should depend on this.
     visibility = [
-      "//:default",
-      "//:webrtc_lib_link_test",
+      ".:default",
+      ":webrtc_lib_link_test",
     ]
 
+    if (build_with_owt) {
+      visibility += [ "//talk/owt" ]
+    }
     sources = []
     complete_static_lib = true
     suppressed_configs += [ "//build/config/compiler:thin_archive" ]

api/video/video_codec_type.h

@@ -21,6 +21,9 @@ enum VideoCodecType {
   kVideoCodecVP9,
   kVideoCodecAV1,
   kVideoCodecH264,
+#ifndef DISABLE_H265
+  kVideoCodecH265,
+#endif
   kVideoCodecMultiplex,
 };
 

api/video_codecs/video_codec.cc

@@ -26,6 +26,9 @@ constexpr char kPayloadNameAv1[] = "AV1";
 // needed.
 constexpr char kPayloadNameAv1x[] = "AV1X";
 constexpr char kPayloadNameH264[] = "H264";
+#ifndef DISABLE_H265
+constexpr char kPayloadNameH265[] = "H265";
+#endif
 constexpr char kPayloadNameGeneric[] = "Generic";
 constexpr char kPayloadNameMultiplex[] = "Multiplex";
 }  // namespace
@@ -52,6 +55,17 @@ bool VideoCodecH264::operator==(const VideoCodecH264& other) const {
           numberOfTemporalLayers == other.numberOfTemporalLayers);
 }
 
+#ifndef DISABLE_H265
+bool VideoCodecH265::operator==(const VideoCodecH265& other) const {
+  return (frameDroppingOn == other.frameDroppingOn &&
+          keyFrameInterval == other.keyFrameInterval &&
+          vpsLen == other.vpsLen && spsLen == other.spsLen &&
+          ppsLen == other.ppsLen &&
+          (spsLen == 0 || memcmp(spsData, other.spsData, spsLen) == 0) &&
+          (ppsLen == 0 || memcmp(ppsData, other.ppsData, ppsLen) == 0));
+}
+#endif
+
 VideoCodec::VideoCodec()
     : codecType(kVideoCodecGeneric),
       width(0),
@@ -102,6 +116,18 @@ const VideoCodecH264& VideoCodec::H264() const {
   return codec_specific_.H264;
 }
 
+#ifndef DISABLE_H265
+VideoCodecH265* VideoCodec::H265() {
+  RTC_DCHECK_EQ(codecType, kVideoCodecH265);
+  return &codec_specific_.H265;
+}
+
+const VideoCodecH265& VideoCodec::H265() const {
+  RTC_DCHECK_EQ(codecType, kVideoCodecH265);
+  return codec_specific_.H265;
+}
+#endif
+
 const char* CodecTypeToPayloadString(VideoCodecType type) {
   switch (type) {
     case kVideoCodecVP8:
@@ -112,9 +138,14 @@ const char* CodecTypeToPayloadString(VideoCodecType type) {
       return kPayloadNameAv1;
     case kVideoCodecH264:
       return kPayloadNameH264;
+#ifndef DISABLE_H265
+    case kVideoCodecH265:
+      return kPayloadNameH265;
+#endif
     case kVideoCodecMultiplex:
       return kPayloadNameMultiplex;
     case kVideoCodecGeneric:
+	default:
       return kPayloadNameGeneric;
   }
   RTC_CHECK_NOTREACHED();
@@ -132,6 +163,10 @@ VideoCodecType PayloadStringToCodecType(const std::string& name) {
     return kVideoCodecH264;
   if (absl::EqualsIgnoreCase(name, kPayloadNameMultiplex))
     return kVideoCodecMultiplex;
+#ifndef DISABLE_H265
+  if (absl::EqualsIgnoreCase(name, kPayloadNameH265))
+    return kVideoCodecH265;
+#endif
   return kVideoCodecGeneric;
 }
 

api/video_codecs/video_codec.h

@@ -97,6 +97,23 @@ struct VideoCodecH264 {
   uint8_t numberOfTemporalLayers;
 };
 
+#ifndef DISABLE_H265
+struct VideoCodecH265 {
+  bool operator==(const VideoCodecH265& other) const;
+  bool operator!=(const VideoCodecH265& other) const {
+    return !(*this == other);
+  }
+  bool frameDroppingOn;
+  int keyFrameInterval;
+  const uint8_t* vpsData;
+  size_t vpsLen;
+  const uint8_t* spsData;
+  size_t spsLen;
+  const uint8_t* ppsData;
+  size_t ppsLen;
+};
+#endif
+
 // Translates from name of codec to codec type and vice versa.
 RTC_EXPORT const char* CodecTypeToPayloadString(VideoCodecType type);
 RTC_EXPORT VideoCodecType PayloadStringToCodecType(const std::string& name);
@@ -105,6 +122,9 @@ union VideoCodecUnion {
   VideoCodecVP8 VP8;
   VideoCodecVP9 VP9;
   VideoCodecH264 H264;
+#ifndef DISABLE_H265
+  VideoCodecH265 H265;
+#endif
 };
 
 enum class VideoCodecMode { kRealtimeVideo, kScreensharing };
@@ -184,6 +204,10 @@ class RTC_EXPORT VideoCodec {
   const VideoCodecVP9& VP9() const;
   VideoCodecH264* H264();
   const VideoCodecH264& H264() const;
+#ifndef DISABLE_H265
+  VideoCodecH265* H265();
+  const VideoCodecH265& H265() const;
+#endif
 
  private:
   // TODO(hta): Consider replacing the union with a pointer type.

api/video_codecs/video_encoder.cc

@@ -57,6 +57,23 @@ VideoCodecH264 VideoEncoder::GetDefaultH264Settings() {
   return h264_settings;
 }
 
+#ifndef DISABLE_H265
+VideoCodecH265 VideoEncoder::GetDefaultH265Settings() {
+  VideoCodecH265 h265_settings;
+  memset(&h265_settings, 0, sizeof(h265_settings));
+
+  // h265_settings.profile = kProfileBase;
+  h265_settings.frameDroppingOn = true;
+  h265_settings.keyFrameInterval = 3000;
+  h265_settings.spsData = nullptr;
+  h265_settings.spsLen = 0;
+  h265_settings.ppsData = nullptr;
+  h265_settings.ppsLen = 0;
+
+  return h265_settings;
+}
+#endif
+
 VideoEncoder::ScalingSettings::ScalingSettings() = default;
 
 VideoEncoder::ScalingSettings::ScalingSettings(KOff) : ScalingSettings() {}

api/video_codecs/video_encoder.h

@@ -335,6 +335,9 @@ class RTC_EXPORT VideoEncoder {
   static VideoCodecVP8 GetDefaultVp8Settings();
   static VideoCodecVP9 GetDefaultVp9Settings();
   static VideoCodecH264 GetDefaultH264Settings();
+#ifndef DISABLE_H265
+  static VideoCodecH265 GetDefaultH265Settings();
+#endif
 
   virtual ~VideoEncoder() {}
 

build_overrides/build.gni

@@ -32,6 +32,12 @@ ubsan_vptr_ignorelist_path =
 # so we just ignore that assert. See https://crbug.com/648948 for more info.
 ignore_elf32_limitations = true
 
+if (is_win || is_ios || is_android) {
+  rtc_use_h265 = true
+} else {
+  rtc_use_h265 = false
+}
+
 # Use bundled hermetic Xcode installation maintainted by Chromium,
 # except for local iOS builds where it's unsupported.
 # Allow for mac cross compile on linux machines.

call/rtp_payload_params.cc

@@ -98,6 +98,14 @@ void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info,
       rtp->simulcastIdx = spatial_index.value_or(0);
       return;
     }
+#ifndef DISABLE_H265
+    case kVideoCodecH265: {
+      auto& h265_header = rtp->video_type_header.emplace<RTPVideoHeaderH265>();
+      h265_header.packetization_mode =
+          info.codecSpecific.H265.packetization_mode;
+    }
+    return;
+#endif
     case kVideoCodecMultiplex:
     case kVideoCodecGeneric:
       rtp->codec = kVideoCodecGeneric;
@@ -341,6 +349,9 @@ void RtpPayloadParams::SetGeneric(const CodecSpecificInfo* codec_specific_info,
                       is_keyframe, rtp_video_header);
       }
       return;
+#ifndef DISABLE_H265
+    case VideoCodecType::kVideoCodecH265:
+#endif
     case VideoCodecType::kVideoCodecMultiplex:
       return;
   }

common_video/BUILD.gn

@@ -23,6 +23,8 @@ rtc_library("common_video") {
     "h264/h264_common.h",
     "h264/pps_parser.cc",
     "h264/pps_parser.h",
+    "h264/prefix_parser.cc",
+    "h264/prefix_parser.h",
     "h264/sps_parser.cc",
     "h264/sps_parser.h",
     "h264/sps_vui_rewriter.cc",
@@ -37,6 +39,21 @@ rtc_library("common_video") {
     "video_frame_buffer_pool.cc",
   ]
 
+  if (rtc_use_h265) {
+    sources += [
+      "h265/h265_bitstream_parser.cc",
+      "h265/h265_bitstream_parser.h",
+      "h265/h265_common.cc",
+      "h265/h265_common.h",
+      "h265/h265_pps_parser.cc",
+      "h265/h265_pps_parser.h",
+      "h265/h265_sps_parser.cc",
+      "h265/h265_sps_parser.h",
+      "h265/h265_vps_parser.cc",
+      "h265/h265_vps_parser.h",
+    ]
+  }
+
   deps = [
     "../api:array_view",
     "../api:make_ref_counted",

common_video/h264/prefix_parser.cc

@@ -0,0 +1,85 @@
+/*
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "common_video/h264/prefix_parser.h"
+
+#include <cstdint>
+#include <vector>
+
+#include "common_video/h264/h264_common.h"
+#include "rtc_base/bit_buffer.h"
+
+namespace {
+typedef absl::optional<webrtc::PrefixParser::PrefixState> OptionalPrefix;
+
+#define RETURN_EMPTY_ON_FAIL(x) \
+  if (!(x)) {                   \
+    return OptionalPrefix();       \
+  }
+}  // namespace
+
+namespace webrtc {
+
+PrefixParser::PrefixState::PrefixState() = default;
+PrefixParser::PrefixState::PrefixState(const PrefixState&) = default;
+PrefixParser::PrefixState::~PrefixState() = default;
+
+// General note: this is based off the 02/2016 version of the H.264 standard.
+// You can find it on this page:
+// http://www.itu.int/rec/T-REC-H.264
+
+// Unpack RBSP and parse SVC extension state from the supplied buffer.
+absl::optional<PrefixParser::PrefixState> PrefixParser::ParsePrefix(
+    const uint8_t* data,
+                                                        size_t length) {
+  std::vector<uint8_t> unpacked_buffer = H264::ParseRbsp(data, length);
+  rtc::BitBuffer bit_buffer(unpacked_buffer.data(), unpacked_buffer.size());
+  return ParsePrefixUpToSvcExtension(&bit_buffer);
+}
+
+absl::optional<PrefixParser::PrefixState> PrefixParser::ParsePrefixUpToSvcExtension(
+    rtc::BitBuffer* buffer) {
+  // Now, we need to use a bit buffer to parse through the actual SVC extension
+  // format. See Section 7.3.1 ("NAL unit syntax") and 7.3.1.1 ("NAL unit header
+  // SVC extension syntax") of the H.264 standard for a complete description.
+
+  PrefixState svc_extension;
+
+  uint32_t svc_extension_flag = 0;
+  // Make sure the svc_extension_flag is on.
+  RETURN_EMPTY_ON_FAIL(buffer->ReadBits(&svc_extension_flag, 1));
+  if (!svc_extension_flag)
+    return OptionalPrefix();
+
+  // idr_flag: u(1)
+  RETURN_EMPTY_ON_FAIL(buffer->ReadBits(&svc_extension.idr_flag, 1));
+  // priority_id: u(6)
+  RETURN_EMPTY_ON_FAIL(buffer->ReadBits(&svc_extension.priority_id, 6));
+  // no_inter_layer_pred_flag: u(1)
+  RETURN_EMPTY_ON_FAIL(
+      buffer->ReadBits(&svc_extension.no_inter_layer_pred_flag, 1));
+  // dependency_id: u(3)
+  RETURN_EMPTY_ON_FAIL(buffer->ReadBits(&svc_extension.dependency_id, 3));
+  // quality_id: u(4)
+  RETURN_EMPTY_ON_FAIL(buffer->ReadBits(&svc_extension.quality_id, 4));
+  // temporal_id: u(3)
+  RETURN_EMPTY_ON_FAIL(buffer->ReadBits(&svc_extension.temporal_id, 3));
+  // use_ref_base_pic_flag: u(1)
+  RETURN_EMPTY_ON_FAIL(
+      buffer->ReadBits(&svc_extension.use_ref_base_pic_flag, 1));
+  // discardable_flag: u(1)
+  RETURN_EMPTY_ON_FAIL(buffer->ReadBits(&svc_extension.discardable_flag, 1));
+  // output_flag: u(1)
+  RETURN_EMPTY_ON_FAIL(buffer->ReadBits(&svc_extension.output_flag, 1));
+
+  return OptionalPrefix(svc_extension);
+}
+
+}  // namespace webrtc