Update M83 code to work with owt-sdk (open-webrtc-toolkit#68)

* Add HEVC support for iOS/Android * Some changes for building with OWT * Enable openssl * Add create_peerconnection_factory to WebRTC.framework. (open-webrtc-toolkit#46) * Set kVTCompressionPropertyKey_RealTime to true. (open-webrtc-toolkit#51) * H265 packetization_mode setting fix (open-webrtc-toolkit#53) * add H.265 QP parsing logic (open-webrtc-toolkit#47) * Fix linux build error. (open-webrtc-toolkit#54) * Add h264 prefix NAL parser implmentation for enabling frame-marking for h.264 (open-webrtc-toolkit#58) * Make hevc rtp depacketizer/tracker conforming to h.264 design Co-authored-by: jianjunz <jianjun.zhu@intel.com> Co-authored-by: Cyril Lashkevich <notorca@gmail.com> Co-authored-by: Piasy <xz4215@gmail.com> Co-authored-by: ShiJinCheng <874042641@qq.com>
jianjunz · Dec 2, 2022 · 5a23199 · 5a23199
1 parent 38b54ec
commit 5a23199
Show file tree

Hide file tree

Showing 24 changed files with 1,400 additions and 15 deletions.
diff --git a/api/video/builtin_video_bitrate_allocator_factory.cc b/api/video/builtin_video_bitrate_allocator_factory.cc
@@ -34,6 +34,7 @@ class BuiltinVideoBitrateAllocatorFactory
       case kVideoCodecAV1:
       case kVideoCodecVP9:
         return std::make_unique<SvcRateAllocator>(codec);
+      // TODO: add an allocator here for H.265
       default:
         return std::make_unique<SimulcastRateAllocator>(codec);
     }

diff --git a/api/video_codecs/video_codec.cc b/api/video_codecs/video_codec.cc
@@ -26,9 +26,7 @@ constexpr char kPayloadNameAv1[] = "AV1";
 // needed.
 constexpr char kPayloadNameAv1x[] = "AV1X";
 constexpr char kPayloadNameH264[] = "H264";
-#ifdef WEBRTC_USE_H265
 constexpr char kPayloadNameH265[] = "H265";
-#endif
 constexpr char kPayloadNameGeneric[] = "Generic";
 constexpr char kPayloadNameMultiplex[] = "Multiplex";
 }  // namespace
@@ -55,6 +53,17 @@ bool VideoCodecH264::operator==(const VideoCodecH264& other) const {
           numberOfTemporalLayers == other.numberOfTemporalLayers);
 }
 
+#ifndef DISABLE_H265
+bool VideoCodecH265::operator==(const VideoCodecH265& other) const {
+  return (frameDroppingOn == other.frameDroppingOn &&
+          keyFrameInterval == other.keyFrameInterval &&
+          vpsLen == other.vpsLen && spsLen == other.spsLen &&
+          ppsLen == other.ppsLen &&
+          (spsLen == 0 || memcmp(spsData, other.spsData, spsLen) == 0) &&
+          (ppsLen == 0 || memcmp(ppsData, other.ppsData, ppsLen) == 0));
+}
+#endif
+
 #ifdef WEBRTC_USE_H265
 bool VideoCodecH265::operator==(const VideoCodecH265& other) const {
   return (frameDroppingOn == other.frameDroppingOn &&
@@ -116,7 +125,6 @@ const VideoCodecH264& VideoCodec::H264() const {
   return codec_specific_.H264;
 }
 
-#ifdef WEBRTC_USE_H265
 VideoCodecH265* VideoCodec::H265() {
   RTC_DCHECK_EQ(codecType, kVideoCodecH265);
   return &codec_specific_.H265;
@@ -138,7 +146,6 @@ const char* CodecTypeToPayloadString(VideoCodecType type) {
       return kPayloadNameAv1;
     case kVideoCodecH264:
       return kPayloadNameH264;
-#ifdef WEBRTC_USE_H265
     case kVideoCodecH265:
       return kPayloadNameH265;
 #endif
@@ -163,7 +170,6 @@ VideoCodecType PayloadStringToCodecType(const std::string& name) {
     return kVideoCodecH264;
   if (absl::EqualsIgnoreCase(name, kPayloadNameMultiplex))
     return kVideoCodecMultiplex;
-#ifdef WEBRTC_USE_H265
   if (absl::EqualsIgnoreCase(name, kPayloadNameH265))
     return kVideoCodecH265;
 #endif

diff --git a/api/video_codecs/video_codec.h b/api/video_codecs/video_codec.h
@@ -112,6 +112,23 @@ struct VideoCodecH265 {
   size_t ppsLen;
 };
 
+#ifndef DISABLE_H265
+struct VideoCodecH265 {
+  bool operator==(const VideoCodecH265& other) const;
+  bool operator!=(const VideoCodecH265& other) const {
+    return !(*this == other);
+  }
+  bool frameDroppingOn;
+  int keyFrameInterval;
+  const uint8_t* vpsData;
+  size_t vpsLen;
+  const uint8_t* spsData;
+  size_t spsLen;
+  const uint8_t* ppsData;
+  size_t ppsLen;
+};
+#endif
+
 // Translates from name of codec to codec type and vice versa.
 RTC_EXPORT const char* CodecTypeToPayloadString(VideoCodecType type);
 RTC_EXPORT VideoCodecType PayloadStringToCodecType(const std::string& name);
@@ -120,7 +137,9 @@ union VideoCodecUnion {
   VideoCodecVP8 VP8;
   VideoCodecVP9 VP9;
   VideoCodecH264 H264;
+#ifndef DISABLE_H265
   VideoCodecH265 H265;
+#endif
 };
 
 enum class VideoCodecMode { kRealtimeVideo, kScreensharing };

diff --git a/build_overrides/build.gni b/build_overrides/build.gni
@@ -32,6 +32,12 @@ ubsan_vptr_ignorelist_path =
 # so we just ignore that assert. See https://crbug.com/648948 for more info.
 ignore_elf32_limitations = true
 
+if (is_win || is_ios || is_android) {
+  rtc_use_h265 = true
+} else {
+  rtc_use_h265 = false
+}
+
 # Use bundled hermetic Xcode installation maintainted by Chromium,
 # except for local iOS builds where it's unsupported.
 # Allow for mac cross compile on linux machines.

diff --git a/call/rtp_payload_params.cc b/call/rtp_payload_params.cc
@@ -349,7 +349,9 @@ void RtpPayloadParams::SetGeneric(const CodecSpecificInfo* codec_specific_info,
                       is_keyframe, rtp_video_header);
       }
       return;
+#ifndef DISABLE_H265
     case VideoCodecType::kVideoCodecH265:
+#endif
     case VideoCodecType::kVideoCodecMultiplex:
       return;
   }

diff --git a/media/base/media_constants.cc b/media/base/media_constants.cc
@@ -116,7 +116,6 @@ const char kH264FmtpSpropParameterSets[] = "sprop-parameter-sets";
 const char kH264FmtpSpsPpsIdrInKeyframe[] = "sps-pps-idr-in-keyframe";
 const char kH264ProfileLevelConstrainedBaseline[] = "42e01f";
 const char kH264ProfileLevelConstrainedHigh[] = "640c1f";
-#ifdef WEBRTC_USE_H265
 // RFC 7798 RTP Payload Format for H.265 video
 const char kH265FmtpProfileSpace[] = "profile-space";
 const char kH265FmtpProfileId[] = "profile-id";

diff --git a/modules/rtp_rtcp/BUILD.gn b/modules/rtp_rtcp/BUILD.gn
@@ -255,6 +255,19 @@ rtc_library("rtp_rtcp") {
     defines = [ "BWE_TEST_LOGGING_COMPILE_TIME_ENABLE=0" ]
   }
 
+  if (rtc_use_h265) {
+    sources += [
+      "source/rtp_format_h265.cc",
+      "source/rtp_format_h265.h",
+      "source/video_rtp_depacketizer_h265.cc",
+      "source/video_rtp_depacketizer_h265.h",
+    ]
+  }
+
+  if (!rtc_use_h265) {
+    defines += ["DISABLE_H265"]
+  }
+
   deps = [
     ":rtp_rtcp_format",
     ":rtp_video_header",

diff --git a/modules/rtp_rtcp/source/create_video_rtp_depacketizer.cc b/modules/rtp_rtcp/source/create_video_rtp_depacketizer.cc
@@ -19,8 +19,9 @@
 #include "modules/rtp_rtcp/source/video_rtp_depacketizer_h264.h"
 #include "modules/rtp_rtcp/source/video_rtp_depacketizer_vp8.h"
 #include "modules/rtp_rtcp/source/video_rtp_depacketizer_vp9.h"
+#ifndef DISABLE_H265
 #include "modules/rtp_rtcp/source/video_rtp_depacketizer_h265.h"
-
+#endif
 namespace webrtc {
 
 std::unique_ptr<VideoRtpDepacketizer> CreateVideoRtpDepacketizer(
@@ -32,8 +33,10 @@ std::unique_ptr<VideoRtpDepacketizer> CreateVideoRtpDepacketizer(
       return std::make_unique<VideoRtpDepacketizerVp8>();
     case kVideoCodecVP9:
       return std::make_unique<VideoRtpDepacketizerVp9>();
+#ifndef DISABLE_H265
     case kVideoCodecH265:
       return std::make_unique<VideoRtpDepacketizerH265>();
+#endif
     case kVideoCodecAV1:
       return std::make_unique<VideoRtpDepacketizerAv1>();
     case kVideoCodecGeneric:

diff --git a/modules/rtp_rtcp/source/rtp_format.cc b/modules/rtp_rtcp/source/rtp_format.cc
@@ -14,7 +14,6 @@
 
 #include "absl/types/variant.h"
 #include "modules/rtp_rtcp/source/rtp_format_h264.h"
-#ifdef WEBRTC_USE_H265
 #include "modules/rtp_rtcp/source/rtp_format_h265.h"
 #endif
 #include "modules/rtp_rtcp/source/rtp_format_video_generic.h"

diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc
@@ -815,7 +815,9 @@ uint8_t RTPSenderVideo::GetTemporalId(const RTPVideoHeader& header) {
     uint8_t operator()(const RTPVideoHeaderLegacyGeneric&) {
       return kNoTemporalIdx;
     }
+#ifndef DISABLE_H265
     uint8_t operator()(const RTPVideoHeaderH265&) { return kNoTemporalIdx; }
+#endif
     uint8_t operator()(const absl::monostate&) { return kNoTemporalIdx; }
   };
   return absl::visit(TemporalIdGetter(), header.video_type_header);

diff --git a/modules/rtp_rtcp/source/rtp_video_header.h b/modules/rtp_rtcp/source/rtp_video_header.h
@@ -37,12 +37,20 @@ struct RTPVideoHeaderLegacyGeneric {
   uint16_t picture_id;
 };
 
+#ifndef DISABLE_H265
 using RTPVideoTypeHeader = absl::variant<absl::monostate,
                                          RTPVideoHeaderVP8,
                                          RTPVideoHeaderVP9,
                                          RTPVideoHeaderH264,
                                          RTPVideoHeaderH265,
-                                         RTPVideoHeaderLegacyGeneric>;
+										 RTPVideoHeaderLegacyGeneric>;
+#else
+using RTPVideoTypeHeader = absl::variant<absl::monostate,
+                                         RTPVideoHeaderVP8,
+                                         RTPVideoHeaderVP9,
+                                         RTPVideoHeaderH264,
+										 RTPVideoHeaderLegacyGeneric>;
+#endif
 
 struct RTPVideoHeader {
   struct GenericDescriptorInfo {

diff --git a/modules/video_coding/encoded_frame.cc b/modules/video_coding/encoded_frame.cc
@@ -145,6 +145,11 @@ void VCMEncodedFrame::CopyCodecSpecific(const RTPVideoHeader* header) {
         _codecSpecificInfo.codecType = kVideoCodecH265;
         break;
       }
+#ifndef DISABLE_H265
+      case kVideoCodecH265: {
+        _codecSpecificInfo.codecType = kVideoCodecH265;
+        break;
+      }
 #endif
       default: {
         _codecSpecificInfo.codecType = kVideoCodecGeneric;

diff --git a/modules/video_coding/include/video_codec_interface.h b/modules/video_coding/include/video_codec_interface.h
@@ -20,7 +20,6 @@
 #include "api/video_codecs/video_encoder.h"
 #include "common_video/generic_frame_descriptor/generic_frame_info.h"
 #include "modules/video_coding/codecs/h264/include/h264_globals.h"
-#ifdef WEBRTC_USE_H265
 #include "modules/video_coding/codecs/h265/include/h265_globals.h"
 #endif
 #include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
@@ -94,7 +93,6 @@ struct CodecSpecificInfoH264 {
   bool idr_frame;
 };
 
-#ifdef WEBRTC_USE_H265
 struct CodecSpecificInfoH265 {
   H265PacketizationMode packetization_mode;
   bool idr_frame;

diff --git a/modules/video_coding/packet_buffer.cc b/modules/video_coding/packet_buffer.cc
@@ -267,6 +267,7 @@ std::vector<std::unique_ptr<PacketBuffer::Packet>> PacketBuffer::FindFrames(
       bool has_h264_pps = false;
       bool has_h264_idr = false;
       bool is_h264_keyframe = false;
+
       bool is_h265 = false;
 #ifdef WEBRTC_USE_H265
       is_h265 = buffer_[start_index]->codec() == kVideoCodecH265;
@@ -442,7 +443,6 @@ std::vector<std::unique_ptr<PacketBuffer::Packet>> PacketBuffer::FindFrames(
         }
       }
 #endif
-      if (is_h264 || full_frame_found) {
         const uint16_t end_seq_num = seq_num + 1;
         // Use uint16_t type to handle sequence number wrap around case.
         uint16_t num_packets = end_seq_num - start_seq_num;

diff --git a/rtc_base/experiments/min_video_bitrate_experiment.cc b/rtc_base/experiments/min_video_bitrate_experiment.cc
@@ -100,7 +100,9 @@ absl::optional<DataRate> GetExperimentalMinVideoBitrate(VideoCodecType type) {
         return min_bitrate_av1.GetOptional();
       case kVideoCodecH264:
         return min_bitrate_h264.GetOptional();
+#ifndef DISABLE_H265
       case kVideoCodecH265:
+#endif
       case kVideoCodecGeneric:
       case kVideoCodecMultiplex:
         return absl::nullopt;

diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn
@@ -1312,6 +1312,11 @@ if (is_ios || is_mac) {
           "objc/components/video_codec/RTCVideoDecoderH264.h",
           "objc/components/video_codec/RTCVideoEncoderFactoryH264.h",
           "objc/components/video_codec/RTCVideoEncoderH264.h",
+          "objc/components/video_codec/RTCH265ProfileLevelId.h",
+          "objc/components/video_codec/RTCVideoDecoderFactoryH265.h",
+          "objc/components/video_codec/RTCVideoDecoderH265.h",
+          "objc/components/video_codec/RTCVideoEncoderFactoryH265.h",
+          "objc/components/video_codec/RTCVideoEncoderH265.h",
           "objc/components/video_frame_buffer/RTCCVPixelBuffer.h",
           "objc/helpers/RTCCameraPreviewView.h",
           "objc/helpers/RTCDispatcher.h",
@@ -1401,6 +1406,7 @@ if (is_ios || is_mac) {
           ":videocodec_objc",
           ":videotoolbox_objc",
           "../api:create_peerconnection_factory",
+          "../api:create_peerconnection_factory",
         ]
         if (rtc_ios_macos_use_opengl_rendering) {
           deps += [ ":opengl_ui_objc" ]
@@ -1516,6 +1522,11 @@ if (is_ios || is_mac) {
           "objc/components/video_codec/RTCVideoDecoderH264.h",
           "objc/components/video_codec/RTCVideoEncoderFactoryH264.h",
           "objc/components/video_codec/RTCVideoEncoderH264.h",
+          "objc/components/video_codec/RTCH265ProfileLevelId.h",
+          "objc/components/video_codec/RTCVideoDecoderFactoryH265.h",
+          "objc/components/video_codec/RTCVideoDecoderH265.h",
+          "objc/components/video_codec/RTCVideoEncoderFactoryH265.h",
+          "objc/components/video_codec/RTCVideoEncoderH265.h",
           "objc/components/video_frame_buffer/RTCCVPixelBuffer.h",
           "objc/helpers/RTCDispatcher.h",
         ]
@@ -1745,6 +1756,14 @@ if (is_ios || is_mac) {
         "objc/components/video_codec/RTCVideoEncoderFactoryH264.m",
         "objc/components/video_codec/RTCVideoEncoderH264.h",
         "objc/components/video_codec/RTCVideoEncoderH264.mm",
+        "objc/components/video_codec/RTCVideoDecoderFactoryH265.h",
+        "objc/components/video_codec/RTCVideoDecoderFactoryH265.m",
+        "objc/components/video_codec/RTCVideoDecoderH265.h",
+        "objc/components/video_codec/RTCVideoDecoderH265.mm",
+        "objc/components/video_codec/RTCVideoEncoderFactoryH265.h",
+        "objc/components/video_codec/RTCVideoEncoderFactoryH265.m",
+        "objc/components/video_codec/RTCVideoEncoderH265.h",
+        "objc/components/video_codec/RTCVideoEncoderH265.mm",
       ]
 
       if (rtc_use_h265) {

diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java
@@ -137,7 +137,7 @@ public VideoEncoder createEncoder(VideoCodecInfo input) {
   public VideoCodecInfo[] getSupportedCodecs() {
     List<VideoCodecInfo> supportedCodecInfos = new ArrayList<VideoCodecInfo>();
     // Generate a list of supported codecs in order of preference:
-    // VP8, VP9, H264 (high profile), and H264 (baseline profile).
+    // VP8, VP9, H.265(optional), H264 (high profile), and H264 (baseline profile).
     for (VideoCodecMimeType type : new VideoCodecMimeType[] {VideoCodecMimeType.VP8,
              VideoCodecMimeType.VP8, VideoCodecMimeType.VP9, VideoCodecMimeType.H264,
 			     VideoCodecMimeType.H265}) {
@@ -257,6 +257,15 @@ private boolean isHardwareSupportedInCurrentSdkH265(MediaCodecInfo info) {
            || vcp.isExtraHardwareSupported(name, "video/hevc", vcp.parseWithTag(vcp.loadWithDom(extraMediaCodecFile), "Decoders"));
   }
 
+  private boolean isHardwareSupportedInCurrentSdkH265(MediaCodecInfo info) {
+    String name = info.getName();
+    // QCOM H265 encoder is supported in KITKAT or later.
+    return (name.startsWith(QCOM_PREFIX) && Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT)
+           // Exynos H265 encoder is supported in LOLLIPOP or later.
+           || (name.startsWith(EXYNOS_PREFIX)
+               && Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP);
+  }
+
   private boolean isMediaCodecAllowed(MediaCodecInfo info) {
     if (codecAllowedPredicate == null) {
       return true;