media/gpu/v4l2: Add HEVC stateful decode support

This adds HEVC decode support to the V4L2 stateful backend. It is only
enabled when the HEVC parser and decoder has been enabled as part of the
build flags. I.e.

enable_hevc_parser_and_hw_decoder = true

This is currently not enabled on any V4L2-based devices.

For Qualcomm devices, it is necessary to group frame NALU's together
while writing the encoded bitstream to the OUTPUT queue. The SPS and PPS
must be submitted with a complete frame, which is different than H.264
for the same devices. The HEVCInputBufferFragmentSplitter method
AdvanceFrameFragment() handles this requirement.

Bug: b:232255167
Test: video_decode_accelerator_tests on trogdor after locally enabling
Test: the HEVC decode build flag.
Change-Id: I233d30134e56e914a8ea0f9fa2bd131147228294
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4068739
Commit-Queue: Nathan Hebert <nhebert@chromium.org>
Reviewed-by: Steve Cho <stevecho@chromium.org>
Reviewed-by: Fritz Koenig <frkoenig@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1078422}

media/gpu/v4l2/v4l2_device.cc

@@ -1578,6 +1578,16 @@ uint32_t V4L2Device::VideoCodecProfileToV4L2PixFmt(VideoCodecProfile profile,
       return V4L2_PIX_FMT_H264_SLICE;
     else
       return V4L2_PIX_FMT_H264;
+#if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
+  } else if (profile == HEVCPROFILE_MAIN) {
+    if (slice_based) {
+      DVLOGF(1) << "Unsupported profile for slice based decode: "
+                << GetProfileName(profile);
+      return 0;
+    } else {
+      return V4L2_PIX_FMT_HEVC;
+    }
+#endif  // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
   } else if (profile >= VP8PROFILE_MIN && profile <= VP8PROFILE_MAX) {
     if (slice_based)
       return V4L2_PIX_FMT_VP8_FRAME;
@@ -1668,6 +1678,11 @@ std::vector<VideoCodecProfile> V4L2Device::V4L2PixFmtToVideoCodecProfiles(
       case VideoCodec::kH264:
         query_id = V4L2_CID_MPEG_VIDEO_H264_PROFILE;
         break;
+#if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
+      case VideoCodec::kHEVC:
+        query_id = V4L2_CID_MPEG_VIDEO_HEVC_PROFILE;
+        break;
+#endif  // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
       case VideoCodec::kVP8:
         query_id = V4L2_CID_MPEG_VIDEO_VP8_PROFILE;
         break;
@@ -1719,6 +1734,17 @@ std::vector<VideoCodecProfile> V4L2Device::V4L2PixFmtToVideoCodecProfiles(
         };
       }
       break;
+#if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
+    case V4L2_PIX_FMT_HEVC:
+      if (!get_supported_profiles(VideoCodec::kHEVC, &profiles)) {
+        DLOG(WARNING) << "Driver doesn't support QUERY HEVC profiles, "
+                      << "use default value, Main";
+        profiles = {
+            HEVCPROFILE_MAIN,
+        };
+      }
+      break;
+#endif  // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
     case V4L2_PIX_FMT_VP8:
     case V4L2_PIX_FMT_VP8_FRAME:
       profiles = {VP8PROFILE_ANY};

media/gpu/v4l2/v4l2_device.h

@@ -82,6 +82,11 @@
 #endif
 #endif
 
+// TODO(b/260863940): Remove this once V4L2 header is updated
+#ifndef V4L2_CID_MPEG_VIDEO_HEVC_PROFILE
+#define V4L2_CID_MPEG_VIDEO_HEVC_PROFILE (V4L2_CID_MPEG_BASE + 615)
+#endif
+
 // TODO(b/132589320): remove this once V4L2 header is updated.
 #ifndef V4L2_PIX_FMT_MM21
 // MTK 8-bit block mode, two non-contiguous planes.

media/gpu/v4l2/v4l2_vda_helpers.cc

@@ -14,6 +14,9 @@
 #include "media/gpu/v4l2/v4l2_device.h"
 #include "media/gpu/v4l2/v4l2_image_processor_backend.h"
 #include "media/video/h264_parser.h"
+#if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
+#include "media/video/h265_parser.h"
+#endif  // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
 
 namespace media {
 namespace v4l2_vda_helpers {
@@ -149,6 +152,11 @@ InputBufferFragmentSplitter::CreateFromProfile(
     case VideoCodec::kH264:
       return std::make_unique<
           v4l2_vda_helpers::H264InputBufferFragmentSplitter>();
+#if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
+    case VideoCodec::kHEVC:
+      return std::make_unique<
+          v4l2_vda_helpers::HEVCInputBufferFragmentSplitter>();
+#endif  // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
     case VideoCodec::kVP8:
     case VideoCodec::kVP9:
       // VP8/VP9 don't need any frame splitting, use the default implementation.
@@ -272,5 +280,142 @@ bool H264InputBufferFragmentSplitter::IsPartialFramePending() const {
   return partial_frame_pending_;
 }
 
+#if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
+HEVCInputBufferFragmentSplitter::HEVCInputBufferFragmentSplitter()
+    : h265_parser_(new H265Parser()) {}
+
+HEVCInputBufferFragmentSplitter::~HEVCInputBufferFragmentSplitter() = default;
+
+bool HEVCInputBufferFragmentSplitter::AdvanceFrameFragment(const uint8_t* data,
+                                                           size_t size,
+                                                           size_t* endpos) {
+  DCHECK(h265_parser_);
+
+  // For HEVC, we need to feed HW one frame at a time. This parses the bitstream
+  // to determine frame boundaries.
+  h265_parser_->SetStream(data, size);
+  H265NALU nalu;
+  H265Parser::Result result;
+  *endpos = 0;
+
+  // Keep on peeking the next NALs while they don't indicate a frame
+  // boundary.
+  while (true) {
+    bool end_of_frame = false;
+    result = h265_parser_->AdvanceToNextNALU(&nalu);
+    if (result == H265Parser::kInvalidStream ||
+        result == H265Parser::kUnsupportedStream) {
+      return false;
+    }
+    if (result == H265Parser::kEOStream) {
+      // We've reached the end of the buffer before finding a frame boundary.
+      if (*endpos != 0)
+        partial_frame_pending_ = true;
+      *endpos = size;
+      return true;
+    }
+    switch (nalu.nal_unit_type) {
+      case H265NALU::BLA_W_LP:
+      case H265NALU::BLA_W_RADL:
+      case H265NALU::BLA_N_LP:
+      case H265NALU::IDR_W_RADL:
+      case H265NALU::IDR_N_LP:
+      case H265NALU::TRAIL_N:
+      case H265NALU::TRAIL_R:
+      case H265NALU::TSA_N:
+      case H265NALU::TSA_R:
+      case H265NALU::STSA_N:
+      case H265NALU::STSA_R:
+      case H265NALU::RADL_N:
+      case H265NALU::RADL_R:
+      case H265NALU::RASL_N:
+      case H265NALU::RASL_R:
+      case H265NALU::CRA_NUT: {
+        // Rec. ITU-T H.265, section 7.3.1.2 NAL unit header syntax
+        constexpr uint8_t kHevcNaluHeaderSize = 2;
+
+        // These NALU's have a slice header which starts after the NAL unit
+        // header. This ensures that there is enough data in the NALU to contain
+        // contain at least one byte of the slice header.
+        if (nalu.size <= kHevcNaluHeaderSize)
+          return false;
+
+        // From Rec. ITU-T H.265, section 7.3.6 Slice segment header syntax, the
+        // first bit in the slice header is first_slice_segment_in_pic_flag. If
+        // it is set, then the slice starts a new frame.
+        if ((nalu.data[kHevcNaluHeaderSize] & 0x80) == 0x80) {
+          if (slice_data_pending_) {
+            end_of_frame = true;
+            break;
+          }
+        }
+        slice_data_pending_ = true;
+        break;
+      }
+      case H265NALU::SPS_NUT:
+      case H265NALU::PPS_NUT:
+        // Only creates a new frame if there is already slice data. This groups
+        // the SPS and PPS with the first frame. This is needed for the Venus
+        // driver for HEVC. Curiously, the same treatment is not needed for
+        // H.264.
+        // TODO(b/227247905): If a different policy is needed for the Stateless
+        // backend, then make the behavior externally configurable.
+        if (slice_data_pending_) {
+          end_of_frame = true;
+        }
+        break;
+      case H265NALU::EOS_NUT:
+      case H265NALU::EOB_NUT:
+      case H265NALU::AUD_NUT:
+      case H265NALU::RSV_NVCL41:
+      case H265NALU::RSV_NVCL42:
+      case H265NALU::RSV_NVCL43:
+      case H265NALU::RSV_NVCL44:
+      case H265NALU::UNSPEC48:
+      case H265NALU::UNSPEC49:
+      case H265NALU::UNSPEC50:
+      case H265NALU::UNSPEC51:
+      case H265NALU::UNSPEC52:
+      case H265NALU::UNSPEC53:
+      case H265NALU::UNSPEC54:
+      case H265NALU::UNSPEC55:
+        // These unconditionally signal a frame boundary.
+        end_of_frame = true;
+        break;
+      default:
+        // For all others, keep going.
+        break;
+    }
+    if (end_of_frame) {
+      if (!partial_frame_pending_ && *endpos == 0) {
+        // The frame was previously restarted, and we haven't filled the
+        // current frame with any contents yet. Start the new frame here and
+        // continue parsing NALs.
+      } else {
+        // The frame wasn't previously restarted and/or we have contents for
+        // the current frame; signal the start of a new frame here: we don't
+        // have a partial frame anymore.
+        partial_frame_pending_ = false;
+        slice_data_pending_ = false;
+        return true;
+      }
+    }
+    *endpos = (nalu.data + nalu.size) - data;
+  }
+  NOTREACHED();
+  return false;
+}
+
+void HEVCInputBufferFragmentSplitter::Reset() {
+  partial_frame_pending_ = false;
+  slice_data_pending_ = false;
+  h265_parser_.reset(new H265Parser());
+}
+
+bool HEVCInputBufferFragmentSplitter::IsPartialFramePending() const {
+  return partial_frame_pending_;
+}
+#endif  // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
+
 }  // namespace v4l2_vda_helpers
 }  // namespace media

media/gpu/v4l2/v4l2_vda_helpers.h

@@ -18,6 +18,9 @@ namespace media {
 
 class V4L2Device;
 class H264Parser;
+#if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
+class H265Parser;
+#endif  // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
 
 // Helper static methods to be shared between V4L2VideoDecodeAccelerator and
 // V4L2SliceVideoDecodeAccelerator. This avoids some code duplication between
@@ -115,6 +118,31 @@ class H264InputBufferFragmentSplitter : public InputBufferFragmentSplitter {
   bool partial_frame_pending_ = false;
 };
 
+#if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
+// Splitter for HEVC, making sure to properly report when a partial frame
+// may be pending.
+class HEVCInputBufferFragmentSplitter : public InputBufferFragmentSplitter {
+ public:
+  explicit HEVCInputBufferFragmentSplitter();
+  ~HEVCInputBufferFragmentSplitter() override;
+
+  bool AdvanceFrameFragment(const uint8_t* data,
+                            size_t size,
+                            size_t* endpos) override;
+  void Reset() override;
+  bool IsPartialFramePending() const override;
+
+ private:
+  // For HEVC decode, hardware requires that we send it frame-sized chunks.
+  // We'll need to parse the stream.
+  std::unique_ptr<H265Parser> h265_parser_;
+  // Set if we have a pending incomplete frame in the input buffer.
+  bool partial_frame_pending_ = false;
+  // Set if we have pending slice data in the input buffer.
+  bool slice_data_pending_ = false;
+};
+#endif  // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
+
 }  // namespace v4l2_vda_helpers
 }  // namespace media
 

media/gpu/v4l2/v4l2_video_decoder.cc

@@ -52,6 +52,9 @@ constexpr uint32_t kSupportedInputFourccs[] = {
     V4L2_PIX_FMT_AV1_FRAME,
     // V4L2 stateful formats
     V4L2_PIX_FMT_H264,
+#if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
+    V4L2_PIX_FMT_HEVC,
+#endif  // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
     V4L2_PIX_FMT_VP8,
     V4L2_PIX_FMT_VP9,
     V4L2_PIX_FMT_AV1,

media/gpu/v4l2/v4l2_video_decoder_backend_stateful.cc

@@ -231,7 +231,7 @@ void V4L2StatefulVideoDecoderBackend::DoDecodeWork() {
   size_t bytes_to_copy = 0;
 
   if (!frame_splitter_->AdvanceFrameFragment(data, data_size, &bytes_to_copy)) {
-    VLOGF(1) << "Invalid H.264 stream detected.";
+    VLOGF(1) << "Invalid bitstream detected.";
     std::move(current_decode_request_->decode_cb)
         .Run(DecoderStatus::Codes::kFailed);
     current_decode_request_.reset();
@@ -741,9 +741,12 @@ bool V4L2StatefulVideoDecoderBackend::IsSupportedProfile(
   DCHECK(device_);
   if (supported_profiles_.empty()) {
     constexpr uint32_t kSupportedInputFourccs[] = {
-        V4L2_PIX_FMT_H264,
-        V4L2_PIX_FMT_VP8,
-        V4L2_PIX_FMT_VP9,
+      V4L2_PIX_FMT_H264,
+#if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
+      V4L2_PIX_FMT_HEVC,
+#endif  // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
+      V4L2_PIX_FMT_VP8,
+      V4L2_PIX_FMT_VP9,
     };
     scoped_refptr<V4L2Device> device = V4L2Device::Create();
     VideoDecodeAccelerator::SupportedProfiles profiles =