RTCVDAdapter: Don't copy EncodedImage in converting DecoderBuffer

The current webrtc video decoder copies a bitstream buffer two times;
(1) webrtc::EncodedImage => media::DecoderBuffer in
RTCVideoDecoderAdapter::Decode(), and
(2) media::DecoderBuffer => mojo pipe in MojoVideoDecoder::Decode().

This CL removes the first copy. chrome://tracing [1] shows this saved
4us out of 21us in 3Mbps decoding,
ConvertToDecoderBuffer: 7us -> 3us
FallbackWrapper::Decode():  21us -> 16us

[1] https://imgur.com/a/cgpDW6s

Bug: b:241349739
Test: webrtc playground

Change-Id: I2c4640e354d473bb6fa77aadca3fa327c88fe271
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3999071
Reviewed-by: Ilya Nikolaevskiy <ilnik@chromium.org>
Commit-Queue: Hirokazu Honda <hiroh@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1070567}

media/base/decoder_buffer.cc

@@ -56,6 +56,10 @@ DecoderBuffer::DecoderBuffer(base::WritableSharedMemoryMapping mapping,
                              size_t size)
     : size_(size), writable_mapping_(std::move(mapping)) {}
 
+DecoderBuffer::DecoderBuffer(std::unique_ptr<ExternalMemory> external_memory)
+    : size_(external_memory->span().size()),
+      external_memory_(std::move(external_memory)) {}
+
 DecoderBuffer::~DecoderBuffer() {
   data_.reset();
   side_data_.reset();
@@ -72,7 +76,7 @@ scoped_refptr<DecoderBuffer> DecoderBuffer::CopyFrom(const uint8_t* data,
                                                      size_t data_size) {
   // If you hit this CHECK you likely have a bug in a demuxer. Go fix it.
   CHECK(data);
-  return base::WrapRefCounted(new DecoderBuffer(data, data_size, NULL, 0));
+  return base::WrapRefCounted(new DecoderBuffer(data, data_size, nullptr, 0));
 }
 
 // static
@@ -126,9 +130,18 @@ scoped_refptr<DecoderBuffer> DecoderBuffer::FromSharedMemoryRegion(
   return base::WrapRefCounted(new DecoderBuffer(std::move(mapping), size));
 }
 
+// static
+scoped_refptr<DecoderBuffer> DecoderBuffer::FromExternalMemory(
+    std::unique_ptr<ExternalMemory> external_memory) {
+  DCHECK(external_memory);
+  if (external_memory->span().empty())
+    return nullptr;
+  return base::WrapRefCounted(new DecoderBuffer(std::move(external_memory)));
+}
+
 // static
 scoped_refptr<DecoderBuffer> DecoderBuffer::CreateEOSBuffer() {
-  return base::WrapRefCounted(new DecoderBuffer(NULL, 0, NULL, 0));
+  return base::WrapRefCounted(new DecoderBuffer(nullptr, 0, nullptr, 0));
 }
 
 bool DecoderBuffer::MatchesMetadataForTesting(

media/base/decoder_buffer.h

@@ -13,6 +13,7 @@
 #include <utility>
 
 #include "base/check.h"
+#include "base/containers/span.h"
 #include "base/memory/read_only_shared_memory_region.h"
 #include "base/memory/ref_counted.h"
 #include "base/memory/shared_memory_mapping.h"
@@ -33,13 +34,20 @@ namespace media {
 class MEDIA_EXPORT DecoderBuffer
     : public base::RefCountedThreadSafe<DecoderBuffer> {
  public:
-  enum {
-    kPaddingSize = 64,
-#if defined(ARCH_CPU_ARM_FAMILY)
-    kAlignmentSize = 16
-#else
-    kAlignmentSize = 32
-#endif
+  // ExternalMemory wraps a class owning a buffer and expose the data interface
+  // through |span|. This class is derived by a class that owns the class owning
+  // the buffer owner class. It is generally better to add the buffer class to
+  // DecoderBuffer. ExternalMemory is for a class that cannot be added; for
+  // instance, rtc::scoped_refptr<webrtc::EncodedImageBufferInterface>, webrtc
+  // class cannot be included in //media/base.
+  struct MEDIA_EXPORT ExternalMemory {
+   public:
+    explicit ExternalMemory(base::span<const uint8_t> span) : span_(span) {}
+    virtual ~ExternalMemory() = default;
+    const base::span<const uint8_t>& span() const { return span_; }
+
+   private:
+    const base::span<const uint8_t> span_;
   };
 
   using DiscardPadding = std::pair<base::TimeDelta, base::TimeDelta>;
@@ -112,6 +120,12 @@ class MEDIA_EXPORT DecoderBuffer
       uint64_t offset,
       size_t size);
 
+  // Creates a DecoderBuffer with ExternalMemory. The buffer accessed through
+  // the created DecoderBuffer is |span| of |external_memory||.
+  // |external_memory| is owned by DecoderBuffer until it is destroyed.
+  static scoped_refptr<DecoderBuffer> FromExternalMemory(
+      std::unique_ptr<ExternalMemory> external_memory);
+
   // Create a DecoderBuffer indicating we've reached end of stream.
   //
   // Calling any method other than end_of_stream() on the resulting buffer
@@ -151,6 +165,8 @@ class MEDIA_EXPORT DecoderBuffer
       return read_only_mapping_.GetMemoryAs<const uint8_t>();
     if (writable_mapping_.IsValid())
       return writable_mapping_.GetMemoryAs<const uint8_t>();
+    if (external_memory_)
+      return external_memory_->span().data();
     return data_.get();
   }
 
@@ -159,6 +175,7 @@ class MEDIA_EXPORT DecoderBuffer
     DCHECK(!end_of_stream());
     DCHECK(!read_only_mapping_.IsValid());
     DCHECK(!writable_mapping_.IsValid());
+    DCHECK(!external_memory_);
     return data_.get();
   }
 
@@ -202,7 +219,7 @@ class MEDIA_EXPORT DecoderBuffer
   // If there's no data in this buffer, it represents end of stream.
   bool end_of_stream() const {
     return !read_only_mapping_.IsValid() && !writable_mapping_.IsValid() &&
-           !data_;
+           !external_memory_ && !data_;
   }
 
   bool is_key_frame() const {
@@ -245,6 +262,8 @@ class MEDIA_EXPORT DecoderBuffer
 
   DecoderBuffer(base::WritableSharedMemoryMapping mapping, size_t size);
 
+  explicit DecoderBuffer(std::unique_ptr<ExternalMemory> external_memory);
+
   virtual ~DecoderBuffer();
 
   // Encoded data, if it is stored on the heap.
@@ -266,6 +285,8 @@ class MEDIA_EXPORT DecoderBuffer
   // Encoded data, if it is stored in a writable shared memory mapping.
   base::WritableSharedMemoryMapping writable_mapping_;
 
+  std::unique_ptr<ExternalMemory> external_memory_;
+
   // Encryption parameters for the encoded data.
   std::unique_ptr<DecryptConfig> decrypt_config_;
 

media/base/decoder_buffer_unittest.cc

@@ -179,6 +179,19 @@ TEST(DecoderBufferTest, FromSharedMemoryRegion_ZeroSize) {
   ASSERT_FALSE(buffer.get());
 }
 
+TEST(DecoderBufferTest, FromExternalMemory) {
+  constexpr uint8_t kData[] = "hello";
+  constexpr size_t kDataSize = std::size(kData);
+  auto external_memory = std::make_unique<DecoderBuffer::ExternalMemory>(
+      base::make_span(kData, kDataSize));
+  auto buffer = DecoderBuffer::FromExternalMemory(std::move(external_memory));
+  ASSERT_TRUE(buffer.get());
+  EXPECT_EQ(buffer->data_size(), kDataSize);
+  EXPECT_EQ(0, memcmp(buffer->data(), kData, kDataSize));
+  EXPECT_FALSE(buffer->end_of_stream());
+  EXPECT_FALSE(buffer->is_key_frame());
+}
+
 TEST(DecoderBufferTest, ReadingWriting) {
   const char kData[] = "hello";
   const size_t kDataSize = std::size(kData);

media/ffmpeg/ffmpeg_common.cc

@@ -47,12 +47,6 @@ VideoDecoderConfig::AlphaMode GetAlphaMode(const AVStream* stream) {
 
 }  // namespace
 
-// Why AV_INPUT_BUFFER_PADDING_SIZE? FFmpeg assumes all input buffers are
-// padded. Check here to ensure FFmpeg only receives data padded to its
-// specifications.
-static_assert(DecoderBuffer::kPaddingSize >= AV_INPUT_BUFFER_PADDING_SIZE,
-              "DecoderBuffer padding size does not fit ffmpeg requirement");
-
 // Alignment requirement by FFmpeg for input and output buffers. This need to
 // be updated to match FFmpeg when it changes.
 #if defined(ARCH_CPU_ARM_FAMILY)
@@ -61,12 +55,6 @@ static const int kFFmpegBufferAddressAlignment = 16;
 static const int kFFmpegBufferAddressAlignment = 32;
 #endif
 
-// Check here to ensure FFmpeg only receives data aligned to its specifications.
-static_assert(
-    DecoderBuffer::kAlignmentSize >= kFFmpegBufferAddressAlignment &&
-    DecoderBuffer::kAlignmentSize % kFFmpegBufferAddressAlignment == 0,
-    "DecoderBuffer alignment size does not fit ffmpeg requirement");
-
 // Allows faster SIMD YUV convert. Also, FFmpeg overreads/-writes occasionally.
 // See video_get_buffer() in libavcodec/utils.c.
 static const int kFFmpegOutputBufferPaddingSize = 16;

third_party/blink/renderer/platform/peerconnection/rtc_video_decoder_adapter.cc

@@ -110,10 +110,39 @@ bool HasSoftwareFallback(media::VideoCodec video_codec) {
 #endif
 }
 
+struct EncodedImageExternalMemory
+    : public media::DecoderBuffer::ExternalMemory {
+ public:
+  explicit EncodedImageExternalMemory(
+      rtc::scoped_refptr<webrtc::EncodedImageBufferInterface> buffer_interface)
+      : ExternalMemory(base::make_span(buffer_interface->data(),
+                                       buffer_interface->size())),
+        buffer_interface_(std::move(buffer_interface)) {}
+  ~EncodedImageExternalMemory() override = default;
+
+ private:
+  rtc::scoped_refptr<webrtc::EncodedImageBufferInterface> buffer_interface_;
+};
+
 scoped_refptr<media::DecoderBuffer> ConvertToDecoderBuffer(
     const webrtc::EncodedImage& input_image) {
-  std::vector<uint32_t> spatial_layer_frame_size;
+  TRACE_EVENT0("webrtc", "RTCVideoDecoderAdapter::ConvertToDecoderBuffer");
+
+  DCHECK(input_image.GetEncodedData());
+  auto buffer = media::DecoderBuffer::FromExternalMemory(
+      std::make_unique<EncodedImageExternalMemory>(
+          input_image.GetEncodedData()));
+  DCHECK(buffer);
+  buffer->set_timestamp(base::Microseconds(input_image.Timestamp()));
+  buffer->set_is_key_frame(input_image._frameType ==
+                           webrtc::VideoFrameType::kVideoFrameKey);
+
   const int max_sl_index = input_image.SpatialIndex().value_or(0);
+  if (max_sl_index == 0)
+    return buffer;
+
+  std::vector<uint32_t> spatial_layer_frame_size;
+  spatial_layer_frame_size.reserve(max_sl_index);
   for (int i = 0; i <= max_sl_index; i++) {
     const absl::optional<size_t>& frame_size =
         input_image.SpatialLayerFrameSize(i);
@@ -123,22 +152,14 @@ scoped_refptr<media::DecoderBuffer> ConvertToDecoderBuffer(
         base::checked_cast<uint32_t>(*frame_size));
   }
 
-  // TODO(sandersd): What is |render_time_ms|?
-  scoped_refptr<media::DecoderBuffer> buffer;
   if (spatial_layer_frame_size.size() > 1) {
     const uint8_t* side_data =
         reinterpret_cast<const uint8_t*>(spatial_layer_frame_size.data());
     size_t side_data_size =
         spatial_layer_frame_size.size() * sizeof(uint32_t) / sizeof(uint8_t);
-    buffer = media::DecoderBuffer::CopyFrom(
-        input_image.data(), input_image.size(), side_data, side_data_size);
-  } else {
-    buffer =
-        media::DecoderBuffer::CopyFrom(input_image.data(), input_image.size());
+    buffer->CopySideDataFrom(side_data, side_data_size);
   }
-  buffer->set_timestamp(base::Microseconds(input_image.Timestamp()));
-  buffer->set_is_key_frame(input_image._frameType ==
-                           webrtc::VideoFrameType::kVideoFrameKey);
+
   return buffer;
 }