Bug 1978232 - Enable batch encoding in EncoderAgent and EncoderTemplate r=media-playback-reviewers,padenot,jolin

This patch enables batch encoding in `EncoderAgent` and `EncoderTemplate`.
`EncoderAgent` now accepts an array of samples rather than a single sample.
With this extended capability, `EncoderTemplate` can now merge multiple
`encode` requests issued from WebCodecs' users into a single one containing
consecutive samples, then submit to `EncoderAgent` as a batch.

A preference is also introduced to control the maximum batch size, making
it easier to disable (when it is set to 0 or 1) or experiement with and measure
performance under different size values.

Differential Revision: https://phabricator.services.mozilla.com/D261676

Author: ChunMinChang

dom/media/platforms/EncoderConfig.h

@@ -240,7 +240,7 @@ class EncoderConfig final {
   uint32_t mBitrate{};
   uint32_t mMinBitrate{};
   uint32_t mMaxBitrate{};
-  Usage mUsage{};
+  Usage mUsage{Usage::Record};
   // Video-only
   HardwarePreference mHardwarePreference{HardwarePreference::None};
   SampleFormat mFormat{dom::ImageBitmapFormat::YUV420P};

dom/media/webcodecs/EncoderAgent.cpp

@@ -295,9 +295,10 @@ RefPtr<ShutdownPromise> EncoderAgent::Shutdown() {
   return encoder->Shutdown();
 }
 
-RefPtr<EncoderAgent::EncodePromise> EncoderAgent::Encode(MediaData* aInput) {
+RefPtr<EncoderAgent::EncodePromise> EncoderAgent::Encode(
+    nsTArray<RefPtr<MediaData>>&& aInputs) {
   MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
-  MOZ_ASSERT(aInput);
+  MOZ_ASSERT(!aInputs.IsEmpty());
   MOZ_ASSERT(mState == State::Configured || mState == State::Error);
   MOZ_ASSERT(mEncodePromise.IsEmpty());
   MOZ_ASSERT(!mEncodeRequest.Exists());
@@ -316,19 +317,20 @@ RefPtr<EncoderAgent::EncodePromise> EncoderAgent::Encode(MediaData* aInput) {
 
   RefPtr<EncodePromise> p = mEncodePromise.Ensure(__func__);
 
-  mEncoder->Encode(aInput)
+  LOGV("EncoderAgent #%zu (%p) is encoding %zu samples", mId, this, aInputs.Length());
+  mEncoder->Encode(std::move(aInputs))
       ->Then(
           mOwnerThread, __func__,
           [self = RefPtr{this}](MediaDataEncoder::EncodedData&& aData) {
             self->mEncodeRequest.Complete();
-            LOGV("EncoderAgent #%zu (%p) encode successful", self->mId,
+            LOGV("EncoderAgent #%zu (%p) encode a batch successful", self->mId,
                  self.get());
             self->SetState(State::Configured);
             self->mEncodePromise.Resolve(std::move(aData), __func__);
           },
           [self = RefPtr{this}](const MediaResult& aError) {
             self->mEncodeRequest.Complete();
-            LOGV("EncoderAgent #%zu (%p) failed to encode", self->mId,
+            LOGV("EncoderAgent #%zu (%p) failed to encode a batch", self->mId,
                  self.get());
             self->SetState(State::Error);
             self->mEncodePromise.Reject(aError, __func__);

dom/media/webcodecs/EncoderAgent.h

@@ -55,7 +55,7 @@ class EncoderAgent final {
       const RefPtr<const EncoderConfigurationChangeList>& aConfigChange);
   RefPtr<ShutdownPromise> Shutdown();
   using EncodePromise = MediaDataEncoder::EncodePromise;
-  RefPtr<EncodePromise> Encode(MediaData* aInput);
+  RefPtr<EncodePromise> Encode(nsTArray<RefPtr<MediaData>>&& aInputs);
   // WebCodecs's flush() flushes out all the pending encoded data in the
   // encoder. It's called Drain internally.
   RefPtr<EncodePromise> Drain();

dom/media/webcodecs/EncoderTemplate.cpp

@@ -6,11 +6,13 @@
 
 #include "EncoderTemplate.h"
 
+#include <algorithm>
 #include <type_traits>
 
 #include "EncoderTypes.h"
 #include "WebCodecsUtils.h"
 #include "mozilla/ScopeExit.h"
+#include "mozilla/StaticPrefs_dom.h"
 #include "mozilla/Try.h"
 #include "mozilla/Unused.h"
 #include "mozilla/dom/BindingDeclarations.h"
@@ -73,9 +75,11 @@ EncoderTemplate<EncoderType>::ConfigureMessage::ConfigureMessage(
 
 template <typename EncoderType>
 EncoderTemplate<EncoderType>::EncodeMessage::EncodeMessage(
-    WebCodecsId aConfigureId, RefPtr<InputTypeInternal>&& aData,
+    WebCodecsId aConfigureId, already_AddRefed<InputTypeInternal> aData,
     Maybe<VideoEncoderEncodeOptions>&& aOptions)
-    : ControlMessage(aConfigureId), mData(aData) {}
+    : ControlMessage(aConfigureId) {
+  PushData(std::move(aData), std::move(aOptions));
+}
 
 template <typename EncoderType>
 EncoderTemplate<EncoderType>::FlushMessage::FlushMessage(
@@ -167,13 +171,10 @@ void EncoderTemplate<EncoderType>::EncodeAudioData(InputType& aInput,
   mAsyncDurationTracker.Start(
       aInput.Timestamp(),
       AutoWebCodecsMarker(EncoderType::Name.get(), ".encode-duration-a"));
-  mEncodeQueueSize += 1;
   // Dummy options here as a shortcut
-  mControlMessageQueue.push(MakeRefPtr<EncodeMessage>(
+  PushEncodeRequest(
       mLatestConfigureId,
-      EncoderType::CreateInputInternal(aInput, VideoEncoderEncodeOptions())));
-  LOGV("%s %p enqueues %s", EncoderType::Name.get(), this,
-       mControlMessageQueue.back()->ToString().get());
+      EncoderType::CreateInputInternal(aInput, VideoEncoderEncodeOptions()));
   ProcessControlMessageQueue();
 }
 
@@ -199,12 +200,9 @@ void EncoderTemplate<EncoderType>::EncodeVideoFrame(
   mAsyncDurationTracker.Start(
       aInput.Timestamp(),
       AutoWebCodecsMarker(EncoderType::Name.get(), ".encode-duration-v"));
-  mEncodeQueueSize += 1;
-  mControlMessageQueue.push(MakeRefPtr<EncodeMessage>(
-      mLatestConfigureId, EncoderType::CreateInputInternal(aInput, aOptions),
-      Some(aOptions)));
-  LOGV("%s %p enqueues %s", EncoderType::Name.get(), this,
-       mControlMessageQueue.back()->ToString().get());
+  PushEncodeRequest(mLatestConfigureId,
+                    EncoderType::CreateInputInternal(aInput, aOptions),
+                    Some(aOptions));
   ProcessControlMessageQueue();
 }
 
@@ -854,6 +852,7 @@ MessageProcessedResult EncoderTemplate<EncoderType>::ProcessEncodeMessage(
   AssertIsOnOwningThread();
   MOZ_ASSERT(mState == CodecState::Configured);
   MOZ_ASSERT(aMessage->AsEncodeMessage());
+  MOZ_ASSERT(mEncodeQueueSize > 0);
 
   AUTO_ENCODER_MARKER(marker, ".encode-process");
 
@@ -867,7 +866,8 @@ MessageProcessedResult EncoderTemplate<EncoderType>::ProcessEncodeMessage(
   LOGV("%s %p processing %s", EncoderType::Name.get(), this,
        aMessage->ToString().get());
 
-  mEncodeQueueSize -= 1;
+  MOZ_ASSERT(AssertedCast<uint32_t>(aMessage->BatchSize()) <= mEncodeQueueSize);
+  mEncodeQueueSize -= AssertedCast<uint32_t>(aMessage->BatchSize());
   ScheduleDequeueEvent();
 
   // Treat it like decode error if no EncoderAgent is available or the encoded
@@ -888,17 +888,15 @@ MessageProcessedResult EncoderTemplate<EncoderType>::ProcessEncodeMessage(
   }
 
   MOZ_ASSERT(mActiveConfig);
-  RefPtr<InputTypeInternal> data = aMessage->mData;
-  if (!data) {
-    LOGE("%s %p, data for %s is empty or invalid", EncoderType::Name.get(),
-         this, aMessage->ToString().get());
+  if (!aMessage->IsValid()) {
+    LOGE("%s %p, %s has empty data", EncoderType::Name.get(), this,
+         aMessage->ToString().get());
     return closeOnError();
   }
 
-  mAgent->Encode(data.get())
+  mAgent->Encode(aMessage->TakeData())
       ->Then(GetCurrentSerialEventTarget(), __func__,
-             [self = RefPtr{this}, id = mAgent->mId, aMessage,
-              m = std::move(marker)](
+             [self = RefPtr{this}, id = mAgent->mId, m = std::move(marker)](
                  EncoderAgent::EncodePromise::ResolveOrRejectValue&&
                      aResult) mutable {
                MOZ_ASSERT(self->mProcessingMessage);
@@ -908,9 +906,11 @@ MessageProcessedResult EncoderTemplate<EncoderType>::ProcessEncodeMessage(
                MOZ_ASSERT(id == self->mAgent->mId);
                MOZ_ASSERT(self->mActiveConfig);
 
-               nsCString msgStr = aMessage->ToString();
+               RefPtr<EncodeMessage> msg =
+                   self->mProcessingMessage->AsEncodeMessage();
+               nsCString msgStr = msg->ToString();
 
-               aMessage->Complete();
+               msg->Complete();
                self->mProcessingMessage = nullptr;
 
                if (aResult.IsReject()) {
@@ -1208,6 +1208,40 @@ void EncoderTemplate<EncoderType>::DestroyEncoderAgentIfAny() {
       });
 }
 
+template <typename EncoderType>
+void EncoderTemplate<EncoderType>::PushEncodeRequest(
+    WebCodecsId aConfigureId, RefPtr<InputTypeInternal>&& aData,
+    Maybe<VideoEncoderEncodeOptions>&& aOptions) {
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == CodecState::Configured);
+
+  const size_t batchSize = std::max<size_t>(
+      StaticPrefs::dom_media_webcodecs_batch_encoding_size(), 1);
+
+  RefPtr<EncodeMessage> msg;
+  if (!mControlMessageQueue.empty()) {
+    msg = mControlMessageQueue.back()->AsEncodeMessage();
+    if (msg &&
+        (msg->mConfigureId != aConfigureId || msg->BatchSize() >= batchSize)) {
+      msg = nullptr;
+    }
+  }
+
+  const bool isNewMessage = !msg;
+  if (isNewMessage) {
+    msg = MakeRefPtr<EncodeMessage>(aConfigureId, aData.forget(),
+                                    std::move(aOptions));
+    mControlMessageQueue.push(msg);
+  } else {
+    msg->PushData(aData.forget(), std::move(aOptions));
+  }
+
+  mEncodeQueueSize += 1;
+  LOGV("%s %p %s %s, encode queue size: %u", EncoderType::Name.get(), this,
+       isNewMessage ? "queued a new" : "appended data to",
+       msg->ToString().get(), mEncodeQueueSize);
+}
+
 template class EncoderTemplate<VideoEncoderTraits>;
 template class EncoderTemplate<AudioEncoderTraits>;
 

dom/media/webcodecs/EncoderTemplate.h

@@ -7,6 +7,7 @@
 #ifndef mozilla_dom_EncoderTemplate_h
 #define mozilla_dom_EncoderTemplate_h
 
+#include <limits>
 #include <queue>
 
 #include "EncoderAgent.h"
@@ -92,25 +93,50 @@ class EncoderTemplate : public DOMEventTargetHelper {
     const RefPtr<ConfigTypeInternal> mConfig;
   };
 
+  // This stores batch encoding requests, created by grouping multiple encode()
+  // calls. See PushEncodeRequest() for further details.
   class EncodeMessage final
       : public ControlMessage,
         public MessageRequestHolder<EncoderAgent::EncodePromise> {
    public:
-    EncodeMessage(WebCodecsId aConfigureId, RefPtr<InputTypeInternal>&& aData,
+    EncodeMessage(WebCodecsId aConfigureId,
+                  already_AddRefed<InputTypeInternal> aData,
                   Maybe<VideoEncoderEncodeOptions>&& aOptions = Nothing());
     nsCString ToString() const override {
       nsCString rv;
-      bool isKeyFrame = mOptions.isSome() && mOptions.ref().mKeyFrame;
-      rv.AppendPrintf("EncodeMessage(#%zu, #%zu): %s%s", this->mConfigureId,
-                      this->mMessageId, mData->ToString().get(),
-                      isKeyFrame ? " (kf)" : "");
+      rv.AppendPrintf(
+          "EncodeMessage(#%zu, #%zu): %zu frames (%zu kfs, %zu held)",
+          this->mConfigureId, this->mMessageId, mFrames, mKeyFrames,
+          mData.Length());
       return rv;
     }
+    bool IsValid() const { return !mHasEmptyData && !mData.IsEmpty(); }
+    size_t BatchSize() const { return mData.Length(); }
+    void PushData(already_AddRefed<InputTypeInternal> aData,
+                  Maybe<VideoEncoderEncodeOptions>&& aOptions = Nothing()) {
+      mFrames += 1;
+      RefPtr<InputTypeInternal> data = aData;
+      if (!data) {
+        mHasEmptyData = true;
+      }
+      MOZ_ASSERT_IF(aOptions.isSome() && aOptions->mKeyFrame, data->mKeyframe);
+      mKeyFrames += data->mKeyframe ? 1 : 0;
+      mData.AppendElement(data.forget());
+    }
+    nsTArray<RefPtr<MediaData>>&& TakeData() {
+      return std::move(mData);
+    }
     virtual void Cancel() override { Disconnect(); }
     virtual bool IsProcessing() override { return Exists(); };
     virtual RefPtr<EncodeMessage> AsEncodeMessage() override { return this; }
-    RefPtr<InputTypeInternal> mData;
-    Maybe<VideoEncoderEncodeOptions> mOptions;
+
+   private:
+    // Stores data in MediaData rather than InputTypeInternal, as
+    // MediaDataEncoder::EncodeBatch expects an array of MediaData.
+    nsTArray<RefPtr<MediaData>> mData;
+    size_t mFrames = 0;
+    size_t mKeyFrames = 0;
+    bool mHasEmptyData = false;
   };
 
   class FlushMessage final
@@ -221,6 +247,10 @@ class EncoderTemplate : public DOMEventTargetHelper {
   bool CreateEncoderAgent(WebCodecsId aId, RefPtr<ConfigTypeInternal> aConfig);
   void DestroyEncoderAgentIfAny();
 
+  void PushEncodeRequest(
+      WebCodecsId aConfigureId, RefPtr<InputTypeInternal>&& aData,
+      Maybe<VideoEncoderEncodeOptions>&& aOptions = Nothing());
+
   // Constant in practice, only set in ctor.
   RefPtr<WebCodecsErrorCallback> mErrorCallback;
   RefPtr<OutputCallbackType> mOutputCallback;

modules/libpref/init/StaticPrefList.yaml

@@ -3586,6 +3586,17 @@
   value: @IS_NIGHTLY_BUILD@
   mirror: always
 
+# WebCodecs API - Batch encoding size
+- name: dom.media.webcodecs.batch-encoding-size
+  type: RelaxedAtomicUint32
+#if defined(XP_WIN)
+  value: 4294967295 # max value of uint32_t
+#else
+  # Bug 1984936: Batch encoding is not optimized on non-Windows platforms yet.
+  value: 1
+#endif
+  mirror: always
+
 # Number of seconds of very quiet or silent audio before considering the audio
 # inaudible.
 - name: dom.media.silence_duration_for_audibility