[WebCodecs] Add 'optimizeForLatency' to VideoDecoderConfig.

This wires up Chrome's existing low delay decoding mode based on a
new 'optimizeForLatency' boolean flag in the VideoDecoderConfig. It
allows clients using the software decoder to guarantee 1-in-1-out
style decoding under certain format conditions.

It also adds some missing logs which can help with debugging.

R=sandersd

Fixed: 1204314
Change-Id: I2243519b36361598b1cca9be41ae0a5aade26fa8
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2859664
Commit-Queue: Dale Curtis <dalecurtis@chromium.org>
Reviewed-by: Dan Sanders <sandersd@chromium.org>
Cr-Commit-Position: refs/heads/master@{#878030}

media/base/media_log.h

@@ -214,8 +214,9 @@ class MEDIA_EXPORT LogHelper {
 
 // Provides a stringstream to collect a log entry to pass to the provided
 // MediaLog at the requested level.
-#define MEDIA_LOG(level, media_log) \
-  LogHelper((MediaLogMessageLevel::k##level), (media_log)).stream()
+#define MEDIA_LOG(level, media_log)                                      \
+  media::LogHelper((media::MediaLogMessageLevel::k##level), (media_log)) \
+      .stream()
 
 // Logs only while |count| < |max|, increments |count| for each log, and warns
 // in the log if |count| has just reached |max|.

third_party/blink/renderer/modules/webcodecs/audio_decoder.cc

@@ -89,6 +89,8 @@ void AudioDecoderTraits::UpdateDecoderLog(const MediaDecoderType& decoder,
       decoder.IsPlatformDecoder());
   media_log->SetProperty<media::MediaLogProperty::kAudioTracks>(
       std::vector<MediaConfigType>{media_config});
+  MEDIA_LOG(INFO, media_log)
+      << "Initialized AudioDecoder: " << media_config.AsHumanReadableString();
 }
 
 // static
@@ -124,6 +126,7 @@ media::StatusOr<AudioDecoderTraits::OutputType*> AudioDecoderTraits::MakeOutput(
 // static
 void AudioDecoderTraits::InitializeDecoder(
     MediaDecoderType& decoder,
+    bool /*low_delay*/,
     const MediaConfigType& media_config,
     MediaDecoderType::InitCB init_cb,
     MediaDecoderType::OutputCB output_cb) {

third_party/blink/renderer/modules/webcodecs/audio_decoder.h

@@ -58,6 +58,7 @@ class MODULES_EXPORT AudioDecoderTraits {
       media::GpuVideoAcceleratorFactories* gpu_factories,
       media::MediaLog* media_log);
   static void InitializeDecoder(MediaDecoderType& decoder,
+                                bool low_delay,
                                 const MediaConfigType& media_config,
                                 MediaDecoderType::InitCB init_cb,
                                 MediaDecoderType::OutputCB output_cb);

third_party/blink/renderer/modules/webcodecs/audio_decoder_broker.cc

@@ -124,8 +124,9 @@ class MediaAudioTaskWrapper {
                            weak_factory_.GetWeakPtr()));
 
     selector_->SelectDecoder(
-        config, WTF::Bind(&MediaAudioTaskWrapper::OnDecoderSelected,
-                          weak_factory_.GetWeakPtr()));
+        config, /*low_delay=*/false,
+        WTF::Bind(&MediaAudioTaskWrapper::OnDecoderSelected,
+                  weak_factory_.GetWeakPtr()));
   }
 
   void Decode(scoped_refptr<media::DecoderBuffer> buffer, int cb_id) {

third_party/blink/renderer/modules/webcodecs/decoder_selector.cc

@@ -50,11 +50,18 @@ class NullDemuxerStream : public media::DemuxerStream {
     return true;
   }
 
+  void set_low_delay(bool low_delay) { low_delay_ = low_delay; }
+  media::DemuxerStream::Liveness liveness() const override {
+    return low_delay_ ? media::DemuxerStream::LIVENESS_LIVE
+                      : media::DemuxerStream::LIVENESS_UNKNOWN;
+  }
+
  private:
   static const media::DemuxerStream::Type stream_type = StreamType;
 
   media::AudioDecoderConfig audio_decoder_config_;
   media::VideoDecoderConfig video_decoder_config_;
+  bool low_delay_ = false;
 };
 
 template <>
@@ -90,9 +97,11 @@ DecoderSelector<StreamType>::~DecoderSelector() = default;
 template <media::DemuxerStream::Type StreamType>
 void DecoderSelector<StreamType>::SelectDecoder(
     const DecoderConfig& config,
+    bool low_delay,
     SelectDecoderCB select_decoder_cb) {
   // |impl_| will internally use this the |config| from our NullDemuxerStream.
   demuxer_stream_->Configure(config);
+  demuxer_stream_->set_low_delay(low_delay);
 
   // media::DecoderSelector will call back with a null decoder if selection is
   // in progress when it is destructed.

third_party/blink/renderer/modules/webcodecs/decoder_selector.h

@@ -54,6 +54,7 @@ class DecoderSelector {
   // be returned via |select_decoder_cb| posted to |task_runner_|. Subsequent
   // calls will again select from the full list of decoders.
   void SelectDecoder(const DecoderConfig& config,
+                     bool low_delay,
                      SelectDecoderCB select_decoder_cb);
 
  private:

third_party/blink/renderer/modules/webcodecs/decoder_selector_test.cc

@@ -58,7 +58,8 @@ class AudioDecoderSelectorTestParam {
   // Decoder::Initialize() takes different parameters depending on the type.
   static void ExpectInitialize(MockDecoder* decoder,
                                DecoderCapability capability,
-                               media::AudioDecoderConfig expected_config) {
+                               media::AudioDecoderConfig expected_config,
+                               bool /*low_delay */) {
     EXPECT_CALL(*decoder, Initialize_(_, _, _, _, _))
         .WillRepeatedly([capability, expected_config](
                             const media::AudioDecoderConfig& config,
@@ -96,8 +97,9 @@ class VideoDecoderSelectorTestParam {
 
   static void ExpectInitialize(MockDecoder* decoder,
                                DecoderCapability capability,
-                               media::VideoDecoderConfig expected_config) {
-    EXPECT_CALL(*decoder, Initialize_(_, _, _, _, _, _))
+                               media::VideoDecoderConfig expected_config,
+                               bool low_delay) {
+    EXPECT_CALL(*decoder, Initialize_(_, low_delay, _, _, _, _))
         .WillRepeatedly([capability, expected_config](
                             const media::VideoDecoderConfig& config,
                             bool low_delay, media::CdmContext*,
@@ -162,7 +164,7 @@ class WebCodecsDecoderSelectorTest : public ::testing::Test {
               /*is_platform_decoder=*/false, /*supports_decryption=*/true,
               info.first);
       TypeParam::ExpectInitialize(decoder.get(), info.second,
-                                  last_set_decoder_config_);
+                                  last_set_decoder_config_, low_delay_);
       decoders.push_back(std::move(decoder));
     }
 
@@ -180,7 +182,7 @@ class WebCodecsDecoderSelectorTest : public ::testing::Test {
   void SelectDecoder(DecoderConfig config = TypeParam::CreateConfig()) {
     last_set_decoder_config_ = config;
     decoder_selector_->SelectDecoder(
-        config,
+        config, low_delay_,
         base::BindOnce(&Self::OnDecoderSelectedThunk, base::Unretained(this)));
     RunUntilIdle();
   }
@@ -196,6 +198,8 @@ class WebCodecsDecoderSelectorTest : public ::testing::Test {
 
   std::vector<std::pair<int, DecoderCapability>> mock_decoders_to_create_;
 
+  bool low_delay_ = false;
+
  private:
   DISALLOW_COPY_AND_ASSIGN(WebCodecsDecoderSelectorTest);
 };
@@ -218,6 +222,14 @@ TYPED_TEST(WebCodecsDecoderSelectorTest, OneDecoder) {
   this->SelectDecoder();
 }
 
+TYPED_TEST(WebCodecsDecoderSelectorTest, LowDelay) {
+  this->low_delay_ = true;
+  this->AddMockDecoder(kDecoder1, kSucceed);
+
+  EXPECT_CALL(*this, OnDecoderSelected(kDecoder1));
+  this->SelectDecoder();
+}
+
 TYPED_TEST(WebCodecsDecoderSelectorTest, TwoDecoders) {
   this->AddMockDecoder(kDecoder1, kFail);
   this->AddMockDecoder(kDecoder2, kSucceed);

third_party/blink/renderer/modules/webcodecs/decoder_template.cc

@@ -128,6 +128,11 @@ HardwarePreference DecoderTemplate<Traits>::GetHardwarePreference(
   return HardwarePreference::kAllow;
 }
 
+template <typename Traits>
+bool DecoderTemplate<Traits>::GetLowDelayPreference(const ConfigType&) {
+  return false;
+}
+
 template <typename Traits>
 void DecoderTemplate<Traits>::SetHardwarePreference(HardwarePreference) {}
 
@@ -163,6 +168,7 @@ void DecoderTemplate<Traits>::configure(const ConfigType* config,
   request->media_config = std::move(media_config);
   request->reset_generation = reset_generation_;
   request->hw_pref = GetHardwarePreference(*config);
+  request->low_delay = GetLowDelayPreference(*config);
   requests_.push_back(request);
   ProcessRequests();
 }
@@ -300,7 +306,7 @@ bool DecoderTemplate<Traits>::ProcessConfigureRequest(Request* request) {
     SetHardwarePreference(pending_request_->hw_pref);
 
     Traits::InitializeDecoder(
-        *decoder_, *pending_request_->media_config,
+        *decoder_, pending_request_->low_delay, *pending_request_->media_config,
         WTF::Bind(&DecoderTemplate::OnInitializeDone, WrapWeakPersistent(this)),
         WTF::BindRepeating(&DecoderTemplate::OnOutput, WrapWeakPersistent(this),
                            reset_generation_));
@@ -509,7 +515,8 @@ void DecoderTemplate<Traits>::OnFlushDone(media::Status status) {
 
   // Processing continues in OnInitializeDone().
   Traits::InitializeDecoder(
-      *decoder_, is_flush ? *active_config_ : *pending_request_->media_config,
+      *decoder_, is_flush ? low_delay_ : pending_request_->low_delay,
+      is_flush ? *active_config_ : *pending_request_->media_config,
       WTF::Bind(&DecoderTemplate::OnInitializeDone, WrapWeakPersistent(this)),
       WTF::BindRepeating(&DecoderTemplate::OnOutput, WrapWeakPersistent(this),
                          reset_generation_));
@@ -547,6 +554,7 @@ void DecoderTemplate<Traits>::OnInitializeDone(media::Status status) {
     Traits::UpdateDecoderLog(*decoder_, *pending_request_->media_config,
                              logger_->log());
 
+    low_delay_ = pending_request_->low_delay;
     active_config_ = std::move(pending_request_->media_config);
     pending_request_.Release();
   }

third_party/blink/renderer/modules/webcodecs/decoder_template.h

@@ -78,6 +78,10 @@ class MODULES_EXPORT DecoderTemplate
   // If derived classes do not override this, this will default to kAllow.
   virtual HardwarePreference GetHardwarePreference(const ConfigType& config);
 
+  // Get the low delay preference from a config.
+  // If derived classes do not override this, this will default to false.
+  virtual bool GetLowDelayPreference(const ConfigType& config);
+
   // Sets the HardwarePreference on the |decoder_|.
   // The default implementation does nothing and must be overridden by derived
   // classes if needed.
@@ -110,6 +114,7 @@ class MODULES_EXPORT DecoderTemplate
     // For kConfigure Requests.
     std::unique_ptr<MediaConfigType> media_config;
     HardwarePreference hw_pref = HardwarePreference::kAllow;
+    bool low_delay = false;
 
     // For kDecode Requests.
     scoped_refptr<media::DecoderBuffer> decoder_buffer;
@@ -166,6 +171,7 @@ class MODULES_EXPORT DecoderTemplate
 
   // Cached config from the last kConfigure request which successfully completed
   // initialization.
+  bool low_delay_ = false;
   std::unique_ptr<MediaConfigType> active_config_;
 
   // TODO(sandersd): Store the last config, flush, and reset so that

third_party/blink/renderer/modules/webcodecs/video_decoder.cc

@@ -299,6 +299,9 @@ VideoDecoderConfig* CopyConfig(const VideoDecoderConfig& config) {
   if (config.hasHardwareAcceleration())
     copy->setHardwareAcceleration(config.hardwareAcceleration());
 
+  if (config.hasOptimizeForLatency())
+    copy->setOptimizeForLatency(config.optimizeForLatency());
+
   return copy;
 }
 
@@ -326,12 +329,12 @@ HardwarePreference VideoDecoder::GetHardwareAccelerationPreference(
 // static
 void VideoDecoderTraits::InitializeDecoder(
     MediaDecoderType& decoder,
+    bool low_delay,
     const MediaConfigType& media_config,
     MediaDecoderType::InitCB init_cb,
     MediaDecoderType::OutputCB output_cb) {
-  decoder.Initialize(media_config, false /* low_delay */,
-                     nullptr /* cdm_context */, std::move(init_cb), output_cb,
-                     media::WaitingCB());
+  decoder.Initialize(media_config, low_delay, nullptr /* cdm_context */,
+                     std::move(init_cb), output_cb, media::WaitingCB());
 }
 
 // static
@@ -346,6 +349,8 @@ void VideoDecoderTraits::UpdateDecoderLog(const MediaDecoderType& decoder,
       decoder.IsPlatformDecoder());
   media_log->SetProperty<media::MediaLogProperty::kVideoTracks>(
       std::vector<MediaConfigType>{media_config});
+  MEDIA_LOG(INFO, media_log)
+      << "Initialized VideoDecoder: " << media_config.AsHumanReadableString();
 }
 
 // static
@@ -452,6 +457,10 @@ HardwarePreference VideoDecoder::GetHardwarePreference(
   return GetHardwareAccelerationPreference(config);
 }
 
+bool VideoDecoder::GetLowDelayPreference(const ConfigType& config) {
+  return config.hasOptimizeForLatency() && config.optimizeForLatency();
+}
+
 void VideoDecoder::SetHardwarePreference(HardwarePreference preference) {
   static_cast<VideoDecoderBroker*>(decoder())->SetHardwarePreference(
       preference);

third_party/blink/renderer/modules/webcodecs/video_decoder.h

@@ -68,6 +68,7 @@ class MODULES_EXPORT VideoDecoderTraits {
       media::GpuVideoAcceleratorFactories* gpu_factories,
       media::MediaLog* media_log);
   static void InitializeDecoder(MediaDecoderType& decoder,
+                                bool low_delay,
                                 const MediaConfigType& media_config,
                                 MediaDecoderType::InitCB init_cb,
                                 MediaDecoderType::OutputCB output_cb);
@@ -128,6 +129,7 @@ class MODULES_EXPORT VideoDecoder : public DecoderTemplate<VideoDecoderTraits> {
  private:
   // DecoderTemplate implementation.
   HardwarePreference GetHardwarePreference(const ConfigType& config) override;
+  bool GetLowDelayPreference(const ConfigType& config) override;
   void SetHardwarePreference(HardwarePreference preference) override;
 };
 

third_party/blink/renderer/modules/webcodecs/video_decoder_broker.cc

@@ -121,7 +121,7 @@ class MediaVideoTaskWrapper {
   MediaVideoTaskWrapper(const MediaVideoTaskWrapper&) = delete;
   MediaVideoTaskWrapper& operator=(const MediaVideoTaskWrapper&) = delete;
 
-  void Initialize(const media::VideoDecoderConfig& config) {
+  void Initialize(const media::VideoDecoderConfig& config, bool low_delay) {
     DVLOG(2) << __func__;
     DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 
@@ -138,8 +138,9 @@ class MediaVideoTaskWrapper {
                            weak_factory_.GetWeakPtr()));
 
     selector_->SelectDecoder(
-        config, WTF::Bind(&MediaVideoTaskWrapper::OnDecoderSelected,
-                          weak_factory_.GetWeakPtr()));
+        config, low_delay,
+        WTF::Bind(&MediaVideoTaskWrapper::OnDecoderSelected,
+                  weak_factory_.GetWeakPtr()));
   }
 
   void Decode(scoped_refptr<media::DecoderBuffer> buffer, int cb_id) {
@@ -376,9 +377,6 @@ void VideoDecoderBroker::Initialize(const media::VideoDecoderConfig& config,
   DCHECK(!init_cb_) << "Initialize already pending";
 
   // The following are not currently supported in WebCodecs.
-  // TODO(chcunningham): Should |low_delay| be supported? Should it be
-  // hard-coded to true?
-  DCHECK(!low_delay);
   DCHECK(!cdm_context);
   DCHECK(!waiting_cb);
 
@@ -393,7 +391,7 @@ void VideoDecoderBroker::Initialize(const media::VideoDecoderConfig& config,
       *media_task_runner_, FROM_HERE,
       WTF::CrossThreadBindOnce(&MediaVideoTaskWrapper::Initialize,
                                WTF::CrossThreadUnretained(media_tasks_.get()),
-                               config));
+                               config, low_delay));
 }
 
 int VideoDecoderBroker::CreateCallbackId() {

third_party/blink/renderer/modules/webcodecs/video_decoder_config.idl

@@ -35,6 +35,9 @@ dictionary VideoDecoderConfig {
 
   HardwarePreference hardwareAcceleration = "allow";
 
+  // Hint that decoders should be configured for latency versus throughput.
+  boolean optimizeForLatency;
+
   // DEPRECATED: Use visibleRegion.
   [EnforceRange] unsigned long cropLeft;
   [EnforceRange] unsigned long cropTop;

third_party/blink/web_tests/external/wpt/webcodecs/README.md

@@ -9,6 +9,12 @@ To add, update or remove a test file, please update the list below.
 Please provide full reference and steps to generate the test file so that
 any people can regenerate or update the file in the future.
 
+## Notes
+* When updating the sample offsets and descriptions for tests using mp4 files, it's easiest to use [mp4box.js](https://gpac.github.io/mp4box.js/test/filereader.html).
+  * Sample offsets can be copied from the "Sample View" tab after unchecking all but offset and size. Use a multi-line edit mode and clang-format to quickly format entries.
+  * Description entries can be found under moov.trak.mdia.minf.stbl.stsd in box view.
+    * avc1.avcC has an offset, size in the same view. Add 8 to offset and subtract 8 from the size to get the values the tests want.
+
 ## List of Test Files
 
 ### four-colors.png
@@ -77,6 +83,11 @@ channels.
 ### four-colors.mp4
 Used a [custom tool](https://storage.googleapis.com/dalecurtis/avif2mp4.html) to convert four-colors.avif into a .mp4 file.
 
+### h264.mp4
+```
+ffmpeg -f lavfi -i testsrc=rate=10:n=1 -t 1 -pix_fmt yuv420p -vcodec h264 -tune zerolatency h264.mp4
+```
+
 ### sfx-opus.ogg
 ```
 sox -n -r 48000 sfx.wav synth 1 sine 480