VideoEncoder produces no frames with latencyMode "realtime" when fram…

…erate/bitrate are not given

https://bugs.webkit.org/show_bug.cgi?id=264894
rdar://118725549

Reviewed by Eric Carlson.

When frame rate and/or bitrate is not set, use default values for them.
For frame rate, we use 30 frames.
For bitrate, we use the max bit rate as per H264 table A.1.

* LayoutTests/http/wpt/webcodecs/h264-encoder-default-config.https.any-expected.txt: Added.
* LayoutTests/http/wpt/webcodecs/h264-encoder-default-config.https.any.html: Added.
* LayoutTests/http/wpt/webcodecs/h264-encoder-default-config.https.any.js: Added.
(async encoderTest):
(promise_test.async return):
(promise_test):
* LayoutTests/http/wpt/webcodecs/h264-encoder-default-config.https.any.worker-expected.txt: Added.
* LayoutTests/http/wpt/webcodecs/h264-encoder-default-config.https.any.worker.html: Added.
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm:
(-[RTCVideoEncoderH264 startEncodeWithSettings:numberOfCores:]):
(-[RTCVideoEncoderH264 setBitrate:framerate:]):
(-[RTCVideoEncoderH264 setEncoderBitrateBps:frameRate:]):

Canonical link: https://commits.webkit.org/271087@main

LayoutTests/http/wpt/webcodecs/h264-encoder-default-config.https.any-expected.txt

@@ -0,0 +1,5 @@
+
+PASS Realtime encoding without framerate and bitrate
+PASS Realtime encoding without bitrate
+PASS Realtime encoding without framerate
+

LayoutTests/http/wpt/webcodecs/h264-encoder-default-config.https.any.html

@@ -0,0 +1 @@
+<!-- This file is required for WebKit test infrastructure to run the templated test -->

LayoutTests/http/wpt/webcodecs/h264-encoder-default-config.https.any.js

@@ -0,0 +1,68 @@
+async function encoderTest(testConfig)
+{
+    const width = 200;
+    const height = 100;
+    const img = new ImageData(width, height);
+
+    for (let r = 0; r < height; r++) {
+        for (let c = 0; c < width; c++) {
+            const index = (r * width + c) * 4;
+            img.data[index + 0] = 127;
+            img.data[index + 1] = 127;
+            img.data[index + 2] = 127;
+            img.data[index + 3] = 255;
+        }
+    }
+    const bitmap = await createImageBitmap(img);
+
+    let framesCount = 0;
+    let errorCount = 0;
+    const encoder = new VideoEncoder({
+      output: (chunk, metadata) => {
+          ++framesCount;
+      }, error: (err) => {
+          ++errorCount;
+      }
+    })
+
+    const encoderConfig = {
+        codec: "avc1.42001f", // Baseline profile (42 00) with level 3.1 (1f)
+        width,
+        height,
+        latencyMode: "realtime",
+        avc: { format: "annexb" },
+    }
+    if (testConfig.framerate)
+        encoderConfig.framerate = testConfig.framerate;
+    if (testConfig.bitrate)
+        encoderConfig.bitrate = testConfig.bitrate;
+
+    encoder.configure(encoderConfig);
+
+    for (let i = 0; i < 20; i++) {
+        const frame = new VideoFrame(bitmap, { timestamp: i });
+        encoder.encode(frame, {keyFrame: i === 0});
+        frame.close();
+    }
+
+    bitmap.close();
+
+    await encoder.flush();
+    encoder.close();
+
+    assert_greater_than(framesCount, 0, "frames count");
+    assert_equals(errorCount, 0, "error count");
+}
+
+promise_test(async () => {
+    return encoderTest({ });
+}, "Realtime encoding without framerate and bitrate");
+
+promise_test(async () => {
+    return encoderTest({ frameRate: 10 });
+}, "Realtime encoding without bitrate");
+
+promise_test(async () => {
+    return encoderTest({ bitrate: 1000 });
+}, "Realtime encoding without framerate");
+

LayoutTests/http/wpt/webcodecs/h264-encoder-default-config.https.any.worker-expected.txt

@@ -0,0 +1,5 @@
+
+PASS Realtime encoding without framerate and bitrate
+PASS Realtime encoding without bitrate
+PASS Realtime encoding without framerate
+

LayoutTests/http/wpt/webcodecs/h264-encoder-default-config.https.any.worker.html

@@ -0,0 +1 @@
+<!-- This file is required for WebKit test infrastructure to run the templated test -->

LayoutTests/platform/glib/TestExpectations

@@ -1208,6 +1208,8 @@ fast/mediastream/play-newly-added-audio-track.html [ Pass Crash ]
 webkit.org/b/264663 media/media-source/media-source-webm-configuration-framerate.html [ Crash Pass ]
 
 webkit.org/b/264665 http/wpt/webcodecs/videoFrame-webglCanvasImageSource.html [ Failure ]
+webkit.org/b/264894 http/wpt/webcodecs/h264-encoder-default-config.https.any.html [ Failure ]
+webkit.org/b/264894 http/wpt/webcodecs/h264-encoder-default-config.https.any.worker.html [ Failure ]
 
 webkit.org/b/264666 imported/w3c/web-platform-tests/encrypted-media/clearkey-generate-request-disallowed-input.https.html [ Failure ]
 

Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm

@@ -78,6 +78,8 @@ - (void)updateBitRateAccordingActualFrameRate;
 const int kBelowFrameRateThreshold = 2;
 const int kBelowFrameRateBitRateDecrease = 3;
 
+const uint32_t kDefaultEncoderFrameRate = 30;
+
 // Struct that we pass to the encoder per frame to encode. We receive it again
 // in the encoder callback.
 struct RTCFrameEncodeParams {
@@ -341,8 +343,54 @@ NSUInteger GetMaxSampleRate(const webrtc::H264ProfileLevelId &profile_level_id)
       return 0;
   }
 }
-}  // namespace
 
+// Table A-1 level limits of https://www.itu.int/rec/dologin_pub.asp?lang=e&id=T-REC-H.264-201610-S!!PDF-E&type=items.
+size_t GetMaxBitRateKBps(const webrtc::H264ProfileLevelId &profile_level_id) {
+  switch (profile_level_id.level) {
+    case webrtc::H264Level::kLevel1:
+      return 64;
+    case webrtc::H264Level::kLevel1_b:
+      return 128;
+    case webrtc::H264Level::kLevel1_1:
+      return 192;
+    case webrtc::H264Level::kLevel1_2:
+      return 384;
+    case webrtc::H264Level::kLevel1_3:
+      return 768;
+    case webrtc::H264Level::kLevel2:
+    case webrtc::H264Level::kLevel2_1:
+    case webrtc::H264Level::kLevel2_2:
+      return 4000;
+    case webrtc::H264Level::kLevel3:
+      return 10000;
+    case webrtc::H264Level::kLevel3_1:
+      return 14000;
+    case webrtc::H264Level::kLevel3_2:
+      return 20000;
+    case webrtc::H264Level::kLevel4:
+      return 20000;
+    case webrtc::H264Level::kLevel4_1:
+    case webrtc::H264Level::kLevel4_2:
+      return 50000;
+    case webrtc::H264Level::kLevel5:
+      return 135000;
+    case webrtc::H264Level::kLevel5_1:
+    case webrtc::H264Level::kLevel5_2:
+      return 240000;
+  }
+}
+
+size_t computeDefaultBitRateKbps(uint32_t proposedBitrateKbps, const webrtc::H264ProfileLevelId& profile_level_id)
+{
+  return proposedBitrateKbps ? proposedBitrateKbps : GetMaxBitRateKBps(profile_level_id);
+}
+
+uint32_t computeFramerate(uint32_t proposedFramerate, uint32_t maxAllowedFramerate)
+{
+  return MIN(proposedFramerate ? proposedFramerate : kDefaultEncoderFrameRate, maxAllowedFramerate);
+}
+
+} // namespace
 @implementation RTCVideoEncoderH264 {
   RTCVideoCodecInfo *_codecInfo;
   std::unique_ptr<webrtc::BitrateAdjuster> _bitrateAdjuster;
@@ -437,11 +485,10 @@ - (NSInteger)startEncodeWithSettings:(RTCVideoEncoderSettings *)settings
   uint32_t aligned_height = (((_height + 15) >> 4) << 4);
   _maxAllowedFrameRate = static_cast<uint32_t>(GetMaxSampleRate(*_profile_level_id) /
                                                (aligned_width * aligned_height));
-
   // We can only set average bitrate on the HW encoder.
-  _targetBitrateBps = settings.startBitrate * 1000;  // startBitrate is in kbps.
+  _targetBitrateBps = computeDefaultBitRateKbps(settings.startBitrate, *_profile_level_id) * 1000;
   _bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps);
-  _encoderFrameRate = MIN(settings.maxFramerate, _maxAllowedFrameRate);
+  _encoderFrameRate = computeFramerate(settings.maxFramerate, _maxAllowedFrameRate);
   if (settings.maxFramerate > _maxAllowedFrameRate && _maxAllowedFrameRate > 0) {
     RTC_LOG(LS_WARNING) << "Initial encoder frame rate setting " << settings.maxFramerate
                         << " is larger than the "
@@ -616,13 +663,13 @@ - (void)setCallback:(RTCVideoEncoderCallback)callback {
 }
 
 - (int)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate {
-  _targetBitrateBps = 1000 * bitrateKbit;
+  _targetBitrateBps = computeDefaultBitRateKbps(bitrateKbit, *_profile_level_id) * 1000;
   _bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps);
   if (framerate > _maxAllowedFrameRate && _maxAllowedFrameRate > 0) {
     RTC_LOG(LS_WARNING) << "Encoder frame rate setting " << framerate << " is larger than the "
                         << "maximal allowed frame rate " << _maxAllowedFrameRate << ".";
   }
-  framerate = MIN(framerate, _maxAllowedFrameRate);
+  framerate = computeFramerate(framerate, _maxAllowedFrameRate);
   [self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps() frameRate:framerate];
   return WEBRTC_VIDEO_CODEC_OK;
 }
@@ -817,7 +864,9 @@ - (void)setBitrateBps:(uint32_t)bitrateBps frameRate:(uint32_t)frameRate {
 - (void)setEncoderBitrateBps:(uint32_t)bitrateBps frameRate:(uint32_t)frameRate {
   if ([self hasCompressionSession]) {
     auto actualTarget = _isBelowExpectedFrameRate ? bitrateBps / kBelowFrameRateBitRateDecrease : bitrateBps;
-    SetVTSessionProperty(_vtCompressionSession, kVTCompressionPropertyKey_AverageBitRate, actualTarget);
+    if (actualTarget) {
+      SetVTSessionProperty(_vtCompressionSession, kVTCompressionPropertyKey_AverageBitRate, actualTarget);
+    }
 
     // With zero |_maxAllowedFrameRate|, we fall back to automatic frame rate detection.
     if (_maxAllowedFrameRate > 0) {