[Cocoa] WebCodecs HEVC decoder is not always reordering frames accord…

…ing presentation time rdar://127655559 https://bugs.webkit.org/show_bug.cgi?id=270645 Reviewed by Eric Carlson. We were not computing correctly the nal unit type in case array_completeness is set to 1. We fix this by masking with 0x3F to also make sure to not be disturbed by the reserved bit after array_completness. We copy test-bframes-hevc.mp4 to test-bframes-hevc-array-completeness.mp4 and update the array_completness bit to 1 there. * LayoutTests/http/tests/webcodecs/hevc-reordering-array-completeness-expected.txt: Added. * LayoutTests/http/tests/webcodecs/hevc-reordering-array-completeness.html: Added. * LayoutTests/media/media-source/content/test-bframes-hevc-array-completeness.mp4: Added. * Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm: (vpsDataFromHvcc): Canonical link: https://commits.webkit.org/278462@main
WebKit · May 7, 2024 · 1fb6c8c · 1fb6c8c
1 parent bdfd69d
commit 1fb6c8c
Show file tree

Hide file tree

Showing 4 changed files with 66 additions and 2 deletions.
diff --git a/LayoutTests/http/tests/webcodecs/hevc-reordering-array-completeness-expected.txt b/LayoutTests/http/tests/webcodecs/hevc-reordering-array-completeness-expected.txt
@@ -0,0 +1,3 @@
+
+PASS Test HEVC reordering
+
diff --git a/LayoutTests/http/tests/webcodecs/hevc-reordering-array-completeness.html b/LayoutTests/http/tests/webcodecs/hevc-reordering-array-completeness.html
@@ -0,0 +1,57 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <script src="../../resources/testharness.js"></script>
+    <script src="../../resources/testharnessreport.js"></script>
+</head>
+<body>
+    <script>
+let decoder;
+promise_test(async () => {
+     const response = await fetch("/media-resources/media-source/content/test-bframes-hevc-array-completeness.mp4");
+     const buffer = await response.arrayBuffer();
+     const hvcCOffset = [705, 130];
+     const frames = [
+         [99, 3973, 46304],
+         [366, 50277, 4531],
+         [233, 54808, 1371],
+         [167, 56179, 379],
+         [133, 56558, 172],
+         [199, 56730, 120],
+         [300, 56850, 313],
+         [266, 57163, 112],
+         [333, 57275, 150],
+         [633, 57425, 6427]
+    ];
+
+    let frameTimestamps = [];
+    decoder =  new VideoDecoder({
+        output(frame) {
+            frameTimestamps.push(frame.timestamp);
+            frame.close();
+        },
+        error(e) {
+            console.log(e);
+        }
+    });
+    decoder.configure({
+        codec: 'hev1.1.6.L120.90',
+        codedWidth: 852,
+        codedHeight: 480,
+        visibleRect: {x: 0, y: 0, width: 852, height: 480},
+        displayWidth: 852,
+        displayHeight: 480,
+        format: 'hevc',
+        description: new Uint8Array(buffer, hvcCOffset[0], hvcCOffset[1])
+    });
+
+    chunks = frames.map((frame, i) => new EncodedVideoChunk({type: i == 0 ? 'key' : 'delta', timestamp: frame[0], duration: 1, data: new Uint8Array(buffer, frame[1], frame[2])}));
+
+    chunks.forEach(chunk => decoder.decode(chunk));
+    await decoder.flush();
+
+    assert_array_equals(frameTimestamps, frames.map(frame => frame[0]).sort((a,b) => a - b), "timestamps are ordered");
+}, "Test HEVC reordering");
+    </script>
+</body>
+</html>
diff --git a/LayoutTests/media/media-source/content/test-bframes-hevc-array-completeness.mp4 b/LayoutTests/media/media-source/content/test-bframes-hevc-array-completeness.mp4
diff --git a/...ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm b/...ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm
@@ -16,6 +16,7 @@
 #import "RTCVideoFrameReorderQueue.h"
 #import "base/RTCVideoFrame.h"
 #import "base/RTCVideoFrameBuffer.h"
+#import "common_video/h265/h265_common.h"
 #import "common_video/h265/h265_vps_parser.h"
 #import "components/video_frame_buffer/RTCCVPixelBuffer.h"
 #import "helpers.h"
@@ -91,18 +92,21 @@ static void overrideColorSpaceAttachments(CVImageBufferRef imageBuffer) {
 
   size_t position = (8 + 8 + 32 + 32 + 16 + 8 + 16 + 8 + 8 + 8 + 8 + 16 + 8 + 8) / 8;
   for (uint32_t j = 0; j < numOfArrays; j++) {
-    // NAL_unit_type
-    auto nalUnitType = reader.Read<uint8_t>();
+    // NAL_unit_type: bit(1) array_completeness; unsigned int(1) reserved = 0; unsigned int(6) NAL_unit_type;
+    auto nalUnitType = reader.Read<uint8_t>() & 0x3F;
+    // numNalus
     auto numOfNalus = reader.Read<uint16_t>();
     position += 3;
     if (!reader.Ok()) {
         return { };
     }
 
     for (uint32_t k = 0; k < numOfNalus; k++) {
+        // nalUnitLength
         auto size = reader.Read<uint16_t>();
 
         position += 2;
+        // nalUnit
         reader.ConsumeBits(8 * size);
 
         static const size_t hevcNalHeaderSize = 2;