Add WebCodecsVideoFrame initial support for serialisation and transfer

https://bugs.webkit.org/show_bug.cgi?id=246854 rdar://problem/101425176 Reviewed by Eric Carlson. Introduce WebCodecsVideoFrameData which contains the necessary data to rebuild an equivalent WebCodecs video frame. Add support for serialization of video frames. This works for postMessaging to workers and for same-process MessageChannel but not cross process MessageChannel since encode/decode is not yet supported. Covered by Worker specific added test. Skipped new test in glib and wk1. * LayoutTests/http/wpt/webcodecs/videoFrame-serialization-expected.txt: Added. * LayoutTests/http/wpt/webcodecs/videoFrame-serialization.html: Added. * LayoutTests/imported/w3c/web-platform-tests/webcodecs/video-frame-serialization.any-expected.txt: * LayoutTests/platform/glib/TestExpectations: * LayoutTests/platform/mac-wk1/TestExpectations: * Source/WebCore/Headers.cmake: * Source/WebCore/Modules/webcodecs/VideoColorSpace.h: (WebCore::VideoColorSpace::state const): * Source/WebCore/Modules/webcodecs/WebCodecsVideoFrame.cpp: (WebCore::WebCodecsVideoFrame::WebCodecsVideoFrame): (WebCore::WebCodecsVideoFrame::create): (WebCore::WebCodecsVideoFrame::initializeFrameFromOtherFrame): (WebCore::WebCodecsVideoFrame::initializeFrameWithResourceAndSize): (WebCore::WebCodecsVideoFrame::allocationSize): (WebCore::WebCodecsVideoFrame::copyTo): (WebCore::WebCodecsVideoFrame::clone): (WebCore::WebCodecsVideoFrame::close): (WebCore::WebCodecsVideoFrame::codedRect const): (WebCore::WebCodecsVideoFrame::visibleRect const): (WebCore::WebCodecsVideoFrame::setDisplaySize): (WebCore::WebCodecsVideoFrame::setVisibleRect): (WebCore::WebCodecsVideoFrame::colorSpace const): * Source/WebCore/Modules/webcodecs/WebCodecsVideoFrame.h: (WebCore::WebCodecsVideoFrame::create): (WebCore::WebCodecsVideoFrame::format const): (WebCore::WebCodecsVideoFrame::codedWidth const): (WebCore::WebCodecsVideoFrame::codedHeight const): (WebCore::WebCodecsVideoFrame::displayWidth const): (WebCore::WebCodecsVideoFrame::displayHeight const): (WebCore::WebCodecsVideoFrame::duration const): (WebCore::WebCodecsVideoFrame::timestamp const): (WebCore::WebCodecsVideoFrame::internalFrame const): (WebCore::WebCodecsVideoFrame::shoudlDiscardAlpha const): (WebCore::WebCodecsVideoFrame::data const): (WebCore::WebCodecsVideoFrame::colorSpace const): Deleted. * Source/WebCore/Modules/webcodecs/WebCodecsVideoFrameAlgorithms.cpp: (WebCore::videoFramePickColorSpace): * Source/WebCore/Modules/webcodecs/WebCodecsVideoFrameAlgorithms.h: * Source/WebCore/Modules/webcodecs/WebCodecsVideoFrameData.h: Added. * Source/WebCore/WebCore.xcodeproj/project.pbxproj: * Source/WebCore/bindings/js/SerializedScriptValue.cpp: (WebCore::CloneSerializer::serialize): (WebCore::CloneSerializer::CloneSerializer): (WebCore::CloneSerializer::dumpWebCodecsVideoFrame): (WebCore::CloneSerializer::dumpIfTerminal): (WebCore::CloneDeserializer::deserialize): (WebCore::CloneDeserializer::readWebCodecsVideoFrame): (WebCore::CloneDeserializer::readTerminal): (WebCore::SerializedScriptValue::SerializedScriptValue): (WebCore::SerializedScriptValue::computeMemoryCost const): (WebCore::SerializedScriptValue::create): (WebCore::SerializedScriptValue::deserialize): * Source/WebCore/bindings/js/SerializedScriptValue.h: (WebCore::SerializedScriptValue::encode const): (WebCore::SerializedScriptValue::decode): Canonical link: https://commits.webkit.org/255949@main
WebKit · Oct 25, 2022 · e813fe5 · e813fe5
1 parent de2edaa
commit e813fe5
Show file tree

Hide file tree

Showing 15 changed files with 363 additions and 124 deletions.
diff --git a/LayoutTests/http/wpt/webcodecs/videoFrame-serialization-expected.txt b/LayoutTests/http/wpt/webcodecs/videoFrame-serialization-expected.txt
@@ -0,0 +1,4 @@
+
+PASS Verify closing frames does not propagate accross contexts with Worker.postMessage.
+PASS Verify transferring frames closes them with Worker.postMessage.
+
diff --git a/LayoutTests/http/wpt/webcodecs/videoFrame-serialization.html b/LayoutTests/http/wpt/webcodecs/videoFrame-serialization.html
@@ -0,0 +1,82 @@
+<!DOCTYPE html>
+<html>
+<header>
+<script src='/resources/testharness.js'></script>
+<script src='/resources/testharnessreport.js'></script>
+</header>
+<body>
+<script>
+var defaultInit = {
+  timestamp : 100,
+  duration : 33,
+}
+
+function createDefaultVideoFrame() {
+  const init = {
+    format: 'I420',
+    timestamp: 1234,
+    codedWidth: 4,
+    codedHeight: 2,
+    timestamp: defaultInit.timestamp,
+    duration: defaultInit.duration
+  };
+  const data = new Uint8Array([
+    1, 2, 3, 4, 5, 6, 7, 8,  // y
+    1, 2,                    // u
+    1, 2,                    // v
+  ]);
+  return new VideoFrame(data, init);
+}
+
+async function createWorker(script)
+{
+  script += "self.postMessage('ready');";
+  const blob = new Blob([script], { type: 'text/javascript' });
+  const url = URL.createObjectURL(blob);
+  const worker = new Worker(URL.createObjectURL(blob));
+  await new Promise(resolve => worker.onmessage = () => {
+      resolve();
+  });
+  URL.revokeObjectURL(url);
+  return worker;
+}
+
+promise_test(async t => {
+  let localFrame = createDefaultVideoFrame();
+
+  const worker = await createWorker(`
+    self.onmessage = (event) => {
+      let externalFrame = event.data;
+      externalFrame.close();
+      self.postMessage("Done");
+    }
+  `);
+
+  const promise = new Promise(resolve => worker.onmessage = resolve);  
+  worker.postMessage(localFrame);
+  await promise;
+
+  assert_equals(localFrame.timestamp, defaultInit.timestamp);
+  localFrame.close();
+}, 'Verify closing frames does not propagate accross contexts with Worker.postMessage.');
+
+promise_test(async t => {
+  let localFrame = createDefaultVideoFrame();
+
+  const worker = await createWorker(`
+    self.onmessage = (event) => {
+      let externalFrame = event.data;
+      self.postMessage(externalFrame.timestamp);
+      externalFrame.close();
+    }
+  `);
+
+  const promise = new Promise(resolve => worker.onmessage = event => resolve(event.data));  
+  worker.postMessage(localFrame);
+
+  assert_equals(await promise, defaultInit.timestamp);
+  localFrame.close();
+}, 'Verify transferring frames closes them with Worker.postMessage.');
+</script>
+</body>
+</html>
diff --git a/...ests/imported/w3c/web-platform-tests/webcodecs/video-frame-serialization.any-expected.txt b/...ests/imported/w3c/web-platform-tests/webcodecs/video-frame-serialization.any-expected.txt
@@ -2,7 +2,7 @@
 PASS Test we can clone a VideoFrame.
 PASS Verify closing a frame doesn't affect its clones.
 PASS Verify cloning a closed frame throws.
-FAIL Verify closing frames does not propagate accross contexts. The object can not be cloned.
-FAIL Verify transferring frames closes them. The object can not be cloned.
+PASS Verify closing frames does not propagate accross contexts.
+PASS Verify transferring frames closes them.
 PASS Verify posting closed frames throws.
 
diff --git a/LayoutTests/platform/glib/TestExpectations b/LayoutTests/platform/glib/TestExpectations
@@ -249,6 +249,7 @@ imported/w3c/web-platform-tests/compat/webkit-box-rtl-flex.html [ Pass ]
 imported/w3c/web-platform-tests/web-animations/timing-model/animations/update-playback-rate-zero.html [ Pass ]
 
 # Tests need rebasing
+http/wpt/webcodecs [ Skip ]
 imported/w3c/web-platform-tests/webcodecs [ Skip ]
 
 # Tests that timeout on Apple platforms.

diff --git a/LayoutTests/platform/mac-wk1/TestExpectations b/LayoutTests/platform/mac-wk1/TestExpectations
@@ -42,6 +42,7 @@ webkit.org/b/235072 imported/w3c/web-platform-tests/video-rvfc/request-video-fra
 
 webkit.org/b/244736 imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback.html [ Pass Failure ]
 
+http/wpt/webcodecs [ Skip ]
 imported/w3c/web-platform-tests/webcodecs [ Skip ]
 
 #//////////////////////////////////////////////////////////////////////////////////////////

diff --git a/Source/WebCore/Headers.cmake b/Source/WebCore/Headers.cmake
@@ -414,6 +414,8 @@ set(WebCore_PRIVATE_FRAMEWORK_HEADERS
     Modules/webcodecs/WebCodecsEncodedVideoChunk.h
     Modules/webcodecs/WebCodecsEncodedVideoChunkData.h
     Modules/webcodecs/WebCodecsEncodedVideoChunkType.h
+    Modules/webcodecs/WebCodecsVideoFrame.h
+    Modules/webcodecs/WebCodecsVideoFrameData.h
 
     Modules/webdatabase/DatabaseDetails.h
     Modules/webdatabase/DatabaseManager.h

diff --git a/Source/WebCore/Modules/webcodecs/VideoColorSpace.h b/Source/WebCore/Modules/webcodecs/VideoColorSpace.h
@@ -57,6 +57,8 @@ class VideoColorSpace : public RefCounted<VideoColorSpace> {
     const std::optional<bool>& fullRange() const { return m_state.fullRange; }
     void setfFullRange(std::optional<bool>&& fullRange) { m_state.fullRange = WTFMove(fullRange); }
 
+    VideoColorSpaceInit state() const { return m_state; }
+
 private:
     VideoColorSpace() = default;
     VideoColorSpace(const VideoColorSpaceInit& init)