Skip to content
Permalink
Browse files
Add WebCodecsVideoFrame initial support for serialisation and transfer
https://bugs.webkit.org/show_bug.cgi?id=246854
rdar://problem/101425176

Reviewed by Eric Carlson.

Introduce WebCodecsVideoFrameData which contains the necessary data to rebuild an equivalent WebCodecs video frame.
Add support for serialization of video frames. This works for postMessaging to workers and for same-process MessageChannel but not cross process MessageChannel since encode/decode is not yet supported.

Covered by Worker specific added test.
Skipped new test in glib and wk1.

* LayoutTests/http/wpt/webcodecs/videoFrame-serialization-expected.txt: Added.
* LayoutTests/http/wpt/webcodecs/videoFrame-serialization.html: Added.
* LayoutTests/imported/w3c/web-platform-tests/webcodecs/video-frame-serialization.any-expected.txt:
* LayoutTests/platform/glib/TestExpectations:
* LayoutTests/platform/mac-wk1/TestExpectations:
* Source/WebCore/Headers.cmake:
* Source/WebCore/Modules/webcodecs/VideoColorSpace.h:
(WebCore::VideoColorSpace::state const):
* Source/WebCore/Modules/webcodecs/WebCodecsVideoFrame.cpp:
(WebCore::WebCodecsVideoFrame::WebCodecsVideoFrame):
(WebCore::WebCodecsVideoFrame::create):
(WebCore::WebCodecsVideoFrame::initializeFrameFromOtherFrame):
(WebCore::WebCodecsVideoFrame::initializeFrameWithResourceAndSize):
(WebCore::WebCodecsVideoFrame::allocationSize):
(WebCore::WebCodecsVideoFrame::copyTo):
(WebCore::WebCodecsVideoFrame::clone):
(WebCore::WebCodecsVideoFrame::close):
(WebCore::WebCodecsVideoFrame::codedRect const):
(WebCore::WebCodecsVideoFrame::visibleRect const):
(WebCore::WebCodecsVideoFrame::setDisplaySize):
(WebCore::WebCodecsVideoFrame::setVisibleRect):
(WebCore::WebCodecsVideoFrame::colorSpace const):
* Source/WebCore/Modules/webcodecs/WebCodecsVideoFrame.h:
(WebCore::WebCodecsVideoFrame::create):
(WebCore::WebCodecsVideoFrame::format const):
(WebCore::WebCodecsVideoFrame::codedWidth const):
(WebCore::WebCodecsVideoFrame::codedHeight const):
(WebCore::WebCodecsVideoFrame::displayWidth const):
(WebCore::WebCodecsVideoFrame::displayHeight const):
(WebCore::WebCodecsVideoFrame::duration const):
(WebCore::WebCodecsVideoFrame::timestamp const):
(WebCore::WebCodecsVideoFrame::internalFrame const):
(WebCore::WebCodecsVideoFrame::shoudlDiscardAlpha const):
(WebCore::WebCodecsVideoFrame::data const):
(WebCore::WebCodecsVideoFrame::colorSpace const): Deleted.
* Source/WebCore/Modules/webcodecs/WebCodecsVideoFrameAlgorithms.cpp:
(WebCore::videoFramePickColorSpace):
* Source/WebCore/Modules/webcodecs/WebCodecsVideoFrameAlgorithms.h:
* Source/WebCore/Modules/webcodecs/WebCodecsVideoFrameData.h: Added.
* Source/WebCore/WebCore.xcodeproj/project.pbxproj:
* Source/WebCore/bindings/js/SerializedScriptValue.cpp:
(WebCore::CloneSerializer::serialize):
(WebCore::CloneSerializer::CloneSerializer):
(WebCore::CloneSerializer::dumpWebCodecsVideoFrame):
(WebCore::CloneSerializer::dumpIfTerminal):
(WebCore::CloneDeserializer::deserialize):
(WebCore::CloneDeserializer::readWebCodecsVideoFrame):
(WebCore::CloneDeserializer::readTerminal):
(WebCore::SerializedScriptValue::SerializedScriptValue):
(WebCore::SerializedScriptValue::computeMemoryCost const):
(WebCore::SerializedScriptValue::create):
(WebCore::SerializedScriptValue::deserialize):
* Source/WebCore/bindings/js/SerializedScriptValue.h:
(WebCore::SerializedScriptValue::encode const):
(WebCore::SerializedScriptValue::decode):

Canonical link: https://commits.webkit.org/255949@main
  • Loading branch information
youennf committed Oct 25, 2022
1 parent de2edaa commit e813fe5c61f40e29f9f9d28a465224f05ae088ae
Show file tree
Hide file tree
Showing 15 changed files with 363 additions and 124 deletions.
@@ -0,0 +1,4 @@

PASS Verify closing frames does not propagate accross contexts with Worker.postMessage.
PASS Verify transferring frames closes them with Worker.postMessage.

@@ -0,0 +1,82 @@
<!DOCTYPE html>
<html>
<header>
<script src='/resources/testharness.js'></script>
<script src='/resources/testharnessreport.js'></script>
</header>
<body>
<script>
var defaultInit = {
timestamp : 100,
duration : 33,
}

function createDefaultVideoFrame() {
const init = {
format: 'I420',
timestamp: 1234,
codedWidth: 4,
codedHeight: 2,
timestamp: defaultInit.timestamp,
duration: defaultInit.duration
};
const data = new Uint8Array([
1, 2, 3, 4, 5, 6, 7, 8, // y
1, 2, // u
1, 2, // v
]);
return new VideoFrame(data, init);
}

async function createWorker(script)
{
script += "self.postMessage('ready');";
const blob = new Blob([script], { type: 'text/javascript' });
const url = URL.createObjectURL(blob);
const worker = new Worker(URL.createObjectURL(blob));
await new Promise(resolve => worker.onmessage = () => {
resolve();
});
URL.revokeObjectURL(url);
return worker;
}

promise_test(async t => {
let localFrame = createDefaultVideoFrame();

const worker = await createWorker(`
self.onmessage = (event) => {
let externalFrame = event.data;
externalFrame.close();
self.postMessage("Done");
}
`);

const promise = new Promise(resolve => worker.onmessage = resolve);
worker.postMessage(localFrame);
await promise;

assert_equals(localFrame.timestamp, defaultInit.timestamp);
localFrame.close();
}, 'Verify closing frames does not propagate accross contexts with Worker.postMessage.');

promise_test(async t => {
let localFrame = createDefaultVideoFrame();

const worker = await createWorker(`
self.onmessage = (event) => {
let externalFrame = event.data;
self.postMessage(externalFrame.timestamp);
externalFrame.close();
}
`);

const promise = new Promise(resolve => worker.onmessage = event => resolve(event.data));
worker.postMessage(localFrame);

assert_equals(await promise, defaultInit.timestamp);
localFrame.close();
}, 'Verify transferring frames closes them with Worker.postMessage.');
</script>
</body>
</html>
@@ -2,7 +2,7 @@
PASS Test we can clone a VideoFrame.
PASS Verify closing a frame doesn't affect its clones.
PASS Verify cloning a closed frame throws.
FAIL Verify closing frames does not propagate accross contexts. The object can not be cloned.
FAIL Verify transferring frames closes them. The object can not be cloned.
PASS Verify closing frames does not propagate accross contexts.
PASS Verify transferring frames closes them.
PASS Verify posting closed frames throws.

@@ -249,6 +249,7 @@ imported/w3c/web-platform-tests/compat/webkit-box-rtl-flex.html [ Pass ]
imported/w3c/web-platform-tests/web-animations/timing-model/animations/update-playback-rate-zero.html [ Pass ]

# Tests need rebasing
http/wpt/webcodecs [ Skip ]
imported/w3c/web-platform-tests/webcodecs [ Skip ]

# Tests that timeout on Apple platforms.
@@ -42,6 +42,7 @@ webkit.org/b/235072 imported/w3c/web-platform-tests/video-rvfc/request-video-fra

webkit.org/b/244736 imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback.html [ Pass Failure ]

http/wpt/webcodecs [ Skip ]
imported/w3c/web-platform-tests/webcodecs [ Skip ]

#//////////////////////////////////////////////////////////////////////////////////////////
@@ -414,6 +414,8 @@ set(WebCore_PRIVATE_FRAMEWORK_HEADERS
Modules/webcodecs/WebCodecsEncodedVideoChunk.h
Modules/webcodecs/WebCodecsEncodedVideoChunkData.h
Modules/webcodecs/WebCodecsEncodedVideoChunkType.h
Modules/webcodecs/WebCodecsVideoFrame.h
Modules/webcodecs/WebCodecsVideoFrameData.h

Modules/webdatabase/DatabaseDetails.h
Modules/webdatabase/DatabaseManager.h
@@ -57,6 +57,8 @@ class VideoColorSpace : public RefCounted<VideoColorSpace> {
const std::optional<bool>& fullRange() const { return m_state.fullRange; }
void setfFullRange(std::optional<bool>&& fullRange) { m_state.fullRange = WTFMove(fullRange); }

VideoColorSpaceInit state() const { return m_state; }

private:
VideoColorSpace() = default;
VideoColorSpace(const VideoColorSpaceInit& init)

0 comments on commit e813fe5

Please sign in to comment.