Set the colorspace of WebCodecs VideoFrame created from canvas source

https://bugs.webkit.org/show_bug.cgi?id=257535 rdar://110062111 Reviewed by Eric Carlson. Set canvas based VideoFrame color space to RGB. We introduce VideoFrame::createFromPixelBuffer, that gets specialized for Cocoa and Gstreamer. * LayoutTests/imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker_h264_annexb-expected.txt: * LayoutTests/imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker_h264_avc-expected.txt: * LayoutTests/imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker_vp8-expected.txt: * LayoutTests/imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker_vp9_p0-expected.txt: * LayoutTests/imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any_h264_annexb-expected.txt: * LayoutTests/imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any_h264_avc-expected.txt: * LayoutTests/imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any_vp8-expected.txt: * LayoutTests/imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any_vp9_p0-expected.txt: * LayoutTests/platform/glib/TestExpectations: * Source/WebCore/Modules/webcodecs/WebCodecsVideoFrame.cpp: (WebCore::WebCodecsVideoFrame::create): * Source/WebCore/html/HTMLCanvasElement.cpp: (WebCore::HTMLCanvasElement::toVideoFrame): * Source/WebCore/platform/VideoFrame.cpp: (WebCore::createFromPixelBuffer): * Source/WebCore/platform/VideoFrame.h: (WebCore::VideoFrame::createFromPixelBuffer): * Source/WebCore/platform/graphics/cv/VideoFrameCV.h: * Source/WebCore/platform/graphics/cv/VideoFrameCV.mm: (WebCore::VideoFrame::createFromPixelBuffer): (WebCore::VideoFrameCV::createFromPixelBuffer): Deleted. * Source/WebCore/platform/graphics/gstreamer/VideoFrameGStreamer.cpp: (WebCore::VideoFrame::createFromPixelBuffer): (WebCore::VideoFrameGStreamer::createFromPixelBuffer): * Source/WebCore/platform/graphics/gstreamer/VideoFrameGStreamer.h: Canonical link: https://commits.webkit.org/264881@main
WebKit · Jun 6, 2023 · e593104 · e593104
1 parent 369109e
commit e593104
Show file tree

Hide file tree

Showing 17 changed files with 58 additions and 28 deletions.
diff --git a/...3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker_h264_annexb-expected.txt b/...3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker_h264_annexb-expected.txt
@@ -1,4 +1,6 @@
 
-FAIL Encoding and decoding cycle assert_not_equals: colorSpace.primaries got disallowed value null
-FAIL Encoding and decoding cycle w/ stripped color space assert_not_equals: colorSpace.primaries got disallowed value null
+Harness Error (FAIL), message = Error in remote https://localhost:9443/webcodecs/full-cycle-test.https.any.js: TypeError: undefined is not an object (evaluating 'encoder_color_space.primaries')
+
+FAIL Encoding and decoding cycle assert_equals: frames_decoded expected 16 but got 0
+FAIL Encoding and decoding cycle w/ stripped color space assert_equals: frames_decoded expected 16 but got 0
 
diff --git a/...d/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker_h264_avc-expected.txt b/...d/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker_h264_avc-expected.txt
@@ -1,4 +1,6 @@
 
-FAIL Encoding and decoding cycle assert_not_equals: colorSpace.primaries got disallowed value null
-FAIL Encoding and decoding cycle w/ stripped color space assert_not_equals: colorSpace.primaries got disallowed value null
+Harness Error (FAIL), message = Error in remote https://localhost:9443/webcodecs/full-cycle-test.https.any.js: TypeError: undefined is not an object (evaluating 'encoder_color_space.primaries')
+
+FAIL Encoding and decoding cycle assert_equals: frames_decoded expected 16 but got 0
+FAIL Encoding and decoding cycle w/ stripped color space assert_equals: frames_decoded expected 16 but got 0
 
diff --git a/...ported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker_vp8-expected.txt b/...ported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker_vp8-expected.txt
@@ -1,4 +1,4 @@
 
-FAIL Encoding and decoding cycle assert_not_equals: colorSpace.primaries got disallowed value null
+PASS Encoding and decoding cycle
 PASS Encoding and decoding cycle w/ stripped color space
 
diff --git a/...ted/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker_vp9_p0-expected.txt b/...ted/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker_vp9_p0-expected.txt
@@ -1,4 +1,4 @@
 
-FAIL Encoding and decoding cycle assert_not_equals: colorSpace.primaries got disallowed value null
-FAIL Encoding and decoding cycle w/ stripped color space assert_not_equals: colorSpace.primaries got disallowed value null
+PASS Encoding and decoding cycle
+PASS Encoding and decoding cycle w/ stripped color space
 
diff --git a/...orted/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any_h264_annexb-expected.txt b/...orted/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any_h264_annexb-expected.txt
@@ -1,4 +1,6 @@
 
-FAIL Encoding and decoding cycle assert_not_equals: colorSpace.primaries got disallowed value null
-FAIL Encoding and decoding cycle w/ stripped color space assert_not_equals: colorSpace.primaries got disallowed value null
+Harness Error (FAIL), message = TypeError: undefined is not an object (evaluating 'encoder_color_space.primaries')
+
+FAIL Encoding and decoding cycle assert_equals: frames_decoded expected 16 but got 0
+FAIL Encoding and decoding cycle w/ stripped color space assert_equals: frames_decoded expected 16 but got 0
 
diff --git a/...imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any_h264_avc-expected.txt b/...imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any_h264_avc-expected.txt
@@ -1,4 +1,6 @@
 
-FAIL Encoding and decoding cycle assert_not_equals: colorSpace.primaries got disallowed value null
-FAIL Encoding and decoding cycle w/ stripped color space assert_not_equals: colorSpace.primaries got disallowed value null
+Harness Error (FAIL), message = TypeError: undefined is not an object (evaluating 'encoder_color_space.primaries')
+
+FAIL Encoding and decoding cycle assert_equals: frames_decoded expected 16 but got 0
+FAIL Encoding and decoding cycle w/ stripped color space assert_equals: frames_decoded expected 16 but got 0
 
diff --git a/...ests/imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any_vp8-expected.txt b/...ests/imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any_vp8-expected.txt
@@ -1,4 +1,4 @@
 
-FAIL Encoding and decoding cycle assert_not_equals: colorSpace.primaries got disallowed value null
+PASS Encoding and decoding cycle
 PASS Encoding and decoding cycle w/ stripped color space
 
diff --git a/...s/imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any_vp9_p0-expected.txt b/...s/imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any_vp9_p0-expected.txt
@@ -1,4 +1,4 @@
 
-FAIL Encoding and decoding cycle assert_not_equals: colorSpace.primaries got disallowed value null
-FAIL Encoding and decoding cycle w/ stripped color space assert_not_equals: colorSpace.primaries got disallowed value null
+PASS Encoding and decoding cycle
+PASS Encoding and decoding cycle w/ stripped color space
 
diff --git a/LayoutTests/platform/glib/TestExpectations b/LayoutTests/platform/glib/TestExpectations
@@ -1137,6 +1137,18 @@ webkit.org/b/239750 media/media-source/media-mp4-xhe-aac.html [ Skip ]
 media/media-hevc-video-as-img.html [ ImageOnlyFailure ]
 
 imported/w3c/web-platform-tests/webcodecs/videoFrame-createImageBitmap.any.worker.html [ Pass ]
+imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.html?av [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.html?h264_annexb [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.html?h264_avc [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.html?vp8 [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.html?vp9_p0 [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.html?vp9_p2 [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker.html?av1 [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker.html?h264_annexb [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker.html?h264_avc [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker.html?vp8 [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker.html?vp9_p0 [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker.html?vp9_p2 [ Failure ]
 
 # Likely bugs related with dav1ddec.
 imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.html?av1 [ Failure ]

diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsVideoFrame.cpp b/Source/WebCore/Modules/webcodecs/WebCodecsVideoFrame.cpp
@@ -241,12 +241,7 @@ ExceptionOr<Ref<WebCodecsVideoFrame>> WebCodecsVideoFrame::create(ScriptExecutio
     if (!pixelBuffer)
         return Exception { InvalidStateError,  "Buffer has no frame"_s };
 
-    RefPtr<VideoFrame> videoFrame;
-#if PLATFORM(COCOA)
-    videoFrame = VideoFrameCV::createFromPixelBuffer(pixelBuffer.releaseNonNull());
-#elif USE(GSTREAMER)
-    videoFrame = VideoFrameGStreamer::createFromPixelBuffer(pixelBuffer.releaseNonNull(), VideoFrameGStreamer::CanvasContentType::Canvas2D);
-#endif
+    auto videoFrame = VideoFrame::createFromPixelBuffer(pixelBuffer.releaseNonNull(), { PlatformVideoColorPrimaries::Bt709, PlatformVideoTransferCharacteristics::Iec6196621, PlatformVideoMatrixCoefficients::Rgb, true });
 
     if (!videoFrame)
         return Exception { InvalidStateError,  "Unable to create frame from buffer"_s };

diff --git a/Source/WebCore/html/HTMLCanvasElement.cpp b/Source/WebCore/html/HTMLCanvasElement.cpp
@@ -839,8 +839,9 @@ RefPtr<VideoFrame> HTMLCanvasElement::toVideoFrame()
     if (!pixelBuffer)
         return nullptr;
 
+    // FIXME: Set color space.
 #if PLATFORM(COCOA)
-    return VideoFrameCV::createFromPixelBuffer(pixelBuffer.releaseNonNull());
+    return VideoFrame::createFromPixelBuffer(pixelBuffer.releaseNonNull());
 #elif USE(GSTREAMER)
     // FIXME: Hardcoding 30fps here is not great. Ideally we should get this from the compositor refresh rate, somehow.
     return VideoFrameGStreamer::createFromPixelBuffer(pixelBuffer.releaseNonNull(), VideoFrameGStreamer::CanvasContentType::Canvas2D, VideoFrameGStreamer::Rotation::None, MediaTime::invalidTime(), { }, 30, false, { });

diff --git a/Source/WebCore/platform/VideoFrame.cpp b/Source/WebCore/platform/VideoFrame.cpp
@@ -56,6 +56,12 @@ RefPtr<VideoFrame> VideoFrame::fromNativeImage(NativeImage&)
     return nullptr;
 }
 
+RefPtr<VideoFrame> createFromPixelBuffer(Ref<PixelBuffer>&&, PlatformVideoColorSpace&&)
+{
+    // FIXME: Add support.
+    return nullptr;
+}
+
 RefPtr<VideoFrame> VideoFrame::createNV12(std::span<const uint8_t>, size_t, size_t, const ComputedPlaneLayout&, const ComputedPlaneLayout&, PlatformVideoColorSpace&&)
 {
     // FIXME: Add support.

diff --git a/Source/WebCore/platform/VideoFrame.h b/Source/WebCore/platform/VideoFrame.h
@@ -42,13 +42,16 @@ namespace WebCore {
 
 class FloatRect;
 class GraphicsContext;
-struct ImageOrientation;
 class NativeImage;
+class PixelBuffer;
 class ProcessIdentity;
 #if USE(AVFOUNDATION) && PLATFORM(COCOA)
 class VideoFrameCV;
 #endif
 
+struct ImageOrientation;
+struct PlatformVideoColorSpace;
+
 struct ComputedPlaneLayout {
     size_t destinationOffset { 0 };
     size_t destinationStride { 0 };
@@ -71,6 +74,7 @@ class VideoFrame : public ThreadSafeRefCounted<VideoFrame> {
     virtual ~VideoFrame() = default;
 
     static RefPtr<VideoFrame> fromNativeImage(NativeImage&);
+    static RefPtr<VideoFrame> createFromPixelBuffer(Ref<PixelBuffer>&&, PlatformVideoColorSpace&& = { });
     static RefPtr<VideoFrame> createNV12(std::span<const uint8_t>, size_t width, size_t height, const ComputedPlaneLayout&, const ComputedPlaneLayout&, PlatformVideoColorSpace&&);
     static RefPtr<VideoFrame> createRGBA(std::span<const uint8_t>, size_t width, size_t height, const ComputedPlaneLayout&, PlatformVideoColorSpace&&);
     static RefPtr<VideoFrame> createBGRA(std::span<const uint8_t>, size_t width, size_t height, const ComputedPlaneLayout&, PlatformVideoColorSpace&&);

diff --git a/Source/WebCore/platform/graphics/cv/VideoFrameCV.h b/Source/WebCore/platform/graphics/cv/VideoFrameCV.h
@@ -42,7 +42,6 @@ class VideoFrameCV : public VideoFrame {
 public:
     WEBCORE_EXPORT static Ref<VideoFrameCV> create(MediaTime presentationTime, bool isMirrored, Rotation, RetainPtr<CVPixelBufferRef>&&, std::optional<PlatformVideoColorSpace>&& = { });
     WEBCORE_EXPORT static Ref<VideoFrameCV> create(CMSampleBufferRef, bool isMirrored, Rotation);
-    static RefPtr<VideoFrameCV> createFromPixelBuffer(Ref<PixelBuffer>&&);
     WEBCORE_EXPORT ~VideoFrameCV();
 
     CVPixelBufferRef pixelBuffer() const final { return m_pixelBuffer.get(); }

diff --git a/Source/WebCore/platform/graphics/cv/VideoFrameCV.mm b/Source/WebCore/platform/graphics/cv/VideoFrameCV.mm
@@ -420,7 +420,7 @@ static void copyPlane(uint8_t* destination, const uint8_t* source, uint64_t sour
     return adoptRef(*new VideoFrameCV(presentationTime, isMirrored, rotation, WTFMove(pixelBuffer), WTFMove(colorSpace)));
 }
 
-RefPtr<VideoFrameCV> VideoFrameCV::createFromPixelBuffer(Ref<PixelBuffer>&& pixelBuffer)
+RefPtr<VideoFrame> VideoFrame::createFromPixelBuffer(Ref<PixelBuffer>&& pixelBuffer, PlatformVideoColorSpace&& colorSpace)
 {
     auto size = pixelBuffer->size();
     auto width = size.width();
@@ -442,7 +442,7 @@ static void copyPlane(uint8_t* destination, const uint8_t* source, uint64_t sour
         return nullptr;
     }
     ASSERT_UNUSED(status, !status);
-    return create({ }, false, Rotation::None, WTFMove(cvPixelBuffer));
+    return RefPtr { VideoFrameCV::create({ }, false, Rotation::None, WTFMove(cvPixelBuffer), WTFMove(colorSpace)) };
 }
 
 static PlatformVideoColorSpace computeVideoFrameColorSpace(CVPixelBufferRef pixelBuffer)

diff --git a/Source/WebCore/platform/graphics/gstreamer/VideoFrameGStreamer.cpp b/Source/WebCore/platform/graphics/gstreamer/VideoFrameGStreamer.cpp
@@ -55,6 +55,11 @@ static void ensureVideoFrameDebugCategoryInitialized()
     });
 }
 
+RefPtr<VideoFrame> VideoFrame::createFromPixelBuffer(Ref<PixelBuffer>&& pixelBuffer, PlatformVideoColorSpace&& colorSpace)
+{
+    return RefPtr { VideoFrameGStreamer::createFromPixelBuffer(WTFMove(pixelBuffer), VideoFrameGStreamer::CanvasContentType::Canvas2D, VideoFrame::Rotation::None, MediaTime::invalidTime(), { }, 1, false, { }, WTFMove(colorSpace)) };
+}
+
 class GstSampleColorConverter {
 public:
     static GstSampleColorConverter& singleton();
@@ -294,7 +299,7 @@ Ref<VideoFrameGStreamer> VideoFrameGStreamer::createWrappedSample(const GRefPtr<
     return adoptRef(*new VideoFrameGStreamer(sample, *presentationSize, presentationTime, videoRotation, WTFMove(colorSpace)));
 }
 
-Ref<VideoFrameGStreamer> VideoFrameGStreamer::createFromPixelBuffer(Ref<PixelBuffer>&& pixelBuffer, CanvasContentType canvasContentType, Rotation videoRotation, const MediaTime& presentationTime, const IntSize& destinationSize, double frameRate, bool videoMirrored, std::optional<VideoFrameTimeMetadata>&& metadata)
+Ref<VideoFrameGStreamer> VideoFrameGStreamer::createFromPixelBuffer(Ref<PixelBuffer>&& pixelBuffer, CanvasContentType canvasContentType, Rotation videoRotation, const MediaTime& presentationTime, const IntSize& destinationSize, double frameRate, bool videoMirrored, std::optional<VideoFrameTimeMetadata>&& metadata, PlatformVideoColorSpace&& colorSpace)
 {
     ensureGStreamerInitialized();
 
@@ -358,7 +363,7 @@ Ref<VideoFrameGStreamer> VideoFrameGStreamer::createFromPixelBuffer(Ref<PixelBuf
         sample = adoptGRef(gst_sample_new(buffer.get(), caps.get(), nullptr, nullptr));
     }
 
-    return adoptRef(*new VideoFrameGStreamer(WTFMove(sample), FloatSize(width, height), presentationTime, videoRotation, videoMirrored, WTFMove(metadata)));
+    return adoptRef(*new VideoFrameGStreamer(WTFMove(sample), FloatSize(width, height), presentationTime, videoRotation, videoMirrored, WTFMove(metadata), WTFMove(colorSpace)));
 }
 
 VideoFrameGStreamer::VideoFrameGStreamer(GRefPtr<GstSample>&& sample, const FloatSize& presentationSize, const MediaTime& presentationTime, Rotation videoRotation, bool videoMirrored, std::optional<VideoFrameTimeMetadata>&& metadata, PlatformVideoColorSpace&& colorSpace)

diff --git a/Source/WebCore/platform/graphics/gstreamer/VideoFrameGStreamer.h b/Source/WebCore/platform/graphics/gstreamer/VideoFrameGStreamer.h
@@ -46,7 +46,7 @@ class VideoFrameGStreamer final : public VideoFrame {
 
     static Ref<VideoFrameGStreamer> createWrappedSample(const GRefPtr<GstSample>&, const MediaTime& presentationTime, Rotation videoRotation = Rotation::None);
 
-    static Ref<VideoFrameGStreamer> createFromPixelBuffer(Ref<PixelBuffer>&&, CanvasContentType canvasContentType, Rotation videoRotation = VideoFrame::Rotation::None, const MediaTime& presentationTime = MediaTime::invalidTime(), const IntSize& destinationSize = { }, double frameRate = 1, bool videoMirrored = false, std::optional<VideoFrameTimeMetadata>&& metadata = std::nullopt);
+    static Ref<VideoFrameGStreamer> createFromPixelBuffer(Ref<PixelBuffer>&&, CanvasContentType canvasContentType, Rotation videoRotation = VideoFrame::Rotation::None, const MediaTime& presentationTime = MediaTime::invalidTime(), const IntSize& destinationSize = { }, double frameRate = 1, bool videoMirrored = false, std::optional<VideoFrameTimeMetadata>&& metadata = std::nullopt, PlatformVideoColorSpace&& = { });
 
     RefPtr<VideoFrameGStreamer> resizeTo(const IntSize&);