[GStreamer][WebCodecs] Video encoding and decoding support

LayoutTests/platform/glib/TestExpectations

@@ -236,10 +236,6 @@ imported/w3c/web-platform-tests/compat/webkit-box-rtl-flex.html [ Pass ]
 
 imported/w3c/web-platform-tests/web-animations/timing-model/animations/update-playback-rate-zero.html [ Pass ]
 
-# Tests need rebasing
-http/wpt/webcodecs [ Skip ]
-imported/w3c/web-platform-tests/webcodecs [ Skip ]
-
 # Tests that timeout on Apple platforms.
 webkit.org/b/185859 imported/w3c/web-platform-tests/css/selectors/focus-visible-002.html [ Pass ]
 webkit.org/b/185859 imported/w3c/web-platform-tests/css/selectors/focus-visible-003.html [ Pass ]
@@ -957,6 +953,21 @@ webkit.org/b/239750 media/media-source/media-mp4-xhe-aac.html [ Skip ]
 # Apple baseline.
 media/media-hevc-video-as-img.html [ ImageOnlyFailure ]
 
+# AV1 WebCodec not supported yet.
+imported/w3c/web-platform-tests/webcodecs/reconfiguring-encoder.https.any.worker.html?av1 [ Skip ]
+imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.html?av1 [ Skip ]
+imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.worker.html?av1 [ Skip ]
+
+# Expected to pass when updating to GStreamer 1.22.
+imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.html?h264_annexb [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.html?h264_avc [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.html?vp8 [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.html?vp9 [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.worker.html?h264_annexb [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.worker.html?h264_avc [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.worker.html?vp8 [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.worker.html?vp9 [ Failure ]
+
 #////////////////////////////////////////////////////////////////////////////////////////
 # End of GStreamer-related bugs
 #////////////////////////////////////////////////////////////////////////////////////////
@@ -1107,6 +1118,10 @@ webkit.org/b/245334 [ Debug ] imported/w3c/web-platform-tests/html/canvas/offscr
 webkit.org/b/245334 [ Debug ] imported/w3c/web-platform-tests/html/canvas/offscreen/manual/transformations/2d.transformation.getTransform.html [ Crash ]
 webkit.org/b/245334 [ Debug ] imported/w3c/web-platform-tests/html/canvas/offscreen/text/ [ Pass Crash ]
 
+# This failure is related with (incomplete?) webgl support for OffscreenCanvas.
+webkit.org/b/245952 [ Debug ] imported/w3c/web-platform-tests/webcodecs/videoFrame-texImage.any.worker.html [ Failure Crash ]
+webkit.org/b/245952 [ Release ] imported/w3c/web-platform-tests/webcodecs/videoFrame-texImage.any.worker.html [ Failure ]
+
 #////////////////////////////////////////////////////////////////////////////////////////
 # End of OffscreenCanvas-related bugs
 #////////////////////////////////////////////////////////////////////////////////////////

LayoutTests/platform/glib/imported/w3c/web-platform-tests/webcodecs/temporal-svc-encoding.https.any.worker_h264-expected.txt

@@ -0,0 +1,6 @@
+
+Harness Error (FAIL), message = Error in remote https://web-platform.test:9443/resources/testharness.js: Error: assert_own_property: expected property "svc" missing
+
+FAIL SVC L1T2 assert_equals: expected 12 but got 0
+FAIL SVC L1T3 assert_equals: expected 6 but got 0
+

LayoutTests/platform/glib/imported/w3c/web-platform-tests/webcodecs/temporal-svc-encoding.https.any.worker_vp8-expected.txt

@@ -0,0 +1,6 @@
+
+Harness Error (FAIL), message = Error in remote https://web-platform.test:9443/resources/testharness.js: Error: assert_own_property: expected property "svc" missing
+
+FAIL SVC L1T2 assert_equals: expected 12 but got 0
+FAIL SVC L1T3 assert_equals: expected 6 but got 0
+

LayoutTests/platform/glib/imported/w3c/web-platform-tests/webcodecs/temporal-svc-encoding.https.any.worker_vp9-expected.txt

@@ -0,0 +1,6 @@
+
+Harness Error (FAIL), message = Error in remote https://web-platform.test:9443/resources/testharness.js: Error: assert_own_property: expected property "svc" missing
+
+FAIL SVC L1T2 assert_equals: expected 12 but got 0
+FAIL SVC L1T3 assert_equals: expected 6 but got 0
+

LayoutTests/platform/glib/imported/w3c/web-platform-tests/webcodecs/videoFrame-createImageBitmap.any-expected.txt

@@ -0,0 +1,9 @@
+
+PASS ImageBitmap<->VideoFrame with canvas(48x36 srgb uint8).
+PASS ImageBitmap<->VideoFrame with canvas(480x360 srgb uint8).
+FAIL ImageBitmap<->VideoFrame with canvas(48x36 display-p3 uint8). Type error
+FAIL ImageBitmap<->VideoFrame with canvas(480x360 display-p3 uint8). Type error
+FAIL ImageBitmap<->VideoFrame with canvas(48x36 rec2020 uint8). Type error
+FAIL ImageBitmap<->VideoFrame with canvas(480x360 rec2020 uint8). Type error
+PASS createImageBitmap uses frame display size
+

LayoutTests/platform/glib/imported/w3c/web-platform-tests/webcodecs/videoFrame-drawImage.any-expected.txt

@@ -0,0 +1,10 @@
+
+PASS drawImage(VideoFrame) with canvas(48x36 srgb uint8).
+PASS drawImage(VideoFrame) with canvas(480x360 srgb uint8).
+FAIL drawImage(VideoFrame) with canvas(48x36 display-p3 uint8). Type error
+FAIL drawImage(VideoFrame) with canvas(480x360 display-p3 uint8). Type error
+FAIL drawImage(VideoFrame) with canvas(48x36 rec2020 uint8). Type error
+PASS drawImage on a closed VideoFrame throws InvalidStateError.
+PASS drawImage of nested frame works properly
+PASS drawImage with display size != visible size
+

Source/WTF/Scripts/Preferences/UnifiedWebPreferences.yaml

@@ -6494,9 +6494,11 @@ WebCodecsEnabled:
       default: false
     WebKit:
       "PLATFORM(COCOA)": true
+      "USE(GSTREAMER)": true
       default: false
     WebCore:
       "PLATFORM(COCOA)": true
+      "USE(GSTREAMER)": true
       default: false
 
 WebGL2Enabled:

Source/WebCore/Modules/webcodecs/WebCodecsVideoFrame.cpp

@@ -47,6 +47,10 @@
 #include "VideoFrameCV.h"
 #endif
 
+#if USE(GSTREAMER)
+#include "VideoFrameGStreamer.h"
+#endif
+
 namespace WebCore {
 
 WebCodecsVideoFrame::WebCodecsVideoFrame(ScriptExecutionContext& context)
@@ -207,7 +211,10 @@ ExceptionOr<Ref<WebCodecsVideoFrame>> WebCodecsVideoFrame::create(ScriptExecutio
     RefPtr<VideoFrame> videoFrame;
 #if PLATFORM(COCOA)
     videoFrame = VideoFrameCV::createFromPixelBuffer(pixelBuffer.releaseNonNull());
+#elif USE(GSTREAMER)
+    videoFrame = VideoFrameGStreamer::createFromPixelBuffer(pixelBuffer.releaseNonNull(), VideoFrameGStreamer::CanvasContentType::Canvas2D);
 #endif
+
     if (!videoFrame)
         return Exception { InvalidStateError,  "Unable to create frame from buffer"_s };
 

Source/WebCore/platform/GStreamer.cmake

@@ -46,6 +46,8 @@ if (ENABLE_VIDEO OR ENABLE_WEB_AUDIO)
         platform/graphics/gstreamer/TextCombinerPadGStreamer.cpp
         platform/graphics/gstreamer/TextSinkGStreamer.cpp
         platform/graphics/gstreamer/TrackPrivateBaseGStreamer.cpp
+        platform/graphics/gstreamer/VideoDecoderGStreamer.cpp
+        platform/graphics/gstreamer/VideoEncoderGStreamer.cpp
         platform/graphics/gstreamer/VideoFrameGStreamer.cpp
         platform/graphics/gstreamer/VideoFrameMetadataGStreamer.cpp
         platform/graphics/gstreamer/VideoSinkGStreamer.cpp
@@ -67,6 +69,7 @@ if (ENABLE_VIDEO OR ENABLE_WEB_AUDIO)
         platform/graphics/gstreamer/mse/WebKitMediaSourceGStreamer.cpp
 
         platform/gstreamer/GStreamerCodecUtilities.cpp
+        platform/gstreamer/GStreamerElementHarness.cpp
 
         platform/mediarecorder/MediaRecorderPrivateGStreamer.cpp
 
@@ -103,6 +106,7 @@ if (ENABLE_VIDEO OR ENABLE_WEB_AUDIO)
     )
 
     list(APPEND WebCore_PRIVATE_FRAMEWORK_HEADERS
+        platform/gstreamer/GStreamerElementHarness.h
         platform/graphics/gstreamer/GRefPtrGStreamer.h
         platform/graphics/gstreamer/GStreamerCommon.h
         platform/graphics/gstreamer/GUniquePtrGStreamer.h

Source/WebCore/platform/VideoDecoder.cpp

@@ -32,6 +32,10 @@
 #include "LibWebRTCVPXVideoDecoder.h"
 #endif
 
+#if USE(GSTREAMER)
+#include "VideoDecoderGStreamer.h"
+#endif
+
 namespace WebCore {
 
 VideoDecoder::CreatorFunction VideoDecoder::s_customCreator = nullptr;
@@ -51,9 +55,10 @@ void VideoDecoder::create(const String& codecName, const Config& config, CreateC
 }
 
 
-void VideoDecoder::createLocalDecoder(const String& codecName, const Config&, CreateCallback&& callback, OutputCallback&& outputCallback, PostTaskCallback&& postCallback)
+void VideoDecoder::createLocalDecoder(const String& codecName, const Config& config, CreateCallback&& callback, OutputCallback&& outputCallback, PostTaskCallback&& postCallback)
 {
 #if USE(LIBWEBRTC)
+    UNUSED_PARAM(config);
     if (codecName == "vp8"_s) {
         LibWebRTCVPXVideoDecoder::create(LibWebRTCVPXVideoDecoder::Type::VP8, WTFMove(callback), WTFMove(outputCallback), WTFMove(postCallback));
         return;
@@ -66,8 +71,12 @@ void VideoDecoder::createLocalDecoder(const String& codecName, const Config&, Cr
         LibWebRTCVPXVideoDecoder::create(LibWebRTCVPXVideoDecoder::Type::AV1, WTFMove(callback), WTFMove(outputCallback), WTFMove(postCallback));
         return;
     }
+#elif USE(GSTREAMER)
+    if (GStreamerVideoDecoder::create(codecName, config, WTFMove(callback), WTFMove(outputCallback), WTFMove(postCallback)))
+        return;
 #else
     UNUSED_PARAM(codecName);
+    UNUSED_PARAM(config);
     UNUSED_PARAM(outputCallback);
     UNUSED_PARAM(postCallback);
 #endif

Source/WebCore/platform/VideoEncoder.cpp

@@ -32,6 +32,10 @@
 #include "LibWebRTCVPXVideoEncoder.h"
 #endif
 
+#if USE(GSTREAMER)
+#include "VideoEncoderGStreamer.h"
+#endif
+
 namespace WebCore {
 
 VideoEncoder::CreatorFunction VideoEncoder::s_customCreator = nullptr;
@@ -65,6 +69,9 @@ void VideoEncoder::createLocalEncoder(const String& codecName, const Config& con
         LibWebRTCVPXVideoEncoder::create(LibWebRTCVPXVideoEncoder::Type::AV1, config, WTFMove(callback), WTFMove(descriptionCallback), WTFMove(outputCallback), WTFMove(postCallback));
         return;
     }
+#elif USE(GSTREAMER)
+    GStreamerVideoEncoder::create(codecName, config, WTFMove(callback), WTFMove(descriptionCallback), WTFMove(outputCallback), WTFMove(postCallback));
+    return;
 #else
     UNUSED_PARAM(codecName);
     UNUSED_PARAM(outputCallback);

Source/WebCore/platform/VideoFrame.cpp

@@ -49,7 +49,7 @@ void VideoFrame::initializeCharacteristics(MediaTime presentationTime, bool isMi
     const_cast<Rotation&>(m_rotation) = rotation;
 }
 
-#if !PLATFORM(COCOA)
+#if !PLATFORM(COCOA) && !USE(GSTREAMER)
 RefPtr<VideoFrame> VideoFrame::fromNativeImage(NativeImage&)
 {
     // FIXME: Add support.
@@ -86,6 +86,13 @@ void VideoFrame::copyTo(Span<uint8_t>, VideoPixelFormat, Vector<ComputedPlaneLay
     callback({ });
 }
 
+void VideoFrame::paintInContext(GraphicsContext&, const FloatRect&, const ImageOrientation&, bool)
+{
+    // FIXME: Add support.
+}
+#endif // !PLATFORM(COCOA)
+
+#if !PLATFORM(COCOA)
 RefPtr<JSC::Uint8ClampedArray> VideoFrame::getRGBAImageData() const
 {
 #if USE(GSTREAMER)
@@ -95,15 +102,7 @@ RefPtr<JSC::Uint8ClampedArray> VideoFrame::getRGBAImageData() const
     // FIXME: Add support.
     return nullptr;
 }
-#endif // !PLATFORM(COCOA)
-
-#if !PLATFORM(COCOA) && !USE(GSTREAMER)
-void VideoFrame::paintInContext(GraphicsContext&, const FloatRect&, const ImageOrientation&, bool)
-{
-    // FIXME: Add support.
-}
-#endif // !PLATFORM(COCOA) && !USE(GSTREAMER)
-
+#endif
 }
 
 #endif // ENABLE(VIDEO)