Implement WebCodecs Validate VideoFrameInit algorithm

https://bugs.webkit.org/show_bug.cgi?id=246796 rdar://problem/101373396 Reviewed by Eric Carlson. Implement https://w3c.github.io/webcodecs/#validate-videoframeinit and add support for alpha from canvas. Covered by rebased test. * LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoFrame-construction.any-expected.txt: * Source/WebCore/Modules/webcodecs/WebCodecsVideoFrame.cpp: (WebCore::WebCodecsVideoFrame::initializeFrameFromOtherFrame): (WebCore::WebCodecsVideoFrame::initializeFrameWithResourceAndSize): * Source/WebCore/Modules/webcodecs/WebCodecsVideoFrame.h: * Source/WebCore/Modules/webcodecs/WebCodecsVideoFrameAlgorithms.cpp: (WebCore::isNegativeOrNonFinite): (WebCore::validateVideoFrameInit): * Source/WebCore/Modules/webcodecs/WebCodecsVideoFrameAlgorithms.h: * Source/WebCore/platform/VideoPixelFormat.cpp: (WebCore::convertVideoFramePixelFormat): * Source/WebCore/platform/VideoPixelFormat.h: Canonical link: https://commits.webkit.org/255786@main
WebKit · Oct 20, 2022 · 9761a5d · 9761a5d
1 parent 7cf8c6d
commit 9761a5d
Show file tree

Hide file tree

Showing 7 changed files with 70 additions and 25 deletions.
diff --git a/...tTests/imported/w3c/web-platform-tests/webcodecs/videoFrame-construction.any-expected.txt b/...tTests/imported/w3c/web-platform-tests/webcodecs/videoFrame-construction.any-expected.txt
@@ -10,9 +10,7 @@ PASS Test constructing w/ unusable image argument throws: HAVE_NOTHING <video>.
 PASS Test constructing w/ unusable image argument throws: emtpy Canvas.
 PASS Test constructing w/ unusable image argument throws: closed ImageBitmap.
 PASS Test constructing w/ unusable image argument throws: closed VideoFrame.
-FAIL Test invalid CanvasImageSource constructed VideoFrames assert_throws_js: negative visibleRect x function "() => new VideoFrame(
-          image,
-          {timestamp: 10, visibleRect: {x: -1, y: 0, width: 10, height: 10}})" did not throw
+PASS Test invalid CanvasImageSource constructed VideoFrames
 PASS Test visibleRect metadata override where source display size = visible size
 PASS Test visibleRect metadata override where source display width = 2 * visible width (anamorphic)
 PASS Test visibleRect metadata override where source display size = 2 * visible size for both width and height
@@ -30,5 +28,5 @@ PASS Test we can construct a VideoFrame from an offscreen canvas.
 PASS Test I420 VideoFrame with odd visible size
 FAIL Test I420A VideoFrame and alpha={keep,discard} VideoPixelFormat is not supported
 PASS Test RGBA, BGRA VideoFrames with alpha={keep,discard}
-FAIL Test a VideoFrame constructed from canvas can drop the alpha channel. assert_true: plane format should not have alpha: BGRA expected true got false
+PASS Test a VideoFrame constructed from canvas can drop the alpha channel.
 
diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsVideoFrame.cpp b/Source/WebCore/Modules/webcodecs/WebCodecsVideoFrame.cpp
@@ -307,13 +307,18 @@ static VideoPixelFormat computeVideoPixelFormat(VideoPixelFormat baseFormat, boo
 }
 
 // https://w3c.github.io/webcodecs/#videoframe-initialize-frame-from-other-frame
-Ref<WebCodecsVideoFrame> WebCodecsVideoFrame::initializeFrameFromOtherFrame(Ref<WebCodecsVideoFrame>&& videoFrame, Init&& init)
+ExceptionOr<Ref<WebCodecsVideoFrame>> WebCodecsVideoFrame::initializeFrameFromOtherFrame(Ref<WebCodecsVideoFrame>&& videoFrame, Init&& init)
 {
-    // FIXME: Call https://w3c.github.io/webcodecs/#validate-videoframeinit
+    auto codedWidth = videoFrame->m_codedWidth;
+    auto codedHeight = videoFrame->m_codedHeight;
+    auto format = computeVideoPixelFormat(videoFrame->m_format.value_or(VideoPixelFormat::I420), init.alpha == WebCodecsAlphaOption::Discard);
+    if (!validateVideoFrameInit(init, codedWidth, codedHeight, format))
+        return Exception { TypeError,  "VideoFrameInit is not valid"_s };
+
     auto result = adoptRef(*new WebCodecsVideoFrame);
     result->m_internalFrame = videoFrame->m_internalFrame;
     if (videoFrame->m_format)
-        result->m_format = computeVideoPixelFormat(*videoFrame->m_format, init.alpha == WebCodecsAlphaOption::Discard);
+        result->m_format = format;
 
     result->m_codedWidth = videoFrame->m_codedWidth;
     result->m_codedHeight = videoFrame->m_codedHeight;
@@ -327,17 +332,19 @@ Ref<WebCodecsVideoFrame> WebCodecsVideoFrame::initializeFrameFromOtherFrame(Ref<
     return result;
 }
 
-Ref<WebCodecsVideoFrame> WebCodecsVideoFrame::initializeFrameFromOtherFrame(Ref<VideoFrame>&& internalVideoFrame, Init&& init)
+ExceptionOr<Ref<WebCodecsVideoFrame>> WebCodecsVideoFrame::initializeFrameFromOtherFrame(Ref<VideoFrame>&& internalVideoFrame, Init&& init)
 {
-    // FIXME: Call https://w3c.github.io/webcodecs/#validate-videoframeinit
-    // FIXME: Implement alpha discarding.
+    auto codedWidth = internalVideoFrame->presentationSize().width();
+    auto codedHeight = internalVideoFrame->presentationSize().height();
+    auto format = convertVideoFramePixelFormat(internalVideoFrame->pixelFormat(), init.alpha == WebCodecsAlphaOption::Discard);
+    if (!validateVideoFrameInit(init, codedWidth, codedHeight, format))
+        return Exception { TypeError,  "VideoFrameInit is not valid"_s };
 
     auto result = adoptRef(*new WebCodecsVideoFrame);
     result->m_internalFrame = WTFMove(internalVideoFrame);
-    result->m_format = convertVideoFramePixelFormat(result->m_internalFrame->pixelFormat());
-
-    result->m_codedWidth = result->m_internalFrame->presentationSize().width();
-    result->m_codedHeight = result->m_internalFrame->presentationSize().height();
+    result->m_format = format;
+    result->m_codedWidth = codedWidth;
+    result->m_codedHeight = codedHeight;
 
     initializeVisibleRectAndDisplaySize(result.get(), init, DOMRectInit { 0, 0 , static_cast<double>(result->m_codedWidth), static_cast<double>(result->m_codedHeight) }, result->m_codedWidth, result->m_codedHeight);
 
@@ -355,14 +362,17 @@ ExceptionOr<Ref<WebCodecsVideoFrame>> WebCodecsVideoFrame::initializeFrameWithRe
     if (!internalVideoFrame)
         return Exception { TypeError,  "image has no resource"_s };
 
-    // FIXME: Call https://w3c.github.io/webcodecs/#validate-videoframeinit
-    // FIXME: Implement alpha discarding.
+    auto codedWidth = image->size().width();
+    auto codedHeight = image->size().height();
+    auto format = convertVideoFramePixelFormat(internalVideoFrame->pixelFormat(), init.alpha == WebCodecsAlphaOption::Discard);
+    if (!validateVideoFrameInit(init, codedWidth, codedHeight, format))
+        return Exception { TypeError,  "VideoFrameInit is not valid"_s };
 
     auto result = adoptRef(*new WebCodecsVideoFrame);
     result->m_internalFrame = WTFMove(internalVideoFrame);
-    result->m_format = convertVideoFramePixelFormat(result->m_internalFrame->pixelFormat());
-    result->m_codedWidth = image->size().width();
-    result->m_codedHeight = image->size().height();
+    result->m_format = format;
+    result->m_codedWidth = codedWidth;
+    result->m_codedHeight = codedHeight;
 
     initializeVisibleRectAndDisplaySize(result.get(), init, DOMRectInit { 0, 0 , static_cast<double>(result->m_codedWidth), static_cast<double>(result->m_codedHeight) }, result->m_codedWidth, result->m_codedHeight);
 

diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsVideoFrame.h b/Source/WebCore/Modules/webcodecs/WebCodecsVideoFrame.h
@@ -129,8 +129,8 @@ class WebCodecsVideoFrame : public RefCounted<WebCodecsVideoFrame> {
     bool shoudlDiscardAlpha() const { return m_format && (*m_format == VideoPixelFormat::RGBX || *m_format == VideoPixelFormat::BGRX); }
 
 private:
-    static Ref<WebCodecsVideoFrame> initializeFrameFromOtherFrame(Ref<WebCodecsVideoFrame>&&, Init&&);
-    static Ref<WebCodecsVideoFrame> initializeFrameFromOtherFrame(Ref<VideoFrame>&&, Init&&);
+    static ExceptionOr<Ref<WebCodecsVideoFrame>> initializeFrameFromOtherFrame(Ref<WebCodecsVideoFrame>&&, Init&&);
+    static ExceptionOr<Ref<WebCodecsVideoFrame>> initializeFrameFromOtherFrame(Ref<VideoFrame>&&, Init&&);
     static ExceptionOr<Ref<WebCodecsVideoFrame>> initializeFrameWithResourceAndSize(Ref<NativeImage>&&, Init&&);
 
     RefPtr<VideoFrame> m_internalFrame;

diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsVideoFrameAlgorithms.cpp b/Source/WebCore/Modules/webcodecs/WebCodecsVideoFrameAlgorithms.cpp
@@ -286,6 +286,41 @@ Ref<VideoColorSpace> videoFramePickColorSpace(const std::optional<VideoColorSpac
     return VideoColorSpace::create({ PlatformVideoColorPrimaries::Bt709, PlatformVideoTransferCharacteristics::Bt709, PlatformVideoMatrixCoefficients::Bt709, false });
 }
 
+// https://w3c.github.io/webcodecs/#validate-videoframeinit
+static bool isNegativeOrNonFinite(double value)
+{
+    return value < 0 || !std::isfinite(value);
+}
+
+bool validateVideoFrameInit(const WebCodecsVideoFrame::Init& init, size_t codedWidth, size_t codedHeight, VideoPixelFormat format)
+{
+    if (init.visibleRect) {
+        auto& visibleRect = *init.visibleRect;
+        if (!verifyRectOffsetAlignment(format, visibleRect))
+            return false;
+
+        if (isNegativeOrNonFinite(visibleRect.x) || isNegativeOrNonFinite(visibleRect.y) || isNegativeOrNonFinite(visibleRect.width) || isNegativeOrNonFinite(visibleRect.height))
+            return false;
+        if (!visibleRect.width || !visibleRect.height)
+            return false;
+
+        if (visibleRect.y + visibleRect.height > codedHeight)
+            return false;
+        if (visibleRect.x + visibleRect.width > codedWidth)
+            return false;
+    }
+
+    if (!codedWidth || !codedHeight)
+        return false;
+
+    if (!!init.displayWidth != !!init.displayHeight)
+        return false;
+    if (init.displayWidth && (!*init.displayWidth || !*init.displayHeight))
+        return false;
+
+    return true;
+}
+
 } // namespace WebCore
 
 #endif // ENABLE(WEB_CODECS)
diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsVideoFrameAlgorithms.h b/Source/WebCore/Modules/webcodecs/WebCodecsVideoFrameAlgorithms.h
@@ -53,6 +53,8 @@ void initializeVisibleRectAndDisplaySize(WebCodecsVideoFrame&, const WebCodecsVi
 
 Ref<VideoColorSpace> videoFramePickColorSpace(const std::optional<VideoColorSpaceInit>&, VideoPixelFormat);
 
+bool validateVideoFrameInit(const WebCodecsVideoFrame::Init&, size_t codedWidth, size_t codedHeight, VideoPixelFormat);
+
 }
 
 #endif
diff --git a/Source/WebCore/platform/VideoPixelFormat.cpp b/Source/WebCore/platform/VideoPixelFormat.cpp
@@ -33,15 +33,15 @@
 
 namespace WebCore {
 
-VideoPixelFormat convertVideoFramePixelFormat(uint32_t format)
+VideoPixelFormat convertVideoFramePixelFormat(uint32_t format, bool shouldDiscardAlpha)
 {
 #if PLATFORM(COCOA)
     if (format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
         return VideoPixelFormat::NV12;
     if (format == kCVPixelFormatType_32BGRA)
-        return VideoPixelFormat::BGRA;
+        return shouldDiscardAlpha ? VideoPixelFormat::BGRX : VideoPixelFormat::BGRA;
     if (format == kCVPixelFormatType_32ARGB)
-        return VideoPixelFormat::RGBA;
+        return shouldDiscardAlpha ? VideoPixelFormat::RGBX : VideoPixelFormat::RGBA;
     ASSERT_NOT_REACHED();
 #else
     UNUSED_PARAM(format);

diff --git a/Source/WebCore/platform/VideoPixelFormat.h b/Source/WebCore/platform/VideoPixelFormat.h
@@ -39,7 +39,7 @@ enum class VideoPixelFormat {
     BGRX
 };
 
-VideoPixelFormat convertVideoFramePixelFormat(uint32_t);
+VideoPixelFormat convertVideoFramePixelFormat(uint32_t, bool shouldDiscardAlpha);
 
 inline bool isRGBVideoPixelFormat(VideoPixelFormat format)
 {