Convert HDR video frames to SDR when drawing to canvas

heycam · heycam · commit adb079825b35 · 2022-09-28T15:39:58.000-07:00
https://bugs.webkit.org/show_bug.cgi?id=240972 rdar://98692644 Reviewed by Jer Noble. Core Animation has a bug with accelerated rendering where drawing an image with HDR color space metadata does not correctly tone map the image. To work around this, we perform our own color space conversion of the image to the destination color space in this case. By using CGIOSurfaceContextGetSurface, we limit this workaround to CGContexts that are backed by an IOSurface (and so use accelerated drawing). Doing this color space conversion inside GraphicsContextCG::drawNativeImage means that we'll do it each time the image is drawn. Ideally this would only be done once for a given { CGImage, DestinationColorSpace } pair. * LayoutTests/fast/canvas/canvas-drawImage-hdr-video-expected.txt: Added. * LayoutTests/fast/canvas/canvas-drawImage-hdr-video.html: Added. * LayoutTests/fast/canvas/resources/hdr.mp4: Added. * LayoutTests/platform/gtk/fast/canvas/canvas-drawImage-hdr-video-expected.txt: Added. * Source/WTF/wtf/PlatformHave.h: * Source/WebCore/PAL/pal/spi/cg/CoreGraphicsSPI.h: * Source/WebCore/platform/graphics/cg/GraphicsContextCG.cpp: (WebCore::GraphicsContextCG::drawNativeImage): * Source/WebCore/platform/graphics/cg/GraphicsContextCG.h: * Source/WebCore/platform/graphics/cocoa/GraphicsContextCocoa.mm: (WebCore::GraphicsContextCG::convertToDestinationColorSpaceIfNeeded): * Source/WebCore/platform/graphics/cocoa/IOSurface.h: * Source/WebCore/platform/graphics/cocoa/IOSurface.mm: (WebCore::IOSurface::bitmapConfiguration const): (WebCore::IOSurface::createCompatibleBitmap): (WebCore::IOSurface::ensurePlatformContext): * Source/WebCore/platform/graphics/win/GraphicsContextCGWin.cpp: (WebCore::GraphicsContextCG::convertToDestinationColorSpaceIfNeeded): Canonical link: https://commits.webkit.org/254973@main
diff --git a/LayoutTests/fast/canvas/canvas-drawImage-hdr-video-expected.txt b/LayoutTests/fast/canvas/canvas-drawImage-hdr-video-expected.txt
@@ -0,0 +1 @@
+Pixel at 1920,540: 0,174,0,255
diff --git a/LayoutTests/fast/canvas/canvas-drawImage-hdr-video.html b/LayoutTests/fast/canvas/canvas-drawImage-hdr-video.html
@@ -0,0 +1,32 @@
+<!DOCTYPE html>
+<script src="../../resources/js-test-pre.js"></script>
+<script src="../../media/utilities.js"></script>
+<pre id=log></pre>
+<script>
+if (window.testRunner) {
+    testRunner.dumpAsText();
+    testRunner.waitUntilDone();
+}
+
+let video = document.createElement("video");
+
+waitForVideoFrame(video, async function() {
+    let canvas = document.createElement("canvas");
+    canvas.width = 3840;
+    canvas.height = 2160;
+
+    let ctx = canvas.getContext("2d");
+    ctx.drawImage(video, 0, 0);
+
+    let x = 1920;
+    let y = 540;
+
+    let data = ctx.getImageData(x, y, 1, 1).data;
+    log.textContent = `Pixel at ${x},${y}: ${[...data]}`;
+
+    if (window.testRunner)
+        testRunner.notifyDone();
+});
+
+video.src = "resources/hdr.mp4";
+</script>
diff --git a/LayoutTests/fast/canvas/resources/hdr.mp4 b/LayoutTests/fast/canvas/resources/hdr.mp4
diff --git a/LayoutTests/platform/gtk/fast/canvas/canvas-drawImage-hdr-video-expected.txt b/LayoutTests/platform/gtk/fast/canvas/canvas-drawImage-hdr-video-expected.txt
@@ -0,0 +1 @@
+Pixel at 1920,540: 16,223,6,255
diff --git a/LayoutTests/platform/win/TestExpectations b/LayoutTests/platform/win/TestExpectations
@@ -3669,6 +3669,7 @@ editing/selection/character-granularity-rect.html [ Skip ]
 editing/selection/ios [ Skip ]
 fast/canvas/canvas-createPattern-video-loading.html [ Skip ]
 fast/canvas/canvas-createPattern-video-modify.html [ Skip ]
+fast/canvas/canvas-drawImage-hdr-video.html [ Skip ]
 fast/events/before-input-prevent-insert-replacement.html [ Skip ]
 fast/events/input-event-insert-replacement.html [ Skip ]
 fast/forms/scroll-into-view-and-show-validation-message.html [ Skip ]
diff --git a/Source/WTF/wtf/PlatformHave.h b/Source/WTF/wtf/PlatformHave.h
@@ -728,6 +728,7 @@
 #define HAVE_WEBP 1
 #define HAVE_IMAGEIO_FIX_FOR_RADAR_59589723 1
 #define HAVE_CGS_FIX_FOR_RADAR_97530095 0
+#define HAVE_CORE_ANIMATION_FIX_FOR_RADAR_93560567 0
 #endif
 
 #if PLATFORM(COCOA)
diff --git a/Source/WebCore/PAL/pal/spi/cg/CoreGraphicsSPI.h b/Source/WebCore/PAL/pal/spi/cg/CoreGraphicsSPI.h
@@ -307,6 +307,7 @@ CGImageRef CGIOSurfaceContextCreateImage(CGContextRef);
 CGImageRef CGIOSurfaceContextCreateImageReference(CGContextRef);
 CGColorSpaceRef CGIOSurfaceContextGetColorSpace(CGContextRef);
 void CGIOSurfaceContextSetDisplayMask(CGContextRef, uint32_t mask);
+IOSurfaceRef CGIOSurfaceContextGetSurface(CGContextRef);
 #endif // HAVE(IOSURFACE)
 
 #if PLATFORM(COCOA)
diff --git a/Source/WebCore/platform/graphics/cg/GraphicsContextCG.cpp b/Source/WebCore/platform/graphics/cg/GraphicsContextCG.cpp
@@ -285,6 +285,8 @@ void GraphicsContextCG::drawNativeImage(NativeImage& nativeImage, const FloatSiz
     CGContextStateSaver stateSaver(context, false);
     auto transform = CGContextGetCTM(context);
 
+    convertToDestinationColorSpaceIfNeeded(image);
+
     auto subImage = image;
     auto currentImageSize = imageLogicalSize(image.get(), options);
 
diff --git a/Source/WebCore/platform/graphics/cg/GraphicsContextCG.h b/Source/WebCore/platform/graphics/cg/GraphicsContextCG.h
@@ -136,6 +136,8 @@ class WEBCORE_EXPORT GraphicsContextCG : public GraphicsContext {
 #endif
 
 private:
+    void convertToDestinationColorSpaceIfNeeded(RetainPtr<CGImageRef>&);
+
     GraphicsContextPlatformPrivate* m_data { nullptr };
 };
 
diff --git a/Source/WebCore/platform/graphics/cocoa/GraphicsContextCocoa.mm b/Source/WebCore/platform/graphics/cocoa/GraphicsContextCocoa.mm
@@ -29,6 +29,7 @@
 #import "DisplayListRecorder.h"
 #import "GraphicsContextCG.h"
 #import "GraphicsContextPlatformPrivateCG.h"
+#import "IOSurface.h"
 #import "IntRect.h"
 #import <pal/spi/cg/CoreGraphicsSPI.h>
 #import <pal/spi/cocoa/FeatureFlagsSPI.h>
@@ -224,4 +225,34 @@ static inline void setPatternPhaseInUserSpace(CGContextRef context, CGPoint phas
     CGContextFillPath(platformContext);
 }
 
+void GraphicsContextCG::convertToDestinationColorSpaceIfNeeded(RetainPtr<CGImageRef>& image)
+{
+#if HAVE(CORE_ANIMATION_FIX_FOR_RADAR_93560567)
+    UNUSED_PARAM(image);
+#else
+    if (!CGColorSpaceUsesITUR_2100TF(CGImageGetColorSpace(image.get())))
+        return;
+
+    auto context = platformContext();
+
+    auto destinationSurface = CGIOSurfaceContextGetSurface(context);
+    if (!destinationSurface)
+        return;
+
+    auto destinationColorSpace = CGIOSurfaceContextGetColorSpace(context);
+    if (!destinationColorSpace)
+        return;
+
+    auto surface = IOSurface::createFromSurface(destinationSurface, DestinationColorSpace(destinationColorSpace));
+
+    auto width = CGImageGetWidth(image.get());
+    auto height = CGImageGetHeight(image.get());
+
+    auto bitmapContext = surface->createCompatibleBitmap(width, height);
+    CGContextDrawImage(bitmapContext.get(), CGRectMake(0, 0, width, height), image.get());
+
+    image = adoptCF(CGBitmapContextCreateImage(bitmapContext.get()));
+#endif
+}
+
 } // namespace WebCore
diff --git a/Source/WebCore/platform/graphics/cocoa/IOSurface.h b/Source/WebCore/platform/graphics/cocoa/IOSurface.h
@@ -164,10 +164,19 @@ class IOSurface final {
 
     void migrateColorSpaceToProperties();
 
+    RetainPtr<CGContextRef> createCompatibleBitmap(unsigned width, unsigned height);
+
 private:
     IOSurface(IntSize, const DestinationColorSpace&, Format, bool& success);
     IOSurface(IOSurfaceRef, const DestinationColorSpace&);
 
+    struct BitmapConfiguration {
+        CGBitmapInfo bitmapInfo;
+        size_t bitsPerComponent;
+    };
+
+    BitmapConfiguration bitmapConfiguration() const;
+
     DestinationColorSpace m_colorSpace;
     IntSize m_size;
     size_t m_totalBytes;
diff --git a/Source/WebCore/platform/graphics/cocoa/IOSurface.mm b/Source/WebCore/platform/graphics/cocoa/IOSurface.mm
@@ -315,15 +315,11 @@ static IntSize computeMaximumSurfaceSize()
     return adoptCF(CGIOSurfaceContextCreateImageReference(surface->ensurePlatformContext()));
 }
 
-CGContextRef IOSurface::ensurePlatformContext(const HostWindow* hostWindow)
+IOSurface::BitmapConfiguration IOSurface::bitmapConfiguration() const
 {
-    if (m_cgContext)
-        return m_cgContext.get();
-
-    CGBitmapInfo bitmapInfo = static_cast<CGBitmapInfo>(kCGImageAlphaPremultipliedFirst) | static_cast<CGBitmapInfo>(kCGBitmapByteOrder32Host);
+    auto bitmapInfo = static_cast<CGBitmapInfo>(kCGImageAlphaPremultipliedFirst) | static_cast<CGBitmapInfo>(kCGBitmapByteOrder32Host);
 
     size_t bitsPerComponent = 8;
-    size_t bitsPerPixel = 32;
     
     switch (format()) {
     case Format::BGRA:
@@ -334,16 +330,35 @@ static IntSize computeMaximumSurfaceSize()
         // A half-float format will be used if CG needs to read back the IOSurface contents,
         // but for an IOSurface-to-IOSurface copy, there should be no conversion.
         bitsPerComponent = 16;
-        bitsPerPixel = 64;
         bitmapInfo = static_cast<CGBitmapInfo>(kCGImageAlphaPremultipliedLast) | static_cast<CGBitmapInfo>(kCGBitmapByteOrder16Host) | static_cast<CGBitmapInfo>(kCGBitmapFloatComponents);
         break;
 #endif
     case Format::YUV422:
         ASSERT_NOT_REACHED();
         break;
     }
-    
-    m_cgContext = adoptCF(CGIOSurfaceContextCreate(m_surface.get(), m_size.width(), m_size.height(), bitsPerComponent, bitsPerPixel, m_colorSpace.platformColorSpace(), bitmapInfo));
+
+    return { bitmapInfo, bitsPerComponent };
+}
+
+RetainPtr<CGContextRef> IOSurface::createCompatibleBitmap(unsigned width, unsigned height)
+{
+    auto configuration = bitmapConfiguration();
+    auto bitsPerPixel = configuration.bitsPerComponent * 4;
+    auto bytesPerRow = width * bitsPerPixel;
+
+    return adoptCF(CGBitmapContextCreate(NULL, width, height, configuration.bitsPerComponent, bytesPerRow, m_colorSpace.platformColorSpace(), configuration.bitmapInfo));
+}
+
+CGContextRef IOSurface::ensurePlatformContext(const HostWindow* hostWindow)
+{
+    if (m_cgContext)
+        return m_cgContext.get();
+
+    auto configuration = bitmapConfiguration();
+    auto bitsPerPixel = configuration.bitsPerComponent * 4;
+
+    m_cgContext = adoptCF(CGIOSurfaceContextCreate(m_surface.get(), m_size.width(), m_size.height(), configuration.bitsPerComponent, bitsPerPixel, m_colorSpace.platformColorSpace(), configuration.bitmapInfo));
 
 #if PLATFORM(MAC)
     if (auto displayMask = primaryOpenGLDisplayMask()) {
diff --git a/Source/WebCore/platform/graphics/win/GraphicsContextCGWin.cpp b/Source/WebCore/platform/graphics/win/GraphicsContextCGWin.cpp
@@ -219,5 +219,9 @@ GraphicsContextPlatformPrivate* GraphicsContextCG::deprecatedPrivateContext() co
     return m_data;
 }
 
+void GraphicsContextCG::convertToDestinationColorSpaceIfNeeded(RetainPtr<CGImageRef>&)
+{
+}
+
 }
 #endif
