Skip to content
Permalink
Browse files
Convert HDR video frames to SDR when drawing to canvas
https://bugs.webkit.org/show_bug.cgi?id=240972
rdar://98692644

Reviewed by Jer Noble.

Core Animation has a bug with accelerated rendering where drawing an
image with HDR color space metadata does not correctly tone map the
image. To work around this, we perform our own color space conversion of
the image to the destination color space in this case.

By using CGIOSurfaceContextGetSurface, we limit this workaround to
CGContexts that are backed by an IOSurface (and so use accelerated
drawing).

Doing this color space conversion inside
GraphicsContextCG::drawNativeImage means that we'll do it each time the
image is drawn. Ideally this would only be done once for a given
{ CGImage, DestinationColorSpace } pair.

* LayoutTests/fast/canvas/canvas-drawImage-hdr-video-expected.txt: Added.
* LayoutTests/fast/canvas/canvas-drawImage-hdr-video.html: Added.
* LayoutTests/fast/canvas/resources/hdr.mp4: Added.
* LayoutTests/platform/gtk/fast/canvas/canvas-drawImage-hdr-video-expected.txt: Added.
* Source/WTF/wtf/PlatformHave.h:
* Source/WebCore/PAL/pal/spi/cg/CoreGraphicsSPI.h:
* Source/WebCore/platform/graphics/cg/GraphicsContextCG.cpp:
(WebCore::GraphicsContextCG::drawNativeImage):
* Source/WebCore/platform/graphics/cg/GraphicsContextCG.h:
* Source/WebCore/platform/graphics/cocoa/GraphicsContextCocoa.mm:
(WebCore::GraphicsContextCG::convertToDestinationColorSpaceIfNeeded):
* Source/WebCore/platform/graphics/cocoa/IOSurface.h:
* Source/WebCore/platform/graphics/cocoa/IOSurface.mm:
(WebCore::IOSurface::bitmapConfiguration const):
(WebCore::IOSurface::createCompatibleBitmap):
(WebCore::IOSurface::ensurePlatformContext):
* Source/WebCore/platform/graphics/win/GraphicsContextCGWin.cpp:
(WebCore::GraphicsContextCG::convertToDestinationColorSpaceIfNeeded):

Canonical link: https://commits.webkit.org/254973@main
  • Loading branch information
heycam committed Sep 28, 2022
1 parent 2daf672 commit adb079825b35c1545c48298fec71d2c514884104
Show file tree
Hide file tree
Showing 13 changed files with 109 additions and 9 deletions.
@@ -0,0 +1 @@
Pixel at 1920,540: 0,174,0,255
@@ -0,0 +1,32 @@
<!DOCTYPE html>
<script src="../../resources/js-test-pre.js"></script>
<script src="../../media/utilities.js"></script>
<pre id=log></pre>
<script>
if (window.testRunner) {
testRunner.dumpAsText();
testRunner.waitUntilDone();
}

let video = document.createElement("video");

waitForVideoFrame(video, async function() {
let canvas = document.createElement("canvas");
canvas.width = 3840;
canvas.height = 2160;

let ctx = canvas.getContext("2d");
ctx.drawImage(video, 0, 0);

let x = 1920;
let y = 540;

let data = ctx.getImageData(x, y, 1, 1).data;
log.textContent = `Pixel at ${x},${y}: ${[...data]}`;

if (window.testRunner)
testRunner.notifyDone();
});

video.src = "resources/hdr.mp4";
</script>
Binary file not shown.
@@ -0,0 +1 @@
Pixel at 1920,540: 16,223,6,255
@@ -3669,6 +3669,7 @@ editing/selection/character-granularity-rect.html [ Skip ]
editing/selection/ios [ Skip ]
fast/canvas/canvas-createPattern-video-loading.html [ Skip ]
fast/canvas/canvas-createPattern-video-modify.html [ Skip ]
fast/canvas/canvas-drawImage-hdr-video.html [ Skip ]
fast/events/before-input-prevent-insert-replacement.html [ Skip ]
fast/events/input-event-insert-replacement.html [ Skip ]
fast/forms/scroll-into-view-and-show-validation-message.html [ Skip ]
@@ -728,6 +728,7 @@
#define HAVE_WEBP 1
#define HAVE_IMAGEIO_FIX_FOR_RADAR_59589723 1
#define HAVE_CGS_FIX_FOR_RADAR_97530095 0
#define HAVE_CORE_ANIMATION_FIX_FOR_RADAR_93560567 0
#endif

#if PLATFORM(COCOA)
@@ -307,6 +307,7 @@ CGImageRef CGIOSurfaceContextCreateImage(CGContextRef);
CGImageRef CGIOSurfaceContextCreateImageReference(CGContextRef);
CGColorSpaceRef CGIOSurfaceContextGetColorSpace(CGContextRef);
void CGIOSurfaceContextSetDisplayMask(CGContextRef, uint32_t mask);
IOSurfaceRef CGIOSurfaceContextGetSurface(CGContextRef);
#endif // HAVE(IOSURFACE)

#if PLATFORM(COCOA)
@@ -285,6 +285,8 @@ void GraphicsContextCG::drawNativeImage(NativeImage& nativeImage, const FloatSiz
CGContextStateSaver stateSaver(context, false);
auto transform = CGContextGetCTM(context);

convertToDestinationColorSpaceIfNeeded(image);

auto subImage = image;
auto currentImageSize = imageLogicalSize(image.get(), options);

@@ -136,6 +136,8 @@ class WEBCORE_EXPORT GraphicsContextCG : public GraphicsContext {
#endif

private:
void convertToDestinationColorSpaceIfNeeded(RetainPtr<CGImageRef>&);

GraphicsContextPlatformPrivate* m_data { nullptr };
};

@@ -29,6 +29,7 @@
#import "DisplayListRecorder.h"
#import "GraphicsContextCG.h"
#import "GraphicsContextPlatformPrivateCG.h"
#import "IOSurface.h"
#import "IntRect.h"
#import <pal/spi/cg/CoreGraphicsSPI.h>
#import <pal/spi/cocoa/FeatureFlagsSPI.h>
@@ -224,4 +225,34 @@ static inline void setPatternPhaseInUserSpace(CGContextRef context, CGPoint phas
CGContextFillPath(platformContext);
}

void GraphicsContextCG::convertToDestinationColorSpaceIfNeeded(RetainPtr<CGImageRef>& image)
{
#if HAVE(CORE_ANIMATION_FIX_FOR_RADAR_93560567)
UNUSED_PARAM(image);
#else
if (!CGColorSpaceUsesITUR_2100TF(CGImageGetColorSpace(image.get())))
return;

auto context = platformContext();

auto destinationSurface = CGIOSurfaceContextGetSurface(context);
if (!destinationSurface)
return;

auto destinationColorSpace = CGIOSurfaceContextGetColorSpace(context);
if (!destinationColorSpace)
return;

auto surface = IOSurface::createFromSurface(destinationSurface, DestinationColorSpace(destinationColorSpace));

auto width = CGImageGetWidth(image.get());
auto height = CGImageGetHeight(image.get());

auto bitmapContext = surface->createCompatibleBitmap(width, height);
CGContextDrawImage(bitmapContext.get(), CGRectMake(0, 0, width, height), image.get());

image = adoptCF(CGBitmapContextCreateImage(bitmapContext.get()));
#endif
}

} // namespace WebCore
@@ -164,10 +164,19 @@ class IOSurface final {

void migrateColorSpaceToProperties();

RetainPtr<CGContextRef> createCompatibleBitmap(unsigned width, unsigned height);

private:
IOSurface(IntSize, const DestinationColorSpace&, Format, bool& success);
IOSurface(IOSurfaceRef, const DestinationColorSpace&);

struct BitmapConfiguration {
CGBitmapInfo bitmapInfo;
size_t bitsPerComponent;
};

BitmapConfiguration bitmapConfiguration() const;

DestinationColorSpace m_colorSpace;
IntSize m_size;
size_t m_totalBytes;
@@ -315,15 +315,11 @@ static IntSize computeMaximumSurfaceSize()
return adoptCF(CGIOSurfaceContextCreateImageReference(surface->ensurePlatformContext()));
}

CGContextRef IOSurface::ensurePlatformContext(const HostWindow* hostWindow)
IOSurface::BitmapConfiguration IOSurface::bitmapConfiguration() const
{
if (m_cgContext)
return m_cgContext.get();

CGBitmapInfo bitmapInfo = static_cast<CGBitmapInfo>(kCGImageAlphaPremultipliedFirst) | static_cast<CGBitmapInfo>(kCGBitmapByteOrder32Host);
auto bitmapInfo = static_cast<CGBitmapInfo>(kCGImageAlphaPremultipliedFirst) | static_cast<CGBitmapInfo>(kCGBitmapByteOrder32Host);

size_t bitsPerComponent = 8;
size_t bitsPerPixel = 32;

switch (format()) {
case Format::BGRA:
@@ -334,16 +330,35 @@ static IntSize computeMaximumSurfaceSize()
// A half-float format will be used if CG needs to read back the IOSurface contents,
// but for an IOSurface-to-IOSurface copy, there should be no conversion.
bitsPerComponent = 16;
bitsPerPixel = 64;
bitmapInfo = static_cast<CGBitmapInfo>(kCGImageAlphaPremultipliedLast) | static_cast<CGBitmapInfo>(kCGBitmapByteOrder16Host) | static_cast<CGBitmapInfo>(kCGBitmapFloatComponents);
break;
#endif
case Format::YUV422:
ASSERT_NOT_REACHED();
break;
}

m_cgContext = adoptCF(CGIOSurfaceContextCreate(m_surface.get(), m_size.width(), m_size.height(), bitsPerComponent, bitsPerPixel, m_colorSpace.platformColorSpace(), bitmapInfo));

return { bitmapInfo, bitsPerComponent };
}

RetainPtr<CGContextRef> IOSurface::createCompatibleBitmap(unsigned width, unsigned height)
{
auto configuration = bitmapConfiguration();
auto bitsPerPixel = configuration.bitsPerComponent * 4;
auto bytesPerRow = width * bitsPerPixel;

return adoptCF(CGBitmapContextCreate(NULL, width, height, configuration.bitsPerComponent, bytesPerRow, m_colorSpace.platformColorSpace(), configuration.bitmapInfo));
}

CGContextRef IOSurface::ensurePlatformContext(const HostWindow* hostWindow)
{
if (m_cgContext)
return m_cgContext.get();

auto configuration = bitmapConfiguration();
auto bitsPerPixel = configuration.bitsPerComponent * 4;

m_cgContext = adoptCF(CGIOSurfaceContextCreate(m_surface.get(), m_size.width(), m_size.height(), configuration.bitsPerComponent, bitsPerPixel, m_colorSpace.platformColorSpace(), configuration.bitmapInfo));

#if PLATFORM(MAC)
if (auto displayMask = primaryOpenGLDisplayMask()) {
@@ -219,5 +219,9 @@ GraphicsContextPlatformPrivate* GraphicsContextCG::deprecatedPrivateContext() co
return m_data;
}

void GraphicsContextCG::convertToDestinationColorSpaceIfNeeded(RetainPtr<CGImageRef>&)
{
}

}
#endif

0 comments on commit adb0798

Please sign in to comment.