[GPU Process] To encode a CGImage, copy its pixels to a SharedMemory

https://bugs.webkit.org/show_bug.cgi?id=254794 rdar://106794138 Reviewed by Simon Fraser. Use CGImageGetDataProvider() to get the CGImage bytes. Copy these bytes to the SharedMemory. Encode some of the CGImage properties so the decoded CGImage matches the encoded one. These properties are BytesPerPixel, BytesPerRow and BitmapInfo Make sure WebProcess can create a PlatformImage out of the ShareableBitmap before sending the pixels to GPUProcess. Otherwise fall back to the old code path and draw the image to a BitmapContext backed by the SharedMemory. * Source/WebKit/Platform/Logging.h: * Source/WebKit/Shared/ShareableBitmap.cpp: (WebKit::ShareableBitmap::create): (WebKit::ShareableBitmap::createFromImageDraw): * Source/WebKit/Shared/ShareableBitmap.h: * Source/WebKit/Shared/cg/ShareableBitmapCG.cpp: (WebKit::ShareableBitmapConfiguration::ShareableBitmapConfiguration): (WebKit::ShareableBitmap::createFromImagePixels): * Source/WebKit/WebProcess/GPU/graphics/RemoteResourceCacheProxy.cpp: (WebKit::createShareableBitmapFromNativeImage): (WebKit::RemoteResourceCacheProxy::recordNativeImageUse): Canonical link: https://commits.webkit.org/262607@main
WebKit · Apr 5, 2023 · c50a18b · c50a18b
1 parent 22b8833
commit c50a18b
Show file tree

Hide file tree

Showing 5 changed files with 106 additions and 34 deletions.
diff --git a/Source/WebKit/Platform/Logging.h b/Source/WebKit/Platform/Logging.h
@@ -60,6 +60,7 @@ extern "C" {
     M(IPCMessages) \
     M(ITPDebug) \
     M(IconDatabase) \
+    M(Images) \
     M(ImageAnalysis) \
     M(IncrementalPDF) \
     M(IncrementalPDFVerbose) \

diff --git a/Source/WebKit/Shared/ShareableBitmap.cpp b/Source/WebKit/Shared/ShareableBitmap.cpp
@@ -75,11 +75,11 @@ CheckedUint32 ShareableBitmapConfiguration::calculateSizeInBytes(const IntSize&
 
 RefPtr<ShareableBitmap> ShareableBitmap::create(const ShareableBitmapConfiguration& configuration)
 {
-    auto numBytes = configuration.sizeInBytes();
-    if (numBytes.hasOverflowed())
+    auto sizeInBytes = configuration.sizeInBytes();
+    if (sizeInBytes.hasOverflowed())
         return nullptr;
 
-    RefPtr<SharedMemory> sharedMemory = SharedMemory::allocate(numBytes);
+    RefPtr<SharedMemory> sharedMemory = SharedMemory::allocate(sizeInBytes);
     if (!sharedMemory)
         return nullptr;
 
@@ -88,18 +88,34 @@ RefPtr<ShareableBitmap> ShareableBitmap::create(const ShareableBitmapConfigurati
 
 RefPtr<ShareableBitmap> ShareableBitmap::create(const ShareableBitmapConfiguration& configuration, Ref<SharedMemory>&& sharedMemory)
 {
-    auto numBytes = configuration.sizeInBytes();
-    if (numBytes.hasOverflowed())
+    auto sizeInBytes = configuration.sizeInBytes();
+    if (sizeInBytes.hasOverflowed())
         return nullptr;
 
-    if (sharedMemory->size() < numBytes) {
+    if (sharedMemory->size() < sizeInBytes) {
         ASSERT_NOT_REACHED();
         return nullptr;
     }
 
     return adoptRef(new ShareableBitmap(configuration, WTFMove(sharedMemory)));
 }
 
+RefPtr<ShareableBitmap> ShareableBitmap::createFromImageDraw(NativeImage& image)
+{
+    auto imageSize = image.size();
+
+    auto bitmap = ShareableBitmap::create({ imageSize, image.colorSpace() });
+    if (!bitmap)
+        return nullptr;
+
+    auto context = bitmap->createGraphicsContext();
+    if (!context)
+        return nullptr;
+
+    context->drawNativeImage(image, imageSize, FloatRect({ }, imageSize), FloatRect({ }, imageSize), { CompositeOperator::Copy });
+    return bitmap;
+}
+
 RefPtr<ShareableBitmap> ShareableBitmap::create(const ShareableBitmapHandle& handle, SharedMemory::Protection protection)
 {
     auto sharedMemory = SharedMemory::map(handle.m_handle, protection);

diff --git a/Source/WebKit/Shared/ShareableBitmap.h b/Source/WebKit/Shared/ShareableBitmap.h
@@ -52,6 +52,9 @@ class ShareableBitmapConfiguration {
         , CGBitmapInfo
 #endif
     );
+#if USE(CG)
+    ShareableBitmapConfiguration(WebCore::NativeImage&);
+#endif
 
     WebCore::IntSize size() const { return m_size; }
     const WebCore::DestinationColorSpace& colorSpace() const { return m_colorSpace ? *m_colorSpace : WebCore::DestinationColorSpace::SRGB(); }
@@ -122,6 +125,12 @@ class ShareableBitmap : public ThreadSafeRefCounted<ShareableBitmap> {
     // Create a shareable bitmap from an already existing shared memory block.
     static RefPtr<ShareableBitmap> create(const ShareableBitmapConfiguration&, Ref<SharedMemory>&&);
 
+    // Create a shareable bitmap from a NativeImage.
+#if USE(CG)
+    static RefPtr<ShareableBitmap> createFromImagePixels(WebCore::NativeImage&);
+#endif
+    static RefPtr<ShareableBitmap> createFromImageDraw(WebCore::NativeImage&);
+
     // Create a shareable bitmap from a handle.
     static RefPtr<ShareableBitmap> create(const ShareableBitmapHandle&, SharedMemory::Protection = SharedMemory::Protection::ReadWrite);
 

diff --git a/Source/WebKit/Shared/cg/ShareableBitmapCG.cpp b/Source/WebKit/Shared/cg/ShareableBitmapCG.cpp
@@ -39,6 +39,15 @@
 namespace WebKit {
 using namespace WebCore;
 
+ShareableBitmapConfiguration::ShareableBitmapConfiguration(NativeImage& image)
+    : m_size(image.size())
+    , m_colorSpace(image.colorSpace())
+    , m_bytesPerPixel(CGImageGetBitsPerPixel(image.platformImage().get()) / 8)
+    , m_bytesPerRow(CGImageGetBytesPerRow(image.platformImage().get()))
+    , m_bitmapInfo(CGImageGetBitmapInfo(image.platformImage().get()))
+{
+}
+
 std::optional<DestinationColorSpace> ShareableBitmapConfiguration::validateColorSpace(std::optional<DestinationColorSpace> colorSpace)
 {
     if (!colorSpace)
@@ -99,6 +108,30 @@ CGBitmapInfo ShareableBitmapConfiguration::calculateBitmapInfo(const Destination
     return info;
 }
 
+RefPtr<ShareableBitmap> ShareableBitmap::createFromImagePixels(NativeImage& image)
+{
+    auto colorSpace = image.colorSpace();
+    if (colorSpace != DestinationColorSpace::SRGB())
+        return nullptr;
+
+    auto pixels = adoptCF(CGDataProviderCopyData(CGImageGetDataProvider(image.platformImage().get())));
+    if (!pixels)
+        return nullptr;
+
+    const auto* bytes = reinterpret_cast<const uint8_t*>(CFDataGetBytePtr(pixels.get()));
+    auto sizeInBytes = CFDataGetLength(pixels.get());
+    if (!bytes || !sizeInBytes)
+        return nullptr;
+
+    RefPtr<SharedMemory> sharedMemory = SharedMemory::allocate(sizeInBytes);
+    if (!sharedMemory)
+        return nullptr;
+
+    memcpy(sharedMemory->data(), bytes, sizeInBytes);
+
+    return adoptRef(new ShareableBitmap(ShareableBitmapConfiguration(image), sharedMemory.releaseNonNull()));
+}
+
 std::unique_ptr<GraphicsContext> ShareableBitmap::createGraphicsContext()
 {
     unsigned bitsPerComponent = m_configuration.bytesPerPixel() * 8 / 4;

diff --git a/Source/WebKit/WebProcess/GPU/graphics/RemoteResourceCacheProxy.cpp b/Source/WebKit/WebProcess/GPU/graphics/RemoteResourceCacheProxy.cpp
@@ -29,6 +29,7 @@
 #if ENABLE(GPU_PROCESS)
 
 #include "ArgumentCoders.h"
+#include "Logging.h"
 #include "RemoteImageBufferProxy.h"
 #include "RemoteRenderingBackendProxy.h"
 #include <WebCore/FontCustomPlatformData.h>
@@ -85,26 +86,42 @@ void RemoteResourceCacheProxy::forgetImageBuffer(RenderingResourceIdentifier ide
     ASSERT_UNUSED(success, success);
 }
 
-inline static RefPtr<ShareableBitmap> createShareableBitmapFromNativeImage(NativeImage& image)
+void RemoteResourceCacheProxy::recordImageBufferUse(WebCore::ImageBuffer& imageBuffer)
 {
-    auto imageSize = image.size();
+    auto iterator = m_imageBuffers.find(imageBuffer.renderingResourceIdentifier());
+    ASSERT_UNUSED(iterator, iterator != m_imageBuffers.end());
+}
 
-    auto bitmap = ShareableBitmap::create({ image.size(), image.colorSpace() });
-    if (!bitmap)
-        return nullptr;
+inline static std::optional<ShareableBitmapHandle> createShareableBitmapFromNativeImage(NativeImage& image)
+{
+    RefPtr<ShareableBitmap> bitmap;
+    PlatformImagePtr platformImage;
 
-    auto context = bitmap->createGraphicsContext();
-    if (!context)
-        return nullptr;
+#if USE(CG)
+    bitmap = ShareableBitmap::createFromImagePixels(image);
+    if (bitmap)
+        platformImage = bitmap->createPlatformImage(DontCopyBackingStore, ShouldInterpolate::Yes);
+#endif
 
-    context->drawNativeImage(image, imageSize, FloatRect({ }, imageSize), FloatRect({ }, imageSize), { WebCore::CompositeOperator::Copy });
-    return bitmap;
-}
+    // If we failed to create ShareableBitmap or PlatformImage, fall back to image-draw method.
+    if (!platformImage)
+        bitmap = ShareableBitmap::createFromImageDraw(image);
 
-void RemoteResourceCacheProxy::recordImageBufferUse(WebCore::ImageBuffer& imageBuffer)
-{
-    auto iterator = m_imageBuffers.find(imageBuffer.renderingResourceIdentifier());
-    ASSERT_UNUSED(iterator, iterator != m_imageBuffers.end());
+    if (!platformImage && bitmap)
+        platformImage = bitmap->createPlatformImage(DontCopyBackingStore, ShouldInterpolate::Yes);
+
+    if (!platformImage)
+        return std::nullopt;
+
+    auto handle = bitmap->createHandle();
+    if (!handle)
+        return std::nullopt;
+
+    handle->takeOwnershipOfMemory(MemoryLedger::Graphics);
+
+    // Replace the PlatformImage of the input NativeImage with the shared one.
+    image.setPlatformImage(WTFMove(platformImage));
+    return handle;
 }
 
 void RemoteResourceCacheProxy::recordNativeImageUse(NativeImage& image)
@@ -115,20 +132,16 @@ void RemoteResourceCacheProxy::recordNativeImageUse(NativeImage& image)
     if (iterator != m_nativeImages.end())
         return;
 
-    auto bitmap = createShareableBitmapFromNativeImage(image);
-    if (!bitmap)
-        return;
-
-    auto handle = bitmap->createHandle();
-    if (!handle)
+    auto handle = createShareableBitmapFromNativeImage(image);
+    if (!handle) {
+        // FIXME: Failing to send the image to GPUP will crash it when referencing this image.
+        LOG_WITH_STREAM(Images, stream
+            << "RemoteResourceCacheProxy::recordNativeImageUse() " << this
+            << " image.size(): " << image.size()
+            << " image.colorSpace(): " << image.colorSpace()
+            << " ShareableBitmap could not be created; bailing.");
         return;
-
-    auto platformImage = bitmap->createPlatformImage(DontCopyBackingStore, ShouldInterpolate::Yes);
-    if (!platformImage)
-        return;
-
-    image.setPlatformImage(WTFMove(platformImage));
-    handle->takeOwnershipOfMemory(MemoryLedger::Graphics);
+    }
 
     m_nativeImages.add(image.renderingResourceIdentifier(), image);