Skip to content

Commit

Permalink
Cherry-pick 9084db41d592. rdar://problem/107371244
Browse files Browse the repository at this point in the history 
    Refactor Canvas noise injection logic and improve support for gradients in post-processed ImageData
    https://bugs.webkit.org/show_bug.cgi?id=255993
    rdar://107371244

    Reviewed by Kimmo Kinnunen.

    Gradients present an interesting problem when post-processing canvas ImageData
    because tweaking colors result in significant visual anomalies. This change
    introduces some naive gradient detection by looking at all of the immediate
    neighbors of a pixel. We decide that a pixel is within a gradient if the
    current color is between its opposing adjacent colors (e.g., above and below,
    left and right, etc). If a color is decided to be within a gradient, then we
    set those adjacent colors as bounds within which we can tweak the current
    color.

    In addition, within this change, we move the noise injection logic into its own
    class, it removes the dependency on supplied colors for post-processing, and it
    reduces the magnitude of noise from ~5% to ~1%.

    * Source/WebCore/Headers.cmake:
    * Source/WebCore/Sources.txt:
    * Source/WebCore/WebCore.xcodeproj/project.pbxproj:
    * Source/WebCore/html/CanvasBase.cpp:
    (WebCore::CanvasBase::makeRenderingResultsAvailable):
    (WebCore::CanvasBase::didDraw): We clip the rect before passing it into
    CanvasNoiseInjection, and if the rect is std::nullopt then we pass in the
    entire canvas.

    (WebCore::CanvasBase::postProcessPixelBufferResults const):
    (WebCore::CanvasBase::postProcessDirtyCanvasBuffer const): Deleted.
    * Source/WebCore/html/CanvasBase.h:
    * Source/WebCore/html/CanvasNoiseInjection.cpp: Added.
    (WebCore::CanvasNoiseInjection::updateDirtyRect):
    (WebCore::isIndexInBounds):
    (WebCore::setTightnessBounds):
    (WebCore::getGradientNeighbors):
    (WebCore::CanvasNoiseInjection::postProcessDirtyCanvasBuffer):
    (WebCore::CanvasNoiseInjection::postProcessPixelBufferResults const):
    * Source/WebCore/html/CanvasNoiseInjection.h: Added.
    * Source/WebCore/html/HTMLCanvasElement.cpp:
    (WebCore::HTMLCanvasElement::getImageData):

    Originally-landed-as: 264019@main (bfc25dc). rdar://107371244

Identifier: 263769.34@safari-7616.1.14.10-branch
  • Loading branch information
@sysrqb
sysrqb authored and Dan Robson committed May 15, 2023
1 parent 6e7d9a1 commit d721aae
Show file tree
Hide file tree
Showing 8 changed files with 303 additions and 91 deletions.
1 change: 1 addition & 0 deletions Source/WebCore/Headers.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -952,6 +952,7 @@ set(WebCore_PRIVATE_FRAMEWORK_HEADERS
html/CachedHTMLCollection.h
html/CachedHTMLCollectionInlines.h
html/CanvasBase.h
html/CanvasNoiseInjection.h
html/CanvasObserver.h
html/CollectionTraversal.h
html/CollectionTraversalInlines.h
Expand Down
1 change: 1 addition & 0 deletions Source/WebCore/Sources.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1258,6 +1258,7 @@ html/BaseDateAndTimeInputType.cpp
html/BaseTextInputType.cpp
html/ButtonInputType.cpp
html/CanvasBase.cpp
html/CanvasNoiseInjection.cpp
html/CheckboxInputType.cpp
html/ColorInputType.cpp
html/CustomPaintCanvas.cpp
Expand Down
6 changes: 6 additions & 0 deletions Source/WebCore/WebCore.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5048,6 +5048,7 @@
D0EDA775143E303C0028E383 /* CachedRawResource.h in Headers */ = {isa = PBXBuildFile; fileRef = D0EDA773143E303C0028E383 /* CachedRawResource.h */; settings = {ATTRIBUTES = (Private, ); }; };
D0FF2A5E11F8C45A007E74E0 /* PingLoader.h in Headers */ = {isa = PBXBuildFile; fileRef = D0FF2A5C11F8C45A007E74E0 /* PingLoader.h */; settings = {ATTRIBUTES = (Private, ); }; };
D2BB5C65290985E40065457D /* NetworkConnectionIntegrity.h in Headers */ = {isa = PBXBuildFile; fileRef = D2BB5C64290985E40065457D /* NetworkConnectionIntegrity.h */; settings = {ATTRIBUTES = (Private, ); }; };
D2C4612E2A0C744E00A5F791 /* CanvasNoiseInjection.h in Headers */ = {isa = PBXBuildFile; fileRef = D2C4612C2A0C744E00A5F791 /* CanvasNoiseInjection.h */; settings = {ATTRIBUTES = (Private, ); }; };
D302754A12A5FE84004BD828 /* RenderDetailsMarker.h in Headers */ = {isa = PBXBuildFile; fileRef = D302754612A5FE84004BD828 /* RenderDetailsMarker.h */; };
D359D78A129CA2710006E5D2 /* HTMLDetailsElement.h in Headers */ = {isa = PBXBuildFile; fileRef = D359D787129CA2710006E5D2 /* HTMLDetailsElement.h */; };
D359D8BF129CA55C0006E5D2 /* JSHTMLDetailsElement.h in Headers */ = {isa = PBXBuildFile; fileRef = D359D8BD129CA55C0006E5D2 /* JSHTMLDetailsElement.h */; };
Expand Down Expand Up @@ -17721,6 +17722,8 @@
D0FF2A5B11F8C45A007E74E0 /* PingLoader.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = PingLoader.cpp; sourceTree = "<group>"; };
D0FF2A5C11F8C45A007E74E0 /* PingLoader.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = PingLoader.h; sourceTree = "<group>"; };
D2BB5C64290985E40065457D /* NetworkConnectionIntegrity.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = NetworkConnectionIntegrity.h; sourceTree = "<group>"; };
D2C4612B2A0C744E00A5F791 /* CanvasNoiseInjection.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = CanvasNoiseInjection.cpp; sourceTree = "<group>"; };
D2C4612C2A0C744E00A5F791 /* CanvasNoiseInjection.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = CanvasNoiseInjection.h; sourceTree = "<group>"; };
D2CDB5ED638F43AF86F07AA2 /* JSErrorEventCustom.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = JSErrorEventCustom.cpp; sourceTree = "<group>"; };
D302754512A5FE84004BD828 /* RenderDetailsMarker.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = RenderDetailsMarker.cpp; sourceTree = "<group>"; };
D302754612A5FE84004BD828 /* RenderDetailsMarker.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RenderDetailsMarker.h; sourceTree = "<group>"; };
Expand Down Expand Up @@ -27633,6 +27636,8 @@
CD814C342996CA6A005A780A /* CachedHTMLCollectionInlines.h */,
313171571FB0969E008D91FC /* CanvasBase.cpp */,
313171541FB079D1008D91FC /* CanvasBase.h */,
D2C4612B2A0C744E00A5F791 /* CanvasNoiseInjection.cpp */,
D2C4612C2A0C744E00A5F791 /* CanvasNoiseInjection.h */,
CD6A1F0C297A1A8F00F7B9FA /* CanvasObserver.h */,
F55B3D7D1251F12D003EF269 /* CheckboxInputType.cpp */,
F55B3D7E1251F12D003EF269 /* CheckboxInputType.h */,
Expand Down Expand Up @@ -36272,6 +36277,7 @@
49484FC2102CF23C00187DD3 /* CanvasGradient.h in Headers */,
7C193BBD1F5E0EED0088F3E6 /* CanvasLineCap.h in Headers */,
7C193BBE1F5E0EED0088F3E6 /* CanvasLineJoin.h in Headers */,
D2C4612E2A0C744E00A5F791 /* CanvasNoiseInjection.h in Headers */,
CD6A1F0D297A1A8F00F7B9FA /* CanvasObserver.h in Headers */,
4671E0661D67A59600C6B497 /* CanvasPath.h in Headers */,
49484FC5102CF23C00187DD3 /* CanvasPattern.h in Headers */,
Expand Down
99 changes: 12 additions & 87 deletions Source/WebCore/html/CanvasBase.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,8 @@ void CanvasBase::makeRenderingResultsAvailable()
{
if (auto* context = renderingContext()) {
context->paintRenderingResultsToCanvas();
postProcessDirtyCanvasBuffer();
if (auto noiseInjectionHashSalt = scriptExecutionContext() ? scriptExecutionContext()->noiseInjectionHashSalt() : std::nullopt)
m_canvasNoiseInjection.postProcessDirtyCanvasBuffer(buffer(), *noiseInjectionHashSalt);
}
}

Expand Down Expand Up @@ -203,8 +204,12 @@ void CanvasBase::notifyObserversCanvasChanged(const std::optional<FloatRect>& re
void CanvasBase::didDraw(const std::optional<FloatRect>& rect, ShouldApplyPostProcessingToDirtyRect shouldApplyPostProcessingToDirtyRect)
{
// FIXME: We should exclude rects with ShouldApplyPostProcessingToDirtyRect::No
if (shouldInjectNoiseBeforeReadback() && shouldApplyPostProcessingToDirtyRect == ShouldApplyPostProcessingToDirtyRect::Yes && rect)
m_postProcessDirtyRect.unite(*rect);
if (shouldInjectNoiseBeforeReadback() && shouldApplyPostProcessingToDirtyRect == ShouldApplyPostProcessingToDirtyRect::Yes) {
if (rect)
m_canvasNoiseInjection.updateDirtyRect(intersection(enclosingIntRect(*rect), { { }, size() }));
else
m_canvasNoiseInjection.updateDirtyRect({ { }, size() });
}
}

void CanvasBase::notifyObserversCanvasResized()
Expand Down Expand Up @@ -373,91 +378,11 @@ bool CanvasBase::shouldInjectNoiseBeforeReadback() const
return scriptExecutionContext() && scriptExecutionContext()->noiseInjectionHashSalt();
}

void CanvasBase::postProcessDirtyCanvasBuffer() const
{
if (!shouldInjectNoiseBeforeReadback())
return;

if (m_postProcessDirtyRect.isEmpty())
return;

ImageBuffer* imageBuffer = buffer();
if (!imageBuffer)
return;

auto imageRect = enclosingIntRect(m_postProcessDirtyRect);
PixelBufferFormat format { AlphaPremultiplication::Unpremultiplied, PixelFormat::RGBA8, imageBuffer->colorSpace() };
auto pixelBuffer = imageBuffer->getPixelBuffer(format, imageRect);
if (!is<ByteArrayPixelBuffer>(pixelBuffer))
return;

if (postProcessPixelBufferResults(*pixelBuffer, { })) {
imageBuffer->putPixelBuffer(*pixelBuffer, imageRect, { 0, 0 });
m_postProcessDirtyRect = { };
}
}

bool CanvasBase::postProcessPixelBufferResults(Ref<PixelBuffer>&& pixelBuffer, const HashSet<uint32_t>& suppliedColors) const
bool CanvasBase::postProcessPixelBufferResults(Ref<PixelBuffer>&& pixelBuffer) const
{
if (!shouldInjectNoiseBeforeReadback())
return false;

ASSERT(pixelBuffer->format().pixelFormat == PixelFormat::RGBA8);

constexpr auto contextString { "Canvas2DContextString"_s };
unsigned salt = computeHash<String, uint64_t>(contextString, *scriptExecutionContext()->noiseInjectionHashSalt());
constexpr int bytesPerPixel = 4;
auto* bytes = pixelBuffer->bytes();
HashMap<uint32_t, uint32_t> pixelColorMap;
bool wasPixelBufferModified { false };

for (size_t i = 0; i < pixelBuffer->sizeInBytes(); i += bytesPerPixel) {
auto& redChannel = bytes[i];
auto& greenChannel = bytes[i + 1];
auto& blueChannel = bytes[i + 2];
auto& alphaChannel = bytes[i + 3];
bool isBlack { !redChannel && !greenChannel && !blueChannel };

uint32_t pixel = (redChannel << 24) | (greenChannel << 16) | (blueChannel << 8) | alphaChannel;
// FIXME: Consider isEquivalentColor comparision instead of exact match
if (!pixel || !alphaChannel || !suppliedColors.isValidValue(pixel) || suppliedColors.contains(pixel))
continue;

if (auto color = pixelColorMap.get(pixel)) {
alphaChannel = static_cast<uint8_t>(color);
blueChannel = static_cast<uint8_t>(color >> 8);
greenChannel = static_cast<uint8_t>(color >> 16);
redChannel = static_cast<uint8_t>(color >> 24);
continue;
}

const uint64_t pixelHash = computeHash(salt, redChannel, greenChannel, blueChannel, alphaChannel);
// +/- ~13 is roughly 5% of the 255 max value.
const auto clampedFivePercent = static_cast<uint32_t>(((pixelHash * 26) / std::numeric_limits<uint32_t>::max()) - 13);

const auto clampedColorComponentOffset = [](int colorComponentOffset, int originalComponentValue) {
if (colorComponentOffset + originalComponentValue > std::numeric_limits<uint8_t>::max())
return std::numeric_limits<uint8_t>::max() - originalComponentValue;
if (colorComponentOffset + originalComponentValue < 0)
return -originalComponentValue;
return colorComponentOffset;
};

// If alpha is non-zero and the color channels are zero, then only tweak the alpha channel's value;
if (isBlack)
alphaChannel += clampedColorComponentOffset(clampedFivePercent, alphaChannel);
else {
// If alpha and any of the color channels are non-zero, then tweak all of the channels;
redChannel += clampedColorComponentOffset(clampedFivePercent, redChannel);
greenChannel += clampedColorComponentOffset(clampedFivePercent, greenChannel);
blueChannel += clampedColorComponentOffset(clampedFivePercent, blueChannel);
alphaChannel += clampedColorComponentOffset(clampedFivePercent, alphaChannel);
}
uint32_t modifiedPixel = (redChannel << 24) | (greenChannel << 16) | (blueChannel << 8) | alphaChannel;
pixelColorMap.set(pixel, modifiedPixel);
wasPixelBufferModified = true;
}
return wasPixelBufferModified;
if (auto noiseInjectionHashSalt = scriptExecutionContext() ? scriptExecutionContext()->noiseInjectionHashSalt() : std::nullopt)
return m_canvasNoiseInjection.postProcessPixelBufferResults(std::forward<Ref<PixelBuffer>>(pixelBuffer), *noiseInjectionHashSalt);
return false;
}

size_t CanvasBase::activePixelMemory()
Expand Down
6 changes: 3 additions & 3 deletions Source/WebCore/html/CanvasBase.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@

#pragma once

#include "CanvasNoiseInjection.h"
#include "FloatRect.h"
#include "IntSize.h"
#include "PixelBuffer.h"
Expand Down Expand Up @@ -130,7 +131,7 @@ class CanvasBase {
virtual void queueTaskKeepingObjectAlive(TaskSource, Function<void()>&&) = 0;
virtual void dispatchEvent(Event&) = 0;

bool postProcessPixelBufferResults(Ref<PixelBuffer>&&, const HashSet<uint32_t>&) const;
bool postProcessPixelBufferResults(Ref<PixelBuffer>&&) const;

protected:
explicit CanvasBase(IntSize);
Expand All @@ -149,16 +150,15 @@ class CanvasBase {

private:
bool shouldInjectNoiseBeforeReadback() const;
void postProcessDirtyCanvasBuffer() const;
virtual void createImageBuffer() const { }

mutable IntSize m_size;
mutable FloatRect m_postProcessDirtyRect;
mutable Lock m_imageBufferAssignmentLock;
mutable RefPtr<ImageBuffer> m_imageBuffer;
mutable size_t m_imageBufferCost { 0 };
mutable std::unique_ptr<GraphicsContextStateSaver> m_contextStateSaver;

CanvasNoiseInjection m_canvasNoiseInjection;
bool m_originClean { true };
#if ASSERT_ENABLED
bool m_didNotifyObserversCanvasDestroyed { false };
Expand Down
Loading

0 comments on commit d721aae

Please sign in to comment.