[MultiRep HEIC] Add support for insertion in editable content

https://bugs.webkit.org/show_bug.cgi?id=268202
rdar://121697249

Reviewed by Richard Robinson.

Insert multi-representation HEIC as a <picture> element with a HEIC <source>
and PNG <img> fallback.

* Source/WebCore/editing/Editor.h:
* Source/WebCore/editing/cocoa/EditorCocoa.mm:
(WebCore::Editor::insertMultiRepresentationHEIC):
* Source/WebCore/platform/graphics/cg/ImageBufferUtilitiesCG.cpp:
(WebCore::imagePropertiesForDestinationUTIAndQuality):
(WebCore::encode):
(WebCore::encodeData):
* Source/WebCore/platform/graphics/cg/ImageBufferUtilitiesCG.h:

Add a method to convert to data into a specific image format.

* Source/WebKit/UIProcess/Cocoa/WebPageProxyCocoa.mm:
(WebKit::WebPageProxy::insertMultiRepresentationHEIC):
* Source/WebKit/WebProcess/WebPage/Cocoa/WebPageCocoa.mm:
(WebKit::WebPage::insertMultiRepresentationHEIC):
* Source/WebKit/WebProcess/WebPage/WebPage.h:
* Source/WebKit/WebProcess/WebPage/WebPage.messages.in:

Canonical link: https://commits.webkit.org/273610@main

Source/WebCore/editing/Editor.h

@@ -550,6 +550,10 @@ class Editor : public CanMakeCheckedPtr {
     WEBCORE_EXPORT void readSelectionFromPasteboard(const String& pasteboardName);
     WEBCORE_EXPORT void replaceNodeFromPasteboard(Node&, const String& pasteboardName, EditAction = EditAction::Paste);
 
+#if ENABLE(MULTI_REPRESENTATION_HEIC)
+    WEBCORE_EXPORT void insertMultiRepresentationHEIC(const std::span<const uint8_t>&);
+#endif
+
     static RefPtr<SharedBuffer> dataInRTFDFormat(NSAttributedString *);
     static RefPtr<SharedBuffer> dataInRTFFormat(NSAttributedString *);
 #endif

Source/WebCore/editing/cocoa/EditorCocoa.mm

@@ -31,6 +31,7 @@
 #import "CSSValuePool.h"
 #import "CachedResourceLoader.h"
 #import "ColorMac.h"
+#import "DOMURL.h"
 #import "DocumentFragment.h"
 #import "DocumentLoader.h"
 #import "Editing.h"
@@ -44,7 +45,10 @@
 #import "HTMLAttachmentElement.h"
 #import "HTMLConverter.h"
 #import "HTMLImageElement.h"
+#import "HTMLPictureElement.h"
+#import "HTMLSourceElement.h"
 #import "HTMLSpanElement.h"
+#import "ImageBufferUtilitiesCG.h"
 #import "ImageOverlay.h"
 #import "LegacyNSPasteboardTypes.h"
 #import "LegacyWebArchive.h"
@@ -56,6 +60,7 @@
 #import "PlatformStrategies.h"
 #import "RenderElement.h"
 #import "RenderStyle.h"
+#import "ReplaceSelectionCommand.h"
 #import "Settings.h"
 #import "SystemSoundManager.h"
 #import "Text.h"
@@ -385,4 +390,32 @@ static void maybeCopyNodeAttributesToFragment(const Node& node, DocumentFragment
 #endif
 }
 
+#if ENABLE(MULTI_REPRESENTATION_HEIC)
+void Editor::insertMultiRepresentationHEIC(const std::span<const uint8_t>& data)
+{
+    auto document = protectedDocument();
+
+    String primaryType = "image/heic"_s;
+
+    String fallbackType = "image/png"_s;
+    auto fallbackData = encodeData(data, fallbackType, std::nullopt);
+
+    auto picture = HTMLPictureElement::create(HTMLNames::pictureTag, document);
+
+    auto source = HTMLSourceElement::create(document);
+    source->setAttributeWithoutSynchronization(srcsetAttr, AtomString { DOMURL::createObjectURL(document, Blob::create(document.ptr(), Vector<uint8_t> { data }, primaryType)) });
+    source->setAttributeWithoutSynchronization(typeAttr, AtomString { primaryType });
+    picture->appendChild(WTFMove(source));
+
+    auto image = HTMLImageElement::create(document);
+    image->setSrc(AtomString { DOMURL::createObjectURL(document, Blob::create(document.ptr(), WTFMove(fallbackData), fallbackType)) });
+    picture->appendChild(WTFMove(image));
+
+    auto fragment = document->createDocumentFragment();
+    fragment->appendChild(WTFMove(picture));
+
+    ReplaceSelectionCommand::create(document.get(), WTFMove(fragment), ReplaceSelectionCommand::PreventNesting, EditAction::Insert)->apply();
+}
+#endif
+
 }

Source/WebCore/platform/graphics/cg/ImageBufferUtilitiesCG.cpp

@@ -93,6 +93,22 @@ ALLOW_DEPRECATED_DECLARATIONS_BEGIN
 ALLOW_DEPRECATED_DECLARATIONS_END
 }
 
+static RetainPtr<CFDictionaryRef> imagePropertiesForDestinationUTIAndQuality(CFStringRef destinationUTI, std::optional<double> quality)
+{
+    if (CFEqual(destinationUTI, jpegUTI()) && quality && *quality >= 0.0 && *quality <= 1.0) {
+        // Apply the compression quality to the JPEG image destination.
+        quality = std::max(*quality, 0.0001); // FIXME: Remove once BigSur is unsupported (rdar://80446736)
+        auto compressionQuality = adoptCF(CFNumberCreate(kCFAllocatorDefault, kCFNumberDoubleType, &*quality));
+        const void* key = kCGImageDestinationLossyCompressionQuality;
+        const void* value = compressionQuality.get();
+        return adoptCF(CFDictionaryCreate(0, &key, &value, 1, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks));
+    }
+    return nullptr;
+
+    // FIXME: Setting kCGImageDestinationBackgroundColor to black for JPEG images in imageProperties would save some math
+    // in the calling functions, but it doesn't seem to work.
+}
+
 static bool encode(CGImageRef image, const String& mimeType, std::optional<double> quality, const ScopedLambda<PutBytesCallback>& function)
 {
     if (!image)
@@ -113,21 +129,7 @@ static bool encode(CGImageRef image, const String& mimeType, std::optional<doubl
     auto consumer = adoptCF(CGDataConsumerCreate(const_cast<ScopedLambda<PutBytesCallback>*>(&function), &callbacks));
     auto destination = adoptCF(CGImageDestinationCreateWithDataConsumer(consumer.get(), destinationUTI.get(), 1, nullptr));
 
-    auto imageProperties = [&] () -> RetainPtr<CFDictionaryRef> {
-        if (CFEqual(destinationUTI.get(), jpegUTI()) && quality && *quality >= 0.0 && *quality <= 1.0) {
-            // Apply the compression quality to the JPEG image destination.
-            quality = std::max(*quality, 0.0001); // FIXME: Remove once BigSur is unsupported (rdar://80446736)
-            auto compressionQuality = adoptCF(CFNumberCreate(kCFAllocatorDefault, kCFNumberDoubleType, &*quality));
-            const void* key = kCGImageDestinationLossyCompressionQuality;
-            const void* value = compressionQuality.get();
-            return adoptCF(CFDictionaryCreate(0, &key, &value, 1, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks));
-        }
-        return nullptr;
-    }();
-
-    // FIXME: Setting kCGImageDestinationBackgroundColor to black for JPEG images in imageProperties would save some math
-    // in the calling functions, but it doesn't seem to work.
-
+    auto imageProperties = imagePropertiesForDestinationUTIAndQuality(destinationUTI.get(), quality);
     CGImageDestinationAddImage(destination.get(), image, imageProperties.get());
 
     return CGImageDestinationFinalize(destination.get());
@@ -184,6 +186,40 @@ static bool encode(const PixelBuffer& source, const String& mimeType, std::optio
     return encode(image.get(), mimeType, quality, function);
 }
 
+static bool encode(const std::span<const uint8_t>& data, const String& mimeType, std::optional<double> quality, const ScopedLambda<PutBytesCallback>& function)
+{
+    if (data.empty())
+        return false;
+
+    auto destinationUTI = utiFromImageBufferMIMEType(mimeType);
+    if (!destinationUTI)
+        return false;
+
+    auto cfData = adoptCF(CFDataCreateWithBytesNoCopy(nullptr, data.data(), data.size(), kCFAllocatorNull));
+    if (!cfData)
+        return false;
+
+    auto source = adoptCF(CGImageSourceCreateWithData(cfData.get(), nullptr));
+    if (!source)
+        return false;
+
+    CGDataConsumerCallbacks callbacks {
+        [](void* context, const void* buffer, size_t count) -> size_t {
+            auto functor = *static_cast<const ScopedLambda<PutBytesCallback>*>(context);
+            return functor(buffer, count);
+        },
+        nullptr
+    };
+
+    auto consumer = adoptCF(CGDataConsumerCreate(const_cast<ScopedLambda<PutBytesCallback>*>(&function), &callbacks));
+    auto destination = adoptCF(CGImageDestinationCreateWithDataConsumer(consumer.get(), destinationUTI.get(), 1, nullptr));
+
+    auto imageProperties = imagePropertiesForDestinationUTIAndQuality(destinationUTI.get(), quality);
+    CGImageDestinationAddImageFromSource(destination.get(), source.get(), 0, nullptr);
+
+    return CGImageDestinationFinalize(destination.get());
+}
+
 template<typename Source> static Vector<uint8_t> encodeToVector(Source&& source, const String& mimeType, std::optional<double> quality)
 {
     Vector<uint8_t> result;
@@ -219,6 +255,11 @@ Vector<uint8_t> encodeData(const PixelBuffer& pixelBuffer, const String& mimeTyp
     return encodeToVector(pixelBuffer, mimeType, quality);
 }
 
+Vector<uint8_t> encodeData(const std::span<const uint8_t>& data, const String& mimeType, std::optional<double> quality)
+{
+    return encodeToVector(data, mimeType, quality);
+}
+
 String dataURL(CGImageRef image, const String& mimeType, std::optional<double> quality)
 {
     return encodeToDataURL(image, mimeType, quality);

Source/WebCore/platform/graphics/cg/ImageBufferUtilitiesCG.h

@@ -40,6 +40,7 @@ RetainPtr<CFStringRef> utiFromImageBufferMIMEType(const String& mimeType);
 CFStringRef jpegUTI();
 Vector<uint8_t> encodeData(CGImageRef, const String& mimeType, std::optional<double> quality);
 Vector<uint8_t> encodeData(const PixelBuffer&, const String& mimeType, std::optional<double> quality);
+Vector<uint8_t> encodeData(const std::span<const uint8_t>&, const String& mimeType, std::optional<double> quality);
 
 WEBCORE_EXPORT String dataURL(CGImageRef, const String& mimeType, std::optional<double> quality);
 String dataURL(const PixelBuffer&, const String& mimeType, std::optional<double> quality);

Source/WebKit/UIProcess/Cocoa/WebPageProxyCocoa.mm

@@ -915,8 +915,10 @@ static bool exceedsRenderTreeSizeSizeThreshold(uint64_t thresholdSize, uint64_t
 #endif
 
 #if ENABLE(MULTI_REPRESENTATION_HEIC)
-void WebPageProxy::insertMultiRepresentationHEIC(NSData *)
+void WebPageProxy::insertMultiRepresentationHEIC(NSData *data)
 {
+    send(Messages::WebPage::InsertMultiRepresentationHEIC(IPC::DataReference(static_cast<const uint8_t*>([data bytes]), [data length])));
+
 }
 #endif
 

Source/WebKit/WebProcess/WebPage/Cocoa/WebPageCocoa.mm

@@ -733,6 +733,16 @@
     completionHandler(true);
 }
 
+#if ENABLE(MULTI_REPRESENTATION_HEIC)
+void WebPage::insertMultiRepresentationHEIC(const IPC::DataReference& data)
+{
+    Ref frame = m_page->focusController().focusedOrMainFrame();
+    if (frame->selection().isNone())
+        return;
+    frame->editor().insertMultiRepresentationHEIC(data);
+}
+#endif
+
 std::pair<URL, DidFilterLinkDecoration> WebPage::applyLinkDecorationFilteringWithResult(const URL& url, LinkDecorationFilteringTrigger trigger)
 {
 #if ENABLE(ADVANCED_PRIVACY_PROTECTIONS)

Source/WebKit/WebProcess/WebPage/WebPage.h

@@ -1065,6 +1065,10 @@ class WebPage : public API::ObjectImpl<API::Object::Type::BundlePage>, public IP
     void shouldDelayWindowOrderingEvent(const WebKit::WebMouseEvent&, CompletionHandler<void(bool)>&&);
     bool performNonEditingBehaviorForSelector(const String&, WebCore::KeyboardEvent*);
 
+#if ENABLE(MULTI_REPRESENTATION_HEIC)
+    void insertMultiRepresentationHEIC(const IPC::DataReference&);
+#endif
+
     void insertDictatedTextAsync(const String& text, const EditingRange& replacementRange, const Vector<WebCore::DictationAlternative>& dictationAlternativeLocations, InsertTextOptions&&);
     void addDictationAlternative(const String& text, WebCore::DictationContext, CompletionHandler<void(bool)>&&);
     void dictationAlternativesAtSelection(CompletionHandler<void(Vector<WebCore::DictationContext>&&)>&&);

Source/WebKit/WebProcess/WebPage/WebPage.messages.in

@@ -494,6 +494,10 @@ GenerateSyntheticEditingCommand(enum:uint8_t WebKit::SyntheticEditingCommandType
     ReadSelectionFromPasteboard(String pasteboardName) -> (bool result) Synchronous
     ReplaceSelectionWithPasteboardData(Vector<String> types, IPC::DataReference data)
 
+#if ENABLE(MULTI_REPRESENTATION_HEIC)
+    InsertMultiRepresentationHEIC(IPC::DataReference data)
+#endif
+
     ShouldDelayWindowOrderingEvent(WebKit::WebMouseEvent event) -> (bool result) Synchronous
 
     SetTextAsync(String text)