Add support for recognizing data interaction gestures in WebKit2

https://bugs.webkit.org/show_bug.cgi?id=167444

Reviewed by Beth Dakin.

Source/WebCore:

Minor tweaks to pasteboard code to support data interaction.

* WebCore.xcodeproj/project.pbxproj:
* platform/PlatformPasteboard.h:
* platform/ios/PasteboardIOS.mm:
(WebCore::Pasteboard::read):
* platform/ios/PlatformPasteboardIOS.mm:
(WebCore::PlatformPasteboard::PlatformPasteboard):

If the pasteboard is the special data interaction type, use the shared item provider pasteboard; otherwise,
fall back to the general pasteboard.

(WebCore::PlatformPasteboard::getTypes):

Actually populate the list of available types using available pasteboardTypes.

(WebCore::PlatformPasteboard::write):

Add UTF8 plaintext type (kUTTypeUTF8PlainText) when vending data representations of rich text.

* platform/ios/WebItemProviderPasteboard.mm:
(-[WebItemProviderPasteboard setItems:]):
(-[WebItemProviderPasteboard dataForPasteboardType:inItemSet:]):
(-[WebItemProviderPasteboard valuesForPasteboardType:inItemSet:]):

Move off of deprecated methods when retrieving and supplying data to the item provider pasteboard.

* platform/spi/ios/UIKitSPI.h:

Source/WebKit2:

Adds a new data interaction gesture recognizer, responsible for determining when to begin data interaction. This
is a new long press gesture recognizer that fires simultaneously with the existing long press gesture
recognizers (for performing long-press actions, and for showing the tap highlight).

Also tweaks logic for determining whether selection gesture recognizers should fire to account for data
interaction -- in particular, we don't want selection gesture recognizers to cause the current selection to
change while data interaction is possible. See -hasSelectablePositionAtPoint and -pointIsInAssistedNode for
more details.

* UIProcess/ios/PageClientImplIOS.mm:
(WebKit::PageClientImpl::didPerformDataInteractionControllerOperation):
(WebKit::PageClientImpl::startDataInteractionWithImage):
* UIProcess/ios/WKContentViewInteraction.h:
* UIProcess/ios/WKContentViewInteraction.mm:
(-[WKContentView _createAndConfigureLongPressGestureRecognizer]):
(-[WKContentView setupInteraction]):
(-[WKContentView cleanupInteraction]):
(-[WKContentView _removeDefaultGestureRecognizers]):
(-[WKContentView _addDefaultGestureRecognizers]):
(-[WKContentView resignFirstResponder]):
(-[WKContentView gestureRecognizer:shouldRecognizeSimultaneouslyWithGestureRecognizer:]):
(-[WKContentView gestureRecognizerShouldBegin:]):
(-[WKContentView hasSelectablePositionAtPoint:]):
(-[WKContentView pointIsInDataInteractionContent:]):
(-[WKContentView pointIsInAssistedNode:]):


Canonical link: https://commits.webkit.org/184503@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@211227 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Source/WebCore/ChangeLog

@@ -1,3 +1,39 @@
+2017-01-26  Wenson Hsieh  <wenson_hsieh@apple.com>
+
+        Add support for recognizing data interaction gestures in WebKit2
+        https://bugs.webkit.org/show_bug.cgi?id=167444
+
+        Reviewed by Beth Dakin.
+
+        Minor tweaks to pasteboard code to support data interaction.
+
+        * WebCore.xcodeproj/project.pbxproj:
+        * platform/PlatformPasteboard.h:
+        * platform/ios/PasteboardIOS.mm:
+        (WebCore::Pasteboard::read):
+        * platform/ios/PlatformPasteboardIOS.mm:
+        (WebCore::PlatformPasteboard::PlatformPasteboard):
+
+        If the pasteboard is the special data interaction type, use the shared item provider pasteboard; otherwise,
+        fall back to the general pasteboard.
+
+        (WebCore::PlatformPasteboard::getTypes):
+
+        Actually populate the list of available types using available pasteboardTypes.
+
+        (WebCore::PlatformPasteboard::write):
+
+        Add UTF8 plaintext type (kUTTypeUTF8PlainText) when vending data representations of rich text.
+
+        * platform/ios/WebItemProviderPasteboard.mm:
+        (-[WebItemProviderPasteboard setItems:]):
+        (-[WebItemProviderPasteboard dataForPasteboardType:inItemSet:]):
+        (-[WebItemProviderPasteboard valuesForPasteboardType:inItemSet:]):
+
+        Move off of deprecated methods when retrieving and supplying data to the item provider pasteboard.
+
+        * platform/spi/ios/UIKitSPI.h:
+
 2017-01-26  Matt Rajca  <mrajca@apple.com>
 
         Notify clients when the user plays media otherwise prevented from autoplaying

Source/WebCore/WebCore.xcodeproj/project.pbxproj

@@ -6511,8 +6511,8 @@
 		F47A5E3E195B8C8A00483100 /* StyleScrollSnapPoints.h in Headers */ = {isa = PBXBuildFile; fileRef = F47A5E3B195B8C8A00483100 /* StyleScrollSnapPoints.h */; settings = {ATTRIBUTES = (Private, ); }; };
 		F47A5E3F195B8E4800483100 /* StyleScrollSnapPoints.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F47A5E3A195B8C8A00483100 /* StyleScrollSnapPoints.cpp */; };
 		F48223101E3869B80066FC79 /* WebItemProviderPasteboard.mm in Sources */ = {isa = PBXBuildFile; fileRef = F482230E1E3869B80066FC79 /* WebItemProviderPasteboard.mm */; };
-		F48223111E3869B80066FC79 /* WebItemProviderPasteboard.h in Headers */ = {isa = PBXBuildFile; fileRef = F482230F1E3869B80066FC79 /* WebItemProviderPasteboard.h */; };
-		F48223131E386E240066FC79 /* AbstractPasteboard.h in Headers */ = {isa = PBXBuildFile; fileRef = F48223121E386E240066FC79 /* AbstractPasteboard.h */; };
+		F48223111E3869B80066FC79 /* WebItemProviderPasteboard.h in Headers */ = {isa = PBXBuildFile; fileRef = F482230F1E3869B80066FC79 /* WebItemProviderPasteboard.h */; settings = {ATTRIBUTES = (Private, ); }; };
+		F48223131E386E240066FC79 /* AbstractPasteboard.h in Headers */ = {isa = PBXBuildFile; fileRef = F48223121E386E240066FC79 /* AbstractPasteboard.h */; settings = {ATTRIBUTES = (Private, ); }; };
 		F4BFB9851E1DDF9B00862C24 /* DumpEditingHistory.js in Copy Scripts */ = {isa = PBXBuildFile; fileRef = F48389831E1DDF2B0076B7EA /* DumpEditingHistory.js */; };
 		F4BFB9861E1DDF9B00862C24 /* EditingHistoryUtil.js in Copy Scripts */ = {isa = PBXBuildFile; fileRef = F48389841E1DDF2B0076B7EA /* EditingHistoryUtil.js */; };
 		F50664F7157F52DC00AC226F /* FormController.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F50664F5157F52DC00AC226F /* FormController.cpp */; };

Source/WebCore/platform/PlatformPasteboard.h

@@ -92,7 +92,7 @@ class PlatformPasteboard {
     RetainPtr<NSPasteboard> m_pasteboard;
 #endif
 #if PLATFORM(IOS)
-    RetainPtr<UIPasteboard> m_pasteboard;
+    RetainPtr<id> m_pasteboard;
 #endif
 #if PLATFORM(GTK)
     GtkClipboard* m_clipboard;

Source/WebCore/platform/ios/PasteboardIOS.mm

@@ -150,9 +150,14 @@ - (BOOL)containsAttachments;
 void Pasteboard::read(PasteboardPlainText& text)
 {
     PasteboardStrategy& strategy = *platformStrategies()->pasteboardStrategy();
+    text.text = strategy.readStringFromPasteboard(0, kUTTypeURL, m_pasteboardName);
+    if (!text.text.isNull() && !text.text.isEmpty()) {
+        text.isURL = true;
+        return;
+    }
+
     text.text = strategy.readStringFromPasteboard(0, kUTTypeText, m_pasteboardName);
-    if (text.text.isEmpty())
-        text.text = strategy.readStringFromPasteboard(0, kUTTypeURL, m_pasteboardName);
+    text.isURL = false;
 }
 
 static NSArray* supportedImageTypes()

Source/WebCore/platform/ios/PlatformPasteboardIOS.mm

@@ -32,6 +32,7 @@
 #import "Pasteboard.h"
 #import "SharedBuffer.h"
 #import "SoftLinking.h"
+#import "WebItemProviderPasteboard.h"
 #import <MobileCoreServices/MobileCoreServices.h>
 
 SOFT_LINK_FRAMEWORK(UIKit)
@@ -53,13 +54,25 @@ - (NSInteger)changeCount;
 {
 }
 
+#if ENABLE(DATA_INTERACTION)
+PlatformPasteboard::PlatformPasteboard(const String& name)
+{
+    if (name == "data interaction pasteboard")
+        m_pasteboard = [WebItemProviderPasteboard sharedInstance];
+    else
+        m_pasteboard = [getUIPasteboardClass() generalPasteboard];
+}
+#else
 PlatformPasteboard::PlatformPasteboard(const String&)
     : m_pasteboard([getUIPasteboardClass() generalPasteboard])
 {
 }
+#endif
 
-void PlatformPasteboard::getTypes(Vector<String>&)
+void PlatformPasteboard::getTypes(Vector<String>& types)
 {
+    for (NSString *pasteboardType in [m_pasteboard pasteboardTypes])
+        types.append(pasteboardType);
 }
 
 RefPtr<SharedBuffer> PlatformPasteboard::bufferForType(const String&)
@@ -145,7 +158,9 @@ - (NSInteger)changeCount;
         [representations setValue:content.dataInRTFDFormat->createNSData().get() forKey:(NSString *)kUTTypeFlatRTFD];
     if (content.dataInRTFFormat)
         [representations setValue:content.dataInRTFFormat->createNSData().get() forKey:(NSString *)kUTTypeRTF];
+
     [representations setValue:content.dataInStringFormat forKey:(NSString *)kUTTypeText];
+    [representations setValue:[(NSString *)content.dataInStringFormat dataUsingEncoding:NSUTF8StringEncoding] forKey:(NSString *)kUTTypeUTF8PlainText];
     [m_pasteboard setItems:@[representations.get()]];
 }
 

Source/WebCore/platform/ios/WebItemProviderPasteboard.mm

@@ -30,6 +30,7 @@
 
 #import "SoftLinking.h"
 #import "UIKitSPI.h"
+#import <Foundation/NSProgress.h>
 #import <MobileCoreServices/MobileCoreServices.h>
 #import <UIKit/UIColor.h>
 #import <UIKit/UIImage.h>
@@ -131,8 +132,10 @@ - (void)setItems:(NSArray *)items
             continue;
         UIItemProvider *itemProvider = [[getUIItemProviderClass() alloc] init];
         for (NSString *typeIdentifier in item) {
-            [itemProvider registerDataRepresentationForTypeIdentifier:typeIdentifier loadHandler:^(UIItemProviderDataLoadCompletionBlock completionBlock, NSDictionary *) {
+            [itemProvider registerDataRepresentationForTypeIdentifier:typeIdentifier options:nil loadHandler:^NSProgress *(UIItemProviderDataLoadCompletionBlock completionBlock)
+            {
                 completionBlock(item[typeIdentifier], nil);
+                return [NSProgress discreteProgressWithTotalUnitCount:100];
             }];
         }
         [providers addObject:itemProvider];
@@ -149,7 +152,7 @@ - (NSArray *)dataForPasteboardType:(NSString *)pasteboardType inItemSet:(NSIndex
         if (!provider)
             return;
 
-        NSData *data = [provider copyDataRepresentationForTypeIdentifier:pasteboardType options:nil error:nil];
+        NSData *data = [provider copyDataRepresentationForTypeIdentifier:pasteboardType error:nil];
         if (data)
             [values addObject:data];
     }];
@@ -165,28 +168,29 @@ - (NSArray *)valuesForPasteboardType:(NSString *)pasteboardType inItemSet:(NSInd
         if (!provider)
             return;
 
-        if (isRichTextType(pasteboardType) && [provider canInstantiateObjectOfClass:[NSAttributedString class]]) {
-            [values addObject:[provider instantiateObjectOfClass:[NSAttributedString class] options:nil error:nil]];
+        // FIXME: These should be refactored to use asynchronous calls.
+        if (isColorType(pasteboardType) && [provider canCreateObjectOfClass:[getUIColorClass() class]]) {
+            [values addObject:[provider createObjectOfClass:[getUIColorClass() class] error:nil]];
             return;
         }
 
-        if (isStringType(pasteboardType) && [provider canInstantiateObjectOfClass:[NSString class]]) {
-            [values addObject:[provider instantiateObjectOfClass:[NSString class] options:nil error:nil]];
+        if (isImageType(pasteboardType) && [provider canCreateObjectOfClass:[getUIImageClass() class]]) {
+            [values addObject:[provider createObjectOfClass:[getUIImageClass() class] error:nil]];
             return;
         }
 
-        if (isColorType(pasteboardType) && [provider canInstantiateObjectOfClass:[getUIColorClass() class]]) {
-            [values addObject:[provider instantiateObjectOfClass:[getUIColorClass() class] options:nil error:nil]];
+        if (isURLType(pasteboardType) && [provider canCreateObjectOfClass:[NSURL class]]) {
+            [values addObject:[provider createObjectOfClass:[NSURL class] error:nil]];
             return;
         }
 
-        if (isURLType(pasteboardType) && [provider canInstantiateObjectOfClass:[NSURL class]]) {
-            [values addObject:[provider instantiateObjectOfClass:[NSURL class] options:nil error:nil]];
+        if (isRichTextType(pasteboardType) && [provider canCreateObjectOfClass:[NSAttributedString class]]) {
+            [values addObject:[provider createObjectOfClass:[NSAttributedString class] error:nil]];
             return;
         }
 
-        if (isImageType(pasteboardType) && [provider canInstantiateObjectOfClass:[getUIImageClass() class]]) {
-            [values addObject:[provider instantiateObjectOfClass:[getUIImageClass() class] options:nil error:nil]];
+        if (isStringType(pasteboardType) && [provider canCreateObjectOfClass:[NSString class]]) {
+            [values addObject:[provider createObjectOfClass:[NSString class] error:nil]];
             return;
         }
 

Source/WebKit2/ChangeLog

@@ -1,3 +1,36 @@
+2017-01-26  Wenson Hsieh  <wenson_hsieh@apple.com>
+
+        Add support for recognizing data interaction gestures in WebKit2
+        https://bugs.webkit.org/show_bug.cgi?id=167444
+
+        Reviewed by Beth Dakin.
+
+        Adds a new data interaction gesture recognizer, responsible for determining when to begin data interaction. This
+        is a new long press gesture recognizer that fires simultaneously with the existing long press gesture
+        recognizers (for performing long-press actions, and for showing the tap highlight).
+
+        Also tweaks logic for determining whether selection gesture recognizers should fire to account for data
+        interaction -- in particular, we don't want selection gesture recognizers to cause the current selection to
+        change while data interaction is possible. See -hasSelectablePositionAtPoint and -pointIsInAssistedNode for
+        more details.
+
+        * UIProcess/ios/PageClientImplIOS.mm:
+        (WebKit::PageClientImpl::didPerformDataInteractionControllerOperation):
+        (WebKit::PageClientImpl::startDataInteractionWithImage):
+        * UIProcess/ios/WKContentViewInteraction.h:
+        * UIProcess/ios/WKContentViewInteraction.mm:
+        (-[WKContentView _createAndConfigureLongPressGestureRecognizer]):
+        (-[WKContentView setupInteraction]):
+        (-[WKContentView cleanupInteraction]):
+        (-[WKContentView _removeDefaultGestureRecognizers]):
+        (-[WKContentView _addDefaultGestureRecognizers]):
+        (-[WKContentView resignFirstResponder]):
+        (-[WKContentView gestureRecognizer:shouldRecognizeSimultaneouslyWithGestureRecognizer:]):
+        (-[WKContentView gestureRecognizerShouldBegin:]):
+        (-[WKContentView hasSelectablePositionAtPoint:]):
+        (-[WKContentView pointIsInDataInteractionContent:]):
+        (-[WKContentView pointIsInAssistedNode:]):
+
 2017-01-26  Matt Rajca  <mrajca@apple.com>
 
         Notify clients when the user plays media otherwise prevented from autoplaying

Source/WebKit2/UIProcess/ios/PageClientImplIOS.mm

@@ -765,10 +765,12 @@ - (void)redoEditing:(id)sender
 #if ENABLE(DATA_INTERACTION)
 void PageClientImpl::didPerformDataInteractionControllerOperation()
 {
+    [m_contentView _didPerformDataInteractionControllerOperation];
 }
 
-void PageClientImpl::startDataInteractionWithImage(const WebCore::IntPoint& clientPosition, const ShareableBitmap::Handle& image, bool isLink)
+void PageClientImpl::startDataInteractionWithImage(const IntPoint& clientPosition, const ShareableBitmap::Handle& image, bool isLink)
 {
+    [m_contentView _startDataInteractionWithImage:ShareableBitmap::create(image)->makeCGImageCopy() atClientPosition:CGPointMake(clientPosition.x(), clientPosition.y()) isLink:isLink];
 }
 #endif
 

Source/WebKit2/UIProcess/ios/WKContentViewInteraction.h

@@ -44,6 +44,10 @@
 #import <wtf/Vector.h>
 #import <wtf/text/WTFString.h>
 
+#if USE(APPLE_INTERNAL_SDK) && __has_include(<WebKitAdditions/WKContentViewInteractionAdditions.h>)
+#import <WebKitAdditions/WKContentViewInteractionAdditions.h>
+#endif
+
 namespace API {
 class OpenPanelParameters;
 }
@@ -179,11 +183,22 @@ struct WKAutoCorrectionData {
 
     BOOL _resigningFirstResponder;
     BOOL _needsDeferredEndScrollingSelectionUpdate;
+
+#if ENABLE(DATA_INTERACTION)
+    RetainPtr<UILongPressGestureRecognizer> _dataInteractionGestureRecognizer;
+    RetainPtr<UIImage> _currentDataInteractionImage;
+    CGPoint _currentDataInteractionOrigin;
+    BOOL _shouldHandleLongPressActionAfterDataInteraction;
+#endif
 }
 
 @end
 
-@interface WKContentView (WKInteraction) <UIGestureRecognizerDelegate, UIWebTouchEventsGestureRecognizerDelegate, UITextInputPrivate, UIWebFormAccessoryDelegate, UIWKInteractionViewProtocol, WKFileUploadPanelDelegate, WKActionSheetAssistantDelegate>
+@interface WKContentView (WKInteraction) <UIGestureRecognizerDelegate, UIWebTouchEventsGestureRecognizerDelegate, UITextInputPrivate, UIWebFormAccessoryDelegate, UIWKInteractionViewProtocol, WKFileUploadPanelDelegate, WKActionSheetAssistantDelegate
+#if ENABLE(DATA_INTERACTION)
+    , WKViewDataInteractionSourceDelegate, WKDataInteractionSessionDelegate, WKViewDataInteractionDestinationDelegate, WKDataInteractionItemVisualTarget
+#endif
+>
 
 @property (nonatomic, readonly) CGPoint lastInteractionLocation;
 @property (nonatomic, readonly) BOOL isEditable;
@@ -232,6 +247,12 @@ struct WKAutoCorrectionData {
 - (NSArray *)_dataDetectionResults;
 - (NSArray<NSValue *> *)_uiTextSelectionRects;
 - (void)accessibilityRetrieveSpeakSelectionContent;
+
+#if ENABLE(DATA_INTERACTION)
+- (void)_didPerformDataInteractionControllerOperation;
+- (void)_startDataInteractionWithImage:(RetainPtr<CGImageRef>)image atClientPosition:(CGPoint)clientPosition isLink:(BOOL)isLink;
+#endif
+
 @end
 
 @interface WKContentView (WKTesting)