[Remote Inspection] Element targeting should additionally find nearby…

… out-of-flow elements

https://bugs.webkit.org/show_bug.cgi?id=271616

Reviewed by Abrar Protyasha.

When targeting elements for remote inspection, additionally surface nearby elements which aren't
precisely under the hit test location. Our strategy for this consists of the following:

1.  While collecting targets, aggregate a `Region` containing the rects of all targeted out-of-flow
    elements (and hit-tested elements underneath targeted elements).

2.  After building the list of targets, scan the entire DOM for out-of-flow renderers that are also
    contained in the "nearby targets" region, which also satisfy the same criteria for element
    targeting.

3.  Add these as "nearby targets" to the final list of target infos, to the end of the array.

See below for more details.

Test: ElementTargeting.NearbyOutOfFlowElements

* Source/WebCore/page/ElementTargeting.cpp:
(WebCore::targetedElementInfo):

Refactor this code to pull common logic into lambdas, and implement the steps detailed above.

(WebCore::findTargetedElements):
* Source/WebCore/page/ElementTargetingTypes.h:
* Source/WebKit/Shared/WebCoreArgumentCoders.serialization.in:
* Source/WebKit/UIProcess/API/APITargetedElementInfo.h:
* Source/WebKit/UIProcess/API/Cocoa/WKWebView.mm:
(-[WKWebView _requestTargetedElementInfo:completionHandler:]):
* Source/WebKit/UIProcess/API/Cocoa/_WKTargetedElementInfo.h:

Add a new `isUnderPoint` property, which is `YES` for elements that are directly hit-tested, and
`NO` for nearby targets.

* Source/WebKit/UIProcess/API/Cocoa/_WKTargetedElementInfo.mm:
(-[_WKTargetedElementInfo isUnderPoint]):
* Source/WebKit/UIProcess/API/Cocoa/_WKTargetedElementRequest.h:

Add a new `canIncludeNearbyElements` (default: `YES`) which determines whether or not element
targeting should include elements that have not been hit-tested, but are visually contained within
another element that has been hit-tested.

* Source/WebKit/UIProcess/API/Cocoa/_WKTargetedElementRequest.mm:
(-[_WKTargetedElementRequest init]):
* Tools/TestWebKitAPI/TestWebKitAPI.xcodeproj/project.pbxproj:
* Tools/TestWebKitAPI/Tests/WebKitCocoa/ElementTargetingTests.mm:
(TestWebKitAPI::TEST):
* Tools/TestWebKitAPI/Tests/WebKitCocoa/element-targeting-2.html: Added.

Add a new API test to exercise the change.

Canonical link: https://commits.webkit.org/276670@main

Source/WebCore/page/ElementTargeting.cpp

@@ -41,6 +41,7 @@
 #include "LocalFrameView.h"
 #include "NodeList.h"
 #include "Page.h"
+#include "Region.h"
 #include "RenderDescendantIterator.h"
 #include "RenderIFrame.h"
 #include "RenderView.h"
@@ -196,7 +197,8 @@ static inline Vector<FrameIdentifier> collectChildFrameIdentifiers(Element& elem
     return identifiers;
 }
 
-static TargetedElementInfo targetedElementInfo(Element& element)
+enum class IsUnderPoint : bool { No, Yes };
+static TargetedElementInfo targetedElementInfo(Element& element, IsUnderPoint isUnderPoint)
 {
     CheckedPtr renderer = element.renderer();
     return {
@@ -208,6 +210,7 @@ static TargetedElementInfo targetedElementInfo(Element& element)
         .boundsInRootView = element.boundingBoxInRootViewCoordinates(),
         .positionType = renderer->style().position(),
         .childFrameIdentifiers = collectChildFrameIdentifiers(element),
+        .isUnderPoint = isUnderPoint == IsUnderPoint::Yes,
     };
 }
 
@@ -260,70 +263,87 @@ Vector<TargetedElementInfo> findTargetedElements(Page& page, TargetedElementRequ
         onlyMainElement = &descendant;
     }
 
-    auto candidates = [&] {
-        auto& results = result.listBasedTestResult();
-        Vector<Ref<Element>> elements;
-        elements.reserveInitialCapacity(results.size());
-        for (auto& node : results) {
-            RefPtr element = dynamicDowncast<Element>(node);
-            if (!element)
-                continue;
+    auto isCandidate = [&](Element& element) {
+        if (!element.renderer())
+            return false;
 
-            if (!element->renderer())
-                continue;
+        if (&element == document->body())
+            return false;
 
-            if (element == document->body())
-                continue;
+        if (&element == document->documentElement())
+            return false;
 
-            if (element == document->documentElement())
-                continue;
+        if (onlyMainElement && (onlyMainElement == &element || element.contains(*onlyMainElement)))
+            return false;
 
-            if (elementAndAncestorsAreOnlyChildren(*element))
-                continue;
+        if (elementAndAncestorsAreOnlyChildren(element))
+            return false;
 
-            if (onlyMainElement && (onlyMainElement == element || element->contains(*onlyMainElement)))
-                continue;
+        return true;
+    };
 
-            elements.append(element.releaseNonNull());
+    auto candidates = [&] {
+        auto& results = result.listBasedTestResult();
+        Vector<Ref<Element>> elements;
+        elements.reserveInitialCapacity(results.size());
+        for (auto& node : results) {
+            if (RefPtr element = dynamicDowncast<Element>(node); element && isCandidate(*element))
+                elements.append(element.releaseNonNull());
         }
         return elements;
     }();
 
-    Vector<Ref<Element>> targets; // The front-most target is last in this list.
-    auto addTarget = [&](Element& newTarget) {
-        targets.append(newTarget);
-        candidates.removeAllMatching([&](auto& element) {
-            return &newTarget == element.ptr() || newTarget.contains(element);
-        });
+    static constexpr auto maximumAreaRatioForAbsolutelyPositionedContent = 0.75;
+    static constexpr auto maximumAreaRatioForInFlowContent = 0.5;
+    static constexpr auto maximumAreaRatioForNearbyTargets = 0.25;
+    static constexpr auto minimumAreaRatioForInFlowContent = 0.01;
+
+    auto computeViewportAreaRatio = [&](IntRect boundingBox) {
+        auto area = boundingBox.area<RecordOverflow>();
+        return area.hasOverflowed() ? std::numeric_limits<float>::max() : area.value() / viewportArea;
     };
 
-    static constexpr auto areaRatioForAbsolutelyPositionedContent = 0.75;
-    static constexpr auto areaRatioForInFlowContent = 0.5;
-    static constexpr auto minimumAreaForNonFixedOrStickyContent = 10000;
+    Vector<Ref<Element>> targets; // The front-most target is last in this list.
+    Region additionalRegionForNearbyElements;
 
     // Prioritize parent elements over their children by traversing backwards over the candidates.
     // This allows us to target only the top-most container elements that satisfy the criteria.
+    // While adding targets, we also accumulate additional regions, wherein we should report any
+    // nearby targets.
     while (!candidates.isEmpty()) {
-        Ref element = candidates.takeLast();
-        CheckedPtr renderer = element->renderer();
-        if (renderer->isFixedPositioned() || renderer->isStickilyPositioned()) {
-            addTarget(element);
+        Ref target = candidates.takeLast();
+        CheckedPtr targetRenderer = target->renderer();
+        auto targetBoundingBox = target->boundingBoxInRootViewCoordinates();
+        auto targetAreaRatio = computeViewportAreaRatio(targetBoundingBox);
+        bool shouldAddTarget = targetRenderer->isFixedPositioned()
+            || targetRenderer->isStickilyPositioned()
+            || (targetRenderer->isAbsolutelyPositioned() && targetAreaRatio < maximumAreaRatioForAbsolutelyPositionedContent)
+            || (minimumAreaRatioForInFlowContent < targetAreaRatio && targetAreaRatio < maximumAreaRatioForInFlowContent);
+
+        if (!shouldAddTarget)
             continue;
-        }
 
-        auto boundingBox = element->boundingBoxInRootViewCoordinates();
-        auto area = boundingBox.area<RecordOverflow>();
-        if (!area.hasOverflowed() && area.value() < minimumAreaForNonFixedOrStickyContent)
-            continue;
+        bool checkForNearbyTargets = request.canIncludeNearbyElements
+            && targetRenderer->isOutOfFlowPositioned()
+            && targetAreaRatio < maximumAreaRatioForNearbyTargets;
 
-        auto elementAreaRatio = area.hasOverflowed() ? std::numeric_limits<float>::max() : area.value() / viewportArea;
-        if (renderer->isAbsolutelyPositioned() && elementAreaRatio < areaRatioForAbsolutelyPositionedContent) {
-            addTarget(element);
-            continue;
-        }
+        if (checkForNearbyTargets && computeViewportAreaRatio(targetBoundingBox) < maximumAreaRatioForNearbyTargets)
+            additionalRegionForNearbyElements.unite(targetBoundingBox);
+
+        candidates.removeAllMatching([&](auto& candidate) {
+            if (target.ptr() != candidate.ptr() && !target->contains(candidate))
+                return false;
+
+            if (checkForNearbyTargets) {
+                auto boundingBox = candidate->boundingBoxInRootViewCoordinates();
+                if (computeViewportAreaRatio(boundingBox) < maximumAreaRatioForNearbyTargets)
+                    additionalRegionForNearbyElements.unite(boundingBox);
+            }
+
+            return true;
+        });
 
-        if (elementAreaRatio < areaRatioForInFlowContent)
-            addTarget(element);
+        targets.append(WTFMove(target));
     }
 
     if (targets.isEmpty())
@@ -332,7 +352,48 @@ Vector<TargetedElementInfo> findTargetedElements(Page& page, TargetedElementRequ
     Vector<TargetedElementInfo> results;
     results.reserveInitialCapacity(targets.size());
     for (auto iterator = targets.rbegin(); iterator != targets.rend(); ++iterator)
-        results.append(targetedElementInfo(*iterator));
+        results.append(targetedElementInfo(*iterator, IsUnderPoint::Yes));
+
+    if (additionalRegionForNearbyElements.isEmpty())
+        return results;
+
+    auto nearbyTargets = [&] {
+        HashSet<Ref<Element>> targets;
+        CheckedPtr bodyRenderer = bodyElement->renderer();
+        if (!bodyRenderer)
+            return targets;
+
+        for (auto& renderer : descendantsOfType<RenderElement>(*bodyRenderer)) {
+            if (!renderer.isOutOfFlowPositioned())
+                continue;
+
+            RefPtr element = renderer.protectedElement();
+            if (!element)
+                continue;
+
+            if (targets.contains(*element))
+                continue;
+
+            if (result.listBasedTestResult().contains(*element))
+                continue;
+
+            if (!isCandidate(*element))
+                continue;
+
+            auto boundingBox = element->boundingBoxInRootViewCoordinates();
+            if (!additionalRegionForNearbyElements.contains(boundingBox))
+                continue;
+
+            if (computeViewportAreaRatio(boundingBox) > maximumAreaRatioForNearbyTargets)
+                continue;
+
+            targets.add(element.releaseNonNull());
+        }
+        return targets;
+    }();
+
+    for (auto& element : nearbyTargets)
+        results.append(targetedElementInfo(element, IsUnderPoint::No));
 
     return results;
 }

Source/WebCore/page/ElementTargetingTypes.h

@@ -39,6 +39,7 @@ namespace WebCore {
 
 struct TargetedElementRequest {
     FloatPoint pointInRootView;
+    bool canIncludeNearbyElements { true };
 };
 
 struct TargetedElementInfo {
@@ -50,6 +51,7 @@ struct TargetedElementInfo {
     FloatRect boundsInRootView;
     PositionType positionType { PositionType::Static };
     Vector<FrameIdentifier> childFrameIdentifiers;
+    bool isUnderPoint { true };
 };
 
 } // namespace WebCore

Source/WebKit/Shared/WebCoreArgumentCoders.serialization.in

@@ -886,6 +886,7 @@ enum class WebCore::ShareDataOriginator : bool
 header: <WebCore/ElementTargetingTypes.h>
 [CustomHeader] struct WebCore::TargetedElementRequest {
     WebCore::FloatPoint pointInRootView
+    bool canIncludeNearbyElements
 };
 
 header: <WebCore/ElementTargetingTypes.h>
@@ -898,6 +899,7 @@ header: <WebCore/ElementTargetingTypes.h>
     WebCore::FloatRect boundsInRootView
     WebCore::PositionType positionType
     Vector<WebCore::FrameIdentifier> childFrameIdentifiers
+    bool isUnderPoint
 };
 
 header: <WebCore/RenderStyleConstants.h>

Source/WebKit/UIProcess/API/APITargetedElementInfo.h

@@ -56,6 +56,8 @@ class TargetedElementInfo final : public ObjectImpl<Object::Type::TargetedElemen
     WebCore::FloatRect boundsInRootView() const { return m_info.boundsInRootView; }
     WebCore::FloatRect boundsInWebView() const;
 
+    bool isUnderPoint() const { return m_info.isUnderPoint; }
+
     void childFrames(CompletionHandler<void(Vector<Ref<FrameTreeNode>>&&)>&&) const;
 
     bool isSameElement(const TargetedElementInfo&) const;

Source/WebKit/UIProcess/API/Cocoa/WKWebView.mm

@@ -2753,6 +2753,7 @@ - (void)_requestTargetedElementInfo:(_WKTargetedElementRequest *)request complet
 #else
         request.point,
 #endif
+        static_cast<bool>(request.canIncludeNearbyElements)
     };
     _page->requestTargetedElement(WTFMove(coreRequest), [completion = makeBlockPtr(completion)](auto& elements) {
         completion(createNSArray(elements, [](auto& element) {

Source/WebKit/UIProcess/API/Cocoa/_WKTargetedElementInfo.h

@@ -42,6 +42,7 @@ WK_CLASS_AVAILABLE(macos(WK_MAC_TBA), ios(WK_IOS_TBA))
 
 @property (nonatomic, readonly) _WKTargetedElementPosition positionType;
 @property (nonatomic, readonly) CGRect bounds;
+@property (nonatomic, readonly, getter=isUnderPoint) BOOL underPoint;
 
 @property (nonatomic, readonly, copy) NSArray<NSString *> *selectors;
 @property (nonatomic, readonly, copy) NSString *renderedText;

Source/WebKit/UIProcess/API/Cocoa/_WKTargetedElementInfo.mm

@@ -110,4 +110,9 @@ - (BOOL)isSameElement:(_WKTargetedElementInfo *)other
     return _info->isSameElement(*other->_info);
 }
 
+- (BOOL)isUnderPoint
+{
+    return _info->isUnderPoint();
+}
+
 @end

Source/WebKit/UIProcess/API/Cocoa/_WKTargetedElementRequest.h

@@ -33,5 +33,6 @@ WK_CLASS_AVAILABLE(macos(WK_MAC_TBA), ios(WK_IOS_TBA))
 @interface _WKTargetedElementRequest : NSObject
 
 @property (nonatomic) CGPoint point;
+@property (nonatomic) BOOL canIncludeNearbyElements;
 
 @end

Source/WebKit/UIProcess/API/Cocoa/_WKTargetedElementRequest.mm

@@ -28,4 +28,13 @@
 
 @implementation _WKTargetedElementRequest
 
+- (instancetype)init
+{
+    if (!(self = [super init]))
+        return nil;
+
+    _canIncludeNearbyElements = YES;
+    return self;
+}
+
 @end

Tools/TestWebKitAPI/TestWebKitAPI.xcodeproj/project.pbxproj

@@ -1168,6 +1168,7 @@
 		F42F081227449892007E0D90 /* multiple-images.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = F42F0811274497C8007E0D90 /* multiple-images.html */; };
 		F434CA1A22E65BCA005DDB26 /* ScrollToRevealSelection.mm in Sources */ = {isa = PBXBuildFile; fileRef = F434CA1922E65BCA005DDB26 /* ScrollToRevealSelection.mm */; };
 		F4352F9F26D403DE00E605E4 /* editable-responsive-body.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = F4352F9E26D4037000E605E4 /* editable-responsive-body.html */; };
+		F4365E2B2BB11829005E8C1A /* element-targeting-2.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = F4365E232BB1181A005E8C1A /* element-targeting-2.html */; };
 		F43CAB1D278A326500C8D0A2 /* CloseWhileCommittingLoad.mm in Sources */ = {isa = PBXBuildFile; fileRef = F43CAB1C278A326500C8D0A2 /* CloseWhileCommittingLoad.mm */; };
 		F43E3BBF20DADA1E00A4E7ED /* WKScrollViewTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = F43E3BBE20DADA1E00A4E7ED /* WKScrollViewTests.mm */; };
 		F43E3BC120DADBC500A4E7ED /* fixed-nav-bar.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = F43E3BC020DADB8000A4E7ED /* fixed-nav-bar.html */; };
@@ -1623,6 +1624,7 @@
 				F44D06451F395C26001A0E29 /* editor-state-test-harness.html in Copy Resources */,
 				F4BDA43027F8D19C00F9647D /* element-fullscreen.html in Copy Resources */,
 				F4DADD072BABAD0F008B398F /* element-targeting-1.html in Copy Resources */,
+				F4365E2B2BB11829005E8C1A /* element-targeting-2.html in Copy Resources */,
 				51C8E1A91F27F49600BF731B /* EmptyGrandfatheredResourceLoadStatistics.plist in Copy Resources */,
 				49D902B328209B3300E2C3B8 /* emptyTable.html in Copy Resources */,
 				A14AAB651E78DC5400C1ADC2 /* encrypted.pdf in Copy Resources */,
@@ -3523,6 +3525,7 @@
 		F42F081727449FFD007E0D90 /* ImageAnalysisTestingUtilities.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ImageAnalysisTestingUtilities.h; path = cocoa/ImageAnalysisTestingUtilities.h; sourceTree = "<group>"; };
 		F434CA1922E65BCA005DDB26 /* ScrollToRevealSelection.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = ScrollToRevealSelection.mm; sourceTree = "<group>"; };
 		F4352F9E26D4037000E605E4 /* editable-responsive-body.html */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.html; path = "editable-responsive-body.html"; sourceTree = "<group>"; };
+		F4365E232BB1181A005E8C1A /* element-targeting-2.html */ = {isa = PBXFileReference; lastKnownFileType = text.html; path = "element-targeting-2.html"; sourceTree = "<group>"; };
 		F43C3823278133190099ABCE /* NSResponderTests.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = NSResponderTests.mm; sourceTree = "<group>"; };
 		F43CAB1C278A326500C8D0A2 /* CloseWhileCommittingLoad.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = CloseWhileCommittingLoad.mm; sourceTree = "<group>"; };
 		F43E3BBE20DADA1E00A4E7ED /* WKScrollViewTests.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = WKScrollViewTests.mm; sourceTree = "<group>"; };
@@ -4823,6 +4826,7 @@
 				F44D06441F395C0D001A0E29 /* editor-state-test-harness.html */,
 				F4BDA42F27F8CF5600F9647D /* element-fullscreen.html */,
 				F4DADCFF2BABA80C008B398F /* element-targeting-1.html */,
+				F4365E232BB1181A005E8C1A /* element-targeting-2.html */,
 				51C8E1A81F27F47300BF731B /* EmptyGrandfatheredResourceLoadStatistics.plist */,
 				F4C2AB211DD6D94100E06D5B /* enormous-video-with-sound.html */,
 				F407FE381F1D0DE60017CF25 /* enormous.svg */,

Tools/TestWebKitAPI/Tests/WebKitCocoa/ElementTargetingTests.mm

@@ -131,4 +131,26 @@ @implementation _WKTargetedElementInfo (TestingAdditions)
     }
 }
 
+TEST(ElementTargeting, NearbyOutOfFlowElements)
+{
+    auto webView = adoptNS([[TestWKWebView alloc] initWithFrame:CGRectMake(0, 0, 800, 600)]);
+    [webView synchronouslyLoadTestPageNamed:@"element-targeting-2"];
+
+    auto elements = [webView targetedElementInfoAt:CGPointMake(100, 100)];
+    EXPECT_EQ(elements.count, 4U);
+    EXPECT_TRUE(elements[0].underPoint);
+    EXPECT_FALSE(elements[1].underPoint);
+    EXPECT_FALSE(elements[2].underPoint);
+    EXPECT_FALSE(elements[3].underPoint);
+    EXPECT_WK_STREQ(".fixed.container", elements[0].selectors.firstObject);
+    __auto_type nextThreeSelectors = [NSSet setWithArray:@[
+        elements[1].selectors.firstObject,
+        elements[2].selectors.firstObject,
+        elements[3].selectors.firstObject,
+    ]];
+    EXPECT_TRUE([nextThreeSelectors containsObject:@".absolute.top-right"]);
+    EXPECT_TRUE([nextThreeSelectors containsObject:@".absolute.bottom-left"]);
+    EXPECT_TRUE([nextThreeSelectors containsObject:@".absolute.bottom-right"]);
+}
+
 } // namespace TestWebKitAPI