[Remote Inspection] AX: Add screen reader support for element targeting

https://bugs.webkit.org/show_bug.cgi?id=274108 rdar://128019127 Reviewed by Wenson Hsieh. To surface targeting information to screen readers, we need to add a new text property, screenReaderText, to expose those bits of rendered text relevant to screen reader users. * Source/WebCore/page/ElementTargetingController.cpp: (WebCore::targetedElementInfo): * Source/WebCore/page/ElementTargetingTypes.h: * Source/WebCore/page/text-extraction/TextExtraction.cpp: (WebCore::TextExtraction::extractRenderedText): * Source/WebCore/page/text-extraction/TextExtraction.h: * Source/WebKit/Shared/WebCoreArgumentCoders.serialization.in: * Source/WebKit/UIProcess/API/APITargetedElementInfo.h: * Source/WebKit/UIProcess/API/Cocoa/_WKTargetedElementInfo.h: * Source/WebKit/UIProcess/API/Cocoa/_WKTargetedElementInfo.mm: (-[_WKTargetedElementInfo screenReaderText]): * Tools/TestWebKitAPI/Tests/WebKitCocoa/ElementTargetingTests.mm: (TestWebKitAPI::TEST(ElementTargeting, TargetedElementScreenReaderText)): Canonical link: https://commits.webkit.org/278780@main
WebKit · May 14, 2024 · ab68b25 · ab68b25
1 parent b251507
commit ab68b25
Show file tree

Hide file tree

Showing 9 changed files with 39 additions and 13 deletions.
diff --git a/Source/WebCore/page/ElementTargetingController.cpp b/Source/WebCore/page/ElementTargetingController.cpp
@@ -598,8 +598,9 @@ static TargetedElementInfo targetedElementInfo(Element& element, IsNearbyTarget
         .elementIdentifier = element.identifier(),
         .documentIdentifier = element.document().identifier(),
         .offsetEdges = computeOffsetEdges(renderer->style()),
-        .renderedText = TextExtraction::extractRenderedText(element),
+        .renderedText = TextExtraction::extractRenderedText(element, TextExtraction::IncludeGeometryText::Yes),
         .searchableText = searchableTextForTarget(element),
+        .screenReaderText = TextExtraction::extractRenderedText(element, TextExtraction::IncludeGeometryText::No),
         .selectors = selectorsForTarget(element, cache),
         .boundsInRootView = element.boundingBoxInRootViewCoordinates(),
         .boundsInClientCoordinates = computeClientRect(*renderer),

diff --git a/Source/WebCore/page/ElementTargetingTypes.h b/Source/WebCore/page/ElementTargetingTypes.h
@@ -57,6 +57,7 @@ struct TargetedElementInfo {
     RectEdges<bool> offsetEdges;
     String renderedText;
     String searchableText;
+    String screenReaderText;
     Vector<Vector<String>> selectors;
     FloatRect boundsInRootView;
     FloatRect boundsInClientCoordinates;

diff --git a/Source/WebCore/page/text-extraction/TextExtraction.cpp b/Source/WebCore/page/text-extraction/TextExtraction.cpp
@@ -466,7 +466,7 @@ static IntSize reducePrecision(FloatSize size)
     };
 }
 
-static void extractRenderedText(Vector<StringsAndBlockOffset>& stringsAndOffsets, ContainerNode& node, BlockFlowDirection direction)
+static void extractRenderedText(Vector<StringsAndBlockOffset>& stringsAndOffsets, ContainerNode& node, BlockFlowDirection direction, IncludeGeometryText includeGeometryText)
 {
     CheckedPtr renderer = node.renderer();
     if (!renderer)
@@ -506,7 +506,7 @@ static void extractRenderedText(Vector<StringsAndBlockOffset>& stringsAndOffsets
 
     if (CheckedPtr frameRenderer = dynamicDowncast<RenderIFrame>(*renderer)) {
         if (auto contentDocument = frameRenderer->iframeElement().protectedContentDocument())
-            extractRenderedText(stringsAndOffsets, *contentDocument, direction);
+            extractRenderedText(stringsAndOffsets, *contentDocument, direction, includeGeometryText);
         return;
     }
 
@@ -518,9 +518,11 @@ static void extractRenderedText(Vector<StringsAndBlockOffset>& stringsAndOffsets
         appendStrings({ makeString('{', roundedSizeIgnoringZoom.width(), ',', roundedSizeIgnoringZoom.height(), '}') }, frameView->contentsToRootView(renderReplaced.absoluteBoundingBoxRect()));
     };
 
-    if (CheckedPtr renderReplaced = dynamicDowncast<RenderReplaced>(*renderer)) {
-        appendReplacedRenderer(*renderReplaced);
-        return;
+    if (includeGeometryText == IncludeGeometryText::Yes) {
+        if (CheckedPtr renderReplaced = dynamicDowncast<RenderReplaced>(*renderer)) {
+            appendReplacedRenderer(*renderReplaced);
+            return;
+        }
     }
 
     for (auto& descendant : descendantsOfType<RenderObject>(*renderer)) {
@@ -545,26 +547,28 @@ static void extractRenderedText(Vector<StringsAndBlockOffset>& stringsAndOffsets
 
         if (CheckedPtr frameRenderer = dynamicDowncast<RenderIFrame>(descendant)) {
             if (auto contentDocument = frameRenderer->iframeElement().protectedContentDocument())
-                extractRenderedText(stringsAndOffsets, *contentDocument, direction);
+                extractRenderedText(stringsAndOffsets, *contentDocument, direction, includeGeometryText);
             continue;
         }
 
-        if (CheckedPtr renderReplaced = dynamicDowncast<RenderReplaced>(descendant)) {
-            appendReplacedRenderer(*renderReplaced);
-            continue;
+        if (includeGeometryText == IncludeGeometryText::Yes) {
+            if (CheckedPtr renderReplaced = dynamicDowncast<RenderReplaced>(descendant)) {
+                appendReplacedRenderer(*renderReplaced);
+                continue;
+            }
         }
     }
 }
 
-String extractRenderedText(Element& element)
+String extractRenderedText(Element& element, IncludeGeometryText includeGeometryText)
 {
     if (!element.renderer())
         return emptyString();
 
     auto direction = element.renderer()->style().blockFlowDirection();
 
     Vector<StringsAndBlockOffset> stringsAndOffsets;
-    extractRenderedText(stringsAndOffsets, element, direction);
+    extractRenderedText(stringsAndOffsets, element, direction, includeGeometryText);
 
     bool ascendingOrder = [&] {
         switch (direction) {

diff --git a/Source/WebCore/page/text-extraction/TextExtraction.h b/Source/WebCore/page/text-extraction/TextExtraction.h
@@ -37,9 +37,11 @@ enum class ExceptionCode : uint8_t;
 
 namespace TextExtraction {
 
+enum class IncludeGeometryText : bool { No, Yes };
+
 WEBCORE_EXPORT Item extractItem(std::optional<WebCore::FloatRect>&& collectionRectInRootView, Page&);
 
-String extractRenderedText(Element&);
+String extractRenderedText(Element&, IncludeGeometryText);
 
 } // namespace TextExtraction
 } // namespace WebCore
diff --git a/Source/WebKit/Shared/WebCoreArgumentCoders.serialization.in b/Source/WebKit/Shared/WebCoreArgumentCoders.serialization.in
@@ -910,6 +910,7 @@ header: <WebCore/ElementTargetingTypes.h>
     WebCore::RectEdges<bool> offsetEdges
     String renderedText
     String searchableText
+    String screenReaderText;
     Vector<Vector<String>> selectors
     WebCore::FloatRect boundsInRootView
     WebCore::FloatRect boundsInClientCoordinates

diff --git a/Source/WebKit/UIProcess/API/APITargetedElementInfo.h b/Source/WebKit/UIProcess/API/APITargetedElementInfo.h
@@ -52,6 +52,7 @@ class TargetedElementInfo final : public ObjectImpl<Object::Type::TargetedElemen
 
     const WTF::String& renderedText() const { return m_info.renderedText; }
     const WTF::String& searchableText() const { return m_info.searchableText; }
+    const WTF::String& screenReaderText() const { return m_info.screenReaderText; }
     const Vector<Vector<WTF::String>>& selectors() const { return m_info.selectors; }
     WebCore::PositionType positionType() const { return m_info.positionType; }
     WebCore::FloatRect boundsInRootView() const { return m_info.boundsInRootView; }

diff --git a/Source/WebKit/UIProcess/API/Cocoa/_WKTargetedElementInfo.h b/Source/WebKit/UIProcess/API/Cocoa/_WKTargetedElementInfo.h
@@ -52,6 +52,7 @@ WK_CLASS_AVAILABLE(macos(WK_MAC_TBA), ios(WK_IOS_TBA), visionos(WK_XROS_TBA))
 @property (nonatomic, readonly, copy) NSArray<NSArray<NSString *> *> *selectorsIncludingShadowHosts;
 @property (nonatomic, readonly, copy) NSString *renderedText;
 @property (nonatomic, readonly, copy) NSString *searchableText;
+@property (nonatomic, readonly, copy) NSString *screenReaderText;
 @property (nonatomic, readonly) _WKRectEdge offsetEdges;
 
 // In root view coordinates. To be deprecated and removed, once clients adopt the more explicit bounds properties above.

diff --git a/Source/WebKit/UIProcess/API/Cocoa/_WKTargetedElementInfo.mm b/Source/WebKit/UIProcess/API/Cocoa/_WKTargetedElementInfo.mm
@@ -114,6 +114,11 @@ - (NSString *)searchableText
     return _info->searchableText();
 }
 
+- (NSString *)screenReaderText
+{
+    return _info->screenReaderText();
+}
+
 - (_WKRectEdge)offsetEdges
 {
     _WKRectEdge edges = _WKRectEdgeNone;

diff --git a/Tools/TestWebKitAPI/Tests/WebKitCocoa/ElementTargetingTests.mm b/Tools/TestWebKitAPI/Tests/WebKitCocoa/ElementTargetingTests.mm
@@ -501,4 +501,14 @@ @implementation _WKTargetedElementInfo (TestingAdditions)
     Util::run(&didAdjustment);
 }
 
+TEST(ElementTargeting, TargetedElementScreenReaderText)
+{
+    auto webView = adoptNS([[TestWKWebView alloc] initWithFrame:CGRectMake(0, 0, 800, 600)]);
+    [webView synchronouslyLoadTestPageNamed:@"element-targeting-7"];
+    RetainPtr element = [[webView targetedElementInfoAt:CGPointMake(100, 100)] firstObject];
+
+    EXPECT_TRUE([[element renderedText] containsString:@"{200,100}"]);
+    EXPECT_FALSE([[element screenReaderText] containsString:@"{200,100}"]);
+}
+
 } // namespace TestWebKitAPI