Skip to content

AX: Add a fast path for assistive technologies that search for all live regions and all non-root web areas on macOS #41898

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 5, 2025
Merged

AX: Add a fast path for assistive technologies that search for all live regions and all non-root web areas on macOS #41898

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions LayoutTests/accessibility/mac/live-region-search-expected.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
This test ensures that live region searches work correctly after dynamic page changes.

PASS: webArea.uiElementCountForSearchPredicate(null, true, 'AXLiveRegionSearchKey', '', false, false) === 0
PASS: webArea.uiElementCountForSearchPredicate(null, true, 'AXLiveRegionSearchKey', '', false, false) === 1
PASS: webArea.uiElementCountForSearchPredicate(null, true, 'AXLiveRegionSearchKey', '', false, false) === 0
PASS: webArea.uiElementCountForSearchPredicate(null, true, 'AXLiveRegionSearchKey', '', false, false) === 1
PASS: webArea.uiElementCountForSearchPredicate(null, true, 'AXLiveRegionSearchKey', '', false, false) === 0
PASS: webArea.uiElementCountForSearchPredicate(null, true, 'AXLiveRegionSearchKey', '', false, false) === 1
PASS: webArea.uiElementCountForSearchPredicate(null, true, 'AXLiveRegionSearchKey', '', false, false) === 2
PASS: webArea.uiElementCountForSearchPredicate(null, true, 'AXLiveRegionSearchKey', '', false, false) === 3
PASS: webArea.uiElementCountForSearchPredicate(null, true, 'AXLiveRegionSearchKey', '', false, false) === 2
PASS: webArea.uiElementCountForSearchPredicate(null, true, 'AXLiveRegionSearchKey', '', false, false) === 1

PASS successfullyParsed is true

TEST COMPLETE
Hello
59 changes: 59 additions & 0 deletions LayoutTests/accessibility/mac/live-region-search.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN">
<html>
<head>
<script src="../../resources/accessibility-helper.js"></script>
<script src="../../resources/js-test.js"></script>
</head>
<body>

<div id="container">
</div>

<script>
var output = "This test ensures that live region searches work correctly after dynamic page changes.\n\n";

if (window.accessibilityController) {
window.jsTestIsAsync = true;

var container = document.getElementById("container");
var webArea = accessibilityController.rootElement.childAtIndex(0);
output += expect("webArea.uiElementCountForSearchPredicate(null, true, 'AXLiveRegionSearchKey', '', false, false)", "0");

var dynamicStatus = document.createElement("div");
dynamicStatus.id = "dynamic-status";
dynamicStatus.role = "status";
dynamicStatus.innerText = "Hello";

var dynamicStatus2 = document.createElement("div");
dynamicStatus2.id = "dynamic-status2";
dynamicStatus2.role = "status";
dynamicStatus2.innerText = "Hello2";

container.appendChild(dynamicStatus);
setTimeout(async function() {
output += await expectAsync("webArea.uiElementCountForSearchPredicate(null, true, 'AXLiveRegionSearchKey', '', false, false)", "1");
dynamicStatus = document.getElementById("container").removeChild(dynamicStatus);
output += await expectAsync("webArea.uiElementCountForSearchPredicate(null, true, 'AXLiveRegionSearchKey', '', false, false)", "0");
container.appendChild(dynamicStatus);
output += await expectAsync("webArea.uiElementCountForSearchPredicate(null, true, 'AXLiveRegionSearchKey', '', false, false)", "1");
dynamicStatus.role = "group";
output += await expectAsync("webArea.uiElementCountForSearchPredicate(null, true, 'AXLiveRegionSearchKey', '', false, false)", "0");
dynamicStatus.setAttribute("aria-live", "polite");
output += await expectAsync("webArea.uiElementCountForSearchPredicate(null, true, 'AXLiveRegionSearchKey', '', false, false)", "1");
container.setAttribute("aria-live", "polite");
output += await expectAsync("webArea.uiElementCountForSearchPredicate(null, true, 'AXLiveRegionSearchKey', '', false, false)", "2");
container.appendChild(dynamicStatus2);
output += await expectAsync("webArea.uiElementCountForSearchPredicate(null, true, 'AXLiveRegionSearchKey', '', false, false)", "3");
container.removeAttribute("aria-live");
output += await expectAsync("webArea.uiElementCountForSearchPredicate(null, true, 'AXLiveRegionSearchKey', '', false, false)", "2");
dynamicStatus2.remove();
output += await expectAsync("webArea.uiElementCountForSearchPredicate(null, true, 'AXLiveRegionSearchKey', '', false, false)", "1");

debug(output);
finishJSTest();
}, 0);
}
</script>
</body>
</html>

7 changes: 7 additions & 0 deletions Source/WebCore/accessibility/AXCoreObject.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,8 @@ enum class ClickHandlerFilter : bool {
IncludeBody,
};

enum class PreSortedObjectType : bool { LiveRegion, WebArea };

enum class DateComponentsType : uint8_t;

enum class AXIDType { };
Expand Down Expand Up @@ -1366,6 +1368,11 @@ class AXCoreObject : public ThreadSafeRefCountedAndCanMakeThreadSafeWeakPtr<AXCo
// ARIA live-region features.
static bool liveRegionStatusIsEnabled(const AtomString&);
bool supportsLiveRegion(bool excludeIfOff = true) const;
#if PLATFORM(MAC)
virtual AccessibilityChildrenVector allSortedLiveRegions() const = 0;
virtual AccessibilityChildrenVector allSortedNonRootWebAreas() const = 0;
AccessibilityChildrenVector sortedDescendants(size_t limit, PreSortedObjectType) const;
#endif // PLATFORM(MAC)
virtual AXCoreObject* liveRegionAncestor(bool excludeIfOff = true) const = 0;
virtual const String liveRegionStatus() const = 0;
virtual const String liveRegionRelevant() const = 0;
Expand Down
67 changes: 53 additions & 14 deletions Source/WebCore/accessibility/AXObjectCache.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1140,9 +1140,18 @@ void AXObjectCache::remove(std::optional<AXID> axID)
return;

#if ENABLE(ACCESSIBILITY_ISOLATED_TREE)
if (auto tree = AXIsolatedTree::treeForPageID(m_pageID))
unsigned liveRegionsRemoved = m_sortedLiveRegionIDs.removeAll(*axID);
unsigned webAreasRemoved = m_sortedNonRootWebAreaIDs.removeAll(*axID);

if (RefPtr tree = AXIsolatedTree::treeForPageID(m_pageID)) {
tree->queueNodeRemoval(*object);
#endif

if (liveRegionsRemoved)
tree->sortedLiveRegionsDidChange(m_sortedLiveRegionIDs);
else if (webAreasRemoved)
tree->sortedNonRootWebAreasDidChange(m_sortedNonRootWebAreaIDs);
}
#endif // ENABLE(ACCESSIBILITY_ISOLATED_TREE)

removeAllRelations(*axID);
object->detach(AccessibilityDetachmentType::ElementDestroyed);
Expand Down Expand Up @@ -1507,8 +1516,15 @@ void AXObjectCache::handleLiveRegionCreated(Element& element)
liveRegionStatus = AtomString { AccessibilityObject::defaultLiveRegionStatusForRole(AccessibilityObject::ariaRoleToWebCoreRole(ariaRole)) };
}

if (AXCoreObject::liveRegionStatusIsEnabled(liveRegionStatus))
postNotification(getOrCreate(element), protectedDocument().get(), AXNotification::LiveRegionCreated);
if (AXCoreObject::liveRegionStatusIsEnabled(liveRegionStatus)) {
RefPtr axObject = getOrCreate(element);
#if PLATFORM(MAC)
if (axObject)
addSortedObject(*axObject, PreSortedObjectType::LiveRegion);
#endif // PLATFORM(MAC)

postNotification(axObject.get(), protectedDocument().get(), AXNotification::LiveRegionCreated);
}
}

void AXObjectCache::deferElementAddedOrRemoved(Element* element)
Expand Down Expand Up @@ -2457,8 +2473,8 @@ void AXObjectCache::postLiveRegionChangeNotification(AccessibilityObject& object
m_liveRegionChangedPostTimer.stop();

Ref objectRef = object;
if (!m_liveRegionObjects.contains(objectRef))
m_liveRegionObjects.add(WTFMove(objectRef));
if (!m_changedLiveRegions.contains(objectRef))
m_changedLiveRegions.add(WTFMove(objectRef));

m_liveRegionChangedPostTimer.startOneShot(0_s);
}
Expand All @@ -2467,12 +2483,12 @@ void AXObjectCache::liveRegionChangedNotificationPostTimerFired()
{
m_liveRegionChangedPostTimer.stop();

if (m_liveRegionObjects.isEmpty())
if (m_changedLiveRegions.isEmpty())
return;

for (auto& object : m_liveRegionObjects)
for (auto& object : m_changedLiveRegions)
postNotification(object.ptr(), object->protectedDocument().get(), AXNotification::LiveRegionChanged);
m_liveRegionObjects.clear();
m_changedLiveRegions.clear();
}

void AXObjectCache::onScrollbarUpdate(ScrollView& view)
Expand Down Expand Up @@ -2620,11 +2636,20 @@ void AXObjectCache::handleRoleChanged(Element& element, const AtomString& oldVal
object->updateRole();
}

void AXObjectCache::handleRoleChanged(AccessibilityObject& axObject)
void AXObjectCache::handleRoleChanged(AccessibilityObject& axObject, AccessibilityRole oldRole)
{
stopCachingComputedObjectAttributes();
axObject.recomputeIsIgnored();

#if PLATFORM(MAC)
if (axObject.supportsLiveRegion())
addSortedObject(axObject, PreSortedObjectType::LiveRegion);
else if (AXCoreObject::liveRegionStatusIsEnabled(AtomString { AccessibilityObject::defaultLiveRegionStatusForRole(oldRole) }))
removeLiveRegion(axObject);
#else
UNUSED_PARAM(oldRole);
#endif // PLATFORM(MAC)

#if ENABLE(ACCESSIBILITY_ISOLATED_TREE)
postNotification(axObject, AXNotification::RoleChanged);
#endif
Expand Down Expand Up @@ -2837,8 +2862,22 @@ void AXObjectCache::handleAttributeChange(Element* element, const QualifiedName&
postNotification(element, AXNotification::KeyShortcutsChanged);
else if (attrName == aria_levelAttr)
postNotification(element, AXNotification::LevelChanged);
else if (attrName == aria_liveAttr)
else if (attrName == aria_liveAttr) {
postNotification(element, AXNotification::LiveRegionStatusChanged);

#if PLATFORM(MAC)
if (m_sortedIDListsInitialized) {
// Once the sorted ID lists have been initialized for the first time, we rely
// on handling these dynamic updates to keep them up-to-date.
if (RefPtr object = getOrCreate(element)) {
if (object->supportsLiveRegion())
addSortedObject(*object, PreSortedObjectType::LiveRegion);
else
removeLiveRegion(*object);
}
}
#endif // PLATFORM(MAC)
}
else if (attrName == aria_placeholderAttr)
postNotification(element, AXNotification::PlaceholderChanged);
else if (attrName == aria_rowindexAttr) {
Expand Down Expand Up @@ -4398,6 +4437,9 @@ void AXObjectCache::performDeferredCacheUpdate(ForceLayout forceLayout)
});
m_deferredRecomputeTableIsExposedList.clear();

AXLOGDeferredCollection("ChildrenChangedList"_s, m_deferredChildrenChangedList);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Out of curiosity, what is the motivation for moving children changed up here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need to update the accessibility tree before any calls to addSortedObject, as otherwise we'll be trying to find the right place for a sorted object based on outdated accessibility tree state. This moves handling children-changed up above where we would potentially call addSortedObject.

handleAllDeferredChildrenChanged();

AXLOGDeferredCollection("ElementAddedOrRemovedList"_s, m_deferredElementAddedOrRemovedList);
auto nodeAddedOrRemovedList = copyToVector(m_deferredElementAddedOrRemovedList);
for (auto& weakNode : nodeAddedOrRemovedList) {
Expand All @@ -4413,9 +4455,6 @@ void AXObjectCache::performDeferredCacheUpdate(ForceLayout forceLayout)
}
m_deferredElementAddedOrRemovedList.clear();

AXLOGDeferredCollection("ChildrenChangedList"_s, m_deferredChildrenChangedList);
handleAllDeferredChildrenChanged();

#if ENABLE(ACCESSIBILITY_ISOLATED_TREE)
AXLOGDeferredCollection("UnconnectedObjects"_s, m_deferredUnconnectedObjects);
if (auto tree = AXIsolatedTree::treeForPageID(m_pageID)) {
Expand Down
25 changes: 23 additions & 2 deletions Source/WebCore/accessibility/AXObjectCache.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -373,7 +373,7 @@ class AXObjectCache final : public CanMakeWeakPtr<AXObjectCache>, public CanMake
void valueChanged(Element&);
void checkedStateChanged(Element&);
void autofillTypeChanged(HTMLInputElement&);
void handleRoleChanged(AccessibilityObject&);
void handleRoleChanged(AccessibilityObject&, AccessibilityRole previousRole);

#if ENABLE(ACCESSIBILITY_ISOLATED_TREE)
void columnIndexChanged(AccessibilityObject&);
Expand All @@ -386,6 +386,7 @@ class AXObjectCache final : public CanMakeWeakPtr<AXObjectCache>, public CanMake
// re-parented into a subtree that does have a renderer.
void onRendererCreated(Element&);
#if PLATFORM(MAC)
void onDocumentRenderTreeCreation(const Document&);
void onSelectedTextChanged(const VisiblePositionRange&);
#endif
#if ENABLE(AX_THREAD_TEXT_APIS)
Expand Down Expand Up @@ -605,6 +606,12 @@ class AXObjectCache final : public CanMakeWeakPtr<AXObjectCache>, public CanMake
#endif

#if PLATFORM(MAC)
AXCoreObject::AccessibilityChildrenVector sortedLiveRegions() const;
AXCoreObject::AccessibilityChildrenVector sortedNonRootWebAreas() const;
void addSortedObject(AccessibilityObject&, PreSortedObjectType);
void removeLiveRegion(AccessibilityObject&);
void initializeSortedIDLists();

static bool clientIsInTestMode();
#endif

Expand Down Expand Up @@ -816,7 +823,18 @@ class AXObjectCache final : public CanMakeWeakPtr<AXObjectCache>, public CanMake
ListHashSet<Ref<AccessibilityObject>> m_passwordNotificationsToPost;

Timer m_liveRegionChangedPostTimer;
ListHashSet<Ref<AccessibilityObject>> m_liveRegionObjects;
ListHashSet<Ref<AccessibilityObject>> m_changedLiveRegions;

#if PLATFORM(MAC)
// This block is PLATFORM(MAC) because the remote search API is currently only used on macOS.

// AX tree-order-sorted list of a few types of objects. This is useful because some assistive
// technologies send us frequent remote search requests for all the live regions or non-root webareas
// on the page.
bool m_sortedIDListsInitialized { false };
Vector<AXID> m_sortedLiveRegionIDs;
Vector<AXID> m_sortedNonRootWebAreaIDs;
#endif // PLATFORM(MAC)

WeakPtr<Element, WeakPtrImplWithEventTargetData> m_currentModalElement;
// Multiple aria-modals behavior is undefined by spec. We keep them sorted based on DOM order here.
Expand Down Expand Up @@ -845,6 +863,9 @@ class AXObjectCache final : public CanMakeWeakPtr<AXObjectCache>, public CanMake
Vector<AttributeChange> m_deferredAttributeChange;
std::optional<std::pair<WeakPtr<Element, WeakPtrImplWithEventTargetData>, WeakPtr<Element, WeakPtrImplWithEventTargetData>>> m_deferredFocusedNodeChange;
WeakHashSet<AccessibilityObject> m_deferredUnconnectedObjects;
#if PLATFORM(MAC)
WeakHashSet<Document, WeakPtrImplWithEventTargetData> m_deferredDocumentAddedList;
#endif

#if ENABLE(ACCESSIBILITY_ISOLATED_TREE)
Timer m_buildIsolatedTreeTimer;
Expand Down
62 changes: 61 additions & 1 deletion Source/WebCore/accessibility/AXSearchManager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,8 +279,37 @@ AXCoreObject::AccessibilityChildrenVector AXSearchManager::findMatchingObjectsIn
AXTRACE("AXSearchManager::findMatchingObjectsInternal"_s);
AXLOG(criteria);

AXCoreObject::AccessibilityChildrenVector results;
if (!criteria.searchKeys.size())
return { };

#if PLATFORM(MAC)
if (criteria.searchKeys.size() == 1 && criteria.usePreCachedResults) {
// Only perform these optimizations if we aren't expected to start from somewhere mid-tree.
// We could probably implement these optimizations when we do have a startObject and get
// performance benefits, but no known assistive technology needs this right now.
if (!criteria.startObject) {
if (criteria.searchKeys[0] == AccessibilitySearchKey::LiveRegion) {
if (criteria.anchorObject->isRootWebArea()) {
// All live regions will be descendants of the root webarea, so we don't need to do
// any ancestry walks as `sortedDescendants` does.
auto liveRegions = criteria.anchorObject->allSortedLiveRegions();
return liveRegions.subvector(0, std::min(liveRegions.size(), static_cast<size_t>(criteria.resultsLimit)));
}
return criteria.anchorObject->sortedDescendants(criteria.resultsLimit, PreSortedObjectType::LiveRegion);
}

if (criteria.searchKeys[0] == AccessibilitySearchKey::Frame) {
if (criteria.anchorObject->isRootWebArea()) {
auto webAreas = criteria.anchorObject->allSortedNonRootWebAreas();
return webAreas.subvector(0, std::min(webAreas.size(), static_cast<size_t>(criteria.resultsLimit)));
}
return criteria.anchorObject->sortedDescendants(criteria.resultsLimit, PreSortedObjectType::WebArea);
}
}
}
#endif // PLATFORM(MAC)

AXCoreObject::AccessibilityChildrenVector results;
// This search algorithm only searches the elements before/after the starting object.
// It does this by stepping up the parent chain and at each level doing a DFS.

Expand Down Expand Up @@ -383,4 +412,35 @@ std::optional<AXTextMarkerRange> AXSearchManager::findMatchingRange(Accessibilit
return std::nullopt;
}

AXCoreObject* AXSearchManager::findNextStartingFrom(AccessibilitySearchKey key, AXCoreObject* start, AXCoreObject& anchor)
{
AccessibilitySearchCriteria criteria;
criteria.startObject = start;
criteria.anchorObject = &anchor;
criteria.searchDirection = AccessibilitySearchDirection::Next;
criteria.searchKeys = { key };
criteria.resultsLimit = 1;
criteria.visibleOnly = false;
criteria.immediateDescendantsOnly = false;
criteria.usePreCachedResults = false;

auto results = findMatchingObjectsInternal(criteria);
return results.size() ? results[0].ptr() : nullptr;
}

AXCoreObject::AccessibilityChildrenVector AXSearchManager::findAllMatchingObjectsIgnoringCache(Vector<AccessibilitySearchKey>&& keys, AXCoreObject& anchor)
{
AccessibilitySearchCriteria criteria;
criteria.anchorObject = &anchor;
criteria.startObject = nullptr;
criteria.searchDirection = AccessibilitySearchDirection::Next;
criteria.searchKeys = WTFMove(keys);
criteria.resultsLimit = std::numeric_limits<unsigned>::max();
criteria.visibleOnly = false;
criteria.immediateDescendantsOnly = false;
criteria.usePreCachedResults = false;

return findMatchingObjectsInternal(criteria);
}

} // namespace WebCore
5 changes: 5 additions & 0 deletions Source/WebCore/accessibility/AXSearchManager.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,11 +87,16 @@ struct AccessibilitySearchCriteria {
unsigned resultsLimit { 0 };
bool visibleOnly { false };
bool immediateDescendantsOnly { false };
// For some types of common searches (e.g. all the live regions on the page), we eagerly
// compute results. This flag determines whether to use these precached results or not.
bool usePreCachedResults { true };
};

class AXSearchManager {
WTF_MAKE_FAST_ALLOCATED_WITH_HEAP_IDENTIFIER(AXSearchManager);
public:
AXCoreObject* findNextStartingFrom(AccessibilitySearchKey, AXCoreObject* start, AXCoreObject& anchor);
AXCoreObject::AccessibilityChildrenVector findAllMatchingObjectsIgnoringCache(Vector<AccessibilitySearchKey>&&, AXCoreObject& anchor);
AXCoreObject::AccessibilityChildrenVector findMatchingObjects(AccessibilitySearchCriteria&&);
std::optional<AXTextMarkerRange> findMatchingRange(AccessibilitySearchCriteria&&);
private:
Expand Down
2 changes: 1 addition & 1 deletion Source/WebCore/accessibility/AccessibilityObject.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2845,7 +2845,7 @@ void AccessibilityObject::updateRole()
m_role = determineAccessibilityRole();
if (previousRole != m_role) {
if (auto* cache = axObjectCache())
cache->handleRoleChanged(*this);
cache->handleRoleChanged(*this, previousRole);
}
}

Expand Down
Loading