Skip to content

Commit

Permalink
[M113][Read Anything] Show selected content in Read Anything
Browse files Browse the repository at this point in the history 
When a user selects content on the main web page, the Read Anything
panel will scroll to and highlight the corresponding content.

If the user's selection has content that is not already displayed in
Read Anything, we will display the user's selection in the Read
Anything panel.

Screenshots:
https://screenshot.googleplex.com/8Zx8vYtqHX2W4o5
https://screenshot.googleplex.com/C2MG7LrMqzftXfP

(cherry picked from commit 8e87011)

Bug: 1266555
Change-Id: I3a4edc48a05a68e7e951078f3e7dadbd5bd09bfa
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4303963
Reviewed-by: Abigail Klein <abigailbklein@google.com>
Commit-Queue: Jocelyn Tran <jocelyntran@google.com>
Cr-Original-Commit-Position: refs/heads/main@{#1123174}
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4382234
Bot-Commit: Rubber Stamper <rubber-stamper@appspot.gserviceaccount.com>
Cr-Commit-Position: refs/branch-heads/5672@{#125}
Cr-Branched-From: 5f2a724-refs/heads/main@{#1121455}
  • Loading branch information
@tranjocelyn
tranjocelyn authored and Chromium LUCI CQ committed Mar 29, 2023
1 parent c05a34a commit 9357366
Show file tree
Hide file tree
Showing 9 changed files with 376 additions and 97 deletions.
8 changes: 8 additions & 0 deletions chrome/browser/resources/side_panel/read_anything/app.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,14 @@
margin: 0 auto 16px auto;
width: fit-content;
}
::selection {
background: var(--selection-color);
}
@media (prefers-color-scheme: dark) {
::selection {
color: var(--google-grey-900);
}
}
</style>
<div id="container" hidden="[[!hasContent_]]"></div>
<div id="empty-state-container" hidden="[[hasContent_]]">
Expand Down
31 changes: 27 additions & 4 deletions chrome/browser/resources/side_panel/read_anything/app.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,11 @@ interface LinkColor {
default: string;
visited: string;
}
const darkThemeBackgroundSkColor = rgbToSkColor(
getComputedStyle(document.body).getPropertyValue('--google-grey-900-rgb'));
const style = getComputedStyle(document.body);
const darkThemeBackgroundSkColor =
rgbToSkColor(style.getPropertyValue('--google-grey-900-rgb'));
const lightThemeBackgroundSkColor =
rgbToSkColor(style.getPropertyValue('--google-grey-50-rgb'));
const darkThemeEmptyStateBodyColor = 'var(--google-grey-500)';
const defaultThemeEmptyStateBodyColor = 'var(--google-grey-700)';
const defaultLinkColors: LinkColor = {
Expand All @@ -33,6 +36,9 @@ const darkThemeLinkColors: LinkColor = {
default: 'var(--google-blue-300)',
visited: 'var(--google-purple-100)',
};
const lightThemeSelectionColor = 'var(--google-yellow-100)';
const darkThemeSelectionColor = 'var(--google-blue-300)';
const defaultThemeSelctionColor = 'var(--google-blue-100)';

// A two-way map where each key is unique and each value is unique. The keys are
// DOM nodes and the values are numbers, representing AXNodeIDs.
Expand Down Expand Up @@ -71,6 +77,12 @@ if (chrome.readAnything) {
readAnythingApp.updateContent();
};

chrome.readAnything.updateSelection = () => {
const readAnythingApp = document.querySelector('read-anything-app');
assert(readAnythingApp);
readAnythingApp.updateSelection();
};

chrome.readAnything.updateTheme = () => {
const readAnythingApp = document.querySelector('read-anything-app');
assert(readAnythingApp);
Expand Down Expand Up @@ -254,10 +266,9 @@ export class ReadAnythingElement extends ReadAnythingElementBase {

this.hasContent_ = true;
container.appendChild(node);
this.updateSelection_();
}

private updateSelection_() {
updateSelection() {
const shadowRoot = this.shadowRoot;
assert(shadowRoot);
const selection = shadowRoot.getSelection();
Expand Down Expand Up @@ -307,6 +318,17 @@ export class ReadAnythingElement extends ReadAnythingElementBase {
defaultThemeEmptyStateBodyColor;
}

private getSelectionColor_(backgroundSkColor: SkColor): string {
switch (backgroundSkColor.value) {
case lightThemeBackgroundSkColor.value:
return lightThemeSelectionColor;
case darkThemeBackgroundSkColor.value:
return darkThemeSelectionColor;
default:
return defaultThemeSelctionColor;
}
}

updateTheme() {
const foregroundColor:
SkColor = {value: chrome.readAnything.foregroundColor};
Expand All @@ -321,6 +343,7 @@ export class ReadAnythingElement extends ReadAnythingElementBase {
'--letter-spacing': chrome.readAnything.letterSpacing + 'em',
'--line-height': chrome.readAnything.lineSpacing,
'--link-color': linkColor.default,
'--selection-color': this.getSelectionColor_(backgroundColor),
'--sp-empty-state-heading-color': skColorToRgba(foregroundColor),
'--sp-empty-state-body-color':
this.getEmptyStateBodyColor_(backgroundColor),
Expand Down
5 changes: 4 additions & 1 deletion chrome/browser/resources/side_panel/read_anything/read_anything.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ declare namespace chrome {

// Called when a user makes a selection change. AnchorNodeID and
// focusAXNodeID are AXNodeIDs which identify the anchor and focus AXNodes
// in the main pane.
// in the main pane. The selection can either be forward or backwards.
function onSelectionChange(
anchorNodeId: number, anchorOffset: number, focusNodeId: number,
focusOffset: number): void;
Expand Down Expand Up @@ -112,6 +112,9 @@ declare namespace chrome {
// and is available to consume.
function updateContent(): void;

// Ping that the selection has been updated.
function updateSelection(): void;

// Ping that the theme choices of the user have been changed using the
// toolbar and are ready to consume.
function updateTheme(): void;
Expand Down
131 changes: 96 additions & 35 deletions chrome/renderer/accessibility/read_anything_app_controller.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
#include <utility>
#include <vector>

#include "base/containers/contains.h"
#include "base/notreached.h"
#include "base/strings/string_util.h"
#include "base/strings/utf_string_conversions.h"
Expand Down Expand Up @@ -315,6 +314,17 @@ void ReadAnythingAppController::AccessibilityEventReceived(
const std::vector<ui::AXTreeUpdate>& updates,
const std::vector<ui::AXEvent>& events) {
model_.AccessibilityEventReceived(tree_id, updates, this);

if (model_.loading() || tree_id != model_.active_tree_id()) {
return;
}

for (auto& event : events) {
if (event.event_type == ax::mojom::Event::kDocumentSelectionChanged &&
event.event_from == ax::mojom::EventFrom::kUser) {
PostProcessSelection();
}
}
}

void ReadAnythingAppController::OnActiveAXTreeIDChanged(
Expand All @@ -335,6 +345,8 @@ void ReadAnythingAppController::OnActiveAXTreeIDChanged(
// replace this function call with firing an event.
std::string script = "chrome.readAnything.showLoading();";
render_frame_->ExecuteJavaScript(base::ASCIIToUTF16(script));
model_.SetLoading(true);

// When the UI first constructs, this function may be called before tree_id
// has been added to the tree list in AccessibilityEventReceived. In that
// case, do not distill.
Expand Down Expand Up @@ -430,31 +442,69 @@ void ReadAnythingAppController::OnAXTreeDistilled(
}
if (!model_.content_node_ids().empty()) {
// If there are content_node_ids, this means the AXTree was successfully
// distilled. Post-process in preparation to display the distilled content.
PostProcessDistillableAXTree();
} else if (model_.has_selection()) {
// Otherwise, if there is a selection, post-process the AXTree to display
// the selected content.
PostProcessAXTreeWithSelection();
} else {
// TODO(crbug.com/1266555): Display a UI giving user instructions if the
// tree was not distillable.
}

Draw();
// distilled.
ComputeDisplayNodeIdsForDistilledTree();
}

// Draw selection (if one exists) and the content.
PostProcessSelection();

// TODO(crbug.com/1266555): If no content nodes were identified, the
// controller should handle drawing the empty state (like how it handles the
// loading state) instead of the JS.

// Once drawing is complete, unserialize all of the pending updates on the
// active tree and send out a new distillation request.
model_.UnserializePendingUpdates(tree_id);
}

void ReadAnythingAppController::PostProcessSelection() {
DCHECK_NE(model_.active_tree_id(), ui::AXTreeIDUnknown());
DCHECK(model_.ContainsTree(model_.active_tree_id()));

// If the previous selection was inside the distilled content, that means we
// are currently displaying the distilled content in Read Anything. We may not
// need to redraw the distilled content if the user's new selection is inside
// the distilled content.
// If the previous selection was outside the distilled content, we will always
// redraw either a) the new selected content or b) the original distilled
// content if the new selection is inside that or if the selection was
// cleared.
bool need_to_draw = !model_.SelectionInsideDisplayNodes();

// Save the current selection
model_.UpdateSelection();

// If the main panel selection contains content outside of the distilled
// content, we need to find the selected nodes to display instead of the
// distilled content.
if (model_.has_selection() && !model_.SelectionInsideDisplayNodes()) {
need_to_draw = true;
ComputeSelectionNodeIds();
}

if (need_to_draw) {
Draw();
}
DrawSelection();
}

void ReadAnythingAppController::Draw() {
// TODO(abigailbklein): Use v8::Function rather than javascript. If possible,
// replace this function call with firing an event.
std::string script = "chrome.readAnything.updateContent();";
render_frame_->ExecuteJavaScript(base::ASCIIToUTF16(script));
model_.SetLoading(false);
}

void ReadAnythingAppController::PostProcessAXTreeWithSelection() {
void ReadAnythingAppController::DrawSelection() {
// TODO(abigailbklein): Use v8::Function rather than javascript. If possible,
// replace this function call with firing an event.
std::string script = "chrome.readAnything.updateSelection();";
render_frame_->ExecuteJavaScript(base::ASCIIToUTF16(script));
}

void ReadAnythingAppController::ComputeSelectionNodeIds() {
DCHECK(model_.has_selection());
DCHECK_NE(model_.active_tree_id(), ui::AXTreeIDUnknown());
DCHECK(model_.ContainsTree(model_.active_tree_id()));
Expand All @@ -466,20 +516,14 @@ void ReadAnythingAppController::PostProcessAXTreeWithSelection() {
ui::AXNode* end_node = model_.GetAXNode(model_.end_node_id());
DCHECK(end_node);

// If start node or end node is ignored, go to the nearest unignored node
// within the selection.
if (start_node->IsIgnored()) {
start_node = start_node->GetNextUnignoredInTreeOrder();
DCHECK(start_node);
model_.SetStart(start_node->id(), 0);
}
if (end_node->IsIgnored()) {
end_node = end_node->GetPreviousUnignoredInTreeOrder();
model_.SetEnd(end_node->id(), end_node->GetTextContentLengthUTF8());
// If start node or end node is ignored, the selection was invalid.
if (start_node->IsIgnored() || end_node->IsIgnored()) {
return;
}

// Display nodes are the nodes which will be displayed by the rendering
// algorithm of Read Anything app.ts. We wish to create a subtree which
// Selection nodes are the nodes which will be displayed by the rendering
// algorithm of Read Anything app.ts if there is a selection that contains
// content outside of the distilled content. We wish to create a subtree which
// stretches from start node to end node with tree root as the root.

// Add all ancestor ids of start node, including the start node itself. This
Expand All @@ -488,22 +532,29 @@ void ReadAnythingAppController::PostProcessAXTreeWithSelection() {
start_node->GetAncestorsCrossingTreeBoundaryAsQueue();
while (!ancestors.empty()) {
ui::AXNodeID ancestor_id = ancestors.front()->id();
model_.InsertDisplayNode(ancestor_id);
ancestors.pop();
if (!model_.IsNodeIgnoredForReadAnything(ancestor_id)) {
model_.InsertSelectionNode(ancestor_id);
}
}

// Do a pre-order walk of the tree from the start node to the end node and add
// all nodes to the list of display node ids.
ui::AXNode* next_node = start_node;
DCHECK(!start_node->IsIgnored());
DCHECK(!end_node->IsIgnored());
while (next_node != end_node) {
// all nodes to the list.
// TODO(crbug.com/1266555): Right now, we are going from start node to an
// unignored node that is before or equal to the end node. This condition was
// changed from next_node != end node because when a paragraph is selected
// with a triple click, we sometimes pass the end node, causing a SEGV_ACCERR.
// We need to investigate this case in more depth.
ui::AXNode* next_node = start_node->GetNextUnignoredInTreeOrder();
while (next_node && next_node->CompareTo(*end_node) <= 0) {
if (!model_.IsNodeIgnoredForReadAnything(next_node->id())) {
model_.InsertSelectionNode(next_node->id());
}
next_node = next_node->GetNextUnignoredInTreeOrder();
model_.InsertDisplayNode(next_node->id());
}
}

void ReadAnythingAppController::PostProcessDistillableAXTree() {
void ReadAnythingAppController::ComputeDisplayNodeIdsForDistilledTree() {
DCHECK(!model_.content_node_ids().empty());

// Display nodes are the nodes which will be displayed by the rendering
Expand Down Expand Up @@ -662,9 +713,12 @@ std::vector<ui::AXNodeID> ReadAnythingAppController::GetChildren(
std::vector<ui::AXNodeID> child_ids;
ui::AXNode* ax_node = model_.GetAXNode(ax_node_id);
DCHECK(ax_node);
const std::set<ui::AXNodeID>* node_ids = model_.selection_node_ids().empty()
? &model_.display_node_ids()
: &model_.selection_node_ids();
for (auto it = ax_node->UnignoredChildrenBegin();
it != ax_node->UnignoredChildrenEnd(); ++it) {
if (base::Contains(model_.display_node_ids(), it->id())) {
if (base::Contains(*node_ids, it->id())) {
child_ids.push_back(it->id());
}
}
Expand Down Expand Up @@ -824,9 +878,16 @@ void ReadAnythingAppController::SetContentForTesting(
v8::Isolate* isolate = blink::MainThreadIsolate();
ui::AXTreeUpdate snapshot =
GetSnapshotFromV8SnapshotLite(isolate, v8_snapshot_lite);
ui::AXEvent selectionEvent;
selectionEvent.event_type = ax::mojom::Event::kDocumentSelectionChanged;
selectionEvent.event_from = ax::mojom::EventFrom::kUser;
AccessibilityEventReceived(snapshot.tree_data.tree_id, {snapshot}, {});
OnActiveAXTreeIDChanged(snapshot.tree_data.tree_id, ukm::kInvalidSourceId);
OnAXTreeDistilled(snapshot.tree_data.tree_id, content_node_ids);

// Trigger a selection event (for testing selections).
AccessibilityEventReceived(snapshot.tree_data.tree_id, {snapshot},
{selectionEvent});
}

AXTreeDistiller* ReadAnythingAppController::SetDistillerForTesting(
Expand Down
22 changes: 15 additions & 7 deletions chrome/renderer/accessibility/read_anything_app_controller.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ class ReadAnythingAppController

void Distill();
void Draw();
void DrawSelection();

void UnserializeUpdates(std::vector<ui::AXTreeUpdate> updates,
const ui::AXTreeID& tree_id);
Expand All @@ -141,13 +142,20 @@ class ReadAnythingAppController

// Helper functions for the rendering algorithm. Post-process the AXTree and
// cache values before sending an `updateContent` notification to the Read
// Anything app.ts. These functions:
// 1. Save state related to selection (start_node_, end_node_, start_offset_,
// end_offset_).
// 2. Save the display_node_ids_, which is a set of all nodes to be displayed
// in Read Anything app.ts.
void PostProcessAXTreeWithSelection();
void PostProcessDistillableAXTree();
// Anything app.ts.
// ComputeDisplayNodeIdsForDistilledTree computes display nodes from the
// content nodes. These display nodes will be displayed in Read Anything
// app.ts by default.
// ComputeSelectionNodeIds computes selection nodes from
// the user's selection. The selection nodes list is only populated when the
// user's selection contains nodes outside of the display nodes list. By
// keeping two separate lists of nodes, we can switch back to displaying the
// default distilled content without recomputing the nodes when the user
// clears their selection or selects content inside the distilled content.
void ComputeDisplayNodeIdsForDistilledTree();
void ComputeSelectionNodeIds();

void PostProcessSelection();

// The following methods are used for testing ReadAnythingAppTest.
// Snapshot_lite is a data structure which resembles an AXTreeUpdate. E.g.:
Expand Down

0 comments on commit 9357366

Please sign in to comment.