From d7358ac088280f46ba8ca8b8aed0ca8d30910311 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Wed, 12 Oct 2022 14:18:37 -0500 Subject: [PATCH 01/74] New per-glyph and per-word indices --- common/src/lib.rs | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/common/src/lib.rs b/common/src/lib.rs index 3b01874df..5a6b2c70a 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -930,20 +930,29 @@ pub struct Node { #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] pub text_direction: Option, + + /// For inline text. The end index (non-inclusive) of each glyph + /// in UTF-8 code units (bytes). For example, if the text box + /// consists of a 1-byte glyph, a 3-byte glyph, and a 1-byte glyph, + /// the indices would be [1, 4, 5]. + #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] + pub glyph_end_indices: Option>, /// For inline text. This is the pixel position of the end of each - /// character within the bounding rectangle of this object, in the - /// direction given by [`Node::text_direction`]. For example, for left-to-right + /// glyph within the bounding rectangle of this object, in the direction + /// given by [`Node::text_direction`]. For example, for left-to-right /// text, the first offset is the right coordinate of the first - /// character within the object's bounds, the second offset - /// is the right coordinate of the second character, and so on. - #[cfg_attr(feature = "serde", serde(default))] - #[cfg_attr(feature = "serde", serde(skip_serializing_if = "is_empty"))] - pub character_offsets: Box<[f32]>, + /// glyph within the object's bounds, the second offset is + /// the right coordinate of the second glyph, and so on. + #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] + pub glyph_end_pixel_offsets: Option>, - /// For inline text. The indices of each word, in UTF-8 code units. - #[cfg_attr(feature = "serde", serde(default))] - #[cfg_attr(feature = "serde", serde(skip_serializing_if = "is_empty"))] - pub words: Box<[Range]>, + /// For inline text. The end index (non-inclusive) of each word + /// in UTF-8 code units (bytes). For example, if the text box + /// consists of a 1-byte word (e.g. a leading space), a 3-byte word + /// (e.g. two ASCII letters followed by a space), and a 1-byte word + /// (e.g. an ASCII letter), the indices would be [1, 4, 5]. + #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] + pub word_end_indices: Option>, #[cfg_attr(feature = "serde", serde(default))] #[cfg_attr(feature = "serde", serde(skip_serializing_if = "is_empty"))] From 0a41c7c5e964a3942b90c1a8c51b4d3daceadaf3 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Wed, 12 Oct 2022 15:47:54 -0500 Subject: [PATCH 02/74] Change and clarify the semantics of TextSelection --- common/src/lib.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/common/src/lib.rs b/common/src/lib.rs index 5a6b2c70a..552d7b28b 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -646,7 +646,14 @@ fn is_empty(slice: &[T]) -> bool { slice.is_empty() } -/// Offsets are in UTF-8 code units. +/// The current text selection for a text field or document. +/// +/// Glyph indices are the indices of items in [`Node::glyph_end_indices`]. +/// The focus glyph index may be the number of glyphs in the focus node, +/// to indicate that the focus of the selection is at the end of the line. +/// This should only be true for the anchor glyph index if the anchor and focus +/// are the same, i.e. there is no selection, only a caret (also known as +/// a degenerate selection). #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "schemars", derive(JsonSchema))] @@ -655,9 +662,9 @@ fn is_empty(slice: &[T]) -> bool { #[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] pub struct TextSelection { anchor_node: NodeId, - anchor_offset: usize, + anchor_glyph_index: u16, focus_node: NodeId, - focus_offset: usize, + focus_glyph_index: u16, } /// A single accessible object. A complete UI is represented as a tree of these. From 1e8cd85ec138a2ec76316a867df9df52e4ab49e4 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Wed, 12 Oct 2022 16:38:03 -0500 Subject: [PATCH 03/74] Forgot to make TextSelection fields public --- common/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/src/lib.rs b/common/src/lib.rs index 552d7b28b..83f8a7927 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -661,10 +661,10 @@ fn is_empty(slice: &[T]) -> bool { #[cfg_attr(feature = "serde", serde(deny_unknown_fields))] #[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] pub struct TextSelection { - anchor_node: NodeId, - anchor_glyph_index: u16, - focus_node: NodeId, - focus_glyph_index: u16, + pub anchor_node: NodeId, + pub anchor_glyph_index: u16, + pub focus_node: NodeId, + pub focus_glyph_index: u16, } /// A single accessible object. A complete UI is represented as a tree of these. From 9e22677b4e3998ed749cc58ec9badc6ecc500977 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Thu, 13 Oct 2022 07:45:02 -0500 Subject: [PATCH 04/74] Refactor TextSelection into two instances of a new TextPosition struct --- common/src/lib.rs | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/common/src/lib.rs b/common/src/lib.rs index 83f8a7927..f138d1620 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -646,14 +646,20 @@ fn is_empty(slice: &[T]) -> bool { slice.is_empty() } -/// The current text selection for a text field or document. -/// -/// Glyph indices are the indices of items in [`Node::glyph_end_indices`]. -/// The focus glyph index may be the number of glyphs in the focus node, -/// to indicate that the focus of the selection is at the end of the line. -/// This should only be true for the anchor glyph index if the anchor and focus -/// are the same, i.e. there is no selection, only a caret (also known as -/// a degenerate selection). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "schemars", derive(JsonSchema))] +#[cfg_attr(feature = "serde", serde(crate = "serde"))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +#[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] +pub struct TextPosition { + /// The node's role must be [`Role::InlineTextBox`]. + pub node: NodeId, + /// The index of an item in [`Node::glyph_end_indices`], or the length + /// of that slice if the position is at the end of the line. + pub glyph_index: u16, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "schemars", derive(JsonSchema))] @@ -661,10 +667,15 @@ fn is_empty(slice: &[T]) -> bool { #[cfg_attr(feature = "serde", serde(deny_unknown_fields))] #[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] pub struct TextSelection { - pub anchor_node: NodeId, - pub anchor_glyph_index: u16, - pub focus_node: NodeId, - pub focus_glyph_index: u16, + /// The position where the selection started, and which does not change + /// as the selection is expanded or contracted. If there is no selection + /// but only a caret, this must be equal to [`focus`]. This is also known + /// as a degenerate selection. + pub anchor: TextPosition, + /// The active end of the selection, which changes as the selection + /// is expanded or contracted, or the position of the caret if there is + /// no selection. + pub focus: TextPosition, } /// A single accessible object. A complete UI is represented as a tree of these. From 074e3f236ec0992dd205214d0d6958767d195c0e Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Thu, 13 Oct 2022 16:25:08 -0500 Subject: [PATCH 05/74] Switch from 'glyph' back to 'character', and document the meaning of characters --- common/src/lib.rs | 44 ++++++++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/common/src/lib.rs b/common/src/lib.rs index f138d1620..d619fd4c4 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -655,9 +655,9 @@ fn is_empty(slice: &[T]) -> bool { pub struct TextPosition { /// The node's role must be [`Role::InlineTextBox`]. pub node: NodeId, - /// The index of an item in [`Node::glyph_end_indices`], or the length + /// The index of an item in [`Node::character_end_indices`], or the length /// of that slice if the position is at the end of the line. - pub glyph_index: u16, + pub character_index: u16, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -949,20 +949,40 @@ pub struct Node { #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] pub text_direction: Option, - /// For inline text. The end index (non-inclusive) of each glyph + /// For inline text. The end index (non-inclusive) of each character /// in UTF-8 code units (bytes). For example, if the text box - /// consists of a 1-byte glyph, a 3-byte glyph, and a 1-byte glyph, - /// the indices would be [1, 4, 5]. - #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] - pub glyph_end_indices: Option>, + /// consists of a 1-byte character, a 3-byte character, and another + /// 1-byte character, the indices would be [1, 4, 5]. + /// + /// A character is defined as the smallest unit of text that + /// can be selected. This isn't necessarily a single Unicode + /// scalar value (code point). This is why AccessKit can't compute + /// the indices of the characters from the text itself; this information + /// must be provided by the text editing implementation. + /// + /// If this node is the last text box in a line that ends with a hard + /// line break, that line break should be included at the end of this + /// node's value as either a CRLF or LF; in both cases, the line break + /// should be counted as a single character for the sake of this slice. + /// When the caret is at the end of such a line, the focus of the text + /// selection should be on the line break, not after it. + #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] + pub character_end_indices: Option>, /// For inline text. This is the pixel position of the end of each - /// glyph within the bounding rectangle of this object, in the direction + /// character within the bounding rectangle of this object, in the direction /// given by [`Node::text_direction`]. For example, for left-to-right /// text, the first offset is the right coordinate of the first - /// glyph within the object's bounds, the second offset is - /// the right coordinate of the second glyph, and so on. - #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] - pub glyph_end_pixel_offsets: Option>, + /// character within the object's bounds, the second offset is + /// the right coordinate of the second character, and so on. + /// + /// When present, the length of this slice should be the same as the length + /// of [`character_end_incides`], including for lines that end + /// with a hard line break. The end offset of such a line break should + /// be non-zero if selecting the line break by itself results in + /// a visible highlight (as in Microsoft Word), or zero if not + /// (as in Windows Notepad). + #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] + pub character_end_pixel_offsets: Option>, /// For inline text. The end index (non-inclusive) of each word /// in UTF-8 code units (bytes). For example, if the text box From fd4c91c3f12df64dbc920b307ece577f1ec4c36e Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Mon, 17 Oct 2022 07:39:45 -0500 Subject: [PATCH 06/74] Work so far on consumer text range API --- common/src/lib.rs | 5 +- consumer/src/lib.rs | 5 ++ consumer/src/node.rs | 16 ++++ consumer/src/text.rs | 205 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 229 insertions(+), 2 deletions(-) create mode 100644 consumer/src/text.rs diff --git a/common/src/lib.rs b/common/src/lib.rs index d619fd4c4..f5eb4ece9 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -966,8 +966,9 @@ pub struct Node { /// should be counted as a single character for the sake of this slice. /// When the caret is at the end of such a line, the focus of the text /// selection should be on the line break, not after it. - #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] - pub character_end_indices: Option>, + #[cfg_attr(feature = "serde", serde(default))] + #[cfg_attr(feature = "serde", serde(skip_serializing_if = "is_empty"))] + pub character_end_indices: Box<[u16]>, /// For inline text. This is the pixel position of the end of each /// character within the bounding rectangle of this object, in the direction /// given by [`Node::text_direction`]. For example, for left-to-right diff --git a/consumer/src/lib.rs b/consumer/src/lib.rs index ff99182e7..5b6ea6a99 100644 --- a/consumer/src/lib.rs +++ b/consumer/src/lib.rs @@ -12,6 +12,11 @@ pub use node::Node; pub(crate) mod iterators; pub use iterators::FilterResult; +pub(crate) mod text; +pub use text::{ + Position as TextPosition, Range as TextRange, Unit as TextUnit, WeakRange as WeakTextRange, +}; + #[cfg(test)] mod tests { use accesskit::kurbo::{Affine, Rect, Vec2}; diff --git a/consumer/src/node.rs b/consumer/src/node.rs index 810eadfc6..c65ec4625 100644 --- a/consumer/src/node.rs +++ b/consumer/src/node.rs @@ -480,6 +480,22 @@ impl<'a> Node<'a> { self.data().selected } + pub fn index_path(&self) -> Vec { + self.relative_index_path(self.tree_state.root_id()) + } + + pub fn relative_index_path(&self, ancestor_id: NodeId) -> Vec { + let mut result = Vec::new(); + let mut current = *self; + while current.id() != ancestor_id { + let (parent, index) = current.parent_and_index().unwrap(); + result.push(index); + current = parent; + } + result.reverse(); + result + } + pub(crate) fn first_filtered_child( &self, filter: &impl Fn(&Node) -> FilterResult, diff --git a/consumer/src/text.rs b/consumer/src/text.rs new file mode 100644 index 000000000..e6f20c5c6 --- /dev/null +++ b/consumer/src/text.rs @@ -0,0 +1,205 @@ +// Copyright 2022 The AccessKit Authors. All rights reserved. +// Licensed under the Apache License, Version 2.0 (found in +// the LICENSE-APACHE file) or the MIT license (found in +// the LICENSE-MIT file), at your option. + +use accesskit::{NodeId, Role, TextPosition as WeakPosition}; +use std::cmp::Ordering; + +use crate::{FilterResult, Node, TreeState}; + +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] +pub enum Unit { + Character, + Format, + Word, + Line, + Paragraph, + Page, + Document, +} + +#[derive(Clone, Copy)] +struct InnerPosition<'a> { + node: Node<'a>, + character_index: u16, +} + +impl<'a> InnerPosition<'a> { + fn upgrade(tree_state: &'a TreeState, weak: WeakPosition) -> Option { + let node = tree_state.node_by_id(weak.node)?; + if node.role() != Role::InlineTextBox { + return None; + } + let character_index = weak.character_index; + if (character_index as usize) > node.data().character_end_indices.len() { + return None; + } + Some(Self { + node, + character_index, + }) + } + + fn comparable(&self, root_node_id: NodeId) -> (Vec, u16) { + ( + self.node.relative_index_path(root_node_id), + self.character_index, + ) + } + + fn downgrade(&self) -> WeakPosition { + WeakPosition { + node: self.node.id(), + character_index: self.character_index, + } + } +} + +impl<'a> PartialEq for InnerPosition<'a> { + fn eq(&self, other: &Self) -> bool { + self.node.id() == other.node.id() && self.character_index == other.character_index + } +} + +impl<'a> Eq for InnerPosition<'a> {} + +#[derive(Clone, Copy)] +pub struct Position<'a> { + root_node: Node<'a>, + inner: InnerPosition<'a>, +} + +impl<'a> PartialEq for Position<'a> { + fn eq(&self, other: &Self) -> bool { + self.root_node.id() == other.root_node.id() && self.inner == other.inner + } +} + +impl<'a> Eq for Position<'a> {} + +impl<'a> PartialOrd for Position<'a> { + fn partial_cmp(&self, other: &Self) -> Option { + let root_node_id = self.root_node.id(); + if root_node_id != other.root_node.id() { + return None; + } + let self_comparable = self.inner.comparable(root_node_id); + let other_comparable = other.inner.comparable(root_node_id); + Some(self_comparable.cmp(&other_comparable)) + } +} + +#[derive(Clone, Copy)] +pub struct Range<'a> { + node: Node<'a>, + start: InnerPosition<'a>, + end: InnerPosition<'a>, +} + +impl<'a> Range<'a> { + fn new(node: Node<'a>, mut start: InnerPosition<'a>, mut end: InnerPosition<'a>) -> Self { + if start.comparable(node.id()) > end.comparable(node.id()) { + std::mem::swap(&mut start, &mut end); + } + Self { node, start, end } + } + + pub fn node(&self) -> &Node { + &self.node + } + + pub fn start(&self) -> Position { + Position { + root_node: self.node, + inner: self.start, + } + } + + pub fn end(&self) -> Position { + Position { + root_node: self.node, + inner: self.end, + } + } + + pub fn downgrade(&self) -> WeakRange { + WeakRange { + node_id: self.node.id(), + start: self.start.downgrade(), + end: self.end.downgrade(), + } + } +} + +impl<'a> PartialEq for Range<'a> { + fn eq(&self, other: &Self) -> bool { + self.node.id() == other.node.id() && self.start == other.start && self.end == other.end + } +} + +impl<'a> Eq for Range<'a> {} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct WeakRange { + node_id: NodeId, + start: WeakPosition, + end: WeakPosition, +} + +impl WeakRange { + pub fn upgrade<'a>(&self, tree_state: &'a TreeState) -> Option> { + let node = tree_state.node_by_id(self.node_id)?; + let start = InnerPosition::upgrade(tree_state, self.start)?; + let end = InnerPosition::upgrade(tree_state, self.end)?; + Some(Range { node, start, end }) + } +} + +impl<'a> Node<'a> { + fn text_node_filter(&self, node: &Node) -> FilterResult { + if node.id() == self.id() || (node.role() == Role::InlineTextBox && !node.is_hidden()) { + FilterResult::Include + } else { + FilterResult::ExcludeNode + } + } + + pub fn supports_text_ranges(&self) -> bool { + let role = self.role(); + if role != Role::StaticText && role != Role::TextField && role != Role::Document { + return false; + } + self.filtered_children(|node| self.text_node_filter(node)) + .next() + .is_some() + } + + fn document_endpoints(&self) -> (InnerPosition, InnerPosition) { + let mut boxes = self.filtered_children(|node| self.text_node_filter(node)); + let first_box = boxes.next().unwrap(); + let start = InnerPosition { + node: first_box, + character_index: 0, + }; + let last_box = boxes.next_back().unwrap(); + let end = InnerPosition { + node: last_box, + character_index: last_box.data().character_end_indices.len() as u16, + }; + (start, end) + } + + pub fn document_range(&self) -> Range { + let (start, end) = self.document_endpoints(); + Range::new(*self, start, end) + } + + pub fn text_selection(&self) -> Option { + self.data().text_selection.map(|selection| { + let anchor = InnerPosition::upgrade(self.tree_state, selection.anchor).unwrap(); + let focus = InnerPosition::upgrade(self.tree_state, selection.focus).unwrap(); + Range::new(*self, anchor, focus) + }) + } +} From afe4ff5c7c3ae5dda1b80e57b5b5527b375e337d Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Mon, 17 Oct 2022 11:12:49 -0500 Subject: [PATCH 07/74] Stub Windows text range implementation --- consumer/src/text.rs | 4 +++ platforms/windows/Cargo.toml | 2 +- platforms/windows/src/lib.rs | 1 + platforms/windows/src/node.rs | 60 +++++++++++++++++++++++++++++++++-- platforms/windows/src/util.rs | 17 ++++++++-- 5 files changed, 79 insertions(+), 5 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index e6f20c5c6..237366a32 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -195,6 +195,10 @@ impl<'a> Node<'a> { Range::new(*self, start, end) } + pub fn has_text_selection(&self) -> bool { + self.data().text_selection.is_some() + } + pub fn text_selection(&self) -> Option { self.data().text_selection.map(|selection| { let anchor = InnerPosition::upgrade(self.tree_state, selection.anchor).unwrap(); diff --git a/platforms/windows/Cargo.toml b/platforms/windows/Cargo.toml index 2aefc7a99..46e260804 100644 --- a/platforms/windows/Cargo.toml +++ b/platforms/windows/Cargo.toml @@ -15,6 +15,7 @@ accesskit = { version = "0.6.1", path = "../../common" } accesskit_consumer = { version = "0.6.1", path = "../../consumer" } arrayvec = "0.7.1" lazy-init = "0.5.0" +parking_lot = "0.11.2" paste = "1.0" [dependencies.windows] @@ -34,6 +35,5 @@ features = [ [dev-dependencies] lazy_static = "1.4.0" -parking_lot = "0.11.2" scopeguard = "1.1.0" winit = "0.27.3" diff --git a/platforms/windows/src/lib.rs b/platforms/windows/src/lib.rs index 27582fb81..c907faf10 100644 --- a/platforms/windows/src/lib.rs +++ b/platforms/windows/src/lib.rs @@ -4,6 +4,7 @@ // the LICENSE-MIT file), at your option. mod node; +mod text; mod util; mod adapter; diff --git a/platforms/windows/src/node.rs b/platforms/windows/src/node.rs index e0a7dc252..0fe221257 100644 --- a/platforms/windows/src/node.rs +++ b/platforms/windows/src/node.rs @@ -21,7 +21,7 @@ use windows::{ Win32::{Foundation::*, Graphics::Gdi::*, System::Com::*, UI::Accessibility::*}, }; -use crate::util::*; +use crate::{text::PlatformRange as PlatformTextRange, util::*}; fn runtime_id_from_node_id(id: NodeId) -> impl std::ops::Deref { let mut result = ArrayVec::() + 1 }>::new(); @@ -408,6 +408,10 @@ impl<'a> NodeWrapper<'a> { } } + fn is_text_pattern_supported(&self) -> bool { + self.node.supports_text_ranges() + } + pub(crate) fn enqueue_property_changes( &self, queue: &mut Vec, @@ -472,6 +476,10 @@ fn element_not_available() -> Error { Error::new(HRESULT(UIA_E_ELEMENTNOTAVAILABLE as i32), "".into()) } +fn not_implemented() -> Error { + Error::new(E_NOTIMPL, "".into()) +} + #[implement( IRawElementProviderSimple, IRawElementProviderFragment, @@ -480,7 +488,8 @@ fn element_not_available() -> Error { IInvokeProvider, IValueProvider, IRangeValueProvider, - ISelectionItemProvider + ISelectionItemProvider, + ITextProvider )] pub(crate) struct PlatformNode { tree: Weak, @@ -839,5 +848,52 @@ patterns! { Err(Error::new(E_FAIL, "".into())) }) } + )), + (Text, is_text_pattern_supported, (), ( + fn GetSelection(&self) -> Result<*mut SAFEARRAY> { + self.resolve(|wrapper| { + if let Some(range) = wrapper.node.text_selection() { + let platform_range: ITextRangeProvider = PlatformTextRange::new(&self.tree, range).into(); + let iunknown: IUnknown = platform_range.into(); + Ok(safe_array_from_com_slice(&[iunknown])) + } else { + Ok(std::ptr::null_mut()) + } + }) + }, + + fn GetVisibleRanges(&self) -> Result<*mut SAFEARRAY> { + // TBD: Do we need this? The Quorum GUI toolkit, which is our + // current point of comparison for text functionality, + // doesn't implement it. + Ok(std::ptr::null_mut()) + }, + + fn RangeFromChild(&self, _child: &Option) -> Result { + // We don't support embedded objects in text. + Err(not_implemented()) + }, + + fn RangeFromPoint(&self, _point: &UiaPoint) -> Result { + // TODO: hit testing for text + Err(not_implemented()) + }, + + fn DocumentRange(&self) -> Result { + self.resolve(|wrapper| { + let range = wrapper.node.document_range(); + Ok(PlatformTextRange::new(&self.tree, range).into()) + }) + }, + + fn SupportedTextSelection(&self) -> Result { + self.resolve(|wrapper| { + if wrapper.node.has_text_selection() { + Ok(SupportedTextSelection_Single) + } else { + Ok(SupportedTextSelection_None) + } + }) + } )) } diff --git a/platforms/windows/src/util.rs b/platforms/windows/src/util.rs index c77833576..35b1e49d4 100644 --- a/platforms/windows/src/util.rs +++ b/platforms/windows/src/util.rs @@ -108,7 +108,7 @@ impl> From> for VariantFactory { } } -fn safe_array_from_slice(vt: VARENUM, slice: &[T]) -> *mut SAFEARRAY { +fn safe_array_from_primitive_slice(vt: VARENUM, slice: &[T]) -> *mut SAFEARRAY { let sa = unsafe { SafeArrayCreateVector(VARENUM(vt.0 as u16), 0, slice.len().try_into().unwrap()) }; if sa.is_null() { @@ -122,7 +122,20 @@ fn safe_array_from_slice(vt: VARENUM, slice: &[T]) -> *mut SAFEARRAY { } pub(crate) fn safe_array_from_i32_slice(slice: &[i32]) -> *mut SAFEARRAY { - safe_array_from_slice(VT_I4, slice) + safe_array_from_primitive_slice(VT_I4, slice) +} + +pub(crate) fn safe_array_from_com_slice(slice: &[IUnknown]) -> *mut SAFEARRAY { + let sa = + unsafe { SafeArrayCreateVector(VT_UNKNOWN, 0, slice.len().try_into().unwrap()) }; + if sa.is_null() { + panic!("SAFEARRAY allocation failed"); + } + for (i, item) in slice.iter().enumerate() { + let i: i32 = i.try_into().unwrap(); + unsafe { SafeArrayPutElement(&*sa, &i, std::mem::transmute_copy(item)) }.unwrap(); + } + sa } pub(crate) enum QueuedEvent { From 1165bfb2e84335e7dab924ab4a32e2090621612a Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Mon, 17 Oct 2022 11:20:46 -0500 Subject: [PATCH 08/74] Refactor utility functions; forgot to commit the actual Windows text module --- platforms/windows/src/node.rs | 4 - platforms/windows/src/text.rs | 136 ++++++++++++++++++++++++++++++++++ platforms/windows/src/util.rs | 5 ++ 3 files changed, 141 insertions(+), 4 deletions(-) create mode 100644 platforms/windows/src/text.rs diff --git a/platforms/windows/src/node.rs b/platforms/windows/src/node.rs index 0fe221257..3b3d303e4 100644 --- a/platforms/windows/src/node.rs +++ b/platforms/windows/src/node.rs @@ -476,10 +476,6 @@ fn element_not_available() -> Error { Error::new(HRESULT(UIA_E_ELEMENTNOTAVAILABLE as i32), "".into()) } -fn not_implemented() -> Error { - Error::new(E_NOTIMPL, "".into()) -} - #[implement( IRawElementProviderSimple, IRawElementProviderFragment, diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs new file mode 100644 index 000000000..6f2932174 --- /dev/null +++ b/platforms/windows/src/text.rs @@ -0,0 +1,136 @@ +// Copyright 2022 The AccessKit Authors. All rights reserved. +// Licensed under the Apache License, Version 2.0 (found in +// the LICENSE-APACHE file) or the MIT license (found in +// the LICENSE-MIT file), at your option. + +use accesskit_consumer::{TextRange as Range, Tree, WeakTextRange as WeakRange}; +use parking_lot::RwLock; +use std::sync::Weak; +use windows::{ + core::*, + Win32::{Foundation::*, System::Com::*, UI::Accessibility::*}, +}; + +#[implement(ITextRangeProvider)] +pub(crate) struct PlatformRange { + tree: Weak, + state: RwLock, +} + +impl PlatformRange { + pub(crate) fn new(tree: &Weak, range: Range) -> Self { + Self { + tree: tree.clone(), + state: RwLock::new(range.downgrade()), + } + } +} + +impl Clone for PlatformRange { + fn clone(&self) -> Self { + PlatformRange { + tree: self.tree.clone(), + state: RwLock::new(*self.state.read()), + } + } +} + +impl ITextRangeProvider_Impl for PlatformRange { + fn Clone(&self) -> Result { + Ok(self.clone().into()) + } + + fn Compare(&self, other: &Option) -> Result { + todo!() + } + + fn CompareEndpoints( + &self, + endpoint: TextPatternRangeEndpoint, + other: &Option, + other_endpoint: TextPatternRangeEndpoint, + ) -> Result { + todo!() + } + + fn ExpandToEnclosingUnit(&self, unit: TextUnit) -> Result<()> { + todo!() + } + + fn FindAttribute( + &self, + id: i32, + value: &VARIANT, + backward: BOOL, + ) -> Result { + todo!() + } + + fn FindText( + &self, + text: &BSTR, + backward: BOOL, + ignore_case: BOOL, + ) -> Result { + todo!() + } + + fn GetAttributeValue(&self, id: i32) -> Result { + todo!() + } + + fn GetBoundingRectangles(&self) -> Result<*mut SAFEARRAY> { + todo!() + } + + fn GetEnclosingElement(&self) -> Result { + todo!() + } + + fn GetText(&self, max_length: i32) -> Result { + todo!() + } + + fn Move(&self, unit: TextUnit, count: i32) -> Result { + todo!() + } + + fn MoveEndpointByUnit( + &self, + endpoint: TextPatternRangeEndpoint, + unit: TextUnit, + count: i32, + ) -> Result { + todo!() + } + + fn MoveEndpointByRange( + &self, + endpoint: TextPatternRangeEndpoint, + other: &Option, + other_endpoint: TextPatternRangeEndpoint, + ) -> Result<()> { + todo!() + } + + fn Select(&self) -> Result<()> { + todo!() + } + + fn AddToSelection(&self) -> Result<()> { + todo!() + } + + fn RemoveFromSelection(&self) -> Result<()> { + todo!() + } + + fn ScrollIntoView(&self, align_to_top: BOOL) -> Result<()> { + todo!() + } + + fn GetChildren(&self) -> Result<*mut SAFEARRAY> { + // We don't support embedded objects in text. + Ok(std::ptr::null_mut()) + } +} diff --git a/platforms/windows/src/util.rs b/platforms/windows/src/util.rs index 35b1e49d4..e0c8d5b82 100644 --- a/platforms/windows/src/util.rs +++ b/platforms/windows/src/util.rs @@ -7,6 +7,7 @@ use std::{convert::TryInto, mem::ManuallyDrop}; use windows::{ core::*, Win32::{ + Foundation::*, System::{Com::*, Ole::*}, UI::Accessibility::*, }, @@ -150,3 +151,7 @@ pub(crate) enum QueuedEvent { new_value: VARIANT, }, } + +pub(crate) fn not_implemented() -> Error { + Error::new(E_NOTIMPL, "".into()) +} From d38e332c2f0467e2f51483386d09e869b778883d Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Mon, 17 Oct 2022 12:59:05 -0500 Subject: [PATCH 09/74] More work on utility functions; implement Compare and CompareEndpoints --- platforms/windows/src/node.rs | 4 --- platforms/windows/src/text.rs | 68 ++++++++++++++++++++++++++++++++--- platforms/windows/src/util.rs | 15 ++++++-- 3 files changed, 77 insertions(+), 10 deletions(-) diff --git a/platforms/windows/src/node.rs b/platforms/windows/src/node.rs index 3b3d303e4..8a087f961 100644 --- a/platforms/windows/src/node.rs +++ b/platforms/windows/src/node.rs @@ -472,10 +472,6 @@ impl<'a> NodeWrapper<'a> { } } -fn element_not_available() -> Error { - Error::new(HRESULT(UIA_E_ELEMENTNOTAVAILABLE as i32), "".into()) -} - #[implement( IRawElementProviderSimple, IRawElementProviderFragment, diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index 6f2932174..1ed50f2de 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -3,14 +3,26 @@ // the LICENSE-APACHE file) or the MIT license (found in // the LICENSE-MIT file), at your option. -use accesskit_consumer::{TextRange as Range, Tree, WeakTextRange as WeakRange}; +#![allow(non_upper_case_globals)] + +use accesskit_consumer::{TextPosition as Position, TextRange as Range, Tree, TreeState, WeakTextRange as WeakRange}; use parking_lot::RwLock; -use std::sync::Weak; +use std::sync::{Arc, Weak}; use windows::{ core::*, Win32::{Foundation::*, System::Com::*, UI::Accessibility::*}, }; +use crate::util::*; + +fn position_from_endpoint<'a>(range: &'a Range, endpoint: TextPatternRangeEndpoint) -> Result> { + match endpoint { + TextPatternRangeEndpoint_Start => Ok(range.start()), + TextPatternRangeEndpoint_End => Ok(range.end()), + _ => Err(invalid_arg()), + } +} + #[implement(ITextRangeProvider)] pub(crate) struct PlatformRange { tree: Weak, @@ -24,6 +36,37 @@ impl PlatformRange { state: RwLock::new(range.downgrade()), } } + + fn upgrade_tree(&self) -> Result> { + if let Some(tree) = self.tree.upgrade() { + Ok(tree) + } else { + Err(element_not_available()) + } + } + + fn with_tree_state(&self, f: F) -> Result + where + F: FnOnce(&TreeState) -> Result, + { + let tree = self.upgrade_tree()?; + let state = tree.read(); + f(&state) + } + + fn read(&self, f: F) -> Result + where + F: FnOnce(&Range) -> Result, + { + self.with_tree_state(|tree_state| { + let state = self.state.read(); + if let Some(range) = state.upgrade(tree_state) { + f(&range) + } else { + Err(element_not_available()) + } + }) + } } impl Clone for PlatformRange { @@ -35,13 +78,19 @@ impl Clone for PlatformRange { } } +// Some text range methods take another text range interface pointer as a +// parameter. We need to cast these interface pointers to their underlying +// implementations. We assume that AccessKit is the only UIA provider +// within this process. This seems a safe assumption for most AccessKit users. + impl ITextRangeProvider_Impl for PlatformRange { fn Clone(&self) -> Result { Ok(self.clone().into()) } fn Compare(&self, other: &Option) -> Result { - todo!() + let other = required_param(other)?.as_impl(); + Ok((*self.state.read() == *other.state.read()).into()) } fn CompareEndpoints( @@ -50,7 +99,18 @@ impl ITextRangeProvider_Impl for PlatformRange { other: &Option, other_endpoint: TextPatternRangeEndpoint, ) -> Result { - todo!() + let other = required_param(other)?.as_impl(); + self.read(|range| { + other.read(|other_range| { + if range.node().id() != other_range.node().id() { + return Err(invalid_arg()); + } + let pos = position_from_endpoint(range, endpoint)?; + let other_pos = position_from_endpoint(other_range, other_endpoint)?; + let result = pos.partial_cmp(&other_pos).unwrap(); + Ok(result as i32) + }) + }) } fn ExpandToEnclosingUnit(&self, unit: TextUnit) -> Result<()> { diff --git a/platforms/windows/src/util.rs b/platforms/windows/src/util.rs index e0c8d5b82..fa87551df 100644 --- a/platforms/windows/src/util.rs +++ b/platforms/windows/src/util.rs @@ -127,8 +127,7 @@ pub(crate) fn safe_array_from_i32_slice(slice: &[i32]) -> *mut SAFEARRAY { } pub(crate) fn safe_array_from_com_slice(slice: &[IUnknown]) -> *mut SAFEARRAY { - let sa = - unsafe { SafeArrayCreateVector(VT_UNKNOWN, 0, slice.len().try_into().unwrap()) }; + let sa = unsafe { SafeArrayCreateVector(VT_UNKNOWN, 0, slice.len().try_into().unwrap()) }; if sa.is_null() { panic!("SAFEARRAY allocation failed"); } @@ -155,3 +154,15 @@ pub(crate) enum QueuedEvent { pub(crate) fn not_implemented() -> Error { Error::new(E_NOTIMPL, "".into()) } + +pub(crate) fn invalid_arg() -> Error { + Error::new(E_INVALIDARG, "".into()) +} + +pub(crate) fn required_param(param: &Option) -> Result<&T> { + param.as_ref().map_or_else(|| Err(invalid_arg()), Ok) +} + +pub(crate) fn element_not_available() -> Error { + Error::new(HRESULT(UIA_E_ELEMENTNOTAVAILABLE as i32), "".into()) +} From 0ca03e8ccd9a5ea7de4e6b686d90a25538af0d99 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Mon, 17 Oct 2022 18:15:11 -0500 Subject: [PATCH 10/74] Partial implementation of ExpandToEnclosingUnit --- consumer/src/lib.rs | 4 +- consumer/src/text.rs | 96 ++++++++++++++++++++++++++--------- platforms/windows/src/text.rs | 46 ++++++++++++++++- 3 files changed, 117 insertions(+), 29 deletions(-) diff --git a/consumer/src/lib.rs b/consumer/src/lib.rs index 5b6ea6a99..cc2b987db 100644 --- a/consumer/src/lib.rs +++ b/consumer/src/lib.rs @@ -13,9 +13,7 @@ pub(crate) mod iterators; pub use iterators::FilterResult; pub(crate) mod text; -pub use text::{ - Position as TextPosition, Range as TextRange, Unit as TextUnit, WeakRange as WeakTextRange, -}; +pub use text::{Position as TextPosition, Range as TextRange, WeakRange as WeakTextRange}; #[cfg(test)] mod tests { diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 237366a32..7f7292544 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -4,21 +4,10 @@ // the LICENSE-MIT file), at your option. use accesskit::{NodeId, Role, TextPosition as WeakPosition}; -use std::cmp::Ordering; +use std::{cmp::Ordering, iter::FusedIterator}; use crate::{FilterResult, Node, TreeState}; -#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] -pub enum Unit { - Character, - Format, - Word, - Line, - Paragraph, - Page, - Document, -} - #[derive(Clone, Copy)] struct InnerPosition<'a> { node: Node<'a>, @@ -48,6 +37,28 @@ impl<'a> InnerPosition<'a> { ) } + fn line_start(&self) -> Self { + let mut node = self.node; + while let Some(id) = node.data().previous_on_line { + node = node.tree_state.node_by_id(id).unwrap(); + } + Self { + node, + character_index: 0, + } + } + + fn line_end(&self) -> Self { + let mut node = self.node; + while let Some(id) = node.data().next_on_line { + node = node.tree_state.node_by_id(id).unwrap(); + } + Self { + node, + character_index: node.data().character_end_indices.len() as _, + } + } + fn downgrade(&self) -> WeakPosition { WeakPosition { node: self.node.id(), @@ -123,6 +134,37 @@ impl<'a> Range<'a> { } } + pub fn expand_to_character(&mut self) { + todo!() + } + + pub fn expand_to_format(&mut self) { + // We don't currently support format runs, so fall back to document. + self.expand_to_document(); + } + + pub fn expand_to_word(&mut self) { + todo!() + } + + pub fn expand_to_line(&mut self) { + self.start = self.start.line_start(); + self.end = self.start.line_end(); + } + + pub fn expand_to_paragraph(&mut self) { + todo!() + } + + pub fn expand_to_page(&mut self) { + // We don't currently support pages, so fall back to document. + self.expand_to_document(); + } + + pub fn expand_to_document(&mut self) { + (self.start, self.end) = self.node.document_endpoints(); + } + pub fn downgrade(&self) -> WeakRange { WeakRange { node_id: self.node.id(), @@ -156,13 +198,20 @@ impl WeakRange { } } +fn text_node_filter(root_id: NodeId, node: &Node) -> FilterResult { + if node.id() == root_id || (node.role() == Role::InlineTextBox && !node.is_hidden()) { + FilterResult::Include + } else { + FilterResult::ExcludeNode + } +} + impl<'a> Node<'a> { - fn text_node_filter(&self, node: &Node) -> FilterResult { - if node.id() == self.id() || (node.role() == Role::InlineTextBox && !node.is_hidden()) { - FilterResult::Include - } else { - FilterResult::ExcludeNode - } + fn inline_text_boxes( + &self, + ) -> impl DoubleEndedIterator> + FusedIterator> + 'a { + let id = self.id(); + self.filtered_children(move |node| text_node_filter(id, node)) } pub fn supports_text_ranges(&self) -> bool { @@ -170,19 +219,16 @@ impl<'a> Node<'a> { if role != Role::StaticText && role != Role::TextField && role != Role::Document { return false; } - self.filtered_children(|node| self.text_node_filter(node)) - .next() - .is_some() + self.inline_text_boxes().next().is_some() } - fn document_endpoints(&self) -> (InnerPosition, InnerPosition) { - let mut boxes = self.filtered_children(|node| self.text_node_filter(node)); - let first_box = boxes.next().unwrap(); + fn document_endpoints(&self) -> (InnerPosition<'a>, InnerPosition<'a>) { + let first_box = self.inline_text_boxes().next().unwrap(); let start = InnerPosition { node: first_box, character_index: 0, }; - let last_box = boxes.next_back().unwrap(); + let last_box = self.inline_text_boxes().next_back().unwrap(); let end = InnerPosition { node: last_box, character_index: last_box.data().character_end_indices.len() as u16, diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index 1ed50f2de..011b3ef96 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -67,6 +67,22 @@ impl PlatformRange { } }) } + + fn write(&self, f: F) -> Result + where + F: FnOnce(&mut Range) -> Result, + { + self.with_tree_state(|tree_state| { + let mut state = self.state.write(); + if let Some(mut range) = state.upgrade(tree_state) { + let result = f(&mut range); + *state = range.downgrade(); + result + } else { + Err(element_not_available()) + } + }) + } } impl Clone for PlatformRange { @@ -114,7 +130,35 @@ impl ITextRangeProvider_Impl for PlatformRange { } fn ExpandToEnclosingUnit(&self, unit: TextUnit) -> Result<()> { - todo!() + self.write(|range| { + match unit { + TextUnit_Character => { + range.expand_to_character(); + } + TextUnit_Format => { + range.expand_to_format(); + } + TextUnit_Word => { + range.expand_to_word(); + } + TextUnit_Line => { + range.expand_to_line(); + } + TextUnit_Paragraph => { + range.expand_to_paragraph(); + } + TextUnit_Page => { + range.expand_to_page(); + } + TextUnit_Document => { + range.expand_to_document(); + } + _ => { + return Err(invalid_arg()); + } + } + Ok(()) + }) } fn FindAttribute( From 755ad305688ab7b7304ecbce949283e393e1b38a Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Tue, 18 Oct 2022 09:22:45 -0500 Subject: [PATCH 11/74] Normalize endpoints for comparison --- consumer/src/text.rs | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 7f7292544..1664443e2 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -30,10 +30,27 @@ impl<'a> InnerPosition<'a> { }) } - fn comparable(&self, root_node_id: NodeId) -> (Vec, u16) { + fn is_box_end(&self) -> bool { + (self.character_index as usize) == self.node.data().character_end_indices.len() + } + + fn normalize_to_box_start(&self, root_node: &Node) -> Self { + if self.is_box_end() { + if let Some(node) = self.node.following_inline_text_boxes(root_node).next() { + return Self { + node, + character_index: 0, + }; + } + } + *self + } + + fn comparable(&self, root_node: &Node) -> (Vec, u16) { + let normalized = self.normalize_to_box_start(root_node); ( - self.node.relative_index_path(root_node_id), - self.character_index, + normalized.node.relative_index_path(root_node.id()), + normalized.character_index, ) } @@ -91,12 +108,11 @@ impl<'a> Eq for Position<'a> {} impl<'a> PartialOrd for Position<'a> { fn partial_cmp(&self, other: &Self) -> Option { - let root_node_id = self.root_node.id(); - if root_node_id != other.root_node.id() { + if self.root_node.id() != other.root_node.id() { return None; } - let self_comparable = self.inner.comparable(root_node_id); - let other_comparable = other.inner.comparable(root_node_id); + let self_comparable = self.inner.comparable(&self.root_node); + let other_comparable = other.inner.comparable(&self.root_node); Some(self_comparable.cmp(&other_comparable)) } } @@ -110,7 +126,7 @@ pub struct Range<'a> { impl<'a> Range<'a> { fn new(node: Node<'a>, mut start: InnerPosition<'a>, mut end: InnerPosition<'a>) -> Self { - if start.comparable(node.id()) > end.comparable(node.id()) { + if start.comparable(&node) > end.comparable(&node) { std::mem::swap(&mut start, &mut end); } Self { node, start, end } @@ -214,6 +230,14 @@ impl<'a> Node<'a> { self.filtered_children(move |node| text_node_filter(id, node)) } + fn following_inline_text_boxes( + &self, + root_node: &Node, + ) -> impl DoubleEndedIterator> + FusedIterator> + 'a { + let id = root_node.id(); + self.following_filtered_siblings(move |node| text_node_filter(id, node)) + } + pub fn supports_text_ranges(&self) -> bool { let role = self.role(); if role != Role::StaticText && role != Role::TextField && role != Role::Document { From 1cad12cbf32a4cd803271c1b387a5421149f0595 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Tue, 18 Oct 2022 11:09:08 -0500 Subject: [PATCH 12/74] Refactor; implement MoveEndpointByRange --- consumer/src/text.rs | 16 ++++++ platforms/windows/src/text.rs | 100 +++++++++++++++++++++++++--------- 2 files changed, 89 insertions(+), 27 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 1664443e2..a3d1238b3 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -181,6 +181,22 @@ impl<'a> Range<'a> { (self.start, self.end) = self.node.document_endpoints(); } + pub fn set_start(&mut self, pos: Position<'a>) { + assert_eq!(pos.root_node.id(), self.node.id()); + self.start = pos.inner; + if self.start.comparable(&self.node) > self.end.comparable(&self.node) { + self.end = self.start; + } + } + + pub fn set_end(&mut self, pos: Position<'a>) { + assert_eq!(pos.root_node.id(), self.node.id()); + self.end = pos.inner; + if self.start.comparable(&self.node) > self.end.comparable(&self.node) { + self.start = self.end; + } + } + pub fn downgrade(&self) -> WeakRange { WeakRange { node_id: self.node.id(), diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index 011b3ef96..af2143118 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -5,7 +5,9 @@ #![allow(non_upper_case_globals)] -use accesskit_consumer::{TextPosition as Position, TextRange as Range, Tree, TreeState, WeakTextRange as WeakRange}; +use accesskit_consumer::{ + TextPosition as Position, TextRange as Range, Tree, TreeState, WeakTextRange as WeakRange, +}; use parking_lot::RwLock; use std::sync::{Arc, Weak}; use windows::{ @@ -15,7 +17,18 @@ use windows::{ use crate::util::*; -fn position_from_endpoint<'a>(range: &'a Range, endpoint: TextPatternRangeEndpoint) -> Result> { +fn upgrade_range<'a>(weak: &WeakRange, tree_state: &'a TreeState) -> Result> { + if let Some(range) = weak.upgrade(tree_state) { + Ok(range) + } else { + Err(element_not_available()) + } +} + +fn position_from_endpoint<'a>( + range: &'a Range, + endpoint: TextPatternRangeEndpoint, +) -> Result> { match endpoint { TextPatternRangeEndpoint_Start => Ok(range.start()), TextPatternRangeEndpoint_End => Ok(range.end()), @@ -54,17 +67,18 @@ impl PlatformRange { f(&state) } + fn upgrade_for_read<'a>(&self, tree_state: &'a TreeState) -> Result> { + let state = self.state.read(); + upgrade_range(&state, tree_state) + } + fn read(&self, f: F) -> Result where F: FnOnce(&Range) -> Result, { self.with_tree_state(|tree_state| { - let state = self.state.read(); - if let Some(range) = state.upgrade(tree_state) { - f(&range) - } else { - Err(element_not_available()) - } + let range = self.upgrade_for_read(tree_state)?; + f(&range) }) } @@ -74,15 +88,20 @@ impl PlatformRange { { self.with_tree_state(|tree_state| { let mut state = self.state.write(); - if let Some(mut range) = state.upgrade(tree_state) { - let result = f(&mut range); - *state = range.downgrade(); - result - } else { - Err(element_not_available()) - } + let mut range = upgrade_range(&state, tree_state)?; + let result = f(&mut range); + *state = range.downgrade(); + result }) } + + fn require_same_tree(&self, other: &PlatformRange) -> Result<()> { + if self.tree.ptr_eq(&other.tree) { + Ok(()) + } else { + Err(invalid_arg()) + } + } } impl Clone for PlatformRange { @@ -106,7 +125,7 @@ impl ITextRangeProvider_Impl for PlatformRange { fn Compare(&self, other: &Option) -> Result { let other = required_param(other)?.as_impl(); - Ok((*self.state.read() == *other.state.read()).into()) + Ok((self.tree.ptr_eq(&other.tree) && *self.state.read() == *other.state.read()).into()) } fn CompareEndpoints( @@ -116,16 +135,17 @@ impl ITextRangeProvider_Impl for PlatformRange { other_endpoint: TextPatternRangeEndpoint, ) -> Result { let other = required_param(other)?.as_impl(); - self.read(|range| { - other.read(|other_range| { - if range.node().id() != other_range.node().id() { - return Err(invalid_arg()); - } - let pos = position_from_endpoint(range, endpoint)?; - let other_pos = position_from_endpoint(other_range, other_endpoint)?; - let result = pos.partial_cmp(&other_pos).unwrap(); - Ok(result as i32) - }) + self.require_same_tree(&other)?; + self.with_tree_state(|tree_state| { + let range = self.upgrade_for_read(&tree_state)?; + let other_range = other.upgrade_for_read(&tree_state)?; + if range.node().id() != other_range.node().id() { + return Err(invalid_arg()); + } + let pos = position_from_endpoint(&range, endpoint)?; + let other_pos = position_from_endpoint(&other_range, other_endpoint)?; + let result = pos.partial_cmp(&other_pos).unwrap(); + Ok(result as i32) }) } @@ -214,7 +234,33 @@ impl ITextRangeProvider_Impl for PlatformRange { other: &Option, other_endpoint: TextPatternRangeEndpoint, ) -> Result<()> { - todo!() + let other = required_param(other)?.as_impl(); + self.require_same_tree(&other)?; + // We have to obtain the tree state and ranges manually to avoid + // lifetime issues, and work with the two locks in a specific order + // to avoid deadlock. + self.with_tree_state(|tree_state| { + let other_range = other.upgrade_for_read(tree_state)?; + let mut state = self.state.write(); + let mut range = upgrade_range(&state, tree_state)?; + if range.node().id() != other_range.node().id() { + return Err(invalid_arg()); + } + let pos = position_from_endpoint(&other_range, other_endpoint)?; + match endpoint { + TextPatternRangeEndpoint_Start => { + range.set_start(pos); + } + TextPatternRangeEndpoint_End => { + range.set_end(pos); + } + _ => { + return Err(invalid_arg()); + } + } + *state = range.downgrade(); + Ok(()) + }) } fn Select(&self) -> Result<()> { From a655f4724ef44ec0d50cd4b8ced116b40d708f47 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Tue, 18 Oct 2022 13:05:07 -0500 Subject: [PATCH 13/74] More small refactors --- platforms/windows/src/text.rs | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index af2143118..e09be02b1 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -36,6 +36,21 @@ fn position_from_endpoint<'a>( } } +fn set_endpoint_position<'a>(range: &mut Range<'a>, endpoint: TextPatternRangeEndpoint, pos: Position<'a>) -> Result<()> { + match endpoint { + TextPatternRangeEndpoint_Start => { + range.set_start(pos); + } + TextPatternRangeEndpoint_End => { + range.set_end(pos); + } + _ => { + return Err(invalid_arg()); + } + } + Ok(()) +} + #[implement(ITextRangeProvider)] pub(crate) struct PlatformRange { tree: Weak, @@ -74,11 +89,11 @@ impl PlatformRange { fn read(&self, f: F) -> Result where - F: FnOnce(&Range) -> Result, + F: FnOnce(Range) -> Result, { self.with_tree_state(|tree_state| { let range = self.upgrade_for_read(tree_state)?; - f(&range) + f(range) }) } @@ -247,17 +262,7 @@ impl ITextRangeProvider_Impl for PlatformRange { return Err(invalid_arg()); } let pos = position_from_endpoint(&other_range, other_endpoint)?; - match endpoint { - TextPatternRangeEndpoint_Start => { - range.set_start(pos); - } - TextPatternRangeEndpoint_End => { - range.set_end(pos); - } - _ => { - return Err(invalid_arg()); - } - } + set_endpoint_position(&mut range, endpoint, pos)?; *state = range.downgrade(); Ok(()) }) From 544a2ff40caf1fd2dcd0b619bb6d9edf7c076271 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Tue, 18 Oct 2022 14:33:31 -0500 Subject: [PATCH 14/74] Skeleton of MoveEndpointByUnit --- platforms/windows/src/text.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index e09be02b1..d0f2c9cfe 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -240,7 +240,12 @@ impl ITextRangeProvider_Impl for PlatformRange { unit: TextUnit, count: i32, ) -> Result { - todo!() + self.write(|range| { + let pos = position_from_endpoint(range, endpoint); + let (pos, moved) = todo!(); + set_endpoint_position(range, endpoint, pos)?; + Ok(moved) + }) } fn MoveEndpointByRange( From 5d8d37c1b880fe99c9717f3e7a34d4e6700067d3 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Tue, 18 Oct 2022 14:44:59 -0500 Subject: [PATCH 15/74] Break apart document_endpoints --- consumer/src/text.rs | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index a3d1238b3..bd7f34aaa 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -178,7 +178,8 @@ impl<'a> Range<'a> { } pub fn expand_to_document(&mut self) { - (self.start, self.end) = self.node.document_endpoints(); + self.start = self.node.document_start(); + self.end = self.node.document_end(); } pub fn set_start(&mut self, pos: Position<'a>) { @@ -262,22 +263,25 @@ impl<'a> Node<'a> { self.inline_text_boxes().next().is_some() } - fn document_endpoints(&self) -> (InnerPosition<'a>, InnerPosition<'a>) { - let first_box = self.inline_text_boxes().next().unwrap(); - let start = InnerPosition { - node: first_box, + fn document_start(&self) -> InnerPosition<'a> { + let node = self.inline_text_boxes().next().unwrap(); + InnerPosition { + node, character_index: 0, - }; - let last_box = self.inline_text_boxes().next_back().unwrap(); - let end = InnerPosition { - node: last_box, - character_index: last_box.data().character_end_indices.len() as u16, - }; - (start, end) + } + } + + fn document_end(&self) -> InnerPosition<'a> { + let node = self.inline_text_boxes().next_back().unwrap(); + InnerPosition { + node, + character_index: node.data().character_end_indices.len() as u16, + } } pub fn document_range(&self) -> Range { - let (start, end) = self.document_endpoints(); + let start = self.document_start(); + let end = self.document_end(); Range::new(*self, start, end) } From ffa975ee1c4e091eb3ed102326c8f91e914783b6 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Tue, 18 Oct 2022 15:48:05 -0500 Subject: [PATCH 16/74] More structure for MoveEndpointByUnit; fix a lifetime issue --- consumer/src/text.rs | 4 ++-- platforms/windows/src/text.rs | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index bd7f34aaa..210649a47 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -136,14 +136,14 @@ impl<'a> Range<'a> { &self.node } - pub fn start(&self) -> Position { + pub fn start(&self) -> Position<'a> { Position { root_node: self.node, inner: self.start, } } - pub fn end(&self) -> Position { + pub fn end(&self) -> Position<'a> { Position { root_node: self.node, inner: self.end, diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index d0f2c9cfe..825a4ebba 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -26,7 +26,7 @@ fn upgrade_range<'a>(weak: &WeakRange, tree_state: &'a TreeState) -> Result( - range: &'a Range, + range: &Range<'a>, endpoint: TextPatternRangeEndpoint, ) -> Result> { match endpoint { @@ -51,6 +51,10 @@ fn set_endpoint_position<'a>(range: &mut Range<'a>, endpoint: TextPatternRangeEn Ok(()) } +fn move_position<'a>(pos: Position<'a>, unit: TextUnit, count: i32) -> (Position<'a>, i32) { + todo!() +} + #[implement(ITextRangeProvider)] pub(crate) struct PlatformRange { tree: Weak, @@ -241,8 +245,8 @@ impl ITextRangeProvider_Impl for PlatformRange { count: i32, ) -> Result { self.write(|range| { - let pos = position_from_endpoint(range, endpoint); - let (pos, moved) = todo!(); + let pos = position_from_endpoint(range, endpoint)?; + let (pos, moved) = move_position(pos, unit, count); set_endpoint_position(range, endpoint, pos)?; Ok(moved) }) From 923e6c001f150e4c82453e5c4222db3873faae44 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Tue, 18 Oct 2022 16:15:48 -0500 Subject: [PATCH 17/74] Finish the generic logic of MoveEndpointByRange; add stub functions in consumer API --- consumer/src/text.rs | 58 +++++++++++++++++++++++++++ platforms/windows/src/text.rs | 75 +++++++++++++++++++++++++++++++++-- 2 files changed, 130 insertions(+), 3 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 210649a47..a3d934048 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -98,6 +98,64 @@ pub struct Position<'a> { inner: InnerPosition<'a>, } +impl<'a> Position<'a> { + pub fn forward_by_character(&self) -> Option { + todo!() + } + + pub fn backward_by_character(&self) -> Option { + todo!() + } + + pub fn forward_by_format(&self) -> Option { + todo!() + } + + pub fn backward_by_format(&self) -> Option { + todo!() + } + + pub fn forward_by_word(&self) -> Option { + todo!() + } + + pub fn backward_by_word(&self) -> Option { + todo!() + } + + pub fn forward_by_line(&self) -> Option { + todo!() + } + + pub fn backward_by_line(&self) -> Option { + todo!() + } + + pub fn forward_by_paragraph(&self) -> Option { + todo!() + } + + pub fn backward_by_paragraph(&self) -> Option { + todo!() + } + + pub fn forward_by_page(&self) -> Option { + todo!() + } + + pub fn backward_by_page(&self) -> Option { + todo!() + } + + pub fn forward_by_document(&self) -> Option { + todo!() + } + + pub fn backward_by_document(&self) -> Option { + todo!() + } +} + impl<'a> PartialEq for Position<'a> { fn eq(&self, other: &Self) -> bool { self.root_node.id() == other.root_node.id() && self.inner == other.inner diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index 825a4ebba..5ae5e88b4 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -51,8 +51,77 @@ fn set_endpoint_position<'a>(range: &mut Range<'a>, endpoint: TextPatternRangeEn Ok(()) } -fn move_position<'a>(pos: Position<'a>, unit: TextUnit, count: i32) -> (Position<'a>, i32) { - todo!() +fn move_position_once<'a>(pos: Position<'a>, unit: TextUnit, forward: bool) -> Result>> { + match unit { + TextUnit_Character => { + if forward { + Ok(pos.forward_by_character()) + } else { + Ok(pos.backward_by_character()) + } + } + TextUnit_Format => { + if forward { + Ok(pos.forward_by_format()) + } else { + Ok(pos.backward_by_format()) + } + } + TextUnit_Word => { + if forward { + Ok(pos.forward_by_word()) + } else { + Ok(pos.backward_by_word()) + } + } + TextUnit_Line => { + if forward { + Ok(pos.forward_by_line()) + } else { + Ok(pos.backward_by_line()) + } + } + TextUnit_Paragraph => { + if forward { + Ok(pos.forward_by_paragraph()) + } else { + Ok(pos.backward_by_paragraph()) + } + } + TextUnit_Page => { + if forward { + Ok(pos.forward_by_page()) + } else { + Ok(pos.backward_by_page()) + } + } + TextUnit_Document => { + if forward { + Ok(pos.forward_by_document()) + } else { + Ok(pos.backward_by_document()) + } + } + _ => Err(invalid_arg()) + } +} + +fn move_position<'a>(mut pos: Position<'a>, unit: TextUnit, count: i32) -> Result<(Position<'a>, i32)> { + let forward = count > 0; + let count = count.abs(); + let mut moved = 0i32; + for _ in 0..count { + if let Some(new_pos) = move_position_once(pos, unit, forward)? { + pos = new_pos; + moved += 1; + } else { + break; + } + } + if !forward { + moved = -moved; + } + Ok((pos, moved)) } #[implement(ITextRangeProvider)] @@ -246,7 +315,7 @@ impl ITextRangeProvider_Impl for PlatformRange { ) -> Result { self.write(|range| { let pos = position_from_endpoint(range, endpoint)?; - let (pos, moved) = move_position(pos, unit, count); + let (pos, moved) = move_position(pos, unit, count)?; set_endpoint_position(range, endpoint, pos)?; Ok(moved) }) From f11a666850541f25746dd326daea9994fcfb43cc Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Tue, 18 Oct 2022 16:59:58 -0500 Subject: [PATCH 18/74] Hoist the document start/end check --- consumer/src/text.rs | 66 +++++++++++++++++++++++++++-------- platforms/windows/src/text.rs | 12 ++++--- 2 files changed, 60 insertions(+), 18 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index a3d934048..1b8a509fc 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -30,10 +30,32 @@ impl<'a> InnerPosition<'a> { }) } + fn is_box_start(&self) -> bool { + self.character_index == 0 + } + fn is_box_end(&self) -> bool { (self.character_index as usize) == self.node.data().character_end_indices.len() } + fn is_document_start(&self, root_node: &Node) -> bool { + self.is_box_start() + && self + .node + .preceding_inline_text_boxes(root_node) + .next() + .is_none() + } + + fn is_document_end(&self, root_node: &Node) -> bool { + self.is_box_end() + && self + .node + .following_inline_text_boxes(root_node) + .next() + .is_none() + } + fn normalize_to_box_start(&self, root_node: &Node) -> Self { if self.is_box_end() { if let Some(node) = self.node.following_inline_text_boxes(root_node).next() { @@ -99,59 +121,67 @@ pub struct Position<'a> { } impl<'a> Position<'a> { - pub fn forward_by_character(&self) -> Option { + pub fn is_document_start(&self) -> bool { + self.inner.is_document_start(&self.root_node) + } + + pub fn is_document_end(&self) -> bool { + self.inner.is_document_end(&self.root_node) + } + + pub fn forward_by_character(&self) -> Self { todo!() } - pub fn backward_by_character(&self) -> Option { + pub fn backward_by_character(&self) -> Self { todo!() } - pub fn forward_by_format(&self) -> Option { + pub fn forward_by_format(&self) -> Self { todo!() } - pub fn backward_by_format(&self) -> Option { + pub fn backward_by_format(&self) -> Self { todo!() } - pub fn forward_by_word(&self) -> Option { + pub fn forward_by_word(&self) -> Self { todo!() } - pub fn backward_by_word(&self) -> Option { + pub fn backward_by_word(&self) -> Self { todo!() } - pub fn forward_by_line(&self) -> Option { + pub fn forward_by_line(&self) -> Self { todo!() } - pub fn backward_by_line(&self) -> Option { + pub fn backward_by_line(&self) -> Self { todo!() } - pub fn forward_by_paragraph(&self) -> Option { + pub fn forward_by_paragraph(&self) -> Self { todo!() } - pub fn backward_by_paragraph(&self) -> Option { + pub fn backward_by_paragraph(&self) -> Self { todo!() } - pub fn forward_by_page(&self) -> Option { + pub fn forward_by_page(&self) -> Self { todo!() } - pub fn backward_by_page(&self) -> Option { + pub fn backward_by_page(&self) -> Self { todo!() } - pub fn forward_by_document(&self) -> Option { + pub fn forward_by_document(&self) -> Self { todo!() } - pub fn backward_by_document(&self) -> Option { + pub fn backward_by_document(&self) -> Self { todo!() } } @@ -313,6 +343,14 @@ impl<'a> Node<'a> { self.following_filtered_siblings(move |node| text_node_filter(id, node)) } + fn preceding_inline_text_boxes( + &self, + root_node: &Node, + ) -> impl DoubleEndedIterator> + FusedIterator> + 'a { + let id = root_node.id(); + self.preceding_filtered_siblings(move |node| text_node_filter(id, node)) + } + pub fn supports_text_ranges(&self) -> bool { let role = self.role(); if role != Role::StaticText && role != Role::TextField && role != Role::Document { diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index 5ae5e88b4..34d5862e1 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -51,7 +51,7 @@ fn set_endpoint_position<'a>(range: &mut Range<'a>, endpoint: TextPatternRangeEn Ok(()) } -fn move_position_once<'a>(pos: Position<'a>, unit: TextUnit, forward: bool) -> Result>> { +fn move_position_once<'a>(pos: Position<'a>, unit: TextUnit, forward: bool) -> Result> { match unit { TextUnit_Character => { if forward { @@ -111,12 +111,16 @@ fn move_position<'a>(mut pos: Position<'a>, unit: TextUnit, count: i32) -> Resul let count = count.abs(); let mut moved = 0i32; for _ in 0..count { - if let Some(new_pos) = move_position_once(pos, unit, forward)? { - pos = new_pos; - moved += 1; + let at_end = if forward { + pos.is_document_end() } else { + pos.is_document_start() + }; + if at_end { break; } + pos = move_position_once(pos, unit, forward)?; + moved += 1; } if !forward { moved = -moved; From 36b35e5e6bd32a58d7a57ec6baad9e3ed1a0286d Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Tue, 18 Oct 2022 17:19:51 -0500 Subject: [PATCH 19/74] Implement forward/backward by character --- consumer/src/text.rs | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 1b8a509fc..f80fee8b2 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -68,6 +68,18 @@ impl<'a> InnerPosition<'a> { *self } + fn normalize_to_box_end(&self, root_node: &Node) -> Self { + if self.is_box_start() { + if let Some(node) = self.node.preceding_inline_text_boxes(root_node).next() { + return Self { + node, + character_index: node.data().character_end_indices.len() as _, + }; + } + } + *self + } + fn comparable(&self, root_node: &Node) -> (Vec, u16) { let normalized = self.normalize_to_box_start(root_node); ( @@ -130,11 +142,25 @@ impl<'a> Position<'a> { } pub fn forward_by_character(&self) -> Self { - todo!() + let normalized = self.inner.normalize_to_box_start(&self.root_node); + Self { + root_node: self.root_node, + inner: InnerPosition { + node: normalized.node, + character_index: normalized.character_index + 1, + }, + } } pub fn backward_by_character(&self) -> Self { - todo!() + let normalized = self.inner.normalize_to_box_end(&self.root_node); + Self { + root_node: self.root_node, + inner: InnerPosition { + node: normalized.node, + character_index: normalized.character_index - 1, + }, + } } pub fn forward_by_format(&self) -> Self { From d9fb75c1ca75bf67772bc3ff3c3469f0e0925901 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Tue, 18 Oct 2022 17:23:47 -0500 Subject: [PATCH 20/74] Implement forward/backward by document --- consumer/src/text.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index f80fee8b2..3a3f50501 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -204,11 +204,17 @@ impl<'a> Position<'a> { } pub fn forward_by_document(&self) -> Self { - todo!() + Self { + root_node: self.root_node, + inner: self.root_node.document_end(), + } } pub fn backward_by_document(&self) -> Self { - todo!() + Self { + root_node: self.root_node, + inner: self.root_node.document_start(), + } } } From 95977f487f6baff97645af486a8b5d6bc9fd1767 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Tue, 18 Oct 2022 17:27:35 -0500 Subject: [PATCH 21/74] Implement forward/backward by line --- consumer/src/text.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 3a3f50501..4fd4ad82f 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -180,11 +180,19 @@ impl<'a> Position<'a> { } pub fn forward_by_line(&self) -> Self { - todo!() + let normalized = self.inner.normalize_to_box_start(&self.root_node); + Self { + root_node: self.root_node, + inner: normalized.line_end(), + } } pub fn backward_by_line(&self) -> Self { - todo!() + let normalized = self.inner.normalize_to_box_end(&self.root_node); + Self { + root_node: self.root_node, + inner: normalized.line_start(), + } } pub fn forward_by_paragraph(&self) -> Self { From d1cc20139e1d0979a82d885f914e437126d508b3 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Tue, 18 Oct 2022 17:47:39 -0500 Subject: [PATCH 22/74] Implement GetEnclosingElement --- platforms/windows/src/node.rs | 10 +++++----- platforms/windows/src/text.rs | 37 ++++++++++++++++++++++++++++------- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/platforms/windows/src/node.rs b/platforms/windows/src/node.rs index 8a087f961..35e433ddc 100644 --- a/platforms/windows/src/node.rs +++ b/platforms/windows/src/node.rs @@ -484,9 +484,9 @@ impl<'a> NodeWrapper<'a> { ITextProvider )] pub(crate) struct PlatformNode { - tree: Weak, - node_id: NodeId, - hwnd: HWND, + pub(crate) tree: Weak, + pub(crate) node_id: NodeId, + pub(crate) hwnd: HWND, } impl PlatformNode { @@ -845,7 +845,7 @@ patterns! { fn GetSelection(&self) -> Result<*mut SAFEARRAY> { self.resolve(|wrapper| { if let Some(range) = wrapper.node.text_selection() { - let platform_range: ITextRangeProvider = PlatformTextRange::new(&self.tree, range).into(); + let platform_range: ITextRangeProvider = PlatformTextRange::new(&self.tree, range, self.hwnd).into(); let iunknown: IUnknown = platform_range.into(); Ok(safe_array_from_com_slice(&[iunknown])) } else { @@ -874,7 +874,7 @@ patterns! { fn DocumentRange(&self) -> Result { self.resolve(|wrapper| { let range = wrapper.node.document_range(); - Ok(PlatformTextRange::new(&self.tree, range).into()) + Ok(PlatformTextRange::new(&self.tree, range, self.hwnd).into()) }) }, diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index 34d5862e1..4f754e686 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -15,7 +15,7 @@ use windows::{ Win32::{Foundation::*, System::Com::*, UI::Accessibility::*}, }; -use crate::util::*; +use crate::{node::PlatformNode, util::*}; fn upgrade_range<'a>(weak: &WeakRange, tree_state: &'a TreeState) -> Result> { if let Some(range) = weak.upgrade(tree_state) { @@ -36,7 +36,11 @@ fn position_from_endpoint<'a>( } } -fn set_endpoint_position<'a>(range: &mut Range<'a>, endpoint: TextPatternRangeEndpoint, pos: Position<'a>) -> Result<()> { +fn set_endpoint_position<'a>( + range: &mut Range<'a>, + endpoint: TextPatternRangeEndpoint, + pos: Position<'a>, +) -> Result<()> { match endpoint { TextPatternRangeEndpoint_Start => { range.set_start(pos); @@ -51,7 +55,11 @@ fn set_endpoint_position<'a>(range: &mut Range<'a>, endpoint: TextPatternRangeEn Ok(()) } -fn move_position_once<'a>(pos: Position<'a>, unit: TextUnit, forward: bool) -> Result> { +fn move_position_once<'a>( + pos: Position<'a>, + unit: TextUnit, + forward: bool, +) -> Result> { match unit { TextUnit_Character => { if forward { @@ -102,11 +110,15 @@ fn move_position_once<'a>(pos: Position<'a>, unit: TextUnit, forward: bool) -> R Ok(pos.backward_by_document()) } } - _ => Err(invalid_arg()) + _ => Err(invalid_arg()), } } -fn move_position<'a>(mut pos: Position<'a>, unit: TextUnit, count: i32) -> Result<(Position<'a>, i32)> { +fn move_position<'a>( + mut pos: Position<'a>, + unit: TextUnit, + count: i32, +) -> Result<(Position<'a>, i32)> { let forward = count > 0; let count = count.abs(); let mut moved = 0i32; @@ -132,13 +144,15 @@ fn move_position<'a>(mut pos: Position<'a>, unit: TextUnit, count: i32) -> Resul pub(crate) struct PlatformRange { tree: Weak, state: RwLock, + hwnd: HWND, } impl PlatformRange { - pub(crate) fn new(tree: &Weak, range: Range) -> Self { + pub(crate) fn new(tree: &Weak, range: Range, hwnd: HWND) -> Self { Self { tree: tree.clone(), state: RwLock::new(range.downgrade()), + hwnd, } } @@ -201,6 +215,7 @@ impl Clone for PlatformRange { PlatformRange { tree: self.tree.clone(), state: RwLock::new(*self.state.read()), + hwnd: self.hwnd, } } } @@ -300,7 +315,15 @@ impl ITextRangeProvider_Impl for PlatformRange { } fn GetEnclosingElement(&self) -> Result { - todo!() + self.read(|range| { + // Revisit this if we eventually support embedded objects. + Ok(PlatformNode { + tree: self.tree.clone(), + node_id: range.node().id(), + hwnd: self.hwnd, + } + .into()) + }) } fn GetText(&self, max_length: i32) -> Result { From 1f97a61d98a8976c7c5c753da7a3343ea226e100 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Tue, 18 Oct 2022 17:53:51 -0500 Subject: [PATCH 23/74] Fix clippy warnings --- platforms/windows/src/text.rs | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index 4f754e686..a1fa5cb87 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -55,11 +55,7 @@ fn set_endpoint_position<'a>( Ok(()) } -fn move_position_once<'a>( - pos: Position<'a>, - unit: TextUnit, - forward: bool, -) -> Result> { +fn move_position_once(pos: Position, unit: TextUnit, forward: bool) -> Result { match unit { TextUnit_Character => { if forward { @@ -114,11 +110,7 @@ fn move_position_once<'a>( } } -fn move_position<'a>( - mut pos: Position<'a>, - unit: TextUnit, - count: i32, -) -> Result<(Position<'a>, i32)> { +fn move_position(mut pos: Position, unit: TextUnit, count: i32) -> Result<(Position, i32)> { let forward = count > 0; let count = count.abs(); let mut moved = 0i32; @@ -242,10 +234,10 @@ impl ITextRangeProvider_Impl for PlatformRange { other_endpoint: TextPatternRangeEndpoint, ) -> Result { let other = required_param(other)?.as_impl(); - self.require_same_tree(&other)?; + self.require_same_tree(other)?; self.with_tree_state(|tree_state| { - let range = self.upgrade_for_read(&tree_state)?; - let other_range = other.upgrade_for_read(&tree_state)?; + let range = self.upgrade_for_read(tree_state)?; + let other_range = other.upgrade_for_read(tree_state)?; if range.node().id() != other_range.node().id() { return Err(invalid_arg()); } @@ -355,7 +347,7 @@ impl ITextRangeProvider_Impl for PlatformRange { other_endpoint: TextPatternRangeEndpoint, ) -> Result<()> { let other = required_param(other)?.as_impl(); - self.require_same_tree(&other)?; + self.require_same_tree(other)?; // We have to obtain the tree state and ranges manually to avoid // lifetime issues, and work with the two locks in a specific order // to avoid deadlock. From 83531375ec11c0eeb7d25e5adca38d3d2a7001c7 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Tue, 18 Oct 2022 17:57:38 -0500 Subject: [PATCH 24/74] Don't check hidden status of inline text boxes in the filter --- consumer/src/text.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 4fd4ad82f..bd29fdab6 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -360,7 +360,7 @@ impl WeakRange { } fn text_node_filter(root_id: NodeId, node: &Node) -> FilterResult { - if node.id() == root_id || (node.role() == Role::InlineTextBox && !node.is_hidden()) { + if node.id() == root_id || node.role() == Role::InlineTextBox { FilterResult::Include } else { FilterResult::ExcludeNode From 3543e5782d11fdb46f820e2b3829b3b1855df5e5 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Tue, 18 Oct 2022 18:59:28 -0500 Subject: [PATCH 25/74] Partial implementation of GetAttributeValue --- consumer/src/text.rs | 34 ++++++++++++++++++++++++++++++++++ platforms/windows/src/text.rs | 8 ++++++-- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index bd29fdab6..5673badd7 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -278,6 +278,40 @@ impl<'a> Range<'a> { } } + pub fn is_degenerate(&self) -> bool { + self.start.comparable(&self.node) == self.end.comparable(&self.node) + } + + fn walk(&self, mut f: impl FnMut(&Node, u16, u16)) { + let start = self.start.normalize_to_box_start(&self.node); + // For a degenerate range, the following avoids having `end` + // come before `start`. + let end = if self.is_degenerate() { + start + } else { + self.end.normalize_to_box_end(&self.node) + }; + if start.node.id() == end.node.id() { + f(&start.node, start.character_index, end.character_index); + return; + } + todo!() + } + + pub fn text(&self) -> String { + let mut result = String::new(); + self.walk(|node, start_index, end_index| { + let character_end_indices = &node.data().character_end_indices; + if start_index == 0 && (end_index as usize) == character_end_indices.len() { + // Fast path + result.push_str(node.value().unwrap()); + return; + } + todo!() + }); + result + } + pub fn expand_to_character(&mut self) { todo!() } diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index a1fa5cb87..1d247a5ec 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -318,8 +318,12 @@ impl ITextRangeProvider_Impl for PlatformRange { }) } - fn GetText(&self, max_length: i32) -> Result { - todo!() + fn GetText(&self, _max_length: i32) -> Result { + // The Microsoft docs imply that the provider isn't _required_ + // to truncate text at the max length, so we just ignore it. + self.read(|range| { + Ok(range.text().into()) + }) } fn Move(&self, unit: TextUnit, count: i32) -> Result { From 357268838e88522375eed4f36ef9973930b0a0f0 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Wed, 19 Oct 2022 09:50:22 -0500 Subject: [PATCH 26/74] Refactor text markers; drop the 'active suggestion' marker type since Chromium doesn't seem to be using it --- common/src/lib.rs | 40 +++++++++++----------------------------- 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/common/src/lib.rs b/common/src/lib.rs index f5eb4ece9..26338a3f1 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -21,7 +21,6 @@ use serde_lib as serde; use serde_lib::{Deserialize, Serialize}; use std::{ num::{NonZeroU128, NonZeroU64}, - ops::Range, sync::Arc, }; @@ -409,19 +408,6 @@ pub enum DropEffect { Popup, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "schemars", derive(JsonSchema))] -#[cfg_attr(feature = "serde", serde(crate = "serde"))] -#[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] -pub enum MarkerType { - SpellingError, - GrammarError, - SearchMatch, - ActiveSuggestion, - Suggestion, -} - #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "schemars", derive(JsonSchema))] @@ -606,19 +592,6 @@ impl From for NodeId { } } -/// A marker spanning a range within text. -#[derive(Clone, Debug, PartialEq, Eq)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "schemars", derive(JsonSchema))] -#[cfg_attr(feature = "serde", serde(crate = "serde"))] -#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] -#[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] -pub struct TextMarker { - pub marker_type: MarkerType, - /// Indices are in UTF-8 code units. - pub range: Range, -} - /// Defines a custom action for a UI element. /// /// For example, a list UI can allow a user to reorder items in the list by dragging the @@ -943,8 +916,17 @@ pub struct Node { pub radio_group: Vec, #[cfg_attr(feature = "serde", serde(default))] - #[cfg_attr(feature = "serde", serde(skip_serializing_if = "is_empty"))] - pub markers: Box<[TextMarker]>, + #[cfg_attr(feature = "serde", serde(skip_serializing_if = "is_false"))] + pub is_spelling_error: bool, + #[cfg_attr(feature = "serde", serde(default))] + #[cfg_attr(feature = "serde", serde(skip_serializing_if = "is_false"))] + pub is_grammar_error: bool, + #[cfg_attr(feature = "serde", serde(default))] + #[cfg_attr(feature = "serde", serde(skip_serializing_if = "is_false"))] + pub is_search_match: bool, + #[cfg_attr(feature = "serde", serde(default))] + #[cfg_attr(feature = "serde", serde(skip_serializing_if = "is_false"))] + pub is_suggestion: bool, #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] pub text_direction: Option, From 714fa26a19d8ed67a27b16b905f5414b6e161c38 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Wed, 19 Oct 2022 11:17:09 -0500 Subject: [PATCH 27/74] Refactor walk and text functions --- consumer/src/text.rs | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 5673badd7..d104334b7 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -282,7 +282,7 @@ impl<'a> Range<'a> { self.start.comparable(&self.node) == self.end.comparable(&self.node) } - fn walk(&self, mut f: impl FnMut(&Node, u16, u16)) { + fn walk(&self, mut f: impl FnMut(&Node)) { let start = self.start.normalize_to_box_start(&self.node); // For a degenerate range, the following avoids having `end` // come before `start`. @@ -291,17 +291,32 @@ impl<'a> Range<'a> { } else { self.end.normalize_to_box_end(&self.node) }; + f(&start.node); if start.node.id() == end.node.id() { - f(&start.node, start.character_index, end.character_index); return; } - todo!() + for node in start.node.following_inline_text_boxes(&self.node) { + f(&node); + if node.id() == end.node.id() { + break; + } + } } pub fn text(&self) -> String { let mut result = String::new(); - self.walk(|node, start_index, end_index| { + self.walk(|node| { let character_end_indices = &node.data().character_end_indices; + let start_index = if node.id() == self.start.node.id() { + self.start.character_index + } else { + 0 + }; + let end_index = if node.id() == self.end.node.id() { + self.end.character_index + } else { + character_end_indices.len() as u16 + }; if start_index == 0 && (end_index as usize) == character_end_indices.len() { // Fast path result.push_str(node.value().unwrap()); From 81031991cc59310602bceca6b98c3d74c008dfe9 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Wed, 19 Oct 2022 12:06:19 -0500 Subject: [PATCH 28/74] Skeleton of new consumer attribute method --- consumer/src/lib.rs | 5 ++++- consumer/src/text.rs | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/consumer/src/lib.rs b/consumer/src/lib.rs index cc2b987db..950bb25a5 100644 --- a/consumer/src/lib.rs +++ b/consumer/src/lib.rs @@ -13,7 +13,10 @@ pub(crate) mod iterators; pub use iterators::FilterResult; pub(crate) mod text; -pub use text::{Position as TextPosition, Range as TextRange, WeakRange as WeakTextRange}; +pub use text::{ + AttributeValue as TextAttributeValue, Position as TextPosition, Range as TextRange, + WeakRange as WeakTextRange, +}; #[cfg(test)] mod tests { diff --git a/consumer/src/text.rs b/consumer/src/text.rs index d104334b7..8f837cd95 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -245,6 +245,11 @@ impl<'a> PartialOrd for Position<'a> { } } +pub enum AttributeValue { + Single(T), + Mixed, +} + #[derive(Clone, Copy)] pub struct Range<'a> { node: Node<'a>, @@ -327,6 +332,16 @@ impl<'a> Range<'a> { result } + pub fn attribute(&self, f: F) -> AttributeValue + where + F: Fn(&Node) -> T, + T: Default + PartialEq, + { + let mut result = None; + todo!(); + AttributeValue::Single(result.unwrap_or_else(Default::default)) + } + pub fn expand_to_character(&mut self) { todo!() } From c48e3a5e6375c3a000eeacab08f5cdf11bfe7300 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Wed, 19 Oct 2022 13:03:10 -0500 Subject: [PATCH 29/74] Refactor walk to allow early result --- consumer/src/text.rs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 8f837cd95..9052e3d2b 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -287,7 +287,7 @@ impl<'a> Range<'a> { self.start.comparable(&self.node) == self.end.comparable(&self.node) } - fn walk(&self, mut f: impl FnMut(&Node)) { + fn walk(&self, mut f: impl FnMut(&Node) -> Option) -> Option { let start = self.start.normalize_to_box_start(&self.node); // For a degenerate range, the following avoids having `end` // come before `start`. @@ -296,21 +296,26 @@ impl<'a> Range<'a> { } else { self.end.normalize_to_box_end(&self.node) }; - f(&start.node); + if let Some(result) = f(&start.node) { + return Some(result); + } if start.node.id() == end.node.id() { - return; + return None; } for node in start.node.following_inline_text_boxes(&self.node) { - f(&node); + if let Some(result) = f(&node) { + return Some(result); + } if node.id() == end.node.id() { break; } } + None } pub fn text(&self) -> String { let mut result = String::new(); - self.walk(|node| { + self.walk::<()>(|node| { let character_end_indices = &node.data().character_end_indices; let start_index = if node.id() == self.start.node.id() { self.start.character_index @@ -325,9 +330,10 @@ impl<'a> Range<'a> { if start_index == 0 && (end_index as usize) == character_end_indices.len() { // Fast path result.push_str(node.value().unwrap()); - return; + return None; } - todo!() + todo!(); + None }); result } From 110e3ce3b464b6e0a84661c6b1f6eb7f2f8ecf35 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Wed, 19 Oct 2022 13:15:08 -0500 Subject: [PATCH 30/74] Implement Range::attribute --- consumer/src/text.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 9052e3d2b..020f67646 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -341,11 +341,21 @@ impl<'a> Range<'a> { pub fn attribute(&self, f: F) -> AttributeValue where F: Fn(&Node) -> T, - T: Default + PartialEq, + T: PartialEq, { - let mut result = None; - todo!(); - AttributeValue::Single(result.unwrap_or_else(Default::default)) + let mut value = None; + self.walk(|node| { + let current = f(node); + if let Some(value) = &value { + if *value != current { + return Some(AttributeValue::Mixed); + } + } else { + value = Some(current); + } + None + }) + .unwrap_or_else(|| AttributeValue::Single(value.unwrap())) } pub fn expand_to_character(&mut self) { From 253940c716e85ceb8eca31ece8a6b797ca47a781 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Wed, 19 Oct 2022 13:18:25 -0500 Subject: [PATCH 31/74] Small refactor to Range::text --- consumer/src/text.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 020f67646..f2a73105b 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -327,12 +327,14 @@ impl<'a> Range<'a> { } else { character_end_indices.len() as u16 }; - if start_index == 0 && (end_index as usize) == character_end_indices.len() { + let value = node.value().unwrap(); + let s = if start_index == 0 && (end_index as usize) == character_end_indices.len() { // Fast path - result.push_str(node.value().unwrap()); - return None; - } - todo!(); + value + } else { + todo!() + }; + result.push_str(s); None }); result From 216b425e952e8af44a82c0ade061516413c5d58e Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Wed, 19 Oct 2022 13:27:15 -0500 Subject: [PATCH 32/74] Find out which attribute NVDA wants first --- platforms/windows/src/text.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index 1d247a5ec..8c1e3b9f2 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -299,7 +299,13 @@ impl ITextRangeProvider_Impl for PlatformRange { } fn GetAttributeValue(&self, id: i32) -> Result { - todo!() + self.read(|range| { + match id { + _ => { + panic!("need attribute {}", id); + } + } + }) } fn GetBoundingRectangles(&self) -> Result<*mut SAFEARRAY> { From 3b125c80946c593ba99062a201fce43853c63aa5 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Wed, 19 Oct 2022 13:30:46 -0500 Subject: [PATCH 33/74] Appease Rust 1.61 --- consumer/src/text.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index f2a73105b..d95a8e0e4 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -287,7 +287,10 @@ impl<'a> Range<'a> { self.start.comparable(&self.node) == self.end.comparable(&self.node) } - fn walk(&self, mut f: impl FnMut(&Node) -> Option) -> Option { + fn walk(&self, mut f: F) -> Option + where + F: FnMut(&Node) -> Option, + { let start = self.start.normalize_to_box_start(&self.node); // For a degenerate range, the following avoids having `end` // come before `start`. @@ -315,7 +318,7 @@ impl<'a> Range<'a> { pub fn text(&self) -> String { let mut result = String::new(); - self.walk::<()>(|node| { + self.walk::<_, ()>(|node| { let character_end_indices = &node.data().character_end_indices; let start_index = if node.id() == self.start.node.id() { self.start.character_index From ff6ae5e5e3d77daa2c4a5f3c36e8afd8229360b9 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Wed, 19 Oct 2022 14:00:34 -0500 Subject: [PATCH 34/74] Basic fallback implementation of GetAttributeValue --- platforms/windows/src/text.rs | 4 +++- platforms/windows/src/util.rs | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index 8c1e3b9f2..6de963a06 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -302,7 +302,9 @@ impl ITextRangeProvider_Impl for PlatformRange { self.read(|range| { match id { _ => { - panic!("need attribute {}", id); + println!("want attribute {}", id); + let value = unsafe { UiaGetReservedNotSupportedValue() }.unwrap(); + Ok(VariantFactory::from(value).into()) } } }) diff --git a/platforms/windows/src/util.rs b/platforms/windows/src/util.rs index fa87551df..9b68a4c21 100644 --- a/platforms/windows/src/util.rs +++ b/platforms/windows/src/util.rs @@ -65,6 +65,17 @@ impl From for VariantFactory { } } +impl From for VariantFactory { + fn from(value: IUnknown) -> Self { + Self( + VT_UNKNOWN, + VARIANT_0_0_0 { + punkVal: ManuallyDrop::new(Some(value)), + }, + ) + } +} + impl From for VariantFactory { fn from(value: i32) -> Self { Self(VT_I4, VARIANT_0_0_0 { lVal: value }) From 6de129ed388ae64079b5c2866321b74d34158470 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Wed, 19 Oct 2022 14:20:47 -0500 Subject: [PATCH 35/74] Get NVDA to read the current character and selection --- consumer/src/text.rs | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index d95a8e0e4..3f789248e 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -335,7 +335,17 @@ impl<'a> Range<'a> { // Fast path value } else { - todo!() + let slice_start = if start_index == 0 { + 0 + } else { + character_end_indices[(start_index - 1) as usize] as usize + }; + let slice_end = if end_index == 0 { + 0 + } else { + character_end_indices[(end_index - 1) as usize] as usize + }; + &value[slice_start..slice_end] }; result.push_str(s); None @@ -364,7 +374,13 @@ impl<'a> Range<'a> { } pub fn expand_to_character(&mut self) { - todo!() + if !self.start.is_document_end(&self.node) { + self.start = self.start.normalize_to_box_start(&self.node); + self.end = InnerPosition { + node: self.start.node, + character_index: self.start.character_index + 1, + }; + } } pub fn expand_to_format(&mut self) { From 59ae45b398b804912dab2d135946171d487ec610 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Wed, 19 Oct 2022 15:39:13 -0500 Subject: [PATCH 36/74] Implement the Windows-specific 'expand' operation atop more generic primitives --- consumer/src/text.rs | 62 +++++++++-------------- platforms/windows/src/text.rs | 94 +++++++++++++++++++++++++---------- 2 files changed, 93 insertions(+), 63 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 3f789248e..c404e96ba 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -34,6 +34,10 @@ impl<'a> InnerPosition<'a> { self.character_index == 0 } + fn is_line_start(&self) -> bool { + self.is_box_start() && self.node.data().previous_on_line.is_none() + } + fn is_box_end(&self) -> bool { (self.character_index as usize) == self.node.data().character_end_indices.len() } @@ -133,6 +137,26 @@ pub struct Position<'a> { } impl<'a> Position<'a> { + pub fn is_format_start(&self) -> bool { + todo!() + } + + pub fn is_word_start(&self) -> bool { + todo!() + } + + pub fn is_line_start(&self) -> bool { + self.inner.is_line_start() + } + + pub fn is_paragraph_start(&self) -> bool { + todo!() + } + + pub fn is_page_start(&self) -> bool { + todo!() + } + pub fn is_document_start(&self) -> bool { self.inner.is_document_start(&self.root_node) } @@ -373,44 +397,6 @@ impl<'a> Range<'a> { .unwrap_or_else(|| AttributeValue::Single(value.unwrap())) } - pub fn expand_to_character(&mut self) { - if !self.start.is_document_end(&self.node) { - self.start = self.start.normalize_to_box_start(&self.node); - self.end = InnerPosition { - node: self.start.node, - character_index: self.start.character_index + 1, - }; - } - } - - pub fn expand_to_format(&mut self) { - // We don't currently support format runs, so fall back to document. - self.expand_to_document(); - } - - pub fn expand_to_word(&mut self) { - todo!() - } - - pub fn expand_to_line(&mut self) { - self.start = self.start.line_start(); - self.end = self.start.line_end(); - } - - pub fn expand_to_paragraph(&mut self) { - todo!() - } - - pub fn expand_to_page(&mut self) { - // We don't currently support pages, so fall back to document. - self.expand_to_document(); - } - - pub fn expand_to_document(&mut self) { - self.start = self.node.document_start(); - self.end = self.node.document_end(); - } - pub fn set_start(&mut self, pos: Position<'a>) { assert_eq!(pos.root_node.id(), self.node.id()); self.start = pos.inner; diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index 6de963a06..81f8e6023 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -55,6 +55,62 @@ fn set_endpoint_position<'a>( Ok(()) } +fn back_to_unit_start(start: Position, unit: TextUnit) -> Result { + match unit { + TextUnit_Character => { + // If we get here, this position is at the start of a non-degenerate + // range, so it's always at the start of a character. + debug_assert!(!start.is_document_end()); + Ok(start) + } + TextUnit_Format => { + if start.is_format_start() { + Ok(start) + } else { + Ok(start.backward_by_format()) + } + } + TextUnit_Word => { + if start.is_word_start() { + Ok(start) + } else { + Ok(start.backward_by_word()) + } + } + TextUnit_Line => { + if start.is_line_start() { + Ok(start) + } else { + Ok(start.backward_by_line()) + } + } + TextUnit_Paragraph => { + if start.is_paragraph_start() { + Ok(start) + } else { + Ok(start.backward_by_paragraph()) + } + } + TextUnit_Page => { + if start.is_page_start() { + Ok(start) + } else { + Ok(start.backward_by_page()) + } + } + TextUnit_Document => { + if start.is_document_start() { + Ok(start) + } else { + Ok(start.backward_by_document()) + } + } + _ => { + Err(invalid_arg()) + } + } +} + fn move_position_once(pos: Position, unit: TextUnit, forward: bool) -> Result { match unit { TextUnit_Character => { @@ -250,31 +306,19 @@ impl ITextRangeProvider_Impl for PlatformRange { fn ExpandToEnclosingUnit(&self, unit: TextUnit) -> Result<()> { self.write(|range| { - match unit { - TextUnit_Character => { - range.expand_to_character(); - } - TextUnit_Format => { - range.expand_to_format(); - } - TextUnit_Word => { - range.expand_to_word(); - } - TextUnit_Line => { - range.expand_to_line(); - } - TextUnit_Paragraph => { - range.expand_to_paragraph(); - } - TextUnit_Page => { - range.expand_to_page(); - } - TextUnit_Document => { - range.expand_to_document(); - } - _ => { - return Err(invalid_arg()); - } + let start = range.start(); + if unit == TextUnit_Character && start.is_document_end() { + // We know from experimentation that some Windows ATs + // expect ExpandToEnclosingUnit(TextUnit_Character) + // to do nothing if the range is degenerate at the end + // of the document. + return Ok(()); + } + let start = back_to_unit_start(start, unit)?; + range.set_start(start); + if !start.is_document_end() { + let end = move_position_once(start, unit, true)?; + range.set_end(end); } Ok(()) }) From b047e83ee66bd1bcf5f39d0f6f87810097fecede Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Wed, 19 Oct 2022 16:03:52 -0500 Subject: [PATCH 37/74] Implement ITextRangeProvider::Move --- platforms/windows/src/text.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index 81f8e6023..c5c4f1a3b 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -379,7 +379,26 @@ impl ITextRangeProvider_Impl for PlatformRange { } fn Move(&self, unit: TextUnit, count: i32) -> Result { - todo!() + self.write(|range| { + let degenerate = range.is_degenerate(); + let start = range.start(); + let start = if degenerate { + start + } else { + back_to_unit_start(start, unit)? + }; + let (start, moved) = move_position(start, unit, count)?; + if moved != 0 { + range.set_start(start); + let end = if degenerate || start.is_document_end() { + start + } else { + move_position_once(start, unit, true)? + }; + range.set_end(end); + } + Ok(moved) + }) } fn MoveEndpointByUnit( From 8a0900966cdb7cac1e7f972f271be0c5997fd5da Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Wed, 19 Oct 2022 18:24:53 -0500 Subject: [PATCH 38/74] Rename normalize functions --- consumer/src/text.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index c404e96ba..9b815ff09 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -60,7 +60,7 @@ impl<'a> InnerPosition<'a> { .is_none() } - fn normalize_to_box_start(&self, root_node: &Node) -> Self { + fn normalize_to_start(&self, root_node: &Node) -> Self { if self.is_box_end() { if let Some(node) = self.node.following_inline_text_boxes(root_node).next() { return Self { @@ -72,7 +72,7 @@ impl<'a> InnerPosition<'a> { *self } - fn normalize_to_box_end(&self, root_node: &Node) -> Self { + fn normalize_to_end(&self, root_node: &Node) -> Self { if self.is_box_start() { if let Some(node) = self.node.preceding_inline_text_boxes(root_node).next() { return Self { @@ -85,7 +85,7 @@ impl<'a> InnerPosition<'a> { } fn comparable(&self, root_node: &Node) -> (Vec, u16) { - let normalized = self.normalize_to_box_start(root_node); + let normalized = self.normalize_to_start(root_node); ( normalized.node.relative_index_path(root_node.id()), normalized.character_index, @@ -166,7 +166,7 @@ impl<'a> Position<'a> { } pub fn forward_by_character(&self) -> Self { - let normalized = self.inner.normalize_to_box_start(&self.root_node); + let normalized = self.inner.normalize_to_start(&self.root_node); Self { root_node: self.root_node, inner: InnerPosition { @@ -177,7 +177,7 @@ impl<'a> Position<'a> { } pub fn backward_by_character(&self) -> Self { - let normalized = self.inner.normalize_to_box_end(&self.root_node); + let normalized = self.inner.normalize_to_end(&self.root_node); Self { root_node: self.root_node, inner: InnerPosition { @@ -204,7 +204,7 @@ impl<'a> Position<'a> { } pub fn forward_by_line(&self) -> Self { - let normalized = self.inner.normalize_to_box_start(&self.root_node); + let normalized = self.inner.normalize_to_start(&self.root_node); Self { root_node: self.root_node, inner: normalized.line_end(), @@ -212,7 +212,7 @@ impl<'a> Position<'a> { } pub fn backward_by_line(&self) -> Self { - let normalized = self.inner.normalize_to_box_end(&self.root_node); + let normalized = self.inner.normalize_to_end(&self.root_node); Self { root_node: self.root_node, inner: normalized.line_start(), @@ -315,13 +315,13 @@ impl<'a> Range<'a> { where F: FnMut(&Node) -> Option, { - let start = self.start.normalize_to_box_start(&self.node); + let start = self.start.normalize_to_start(&self.node); // For a degenerate range, the following avoids having `end` // come before `start`. let end = if self.is_degenerate() { start } else { - self.end.normalize_to_box_end(&self.node) + self.end.normalize_to_end(&self.node) }; if let Some(result) = f(&start.node) { return Some(result); From 50be667c54e80516a083fd4dac8290d8f97a733a Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Wed, 19 Oct 2022 18:52:22 -0500 Subject: [PATCH 39/74] Try automatically normalizing range endpoints, with a special case for collapsing the range --- consumer/src/text.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 9b815ff09..a813cace2 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -399,7 +399,14 @@ impl<'a> Range<'a> { pub fn set_start(&mut self, pos: Position<'a>) { assert_eq!(pos.root_node.id(), self.node.id()); - self.start = pos.inner; + let pos = pos.inner; + self.start = if pos == self.end { + // Don't normalize when collapsing, as we want to preserve + // the start versus end distinction in that special case. + pos + } else { + pos.normalize_to_start(&self.node) + }; if self.start.comparable(&self.node) > self.end.comparable(&self.node) { self.end = self.start; } @@ -407,7 +414,14 @@ impl<'a> Range<'a> { pub fn set_end(&mut self, pos: Position<'a>) { assert_eq!(pos.root_node.id(), self.node.id()); - self.end = pos.inner; + let pos = pos.inner; + self.end = if pos == self.start { + // Don't normalize when collapsing, as we want to preserve + // the start versus end distinction in that special case. + pos + } else { + pos.normalize_to_end(&self.node) + }; if self.start.comparable(&self.node) > self.end.comparable(&self.node) { self.start = self.end; } From 9872e3d0a22aed3b04dcecf96e9be9c58112972d Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Thu, 20 Oct 2022 13:54:52 -0500 Subject: [PATCH 40/74] reformat --- platforms/windows/src/text.rs | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index c5c4f1a3b..f1f996216 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -105,9 +105,7 @@ fn back_to_unit_start(start: Position, unit: TextUnit) -> Result { Ok(start.backward_by_document()) } } - _ => { - Err(invalid_arg()) - } + _ => Err(invalid_arg()), } } @@ -343,13 +341,11 @@ impl ITextRangeProvider_Impl for PlatformRange { } fn GetAttributeValue(&self, id: i32) -> Result { - self.read(|range| { - match id { - _ => { - println!("want attribute {}", id); - let value = unsafe { UiaGetReservedNotSupportedValue() }.unwrap(); - Ok(VariantFactory::from(value).into()) - } + self.read(|range| match id { + _ => { + println!("want attribute {}", id); + let value = unsafe { UiaGetReservedNotSupportedValue() }.unwrap(); + Ok(VariantFactory::from(value).into()) } }) } @@ -373,9 +369,7 @@ impl ITextRangeProvider_Impl for PlatformRange { fn GetText(&self, _max_length: i32) -> Result { // The Microsoft docs imply that the provider isn't _required_ // to truncate text at the max length, so we just ignore it. - self.read(|range| { - Ok(range.text().into()) - }) + self.read(|range| Ok(range.text().into())) } fn Move(&self, unit: TextUnit, count: i32) -> Result { From c42850d92b5b2161684851a07f599de03c20ddf0 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Thu, 20 Oct 2022 13:54:59 -0500 Subject: [PATCH 41/74] emit text selection change events --- platforms/windows/src/node.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/platforms/windows/src/node.rs b/platforms/windows/src/node.rs index 35e433ddc..ee3a901b1 100644 --- a/platforms/windows/src/node.rs +++ b/platforms/windows/src/node.rs @@ -438,6 +438,15 @@ impl<'a> NodeWrapper<'a> { event_id: UIA_SelectionItem_ElementSelectedEventId, }); } + if self.is_text_pattern_supported() + && old.is_text_pattern_supported() + && self.node.text_selection() != old.node.text_selection() + { + queue.push(QueuedEvent::Simple { + element: element.clone(), + event_id: UIA_Text_TextSelectionChangedEventId, + }); + } } fn enqueue_property_change( From 777c5f0c3d59e0001ac2c0de85aa1c244e48db8e Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Thu, 20 Oct 2022 15:31:20 -0500 Subject: [PATCH 42/74] Don't store TextPosition::character_index as u16; it's not worth it --- common/src/lib.rs | 2 +- consumer/src/text.rs | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/common/src/lib.rs b/common/src/lib.rs index 26338a3f1..708dac0e9 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -630,7 +630,7 @@ pub struct TextPosition { pub node: NodeId, /// The index of an item in [`Node::character_end_indices`], or the length /// of that slice if the position is at the end of the line. - pub character_index: u16, + pub character_index: usize, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/consumer/src/text.rs b/consumer/src/text.rs index a813cace2..cd15d4e53 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -11,7 +11,7 @@ use crate::{FilterResult, Node, TreeState}; #[derive(Clone, Copy)] struct InnerPosition<'a> { node: Node<'a>, - character_index: u16, + character_index: usize, } impl<'a> InnerPosition<'a> { @@ -21,7 +21,7 @@ impl<'a> InnerPosition<'a> { return None; } let character_index = weak.character_index; - if (character_index as usize) > node.data().character_end_indices.len() { + if character_index > node.data().character_end_indices.len() { return None; } Some(Self { @@ -39,7 +39,7 @@ impl<'a> InnerPosition<'a> { } fn is_box_end(&self) -> bool { - (self.character_index as usize) == self.node.data().character_end_indices.len() + self.character_index == self.node.data().character_end_indices.len() } fn is_document_start(&self, root_node: &Node) -> bool { @@ -77,14 +77,14 @@ impl<'a> InnerPosition<'a> { if let Some(node) = self.node.preceding_inline_text_boxes(root_node).next() { return Self { node, - character_index: node.data().character_end_indices.len() as _, + character_index: node.data().character_end_indices.len(), }; } } *self } - fn comparable(&self, root_node: &Node) -> (Vec, u16) { + fn comparable(&self, root_node: &Node) -> (Vec, usize) { let normalized = self.normalize_to_start(root_node); ( normalized.node.relative_index_path(root_node.id()), @@ -110,7 +110,7 @@ impl<'a> InnerPosition<'a> { } Self { node, - character_index: node.data().character_end_indices.len() as _, + character_index: node.data().character_end_indices.len(), } } @@ -352,22 +352,22 @@ impl<'a> Range<'a> { let end_index = if node.id() == self.end.node.id() { self.end.character_index } else { - character_end_indices.len() as u16 + character_end_indices.len() }; let value = node.value().unwrap(); - let s = if start_index == 0 && (end_index as usize) == character_end_indices.len() { + let s = if start_index == 0 && end_index == character_end_indices.len() { // Fast path value } else { let slice_start = if start_index == 0 { 0 } else { - character_end_indices[(start_index - 1) as usize] as usize + character_end_indices[start_index - 1] as usize }; let slice_end = if end_index == 0 { 0 } else { - character_end_indices[(end_index - 1) as usize] as usize + character_end_indices[end_index - 1] as usize }; &value[slice_start..slice_end] }; @@ -512,7 +512,7 @@ impl<'a> Node<'a> { let node = self.inline_text_boxes().next_back().unwrap(); InnerPosition { node, - character_index: node.data().character_end_indices.len() as u16, + character_index: node.data().character_end_indices.len(), } } From ac32f48c316ea2ad315be45fd35401641f6ad61a Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Fri, 21 Oct 2022 13:43:33 -0500 Subject: [PATCH 43/74] Refactor character and word boundaries --- common/src/lib.rs | 58 ++++++++++++++++++++++++++------------------ consumer/src/text.rs | 40 +++++++++++++++--------------- 2 files changed, 56 insertions(+), 42 deletions(-) diff --git a/common/src/lib.rs b/common/src/lib.rs index 708dac0e9..4b8b3411a 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -628,7 +628,7 @@ fn is_empty(slice: &[T]) -> bool { pub struct TextPosition { /// The node's role must be [`Role::InlineTextBox`]. pub node: NodeId, - /// The index of an item in [`Node::character_end_indices`], or the length + /// The index of an item in [`Node::character_lengths`], or the length /// of that slice if the position is at the end of the line. pub character_index: usize, } @@ -931,15 +931,14 @@ pub struct Node { #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] pub text_direction: Option, - /// For inline text. The end index (non-inclusive) of each character - /// in UTF-8 code units (bytes). For example, if the text box - /// consists of a 1-byte character, a 3-byte character, and another - /// 1-byte character, the indices would be [1, 4, 5]. + /// For inline text. The length (non-inclusive) of each character + /// in UTF-8 code units (bytes). The sum of these lengths must equal + /// the length of [`Node::value`], also in bytes. /// /// A character is defined as the smallest unit of text that /// can be selected. This isn't necessarily a single Unicode /// scalar value (code point). This is why AccessKit can't compute - /// the indices of the characters from the text itself; this information + /// the lengths of the characters from the text itself; this information /// must be provided by the text editing implementation. /// /// If this node is the last text box in a line that ends with a hard @@ -950,30 +949,43 @@ pub struct Node { /// selection should be on the line break, not after it. #[cfg_attr(feature = "serde", serde(default))] #[cfg_attr(feature = "serde", serde(skip_serializing_if = "is_empty"))] - pub character_end_indices: Box<[u16]>, - /// For inline text. This is the pixel position of the end of each - /// character within the bounding rectangle of this object, in the direction - /// given by [`Node::text_direction`]. For example, for left-to-right - /// text, the first offset is the right coordinate of the first - /// character within the object's bounds, the second offset is - /// the right coordinate of the second character, and so on. + pub character_lengths: Box<[u8]>, + /// For inline text. This is the length of each character in pixels + /// within the bounding rectangle of this object, in the direction + /// given by [`Node::text_direction`]. /// /// When present, the length of this slice should be the same as the length - /// of [`character_end_incides`], including for lines that end - /// with a hard line break. The end offset of such a line break should + /// of [`Node::character_lengths`], including for lines that end + /// with a hard line break. The pixel length of such a line break should /// be non-zero if selecting the line break by itself results in /// a visible highlight (as in Microsoft Word), or zero if not /// (as in Windows Notepad). #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] - pub character_end_pixel_offsets: Option>, + pub character_pixel_lengths: Option>, - /// For inline text. The end index (non-inclusive) of each word - /// in UTF-8 code units (bytes). For example, if the text box - /// consists of a 1-byte word (e.g. a leading space), a 3-byte word - /// (e.g. two ASCII letters followed by a space), and a 1-byte word - /// (e.g. an ASCII letter), the indices would be [1, 4, 5]. - #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] - pub word_end_indices: Option>, + /// For inline text. The length of each word in characters, as defined + /// in [`Node::character_lengths`]. The sum of these lengths must equal + /// the length of [`Node::character_lengths`]. + /// + /// The end of each word is the beginning of the next word; there are no + /// characters that are not considered part of a word. Trailing whitespace + /// is typically considered part of the word that precedes it, while + /// a line's leading whitepsace is considered its own word. Whether + /// punctuation is considered a separate word or part of the preceding + /// word depends on the particular text editing implementation. + /// Some editors may have their own definition of a word; for example, + /// in an IDE, words may correspond to programming language tokens. + /// + /// Not all assistive technologies require information about word + /// boundaries, and not all platform accessibility APIs even expose + /// this information, but for assistive technologies that do use + /// this information, users will get unpredictable results if the word + /// boundaries exposed by the accessibility tree don't match + /// the editor's behavior. This is why AccessKit does not determine + /// word boundaries itself. + #[cfg_attr(feature = "serde", serde(default))] + #[cfg_attr(feature = "serde", serde(skip_serializing_if = "is_empty"))] + pub word_lengths: Box<[u8]>, #[cfg_attr(feature = "serde", serde(default))] #[cfg_attr(feature = "serde", serde(skip_serializing_if = "is_empty"))] diff --git a/consumer/src/text.rs b/consumer/src/text.rs index cd15d4e53..e51be2da1 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -21,7 +21,7 @@ impl<'a> InnerPosition<'a> { return None; } let character_index = weak.character_index; - if character_index > node.data().character_end_indices.len() { + if character_index > node.data().character_lengths.len() { return None; } Some(Self { @@ -39,7 +39,7 @@ impl<'a> InnerPosition<'a> { } fn is_box_end(&self) -> bool { - self.character_index == self.node.data().character_end_indices.len() + self.character_index == self.node.data().character_lengths.len() } fn is_document_start(&self, root_node: &Node) -> bool { @@ -77,7 +77,7 @@ impl<'a> InnerPosition<'a> { if let Some(node) = self.node.preceding_inline_text_boxes(root_node).next() { return Self { node, - character_index: node.data().character_end_indices.len(), + character_index: node.data().character_lengths.len(), }; } } @@ -110,7 +110,7 @@ impl<'a> InnerPosition<'a> { } Self { node, - character_index: node.data().character_end_indices.len(), + character_index: node.data().character_lengths.len(), } } @@ -343,7 +343,7 @@ impl<'a> Range<'a> { pub fn text(&self) -> String { let mut result = String::new(); self.walk::<_, ()>(|node| { - let character_end_indices = &node.data().character_end_indices; + let character_lengths = &node.data().character_lengths; let start_index = if node.id() == self.start.node.id() { self.start.character_index } else { @@ -352,23 +352,25 @@ impl<'a> Range<'a> { let end_index = if node.id() == self.end.node.id() { self.end.character_index } else { - character_end_indices.len() + character_lengths.len() }; let value = node.value().unwrap(); - let s = if start_index == 0 && end_index == character_end_indices.len() { - // Fast path + let s = if start_index == end_index { + "" + } else if start_index == 0 && end_index == character_lengths.len() { value } else { - let slice_start = if start_index == 0 { - 0 - } else { - character_end_indices[start_index - 1] as usize - }; - let slice_end = if end_index == 0 { - 0 - } else { - character_end_indices[end_index - 1] as usize - }; + let slice_start = character_lengths[..start_index] + .iter() + .copied() + .map(usize::from) + .sum::(); + let slice_end = slice_start + + character_lengths[start_index..end_index] + .iter() + .copied() + .map(usize::from) + .sum::(); &value[slice_start..slice_end] }; result.push_str(s); @@ -512,7 +514,7 @@ impl<'a> Node<'a> { let node = self.inline_text_boxes().next_back().unwrap(); InnerPosition { node, - character_index: node.data().character_end_indices.len(), + character_index: node.data().character_lengths.len(), } } From 8e0b2a09d5907e0cbfb00efb37190bf507da0c21 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Fri, 21 Oct 2022 13:44:38 -0500 Subject: [PATCH 44/74] Drop the println in GetAttributeValue for now --- platforms/windows/src/text.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index f1f996216..d95d0c169 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -342,8 +342,8 @@ impl ITextRangeProvider_Impl for PlatformRange { fn GetAttributeValue(&self, id: i32) -> Result { self.read(|range| match id { + // TODO: implement attributes _ => { - println!("want attribute {}", id); let value = unsafe { UiaGetReservedNotSupportedValue() }.unwrap(); Ok(VariantFactory::from(value).into()) } From 5b43c8c86f0c243638fc889bf14b326c036d906e Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Fri, 21 Oct 2022 15:56:45 -0500 Subject: [PATCH 45/74] Movement by word --- consumer/src/text.rs | 54 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index e51be2da1..44249df1b 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -30,6 +30,17 @@ impl<'a> InnerPosition<'a> { }) } + fn is_word_start(&self) -> bool { + let mut total_length = 0usize; + for length in self.node.data().word_lengths.iter() { + if total_length == self.character_index { + return true; + } + total_length += *length as usize; + } + false + } + fn is_box_start(&self) -> bool { self.character_index == 0 } @@ -92,6 +103,35 @@ impl<'a> InnerPosition<'a> { ) } + fn previous_word_start(&self) -> Self { + let mut total_length_before = 0usize; + for length in self.node.data().word_lengths.iter() { + let new_total_length = total_length_before + (*length as usize); + if new_total_length >= self.character_index { + break; + } + total_length_before = new_total_length; + } + Self { + node: self.node, + character_index: total_length_before, + } + } + + fn word_end(&self) -> Self { + let mut total_length = 0usize; + for length in self.node.data().word_lengths.iter() { + total_length += *length as usize; + if total_length > self.character_index { + break; + } + } + Self { + node: self.node, + character_index: total_length, + } + } + fn line_start(&self) -> Self { let mut node = self.node; while let Some(id) = node.data().previous_on_line { @@ -142,7 +182,7 @@ impl<'a> Position<'a> { } pub fn is_word_start(&self) -> bool { - todo!() + self.inner.is_word_start() } pub fn is_line_start(&self) -> bool { @@ -196,11 +236,19 @@ impl<'a> Position<'a> { } pub fn forward_by_word(&self) -> Self { - todo!() + let normalized = self.inner.normalize_to_start(&self.root_node); + Self { + root_node: self.root_node, + inner: normalized.word_end(), + } } pub fn backward_by_word(&self) -> Self { - todo!() + let normalized = self.inner.normalize_to_end(&self.root_node); + Self { + root_node: self.root_node, + inner: normalized.previous_word_start(), + } } pub fn forward_by_line(&self) -> Self { From 92e5d7bfb3b95092b661c4b58ebecf9033929ec4 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Thu, 27 Oct 2022 14:12:58 -0500 Subject: [PATCH 46/74] Return invalid operation error in AddToSelection and RemoveFromSelection --- platforms/windows/src/text.rs | 6 ++++-- platforms/windows/src/util.rs | 4 ++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index d95d0c169..5fd19aa75 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -439,11 +439,13 @@ impl ITextRangeProvider_Impl for PlatformRange { } fn AddToSelection(&self) -> Result<()> { - todo!() + // AccessKit doesn't support multiple text selections. + Err(invalid_operation()) } fn RemoveFromSelection(&self) -> Result<()> { - todo!() + // AccessKit doesn't support multiple text selections. + Err(invalid_operation()) } fn ScrollIntoView(&self, align_to_top: BOOL) -> Result<()> { diff --git a/platforms/windows/src/util.rs b/platforms/windows/src/util.rs index 9b68a4c21..a43bba7ba 100644 --- a/platforms/windows/src/util.rs +++ b/platforms/windows/src/util.rs @@ -177,3 +177,7 @@ pub(crate) fn required_param(param: &Option) -> Result<&T> { pub(crate) fn element_not_available() -> Error { Error::new(HRESULT(UIA_E_ELEMENTNOTAVAILABLE as i32), "".into()) } + +pub(crate) fn invalid_operation() -> Error { + Error::new(HRESULT(UIA_E_INVALIDOPERATION as i32), "".into()) +} From e979b20ca5974867722dbbb38a032da84b344e85 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Thu, 27 Oct 2022 14:42:03 -0500 Subject: [PATCH 47/74] Implement ScrollIntoView --- consumer/src/text.rs | 8 ++++---- consumer/src/tree.rs | 18 +++++++++++++++++- platforms/windows/src/text.rs | 20 +++++++++++++++++++- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 44249df1b..e22e5182f 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -9,9 +9,9 @@ use std::{cmp::Ordering, iter::FusedIterator}; use crate::{FilterResult, Node, TreeState}; #[derive(Clone, Copy)] -struct InnerPosition<'a> { - node: Node<'a>, - character_index: usize, +pub(crate) struct InnerPosition<'a> { + pub(crate) node: Node<'a>, + pub(crate) character_index: usize, } impl<'a> InnerPosition<'a> { @@ -173,7 +173,7 @@ impl<'a> Eq for InnerPosition<'a> {} #[derive(Clone, Copy)] pub struct Position<'a> { root_node: Node<'a>, - inner: InnerPosition<'a>, + pub(crate) inner: InnerPosition<'a>, } impl<'a> Position<'a> { diff --git a/consumer/src/tree.rs b/consumer/src/tree.rs index 4ca21bf50..b1348b903 100644 --- a/consumer/src/tree.rs +++ b/consumer/src/tree.rs @@ -14,7 +14,7 @@ use std::{ sync::Arc, }; -use crate::Node; +use crate::{text::Position as TextPosition, Node}; #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) struct ParentAndIndex(pub(crate) NodeId, pub(crate) usize); @@ -340,6 +340,22 @@ impl Tree { data: Some(ActionData::NumericValue(value)), }) } + + pub fn scroll_into_view(&self, target: NodeId) { + self.action_handler.do_action(ActionRequest { + action: Action::ScrollIntoView, + target, + data: None, + }) + } + + pub fn scroll_text_position_into_view(&self, target: &TextPosition) { + self.action_handler.do_action(ActionRequest { + action: Action::ScrollIntoView, + target: target.inner.node.id(), + data: None, + }) + } } #[cfg(test)] diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index 5fd19aa75..deaa73a68 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -247,6 +247,16 @@ impl PlatformRange { }) } + fn action(&self, f: F) -> Result<()> + where + for<'a> F: FnOnce(&'a Tree, Range<'a>) -> Result<()>, + { + let tree = self.upgrade_tree()?; + let tree_state = tree.read(); + let range = self.upgrade_for_read(&tree_state)?; + f(&tree, range) + } + fn require_same_tree(&self, other: &PlatformRange) -> Result<()> { if self.tree.ptr_eq(&other.tree) { Ok(()) @@ -449,7 +459,15 @@ impl ITextRangeProvider_Impl for PlatformRange { } fn ScrollIntoView(&self, align_to_top: BOOL) -> Result<()> { - todo!() + self.action(|tree, range| { + let position = if align_to_top.into() { + range.start() + } else { + range.end() + }; + tree.scroll_text_position_into_view(&position); + Ok(()) + }) } fn GetChildren(&self) -> Result<*mut SAFEARRAY> { From b5beeb5e1de2f1624aae7cf9e10965cb2cb564cd Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Thu, 27 Oct 2022 14:54:46 -0500 Subject: [PATCH 48/74] Implement Select --- consumer/src/text.rs | 8 ++++---- consumer/src/tree.rs | 21 ++++++++++++++++++--- platforms/windows/src/text.rs | 5 ++++- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index e22e5182f..459775720 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -154,7 +154,7 @@ impl<'a> InnerPosition<'a> { } } - fn downgrade(&self) -> WeakPosition { + pub(crate) fn downgrade(&self) -> WeakPosition { WeakPosition { node: self.node.id(), character_index: self.character_index, @@ -324,9 +324,9 @@ pub enum AttributeValue { #[derive(Clone, Copy)] pub struct Range<'a> { - node: Node<'a>, - start: InnerPosition<'a>, - end: InnerPosition<'a>, + pub(crate) node: Node<'a>, + pub(crate) start: InnerPosition<'a>, + pub(crate) end: InnerPosition<'a>, } impl<'a> Range<'a> { diff --git a/consumer/src/tree.rs b/consumer/src/tree.rs index b1348b903..6fea28c2d 100644 --- a/consumer/src/tree.rs +++ b/consumer/src/tree.rs @@ -4,8 +4,8 @@ // the LICENSE-MIT file), at your option. use accesskit::{ - Action, ActionData, ActionHandler, ActionRequest, Node as NodeData, NodeId, Tree as TreeData, - TreeUpdate, + Action, ActionData, ActionHandler, ActionRequest, Node as NodeData, NodeId, TextSelection, + Tree as TreeData, TreeUpdate, }; use parking_lot::{RwLock, RwLockWriteGuard}; use std::{ @@ -14,7 +14,10 @@ use std::{ sync::Arc, }; -use crate::{text::Position as TextPosition, Node}; +use crate::{ + text::{Position as TextPosition, Range as TextRange}, + Node, +}; #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) struct ParentAndIndex(pub(crate) NodeId, pub(crate) usize); @@ -356,6 +359,18 @@ impl Tree { data: None, }) } + + pub fn select_text_range(&self, range: &TextRange) { + let selection = TextSelection { + anchor: range.start.downgrade(), + focus: range.end.downgrade(), + }; + self.action_handler.do_action(ActionRequest { + action: Action::SetTextSelection, + target: range.node.id(), + data: Some(ActionData::SetTextSelection(selection)), + }) + } } #[cfg(test)] diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index deaa73a68..4c3eebdef 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -445,7 +445,10 @@ impl ITextRangeProvider_Impl for PlatformRange { } fn Select(&self) -> Result<()> { - todo!() + self.action(|tree, range| { + tree.select_text_range(&range); + Ok(()) + }) } fn AddToSelection(&self) -> Result<()> { From a39211d64451f7b9cbd2872a05a0353b5e63006f Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Thu, 27 Oct 2022 15:07:32 -0500 Subject: [PATCH 49/74] Page is synonymous with document for now --- consumer/src/text.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 459775720..db7b8cf3a 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -194,7 +194,7 @@ impl<'a> Position<'a> { } pub fn is_page_start(&self) -> bool { - todo!() + self.is_document_start() } pub fn is_document_start(&self) -> bool { @@ -276,11 +276,11 @@ impl<'a> Position<'a> { } pub fn forward_by_page(&self) -> Self { - todo!() + self.forward_by_document() } pub fn backward_by_page(&self) -> Self { - todo!() + self.backward_by_document() } pub fn forward_by_document(&self) -> Self { From 4bb59db44641dbbd5c1e5da5168c3b69b84e2e2e Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Thu, 27 Oct 2022 15:27:45 -0500 Subject: [PATCH 50/74] Decide that the format unit is also synonymous with document for now, but add a todo comment to eventually change that --- consumer/src/text.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index db7b8cf3a..33fb03716 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -178,7 +178,8 @@ pub struct Position<'a> { impl<'a> Position<'a> { pub fn is_format_start(&self) -> bool { - todo!() + // TODO: support variable text formatting (part of rich text) + self.is_document_start() } pub fn is_word_start(&self) -> bool { @@ -228,11 +229,13 @@ impl<'a> Position<'a> { } pub fn forward_by_format(&self) -> Self { - todo!() + // TODO: support variable text formatting (part of rich text) + self.forward_by_document() } pub fn backward_by_format(&self) -> Self { - todo!() + // TODO: support variable text formatting (part of rich text) + self.backward_by_document() } pub fn forward_by_word(&self) -> Self { From e1c2b8697b8270d6b5321b04222c034732c90bed Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Thu, 27 Oct 2022 16:11:15 -0500 Subject: [PATCH 51/74] Implement movement by paragraph --- consumer/src/text.rs | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 33fb03716..c2b08625c 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -53,6 +53,14 @@ impl<'a> InnerPosition<'a> { self.character_index == self.node.data().character_lengths.len() } + fn is_line_end(&self) -> bool { + self.is_box_end() && self.node.data().next_on_line.is_none() + } + + fn is_paragraph_end(&self) -> bool { + self.is_line_end() && self.node.value().unwrap().ends_with('\n') + } + fn is_document_start(&self, root_node: &Node) -> bool { self.is_box_start() && self @@ -191,7 +199,12 @@ impl<'a> Position<'a> { } pub fn is_paragraph_start(&self) -> bool { - todo!() + self.is_document_start() + || (self.is_line_start() + && self + .inner + .normalize_to_end(&self.root_node) + .is_paragraph_end()) } pub fn is_page_start(&self) -> bool { @@ -271,11 +284,25 @@ impl<'a> Position<'a> { } pub fn forward_by_paragraph(&self) -> Self { - todo!() + let mut current = *self; + loop { + current = current.forward_by_line(); + if current.is_document_end() || current.inner.is_paragraph_end() { + break; + } + } + current } pub fn backward_by_paragraph(&self) -> Self { - todo!() + let mut current = *self; + loop { + current = current.backward_by_line(); + if current.is_paragraph_start() { + break; + } + } + current } pub fn forward_by_page(&self) -> Self { From 7136fa0a12856541967a5f1941f0deb939227c1e Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Thu, 27 Oct 2022 16:39:43 -0500 Subject: [PATCH 52/74] Decide that FindAttribute and FindText are out of scope for now --- platforms/windows/src/text.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index 4c3eebdef..4cc6edec3 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -334,20 +334,25 @@ impl ITextRangeProvider_Impl for PlatformRange { fn FindAttribute( &self, - id: i32, - value: &VARIANT, - backward: BOOL, + _id: i32, + _value: &VARIANT, + _backward: BOOL, ) -> Result { - todo!() + // TODO: implement when we support variable formatting (part of rich text) + // Justification: JUCE doesn't implement this. + Err(Error::OK) } fn FindText( &self, - text: &BSTR, - backward: BOOL, - ignore_case: BOOL, + _text: &BSTR, + _backward: BOOL, + _ignore_case: BOOL, ) -> Result { - todo!() + // TODO: implement when there's a real-world use case that requires it + // Justification: Quorum doesn't implement this and is being used + // by blind students. + Err(Error::OK) } fn GetAttributeValue(&self, id: i32) -> Result { From e13b22f3b2c9dee81cfd24988b121c172967217b Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Thu, 27 Oct 2022 16:46:39 -0500 Subject: [PATCH 53/74] support one actual text attribute so far: read-only --- platforms/windows/src/text.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index 4cc6edec3..6d1ea3b4c 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -357,7 +357,12 @@ impl ITextRangeProvider_Impl for PlatformRange { fn GetAttributeValue(&self, id: i32) -> Result { self.read(|range| match id { - // TODO: implement attributes + UIA_IsReadOnlyAttributeId => { + // TBD: do we ever want to support mixed read-only/editable text? + let value = range.node().is_read_only(); + Ok(VariantFactory::from(value).into()) + } + // TODO: implement more attributes _ => { let value = unsafe { UiaGetReservedNotSupportedValue() }.unwrap(); Ok(VariantFactory::from(value).into()) From a4e4c63b43d818b665fde0f04b6b16e79c832cf9 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Thu, 27 Oct 2022 16:52:16 -0500 Subject: [PATCH 54/74] Don't crash in GetBoundingRectangles; still not implemented yet --- platforms/windows/src/text.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index 6d1ea3b4c..8cf221afd 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -371,7 +371,9 @@ impl ITextRangeProvider_Impl for PlatformRange { } fn GetBoundingRectangles(&self) -> Result<*mut SAFEARRAY> { - todo!() + // TODO: necessary for screen magnifiers and other visual aids + // (e.g. Narrator highlight cursor) + Err(not_implemented()) } fn GetEnclosingElement(&self) -> Result { From 59c33f775185ca4d6291ea3a89714c72edeeb777 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Thu, 27 Oct 2022 17:53:43 -0500 Subject: [PATCH 55/74] Raise text change events --- platforms/windows/src/adapter.rs | 37 +++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/platforms/windows/src/adapter.rs b/platforms/windows/src/adapter.rs index 1e327c88e..8ad793646 100644 --- a/platforms/windows/src/adapter.rs +++ b/platforms/windows/src/adapter.rs @@ -3,9 +3,9 @@ // the LICENSE-APACHE file) or the MIT license (found in // the LICENSE-MIT file), at your option. -use std::sync::Arc; +use std::{collections::HashSet, sync::Arc}; -use accesskit::{ActionHandler, Live, TreeUpdate}; +use accesskit::{ActionHandler, Live, NodeId, Role, TreeUpdate}; use accesskit_consumer::{FilterResult, Node, Tree, TreeChangeHandler}; use lazy_init::LazyTransform; use windows::Win32::{ @@ -100,9 +100,34 @@ impl Adapter { tree: &'a Arc, hwnd: HWND, queue: Vec, + text_changed: HashSet, + } + impl Handler<'_> { + fn enqueue_text_change_if_needed(&mut self, node: &Node) { + if node.role() != Role::InlineTextBox { + return; + } + if let Some(node) = node.filtered_parent(&filter) { + if !node.supports_text_ranges() { + return; + } + let id = node.id(); + if self.text_changed.contains(&id) { + return; + } + let platform_node = PlatformNode::new(self.tree, node.id(), self.hwnd); + let element: IRawElementProviderSimple = platform_node.into(); + self.queue.push(QueuedEvent::Simple { + element, + event_id: UIA_Text_TextChangedEventId, + }); + self.text_changed.insert(id); + } + } } impl TreeChangeHandler for Handler<'_> { fn node_added(&mut self, node: &Node) { + self.enqueue_text_change_if_needed(node); if filter(node) != FilterResult::Include { return; } @@ -116,6 +141,9 @@ impl Adapter { } } fn node_updated(&mut self, old_node: &Node, new_node: &Node) { + if old_node.value() != new_node.value() { + self.enqueue_text_change_if_needed(new_node); + } if filter(new_node) != FilterResult::Include { return; } @@ -146,13 +174,16 @@ impl Adapter { }); } } - fn node_removed(&mut self, _node: &Node) {} + fn node_removed(&mut self, node: &Node) { + self.enqueue_text_change_if_needed(node); + } // TODO: handle other events (#20) } let mut handler = Handler { tree, hwnd: self.hwnd, queue: Vec::new(), + text_changed: HashSet::new(), }; tree.update_and_process_changes(update, &mut handler); QueuedEvents(handler.queue) From 6b9e6f969467aad8696eb2807e0b20583a8b3fa4 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Thu, 27 Oct 2022 19:18:54 -0500 Subject: [PATCH 56/74] Require the last inline text box to end with a line break --- common/src/lib.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/common/src/lib.rs b/common/src/lib.rs index 4b8b3411a..e2a262185 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -947,6 +947,16 @@ pub struct Node { /// should be counted as a single character for the sake of this slice. /// When the caret is at the end of such a line, the focus of the text /// selection should be on the line break, not after it. + /// + /// The last inline text box in a text field or document must end + /// with a hard line break as described above. If the last line + /// is blank, the text box for that line must consist of only + /// a hard line break. If the text field is empty, it must consist + /// of a single inline text box containing only a hard line break. + /// This requirement applies even to text fields that only ever contain + /// one line of text. This consistency eliminates special cases + /// in text navigation which can lead to bugs with some assistive + /// technologies. #[cfg_attr(feature = "serde", serde(default))] #[cfg_attr(feature = "serde", serde(skip_serializing_if = "is_empty"))] pub character_lengths: Box<[u8]>, From 96a44f1f2e5045f56d822649ee3a90f1b02eef5c Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Fri, 28 Oct 2022 08:14:31 -0500 Subject: [PATCH 57/74] Allow a couple of functions on a text range whose endpoints are no longer valid --- consumer/src/text.rs | 6 +++++- platforms/windows/src/text.rs | 39 ++++++++++++++++++++++++++++------- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index c2b08625c..5b63a8fd5 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -532,8 +532,12 @@ pub struct WeakRange { } impl WeakRange { + pub fn upgrade_node<'a>(&self, tree_state: &'a TreeState) -> Option> { + tree_state.node_by_id(self.node_id) + } + pub fn upgrade<'a>(&self, tree_state: &'a TreeState) -> Option> { - let node = tree_state.node_by_id(self.node_id)?; + let node = self.upgrade_node(tree_state)?; let start = InnerPosition::upgrade(tree_state, self.start)?; let end = InnerPosition::upgrade(tree_state, self.end)?; Some(Range { node, start, end }) diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index 8cf221afd..70039a993 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -6,7 +6,7 @@ #![allow(non_upper_case_globals)] use accesskit_consumer::{ - TextPosition as Position, TextRange as Range, Tree, TreeState, WeakTextRange as WeakRange, + Node, TextPosition as Position, TextRange as Range, Tree, TreeState, WeakTextRange as WeakRange, }; use parking_lot::RwLock; use std::sync::{Arc, Weak}; @@ -25,6 +25,14 @@ fn upgrade_range<'a>(weak: &WeakRange, tree_state: &'a TreeState) -> Result(weak: &WeakRange, tree_state: &'a TreeState) -> Result> { + if let Some(node) = weak.upgrade_node(tree_state) { + Ok(node) + } else { + Err(element_not_available()) + } +} + fn position_from_endpoint<'a>( range: &Range<'a>, endpoint: TextPatternRangeEndpoint, @@ -219,6 +227,21 @@ impl PlatformRange { f(&state) } + fn upgrade_node<'a>(&self, tree_state: &'a TreeState) -> Result> { + let state = self.state.read(); + upgrade_range_node(&state, tree_state) + } + + fn with_node(&self, f: F) -> Result + where + F: FnOnce(Node) -> Result, + { + self.with_tree_state(|tree_state| { + let node = self.upgrade_node(tree_state)?; + f(node) + }) + } + fn upgrade_for_read<'a>(&self, tree_state: &'a TreeState) -> Result> { let state = self.state.read(); upgrade_range(&state, tree_state) @@ -356,18 +379,20 @@ impl ITextRangeProvider_Impl for PlatformRange { } fn GetAttributeValue(&self, id: i32) -> Result { - self.read(|range| match id { + match id { UIA_IsReadOnlyAttributeId => { // TBD: do we ever want to support mixed read-only/editable text? - let value = range.node().is_read_only(); - Ok(VariantFactory::from(value).into()) + self.with_node(|node| { + let value = node.is_read_only(); + Ok(VariantFactory::from(value).into()) + }) } // TODO: implement more attributes _ => { let value = unsafe { UiaGetReservedNotSupportedValue() }.unwrap(); Ok(VariantFactory::from(value).into()) } - }) + } } fn GetBoundingRectangles(&self) -> Result<*mut SAFEARRAY> { @@ -377,11 +402,11 @@ impl ITextRangeProvider_Impl for PlatformRange { } fn GetEnclosingElement(&self) -> Result { - self.read(|range| { + self.with_node(|node| { // Revisit this if we eventually support embedded objects. Ok(PlatformNode { tree: self.tree.clone(), - node_id: range.node().id(), + node_id: node.id(), hwnd: self.hwnd, } .into()) From 60470518933d5442012b41ac5ccbfe9809b962b0 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Fri, 28 Oct 2022 10:14:29 -0500 Subject: [PATCH 58/74] Special case for ExpandToEnclosingUnit on document --- platforms/windows/src/text.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index 70039a993..f274f411d 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -336,6 +336,17 @@ impl ITextRangeProvider_Impl for PlatformRange { } fn ExpandToEnclosingUnit(&self, unit: TextUnit) -> Result<()> { + if unit == TextUnit_Document { + // Handle document as a special case so we can get to a document + // range even if the current endpoints are now invalid. + // Based on observed behavior, Narrator needs this ability. + return self.with_tree_state(|tree_state| { + let mut state = self.state.write(); + let node = upgrade_range_node(&state, tree_state)?; + *state = node.document_range().downgrade(); + Ok(()) + }); + } self.write(|range| { let start = range.start(); if unit == TextUnit_Character && start.is_document_end() { From 76c98a818d52415abd37853670fe794414965ffe Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Fri, 28 Oct 2022 12:21:23 -0500 Subject: [PATCH 59/74] Clarify why character_pixel_lengths is optional --- common/src/lib.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/common/src/lib.rs b/common/src/lib.rs index e2a262185..73521c1d5 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -970,6 +970,11 @@ pub struct Node { /// be non-zero if selecting the line break by itself results in /// a visible highlight (as in Microsoft Word), or zero if not /// (as in Windows Notepad). + /// + /// This field is optional. Without it, AccessKit can't support some + /// use cases, such as screen magnifiers that track the caret position + /// or screen readers that display a highlight cursor. However, + /// most text functionality still works without this information. #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] pub character_pixel_lengths: Option>, From cc2acfecd9208c984084048d2bf00914bb2ed66c Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Fri, 28 Oct 2022 14:21:27 -0500 Subject: [PATCH 60/74] Implement GetBoundingRectangles --- consumer/src/text.rs | 86 ++++++++++++++++++++++++++++++++++- platforms/windows/src/node.rs | 8 +--- platforms/windows/src/text.rs | 19 ++++++-- platforms/windows/src/util.rs | 14 ++++++ 4 files changed, 117 insertions(+), 10 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 5b63a8fd5..396dccecc 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -3,7 +3,8 @@ // the LICENSE-APACHE file) or the MIT license (found in // the LICENSE-MIT file), at your option. -use accesskit::{NodeId, Role, TextPosition as WeakPosition}; +use accesskit::kurbo::Rect; +use accesskit::{NodeId, Role, TextDirection, TextPosition as WeakPosition}; use std::{cmp::Ordering, iter::FusedIterator}; use crate::{FilterResult, Node, TreeState}; @@ -457,6 +458,89 @@ impl<'a> Range<'a> { result } + /// Returns the range's transformed bounding boxes relative to the tree's + /// container (e.g. window). + /// + /// If the return value is empty, it means that the source tree doesn't + /// provide enough information to calculate bounding boxes. Otherwise, + /// there will always be at least one box, even if it's zero-width, + /// as it is for a degenerate range. + pub fn bounding_boxes(&self) -> Vec { + let mut result = Vec::new(); + self.walk(|node| { + let mut rect = match &node.data().bounds { + Some(rect) => *rect, + None => { + return Some(Vec::new()); + } + }; + let pixel_lengths = match &node.data().character_pixel_lengths { + Some(lengths) => lengths, + None => { + return Some(Vec::new()); + } + }; + let direction = match node.data().text_direction { + Some(direction) => direction, + None => { + return Some(Vec::new()); + } + }; + let character_lengths = &node.data().character_lengths; + let start_index = if node.id() == self.start.node.id() { + self.start.character_index + } else { + 0 + }; + let end_index = if node.id() == self.end.node.id() { + self.end.character_index + } else { + character_lengths.len() + }; + if start_index != 0 || end_index != character_lengths.len() { + let pixel_start = pixel_lengths[..start_index] + .iter() + .copied() + .map(f64::from) + .sum::(); + let pixel_end = pixel_start + + pixel_lengths[start_index..end_index] + .iter() + .copied() + .map(f64::from) + .sum::(); + match direction { + TextDirection::LeftToRight => { + let orig_left = rect.x0; + rect.x0 = orig_left + pixel_start; + rect.x1 = orig_left + pixel_end; + } + TextDirection::RightToLeft => { + let orig_right = rect.x1; + rect.x1 = orig_right - pixel_start; + rect.x0 = orig_right - pixel_end; + } + // Note: The following directions that the rectangle, + // before being transformed, is y-down. TBD: Will we + // ever encounter a case where this isn't true? + TextDirection::TopToBottom => { + let orig_top = rect.y0; + rect.y0 = orig_top + pixel_start; + rect.y1 = orig_top + pixel_end; + } + TextDirection::BottomToTop => { + let orig_bottom = rect.y1; + rect.y1 = orig_bottom - pixel_start; + rect.y0 = orig_bottom - pixel_end; + } + } + } + result.push(node.transform().transform_rect_bbox(rect)); + None + }) + .unwrap_or(result) + } + pub fn attribute(&self, f: F) -> AttributeValue where F: Fn(&Node) -> T, diff --git a/platforms/windows/src/node.rs b/platforms/windows/src/node.rs index ee3a901b1..42a184532 100644 --- a/platforms/windows/src/node.rs +++ b/platforms/windows/src/node.rs @@ -18,7 +18,7 @@ use paste::paste; use std::sync::{Arc, Weak}; use windows::{ core::*, - Win32::{Foundation::*, Graphics::Gdi::*, System::Com::*, UI::Accessibility::*}, + Win32::{Foundation::*, System::Com::*, UI::Accessibility::*}, }; use crate::{text::PlatformRange as PlatformTextRange, util::*}; @@ -563,11 +563,7 @@ impl PlatformNode { } fn client_top_left(&self) -> Point { - let mut result = POINT::default(); - // If ClientToScreen fails, that means the window is gone. - // That's an unexpected condition, so we should fail loudly. - unsafe { ClientToScreen(self.hwnd, &mut result) }.unwrap(); - Point::new(result.x.into(), result.y.into()) + client_top_left(self.hwnd) } } diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index f274f411d..63ab1edb8 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -407,9 +407,22 @@ impl ITextRangeProvider_Impl for PlatformRange { } fn GetBoundingRectangles(&self) -> Result<*mut SAFEARRAY> { - // TODO: necessary for screen magnifiers and other visual aids - // (e.g. Narrator highlight cursor) - Err(not_implemented()) + self.read(|range| { + let rects = range.bounding_boxes(); + if rects.is_empty() { + return Ok(std::ptr::null_mut()); + } + let client_top_left = client_top_left(self.hwnd); + let mut result = Vec::::new(); + result.reserve(rects.len() * 4); + for rect in rects { + result.push(rect.x0 + client_top_left.x); + result.push(rect.y0 + client_top_left.y); + result.push(rect.width()); + result.push(rect.height()); + } + Ok(safe_array_from_f64_slice(&result)) + }) } fn GetEnclosingElement(&self) -> Result { diff --git a/platforms/windows/src/util.rs b/platforms/windows/src/util.rs index a43bba7ba..717b7c58b 100644 --- a/platforms/windows/src/util.rs +++ b/platforms/windows/src/util.rs @@ -3,11 +3,13 @@ // the LICENSE-APACHE file) or the MIT license (found in // the LICENSE-MIT file), at your option. +use accesskit::kurbo::Point; use std::{convert::TryInto, mem::ManuallyDrop}; use windows::{ core::*, Win32::{ Foundation::*, + Graphics::Gdi::*, System::{Com::*, Ole::*}, UI::Accessibility::*, }, @@ -137,6 +139,10 @@ pub(crate) fn safe_array_from_i32_slice(slice: &[i32]) -> *mut SAFEARRAY { safe_array_from_primitive_slice(VT_I4, slice) } +pub(crate) fn safe_array_from_f64_slice(slice: &[f64]) -> *mut SAFEARRAY { + safe_array_from_primitive_slice(VT_R8, slice) +} + pub(crate) fn safe_array_from_com_slice(slice: &[IUnknown]) -> *mut SAFEARRAY { let sa = unsafe { SafeArrayCreateVector(VT_UNKNOWN, 0, slice.len().try_into().unwrap()) }; if sa.is_null() { @@ -181,3 +187,11 @@ pub(crate) fn element_not_available() -> Error { pub(crate) fn invalid_operation() -> Error { Error::new(HRESULT(UIA_E_INVALIDOPERATION as i32), "".into()) } + +pub(crate) fn client_top_left(hwnd: HWND) -> Point { + let mut result = POINT::default(); + // If ClientToScreen fails, that means the window is gone. + // That's an unexpected condition, so we should fail loudly. + unsafe { ClientToScreen(hwnd, &mut result) }.unwrap(); + Point::new(result.x.into(), result.y.into()) +} From ca30bfcc34cdc9fcdade604f2352290acd2b0e2f Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Mon, 31 Oct 2022 15:40:12 -0500 Subject: [PATCH 61/74] Don't require the last line to end with a newline. Fix the bug that caused problems with Narrator when the last line was a blank line with no trailing newline. General improvements to normalization/bias logic. --- common/src/lib.rs | 10 ------ consumer/src/text.rs | 83 +++++++++++++++++++++----------------------- 2 files changed, 40 insertions(+), 53 deletions(-) diff --git a/common/src/lib.rs b/common/src/lib.rs index 73521c1d5..e149600e6 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -947,16 +947,6 @@ pub struct Node { /// should be counted as a single character for the sake of this slice. /// When the caret is at the end of such a line, the focus of the text /// selection should be on the line break, not after it. - /// - /// The last inline text box in a text field or document must end - /// with a hard line break as described above. If the last line - /// is blank, the text box for that line must consist of only - /// a hard line break. If the text field is empty, it must consist - /// of a single inline text box containing only a hard line break. - /// This requirement applies even to text fields that only ever contain - /// one line of text. This consistency eliminates special cases - /// in text navigation which can lead to bugs with some assistive - /// technologies. #[cfg_attr(feature = "serde", serde(default))] #[cfg_attr(feature = "serde", serde(skip_serializing_if = "is_empty"))] pub character_lengths: Box<[u8]>, diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 396dccecc..2fe1b6ea0 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -80,7 +80,7 @@ impl<'a> InnerPosition<'a> { .is_none() } - fn normalize_to_start(&self, root_node: &Node) -> Self { + fn biased_to_start(&self, root_node: &Node) -> Self { if self.is_box_end() { if let Some(node) = self.node.following_inline_text_boxes(root_node).next() { return Self { @@ -92,7 +92,7 @@ impl<'a> InnerPosition<'a> { *self } - fn normalize_to_end(&self, root_node: &Node) -> Self { + fn biased_to_end(&self, root_node: &Node) -> Self { if self.is_box_start() { if let Some(node) = self.node.preceding_inline_text_boxes(root_node).next() { return Self { @@ -104,8 +104,16 @@ impl<'a> InnerPosition<'a> { *self } + fn normalized(&self, root_node: &Node) -> Self { + if self.is_line_end() && !self.is_paragraph_end() { + *self + } else { + self.biased_to_start(root_node) + } + } + fn comparable(&self, root_node: &Node) -> (Vec, usize) { - let normalized = self.normalize_to_start(root_node); + let normalized = self.biased_to_start(root_node); ( normalized.node.relative_index_path(root_node.id()), normalized.character_index, @@ -202,10 +210,7 @@ impl<'a> Position<'a> { pub fn is_paragraph_start(&self) -> bool { self.is_document_start() || (self.is_line_start() - && self - .inner - .normalize_to_end(&self.root_node) - .is_paragraph_end()) + && self.inner.biased_to_end(&self.root_node).is_paragraph_end()) } pub fn is_page_start(&self) -> bool { @@ -221,24 +226,26 @@ impl<'a> Position<'a> { } pub fn forward_by_character(&self) -> Self { - let normalized = self.inner.normalize_to_start(&self.root_node); + let pos = self.inner.biased_to_start(&self.root_node); Self { root_node: self.root_node, inner: InnerPosition { - node: normalized.node, - character_index: normalized.character_index + 1, - }, + node: pos.node, + character_index: pos.character_index + 1, + } + .normalized(&self.root_node), } } pub fn backward_by_character(&self) -> Self { - let normalized = self.inner.normalize_to_end(&self.root_node); + let pos = self.inner.biased_to_end(&self.root_node); Self { root_node: self.root_node, inner: InnerPosition { - node: normalized.node, - character_index: normalized.character_index - 1, - }, + node: pos.node, + character_index: pos.character_index - 1, + } + .normalized(&self.root_node), } } @@ -253,34 +260,34 @@ impl<'a> Position<'a> { } pub fn forward_by_word(&self) -> Self { - let normalized = self.inner.normalize_to_start(&self.root_node); + let pos = self.inner.biased_to_start(&self.root_node); Self { root_node: self.root_node, - inner: normalized.word_end(), + inner: pos.word_end().normalized(&self.root_node), } } pub fn backward_by_word(&self) -> Self { - let normalized = self.inner.normalize_to_end(&self.root_node); + let pos = self.inner.biased_to_end(&self.root_node); Self { root_node: self.root_node, - inner: normalized.previous_word_start(), + inner: pos.previous_word_start().normalized(&self.root_node), } } pub fn forward_by_line(&self) -> Self { - let normalized = self.inner.normalize_to_start(&self.root_node); + let pos = self.inner.biased_to_start(&self.root_node); Self { root_node: self.root_node, - inner: normalized.line_end(), + inner: pos.line_end().normalized(&self.root_node), } } pub fn backward_by_line(&self) -> Self { - let normalized = self.inner.normalize_to_end(&self.root_node); + let pos = self.inner.biased_to_end(&self.root_node); Self { root_node: self.root_node, - inner: normalized.line_start(), + inner: pos.line_start().normalized(&self.root_node), } } @@ -394,13 +401,13 @@ impl<'a> Range<'a> { where F: FnMut(&Node) -> Option, { - let start = self.start.normalize_to_start(&self.node); + let start = self.start.biased_to_start(&self.node); // For a degenerate range, the following avoids having `end` // come before `start`. let end = if self.is_degenerate() { start } else { - self.end.normalize_to_end(&self.node) + self.end.biased_to_end(&self.node) }; if let Some(result) = f(&start.node) { return Some(result); @@ -563,30 +570,20 @@ impl<'a> Range<'a> { pub fn set_start(&mut self, pos: Position<'a>) { assert_eq!(pos.root_node.id(), self.node.id()); - let pos = pos.inner; - self.start = if pos == self.end { - // Don't normalize when collapsing, as we want to preserve - // the start versus end distinction in that special case. - pos - } else { - pos.normalize_to_start(&self.node) - }; - if self.start.comparable(&self.node) > self.end.comparable(&self.node) { + self.start = pos.inner; + // We use `>=` here because if the two endpoints are equivalent + // but with a different bias, we want to normalize the bias. + if self.start.comparable(&self.node) >= self.end.comparable(&self.node) { self.end = self.start; } } pub fn set_end(&mut self, pos: Position<'a>) { assert_eq!(pos.root_node.id(), self.node.id()); - let pos = pos.inner; - self.end = if pos == self.start { - // Don't normalize when collapsing, as we want to preserve - // the start versus end distinction in that special case. - pos - } else { - pos.normalize_to_end(&self.node) - }; - if self.start.comparable(&self.node) > self.end.comparable(&self.node) { + self.end = pos.inner; + // We use `>=` here because if the two endpoints are equivalent + // but with a different bias, we want to normalize the bias. + if self.start.comparable(&self.node) >= self.end.comparable(&self.node) { self.start = self.end; } } From 0224d239c368fa396b9bf941e93215001a7036af Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Mon, 31 Oct 2022 16:40:54 -0500 Subject: [PATCH 62/74] typo --- common/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/lib.rs b/common/src/lib.rs index e149600e6..71c98e0af 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -975,7 +975,7 @@ pub struct Node { /// The end of each word is the beginning of the next word; there are no /// characters that are not considered part of a word. Trailing whitespace /// is typically considered part of the word that precedes it, while - /// a line's leading whitepsace is considered its own word. Whether + /// a line's leading whitespace is considered its own word. Whether /// punctuation is considered a separate word or part of the preceding /// word depends on the particular text editing implementation. /// Some editors may have their own definition of a word; for example, From f380650df108207276454341a1eba24aaae1e3e2 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Tue, 1 Nov 2022 10:38:08 -0500 Subject: [PATCH 63/74] Make sure text change events are fired before selection change events --- platforms/windows/src/adapter.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/platforms/windows/src/adapter.rs b/platforms/windows/src/adapter.rs index 8ad793646..93fd678dc 100644 --- a/platforms/windows/src/adapter.rs +++ b/platforms/windows/src/adapter.rs @@ -103,7 +103,7 @@ impl Adapter { text_changed: HashSet, } impl Handler<'_> { - fn enqueue_text_change_if_needed(&mut self, node: &Node) { + fn insert_text_change_if_needed(&mut self, node: &Node) { if node.role() != Role::InlineTextBox { return; } @@ -117,17 +117,23 @@ impl Adapter { } let platform_node = PlatformNode::new(self.tree, node.id(), self.hwnd); let element: IRawElementProviderSimple = platform_node.into(); - self.queue.push(QueuedEvent::Simple { - element, - event_id: UIA_Text_TextChangedEventId, - }); + // Text change events must come before selection change + // events. It doesn't matter if text change events come + // before other events. + self.queue.insert( + 0, + QueuedEvent::Simple { + element, + event_id: UIA_Text_TextChangedEventId, + }, + ); self.text_changed.insert(id); } } } impl TreeChangeHandler for Handler<'_> { fn node_added(&mut self, node: &Node) { - self.enqueue_text_change_if_needed(node); + self.insert_text_change_if_needed(node); if filter(node) != FilterResult::Include { return; } @@ -142,7 +148,7 @@ impl Adapter { } fn node_updated(&mut self, old_node: &Node, new_node: &Node) { if old_node.value() != new_node.value() { - self.enqueue_text_change_if_needed(new_node); + self.insert_text_change_if_needed(new_node); } if filter(new_node) != FilterResult::Include { return; @@ -175,7 +181,7 @@ impl Adapter { } } fn node_removed(&mut self, node: &Node) { - self.enqueue_text_change_if_needed(node); + self.insert_text_change_if_needed(node); } // TODO: handle other events (#20) } From 7ae20029bef0269080bd693aa353f64076729d45 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Tue, 1 Nov 2022 11:52:09 -0500 Subject: [PATCH 64/74] Allow endpoints of the same range to be compared even if the range is no longer valid --- consumer/src/text.rs | 18 +++++++++++++++++- platforms/windows/src/text.rs | 25 ++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 2fe1b6ea0..8fc1c619f 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -593,6 +593,8 @@ impl<'a> Range<'a> { node_id: self.node.id(), start: self.start.downgrade(), end: self.end.downgrade(), + start_comparable: self.start.comparable(&self.node), + end_comparable: self.end.comparable(&self.node), } } } @@ -605,14 +607,28 @@ impl<'a> PartialEq for Range<'a> { impl<'a> Eq for Range<'a> {} -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct WeakRange { node_id: NodeId, start: WeakPosition, end: WeakPosition, + start_comparable: (Vec, usize), + end_comparable: (Vec, usize), } impl WeakRange { + pub fn node_id(&self) -> NodeId { + self.node_id + } + + pub fn start_comparable(&self) -> &(Vec, usize) { + &self.start_comparable + } + + pub fn end_comparable(&self) -> &(Vec, usize) { + &self.end_comparable + } + pub fn upgrade_node<'a>(&self, tree_state: &'a TreeState) -> Option> { tree_state.node_by_id(self.node_id) } diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index 63ab1edb8..a0892019d 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -33,6 +33,17 @@ fn upgrade_range_node<'a>(weak: &WeakRange, tree_state: &'a TreeState) -> Result } } +fn weak_comparable_position_from_endpoint( + range: &WeakRange, + endpoint: TextPatternRangeEndpoint, +) -> Result<&(Vec, usize)> { + match endpoint { + TextPatternRangeEndpoint_Start => Ok(range.start_comparable()), + TextPatternRangeEndpoint_End => Ok(range.end_comparable()), + _ => Err(invalid_arg()), + } +} + fn position_from_endpoint<'a>( range: &Range<'a>, endpoint: TextPatternRangeEndpoint, @@ -293,7 +304,7 @@ impl Clone for PlatformRange { fn clone(&self) -> Self { PlatformRange { tree: self.tree.clone(), - state: RwLock::new(*self.state.read()), + state: RwLock::new(self.state.read().clone()), hwnd: self.hwnd, } } @@ -321,6 +332,18 @@ impl ITextRangeProvider_Impl for PlatformRange { other_endpoint: TextPatternRangeEndpoint, ) -> Result { let other = required_param(other)?.as_impl(); + if std::ptr::eq(other as *const _, self as *const _) { + // Comparing endpoints within the same range can be done + // safely without upgrading the range. This allows ATs + // to determine whether an old range is degenerate even if + // that range is no longer valid. + let state = self.state.read(); + let other_state = other.state.read(); + let pos = weak_comparable_position_from_endpoint(&*state, endpoint)?; + let other_pos = weak_comparable_position_from_endpoint(&*other_state, other_endpoint)?; + let result = pos.cmp(other_pos); + return Ok(result as i32); + } self.require_same_tree(other)?; self.with_tree_state(|tree_state| { let range = self.upgrade_for_read(tree_state)?; From 061e768fc6a260bf513a90eb2f4013396de09bc2 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Mon, 7 Nov 2022 10:41:12 -0600 Subject: [PATCH 65/74] Rename character_pixel_lengths and adjust its documentation --- common/src/lib.rs | 10 +++++----- consumer/src/text.rs | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/common/src/lib.rs b/common/src/lib.rs index 71c98e0af..bce4b7dea 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -950,13 +950,13 @@ pub struct Node { #[cfg_attr(feature = "serde", serde(default))] #[cfg_attr(feature = "serde", serde(skip_serializing_if = "is_empty"))] pub character_lengths: Box<[u8]>, - /// For inline text. This is the length of each character in pixels - /// within the bounding rectangle of this object, in the direction - /// given by [`Node::text_direction`]. + /// For inline text. This is the advance width of each character, + /// in the direction given by [`Node::text_direction`], in the coordinate + /// space of this node. /// /// When present, the length of this slice should be the same as the length /// of [`Node::character_lengths`], including for lines that end - /// with a hard line break. The pixel length of such a line break should + /// with a hard line break. The width of such a line break should /// be non-zero if selecting the line break by itself results in /// a visible highlight (as in Microsoft Word), or zero if not /// (as in Windows Notepad). @@ -966,7 +966,7 @@ pub struct Node { /// or screen readers that display a highlight cursor. However, /// most text functionality still works without this information. #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] - pub character_pixel_lengths: Option>, + pub character_widths: Option>, /// For inline text. The length of each word in characters, as defined /// in [`Node::character_lengths`]. The sum of these lengths must equal diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 8fc1c619f..f212d3e8e 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -481,7 +481,7 @@ impl<'a> Range<'a> { return Some(Vec::new()); } }; - let pixel_lengths = match &node.data().character_pixel_lengths { + let widths = match &node.data().character_widths { Some(lengths) => lengths, None => { return Some(Vec::new()); @@ -505,13 +505,13 @@ impl<'a> Range<'a> { character_lengths.len() }; if start_index != 0 || end_index != character_lengths.len() { - let pixel_start = pixel_lengths[..start_index] + let pixel_start = widths[..start_index] .iter() .copied() .map(f64::from) .sum::(); let pixel_end = pixel_start - + pixel_lengths[start_index..end_index] + + widths[start_index..end_index] .iter() .copied() .map(f64::from) From d5d6ae9b82c3e7f5f181bacdd30aeee27715785f Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Mon, 7 Nov 2022 10:42:30 -0600 Subject: [PATCH 66/74] Fix inconsistent internal naming --- consumer/src/text.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index f212d3e8e..32b751498 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -482,7 +482,7 @@ impl<'a> Range<'a> { } }; let widths = match &node.data().character_widths { - Some(lengths) => lengths, + Some(widths) => widths, None => { return Some(Vec::new()); } From da7f8fb994db0082d22c532118b0efef0b7cddac Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Mon, 7 Nov 2022 11:41:59 -0600 Subject: [PATCH 67/74] Change the way we calculate bounding rects of text ranges --- common/src/lib.rs | 15 +++++++++++++++ consumer/src/text.rs | 29 ++++++++++++++++++----------- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/common/src/lib.rs b/common/src/lib.rs index bce4b7dea..d7054b9c9 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -950,6 +950,21 @@ pub struct Node { #[cfg_attr(feature = "serde", serde(default))] #[cfg_attr(feature = "serde", serde(skip_serializing_if = "is_empty"))] pub character_lengths: Box<[u8]>, + /// For inline text. This is the position of each character within + /// the node's bounding box, in the direction given by + /// [`Node::text_direction`], in the coordinate space of this node. + /// + /// When present, the length of this slice should be the same as the length + /// of [`Node::character_lengths`], including for lines that end + /// with a hard line break. The position of such a line break should + /// be the position where an end-of-paragraph marker would be rendered. + /// + /// This field is optional. Without it, AccessKit can't support some + /// use cases, such as screen magnifiers that track the caret position + /// or screen readers that display a highlight cursor. However, + /// most text functionality still works without this information. + #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] + pub character_positions: Option>, /// For inline text. This is the advance width of each character, /// in the direction given by [`Node::text_direction`], in the coordinate /// space of this node. diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 32b751498..18578b424 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -481,6 +481,12 @@ impl<'a> Range<'a> { return Some(Vec::new()); } }; + let positions = match &node.data().character_positions { + Some(positions) => positions, + None => { + return Some(Vec::new()); + } + }; let widths = match &node.data().character_widths { Some(widths) => widths, None => { @@ -505,17 +511,18 @@ impl<'a> Range<'a> { character_lengths.len() }; if start_index != 0 || end_index != character_lengths.len() { - let pixel_start = widths[..start_index] - .iter() - .copied() - .map(f64::from) - .sum::(); - let pixel_end = pixel_start - + widths[start_index..end_index] - .iter() - .copied() - .map(f64::from) - .sum::(); + let pixel_start = if start_index < character_lengths.len() { + positions[start_index] + } else { + positions[start_index - 1] + widths[start_index - 1] + }; + let pixel_end = if end_index == start_index { + pixel_start + } else { + positions[end_index - 1] + widths[end_index - 1] + }; + let pixel_start = f64::from(pixel_start); + let pixel_end = f64::from(pixel_end); match direction { TextDirection::LeftToRight => { let orig_left = rect.x0; From 9e81cd921bee059d49aa685dc8cc81fe86d5ba80 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Mon, 7 Nov 2022 11:58:00 -0600 Subject: [PATCH 68/74] Support the UIA CaretPosition attribute --- consumer/src/text.rs | 4 ++++ platforms/windows/src/text.rs | 12 ++++++++++++ platforms/windows/src/util.rs | 6 ++++++ 3 files changed, 22 insertions(+) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 18578b424..66427e255 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -207,6 +207,10 @@ impl<'a> Position<'a> { self.inner.is_line_start() } + pub fn is_line_end(&self) -> bool { + self.inner.is_line_end() + } + pub fn is_paragraph_start(&self) -> bool { self.is_document_start() || (self.is_line_start() diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index a0892019d..ebc2e6bb5 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -421,6 +421,18 @@ impl ITextRangeProvider_Impl for PlatformRange { Ok(VariantFactory::from(value).into()) }) } + UIA_CaretPositionAttributeId => self.read(|range| { + let mut value = CaretPosition_Unknown; + if range.is_degenerate() { + let pos = range.start(); + if pos.is_line_start() { + value = CaretPosition_BeginningOfLine; + } else if pos.is_line_end() { + value = CaretPosition_EndOfLine; + } + } + Ok(VariantFactory::from(value).into()) + }), // TODO: implement more attributes _ => { let value = unsafe { UiaGetReservedNotSupportedValue() }.unwrap(); diff --git a/platforms/windows/src/util.rs b/platforms/windows/src/util.rs index 717b7c58b..0eba7dee2 100644 --- a/platforms/windows/src/util.rs +++ b/platforms/windows/src/util.rs @@ -102,6 +102,12 @@ impl From for VariantFactory { } } +impl From for VariantFactory { + fn from(value: CaretPosition) -> Self { + value.0.into() + } +} + const VARIANT_FALSE: i16 = 0i16; const VARIANT_TRUE: i16 = -1i16; From 6a6e5210577c1af08cfe797131c3029ca6ba6aac Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Mon, 7 Nov 2022 12:04:07 -0600 Subject: [PATCH 69/74] Fix new clippy warning --- platforms/windows/src/text.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platforms/windows/src/text.rs b/platforms/windows/src/text.rs index ebc2e6bb5..ec0f736d0 100644 --- a/platforms/windows/src/text.rs +++ b/platforms/windows/src/text.rs @@ -339,8 +339,8 @@ impl ITextRangeProvider_Impl for PlatformRange { // that range is no longer valid. let state = self.state.read(); let other_state = other.state.read(); - let pos = weak_comparable_position_from_endpoint(&*state, endpoint)?; - let other_pos = weak_comparable_position_from_endpoint(&*other_state, other_endpoint)?; + let pos = weak_comparable_position_from_endpoint(&state, endpoint)?; + let other_pos = weak_comparable_position_from_endpoint(&other_state, other_endpoint)?; let result = pos.cmp(other_pos); return Ok(result as i32); } From 3e04a646a8cf8aea33137c1b815cc3d1a494fd00 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Mon, 7 Nov 2022 12:10:45 -0600 Subject: [PATCH 70/74] Fix calculation of bounding rectangle for the caret at the end of a wrapped line --- consumer/src/text.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 66427e255..25da59b8a 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -405,13 +405,15 @@ impl<'a> Range<'a> { where F: FnMut(&Node) -> Option, { - let start = self.start.biased_to_start(&self.node); - // For a degenerate range, the following avoids having `end` - // come before `start`. - let end = if self.is_degenerate() { - start + // If the range is degenerate, we don't want to normalize it. + // This is important e.g. when getting the bounding rectangle + // of the caret range when the caret is at the end of a wrapped line. + let (start, end) = if self.is_degenerate() { + (self.start, self.start) } else { - self.end.biased_to_end(&self.node) + let start = self.start.biased_to_start(&self.node); + let end = self.end.biased_to_end(&self.node); + (start, end) }; if let Some(result) = f(&start.node) { return Some(result); From a9141e15fb6e0617e963d0cbd04d90a1f40124cd Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Mon, 7 Nov 2022 16:13:15 -0600 Subject: [PATCH 71/74] Add a realistic test tree for multi-line text and write a few basic tests --- consumer/src/text.rs | 264 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 264 insertions(+) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 25da59b8a..d9de5a00a 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -728,3 +728,267 @@ impl<'a> Node<'a> { }) } } + +#[cfg(test)] +mod tests { + use accesskit::{NodeId, TextSelection}; + use std::{num::NonZeroU128, sync::Arc}; + + use crate::tests::NullActionHandler; + + const NODE_ID_1: NodeId = NodeId(unsafe { NonZeroU128::new_unchecked(1) }); + const NODE_ID_2: NodeId = NodeId(unsafe { NonZeroU128::new_unchecked(2) }); + const NODE_ID_3: NodeId = NodeId(unsafe { NonZeroU128::new_unchecked(3) }); + const NODE_ID_4: NodeId = NodeId(unsafe { NonZeroU128::new_unchecked(4) }); + const NODE_ID_5: NodeId = NodeId(unsafe { NonZeroU128::new_unchecked(5) }); + const NODE_ID_6: NodeId = NodeId(unsafe { NonZeroU128::new_unchecked(6) }); + const NODE_ID_7: NodeId = NodeId(unsafe { NonZeroU128::new_unchecked(7) }); + const NODE_ID_8: NodeId = NodeId(unsafe { NonZeroU128::new_unchecked(8) }); + + // This is based on an actual tree produced by egui. + fn main_multiline_tree(selection: Option) -> crate::Tree { + use accesskit::kurbo::{Affine, Rect}; + use accesskit::{Node, Role, TextDirection, Tree, TreeUpdate}; + + let update = TreeUpdate { + nodes: vec![ + ( + NODE_ID_1, + Arc::new(Node { + role: Role::Window, + transform: Some(Box::new(Affine::scale(1.5))), + children: vec![NODE_ID_2], + ..Default::default() + }), + ), + ( + NODE_ID_2, + Arc::new(Node { + role: Role::TextField, + bounds: Some(Rect { + x0: 8.0, + y0: 31.666664123535156, + x1: 296.0, + y1: 123.66666412353516, + }), + children: vec![ + NODE_ID_3, NODE_ID_4, NODE_ID_5, NODE_ID_6, NODE_ID_7, NODE_ID_8, + ], + focusable: true, + text_selection: selection, + ..Default::default() + }), + ), + ( + NODE_ID_3, + Arc::new(Node { + role: Role::InlineTextBox, + bounds: Some(Rect { + x0: 12.0, + y0: 33.666664123535156, + x1: 290.9189147949219, + y1: 48.33333206176758, + }), + value: Some("This paragraph is long enough to wrap ".into()), + text_direction: Some(TextDirection::LeftToRight), + character_lengths: vec![ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + ] + .into(), + character_positions: Some( + vec![ + 0.0, 7.3333335, 14.666667, 22.0, 29.333334, 36.666668, 44.0, + 51.333332, 58.666668, 66.0, 73.333336, 80.666664, 88.0, 95.333336, + 102.666664, 110.0, 117.333336, 124.666664, 132.0, 139.33333, + 146.66667, 154.0, 161.33333, 168.66667, 176.0, 183.33333, + 190.66667, 198.0, 205.33333, 212.66667, 220.0, 227.33333, + 234.66667, 242.0, 249.33333, 256.66666, 264.0, 271.33334, + ] + .into(), + ), + character_widths: Some( + vec![ + 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, + 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, + 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, + 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, + 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, + 7.58557, 7.58557, 7.58557, + ] + .into(), + ), + word_lengths: vec![5, 10, 3, 5, 7, 3, 5].into(), + ..Default::default() + }), + ), + ( + NODE_ID_4, + Arc::new(Node { + role: Role::InlineTextBox, + bounds: Some(Rect { + x0: 12.0, + y0: 48.33333206176758, + x1: 129.5855712890625, + y1: 63.0, + }), + value: Some("to another line.\n".into()), + text_direction: Some(TextDirection::LeftToRight), + character_lengths: vec![1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + .into(), + character_positions: Some( + vec![ + 0.0, 7.3333435, 14.666687, 22.0, 29.333344, 36.666687, 44.0, + 51.333344, 58.666687, 66.0, 73.33334, 80.66669, 88.0, 95.33334, + 102.66669, 110.0, 117.58557, + ] + .into(), + ), + character_widths: Some( + vec![ + 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, + 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, + 7.58557, 7.58557, 0.0, + ] + .into(), + ), + word_lengths: vec![3, 8, 6].into(), + ..Default::default() + }), + ), + ( + NODE_ID_5, + Arc::new(Node { + role: Role::InlineTextBox, + bounds: Some(Rect { + x0: 12.0, + y0: 63.0, + x1: 144.25222778320313, + y1: 77.66666412353516, + }), + value: Some("Another paragraph.\n".into()), + text_direction: Some(TextDirection::LeftToRight), + character_lengths: vec![ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + ] + .into(), + character_positions: Some( + vec![ + 0.0, 7.3333335, 14.666667, 22.0, 29.333334, 36.666668, 44.0, + 51.333332, 58.666668, 66.0, 73.333336, 80.666664, 88.0, 95.333336, + 102.666664, 110.0, 117.333336, 124.666664, 132.25223, + ] + .into(), + ), + character_widths: Some( + vec![ + 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, + 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, + 7.58557, 7.58557, 7.58557, 7.58557, 0.0, + ] + .into(), + ), + word_lengths: vec![8, 11].into(), + ..Default::default() + }), + ), + ( + NODE_ID_6, + Arc::new(Node { + role: Role::InlineTextBox, + bounds: Some(Rect { + x0: 12.0, + y0: 77.66666412353516, + x1: 12.0, + y1: 92.33332824707031, + }), + value: Some("\n".into()), + text_direction: Some(TextDirection::LeftToRight), + character_lengths: vec![1].into(), + character_positions: Some(vec![0.0].into()), + character_widths: Some(vec![0.0].into()), + word_lengths: vec![1].into(), + ..Default::default() + }), + ), + ( + NODE_ID_7, + Arc::new(Node { + role: Role::InlineTextBox, + bounds: Some(Rect { + x0: 12.0, + y0: 92.33332824707031, + x1: 158.9188995361328, + y1: 107.0, + }), + value: Some("Last non-blank line.\n".into()), + text_direction: Some(TextDirection::LeftToRight), + character_lengths: vec![ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + ] + .into(), + character_positions: Some( + vec![ + 0.0, 7.3333335, 14.666667, 22.0, 29.333334, 36.666668, 44.0, + 51.333332, 58.666668, 66.0, 73.333336, 80.666664, 88.0, 95.333336, + 102.666664, 110.0, 117.333336, 124.666664, 132.0, 139.33333, + 146.9189, + ] + .into(), + ), + character_widths: Some( + vec![ + 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, + 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, + 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 7.58557, 0.0, + ] + .into(), + ), + word_lengths: vec![5, 4, 6, 6].into(), + ..Default::default() + }), + ), + ( + NODE_ID_8, + Arc::new(Node { + role: Role::InlineTextBox, + bounds: Some(Rect { + x0: 12.0, + y0: 107.0, + x1: 12.0, + y1: 121.66666412353516, + }), + value: Some("".into()), + text_direction: Some(TextDirection::LeftToRight), + character_lengths: vec![].into(), + character_positions: Some(vec![].into()), + character_widths: Some(vec![].into()), + word_lengths: vec![0].into(), + ..Default::default() + }), + ), + ], + tree: Some(Tree::new(NODE_ID_1)), + focus: Some(NODE_ID_2), + }; + + crate::Tree::new(update, Box::new(NullActionHandler {})) + } + + #[test] + fn supports_text_ranges() { + let tree = main_multiline_tree(None); + let state = tree.read(); + assert!(!state.node_by_id(NODE_ID_1).unwrap().supports_text_ranges()); + assert!(state.node_by_id(NODE_ID_2).unwrap().supports_text_ranges()); + } + + #[test] + fn multiline_document_range_text() { + let tree = main_multiline_tree(None); + let state = tree.read(); + let node = state.node_by_id(NODE_ID_2).unwrap(); + let range = node.document_range(); + assert_eq!(range.text(), "This paragraph is long enough to wrap to another line.\nAnother paragraph.\n\nLast non-blank line.\n"); + } +} From 3558a460b8d633a5c0a127c720c223b6fba52f24 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Mon, 7 Nov 2022 18:00:27 -0600 Subject: [PATCH 72/74] More tests; fix a bug in paragraph navigation --- consumer/src/text.rs | 283 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 280 insertions(+), 3 deletions(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index d9de5a00a..da3b84cff 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -299,7 +299,12 @@ impl<'a> Position<'a> { let mut current = *self; loop { current = current.forward_by_line(); - if current.is_document_end() || current.inner.is_paragraph_end() { + if current.is_document_end() + || current + .inner + .biased_to_end(&self.root_node) + .is_paragraph_end() + { break; } } @@ -731,6 +736,7 @@ impl<'a> Node<'a> { #[cfg(test)] mod tests { + use accesskit::kurbo::Rect; use accesskit::{NodeId, TextSelection}; use std::{num::NonZeroU128, sync::Arc}; @@ -747,7 +753,7 @@ mod tests { // This is based on an actual tree produced by egui. fn main_multiline_tree(selection: Option) -> crate::Tree { - use accesskit::kurbo::{Affine, Rect}; + use accesskit::kurbo::Affine; use accesskit::{Node, Role, TextDirection, Tree, TreeUpdate}; let update = TreeUpdate { @@ -975,6 +981,51 @@ mod tests { crate::Tree::new(update, Box::new(NullActionHandler {})) } + fn multiline_end_selection() -> TextSelection { + use accesskit::TextPosition; + + TextSelection { + anchor: TextPosition { + node: NODE_ID_8, + character_index: 0, + }, + focus: TextPosition { + node: NODE_ID_8, + character_index: 0, + }, + } + } + + fn multiline_wrapped_line_end_selection() -> TextSelection { + use accesskit::TextPosition; + + TextSelection { + anchor: TextPosition { + node: NODE_ID_3, + character_index: 38, + }, + focus: TextPosition { + node: NODE_ID_3, + character_index: 38, + }, + } + } + + fn multiline_second_line_middle_selection() -> TextSelection { + use accesskit::TextPosition; + + TextSelection { + anchor: TextPosition { + node: NODE_ID_4, + character_index: 5, + }, + focus: TextPosition { + node: NODE_ID_4, + character_index: 5, + }, + } + } + #[test] fn supports_text_ranges() { let tree = main_multiline_tree(None); @@ -984,11 +1035,237 @@ mod tests { } #[test] - fn multiline_document_range_text() { + fn multiline_document_range() { let tree = main_multiline_tree(None); let state = tree.read(); let node = state.node_by_id(NODE_ID_2).unwrap(); let range = node.document_range(); + let start = range.start(); + assert!(start.is_word_start()); + assert!(start.is_line_start()); + assert!(!start.is_line_end()); + assert!(start.is_paragraph_start()); + assert!(start.is_document_start()); + assert!(!start.is_document_end()); + let end = range.end(); + assert!(start < end); + assert!(end.is_word_start()); + assert!(end.is_line_start()); + assert!(end.is_line_end()); + assert!(end.is_paragraph_start()); + assert!(!end.is_document_start()); + assert!(end.is_document_end()); assert_eq!(range.text(), "This paragraph is long enough to wrap to another line.\nAnother paragraph.\n\nLast non-blank line.\n"); + assert_eq!( + range.bounding_boxes(), + vec![ + Rect { + x0: 18.0, + y0: 50.499996185302734, + x1: 436.3783721923828, + y1: 72.49999809265137 + }, + Rect { + x0: 18.0, + y0: 72.49999809265137, + x1: 194.37835693359375, + y1: 94.5 + }, + Rect { + x0: 18.0, + y0: 94.5, + x1: 216.3783416748047, + y1: 116.49999618530273 + }, + Rect { + x0: 18.0, + y0: 116.49999618530273, + x1: 18.0, + y1: 138.49999237060547 + }, + Rect { + x0: 18.0, + y0: 138.49999237060547, + x1: 238.37834930419922, + y1: 160.5 + } + ] + ); + } + + #[test] + fn multiline_end_degenerate_range() { + let tree = main_multiline_tree(Some(multiline_end_selection())); + let state = tree.read(); + let node = state.node_by_id(NODE_ID_2).unwrap(); + let range = node.text_selection().unwrap(); + assert!(range.is_degenerate()); + let pos = range.start(); + assert!(pos.is_word_start()); + assert!(pos.is_line_start()); + assert!(pos.is_line_end()); + assert!(pos.is_paragraph_start()); + assert!(!pos.is_document_start()); + assert!(pos.is_document_end()); + assert_eq!(range.text(), ""); + assert_eq!( + range.bounding_boxes(), + vec![Rect { + x0: 18.0, + y0: 160.5, + x1: 18.0, + y1: 182.49999618530273, + }] + ); + } + + #[test] + fn multiline_wrapped_line_end_range() { + let tree = main_multiline_tree(Some(multiline_wrapped_line_end_selection())); + let state = tree.read(); + let node = state.node_by_id(NODE_ID_2).unwrap(); + let range = node.text_selection().unwrap(); + assert!(range.is_degenerate()); + let pos = range.start(); + assert!(!pos.is_word_start()); + assert!(!pos.is_line_start()); + assert!(pos.is_line_end()); + assert!(!pos.is_paragraph_start()); + assert!(!pos.is_document_start()); + assert!(!pos.is_document_end()); + assert_eq!(range.text(), ""); + assert_eq!( + range.bounding_boxes(), + vec![Rect { + x0: 436.3783721923828, + y0: 50.499996185302734, + x1: 436.3783721923828, + y1: 72.49999809265137 + }] + ); + let next_char_pos = pos.forward_by_character(); + let mut line_start_range = range; + line_start_range.set_end(next_char_pos); + assert!(!line_start_range.is_degenerate()); + assert_eq!(line_start_range.text(), "t"); + assert_eq!( + line_start_range.bounding_boxes(), + vec![Rect { + x0: 18.0, + y0: 72.49999809265137, + x1: 29.378354787826538, + y1: 94.5 + }] + ); + let prev_char_pos = pos.backward_by_character(); + let mut prev_char_range = range; + prev_char_range.set_start(prev_char_pos); + assert!(!prev_char_range.is_degenerate()); + assert_eq!(prev_char_range.text(), " "); + assert_eq!( + prev_char_range.bounding_boxes(), + vec![Rect { + x0: 425.00001525878906, + y0: 50.499996185302734, + x1: 436.3783721923828, + y1: 72.49999809265137 + }] + ); + } + + #[test] + fn multiline_find_line_ends_from_middle() { + let tree = main_multiline_tree(Some(multiline_second_line_middle_selection())); + let state = tree.read(); + let node = state.node_by_id(NODE_ID_2).unwrap(); + let mut range = node.text_selection().unwrap(); + assert!(range.is_degenerate()); + let pos = range.start(); + assert!(!pos.is_line_start()); + assert!(!pos.is_line_end()); + assert!(!pos.is_document_start()); + assert!(!pos.is_document_end()); + let line_start = pos.backward_by_line(); + range.set_start(line_start); + let line_end = line_start.forward_by_line(); + range.set_end(line_end); + assert!(!range.is_degenerate()); + assert_eq!(range.text(), "to another line.\n"); + assert_eq!( + range.bounding_boxes(), + vec![Rect { + x0: 18.0, + y0: 72.49999809265137, + x1: 194.37835693359375, + y1: 94.5 + },] + ); + } + + #[test] + fn multiline_find_paragraph_ends_from_middle() { + let tree = main_multiline_tree(Some(multiline_second_line_middle_selection())); + let state = tree.read(); + let node = state.node_by_id(NODE_ID_2).unwrap(); + let mut range = node.text_selection().unwrap(); + assert!(range.is_degenerate()); + let pos = range.start(); + assert!(!pos.is_paragraph_start()); + assert!(!pos.is_document_start()); + assert!(!pos.is_document_end()); + let paragraph_start = pos.backward_by_paragraph(); + range.set_start(paragraph_start); + let paragraph_end = paragraph_start.forward_by_paragraph(); + range.set_end(paragraph_end); + assert!(!range.is_degenerate()); + assert_eq!( + range.text(), + "This paragraph is long enough to wrap to another line.\n" + ); + assert_eq!( + range.bounding_boxes(), + vec![ + Rect { + x0: 18.0, + y0: 50.499996185302734, + x1: 436.3783721923828, + y1: 72.49999809265137 + }, + Rect { + x0: 18.0, + y0: 72.49999809265137, + x1: 194.37835693359375, + y1: 94.5 + }, + ] + ); + } + + #[test] + fn multiline_find_word_ends_from_middle() { + let tree = main_multiline_tree(Some(multiline_second_line_middle_selection())); + let state = tree.read(); + let node = state.node_by_id(NODE_ID_2).unwrap(); + let mut range = node.text_selection().unwrap(); + assert!(range.is_degenerate()); + let pos = range.start(); + assert!(!pos.is_word_start()); + assert!(!pos.is_document_start()); + assert!(!pos.is_document_end()); + let word_start = pos.backward_by_word(); + range.set_start(word_start); + let word_end = word_start.forward_by_word(); + range.set_end(word_end); + assert!(!range.is_degenerate()); + assert_eq!(range.text(), "another "); + assert_eq!( + range.bounding_boxes(), + vec![Rect { + x0: 51.0, + y0: 72.49999809265137, + x1: 139.3783721923828, + y1: 94.5 + }] + ); } } From dc8eb6a5add366de0c51365007ba6c8fca181042 Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Fri, 11 Nov 2022 13:55:02 -0600 Subject: [PATCH 73/74] Implement hit-testing within text --- consumer/src/node.rs | 42 +++++-- consumer/src/text.rs | 215 +++++++++++++++++++++++++++++++++- platforms/windows/src/node.rs | 12 +- 3 files changed, 255 insertions(+), 14 deletions(-) diff --git a/consumer/src/node.rs b/consumer/src/node.rs index c65ec4625..977d30aae 100644 --- a/consumer/src/node.rs +++ b/consumer/src/node.rs @@ -226,6 +226,19 @@ impl<'a> Node<'a> { * self.direct_transform() } + pub(crate) fn relative_transform(&self, stop_at: &Node) -> Affine { + let parent_transform = if let Some(parent) = self.parent() { + if parent.id() == stop_at.id() { + Affine::IDENTITY + } else { + parent.relative_transform(stop_at) + } + } else { + Affine::IDENTITY + }; + parent_transform * self.direct_transform() + } + /// Returns the node's transformed bounding box relative to the tree's /// container (e.g. window). pub fn bounding_box(&self) -> Option { @@ -235,13 +248,18 @@ impl<'a> Node<'a> { .map(|rect| self.transform().transform_rect_bbox(*rect)) } - /// Returns the deepest filtered node, either this node or a descendant, - /// at the given point in this node's coordinate space. - pub fn node_at_point( + pub(crate) fn bounding_box_in_coordinate_space(&self, other: &Node) -> Option { + self.data() + .bounds + .as_ref() + .map(|rect| self.relative_transform(other).transform_rect_bbox(*rect)) + } + + pub(crate) fn hit_test( &self, point: Point, filter: &impl Fn(&Node) -> FilterResult, - ) -> Option> { + ) -> Option<(Node<'a>, Point)> { let filter_result = filter(self); if filter_result == FilterResult::ExcludeSubtree { @@ -250,15 +268,15 @@ impl<'a> Node<'a> { for child in self.children().rev() { let point = child.direct_transform().inverse() * point; - if let Some(node) = child.node_at_point(point, filter) { - return Some(node); + if let Some(result) = child.hit_test(point, filter) { + return Some(result); } } if filter_result == FilterResult::Include { if let Some(rect) = &self.data().bounds { if rect.contains(point) { - return Some(*self); + return Some((*self, point)); } } } @@ -266,6 +284,16 @@ impl<'a> Node<'a> { None } + /// Returns the deepest filtered node, either this node or a descendant, + /// at the given point in this node's coordinate space. + pub fn node_at_point( + &self, + point: Point, + filter: &impl Fn(&Node) -> FilterResult, + ) -> Option> { + self.hit_test(point, filter).map(|(node, _)| node) + } + pub fn id(&self) -> NodeId { self.state.id } diff --git a/consumer/src/text.rs b/consumer/src/text.rs index da3b84cff..f2668e772 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -3,7 +3,7 @@ // the LICENSE-APACHE file) or the MIT license (found in // the LICENSE-MIT file), at your option. -use accesskit::kurbo::Rect; +use accesskit::kurbo::{Point, Rect}; use accesskit::{NodeId, Role, TextDirection, TextPosition as WeakPosition}; use std::{cmp::Ordering, iter::FusedIterator}; @@ -229,6 +229,10 @@ impl<'a> Position<'a> { self.inner.is_document_end(&self.root_node) } + pub fn to_degenerate_range(&self) -> Range { + Range::new(self.root_node, self.inner, self.inner) + } + pub fn forward_by_character(&self) -> Self { let pos = self.inner.biased_to_start(&self.root_node); Self { @@ -545,8 +549,8 @@ impl<'a> Range<'a> { rect.x1 = orig_right - pixel_start; rect.x0 = orig_right - pixel_end; } - // Note: The following directions that the rectangle, - // before being transformed, is y-down. TBD: Will we + // Note: The following directions assume that the rectangle, + // in the node's coordinate space, is y-down. TBD: Will we // ever encounter a case where this isn't true? TextDirection::TopToBottom => { let orig_top = rect.y0; @@ -667,6 +671,46 @@ fn text_node_filter(root_id: NodeId, node: &Node) -> FilterResult { } } +fn character_index_at_point(node: &Node, point: Point) -> usize { + // We know the node has a bounding rectangle because it was returned + // by a hit test. + let rect = node.data().bounds.as_ref().unwrap(); + let character_lengths = &node.data().character_lengths; + let positions = match &node.data().character_positions { + Some(positions) => positions, + None => { + return 0; + } + }; + let widths = match &node.data().character_widths { + Some(widths) => widths, + None => { + return 0; + } + }; + let direction = match node.data().text_direction { + Some(direction) => direction, + None => { + return 0; + } + }; + for (i, (position, width)) in positions.iter().zip(widths.iter()).enumerate().rev() { + let relative_pos = match direction { + TextDirection::LeftToRight => point.x - rect.x0, + TextDirection::RightToLeft => rect.x1 - point.x, + // Note: The following directions assume that the rectangle, + // in the node's coordinate space, is y-down. TBD: Will we + // ever encounter a case where this isn't true? + TextDirection::TopToBottom => point.y - rect.y0, + TextDirection::BottomToTop => rect.y1 - point.y, + }; + if relative_pos >= f64::from(*position) && relative_pos < f64::from(*position + *width) { + return i; + } + } + character_lengths.len() +} + impl<'a> Node<'a> { fn inline_text_boxes( &self, @@ -732,11 +776,82 @@ impl<'a> Node<'a> { Range::new(*self, anchor, focus) }) } + + /// Returns the nearest text position to the given point + /// in this node's coordinate space. + pub fn text_position_at_point(&self, point: Point) -> Position { + let id = self.id(); + if let Some((node, point)) = self.hit_test(point, &move |node| text_node_filter(id, node)) { + if node.role() == Role::InlineTextBox { + let pos = InnerPosition { + node, + character_index: character_index_at_point(&node, point), + }; + return Position { + root_node: *self, + inner: pos, + }; + } + } + + // The following tests can assume that the point is not within + // any inline text box. + + if let Some(node) = self.inline_text_boxes().next() { + if let Some(rect) = node.bounding_box_in_coordinate_space(self) { + let origin = rect.origin(); + if point.x < origin.x || point.y < origin.y { + return Position { + root_node: *self, + inner: self.document_start(), + }; + } + } + } + + for node in self.inline_text_boxes() { + if let Some(rect) = node.bounding_box_in_coordinate_space(self) { + if let Some(direction) = node.data().text_direction { + let is_past_end = match direction { + TextDirection::LeftToRight => { + point.y >= rect.y0 && point.y < rect.y1 && point.x >= rect.x1 + } + TextDirection::RightToLeft => { + point.y >= rect.y0 && point.y < rect.y1 && point.x < rect.x0 + } + // Note: The following directions assume that the rectangle, + // in the root node's coordinate space, is y-down. TBD: Will we + // ever encounter a case where this isn't true? + TextDirection::TopToBottom => { + point.x >= rect.x0 && point.x < rect.x1 && point.y >= rect.y1 + } + TextDirection::BottomToTop => { + point.x >= rect.x0 && point.x < rect.x1 && point.y < rect.y0 + } + }; + if is_past_end { + return Position { + root_node: *self, + inner: InnerPosition { + node, + character_index: node.data().character_lengths.len(), + }, + }; + } + } + } + } + + Position { + root_node: *self, + inner: self.document_end(), + } + } } #[cfg(test)] mod tests { - use accesskit::kurbo::Rect; + use accesskit::kurbo::{Point, Rect}; use accesskit::{NodeId, TextSelection}; use std::{num::NonZeroU128, sync::Arc}; @@ -1268,4 +1383,96 @@ mod tests { }] ); } + + #[test] + fn text_position_at_point() { + let tree = main_multiline_tree(None); + let state = tree.read(); + let node = state.node_by_id(NODE_ID_2).unwrap(); + + { + let pos = node.text_position_at_point(Point::new(8.0, 31.666664123535156)); + assert!(pos.is_document_start()); + } + + { + let pos = node.text_position_at_point(Point::new(12.0, 33.666664123535156)); + assert!(pos.is_document_start()); + } + + { + let pos = node.text_position_at_point(Point::new(16.0, 40.0)); + assert!(pos.is_document_start()); + } + + { + let pos = node.text_position_at_point(Point::new(144.0, 40.0)); + assert!(!pos.is_document_start()); + assert!(!pos.is_document_end()); + assert!(!pos.is_line_end()); + let mut range = pos.to_degenerate_range(); + range.set_end(pos.forward_by_character()); + assert_eq!(range.text(), "l"); + } + + { + let pos = node.text_position_at_point(Point::new(150.0, 40.0)); + assert!(!pos.is_document_start()); + assert!(!pos.is_document_end()); + assert!(!pos.is_line_end()); + let mut range = pos.to_degenerate_range(); + range.set_end(pos.forward_by_character()); + assert_eq!(range.text(), "l"); + } + + { + let pos = node.text_position_at_point(Point::new(291.0, 40.0)); + assert!(!pos.is_document_start()); + assert!(!pos.is_document_end()); + assert!(pos.is_line_end()); + let mut range = pos.to_degenerate_range(); + range.set_start(pos.backward_by_word()); + assert_eq!(range.text(), "wrap "); + } + + { + let pos = node.text_position_at_point(Point::new(12.0, 50.0)); + assert!(!pos.is_document_start()); + assert!(pos.is_line_start()); + assert!(!pos.is_paragraph_start()); + let mut range = pos.to_degenerate_range(); + range.set_end(pos.forward_by_word()); + assert_eq!(range.text(), "to "); + } + + { + let pos = node.text_position_at_point(Point::new(130.0, 50.0)); + assert!(!pos.is_document_start()); + assert!(!pos.is_document_end()); + assert!(pos.is_line_end()); + let mut range = pos.to_degenerate_range(); + range.set_start(pos.backward_by_word()); + assert_eq!(range.text(), "line.\n"); + } + + { + let pos = node.text_position_at_point(Point::new(12.0, 80.0)); + assert!(!pos.is_document_start()); + assert!(!pos.is_document_end()); + assert!(pos.is_line_end()); + let mut range = pos.to_degenerate_range(); + range.set_start(pos.backward_by_line()); + assert_eq!(range.text(), "\n"); + } + + { + let pos = node.text_position_at_point(Point::new(12.0, 120.0)); + assert!(pos.is_document_end()); + } + + { + let pos = node.text_position_at_point(Point::new(250.0, 122.0)); + assert!(pos.is_document_end()); + } + } } diff --git a/platforms/windows/src/node.rs b/platforms/windows/src/node.rs index 42a184532..56265e2c5 100644 --- a/platforms/windows/src/node.rs +++ b/platforms/windows/src/node.rs @@ -871,9 +871,15 @@ patterns! { Err(not_implemented()) }, - fn RangeFromPoint(&self, _point: &UiaPoint) -> Result { - // TODO: hit testing for text - Err(not_implemented()) + fn RangeFromPoint(&self, point: &UiaPoint) -> Result { + self.resolve(|wrapper| { + let client_top_left = self.client_top_left(); + let point = Point::new(point.x - client_top_left.x, point.y - client_top_left.y); + let point = wrapper.node.transform().inverse() * point; + let pos = wrapper.node.text_position_at_point(point); + let range = pos.to_degenerate_range(); + Ok(PlatformTextRange::new(&self.tree, range, self.hwnd).into()) + }) }, fn DocumentRange(&self) -> Result { From 0cd50bc2d73dacdb1f9fa853bdcdc3a65d98cd0e Mon Sep 17 00:00:00 2001 From: Matt Campbell Date: Fri, 11 Nov 2022 14:19:14 -0600 Subject: [PATCH 74/74] Traverse text boxes in reverse for the 'past end' check. Will be needed when we have lines made up of multiple text boxes. --- consumer/src/text.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consumer/src/text.rs b/consumer/src/text.rs index f2668e772..e6f6ba758 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -809,7 +809,7 @@ impl<'a> Node<'a> { } } - for node in self.inline_text_boxes() { + for node in self.inline_text_boxes().rev() { if let Some(rect) = node.bounding_box_in_coordinate_space(self) { if let Some(direction) = node.data().text_direction { let is_past_end = match direction {