accessibility dictation: first round

dictation/app_overrides/messages.py

@@ -0,0 +1,20 @@
+from talon import Context, Module
+
+ctx = Context()
+ctx.matches = """
+os: mac
+app: messages
+"""
+mod = Module()
+
+@ctx.action_class("self")
+class Actions:
+
+    def accessibility_adjust_context_for_application(el, context):
+        # Messages reports an empty buffer as having None as content instead of "".
+        # We use None as a signal for "accessibility not available", so make sure it is reported as "".
+        if context.content is None:
+            context.content = ""
+
+        return context
+

dictation/debugging.py

@@ -0,0 +1,94 @@
+import time
+import traceback
+
+from talon import Module, actions, cron, noise, ui
+from talon.mac.ui import Element
+
+HISS_DEBUG_ENABLED = True
+
+mod = Module()
+setting_enabled = mod.setting(
+    "hiss_to_debug_accessibility",
+    type=bool,
+    default=False,
+    desc="Use a hissing sound to print accessibility debugging information to the Talon log.",
+)
+setting_threshold = mod.setting(
+    "hiss_to_debug_accessibility_threshold",
+    type=float,
+    default=0.35,
+    desc="If hiss_to_debug_accessibility is enabled, the hissing duration (in seconds) needed to trigger the debug output.",
+)
+
+
+@mod.action_class
+class Actions:
+
+    def debug_accessibility(el: Element = None):
+        """Prints information about the currently focused UI element to the terminal, for debugging"""
+
+        if not el:
+            el = ui.focused_element()
+
+        try:
+            # TODO(pcohen): make this work without Rich installed
+            from rich.console import Console
+            console = Console(color_system="truecolor", soft_wrap=True)
+
+            console.rule(f"{str(el)}'s attributes:")
+
+            # Attempt to sort the keys by relying on insertion order.
+            attributed = {}
+            for k in sorted(el.attrs):
+                attributed[k] = el.get(k)
+
+            console.print(attributed, markup=False)
+        except Exception as e:
+            print(f"Exception while debugging accessibility: \"{e}\":")
+            traceback.print_exc()
+
+
+active_hiss = {"cron": None}
+
+
+def hiss_over_threshold():
+    if not active_hiss.get("start"):
+        return False
+
+    return time.time() - active_hiss["start"] > setting_threshold.get()
+
+
+def stop_hiss():
+    trigger = hiss_over_threshold()
+
+    if active_hiss["cron"]:
+        cron.cancel(active_hiss["cron"])
+        active_hiss["cron"] = None
+
+    active_hiss["start"] = None
+
+    if trigger:
+        actions.user.debug_accessibility()
+
+
+def check_hiss():
+    if hiss_over_threshold():
+        stop_hiss()
+
+
+def start_hiss():
+    active_hiss["start"] = time.time()
+    active_hiss["cron"] = cron.interval("32ms", check_hiss)
+
+
+def on_hiss(noise_active: bool):
+    if not setting_enabled.get():
+        return
+
+    if noise_active:
+        start_hiss()
+    else:
+        stop_hiss()
+
+
+noise.register('hiss', on_hiss)

dictation/dictation_context.py

@@ -0,0 +1,140 @@
+import traceback
+from dataclasses import dataclass
+from enum import Enum
+from typing import Optional
+
+from talon import Context, Module, actions, ui
+from talon.mac.ui import Element
+from talon.types import Span
+
+ctx = Context()
+ctx.matches = "os: mac"
+
+mod = Module()
+setting_accessibility_dictation = mod.setting(
+    "accessibility_dictation",
+    type=bool,
+    default=False,
+    desc="Use accessibility APIs to implement context aware dictation.",
+)
+
+# Default number of characters to use to acquire context. Somewhat arbitrary.
+# The current dictation formatter doesn't need very many, but that could change in the future.
+DEFAULT_CONTEXT_CHARACTERS = 30
+
+
+@dataclass
+class AccessibilityContext:
+    """Records the context needed for dictation"""
+    content: str
+    selection: Span
+
+    def left_context(self, num_chars: int = DEFAULT_CONTEXT_CHARACTERS) -> str:
+        """Returns `num_chars`' worth of context to the left of the cursor"""
+        start = max(0, self.selection.left - num_chars)
+        return self.content[start:self.selection.left]
+
+    def right_context(self, num_chars: int = DEFAULT_CONTEXT_CHARACTERS) -> str:
+        """Returns `num_chars`' worth of context to the right of the cursor"""
+        end = min(self.selection.right + num_chars, len(self.content))
+        return self.content[self.selection.right:end]
+
+
+@mod.action_class
+class ModActions:
+
+    def accessibility_dictation_enabled() -> bool:
+        """Returns whether accessibility dictation should be used"""
+        # NB: for access within other files, since they can't import `setting_accessibility_dictation`
+        return setting_accessibility_dictation.get()
+
+    def dictation_current_element() -> Element:
+        """Returns the accessibility element that should be used for dictation (i.e. the current input textbox).
+
+        This is almost always the focused (current) element, however, this action
+        exists so that context can overwrite it, for applications with strange behavior.
+        """
+        return ui.focused_element()
+
+    def accessibility_adjust_context_for_application(el: Element,
+                                                     context: AccessibilityContext) -> AccessibilityContext:
+        """Hook for applications to override the reported buffer contents/cursor location.
+
+        Sometimes the accessibility context reported by the application is wrong, but fixable in predictable ways (this is most common in Electron apps). This method can be overwritten in those applications to do so.
+
+        TODO(pcohen): it's a it strange to have both this and dictation_current_element;
+        possibly refactor.
+        """
+        return context
+
+    def accessibility_create_dictation_context(el: Element) -> Optional[AccessibilityContext]:
+        """Creates a `AccessibilityContext` representing the state of the input buffer for dictation mode
+        """
+        if not actions.user.accessibility_dictation_enabled():
+            return None
+
+        if not el or not el.attrs:
+            # No accessibility support.
+            return None
+
+        # NOTE(pcohen): In Microsoft apps (Word, OneNote), selection will be none when the cursor
+        # is that the start of the input buffer.
+        # TODO(pcohen): this should probably be an app-specific `accessibility_adjust_context_for_application`
+        selection = el.get("AXSelectedTextRange")
+        if selection is None:
+            selection = Span(0, 0)
+
+        context = AccessibilityContext(content=el.get("AXValue"), selection=selection)
+
+        # Support application-specific overrides:
+        context = actions.user.accessibility_adjust_context_for_application(el, context)
+
+        # If we don't appear to have any accessibility information, don't use it.
+        if context.content is None or context.selection is None:
+            return None
+
+        return context
+
+
+# TODO(pcohen): relocate this
+class Colors(Enum):
+    RESET = '\033[0m'
+    RED = '\033[31m'
+    YELLOW = '\033[33m'
+
+@ctx.action_class("self")
+class Actions:
+    """Wires this into the knausj dictation formatter"""
+
+    def dictation_peek_left(clobber=False):
+        try:
+            el = actions.user.dictation_current_element()
+            context = actions.user.accessibility_create_dictation_context(el)
+            if context is None:
+                print(f"{Colors.YELLOW.value}Accessibility not available for context-aware dictation{Colors.RESET.value}; falling back to cursor method")
+                return actions.next()
+
+            return context.left_context()
+        except Exception as e:
+            print(f"{Colors.RED.value}{type(e).__name__} while querying accessibility for context-aware dictation:{Colors.RESET.value} '{e}':")
+            traceback.print_exc()
+
+            # Fallback to the original (keystrokes) knausj method.
+            actions.next()
+
+    def dictation_peek_right():
+        try:
+            el = actions.user.dictation_current_element()
+            context = actions.user.accessibility_create_dictation_context(el)
+            if context is None:
+                print(
+                    f"{Colors.YELLOW.value}Accessibility not available for context-aware dictation{Colors.RESET.value}; falling back to cursor method")
+                return actions.next()
+
+            return context.right_context()
+        except Exception as e:
+            print(f"{Colors.RED.value}{type(e).__name__} while querying accessibility for context-aware dictation:{Colors.RESET.value} '{e}':")
+            traceback.print_exc()
+
+            # Fallback to the original (keystrokes) knausj method.
+            actions.next()

dictation/electron.py

@@ -0,0 +1,36 @@
+from typing import Optional
+
+from talon import Context, Module, actions, ui
+from talon.mac.ui import App
+from talon.ui import UIErr
+
+ctx = Context()
+ctx.matches = "os: mac"
+
+mod = Module()
+setting_electron_accessibility = mod.setting(
+    "enable_electron_accessibility",
+    type=bool,
+    default=False,
+    desc="Tells Electron apps to enable their accessibility trees, so that you can use accessibility dictation with them. Note that this could cause worse performance, depending on the app.",
+)
+
+@mod.action_class
+class ModActions:
+    def enable_electron_accessibility(app: Optional[App] = None):
+        """Enables AX support in Electron - may affect performance"""
+        if not app:
+            app = ui.active_app()
+
+        try:
+            app.element.AXManualAccessibility = True
+        except UIErr as e:
+            # this will raise "Error setting element attribute" even on success.
+            pass
+
+
+def app_activate(app):
+    if setting_electron_accessibility.get():
+        actions.user.enable_electron_accessibility(app)
+
+ui.register("app_activate", app_activate)