[WIP] Pass keyboard events to AT-SPI

platforms/unix/Cargo.toml

@@ -13,7 +13,7 @@ edition = "2021"
 [dependencies]
 accesskit = { version = "0.8.1", path = "../../common" }
 accesskit_consumer = { version = "0.11.0", path = "../../consumer" }
-atspi = "0.6.0"
+atspi = "0.7.8"
 futures = "0.3"
 parking_lot = "0.12.1"
 serde = "1.0"

platforms/unix/src/adapter.rs

@@ -13,22 +13,30 @@ use crate::{
     },
     node::{filter, filter_detached, NodeWrapper, PlatformNode, PlatformRootNode},
     util::{AppContext, WindowBounds},
+    KeyState, KeyboardEvent,
 };
 use accesskit::{kurbo::Rect, ActionHandler, NodeId, Role, TreeUpdate};
 use accesskit_consumer::{DetachedNode, FilterResult, Node, Tree, TreeChangeHandler, TreeState};
-use atspi::{Interface, InterfaceSet, State};
+use atspi::{
+    device_event_controller::{DeviceEvent, DeviceEventControllerProxy, EventType},
+    Interface, InterfaceSet, State,
+};
 use futures::{
     channel::mpsc::{self, UnboundedReceiver, UnboundedSender},
     StreamExt,
 };
 use parking_lot::RwLock;
-use std::sync::Arc;
+use std::{sync::Arc, time::SystemTime};
 use zbus::Task;
 
+type KeyboardEventMessage = (KeyboardEvent, Box<dyn FnOnce() + Send>);
+
 pub struct Adapter {
     atspi_bus: Bus,
-    _event_task: Task<()>,
-    events: UnboundedSender<Event>,
+    _tree_event_task: Task<()>,
+    _keyboard_event_task: Task<()>,
+    tree_events: UnboundedSender<Event>,
+    keyboard_events: UnboundedSender<KeyboardEventMessage>,
     _app_context: Arc<RwLock<AppContext>>,
     root_window_bounds: Arc<RwLock<WindowBounds>>,
     tree: Arc<Tree>,
@@ -43,13 +51,21 @@ impl Adapter {
         action_handler: Box<dyn ActionHandler>,
     ) -> Option<Self> {
         let mut atspi_bus = Bus::a11y_bus()?;
-        let (event_sender, event_receiver) = mpsc::unbounded();
+        let (tree_event_tx, tree_event_rx) = mpsc::unbounded();
+        let atspi_bus_copy = atspi_bus.clone();
+        let tree_event_task = atspi_bus.connection().inner().executor().spawn(
+            async move {
+                handle_tree_events(atspi_bus_copy, tree_event_rx).await;
+            },
+            "accesskit_tree_events",
+        );
+        let (keyboard_event_tx, keyboard_event_rx) = mpsc::unbounded();
         let atspi_bus_copy = atspi_bus.clone();
-        let event_task = atspi_bus.connection().inner().executor().spawn(
+        let keyboard_event_task = atspi_bus.connection().inner().executor().spawn(
             async move {
-                handle_events(atspi_bus_copy, event_receiver).await;
+                handle_keyboard_events(atspi_bus_copy, keyboard_event_rx).await;
             },
-            "accesskit_event_task",
+            "accesskit_keyboard_events",
         );
         let tree = Arc::new(Tree::new(initial_state(), action_handler));
         let app_context = Arc::new(RwLock::new(AppContext::new(
@@ -62,8 +78,10 @@ impl Adapter {
             .ok()?;
         let adapter = Adapter {
             atspi_bus,
-            _event_task: event_task,
-            events: event_sender,
+            _tree_event_task: tree_event_task,
+            _keyboard_event_task: keyboard_event_task,
+            tree_events: tree_event_tx,
+            keyboard_events: keyboard_event_tx,
             _app_context: app_context,
             root_window_bounds: Arc::new(RwLock::new(WindowBounds::default())),
             tree,
@@ -149,6 +167,15 @@ impl Adapter {
         Ok(true)
     }
 
+    pub fn on_keyboard_event<F>(&self, event: KeyboardEvent, callback: F)
+    where
+        F: FnOnce() + Send + 'static,
+    {
+        self.keyboard_events
+            .unbounded_send((event, Box::new(callback)))
+            .unwrap();
+    }
+
     pub fn set_root_window_bounds(&self, outer: Rect, inner: Rect) {
         let mut bounds = self.root_window_bounds.write();
         bounds.outer = outer;
@@ -170,7 +197,7 @@ impl Adapter {
             fn remove_node(&mut self, node: &DetachedNode) {
                 let node = NodeWrapper::DetachedNode(node);
                 self.adapter
-                    .events
+                    .tree_events
                     .unbounded_send(Event::Object {
                         target: node.id(),
                         event: ObjectEvent::StateChanged(State::Defunct, true),
@@ -214,7 +241,7 @@ impl Adapter {
                         .unwrap();
                     new_wrapper.notify_changes(
                         &self.adapter.root_window_bounds.read(),
-                        &self.adapter.events,
+                        &self.adapter.tree_events,
                         &old_wrapper,
                     );
                 }
@@ -224,18 +251,18 @@ impl Adapter {
                     if old_node.is_none() && new_node.is_some() {
                         self.adapter.window_activated(
                             &NodeWrapper::Node(&root_window),
-                            &self.adapter.events,
+                            &self.adapter.tree_events,
                         );
                     } else if old_node.is_some() && new_node.is_none() {
                         self.adapter.window_deactivated(
                             &NodeWrapper::Node(&root_window),
-                            &self.adapter.events,
+                            &self.adapter.tree_events,
                         );
                     }
                 }
                 if let Some(node) = new_node.map(NodeWrapper::Node) {
                     self.adapter
-                        .events
+                        .tree_events
                         .unbounded_send(Event::Object {
                             target: node.id(),
                             event: ObjectEvent::StateChanged(State::Focused, true),
@@ -244,7 +271,7 @@ impl Adapter {
                 }
                 if let Some(node) = old_node.map(NodeWrapper::DetachedNode) {
                     self.adapter
-                        .events
+                        .tree_events
                         .unbounded_send(Event::Object {
                             target: node.id(),
                             event: ObjectEvent::StateChanged(State::Focused, false),
@@ -308,7 +335,7 @@ fn root_window(current_state: &TreeState) -> Option<Node> {
     }
 }
 
-async fn handle_events(bus: Bus, mut events: UnboundedReceiver<Event>) {
+async fn handle_tree_events(bus: Bus, mut events: UnboundedReceiver<Event>) {
     while let Some(event) = events.next().await {
         let _ = match event {
             Event::Object { target, event } => bus.emit_object_event(target, event).await,
@@ -320,3 +347,33 @@ async fn handle_events(bus: Bus, mut events: UnboundedReceiver<Event>) {
         };
     }
 }
+
+async fn handle_keyboard_events(bus: Bus, mut events: UnboundedReceiver<KeyboardEventMessage>) {
+    let controller = DeviceEventControllerProxy::new(bus.connection().inner())
+        .await
+        .unwrap();
+    while let Some((event, callback)) = events.next().await {
+        let timestamp = SystemTime::now()
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .unwrap()
+            .as_secs();
+        let consumed = controller
+            .notify_listeners_sync(&DeviceEvent {
+                event_type: match event.state {
+                    KeyState::Pressed => EventType::KeyPressed,
+                    KeyState::Released => EventType::KeyReleased,
+                },
+                id: event.id,
+                hw_code: event.hw_code,
+                modifiers: event.modifiers,
+                timestamp: timestamp as i32,
+                event_string: &event.event_string,
+                is_text: event.character.is_some(),
+            })
+            .await
+            .unwrap();
+        if !consumed {
+            callback();
+        }
+    }
+}

platforms/unix/src/atspi/interfaces/events.rs

@@ -39,3 +39,17 @@ pub(crate) enum WindowEvent {
     Activated,
     Deactivated,
 }
+
+pub enum KeyState {
+    Pressed,
+    Released,
+}
+
+pub struct KeyboardEvent {
+    pub state: KeyState,
+    pub id: i32,
+    pub hw_code: i32,
+    pub modifiers: i32,
+    pub event_string: String,
+    pub character: Option<char>,
+}

platforms/unix/src/atspi/interfaces/mod.rs

@@ -39,4 +39,5 @@ pub(crate) use action::*;
 pub(crate) use application::*;
 pub(crate) use component::*;
 pub(crate) use events::*;
+pub use events::{KeyState, KeyboardEvent};
 pub(crate) use value::*;

platforms/unix/src/atspi/mod.rs

@@ -41,6 +41,7 @@ impl From<accesskit::kurbo::Rect> for Rect {
 }
 
 pub(crate) use bus::Bus;
+pub use interfaces::{KeyState, KeyboardEvent};
 pub(crate) use object_address::*;
 pub(crate) use object_id::*;
 pub(crate) use object_ref::*;

platforms/unix/src/lib.rs

@@ -11,5 +11,8 @@ mod atspi;
 mod node;
 mod util;
 
+pub use ::atspi::device_event_controller::EventType;
+
+pub use crate::atspi::{KeyState, KeyboardEvent};
 pub use adapter::Adapter;
 pub(crate) use node::{unknown_object, PlatformNode, PlatformRootNode};

platforms/winit/src/lib.rs

@@ -5,50 +5,56 @@
 use accesskit::{ActionHandler, ActionRequest, TreeUpdate};
 use parking_lot::Mutex;
 use winit::{
-    event::WindowEvent,
+    event::{KeyboardInput, WindowEvent},
     event_loop::EventLoopProxy,
     window::{Window, WindowId},
 };
 
 mod platform_impl;
 
 #[derive(Debug)]
-pub struct ActionRequestEvent {
+pub struct AccessKitEvent {
     pub window_id: WindowId,
-    pub request: ActionRequest,
+    pub kind: AccessKitEventKind,
 }
 
-struct WinitActionHandler<T: From<ActionRequestEvent> + Send + 'static> {
+#[derive(Debug)]
+pub enum AccessKitEventKind {
+    ActionRequest(ActionRequest),
+    Keyboard(KeyboardInput),
+}
+
+struct WinitActionHandler<T: From<AccessKitEvent> + Send + 'static> {
     window_id: WindowId,
     proxy: Mutex<EventLoopProxy<T>>,
 }
 
-impl<T: From<ActionRequestEvent> + Send + 'static> ActionHandler for WinitActionHandler<T> {
+impl<T: From<AccessKitEvent> + Send + 'static> ActionHandler for WinitActionHandler<T> {
     fn do_action(&self, request: ActionRequest) {
         let proxy = self.proxy.lock();
-        let event = ActionRequestEvent {
+        let event = AccessKitEvent {
             window_id: self.window_id,
-            request,
+            kind: AccessKitEventKind::ActionRequest(request),
         };
         proxy.send_event(event.into()).ok();
     }
 }
 
-pub struct Adapter {
-    adapter: platform_impl::Adapter,
+pub struct Adapter<T: From<AccessKitEvent> + Send + 'static> {
+    adapter: platform_impl::Adapter<T>,
 }
 
-impl Adapter {
-    pub fn new<T: From<ActionRequestEvent> + Send + 'static>(
+impl<T: From<AccessKitEvent> + Send + 'static> Adapter<T> {
+    pub fn new(
         window: &Window,
         source: impl 'static + FnOnce() -> TreeUpdate + Send,
         event_loop_proxy: EventLoopProxy<T>,
     ) -> Self {
         let action_handler = WinitActionHandler {
             window_id: window.id(),
-            proxy: Mutex::new(event_loop_proxy),
+            proxy: Mutex::new(event_loop_proxy.clone()),
         };
-        Self::with_action_handler(window, source, Box::new(action_handler))
+        Self::with_action_handler(window, source, Box::new(action_handler), event_loop_proxy)
     }
 
     /// Use this if you need to provide your own AccessKit action handler
@@ -59,8 +65,9 @@ impl Adapter {
         window: &Window,
         source: impl 'static + FnOnce() -> TreeUpdate + Send,
         action_handler: Box<dyn ActionHandler>,
+        event_loop_proxy: EventLoopProxy<T>,
     ) -> Self {
-        let adapter = platform_impl::Adapter::new(window, source, action_handler);
+        let adapter = platform_impl::Adapter::new(window, source, action_handler, event_loop_proxy);
         Self { adapter }
     }
 
@@ -84,44 +91,7 @@ impl Adapter {
     ))]
     #[must_use]
     pub fn on_event(&self, window: &Window, event: &WindowEvent) -> bool {
-        use accesskit::kurbo::Rect;
-
-        match event {
-            WindowEvent::Moved(outer_position) => {
-                let outer_position: (_, _) = outer_position.cast::<f64>().into();
-                let outer_size: (_, _) = window.outer_size().cast::<f64>().into();
-                let inner_position: (_, _) = window
-                    .inner_position()
-                    .unwrap_or_default()
-                    .cast::<f64>()
-                    .into();
-                let inner_size: (_, _) = window.inner_size().cast::<f64>().into();
-                self.adapter.set_root_window_bounds(
-                    Rect::from_origin_size(outer_position, outer_size),
-                    Rect::from_origin_size(inner_position, inner_size),
-                )
-            }
-            WindowEvent::Resized(outer_size) => {
-                let outer_position: (_, _) = window
-                    .outer_position()
-                    .unwrap_or_default()
-                    .cast::<f64>()
-                    .into();
-                let outer_size: (_, _) = outer_size.cast::<f64>().into();
-                let inner_position: (_, _) = window
-                    .inner_position()
-                    .unwrap_or_default()
-                    .cast::<f64>()
-                    .into();
-                let inner_size: (_, _) = window.inner_size().cast::<f64>().into();
-                self.adapter.set_root_window_bounds(
-                    Rect::from_origin_size(outer_position, outer_size),
-                    Rect::from_origin_size(inner_position, inner_size),
-                )
-            }
-            _ => (),
-        }
-        true
+        self.adapter.on_event(window, event)
     }
 
     pub fn update(&self, update: TreeUpdate) {

platforms/winit/src/platform_impl/macos.rs

@@ -2,23 +2,30 @@
 // Licensed under the Apache License, Version 2.0 (found in
 // the LICENSE-APACHE file).
 
+use crate::AccessKitEvent;
 use accesskit::{ActionHandler, TreeUpdate};
 use accesskit_macos::SubclassingAdapter;
-use winit::{platform::macos::WindowExtMacOS, window::Window};
+use std::marker::PhantomData;
+use winit::{event_loop::EventLoopProxy, platform::macos::WindowExtMacOS, window::Window};
 
-pub struct Adapter {
+pub struct Adapter<T: From<AccessKitEvent> + Send + 'static> {
     adapter: SubclassingAdapter,
+    _proxy: PhantomData<T>,
 }
 
-impl Adapter {
+impl<T: From<AccessKitEvent> + Send + 'static> Adapter<T> {
     pub fn new(
         window: &Window,
         source: impl 'static + FnOnce() -> TreeUpdate,
         action_handler: Box<dyn ActionHandler>,
+        _event_loop_proxy: EventLoopProxy<T>,
     ) -> Self {
         let view = window.ns_view();
         let adapter = unsafe { SubclassingAdapter::new(view, source, action_handler) };
-        Self { adapter }
+        Self {
+            adapter,
+            _proxy: PhantomData,
+        }
     }
 
     pub fn update(&self, update: TreeUpdate) {