diff --git a/components/compositing/compositor.rs b/components/compositing/compositor.rs index f0474363fa46..192be87e95a6 100644 --- a/components/compositing/compositor.rs +++ b/components/compositing/compositor.rs @@ -522,19 +522,21 @@ impl IOCompositor { Msg::WebDriverMouseButtonEvent(mouse_event_type, mouse_button, x, y), ShutdownState::NotShuttingDown, ) => { + let dppx = self.device_pixels_per_page_px(); + let point = dppx.transform_point(Point2D::new(x, y)); self.on_mouse_window_event_class(match mouse_event_type { - MouseEventType::Click => { - MouseWindowEvent::Click(mouse_button, DevicePoint::new(x, y)) - }, - MouseEventType::MouseDown => { - MouseWindowEvent::MouseDown(mouse_button, DevicePoint::new(x, y)) - }, - MouseEventType::MouseUp => { - MouseWindowEvent::MouseUp(mouse_button, DevicePoint::new(x, y)) - }, + MouseEventType::Click => MouseWindowEvent::Click(mouse_button, point), + MouseEventType::MouseDown => MouseWindowEvent::MouseDown(mouse_button, point), + MouseEventType::MouseUp => MouseWindowEvent::MouseUp(mouse_button, point), }); }, + (Msg::WebDriverMouseMoveEvent(x, y), ShutdownState::NotShuttingDown) => { + let dppx = self.device_pixels_per_page_px(); + let point = dppx.transform_point(Point2D::new(x, y)); + self.on_mouse_window_move_event_class(DevicePoint::new(point.x, point.y)); + }, + (Msg::PendingPaintMetric(pipeline_id, epoch), _) => { self.pending_paint_metrics.insert(pipeline_id, epoch); }, diff --git a/components/compositing/compositor_thread.rs b/components/compositing/compositor_thread.rs index e10aff78844e..f5a14352c964 100644 --- a/components/compositing/compositor_thread.rs +++ b/components/compositing/compositor_thread.rs @@ -110,6 +110,8 @@ pub enum Msg { LoadComplete(TopLevelBrowsingContextId), /// WebDriver mouse button event WebDriverMouseButtonEvent(MouseEventType, MouseButton, f32, f32), + /// WebDriver mouse move event + WebDriverMouseMoveEvent(f32, f32), /// Get Window Informations size and position. GetClientWindow(IpcSender<(DeviceIntSize, DeviceIntPoint)>), @@ -137,6 +139,7 @@ impl Debug for Msg { Msg::PendingPaintMetric(..) => write!(f, "PendingPaintMetric"), Msg::LoadComplete(..) => write!(f, "LoadComplete"), Msg::WebDriverMouseButtonEvent(..) => write!(f, "WebDriverMouseButtonEvent"), + Msg::WebDriverMouseMoveEvent(..) => write!(f, "WebDriverMouseMoveEvent"), Msg::GetClientWindow(..) => write!(f, "GetClientWindow"), Msg::GetScreenSize(..) => write!(f, "GetScreenSize"), Msg::GetScreenAvailSize(..) => write!(f, "GetScreenAvailSize"), diff --git a/components/constellation/constellation.rs b/components/constellation/constellation.rs index e2169c080f36..9eef83f33a08 100644 --- a/components/constellation/constellation.rs +++ b/components/constellation/constellation.rs @@ -3528,6 +3528,10 @@ where y, )); }, + WebDriverCommandMsg::MouseMoveAction(x, y) => { + self.compositor_proxy + .send(ToCompositorMsg::WebDriverMouseMoveEvent(x, y)); + }, WebDriverCommandMsg::TakeScreenshot(_, rect, reply) => { self.compositor_proxy .send(ToCompositorMsg::CreatePng(rect, reply)); diff --git a/components/script/script_thread.rs b/components/script/script_thread.rs index 3f547d7cfd67..eaffa1d6ef65 100644 --- a/components/script/script_thread.rs +++ b/components/script/script_thread.rs @@ -2209,6 +2209,14 @@ impl ScriptThread { WebDriverScriptCommand::GetElementText(node_id, reply) => { webdriver_handlers::handle_get_text(&*documents, pipeline_id, node_id, reply) }, + WebDriverScriptCommand::GetElementInViewCenterPoint(node_id, reply) => { + webdriver_handlers::handle_get_element_in_view_center_point( + &*documents, + pipeline_id, + node_id, + reply, + ) + }, WebDriverScriptCommand::GetBrowsingContextId(webdriver_frame_id, reply) => { webdriver_handlers::handle_get_browsing_context_id( &*documents, diff --git a/components/script/webdriver_handlers.rs b/components/script/webdriver_handlers.rs index 7293774b7a43..fdb688b1f603 100644 --- a/components/script/webdriver_handlers.rs +++ b/components/script/webdriver_handlers.rs @@ -53,6 +53,7 @@ use script_traits::webdriver_msg::{ WebDriverFrameId, WebDriverJSError, WebDriverJSResult, WebDriverJSValue, }; use servo_url::ServoUrl; +use std::cmp; use std::collections::HashMap; use std::ffi::CString; use webdriver::common::{WebElement, WebFrame, WebWindow}; @@ -360,6 +361,56 @@ pub fn handle_get_browsing_context_id( .unwrap(); } +// https://w3c.github.io/webdriver/#dfn-center-point +fn get_element_in_view_center_point(element: &Element) -> Option> { + element + .GetClientRects() + .iter() + // Step 1 + .next() + .map(|rectangle| { + let x = rectangle.X().round() as i64; + let y = rectangle.Y().round() as i64; + let width = rectangle.Width().round() as i64; + let height = rectangle.Height().round() as i64; + + let window = window_from_node(element.upcast::()); + let document = window.Document(); + let document_element = document.upcast::().downcast::().unwrap(); + let clientWidth = document_element.ClientWidth() as i64; + let clientHeight = document_element.ClientHeight() as i64; + + // Steps 2 - 5 + let left = cmp::max(0, cmp::min(x, x + width)); + let right = cmp::min(clientWidth, cmp::max(x, x + width)); + let top = cmp::max(0, cmp::min(y, y + height)); + let bottom = cmp::min(clientHeight, cmp::max(y, y + height)); + + // Steps 6 - 7 + let x = (left + right) / 2; + let y = (top + bottom) / 2; + + // Step 8 + Point2D::new(x, y) + }) +} + +pub fn handle_get_element_in_view_center_point( + documents: &Documents, + pipeline: PipelineId, + element_id: String, + reply: IpcSender, ErrorStatus>>, +) { + reply + .send( + find_node_by_unique_id(documents, pipeline, element_id).map(|node| { + get_element_in_view_center_point(node.downcast::().unwrap()) + .map(|point| (point.x, point.y)) + }), + ) + .unwrap(); +} + pub fn handle_find_element_css( documents: &Documents, pipeline: PipelineId, diff --git a/components/script_traits/lib.rs b/components/script_traits/lib.rs index f0aa1516ca2e..1aa37e565337 100644 --- a/components/script_traits/lib.rs +++ b/components/script_traits/lib.rs @@ -797,6 +797,8 @@ pub enum WebDriverCommandMsg { KeyboardAction(BrowsingContextId, KeyboardEvent), /// Act as if the mouse was clicked in the browsing context with the given ID. MouseButtonAction(MouseEventType, MouseButton, f32, f32), + /// Act as if the mouse was moved in the browsing context with the given ID. + MouseMoveAction(f32, f32), /// Set the window size. SetWindowSize( TopLevelBrowsingContextId, diff --git a/components/script_traits/webdriver_msg.rs b/components/script_traits/webdriver_msg.rs index 23fa60efddb8..8a691d75eb5c 100644 --- a/components/script_traits/webdriver_msg.rs +++ b/components/script_traits/webdriver_msg.rs @@ -75,6 +75,7 @@ pub enum WebDriverScriptCommand { GetElementRect(String, IpcSender, ErrorStatus>>), GetElementTagName(String, IpcSender>), GetElementText(String, IpcSender>), + GetElementInViewCenterPoint(String, IpcSender, ErrorStatus>>), GetBoundingClientRect(String, IpcSender, ErrorStatus>>), GetBrowsingContextId( WebDriverFrameId, diff --git a/components/webdriver_server/actions.rs b/components/webdriver_server/actions.rs index 0182a16bac31..28255e44cd9c 100644 --- a/components/webdriver_server/actions.rs +++ b/components/webdriver_server/actions.rs @@ -3,30 +3,40 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ use crate::Handler; +use crossbeam_channel::Sender; +use ipc_channel::ipc; use keyboard_types::webdriver::KeyInputState; +use script_traits::webdriver_msg::WebDriverScriptCommand; use script_traits::{ConstellationMsg, MouseButton, MouseEventType, WebDriverCommandMsg}; use std::cmp; use std::collections::HashSet; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; use webdriver::actions::{ActionSequence, ActionsType, GeneralAction, NullActionItem}; use webdriver::actions::{KeyAction, KeyActionItem, KeyDownAction, KeyUpAction}; use webdriver::actions::{ PointerAction, PointerActionItem, PointerActionParameters, PointerDownAction, }; -use webdriver::actions::{PointerType, PointerUpAction}; +use webdriver::actions::{PointerMoveAction, PointerOrigin, PointerType, PointerUpAction}; +use webdriver::error::ErrorStatus; + +// Interval between pointerMove increments in ms, based on common vsync +static POINTERMOVE_INTERVAL: u64 = 17; // https://w3c.github.io/webdriver/#dfn-input-source-state pub(crate) enum InputSourceState { Null, Key(KeyInputState), - Pointer(PointerInputState), + Pointer(Arc>), } // https://w3c.github.io/webdriver/#dfn-pointer-input-source pub(crate) struct PointerInputState { subtype: PointerType, pressed: HashSet, - x: u64, - y: u64, + x: i64, + y: i64, } impl PointerInputState { @@ -72,6 +82,73 @@ fn compute_tick_duration(tick_actions: &ActionSequence) -> u64 { duration } +// https://w3c.github.io/webdriver/#dfn-perform-a-pointer-move +fn perform_pointer_move( + constellation_chan: Sender, + pointer_input_state: Arc>, + duration: u64, + start_x: i64, + start_y: i64, + target_x: i64, + target_y: i64, + tick_start: Instant, +) { + let mut pointer_input_state = pointer_input_state.lock().unwrap(); + + loop { + // Step 1 + let time_delta = tick_start.elapsed().as_millis(); + + // Step 2 + let duration_ratio = if duration > 0 { + time_delta as f64 / duration as f64 + } else { + 1.0 + }; + + // Step 3 + let last = if 1.0 - duration_ratio < 0.001 { + true + } else { + false + }; + + // Step 4 + let (x, y) = if last { + (target_x, target_y) + } else { + ( + (duration_ratio * (target_x - start_x) as f64) as i64 + start_x, + (duration_ratio * (target_y - start_y) as f64) as i64 + start_y, + ) + }; + + // Steps 5 - 6 + let current_x = pointer_input_state.x; + let current_y = pointer_input_state.y; + + // Step 7 + if x != current_x || y != current_y { + // Step 7.2 + let cmd_msg = WebDriverCommandMsg::MouseMoveAction(x as f32, y as f32); + constellation_chan + .send(ConstellationMsg::WebDriverCommand(cmd_msg)) + .unwrap(); + // Step 7.3 + pointer_input_state.x = x; + pointer_input_state.y = y; + } + + // Step 8 + if last { + return; + } + + // Step 9 + thread::sleep(Duration::from_millis(POINTERMOVE_INTERVAL)); + } +} + fn u64_to_mouse_button(button: u64) -> Option { if MouseButton::Left as u64 == button { Some(MouseButton::Left) @@ -86,11 +163,15 @@ fn u64_to_mouse_button(button: u64) -> Option { impl Handler { // https://w3c.github.io/webdriver/#dfn-dispatch-actions - pub(crate) fn dispatch_actions(&mut self, actions_by_tick: &[ActionSequence]) { + pub(crate) fn dispatch_actions( + &mut self, + actions_by_tick: &[ActionSequence], + ) -> Result<(), ErrorStatus> { for tick_actions in actions_by_tick.iter() { let tick_duration = compute_tick_duration(&tick_actions); - self.dispatch_tick_actions(&tick_actions, tick_duration); + self.dispatch_tick_actions(&tick_actions, tick_duration)?; } + Ok(()) } fn dispatch_general_action(&mut self, source_id: &str) { @@ -104,7 +185,11 @@ impl Handler { } // https://w3c.github.io/webdriver/#dfn-dispatch-tick-actions - fn dispatch_tick_actions(&mut self, tick_actions: &ActionSequence, _tick_duration: u64) { + fn dispatch_tick_actions( + &mut self, + tick_actions: &ActionSequence, + tick_duration: u64, + ) -> Result<(), ErrorStatus> { let source_id = &tick_actions.id; match &tick_actions.actions { ActionsType::Null { actions } => { @@ -150,15 +235,19 @@ impl Handler { .unwrap() .input_state_table .entry(source_id.to_string()) - .or_insert(InputSourceState::Pointer(PointerInputState::new( - ¶meters.pointer_type, - ))); + .or_insert(InputSourceState::Pointer(Arc::new(Mutex::new( + PointerInputState::new(¶meters.pointer_type), + )))); match action { PointerAction::Cancel => (), PointerAction::Down(action) => { self.dispatch_pointerdown_action(&source_id, &action) }, - PointerAction::Move(_action) => (), + PointerAction::Move(action) => self.dispatch_pointermove_action( + &source_id, + &action, + tick_duration, + )?, PointerAction::Up(action) => { self.dispatch_pointerup_action(&source_id, &action) }, @@ -168,6 +257,8 @@ impl Handler { } }, } + + Ok(()) } // https://w3c.github.io/webdriver/#dfn-dispatch-a-keydown-action @@ -231,10 +322,10 @@ impl Handler { fn dispatch_pointerdown_action(&mut self, source_id: &str, action: &PointerDownAction) { let session = self.session.as_mut().unwrap(); - let pointer_input_state = match session.input_state_table.get_mut(source_id).unwrap() { + let mut pointer_input_state = match session.input_state_table.get(source_id).unwrap() { InputSourceState::Null => unreachable!(), InputSourceState::Key(_) => unreachable!(), - InputSourceState::Pointer(pointer_input_state) => pointer_input_state, + InputSourceState::Pointer(pointer_input_state) => pointer_input_state.lock().unwrap(), }; if pointer_input_state.pressed.contains(&action.button) { @@ -277,10 +368,10 @@ impl Handler { fn dispatch_pointerup_action(&mut self, source_id: &str, action: &PointerUpAction) { let session = self.session.as_mut().unwrap(); - let pointer_input_state = match session.input_state_table.get_mut(source_id).unwrap() { + let mut pointer_input_state = match session.input_state_table.get(source_id).unwrap() { InputSourceState::Null => unreachable!(), InputSourceState::Key(_) => unreachable!(), - InputSourceState::Pointer(pointer_input_state) => pointer_input_state, + InputSourceState::Pointer(pointer_input_state) => pointer_input_state.lock().unwrap(), }; if !pointer_input_state.pressed.contains(&action.button) { @@ -318,4 +409,116 @@ impl Handler { .unwrap(); } } + + // https://w3c.github.io/webdriver/#dfn-dispatch-a-pointermove-action + fn dispatch_pointermove_action( + &mut self, + source_id: &str, + action: &PointerMoveAction, + tick_duration: u64, + ) -> Result<(), ErrorStatus> { + let tick_start = Instant::now(); + + // Steps 1 - 2 + let x_offset = action.x.unwrap_or(0); + let y_offset = action.y.unwrap_or(0); + + // Steps 3 - 4 + let (start_x, start_y) = match self + .session + .as_ref() + .unwrap() + .input_state_table + .get(source_id) + .unwrap() + { + InputSourceState::Null => unreachable!(), + InputSourceState::Key(_) => unreachable!(), + InputSourceState::Pointer(pointer_input_state) => { + let pointer_input_state = pointer_input_state.lock().unwrap(); + (pointer_input_state.x, pointer_input_state.y) + }, + }; + + // Step 5 - 6 + let (x, y) = match action.origin { + PointerOrigin::Viewport => (x_offset, y_offset), + PointerOrigin::Pointer => (start_x + x_offset, start_y + y_offset), + PointerOrigin::Element(ref x) => { + let (sender, receiver) = ipc::channel().unwrap(); + self.top_level_script_command(WebDriverScriptCommand::GetElementInViewCenterPoint( + x.to_string(), + sender, + )) + .unwrap(); + + match receiver.recv().unwrap() { + Ok(point) => match point { + Some(point) => point, + None => return Err(ErrorStatus::UnknownError), + }, + Err(_) => return Err(ErrorStatus::UnknownError), + } + }, + }; + + let (sender, receiver) = ipc::channel().unwrap(); + let cmd_msg = WebDriverCommandMsg::GetWindowSize( + self.session.as_ref().unwrap().top_level_browsing_context_id, + sender, + ); + self.constellation_chan + .send(ConstellationMsg::WebDriverCommand(cmd_msg)) + .unwrap(); + + // Steps 7 - 8 + let viewport = receiver.recv().unwrap().initial_viewport; + if x < 0 || x as f32 > viewport.width || y < 0 || y as f32 > viewport.height { + return Err(ErrorStatus::MoveTargetOutOfBounds); + } + + // Step 9 + let duration = match action.duration { + Some(duration) => duration, + None => tick_duration, + }; + + // Step 10 + if duration > 0 { + thread::sleep(Duration::from_millis(POINTERMOVE_INTERVAL)); + } + + let pointer_input_state = match self + .session + .as_ref() + .unwrap() + .input_state_table + .get(source_id) + .unwrap() + { + InputSourceState::Null => unreachable!(), + InputSourceState::Key(_) => unreachable!(), + InputSourceState::Pointer(pointer_input_state) => pointer_input_state, + }; + + let constellation_chan = self.constellation_chan.clone(); + let pointer_input_state = pointer_input_state.clone(); + + // Step 11 + thread::spawn(move || { + perform_pointer_move( + constellation_chan, + pointer_input_state, + duration, + start_x, + start_y, + x, + y, + tick_start, + ); + }); + + // Step 12 + Ok(()) + } } diff --git a/components/webdriver_server/lib.rs b/components/webdriver_server/lib.rs index 954f2fd0a52d..431ac5eee9ef 100644 --- a/components/webdriver_server/lib.rs +++ b/components/webdriver_server/lib.rs @@ -1345,9 +1345,10 @@ impl Handler { &mut self, parameters: &ActionsParameters, ) -> WebDriverResult { - self.dispatch_actions(¶meters.actions); - - Ok(WebDriverResponse::Void) + match self.dispatch_actions(¶meters.actions) { + Ok(_) => Ok(WebDriverResponse::Void), + Err(error) => Err(WebDriverError::new(error, "")), + } } fn handle_release_actions(&mut self) -> WebDriverResult { @@ -1356,7 +1357,10 @@ impl Handler { session.input_cancel_list.reverse(); mem::replace(&mut session.input_cancel_list, Vec::new()) }; - self.dispatch_actions(&input_cancel_list); + + if let Err(error) = self.dispatch_actions(&input_cancel_list) { + return Err(WebDriverError::new(error, "")); + } let session = self.session_mut()?; session.input_state_table = HashMap::new();