Skip to content

Commit

Permalink
UWP: support virtual keyboard
Browse files Browse the repository at this point in the history
  • Loading branch information
paulrouget committed Jul 2, 2020
1 parent 19b36bd commit 34265c8
Show file tree
Hide file tree
Showing 11 changed files with 143 additions and 43 deletions.
4 changes: 2 additions & 2 deletions components/embedder_traits/lib.rs
Expand Up @@ -19,7 +19,7 @@ use keyboard_types::KeyboardEvent;
use msg::constellation_msg::{InputMethodType, PipelineId, TopLevelBrowsingContextId};
use servo_url::ServoUrl;
use std::fmt::{Debug, Error, Formatter};
use webrender_api::units::{DeviceIntPoint, DeviceIntSize};
use webrender_api::units::{DeviceIntPoint, DeviceIntRect, DeviceIntSize};

pub use webxr_api::MainThreadWaker as EventLoopWaker;

Expand Down Expand Up @@ -197,7 +197,7 @@ pub enum EmbedderMsg {
/// Open interface to request permission specified by prompt.
PromptPermission(PermissionPrompt, IpcSender<PermissionRequest>),
/// Request to present an IME to the user when an editable element is focused.
ShowIME(InputMethodType),
ShowIME(InputMethodType, Option<String>, DeviceIntRect),
/// Request to hide the IME when the editable element is blurred.
HideIME,
/// Servo has shut down
Expand Down
25 changes: 23 additions & 2 deletions components/script/dom/document.rs
Expand Up @@ -15,6 +15,8 @@ use crate::dom::bindings::codegen::Bindings::DocumentBinding::{
};
use crate::dom::bindings::codegen::Bindings::EventBinding::EventBinding::EventMethods;
use crate::dom::bindings::codegen::Bindings::HTMLIFrameElementBinding::HTMLIFrameElementBinding::HTMLIFrameElementMethods;
use crate::dom::bindings::codegen::Bindings::HTMLInputElementBinding::HTMLInputElementMethods;
use crate::dom::bindings::codegen::Bindings::HTMLTextAreaElementBinding::HTMLTextAreaElementMethods;
use crate::dom::bindings::codegen::Bindings::NodeBinding::NodeMethods;
use crate::dom::bindings::codegen::Bindings::NodeFilterBinding::NodeFilter;
use crate::dom::bindings::codegen::Bindings::PerformanceBinding::PerformanceMethods;
Expand Down Expand Up @@ -67,7 +69,9 @@ use crate::dom::htmlheadelement::HTMLHeadElement;
use crate::dom::htmlhtmlelement::HTMLHtmlElement;
use crate::dom::htmliframeelement::HTMLIFrameElement;
use crate::dom::htmlimageelement::HTMLImageElement;
use crate::dom::htmlinputelement::HTMLInputElement;
use crate::dom::htmlscriptelement::{HTMLScriptElement, ScriptResult};
use crate::dom::htmltextareaelement::HTMLTextAreaElement;
use crate::dom::htmltitleelement::HTMLTitleElement;
use crate::dom::keyboardevent::KeyboardEvent;
use crate::dom::location::Location;
Expand Down Expand Up @@ -113,7 +117,7 @@ use devtools_traits::ScriptToDevtoolsControlMsg;
use dom_struct::dom_struct;
use embedder_traits::EmbedderMsg;
use encoding_rs::{Encoding, UTF_8};
use euclid::default::Point2D;
use euclid::default::{Point2D, Rect, Size2D};
use html5ever::{LocalName, Namespace, QualName};
use hyper_serde::Serde;
use ipc_channel::ipc::{self, IpcSender};
Expand Down Expand Up @@ -168,6 +172,7 @@ use style::stylesheet_set::DocumentStylesheetSet;
use style::stylesheets::{Origin, OriginSet, Stylesheet};
use url::Host;
use uuid::Uuid;
use webrender_api::units::DeviceIntRect;

/// The number of times we are allowed to see spurious `requestAnimationFrame()` calls before
/// falling back to fake ones.
Expand Down Expand Up @@ -1105,7 +1110,23 @@ impl Document {

// Notify the embedder to display an input method.
if let Some(kind) = elem.input_method_type() {
self.send_to_embedder(EmbedderMsg::ShowIME(kind));
let rect = elem.upcast::<Node>().bounding_content_box_or_zero();
let rect = Rect::new(
Point2D::new(rect.origin.x.to_px(), rect.origin.y.to_px()),
Size2D::new(rect.size.width.to_px(), rect.size.height.to_px()),
);
let text = if let Some(input) = elem.downcast::<HTMLInputElement>() {
Some((&input.Value()).to_string())
} else if let Some(textarea) = elem.downcast::<HTMLTextAreaElement>() {
Some((&textarea.Value()).to_string())
} else {
None
};
self.send_to_embedder(EmbedderMsg::ShowIME(
kind,
text,
DeviceIntRect::from_untyped(&rect),
));
}
}
}
Expand Down
20 changes: 16 additions & 4 deletions ports/libmlservo/src/lib.rs
Expand Up @@ -14,10 +14,11 @@ use rust_webvr::api::MagicLeapVRService;
use servo::euclid::Scale;
use servo::keyboard_types::Key;
use servo::servo_url::ServoUrl;
use servo::webrender_api::units::{DevicePixel, DevicePoint, LayoutPixel};
use servo::webrender_api::units::{DeviceIntRect, DevicePixel, DevicePoint, LayoutPixel};
use simpleservo::{self, deinit, gl_glue, MouseButton, ServoGlue, SERVO};
use simpleservo::{
Coordinates, EventLoopWaker, HostTrait, InitOptions, PromptResult, VRInitOptions,
Coordinates, EventLoopWaker, HostTrait, InitOptions, InputMethodType, PromptResult,
VRInitOptions,
};
use smallvec::SmallVec;
use std::cell::Cell;
Expand Down Expand Up @@ -412,9 +413,20 @@ impl HostTrait for HostCallbacks {
self.shut_down_complete.set(true);
}

fn on_ime_state_changed(&self, show: bool) {
fn on_ime_show(
&self,
_input_type: InputMethodType,
_text: Option<String>,
_bounds: DeviceIntRect,
) {
if let Some(keyboard) = self.keyboard.0 {
keyboard(self.app, show)
keyboard(self.app, true)
}
}

fn on_ime_hide(&self) {
if let Some(keyboard) = self.keyboard.0 {
keyboard(self.app, false)
}
}

Expand Down
14 changes: 10 additions & 4 deletions ports/libsimpleservo/api/src/lib.rs
Expand Up @@ -11,7 +11,9 @@ pub use servo::config::prefs::{add_user_prefs, PrefValue};
pub use servo::embedder_traits::{
ContextMenuResult, MediaSessionPlaybackState, PermissionPrompt, PermissionRequest, PromptResult,
};
pub use servo::msg::constellation_msg::InputMethodType;
pub use servo::script_traits::{MediaSessionActionType, MouseButton};
pub use servo::webrender_api::units::DeviceIntRect;

use getopts::Options;
use ipc_channel::ipc::IpcSender;
Expand Down Expand Up @@ -133,7 +135,9 @@ pub trait HostTrait {
/// Servo finished shutting down.
fn on_shutdown_complete(&self);
/// A text input is focused.
fn on_ime_state_changed(&self, show: bool);
fn on_ime_show(&self, input_type: InputMethodType, text: Option<String>, bounds: DeviceIntRect);
/// Input lost focus
fn on_ime_hide(&self);
/// Gets sytem clipboard contents.
fn get_clipboard_contents(&self) -> Option<String>;
/// Sets system clipboard contents.
Expand Down Expand Up @@ -721,11 +725,13 @@ impl ServoGlue {

let _ = sender.send(result);
},
EmbedderMsg::ShowIME(..) => {
self.callbacks.host_callbacks.on_ime_state_changed(true);
EmbedderMsg::ShowIME(kind, text, bounds) => {
self.callbacks
.host_callbacks
.on_ime_show(kind, text, bounds);
},
EmbedderMsg::HideIME => {
self.callbacks.host_callbacks.on_ime_state_changed(false);
self.callbacks.host_callbacks.on_ime_hide();
},
EmbedderMsg::MediaSessionEvent(event) => {
match event {
Expand Down
34 changes: 28 additions & 6 deletions ports/libsimpleservo/capi/src/lib.rs
Expand Up @@ -20,8 +20,8 @@ use keyboard_types::Key;
use log::LevelFilter;
use simpleservo::{self, gl_glue, ServoGlue, SERVO};
use simpleservo::{
ContextMenuResult, Coordinates, EventLoopWaker, HostTrait, InitOptions, MediaSessionActionType,
MediaSessionPlaybackState, MouseButton, PromptResult,
ContextMenuResult, Coordinates, DeviceIntRect, EventLoopWaker, HostTrait, InitOptions,
InputMethodType, MediaSessionActionType, MediaSessionPlaybackState, MouseButton, PromptResult,
};
use std::ffi::{CStr, CString};
#[cfg(target_os = "windows")]
Expand Down Expand Up @@ -212,7 +212,8 @@ pub struct CHostCallbacks {
pub on_history_changed: extern "C" fn(can_go_back: bool, can_go_forward: bool),
pub on_animating_changed: extern "C" fn(animating: bool),
pub on_shutdown_complete: extern "C" fn(),
pub on_ime_state_changed: extern "C" fn(show: bool),
pub on_ime_show: extern "C" fn(text: *const c_char, x: i32, y: i32, width: i32, height: i32),
pub on_ime_hide: extern "C" fn(),
pub get_clipboard_contents: extern "C" fn() -> *const c_char,
pub set_clipboard_contents: extern "C" fn(contents: *const c_char),
pub on_media_session_metadata:
Expand Down Expand Up @@ -834,9 +835,30 @@ impl HostTrait for HostCallbacks {
(self.0.on_shutdown_complete)();
}

fn on_ime_state_changed(&self, show: bool) {
debug!("on_ime_state_changed");
(self.0.on_ime_state_changed)(show);
fn on_ime_show(
&self,
_input_type: InputMethodType,
text: Option<String>,
bounds: DeviceIntRect,
) {
debug!("on_ime_show");
let text = text.and_then(|s| CString::new(s).ok());
let text_ptr = text
.as_ref()
.map(|cstr| cstr.as_ptr())
.unwrap_or(std::ptr::null());
(self.0.on_ime_show)(
text_ptr,
bounds.origin.x,
bounds.origin.y,
bounds.size.width,
bounds.size.height,
);
}

fn on_ime_hide(&self) {
debug!("on_ime_hide");
(self.0.on_ime_hide)();
}

fn get_clipboard_contents(&self) -> Option<String> {
Expand Down
8 changes: 5 additions & 3 deletions ports/libsimpleservo/jniapi/src/lib.rs
Expand Up @@ -14,10 +14,11 @@ use jni::sys::{jboolean, jfloat, jint, jstring, JNI_TRUE};
use jni::{errors, JNIEnv, JavaVM};
use libc::{dup2, pipe, read};
use log::Level;
use simpleservo::{self, deinit, gl_glue, MouseButton, ServoGlue, SERVO};
use simpleservo::{self, gl_glue, ServoGlue, SERVO};
use simpleservo::{
Coordinates, EventLoopWaker, HostTrait, InitOptions, MediaSessionPlaybackState, PromptResult,
VRInitOptions,
Coordinates, DeviceIntRect, EventLoopWaker, HostTrait, InitOptions, InputMethodType,
MediaSessionPlaybackState, PromptResult, VRInitOptions,
};
use std::os::raw::{c_char, c_int, c_void};
use std::ptr::{null, null_mut};
Expand Down Expand Up @@ -529,7 +530,8 @@ impl HostTrait for HostCallbacks {
.unwrap();
}

fn on_ime_state_changed(&self, _show: bool) {}
fn on_ime_show(&self, _type: InputEncoding, _text: Option<String>, _rect: DeviceIntRect) {}
fn on_ime_hide(&self) {}

fn get_clipboard_contents(&self) -> Option<String> {
None
Expand Down
2 changes: 1 addition & 1 deletion ports/winit/browser.rs
Expand Up @@ -498,7 +498,7 @@ where
let permission_state = prompt_user(prompt);
let _ = sender.send(permission_state);
}
EmbedderMsg::ShowIME(_kind) => {
EmbedderMsg::ShowIME(_kind, _text, _rect) => {
debug!("ShowIME received");
},
EmbedderMsg::HideIME => {
Expand Down
14 changes: 11 additions & 3 deletions support/hololens/ServoApp/ServoControl/Servo.cpp
Expand Up @@ -43,10 +43,17 @@ void on_panic(const char *backtrace) {
throw hresult_error(E_FAIL, char2hstring(backtrace));
}

void on_ime_state_changed(bool aShow) {
sServo->Delegate().OnServoIMEStateChanged(aShow);
void on_ime_show(const char *text, int32_t x, int32_t y, int32_t width,
int32_t height) {
hstring htext = L"";
if (text != nullptr) {
htext = char2hstring(text);
}
sServo->Delegate().OnServoIMEShow(htext, x, y, width, height);
}

void on_ime_hide() { sServo->Delegate().OnServoIMEHide(); }

void set_clipboard_contents(const char *) {
// FIXME
}
Expand Down Expand Up @@ -256,7 +263,8 @@ Servo::Servo(hstring args, GLsizei width, GLsizei height,
c.on_animating_changed = &on_animating_changed;
c.on_shutdown_complete = &on_shutdown_complete;
c.on_allow_navigation = &on_allow_navigation;
c.on_ime_state_changed = &on_ime_state_changed;
c.on_ime_show = &on_ime_show;
c.on_ime_hide = &on_ime_hide;
c.get_clipboard_contents = &get_clipboard_contents;
c.set_clipboard_contents = &set_clipboard_contents;
c.on_media_session_metadata = &on_media_session_metadata;
Expand Down
4 changes: 3 additions & 1 deletion support/hololens/ServoApp/ServoControl/Servo.h
Expand Up @@ -109,7 +109,9 @@ class ServoDelegate {
virtual void OnServoURLChanged(hstring) = 0;
virtual bool OnServoAllowNavigation(hstring) = 0;
virtual void OnServoAnimatingChanged(bool) = 0;
virtual void OnServoIMEStateChanged(bool) = 0;
virtual void OnServoIMEShow(hstring text, int32_t x, int32_t y, int32_t width,
int32_t height) = 0;
virtual void OnServoIMEHide() = 0;
virtual void OnServoDevtoolsStarted(bool, const unsigned int, hstring) = 0;
virtual void OnServoMediaSessionMetadata(hstring, hstring, hstring) = 0;
virtual void OnServoMediaSessionPlaybackStateChange(int) = 0;
Expand Down
55 changes: 39 additions & 16 deletions support/hololens/ServoApp/ServoControl/ServoControl.cpp
Expand Up @@ -94,26 +94,37 @@ void ServoControl::InitializeTextController() {
auto manager = CoreTextServicesManager::GetForCurrentView();
mEditContext = manager.CreateEditContext();
mEditContext->InputPaneDisplayPolicy(CoreTextInputPaneDisplayPolicy::Manual);
mEditContext->InputScope(CoreTextInputScope::Text);

mEditContext->TextRequested([=](const auto &, const auto &e) {
e.Request().Text(L"");
CoreTextRange sel;
sel.StartCaretPosition = 0;
sel.EndCaretPosition = 0;
e.Request().Range() = sel;
e.Request().Text(*mFocusedInputText);
});

mEditContext->SelectionRequested([=](const auto &, const auto &) {});

mEditContext->LayoutRequested([=](const auto &, const auto &e) {
// Necessary to show the preview
e.Request().LayoutBounds().TextBounds(*mFocusedInputRect);
e.Request().LayoutBounds().ControlBounds(*mFocusedInputRect);
});

mEditContext->TextUpdating([=](const auto &, const auto &e) {
RunOnGLThread([=] {
auto keystr = *hstring2char((e.Text()));
mServo->KeyDown(keystr);
auto text = *hstring2char(e.Text());
size_t size = strlen(text);
for (int i = 0; i < size; i++) {
char letter[2];
memcpy(letter, &text[i], 1);
letter[1] = '\0';
mServo->KeyDown(letter);
mServo->KeyUp(letter);
}
});
e.Result(CoreTextTextUpdatingResult::Succeeded);
});

mEditContext->SelectionRequested([](const auto &, const auto &) {});

GotFocus(
[=](const auto &, const auto &) { mEditContext->NotifyFocusEnter(); });

LostFocus(
[=](const auto &, const auto &) { mEditContext->NotifyFocusLeave(); });

Expand All @@ -126,6 +137,7 @@ void ServoControl::InitializeTextController() {
});
}
});

PreviewKeyUp([=](const auto &, const auto &e) {
auto keystr = KeyToString(e.Key());
if (keystr.has_value()) {
Expand Down Expand Up @@ -489,13 +501,24 @@ void ServoControl::OnServoAnimatingChanged(bool animating) {
WakeConditionVariable(&mGLCondVar);
}

void ServoControl::OnServoIMEStateChanged(bool focused) {
void ServoControl::OnServoIMEHide() {
RunOnUIThread([=] { mInputPane->TryHide(); });
}

void ServoControl::OnServoIMEShow(hstring text, int32_t x, int32_t y,
int32_t width, int32_t height) {
RunOnUIThread([=] {
if (focused) {
mInputPane->TryShow();
} else {
mInputPane->TryHide();
}
mEditContext->NotifyFocusEnter();
// FIXME: The simpleservo on_ime_show callback comes with a input method
// type parameter that could be used to set the input scope here.
mEditContext->InputScope(CoreTextInputScope::Text);
// offset of the Servo SwapChainPanel.
auto transform = Panel().TransformToVisual(Window::Current().Content());
auto offset = transform.TransformPoint(Point(0, 0));
mFocusedInputRect =
Rect(x + offset.X, y + offset.Y, (float)width, (float)height);
mFocusedInputText = text;
mInputPane->TryShow();
});
}

Expand Down
6 changes: 5 additions & 1 deletion support/hololens/ServoApp/ServoControl/ServoControl.h
Expand Up @@ -173,7 +173,8 @@ struct ServoControl : ServoControlT<ServoControl>, public servo::ServoDelegate {
virtual void OnServoURLChanged(winrt::hstring);
virtual bool OnServoAllowNavigation(winrt::hstring);
virtual void OnServoAnimatingChanged(bool);
virtual void OnServoIMEStateChanged(bool);
virtual void OnServoIMEHide();
virtual void OnServoIMEShow(hstring text, int32_t, int32_t, int32_t, int32_t);
virtual void OnServoMediaSessionMetadata(winrt::hstring, winrt::hstring,
winrt::hstring);
virtual void OnServoMediaSessionPlaybackStateChange(int);
Expand Down Expand Up @@ -281,6 +282,9 @@ struct ServoControl : ServoControlT<ServoControl>, public servo::ServoDelegate {

std::optional<Windows::UI::Text::Core::CoreTextEditContext> mEditContext;
std::optional<Windows::UI::ViewManagement::InputPane> mInputPane;

std::optional<Windows::Foundation::Rect> mFocusedInputRect;
std::optional<hstring> mFocusedInputText;
};
} // namespace winrt::ServoApp::implementation

Expand Down

0 comments on commit 34265c8

Please sign in to comment.