Add basic keyboard support for magicleap

ports/libmlservo/Cargo.toml

@@ -13,6 +13,7 @@ test = false
 bench = false
 
 [dependencies]
+keyboard-types = "0.4"
 libservo = { path = "../../components/servo" }
 log = "0.4"
 servo-egl = "0.2"

ports/libmlservo/src/lib.rs

@@ -8,6 +8,9 @@ use egl::egl::EGLSurface;
 use egl::egl::MakeCurrent;
 use egl::egl::SwapBuffers;
 use egl::eglext::eglGetProcAddress;
+use keyboard_types::Key;
+use keyboard_types::KeyState;
+use keyboard_types::KeyboardEvent;
 use log::info;
 use log::warn;
 use servo::compositing::windowing::AnimationState;
@@ -58,12 +61,40 @@ pub enum MLLogLevel {
     Verbose = 5,
 }
 
+#[repr(C)]
+#[allow(non_camel_case_types)]
+pub enum MLKeyType {
+    kNone,
+    kCharacter,
+    kBackspace,
+    kShift,
+    kSpeechToText,
+    kPageEmoji,
+    kPageLowerLetters,
+    kPageNumericSymbols,
+    kCancel,
+    kSubmit,
+    kPrevious,
+    kNext,
+    kClear,
+    kClose,
+    kEnter,
+    kCustom1,
+    kCustom2,
+    kCustom3,
+    kCustom4,
+    kCustom5,
+}
+
 #[repr(transparent)]
 pub struct MLLogger(extern "C" fn(MLLogLevel, *const c_char));
 
 #[repr(transparent)]
 pub struct MLHistoryUpdate(extern "C" fn(MLApp, bool, *const c_char, bool));
 
+#[repr(transparent)]
+pub struct MLKeyboard(extern "C" fn(MLApp, bool));
+
 #[repr(transparent)]
 #[derive(Clone, Copy)]
 pub struct MLApp(*mut c_void);
@@ -78,6 +109,7 @@ pub unsafe extern "C" fn init_servo(
     app: MLApp,
     logger: MLLogger,
     history_update: MLHistoryUpdate,
+    keyboard: MLKeyboard,
     url: *const c_char,
     width: u32,
     height: u32,
@@ -116,6 +148,7 @@ pub unsafe extern "C" fn init_servo(
         app: app,
         browser_id: browser_id,
         history_update: history_update,
+        keyboard: keyboard,
         scroll_state: ScrollState::TriggerUp,
         scroll_scale: TypedScale::new(SCROLL_SCALE / hidpi),
         servo: servo,
@@ -162,6 +195,8 @@ pub unsafe extern "C" fn heartbeat_servo(servo: *mut ServoInstance) {
                         }
                     }
                 },
+                EmbedderMsg::ShowIME(..) => (servo.keyboard.0)(servo.app, true),
+                EmbedderMsg::HideIME => (servo.keyboard.0)(servo.app, false),
                 // Ignore most messages for now
                 EmbedderMsg::ChangePageTitle(..) |
                 EmbedderMsg::BrowserCreated(..) |
@@ -177,15 +212,46 @@ pub unsafe extern "C" fn heartbeat_servo(servo: *mut ServoInstance) {
                 EmbedderMsg::NewFavicon(..) |
                 EmbedderMsg::HeadParsed |
                 EmbedderMsg::SetFullscreenState(..) |
-                EmbedderMsg::ShowIME(..) |
-                EmbedderMsg::HideIME |
                 EmbedderMsg::Shutdown |
                 EmbedderMsg::Panic(..) => {},
             }
         }
     }
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn keyboard_servo(
+    servo: *mut ServoInstance,
+    key_code: char,
+    key_type: MLKeyType,
+) {
+    if let Some(servo) = servo.as_mut() {
+        let key = match key_type {
+            MLKeyType::kCharacter => Key::Character([key_code].iter().collect()),
+            MLKeyType::kBackspace => Key::Backspace,
+            MLKeyType::kEnter => Key::Enter,
+            _ => return,
+        };
+
+        let key_down = KeyboardEvent {
+            state: KeyState::Down,
+            key: key,
+            ..KeyboardEvent::default()
+        };
+
+        let key_up = KeyboardEvent {
+            state: KeyState::Up,
+            ..key_down.clone()
+        };
+
+        // TODO: can the ML1 generate separate press and release events?
+        servo.servo.handle_events(vec![
+            WindowEvent::Keyboard(key_down),
+            WindowEvent::Keyboard(key_up),
+        ]);
+    }
+}
+
 // Some magic numbers.
 
 // How far does the cursor have to move for it to count as a drag rather than a click?
@@ -353,6 +419,7 @@ pub struct ServoInstance {
     app: MLApp,
     browser_id: BrowserId,
     history_update: MLHistoryUpdate,
+    keyboard: MLKeyboard,
     servo: Servo<WindowInstance>,
     scroll_state: ScrollState,
     scroll_scale: TypedScale<f32, DevicePixel, LayoutPixel>,

support/magicleap/Servo2D/code/inc/Servo2D.h

@@ -61,6 +61,11 @@ class Servo2D : public lumin::LandscapeApp {
    */
   void updateHistory(bool canGoBack, const char* url, bool canGoForward);
 
+  /**
+   * Make the keyboard visible
+   */
+  void keyboardVisible(bool visible);
+
 protected:
   /**
    * Initializes the Landscape Application.
@@ -107,6 +112,7 @@ class Servo2D : public lumin::LandscapeApp {
   bool pose6DofEventListener(lumin::ControlPose6DofInputEventData* event);
   void urlBarEventListener();
   bool gestureEventListener(lumin::GestureInputEventData* event);
+  bool keyboardEventListener(const lumin::ui::KeyboardEvent::EventData& event);
 
   /**
    * Convert a point in prism coordinates to viewport coordinates

support/magicleap/Servo2D/code/src/Servo2D.cpp

@@ -9,6 +9,7 @@
 #include <ml_logging.h>
 #include <scenesGen.h>
 #include <SceneDescriptor.h>
+#include <lumin/ui/Keyboard.h>
 #include <EGL/egl.h>
 #include <GLES/gl.h>
 #include <glm/gtc/matrix_transform.hpp>
@@ -35,6 +36,9 @@ const float KEYBOARD_W = 0.666;
 // The home page
 const char* HOME_PAGE = "https://servo.org/ml-home";
 
+// The locale (currently ML only supports en)
+const lumin::ui::Locale::Code DEFAULT_LOCALE = lumin::ui::Locale::Code::kEn;
+
 // A function which calls the ML logger, suitable for passing into Servo
 typedef void (*MLLogger)(MLLogLevel lvl, char* msg);
 void logger(MLLogLevel lvl, char* msg) {
@@ -49,11 +53,18 @@ void history(Servo2D* app, bool canGoBack, char* url, bool canGoForward) {
   app->updateHistory(canGoBack, url, canGoForward);
 }
 
+// A function to show or hide the keyboard
+typedef void (*MLKeyboard)(Servo2D* app, bool visible);
+void keyboard(Servo2D* app, bool visible) {
+  app->keyboardVisible(visible);
+}
+
 // The functions Servo provides for hooking up to the ML.
 extern "C" ServoInstance* init_servo(EGLContext, EGLSurface, EGLDisplay,
-                                     Servo2D*, MLLogger, MLHistoryUpdate,
+                                     Servo2D*, MLLogger, MLHistoryUpdate, MLKeyboard,
                                      const char* url, int width, int height, float hidpi);
 extern "C" void heartbeat_servo(ServoInstance*);
+extern "C" void keyboard_servo(ServoInstance*, char32_t code, lumin::ui::KeyType keyType);
 extern "C" void trigger_servo(ServoInstance*, float x, float y, bool down);
 extern "C" void move_servo(ServoInstance*, float x, float y);
 extern "C" void traverse_servo(ServoInstance*, int delta);
@@ -95,6 +106,18 @@ int Servo2D::init() {
   lumin::ui::Cursor::SetScale(prism_, 0.03f);
   instanceInitialScenes();
 
+  // Check privileges
+  if (checkPrivilege(lumin::PrivilegeId::kInternet) != lumin::PrivilegeResult::kGranted) {
+    ML_LOG(Error, "Servo2D Failed to get internet access");
+    abort();
+    return 1;
+  }
+  if (checkPrivilege(lumin::PrivilegeId::kControllerPose) != lumin::PrivilegeResult::kGranted) {
+    ML_LOG(Error, "Servo2D Failed to get controller access");
+    abort();
+    return 1;
+  }
+
   // Get the planar resource that holds the EGL context
   lumin::RootNode* root_node = prism_->getRootNode();
   if (!root_node) {
@@ -142,7 +165,7 @@ int Servo2D::init() {
   EGLDisplay dpy = eglGetDisplay(EGL_DEFAULT_DISPLAY);
 
   // Hook into servo
-  servo_ = init_servo(ctx, surf, dpy, this, logger, history, HOME_PAGE, VIEWPORT_W, VIEWPORT_H, HIDPI);
+  servo_ = init_servo(ctx, surf, dpy, this, logger, history, keyboard, HOME_PAGE, VIEWPORT_W, VIEWPORT_H, HIDPI);
   if (!servo_) {
     ML_LOG(Error, "Servo2D Failed to init servo instance");
     abort();
@@ -351,6 +374,33 @@ void Servo2D::urlBarEventListener() {
   navigate_servo(servo_, url_bar_->getText().c_str());
 }
 
+void Servo2D::keyboardVisible(bool visible) {
+  lumin::ui::Keyboard* keys = lumin::ui::Keyboard::Get();
+  if (visible) {
+    lumin::ui::KeyboardProperties properties;
+    properties.keyboardZPosition = lumin::ui::KeyboardProperties::KeyboardZPosition::kVolumeCursorPlane;
+    properties.width = KEYBOARD_W;
+    keys->show(
+      prism_,
+      DEFAULT_LOCALE,
+      properties,
+      std::bind(&Servo2D::keyboardEventListener, this, std::placeholders::_1)
+    );
+  } else {
+    keys->hide();
+  }
+}
+
+bool Servo2D::keyboardEventListener(const lumin::ui::KeyboardEvent::EventData& event) {
+  if (event.getEventType() != lumin::ui::KeyboardEvent::EventType::KEY_PRESSED) {
+    return false;
+  }
+  const lumin::ui::KeyboardEvent::KeyPressedData* keyPress =
+    static_cast<const lumin::ui::KeyboardEvent::KeyPressedData*>(&event);
+  keyboard_servo(servo_, keyPress->getCharCode(), keyPress->getKeyType());
+  return true;
+}
+
 void Servo2D::updateHistory(bool canGoBack, const char* url, bool canGoForward) {
   back_button_->setEnabled(canGoBack);
   fwd_button_->setEnabled(canGoForward);