arc: Add display overlay with input mapping view

Display overlay is added as exo/surface overlay. Once the window is unregistered, the display overlay is removed too. The UIs still need to be polished according to the UX requirement when it's ready. This is simple version and the overlay structure. And then it can add other components like menu and education dialog. Since display overlay is added on exo/surface, rewrote some unit_tests depending on exo::test::ExoTestBase and added class to create test window with exo/surface. DisplayOverlayControllerTest*/TouchInjectorTest* Bug: b:188452744 Test: unit_tests--gtest_filter=ArcInputOverlayManagerTest*/ Change-Id: I8620cf054fe5025282b34099236971184cf2826d Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3273312 Reviewed-by: Yuichiro Hanada <yhanada@chromium.org> Auto-Submit: Cici Ruan <cuicuiruan@google.com> Commit-Queue: Cici Ruan <cuicuiruan@google.com> Cr-Commit-Position: refs/heads/main@{#948633}
chromium · Dec 6, 2021 · 5241024 · 5241024
1 parent 0d1001c
commit 5241024
Show file tree

Hide file tree

Showing 19 changed files with 626 additions and 83 deletions.
diff --git a/chrome/browser/ash/arc/input_overlay/actions/action.cc b/chrome/browser/ash/arc/input_overlay/actions/action.cc
@@ -9,6 +9,7 @@
 #include "ui/events/base_event_utils.h"
 #include "ui/events/keycodes/dom/dom_code.h"
 #include "ui/events/keycodes/dom/keycode_converter.h"
+#include "ui/views/controls/label.h"
 
 namespace arc {
 namespace input_overlay {
@@ -35,14 +36,27 @@ void LogEvent(const ui::Event& event) {
             << "}. Pointer detail {" << touch_event.pointer_details().ToString()
             << "}, TouchID {" << touch_event.pointer_details().id << "}.";
   }
-  // TODO (cuicuiruan): Add logging other events as needed.
+  // TODO(cuicuiruan): Add logging other events as needed.
 }
 
 void LogTouchEvents(const std::list<ui::TouchEvent>& events) {
   for (auto& event : events)
     LogEvent(event);
 }
 
+std::string GetDisplayText(const std::string& dom_code_string) {
+  if (base::StartsWith(dom_code_string, "Key", base::CompareCase::SENSITIVE))
+    return dom_code_string.substr(3);
+  if (base::StartsWith(dom_code_string, "Digit", base::CompareCase::SENSITIVE))
+    return dom_code_string.substr(5);
+  auto lower = base::ToLowerASCII(dom_code_string);
+  if (lower == "escape")
+    return "esc";
+  // TODO(cuicuiruan): adjust more display text according to UX design
+  // requirement.
+  return lower;
+}
+
 Action::Action(aura::Window* window) : target_window_(window) {}
 
 Action::~Action() = default;

diff --git a/chrome/browser/ash/arc/input_overlay/actions/action.h b/chrome/browser/ash/arc/input_overlay/actions/action.h
@@ -11,6 +11,7 @@
 #include <vector>
 
 #include "base/values.h"
+#include "chrome/browser/ash/arc/input_overlay/actions/action_label.h"
 #include "chrome/browser/ash/arc/input_overlay/actions/position.h"
 #include "ui/aura/window.h"
 #include "ui/aura/window_tree_host.h"
@@ -21,6 +22,10 @@ namespace input_overlay {
 // Log events for debugging.
 void LogEvent(const ui::Event& event);
 void LogTouchEvents(const std::list<ui::TouchEvent>& events);
+// TODO(cuicuiruan): Currently, it shows the dom_code.
+// Will replace it with showing the result of dom_key / keyboard key depending
+// on different keyboard layout.
+std::string GetDisplayText(const std::string& dom_code_string);
 
 // This is the base touch action which converts other events to touch
 // events for input overlay.
@@ -41,6 +46,10 @@ class Action {
   virtual bool RewriteEvent(const ui::Event& origin,
                             std::list<ui::TouchEvent>& touch_events,
                             const gfx::RectF& content_bounds) = 0;
+  // Get the UI location in the content view.
+  virtual gfx::PointF GetUIPosition(const gfx::RectF& content_bounds) = 0;
+  virtual std::unique_ptr<ActionLabel> CreateView(
+      const gfx::RectF& content_bounds) = 0;
 
   const std::string& name() { return name_; }
   const std::vector<std::unique_ptr<Position>>& locations() const {

diff --git a/chrome/browser/ash/arc/input_overlay/actions/action_label.cc b/chrome/browser/ash/arc/input_overlay/actions/action_label.cc
@@ -0,0 +1,48 @@
+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/ash/arc/input_overlay/actions/action_label.h"
+
+#include "third_party/skia/include/core/SkColor.h"
+#include "ui/gfx/color_palette.h"
+#include "ui/views/background.h"
+
+namespace arc {
+namespace input_overlay {
+
+constexpr int kWidthPadding = 10;
+constexpr int kMinimumLabelWidth = 20;
+
+ActionLabel::ActionLabel() : views::Label() {}
+ActionLabel::ActionLabel(const std::u16string& text) : views::Label(text) {
+  SetDefaultViewMode();
+}
+
+ActionLabel::~ActionLabel() = default;
+
+void ActionLabel::SetDefaultViewMode() {
+  // TODO(cuicuiruan): Replace it with required color once UI/UX specs are
+  // confirmed.
+  SetBackground(views::CreateSolidBackground(gfx::kGoogleGrey400));
+}
+
+void ActionLabel::SetPositionFromCenterPosition(gfx::PointF& center_position) {
+  auto size = GetPreferredSize();
+  SetSize(size);
+  int left = std::max(0, (int)(center_position.x() - size.width() / 2));
+  int top = std::max(0, (int)(center_position.y() - size.height() / 2));
+  // SetPosition function needs the top-left position.
+  SetPosition(gfx::Point(left, top));
+}
+
+gfx::Size ActionLabel::CalculatePreferredSize() const {
+  auto size = Label::CalculatePreferredSize();
+  size.set_width(size.width() + kWidthPadding);
+  if (size.width() < kMinimumLabelWidth)
+    size.set_width(kMinimumLabelWidth);
+  return size;
+}
+
+}  // namespace input_overlay
+}  // namespace arc
diff --git a/chrome/browser/ash/arc/input_overlay/actions/action_label.h b/chrome/browser/ash/arc/input_overlay/actions/action_label.h
@@ -0,0 +1,34 @@
+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_ASH_ARC_INPUT_OVERLAY_ACTIONS_ACTION_LABEL_H_
+#define CHROME_BROWSER_ASH_ARC_INPUT_OVERLAY_ACTIONS_ACTION_LABEL_H_
+
+#include "ui/views/controls/label.h"
+
+namespace arc {
+namespace input_overlay {
+// ActionLabel is the basic UI label for the action. It can set default view
+// mode and edit mode.
+class ActionLabel : public views::Label {
+ public:
+  ActionLabel();
+  explicit ActionLabel(const std::u16string& text);
+
+  ActionLabel(const ActionLabel&) = delete;
+  ActionLabel& operator=(const ActionLabel&) = delete;
+  ~ActionLabel() override;
+
+  // Set it with default view mode.
+  void SetDefaultViewMode();
+  // Set position from its center position.
+  void SetPositionFromCenterPosition(gfx::PointF& center_position);
+
+  // views::View:
+  gfx::Size CalculatePreferredSize() const override;
+};
+}  // namespace input_overlay
+}  // namespace arc
+
+#endif  // CHROME_BROWSER_ASH_ARC_INPUT_OVERLAY_ACTIONS_ACTION_LABEL_H_
diff --git a/chrome/browser/ash/arc/input_overlay/actions/action_move_key.cc b/chrome/browser/ash/arc/input_overlay/actions/action_move_key.cc
@@ -91,6 +91,25 @@ bool ActionMoveKey::RewriteEvent(const ui::Event& origin,
   return rewritten;
 }
 
+gfx::PointF ActionMoveKey::GetUIPosition(const gfx::RectF& content_bounds) {
+  // TODO(cuicuiruan): will update the UI position according to design specs.
+  auto* position = locations().front().get();
+  return position->CalculatePosition(content_bounds);
+}
+
+std::unique_ptr<ActionLabel> ActionMoveKey::CreateView(
+    const gfx::RectF& content_bounds) {
+  // TODO(cuicuiruan): will update the view according to design specs.
+  std::string text;
+  for (auto key : keys_) {
+    text += GetDisplayText(ui::KeycodeConverter::DomCodeToCodeString(key));
+  }
+  auto view = std::make_unique<ActionLabel>(base::UTF8ToUTF16(text));
+  auto center_pos = GetUIPosition(content_bounds);
+  view->SetPositionFromCenterPosition(center_pos);
+  return view;
+}
+
 bool ActionMoveKey::RewriteKeyEvent(const ui::KeyEvent& key_event,
                                     std::list<ui::TouchEvent>& rewritten_events,
                                     const gfx::RectF& content_bounds) {

diff --git a/chrome/browser/ash/arc/input_overlay/actions/action_move_key.h b/chrome/browser/ash/arc/input_overlay/actions/action_move_key.h
@@ -40,6 +40,9 @@ class ActionMoveKey : public Action {
   bool RewriteEvent(const ui::Event& origin,
                     std::list<ui::TouchEvent>& touch_events,
                     const gfx::RectF& content_bounds) override;
+  gfx::PointF GetUIPosition(const gfx::RectF& content_bounds) override;
+  std::unique_ptr<ActionLabel> CreateView(
+      const gfx::RectF& content_bounds) override;
 
   const std::vector<ui::DomCode>& keys() const { return keys_; }
 

diff --git a/chrome/browser/ash/arc/input_overlay/actions/action_tap_key.cc b/chrome/browser/ash/arc/input_overlay/actions/action_tap_key.cc
@@ -10,6 +10,8 @@
 #include "ui/events/base_event_utils.h"
 #include "ui/events/keycodes/dom/dom_code.h"
 #include "ui/events/keycodes/dom/keycode_converter.h"
+#include "ui/gfx/geometry/size.h"
+#include "ui/views/controls/label.h"
 
 namespace arc {
 namespace input_overlay {
@@ -111,5 +113,21 @@ bool ActionTapKey::RewriteKeyEvent(const ui::KeyEvent& key_event,
   return true;
 }
 
+gfx::PointF ActionTapKey::GetUIPosition(const gfx::RectF& content_bounds) {
+  // TODO(cuicuiruan): will update the UI position according to design specs.
+  auto* position = locations().front().get();
+  return position->CalculatePosition(content_bounds);
+}
+
+std::unique_ptr<ActionLabel> ActionTapKey::CreateView(
+    const gfx::RectF& content_bounds) {
+  // TODO(cuicuiruan): will update the view according to design specs.
+  auto text = GetDisplayText(ui::KeycodeConverter::DomCodeToCodeString(key_));
+  auto view = std::make_unique<ActionLabel>(base::UTF8ToUTF16(text));
+  auto center_pos = GetUIPosition(content_bounds);
+  view->SetPositionFromCenterPosition(center_pos);
+  return view;
+}
+
 }  // namespace input_overlay
 }  // namespace arc
diff --git a/chrome/browser/ash/arc/input_overlay/actions/action_tap_key.h b/chrome/browser/ash/arc/input_overlay/actions/action_tap_key.h
@@ -34,6 +34,9 @@ class ActionTapKey : public Action {
   bool RewriteEvent(const ui::Event& origin,
                     std::list<ui::TouchEvent>& touch_events,
                     const gfx::RectF& content_bounds) override;
+  gfx::PointF GetUIPosition(const gfx::RectF& content_bounds) override;
+  std::unique_ptr<ActionLabel> CreateView(
+      const gfx::RectF& content_bounds) override;
 
   ui::DomCode key() { return key_; }
 

diff --git a/chrome/browser/ash/arc/input_overlay/arc_input_overlay_manager.cc b/chrome/browser/ash/arc/input_overlay/arc_input_overlay_manager.cc
@@ -138,16 +138,19 @@ void ArcInputOverlayManager::RemoveObserverFromInputMethod() {
   input_method_ = nullptr;
 }
 
-void ArcInputOverlayManager::RegisterWindow(aura::Window* top_level_window) {
-  if (!top_level_window || registered_top_level_window_ == top_level_window)
+void ArcInputOverlayManager::RegisterWindow(aura::Window* window) {
+  if (!window || window != window->GetToplevelWindow() ||
+      registered_top_level_window_ == window) {
     return;
-  auto it = input_overlay_enabled_windows_.find(top_level_window);
+  }
+  auto it = input_overlay_enabled_windows_.find(window);
   if (it == input_overlay_enabled_windows_.end())
     return;
   DCHECK(!registered_top_level_window_);
   it->second->RegisterEventRewriter();
-  registered_top_level_window_ = top_level_window;
+  registered_top_level_window_ = window;
   AddObserverToInputMethod();
+  AddDisplayOverlayController();
   // If the window is on the extended window, it turns out only primary root
   // window catches the key event. So it needs to forward the key event from
   // primary root window to extended root window event source.
@@ -158,20 +161,36 @@ void ArcInputOverlayManager::RegisterWindow(aura::Window* top_level_window) {
   }
 }
 
-void ArcInputOverlayManager::UnRegisterWindow(aura::Window* top_level_window) {
-  if (!registered_top_level_window_ ||
-      registered_top_level_window_ != top_level_window) {
+void ArcInputOverlayManager::UnRegisterWindow(aura::Window* window) {
+  if (!registered_top_level_window_ || registered_top_level_window_ != window)
     return;
-  }
   auto it = input_overlay_enabled_windows_.find(registered_top_level_window_);
   DCHECK(it != input_overlay_enabled_windows_.end());
   if (it == input_overlay_enabled_windows_.end())
     return;
   if (key_event_source_rewriter_)
     key_event_source_rewriter_.reset();
   it->second->UnRegisterEventRewriter();
-  registered_top_level_window_ = nullptr;
+  RemoveDisplayOverlayController();
   RemoveObserverFromInputMethod();
+  registered_top_level_window_ = nullptr;
+}
+
+void ArcInputOverlayManager::AddDisplayOverlayController() {
+  if (!registered_top_level_window_)
+    return;
+  DCHECK(!display_overlay_controller_);
+  auto it = input_overlay_enabled_windows_.find(registered_top_level_window_);
+  DCHECK(it != input_overlay_enabled_windows_.end());
+  display_overlay_controller_ =
+      std::make_unique<DisplayOverlayController>(it->second.get());
+}
+
+void ArcInputOverlayManager::RemoveDisplayOverlayController() {
+  if (!registered_top_level_window_)
+    return;
+  DCHECK(display_overlay_controller_);
+  display_overlay_controller_.reset();
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -211,7 +230,7 @@ void ArcInputOverlayManager::OnWindowDestroying(aura::Window* window) {
 void ArcInputOverlayManager::OnWindowAddedToRootWindow(aura::Window* window) {
   if (!window)
     return;
-  RegisterWindow(window->GetToplevelWindow());
+  RegisterWindow(window);
 }
 
 void ArcInputOverlayManager::OnWindowRemovingFromRootWindow(
@@ -224,6 +243,17 @@ void ArcInputOverlayManager::OnWindowRemovingFromRootWindow(
   UnRegisterWindow(window);
 }
 
+void ArcInputOverlayManager::OnWindowBoundsChanged(
+    aura::Window* window,
+    const gfx::Rect& old_bounds,
+    const gfx::Rect& new_bounds,
+    ui::PropertyChangeReason reason) {
+  if (!window || window != registered_top_level_window_)
+    return;
+  if (display_overlay_controller_)
+    display_overlay_controller_->OnWindowBoundsChanged();
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // KeyedService:
 void ArcInputOverlayManager::Shutdown() {

diff --git a/chrome/browser/ash/arc/input_overlay/arc_input_overlay_manager.h b/chrome/browser/ash/arc/input_overlay/arc_input_overlay_manager.h
@@ -9,6 +9,7 @@
 #include "base/scoped_multi_source_observation.h"
 #include "base/scoped_observation.h"
 #include "base/strings/string_util.h"
+#include "chrome/browser/ash/arc/input_overlay/display_overlay_controller.h"
 #include "chrome/browser/ash/arc/input_overlay/key_event_source_rewriter.h"
 #include "chrome/browser/ash/arc/input_overlay/touch_injector.h"
 #include "components/keyed_service/core/keyed_service.h"
@@ -56,6 +57,10 @@ class ArcInputOverlayManager : public KeyedService,
   void OnWindowAddedToRootWindow(aura::Window* window) override;
   void OnWindowRemovingFromRootWindow(aura::Window* window,
                                       aura::Window* new_root) override;
+  void OnWindowBoundsChanged(aura::Window* window,
+                             const gfx::Rect& old_bounds,
+                             const gfx::Rect& new_bounds,
+                             ui::PropertyChangeReason reason) override;
 
   // KeyedService overrides:
   void Shutdown() override;
@@ -84,14 +89,18 @@ class ArcInputOverlayManager : public KeyedService,
   // each time.
   aura::Window* registered_top_level_window_ = nullptr;
   std::unique_ptr<KeyEventSourceRewriter> key_event_source_rewriter_;
+  std::unique_ptr<DisplayOverlayController> display_overlay_controller_;
 
   void ReadData(const std::string& package_name,
                 aura::Window* top_level_window);
   void NotifyTextInputState();
   void AddObserverToInputMethod();
   void RemoveObserverFromInputMethod();
-  void RegisterWindow(aura::Window* top_level_window);
-  void UnRegisterWindow(aura::Window* top_level_window);
+  // Only top level window will be registered successfully.
+  void RegisterWindow(aura::Window* window);
+  void UnRegisterWindow(aura::Window* window);
+  void AddDisplayOverlayController();
+  void RemoveDisplayOverlayController();
 
   base::WeakPtrFactory<ArcInputOverlayManager> weak_ptr_factory_{this};
 };