Implement delayed key event synthesis for Windows (#23524)

This changes the Windows text handling so that keyboard events are sent to the framework first for handling, and then passed to the text input plugin, so that the framework has a chance to handle keys before they get given to the text field.

This is complicated by the async nature of the interaction with the framework, since Windows wants a synchronous response. So, in this change, I always tell Windows that the event was handled, and if the framework (eventually) responds that it wasn't, then I synthesize a new event and send it with SendEvent.

I also added support for detecting "extended" keys, since that was missing, and converted the OnKey handlers in the API to return a bool to indicate whether or not they have handled the event.

ci/licenses_golden/licenses_flutter

@@ -1475,6 +1475,7 @@ FILE: ../../../flutter/shell/platform/windows/flutter_windows_win32.cc
 FILE: ../../../flutter/shell/platform/windows/flutter_windows_winuwp.cc
 FILE: ../../../flutter/shell/platform/windows/key_event_handler.cc
 FILE: ../../../flutter/shell/platform/windows/key_event_handler.h
+FILE: ../../../flutter/shell/platform/windows/key_event_handler_unittests.cc
 FILE: ../../../flutter/shell/platform/windows/keyboard_hook_handler.h
 FILE: ../../../flutter/shell/platform/windows/platform_handler.cc
 FILE: ../../../flutter/shell/platform/windows/platform_handler.h

shell/platform/windows/BUILD.gn

@@ -167,11 +167,14 @@ if (target_os == "winuwp") {
       "flutter_project_bundle_unittests.cc",
       "flutter_windows_engine_unittests.cc",
       "flutter_windows_texture_registrar_unittests.cc",
+      "key_event_handler_unittests.cc",
       "string_conversion_unittests.cc",
       "system_utils_unittests.cc",
       "testing/engine_embedder_api_modifier.h",
       "testing/mock_win32_window.cc",
       "testing/mock_win32_window.h",
+      "testing/mock_window_binding_handler.cc",
+      "testing/mock_window_binding_handler.h",
       "testing/win32_flutter_window_test.cc",
       "testing/win32_flutter_window_test.h",
       "win32_dpi_utils_unittests.cc",

shell/platform/windows/flutter_windows_engine_unittests.cc

@@ -102,7 +102,7 @@ TEST(FlutterWindowsEngine, SendPlatformMessageWithoutResponse) {
   const char* channel = "test";
   const std::vector<uint8_t> test_message = {1, 2, 3, 4};
 
-  // Without a respones, SendPlatformMessage should be a simple passthrough.
+  // Without a response, SendPlatformMessage should be a simple pass-through.
   bool called = false;
   modifier.embedder_api().SendPlatformMessage = MOCK_ENGINE_PROC(
       SendPlatformMessage, ([&called, test_message](auto engine, auto message) {

shell/platform/windows/flutter_windows_view.cc

@@ -35,10 +35,7 @@ void FlutterWindowsView::SetEngine(
 
   // Set up the system channel handlers.
   auto internal_plugin_messenger = internal_plugin_registrar_->messenger();
-  keyboard_hook_handlers_.push_back(
-      std::make_unique<flutter::KeyEventHandler>(internal_plugin_messenger));
-  keyboard_hook_handlers_.push_back(std::make_unique<flutter::TextInputPlugin>(
-      internal_plugin_messenger, this));
+  RegisterKeyboardHookHandlers(internal_plugin_messenger);
   platform_handler_ = PlatformHandler::Create(internal_plugin_messenger, this);
   cursor_handler_ = std::make_unique<flutter::CursorHandler>(
       internal_plugin_messenger, binding_handler_.get());
@@ -49,6 +46,18 @@ void FlutterWindowsView::SetEngine(
                     binding_handler_->GetDpiScale());
 }
 
+void FlutterWindowsView::RegisterKeyboardHookHandlers(
+    flutter::BinaryMessenger* messenger) {
+  AddKeyboardHookHandler(std::make_unique<flutter::KeyEventHandler>(messenger));
+  AddKeyboardHookHandler(
+      std::make_unique<flutter::TextInputPlugin>(messenger, this));
+}
+
+void FlutterWindowsView::AddKeyboardHookHandler(
+    std::unique_ptr<flutter::KeyboardHookHandler> handler) {
+  keyboard_hook_handlers_.push_back(std::move(handler));
+}
+
 uint32_t FlutterWindowsView::GetFrameBufferId(size_t width, size_t height) {
   // Called on an engine-controlled (non-platform) thread.
   std::unique_lock<std::mutex> lock(resize_mutex_);
@@ -120,11 +129,12 @@ void FlutterWindowsView::OnText(const std::u16string& text) {
   SendText(text);
 }
 
-void FlutterWindowsView::OnKey(int key,
+bool FlutterWindowsView::OnKey(int key,
                                int scancode,
                                int action,
-                               char32_t character) {
-  SendKey(key, scancode, action, character);
+                               char32_t character,
+                               bool extended) {
+  return SendKey(key, scancode, action, character, extended);
 }
 
 void FlutterWindowsView::OnScroll(double x,
@@ -215,13 +225,19 @@ void FlutterWindowsView::SendText(const std::u16string& text) {
   }
 }
 
-void FlutterWindowsView::SendKey(int key,
+bool FlutterWindowsView::SendKey(int key,
                                  int scancode,
                                  int action,
-                                 char32_t character) {
+                                 char32_t character,
+                                 bool extended) {
   for (const auto& handler : keyboard_hook_handlers_) {
-    handler->KeyboardHook(this, key, scancode, action, character);
+    if (handler->KeyboardHook(this, key, scancode, action, character,
+                              extended)) {
+      // key event was handled, so don't send to other handlers.
+      return true;
+    }
   }
+  return false;
 }
 
 void FlutterWindowsView::SendScroll(double x,

shell/platform/windows/flutter_windows_view.h

@@ -38,14 +38,14 @@ inline constexpr uint32_t kWindowFrameBufferID = 0;
 class FlutterWindowsView : public WindowBindingHandlerDelegate,
                            public TextInputPluginDelegate {
  public:
-  // Creates a FlutterWindowsView with the given implementator of
+  // Creates a FlutterWindowsView with the given implementor of
   // WindowBindingHandler.
   //
   // In order for object to render Flutter content the SetEngine method must be
   // called with a valid FlutterWindowsEngine instance.
   FlutterWindowsView(std::unique_ptr<WindowBindingHandler> window_binding);
 
-  ~FlutterWindowsView();
+  virtual ~FlutterWindowsView();
 
   // Configures the window instance with an instance of a running Flutter
   // engine.
@@ -100,7 +100,11 @@ class FlutterWindowsView : public WindowBindingHandlerDelegate,
   void OnText(const std::u16string&) override;
 
   // |WindowBindingHandlerDelegate|
-  void OnKey(int key, int scancode, int action, char32_t character) override;
+  bool OnKey(int key,
+             int scancode,
+             int action,
+             char32_t character,
+             bool extended) override;
 
   // |WindowBindingHandlerDelegate|
   void OnScroll(double x,
@@ -112,6 +116,15 @@ class FlutterWindowsView : public WindowBindingHandlerDelegate,
   // |TextInputPluginDelegate|
   void OnCursorRectUpdated(const Rect& rect) override;
 
+ protected:
+  // Called to create the keyboard hook handlers.
+  virtual void RegisterKeyboardHookHandlers(
+      flutter::BinaryMessenger* messenger);
+
+  // Used by RegisterKeyboardHookHandlers to add a new keyboard hook handler.
+  void AddKeyboardHookHandler(
+      std::unique_ptr<flutter::KeyboardHookHandler> handler);
+
  private:
   // Struct holding the mouse state. The engine doesn't keep track of which
   // mouse buttons have been pressed, so it's the embedding's responsibility.
@@ -166,7 +179,11 @@ class FlutterWindowsView : public WindowBindingHandlerDelegate,
   void SendText(const std::u16string&);
 
   // Reports a raw keyboard message to Flutter engine.
-  void SendKey(int key, int scancode, int action, char32_t character);
+  bool SendKey(int key,
+               int scancode,
+               int action,
+               char32_t character,
+               bool extended);
 
   // Reports scroll wheel events to Flutter engine.
   void SendScroll(double x,

shell/platform/windows/key_event_handler.cc

@@ -10,6 +10,10 @@
 
 #include "flutter/shell/platform/common/cpp/json_message_codec.h"
 
+namespace flutter {
+
+namespace {
+
 static constexpr char kChannelName[] = "flutter/keyevent";
 
 static constexpr char kKeyCodeKey[] = "keyCode";
@@ -18,33 +22,35 @@ static constexpr char kCharacterCodePointKey[] = "characterCodePoint";
 static constexpr char kModifiersKey[] = "modifiers";
 static constexpr char kKeyMapKey[] = "keymap";
 static constexpr char kTypeKey[] = "type";
+static constexpr char kHandledKey[] = "handled";
 
 static constexpr char kWindowsKeyMap[] = "windows";
 static constexpr char kKeyUp[] = "keyup";
 static constexpr char kKeyDown[] = "keydown";
 
-namespace flutter {
+// The maximum number of pending events to keep before
+// emitting a warning on the console about unhandled events.
+static constexpr int kMaxPendingEvents = 1000;
 
 // Re-definition of the modifiers for compatibility with the Flutter framework.
 // These have to be in sync with the framework's RawKeyEventDataWindows
 // modifiers definition.
 // https://github.com/flutter/flutter/blob/19ff596979e407c484a32f4071420fca4f4c885f/packages/flutter/lib/src/services/raw_keyboard_windows.dart#L203
-const int kShift = 1 << 0;
-const int kShiftLeft = 1 << 1;
-const int kShiftRight = 1 << 2;
-const int kControl = 1 << 3;
-const int kControlLeft = 1 << 4;
-const int kControlRight = 1 << 5;
-const int kAlt = 1 << 6;
-const int kAltLeft = 1 << 7;
-const int kAltRight = 1 << 8;
-const int kWinLeft = 1 << 9;
-const int kWinRight = 1 << 10;
-const int kCapsLock = 1 << 11;
-const int kNumLock = 1 << 12;
-const int kScrollLock = 1 << 13;
+static constexpr int kShift = 1 << 0;
+static constexpr int kShiftLeft = 1 << 1;
+static constexpr int kShiftRight = 1 << 2;
+static constexpr int kControl = 1 << 3;
+static constexpr int kControlLeft = 1 << 4;
+static constexpr int kControlRight = 1 << 5;
+static constexpr int kAlt = 1 << 6;
+static constexpr int kAltLeft = 1 << 7;
+static constexpr int kAltRight = 1 << 8;
+static constexpr int kWinLeft = 1 << 9;
+static constexpr int kWinRight = 1 << 10;
+static constexpr int kCapsLock = 1 << 11;
+static constexpr int kNumLock = 1 << 12;
+static constexpr int kScrollLock = 1 << 13;
 
-namespace {
 /// Calls GetKeyState() an all modifier keys and packs the result in an int,
 /// with the re-defined values declared above for compatibility with the Flutter
 /// framework.
@@ -81,25 +87,130 @@ int GetModsForKeyState() {
     mods |= kScrollLock;
   return mods;
 }
+
+// This uses event data instead of generating a serial number because
+// information can't be attached to the redispatched events, so it has to be
+// possible to compute an ID from the identifying data in the event when it is
+// received again in order to differentiate between events that are new, and
+// events that have been redispatched.
+//
+// Another alternative would be to compute a checksum from all the data in the
+// event (just compute it over the bytes in the struct, probably skipping
+// timestamps), but the fields used below are enough to differentiate them, and
+// since Windows does some processing on the events (coming up with virtual key
+// codes, setting timestamps, etc.), it's not clear that the redispatched
+// events would have the same checksums.
+uint64_t CalculateEventId(int scancode, int action, bool extended) {
+  // Calculate a key event ID based on the scan code of the key pressed,
+  // and the flags we care about.
+  return scancode | (((action == WM_KEYUP ? KEYEVENTF_KEYUP : 0x0) |
+                      (extended ? KEYEVENTF_EXTENDEDKEY : 0x0))
+                     << 16);
+}
+
 }  // namespace
 
-KeyEventHandler::KeyEventHandler(flutter::BinaryMessenger* messenger)
+KeyEventHandler::KeyEventHandler(flutter::BinaryMessenger* messenger,
+                                 KeyEventHandler::SendInputDelegate send_input)
     : channel_(
           std::make_unique<flutter::BasicMessageChannel<rapidjson::Document>>(
               messenger,
               kChannelName,
-              &flutter::JsonMessageCodec::GetInstance())) {}
+              &flutter::JsonMessageCodec::GetInstance())),
+      send_input_(send_input) {
+  assert(send_input != nullptr);
+}
 
 KeyEventHandler::~KeyEventHandler() = default;
 
 void KeyEventHandler::TextHook(FlutterWindowsView* view,
                                const std::u16string& code_point) {}
 
-void KeyEventHandler::KeyboardHook(FlutterWindowsView* view,
+KEYBDINPUT* KeyEventHandler::FindPendingEvent(uint64_t id) {
+  if (pending_events_.empty()) {
+    return nullptr;
+  }
+  for (auto iter = pending_events_.begin(); iter != pending_events_.end();
+       ++iter) {
+    if (iter->first == id) {
+      return &iter->second;
+    }
+  }
+  return nullptr;
+}
+
+void KeyEventHandler::RemovePendingEvent(uint64_t id) {
+  for (auto iter = pending_events_.begin(); iter != pending_events_.end();
+       ++iter) {
+    if (iter->first == id) {
+      pending_events_.erase(iter);
+      return;
+    }
+  }
+  std::cerr << "Tried to remove pending event with id " << id
+            << ", but the event was not found." << std::endl;
+}
+
+void KeyEventHandler::AddPendingEvent(uint64_t id,
+                                      int scancode,
+                                      int action,
+                                      bool extended) {
+  if (pending_events_.size() > kMaxPendingEvents) {
+    std::cerr
+        << "There are " << pending_events_.size()
+        << " keyboard events that have not yet received a response from the "
+        << "framework. Are responses being sent?" << std::endl;
+  }
+  KEYBDINPUT key_event = KEYBDINPUT{0};
+  key_event.wScan = scancode;
+  key_event.dwFlags = KEYEVENTF_SCANCODE |
+                      (extended ? KEYEVENTF_EXTENDEDKEY : 0x0) |
+                      (action == WM_KEYUP ? KEYEVENTF_KEYUP : 0x0);
+  pending_events_.push_back(std::make_pair(id, key_event));
+}
+
+void KeyEventHandler::HandleResponse(bool handled,
+                                     uint64_t id,
+                                     int action,
+                                     bool extended,
+                                     int scancode,
+                                     int character) {
+  if (handled) {
+    this->RemovePendingEvent(id);
+  } else {
+    // Since the framework didn't handle the event, we inject a newly
+    // synthesized one. We let Windows figure out the virtual key and
+    // character for the given scancode, as well as a new timestamp.
+    const KEYBDINPUT* key_event = this->FindPendingEvent(id);
+    if (key_event == nullptr) {
+      std::cerr << "Unable to find event " << id << " in pending events queue.";
+      return;
+    }
+    INPUT input_event;
+    input_event.type = INPUT_KEYBOARD;
+    input_event.ki = *key_event;
+    UINT accepted = send_input_(1, &input_event, sizeof(input_event));
+    if (accepted != 1) {
+      std::cerr << "Unable to synthesize event for unhandled keyboard event "
+                   "with scancode "
+                << scancode << " (character " << character << ")" << std::endl;
+    }
+  }
+}
+
+bool KeyEventHandler::KeyboardHook(FlutterWindowsView* view,
                                    int key,
                                    int scancode,
                                    int action,
-                                   char32_t character) {
+                                   char32_t character,
+                                   bool extended) {
+  const uint64_t id = CalculateEventId(scancode, action, extended);
+  if (FindPendingEvent(id) != nullptr) {
+    // Don't pass messages that we synthesized to the framework again.
+    RemovePendingEvent(id);
+    return false;
+  }
+
   // TODO: Translate to a cross-platform key code system rather than passing
   // the native key code.
   rapidjson::Document event(rapidjson::kObjectType);
@@ -119,9 +230,17 @@ void KeyEventHandler::KeyboardHook(FlutterWindowsView* view,
       break;
     default:
       std::cerr << "Unknown key event action: " << action << std::endl;
-      return;
+      return false;
   }
-  channel_->Send(event);
+  AddPendingEvent(id, scancode, action, extended);
+  channel_->Send(event, [this, id, action, extended, scancode, character](
+                            const uint8_t* reply, size_t reply_size) {
+    auto decoded = flutter::JsonMessageCodec::GetInstance().DecodeMessage(
+        reply, reply_size);
+    bool handled = (*decoded)[kHandledKey].GetBool();
+    this->HandleResponse(handled, id, action, extended, scancode, character);
+  });
+  return true;
 }
 
 }  // namespace flutter