LibWebView+WebContent: Propagate unconsumed input events out of OOPWV

Since 9e2bd9d, the OOPWV has been
consuming all mouse and keyboard events, preventing action shortcuts
from working. So let's fix that. :^)

OOPWV now queues up input events, sending them one at a time to the
WebContent process and waiting for the new
`did_finish_handling_input_event(bool event_was_accepted) =|` IPC call
before sending the next one. If the event was not accepted, OOPWV
imitates the usual event bubbling: first passing the event to its
superclass, then to its parent widget, and finally propagating to any
Action shortcuts.

With this, shortcuts like Ctrl+I to open Browser's JS console work
again, except when a contenteditable field is selected. That's a
whole separate stack of yaks.

Co-authored-by: Zaggy1024 <zaggy1024@gmail.com>

Author: 2 people

Userland/Libraries/LibWebView/OutOfProcessWebView.cpp

@@ -158,37 +158,37 @@ void OutOfProcessWebView::handle_resize()
 
 void OutOfProcessWebView::keydown_event(GUI::KeyEvent& event)
 {
-    client().async_key_down(event.key(), event.modifiers(), event.code_point());
+    enqueue_input_event(event);
 }
 
 void OutOfProcessWebView::keyup_event(GUI::KeyEvent& event)
 {
-    client().async_key_up(event.key(), event.modifiers(), event.code_point());
+    enqueue_input_event(event);
 }
 
 void OutOfProcessWebView::mousedown_event(GUI::MouseEvent& event)
 {
-    client().async_mouse_down(to_content_position(event.position()), event.button(), event.buttons(), event.modifiers());
+    enqueue_input_event(event);
 }
 
 void OutOfProcessWebView::mouseup_event(GUI::MouseEvent& event)
 {
-    client().async_mouse_up(to_content_position(event.position()), event.button(), event.buttons(), event.modifiers());
+    enqueue_input_event(event);
 }
 
 void OutOfProcessWebView::mousemove_event(GUI::MouseEvent& event)
 {
-    client().async_mouse_move(to_content_position(event.position()), event.button(), event.buttons(), event.modifiers());
+    enqueue_input_event(event);
 }
 
 void OutOfProcessWebView::mousewheel_event(GUI::MouseEvent& event)
 {
-    client().async_mouse_wheel(to_content_position(event.position()), event.button(), event.buttons(), event.modifiers(), event.wheel_delta_x(), event.wheel_delta_y());
+    enqueue_input_event(event);
 }
 
 void OutOfProcessWebView::doubleclick_event(GUI::MouseEvent& event)
 {
-    client().async_doubleclick(to_content_position(event.position()), event.button(), event.buttons(), event.modifiers());
+    enqueue_input_event(event);
 }
 
 void OutOfProcessWebView::theme_change_event(GUI::ThemeChangeEvent& event)
@@ -695,4 +695,133 @@ void OutOfProcessWebView::hide_event(GUI::HideEvent&)
     set_system_visibility_state(false);
 }
 
+void OutOfProcessWebView::enqueue_input_event(InputEvent const& event)
+{
+    m_pending_input_events.enqueue(event);
+    process_next_input_event();
+}
+
+void OutOfProcessWebView::process_next_input_event()
+{
+    if (m_pending_input_events.is_empty())
+        return;
+
+    if (m_is_awaiting_response_for_input_event)
+        return;
+    m_is_awaiting_response_for_input_event = true;
+
+    // Send the next event over to the web content to be handled by JS.
+    // We'll later get a message to say whether JS prevented the default event behavior,
+    // at which point we either discard or handle that event, then try and process the next one.
+    auto event = m_pending_input_events.head();
+    event.visit(
+        [this](GUI::KeyEvent const& event) {
+            switch (event.type()) {
+            case GUI::Event::Type::KeyDown:
+                client().async_key_down(event.key(), event.modifiers(), event.code_point());
+                break;
+            case GUI::Event::Type::KeyUp:
+                client().async_key_up(event.key(), event.modifiers(), event.code_point());
+                break;
+            default:
+                dbgln("Unrecognized key event type in OOPWV input event queue: {}", event.type());
+                VERIFY_NOT_REACHED();
+            }
+        },
+        [this](GUI::MouseEvent const& event) {
+            switch (event.type()) {
+            case GUI::Event::Type::MouseDown:
+                client().async_mouse_down(to_content_position(event.position()), event.button(), event.buttons(), event.modifiers());
+                break;
+            case GUI::Event::Type::MouseUp:
+                client().async_mouse_up(to_content_position(event.position()), event.button(), event.buttons(), event.modifiers());
+                break;
+            case GUI::Event::Type::MouseMove:
+                client().async_mouse_move(to_content_position(event.position()), event.button(), event.buttons(), event.modifiers());
+                break;
+            case GUI::Event::Type::MouseWheel:
+                client().async_mouse_wheel(to_content_position(event.position()), event.button(), event.buttons(), event.modifiers(), event.wheel_delta_x(), event.wheel_delta_y());
+                break;
+            case GUI::Event::Type::MouseDoubleClick:
+                client().async_doubleclick(to_content_position(event.position()), event.button(), event.buttons(), event.modifiers());
+                break;
+            default:
+                dbgln("Unrecognized mouse event type in OOPWV input event queue: {}", event.type());
+                VERIFY_NOT_REACHED();
+            }
+        });
+}
+
+void OutOfProcessWebView::notify_server_did_finish_handling_input_event(bool event_was_accepted)
+{
+    VERIFY(m_is_awaiting_response_for_input_event);
+
+    auto event = m_pending_input_events.dequeue();
+    m_is_awaiting_response_for_input_event = false;
+
+    if (!event_was_accepted) {
+        // Here we handle events that were not consumed or cancelled by web content.
+        // That is, we manually implement the steps that would have happened if the original
+        // OutOfProcessWebView::foo_event() had called event.ignore().
+        //
+        // The first step is to give our superclass a chance to handle the event.
+        //
+        // Then, if it does not, we dispatch the event to our parent widget, but limited so
+        // that it will never bubble up to the Window. (Otherwise, it would then dispatch the
+        // event to us since we are the focused widget, and it would go round indefinitely.)
+        //
+        // Finally, any unhandled KeyDown events are propagated to trigger any Actions.
+        event.visit(
+            [this](GUI::KeyEvent& event) {
+                switch (event.type()) {
+                case GUI::Event::Type::KeyDown:
+                    Super::keydown_event(event);
+                    break;
+                case GUI::Event::Type::KeyUp:
+                    Super::keyup_event(event);
+                    break;
+                default:
+                    dbgln("Unhandled key event type in OOPWV input event queue: {}", event.type());
+                    VERIFY_NOT_REACHED();
+                }
+
+                if (!event.is_accepted()) {
+                    parent_widget()->dispatch_event(event, window());
+
+                    // NOTE: If other events can ever trigger shortcuts, propagate those here.
+                    if (!event.is_accepted() && event.type() == GUI::Event::Type::KeyDown)
+                        window()->propagate_shortcuts_up_to_application(event, this);
+                }
+            },
+            [this](GUI::MouseEvent& event) {
+                switch (event.type()) {
+                case GUI::Event::Type::MouseDown:
+                    Super::mousedown_event(event);
+                    break;
+                case GUI::Event::Type::MouseUp:
+                    Super::mouseup_event(event);
+                    break;
+                case GUI::Event::Type::MouseMove:
+                    Super::mousemove_event(event);
+                    break;
+                case GUI::Event::Type::MouseWheel:
+                    Super::mousewheel_event(event);
+                    break;
+                case GUI::Event::Type::MouseDoubleClick:
+                    Super::doubleclick_event(event);
+                    break;
+                default:
+                    dbgln("Unhandled mouse event type in OOPWV input event queue: {}", event.type());
+                    VERIFY_NOT_REACHED();
+                }
+
+                if (!event.is_accepted())
+                    parent_widget()->dispatch_event(event, window());
+                // FIXME: Propagate event for mouse-button shortcuts once that is implemented.
+            });
+    }
+
+    process_next_input_event();
+}
+
 }

Userland/Libraries/LibWebView/OutOfProcessWebView.h

@@ -1,11 +1,13 @@
 /*
  * Copyright (c) 2020-2022, Andreas Kling <kling@serenityos.org>
+ * Copyright (c) 2022, Sam Atkins <atkinssj@serenityos.org>
  *
  * SPDX-License-Identifier: BSD-2-Clause
  */
 
 #pragma once
 
+#include <AK/Queue.h>
 #include <AK/URL.h>
 #include <LibGUI/AbstractScrollableWidget.h>
 #include <LibGUI/Widget.h>
@@ -182,6 +184,7 @@ class OutOfProcessWebView final
     virtual Gfx::IntRect notify_server_did_request_minimize_window() override;
     virtual Gfx::IntRect notify_server_did_request_fullscreen_window() override;
     virtual void notify_server_did_request_file(Badge<WebContentClient>, String const& path, i32) override;
+    virtual void notify_server_did_finish_handling_input_event(bool event_was_accepted) override;
 
     void request_repaint();
     void handle_resize();
@@ -191,6 +194,10 @@ class OutOfProcessWebView final
 
     void handle_web_content_process_crash();
 
+    using InputEvent = Variant<GUI::KeyEvent, GUI::MouseEvent>;
+    void enqueue_input_event(InputEvent const&);
+    void process_next_input_event();
+
     AK::URL m_url;
 
     struct SharedBitmap {
@@ -210,6 +217,9 @@ class OutOfProcessWebView final
 
     RefPtr<Gfx::Bitmap> m_backup_bitmap;
     RefPtr<GUI::Dialog> m_dialog;
+
+    bool m_is_awaiting_response_for_input_event { false };
+    Queue<InputEvent> m_pending_input_events;
 };
 
 }

Userland/Libraries/LibWebView/ViewImplementation.h

@@ -66,6 +66,7 @@ class ViewImplementation {
     virtual Gfx::IntRect notify_server_did_request_minimize_window() = 0;
     virtual Gfx::IntRect notify_server_did_request_fullscreen_window() = 0;
     virtual void notify_server_did_request_file(Badge<WebContentClient>, String const& path, i32) = 0;
+    virtual void notify_server_did_finish_handling_input_event(bool event_was_accepted) = 0;
 };
 
 }

Userland/Libraries/LibWebView/WebContentClient.cpp

@@ -280,4 +280,9 @@ void WebContentClient::did_request_file(String const& path, i32 request_id)
     m_view.notify_server_did_request_file({}, path, request_id);
 }
 
+void WebContentClient::did_finish_handling_input_event(bool event_was_accepted)
+{
+    m_view.notify_server_did_finish_handling_input_event(event_was_accepted);
+}
+
 }

Userland/Libraries/LibWebView/WebContentClient.h

@@ -76,6 +76,7 @@ class WebContentClient final
     virtual Messages::WebContentClient::DidRequestMinimizeWindowResponse did_request_minimize_window() override;
     virtual Messages::WebContentClient::DidRequestFullscreenWindowResponse did_request_fullscreen_window() override;
     virtual void did_request_file(String const& path, i32) override;
+    virtual void did_finish_handling_input_event(bool event_was_accepted) override;
 
     ViewImplementation& m_view;
 };

Userland/Services/WebContent/ConnectionFromClient.cpp

@@ -157,37 +157,42 @@ void ConnectionFromClient::flush_pending_paint_requests()
 
 void ConnectionFromClient::mouse_down(Gfx::IntPoint const& position, unsigned int button, unsigned int buttons, unsigned int modifiers)
 {
-    page().handle_mousedown(position, button, buttons, modifiers);
+    report_finished_handling_input_event(page().handle_mousedown(position, button, buttons, modifiers));
 }
 
 void ConnectionFromClient::mouse_move(Gfx::IntPoint const& position, [[maybe_unused]] unsigned int button, unsigned int buttons, unsigned int modifiers)
 {
-    page().handle_mousemove(position, buttons, modifiers);
+    report_finished_handling_input_event(page().handle_mousemove(position, buttons, modifiers));
 }
 
 void ConnectionFromClient::mouse_up(Gfx::IntPoint const& position, unsigned int button, unsigned int buttons, unsigned int modifiers)
 {
-    page().handle_mouseup(position, button, buttons, modifiers);
+    report_finished_handling_input_event(page().handle_mouseup(position, button, buttons, modifiers));
 }
 
 void ConnectionFromClient::mouse_wheel(Gfx::IntPoint const& position, unsigned int button, unsigned int buttons, unsigned int modifiers, i32 wheel_delta_x, i32 wheel_delta_y)
 {
-    page().handle_mousewheel(position, button, buttons, modifiers, wheel_delta_x, wheel_delta_y);
+    report_finished_handling_input_event(page().handle_mousewheel(position, button, buttons, modifiers, wheel_delta_x, wheel_delta_y));
 }
 
 void ConnectionFromClient::doubleclick(Gfx::IntPoint const& position, unsigned int button, unsigned int buttons, unsigned int modifiers)
 {
-    page().handle_doubleclick(position, button, buttons, modifiers);
+    report_finished_handling_input_event(page().handle_doubleclick(position, button, buttons, modifiers));
 }
 
 void ConnectionFromClient::key_down(i32 key, unsigned int modifiers, u32 code_point)
 {
-    page().handle_keydown((KeyCode)key, modifiers, code_point);
+    report_finished_handling_input_event(page().handle_keydown((KeyCode)key, modifiers, code_point));
 }
 
 void ConnectionFromClient::key_up(i32 key, unsigned int modifiers, u32 code_point)
 {
-    page().handle_keyup((KeyCode)key, modifiers, code_point);
+    report_finished_handling_input_event(page().handle_keyup((KeyCode)key, modifiers, code_point));
+}
+
+void ConnectionFromClient::report_finished_handling_input_event(bool event_was_handled)
+{
+    async_did_finish_handling_input_event(event_was_handled);
 }
 
 void ConnectionFromClient::debug_request(String const& request, String const& argument)

Userland/Services/WebContent/ConnectionFromClient.h

@@ -98,6 +98,8 @@ class ConnectionFromClient final
 
     void flush_pending_paint_requests();
 
+    void report_finished_handling_input_event(bool event_was_handled);
+
     NonnullOwnPtr<PageHost> m_page_host;
     struct PaintRequest {
         Gfx::IntRect content_rect;

Userland/Services/WebContent/WebContentClient.ipc

@@ -52,6 +52,7 @@ endpoint WebContentClient
     did_request_minimize_window() => (Gfx::IntRect window_rect)
     did_request_fullscreen_window() => (Gfx::IntRect window_rect)
     did_request_file(String path, i32 request_id) =|
+    did_finish_handling_input_event(bool event_was_accepted) =|
 
     did_output_js_console_message(i32 message_index) =|
     did_get_js_console_messages(i32 start_index, Vector<String> message_types, Vector<String> messages) =|