Add support for mouse input

README.md

@@ -148,7 +148,7 @@ To enable TinyPilot's Git hooks, run:
 To run TinyPilot on a non-Pi machine, run:
 
 ```bash
-PORT=8000 KEYBOARD_PATH=/dev/null ./app/main.py
+PORT=8000 KEYBOARD_PATH=/dev/null MOUSE_PATH=/dev/null ./app/main.py
 ```
 
 ## Options
@@ -160,6 +160,7 @@ TinyPilot accepts various options through environment variables:
 | `HOST`               | `0.0.0.0`    | Network interface to listen for incoming connections. |
 | `PORT`               | `8000`       | HTTP port to listen for incoming connections. |
 | `KEYBOARD_PATH`      | `/dev/hidg0` | Path to keyboard HID interface. |
+| `MOUSE_PATH`         | `/dev/hidg1` | Path to mouse HID interface. |
 
 ## Upgrades
 

app/hid/mouse.py

@@ -0,0 +1,22 @@
+from hid import write as hid_write
+
+
+def send_mouse_event(mouse_path, buttons, relative_x, relative_y):
+    x, y = _scale_mouse_coordinates(relative_x, relative_y)
+
+    buf = [0] * 5
+    buf[0] = buttons
+    buf[1] = x & 0xff
+    buf[2] = (x >> 8) & 0xff
+    buf[3] = y & 0xff
+    buf[4] = (y >> 8) & 0xff
+
+    hid_write.write_to_hid_interface(mouse_path, buf)
+
+
+def _scale_mouse_coordinates(relative_x, relative_y):
+    # This comes from LOGICAL_MAXIMUM in the mouse HID descriptor.
+    max_hid_value = 32767.0
+    x = int(relative_x * max_hid_value)
+    y = int(relative_y * max_hid_value)
+    return x, y

app/main.py

@@ -10,7 +10,9 @@
 import js_to_hid
 import local_system
 from hid import keyboard as fake_keyboard
+from hid import mouse as fake_mouse
 from hid import write as hid_write
+from request_parsers import mouse_event as mouse_event_request
 
 root_logger = logging.getLogger()
 handler = logging.StreamHandler()
@@ -29,6 +31,8 @@
 use_reloader = os.environ.get('USE_RELOADER', '1') == '1'
 # Location of file path at which to write keyboard HID input.
 keyboard_path = os.environ.get('KEYBOARD_PATH', '/dev/hidg0')
+# Location of file path at which to write mouse HID input.
+mouse_path = os.environ.get('MOUSE_PATH', '/dev/hidg1')
 
 # Socket.io logs are too chatty at INFO level.
 if not debug:
@@ -49,6 +53,7 @@
 app.config['SECRET_KEY'] = os.urandom(32)
 
 
+# TODO(mtlynch): Move this to request_parsers module.
 def _parse_key_event(payload):
     return js_to_hid.JavaScriptKeyEvent(meta_modifier=payload['metaKey'],
                                         alt_modifier=payload['altKey'],
@@ -90,6 +95,22 @@ def socket_keystroke(message):
     socketio.emit('keystroke-received', processing_result)
 
 
+@socketio.on('mouse-event')
+def socket_mouse_event(message):
+    try:
+        mouse_move_event = mouse_event_request.parse_mouse_event(message)
+    except mouse_event_request.Error as e:
+        logger.error('Failed to parse mouse event request: %s', e)
+        return
+    try:
+        fake_mouse.send_mouse_event(mouse_path, mouse_move_event.buttons,
+                                    mouse_move_event.relative_x,
+                                    mouse_move_event.relative_y)
+    except hid_write.WriteError as e:
+        logger.error('Failed to forward mouse event: %s', e)
+        return
+
+
 @socketio.on('keyRelease')
 def socket_key_release():
     try:

app/request_parsers/mouse_event.py

@@ -0,0 +1,60 @@
+import dataclasses
+
+
+class Error(Exception):
+    pass
+
+
+class InvalidButtonState(Exception):
+    pass
+
+
+class InvalidRelativePosition(Exception):
+    pass
+
+
+# JavaScript only supports 5 mouse buttons.
+# https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent/buttons
+_MAX_BUTTONS = 5
+_MAX_BUTTON_STATE = pow(2, _MAX_BUTTONS) - 1
+
+
+@dataclasses.dataclass
+class MouseEvent:
+    # A bitmask of buttons pressed during the mouse event.
+    buttons: int
+
+    # A value from 0.0 to 1.0 representing the cursor's relative position on the
+    # screen.
+    relative_x: int
+    relative_y: int
+
+
+def parse_mouse_event(message):
+    return MouseEvent(
+        buttons=_parse_button_state(message['buttons']),
+        relative_x=_parse_relative_position(message['relativeX']),
+        relative_y=_parse_relative_position(message['relativeY']),
+    )
+
+
+def _parse_button_state(buttons):
+    if type(buttons) is not int:
+        raise InvalidButtonState('Button state must be an integer value: %s' %
+                                 buttons)
+    if not (0 <= buttons <= _MAX_BUTTON_STATE):
+        raise InvalidButtonState('Button state must be <= 0x%x: %s' %
+                                 (_MAX_BUTTON_STATE, buttons))
+    return buttons
+
+
+def _parse_relative_position(relative_position):
+    if type(relative_position) is not float:
+        raise InvalidRelativePosition(
+            'Relative position must be a float between 0.0 and 1.0: %s' %
+            relative_position)
+    if not (0.0 <= relative_position <= 1.0):
+        raise InvalidRelativePosition(
+            'Relative position must be a float between 0.0 and 1.0: %s' %
+            relative_position)
+    return relative_position

app/static/css/style.css

@@ -155,7 +155,7 @@ button:active {
 
 #remote-screen img {
   width: 100%;
-  cursor: not-allowed;
+  cursor: crosshair;
 }
 
 .keyboard-status {

app/static/js/app.js

@@ -261,6 +261,21 @@ function onKeyDown(evt) {
   clearManualModifiers();
 }
 
+function sendMouseEvent(evt) {
+  const boundingRect = evt.target.getBoundingClientRect();
+  const cursorX = Math.max(0, evt.clientX - boundingRect.left);
+  const cursorY = Math.max(0, evt.clientY - boundingRect.top);
+  const width = boundingRect.right - boundingRect.left;
+  const height = boundingRect.bottom - boundingRect.top;
+  const relativeX = Math.min(1.0, Math.max(0.0, cursorX / width));
+  const relativeY = Math.min(1.0, Math.max(0.0, cursorY / height));
+  socket.emit("mouse-event", {
+    buttons: evt.buttons,
+    relativeX: relativeX,
+    relativeY: relativeY,
+  });
+}
+
 function onKeyUp(evt) {
   keyState[evt.keyCode] = false;
   if (!connectedToServer) {
@@ -291,6 +306,21 @@ function onDisplayHistoryChanged(evt) {
 
 document.querySelector("body").addEventListener("keydown", onKeyDown);
 document.querySelector("body").addEventListener("keyup", onKeyUp);
+
+// Forward all mouse activity that occurs over the image of the remote screen.
+const screenImg = document.getElementById("remote-screen-img");
+screenImg.addEventListener("mousemove", function (evt) {
+  // Ensure that mouse drags don't attempt to drag the image on the screen.
+  evt.preventDefault();
+  sendMouseEvent(evt);
+});
+screenImg.addEventListener("mousedown", sendMouseEvent);
+screenImg.addEventListener("mouseup", sendMouseEvent);
+// Ignore the context menu so that it doesn't block the screen when the user
+// right-clicks.
+screenImg.addEventListener("contextmenu", function (evt) {
+  evt.preventDefault();
+});
 document
   .getElementById("display-history-checkbox")
   .addEventListener("change", onDisplayHistoryChanged);

app/templates/index.html

@@ -75,7 +75,7 @@ <h3>Shut Down TinyPilot Device?</h3>
         <button id="cancel-shutdown" class="btn" type="button">Cancel</button>
       </div>
       <div id="remote-screen">
-        <img src="/stream?advance_headers=1" />
+        <img id="remote-screen-img" src="/stream?advance_headers=1" />
       </div>
       <div id="keystroke-history" class="container">
         <div id="recent-keys"></div>

app/tests/request_parsers/test_mouse_event.py

@@ -0,0 +1,96 @@
+import unittest
+
+from request_parsers import mouse_event
+
+
+class MouseEventTest(unittest.TestCase):
+
+    def test_parses_valid_mouse_event(self):
+        self.assertEqual(
+            mouse_event.MouseEvent(buttons=1, relative_x=0.5, relative_y=0.75),
+            mouse_event.parse_mouse_event({
+                'buttons': 1,
+                'relativeX': 0.5,
+                'relativeY': 0.75,
+            }))
+
+    def test_parses_valid_mouse_event_with_all_buttons_pressed(self):
+        self.assertEqual(
+            mouse_event.MouseEvent(buttons=31, relative_x=0.5, relative_y=0.75),
+            mouse_event.parse_mouse_event({
+                'buttons': 31,
+                'relativeX': 0.5,
+                'relativeY': 0.75,
+            }))
+
+    def test_rejects_negative_buttons_value(self):
+        with self.assertRaises(mouse_event.InvalidButtonState):
+            mouse_event.parse_mouse_event({
+                'buttons': -1,
+                'relativeX': 0.5,
+                'relativeY': 0.75,
+            })
+
+    def test_rejects_too_high_buttons_value(self):
+        with self.assertRaises(mouse_event.InvalidButtonState):
+            mouse_event.parse_mouse_event({
+                'buttons': 32,
+                'relativeX': 0.5,
+                'relativeY': 0.75,
+            })
+
+    def test_rejects_non_numeric_buttons_value(self):
+        with self.assertRaises(mouse_event.InvalidButtonState):
+            mouse_event.parse_mouse_event({
+                'buttons': 'a',
+                'relativeX': 0.5,
+                'relativeY': 0.75,
+            })
+
+    def test_rejects_negative_relative_x_value(self):
+        with self.assertRaises(mouse_event.InvalidRelativePosition):
+            mouse_event.parse_mouse_event({
+                'buttons': 0,
+                'relativeX': -0.001,
+                'relativeY': 0.75,
+            })
+
+    def test_rejects_negative_relative_y_value(self):
+        with self.assertRaises(mouse_event.InvalidRelativePosition):
+            mouse_event.parse_mouse_event({
+                'buttons': 0,
+                'relativeX': 0.5,
+                'relativeY': -0.001,
+            })
+
+    def test_rejects_too_high_relative_x_value(self):
+        with self.assertRaises(mouse_event.InvalidRelativePosition):
+            mouse_event.parse_mouse_event({
+                'buttons': 0,
+                'relativeX': 1.001,
+                'relativeY': 0.75,
+            })
+
+    def test_rejects_too_high_relative_y_value(self):
+        with self.assertRaises(mouse_event.InvalidRelativePosition):
+            mouse_event.parse_mouse_event({
+                'buttons': 0,
+                'relativeX': 0.5,
+                'relativeY': 1.001,
+            })
+
+    def test_rejects_non_float_relative_x_value(self):
+        with self.assertRaises(mouse_event.InvalidRelativePosition):
+            mouse_event.parse_mouse_event({
+                'buttons': 0,
+                'relativeX': 'a',
+                'relativeY': 0.75,
+            })
+
+    def test_rejects_non_float_relative_y_value(self):
+        with self.assertRaises(mouse_event.InvalidRelativePosition):
+            mouse_event.parse_mouse_event({
+                'buttons': 0,
+                'relativeX': 0.5,
+                'relativeY': 'b',
+            })

scripts/usb-gadget/init-usb-gadget

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Adapted from https://github.com/girst/hardpass-sendHID/blob/master/README.md
+# Configures USB gadgets per: https://www.kernel.org/doc/Documentation/usb/gadget_configfs.txt
 
 # Exit on first error.
 set -e
@@ -13,6 +13,8 @@ set -u
 
 modprobe libcomposite
 
+# Adapted from https://github.com/girst/hardpass-sendHID/blob/master/README.md
+
 cd /sys/kernel/config/usb_gadget/
 mkdir -p g1
 cd g1
@@ -26,9 +28,10 @@ STRINGS_DIR="strings/0x409"
 mkdir -p "$STRINGS_DIR"
 echo "6b65796d696d6570690" > "${STRINGS_DIR}/serialnumber"
 echo "tinypilot" > "${STRINGS_DIR}/manufacturer"
-echo "Generic USB Keyboard" > "${STRINGS_DIR}/product"
+echo "Multifunction USB Device" > "${STRINGS_DIR}/product"
 
-KEYBOARD_FUNCTIONS_DIR="functions/hid.usb0"
+# Keyboard
+KEYBOARD_FUNCTIONS_DIR="functions/hid.keyboard"
 mkdir -p "$KEYBOARD_FUNCTIONS_DIR"
 echo 1 > "${KEYBOARD_FUNCTIONS_DIR}/protocol" # Keyboard
 echo 1 > "${KEYBOARD_FUNCTIONS_DIR}/subclass" # Boot interface subclass
@@ -70,6 +73,38 @@ echo -ne \\x81\\x00 >> "$D" #   INPUT (Data,Ary,Abs)
 echo -ne \\xc0 >> "$D"      # END_COLLECTION
 cp "$D" "${KEYBOARD_FUNCTIONS_DIR}/report_desc"
 
+# Mouse
+MOUSE_FUNCTIONS_DIR="functions/hid.mouse"
+mkdir -p "$MOUSE_FUNCTIONS_DIR"
+echo 0 > "${MOUSE_FUNCTIONS_DIR}/protocol"
+echo 0 > "${MOUSE_FUNCTIONS_DIR}/subclass"
+echo 5 > "${MOUSE_FUNCTIONS_DIR}/report_length"
+# Write the report descriptor
+D=$(mktemp)
+echo -ne \\x05\\x01 >> "$D"      # USAGE_PAGE (Generic Desktop)
+echo -ne \\x09\\x02 >> "$D"      # USAGE (Mouse)
+echo -ne \\xA1\\x01 >> "$D"      # COLLECTION (Application)
+                                 #   8-buttons
+echo -ne \\x05\\x09 >> "$D"      #   USAGE_PAGE (Button)
+echo -ne \\x19\\x01 >> "$D"      #   USAGE_MINIMUM (Button 1)
+echo -ne \\x29\\x08 >> "$D"      #   USAGE_MAXIMUM (Button 8)
+echo -ne \\x15\\x00 >> "$D"      #   LOGICAL_MINIMUM (0)
+echo -ne \\x25\\x01 >> "$D"      #   LOGICAL_MAXIMUM (1)
+echo -ne \\x95\\x08 >> "$D"      #   REPORT_COUNT (8)
+echo -ne \\x75\\x01 >> "$D"      #   REPORT_SIZE (1)
+                                 #   x,y absolute coordinates
+echo -ne \\x81\\x02 >> "$D"      #   INPUT (Data,Var,Abs)
+echo -ne \\x05\\x01 >> "$D"      #   USAGE_PAGE (Generic Desktop)
+echo -ne \\x09\\x30 >> "$D"      #   USAGE (X)
+echo -ne \\x09\\x31 >> "$D"      #   USAGE (Y)
+echo -ne \\x16\\x00\\x00 >> "$D" #   LOGICAL_MINIMUM (0)
+echo -ne \\x26\\xFF\\x7F >> "$D" #   LOGICAL_MAXIMUM (32767)
+echo -ne \\x75\\x10 >> "$D"      #   REPORT_SIZE (16)
+echo -ne \\x95\\x02 >> "$D"      #   REPORT_COUNT (2)
+echo -ne \\x81\\x02 >> "$D"      #   INPUT (Data,Var,Abs)
+echo -ne \\xC0 >> "$D"           # END_COLLECTION
+cp "$D" "${MOUSE_FUNCTIONS_DIR}/report_desc"
+
 CONFIG_INDEX=1
 CONFIGS_DIR="configs/c.${CONFIG_INDEX}"
 mkdir -p "$CONFIGS_DIR"
@@ -80,6 +115,8 @@ mkdir -p "$CONFIGS_STRINGS_DIR"
 echo "Config ${CONFIG_INDEX}: ECM network" > "${CONFIGS_STRINGS_DIR}/configuration"
 
 ln -s "$KEYBOARD_FUNCTIONS_DIR" "${CONFIGS_DIR}/"
+ln -s "$MOUSE_FUNCTIONS_DIR" "${CONFIGS_DIR}/"
 ls /sys/class/udc > UDC
 
 chmod 777 /dev/hidg0
+chmod 777 /dev/hidg1