Physical keyboard to android support

Cargo.toml

@@ -64,7 +64,7 @@ default-net = "0.14"
 wol-rs = "1.0"
 flutter_rust_bridge = { version = "1.75", features = ["uuid"], optional = true}
 errno = "0.3"
-rdev = { git = "https://github.com/fufesou/rdev" }
+rdev = { git = "https://github.com/mcfans/rdev" }
 url = { version = "2.3", features = ["serde"] }
 crossbeam-queue = "0.3"
 hex = "0.4"

flutter/android/app/build.gradle

@@ -1,3 +1,8 @@
+import com.google.protobuf.gradle.*
+plugins {
+    id "com.google.protobuf" version "0.9.4"
+}
+
 def keystoreProperties = new Properties()
 def keystorePropertiesFile = rootProject.file('key.properties')
 if (keystorePropertiesFile.exists()) {
@@ -31,10 +36,33 @@ apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
 apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle"
 
+dependencies {
+    implementation 'com.google.protobuf:protobuf-javalite:3.20.1'
+}
+
+protobuf {
+    protoc {
+        artifact = 'com.google.protobuf:protoc:3.20.1'
+    }
+
+    generateProtoTasks {
+        all().configureEach { task ->
+            task.builtins {
+                java {
+                    option "lite"
+                }
+            }
+        }
+    }
+}
+
 android {
     compileSdkVersion 33
     sourceSets {
         main.java.srcDirs += 'src/main/kotlin'
+
+        main.proto.srcDirs += '../../../libs/hbb_common/protos'
+        main.proto.includes += "message.proto"
     }
 
     compileOptions {

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt

@@ -10,12 +10,21 @@ import android.accessibilityservice.AccessibilityService
 import android.accessibilityservice.GestureDescription
 import android.graphics.Path
 import android.os.Build
+import android.os.Bundle
+import android.os.Handler
+import android.os.Looper
 import android.util.Log
+import android.widget.EditText
 import android.view.accessibility.AccessibilityEvent
+import android.view.accessibility.AccessibilityNodeInfo
+import android.accessibilityservice.AccessibilityServiceInfo
+import android.accessibilityservice.AccessibilityServiceInfo.FLAG_INPUT_METHOD_EDITOR
 import androidx.annotation.RequiresApi
 import java.util.*
 import kotlin.math.abs
 import kotlin.math.max
+import hbb.MessageOuterClass.KeyEvent;
+import hbb.KeyEventConverter;
 
 const val LIFT_DOWN = 9
 const val LIFT_MOVE = 8
@@ -252,9 +261,27 @@ class InputService : AccessibilityService() {
         }
     }
 
+    @RequiresApi(Build.VERSION_CODES.N)
+    fun onKeyEvent(data: ByteArray) {
+        val keyEvent = KeyEvent.parseFrom(data);
+        getInputMethod()?.let { inputMethod ->
+            inputMethod.getCurrentInputConnection()?.let { inputConnection ->
+                KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event ->
+                    inputConnection.sendKeyEvent(event)
+                }
+            }
+        }
+    }
+
+
+    override fun onAccessibilityEvent(event: AccessibilityEvent) {}
+
     override fun onServiceConnected() {
         super.onServiceConnected()
         ctx = this
+        val info = AccessibilityServiceInfo()
+        info.flags = FLAG_INPUT_METHOD_EDITOR
+        setServiceInfo(info)
         Log.d(logTag, "onServiceConnected!")
     }
 
@@ -263,7 +290,5 @@ class InputService : AccessibilityService() {
         super.onDestroy()
     }
 
-    override fun onAccessibilityEvent(event: AccessibilityEvent?) {}
-
     override fun onInterrupt() {}
 }

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt

@@ -0,0 +1,116 @@
+package hbb;
+import android.view.KeyEvent
+import android.view.KeyCharacterMap
+import hbb.MessageOuterClass.KeyboardMode
+import hbb.MessageOuterClass.ControlKey
+
+object KeyEventConverter {
+    fun toAndroidKeyEvent(keyEventProto: hbb.MessageOuterClass.KeyEvent): KeyEvent {
+        var chrValue = 0
+        var modifiers = 0
+
+        val keyboardMode = keyEventProto.getMode()
+
+        if (keyEventProto.hasChr()) {
+            if (keyboardMode == KeyboardMode.Map || keyboardMode == KeyboardMode.Translate) {
+                chrValue = keyEventProto.getChr()
+            } else {
+                chrValue = convertUnicodeToKeyCode(keyEventProto.getChr() as Int)
+            }
+        } else if (keyEventProto.hasControlKey()) {
+            chrValue = convertControlKeyToKeyCode(keyEventProto.getControlKey())
+        }
+
+        var modifiersList = keyEventProto.getModifiersList()
+
+        if (modifiersList != null) {
+            for (modifier in keyEventProto.getModifiersList()) {
+                val modifierValue = convertModifier(modifier)
+                modifiers = modifiers or modifierValue
+            }
+        }
+
+        var action = 0
+        if (keyEventProto.getDown()) {
+            action = KeyEvent.ACTION_DOWN
+        } else {
+            action = KeyEvent.ACTION_UP
+        }
+
+        return KeyEvent(0, 0, action, chrValue, 0, modifiers)
+    }
+
+    private fun convertModifier(controlKey: hbb.MessageOuterClass.ControlKey): Int {
+        // Add logic to map ControlKey values to Android KeyEvent key codes.
+        // You'll need to provide the mapping for each key.
+        return when (controlKey) {
+            ControlKey.Alt -> KeyEvent.META_ALT_ON
+            ControlKey.Control -> KeyEvent.META_CTRL_ON
+            ControlKey.CapsLock -> KeyEvent.META_CAPS_LOCK_ON
+            ControlKey.Meta -> KeyEvent.META_META_ON
+            ControlKey.NumLock -> KeyEvent.META_NUM_LOCK_ON
+            ControlKey.RShift -> KeyEvent.META_SHIFT_RIGHT_ON
+            ControlKey.Shift -> KeyEvent.META_SHIFT_ON
+            ControlKey.RAlt -> KeyEvent.META_ALT_RIGHT_ON
+            ControlKey.RControl -> KeyEvent.META_CTRL_RIGHT_ON
+            else -> 0 // Default to unknown.
+        }
+    }
+
+    private val tag = "KeyEventConverter"
+
+    private fun convertUnicodeToKeyCode(unicode: Int): Int {
+        val charMap = KeyCharacterMap.load(KeyCharacterMap.VIRTUAL_KEYBOARD)
+        android.util.Log.d(tag, "unicode: $unicode")
+        val events = charMap.getEvents(charArrayOf(unicode.toChar()))
+        if (events != null && events.size > 0) {
+            android.util.Log.d(tag, "keycode ${events[0].keyCode}")
+            return events[0].keyCode
+        }
+        return 0
+    }
+
+    private fun convertControlKeyToKeyCode(controlKey: hbb.MessageOuterClass.ControlKey): Int {
+        // Add logic to map ControlKey values to Android KeyEvent key codes.
+        // You'll need to provide the mapping for each key.
+        return when (controlKey) {
+            ControlKey.Alt -> KeyEvent.KEYCODE_ALT_LEFT
+            ControlKey.Backspace -> KeyEvent.KEYCODE_DEL
+            ControlKey.Control -> KeyEvent.KEYCODE_CTRL_LEFT
+            ControlKey.CapsLock -> KeyEvent.KEYCODE_CAPS_LOCK
+            ControlKey.Meta -> KeyEvent.KEYCODE_META_LEFT
+            ControlKey.NumLock -> KeyEvent.KEYCODE_NUM_LOCK
+            ControlKey.RShift -> KeyEvent.KEYCODE_SHIFT_RIGHT
+            ControlKey.Shift -> KeyEvent.KEYCODE_SHIFT_LEFT
+            ControlKey.RAlt -> KeyEvent.KEYCODE_ALT_RIGHT
+            ControlKey.RControl -> KeyEvent.KEYCODE_CTRL_RIGHT
+            ControlKey.DownArrow -> KeyEvent.KEYCODE_DPAD_DOWN
+            ControlKey.LeftArrow -> KeyEvent.KEYCODE_DPAD_LEFT
+            ControlKey.RightArrow -> KeyEvent.KEYCODE_DPAD_RIGHT
+            ControlKey.UpArrow -> KeyEvent.KEYCODE_DPAD_UP
+            ControlKey.End -> KeyEvent.KEYCODE_MOVE_END
+            ControlKey.Home -> KeyEvent.KEYCODE_MOVE_HOME
+            ControlKey.Insert -> KeyEvent.KEYCODE_INSERT
+            ControlKey.Escape -> KeyEvent.KEYCODE_ESCAPE
+            ControlKey.F1 -> KeyEvent.KEYCODE_F1
+            ControlKey.F2 -> KeyEvent.KEYCODE_F2
+            ControlKey.F3 -> KeyEvent.KEYCODE_F3
+            ControlKey.F4 -> KeyEvent.KEYCODE_F4
+            ControlKey.F5 -> KeyEvent.KEYCODE_F5
+            ControlKey.F6 -> KeyEvent.KEYCODE_F6
+            ControlKey.F7 -> KeyEvent.KEYCODE_F7
+            ControlKey.F8 -> KeyEvent.KEYCODE_F8
+            ControlKey.F9 -> KeyEvent.KEYCODE_F9
+            ControlKey.F10 -> KeyEvent.KEYCODE_F10
+            ControlKey.F11 -> KeyEvent.KEYCODE_F11
+            ControlKey.F12 -> KeyEvent.KEYCODE_F12
+            ControlKey.Space -> KeyEvent.KEYCODE_SPACE
+            ControlKey.Tab -> KeyEvent.KEYCODE_TAB
+            ControlKey.Return -> KeyEvent.KEYCODE_ENTER
+            ControlKey.Delete -> KeyEvent.KEYCODE_FORWARD_DEL
+            ControlKey.Clear -> KeyEvent.KEYCODE_CLEAR
+            ControlKey.Pause -> KeyEvent.KEYCODE_BREAK
+            else -> 0 // Default to unknown.
+        }
+    }
+}

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt

@@ -44,7 +44,6 @@ import java.nio.ByteBuffer
 import kotlin.math.max
 import kotlin.math.min
 
-
 const val DEFAULT_NOTIFY_TITLE = "RustDesk"
 const val DEFAULT_NOTIFY_TEXT = "Service is running"
 const val DEFAULT_NOTIFY_ID = 1
@@ -94,6 +93,12 @@ class MainService : Service() {
         }
     }
 
+    @Keep
+    @RequiresApi(Build.VERSION_CODES.N)
+    fun rustKeyEventInput(input: ByteArray) {
+        InputService.ctx?.onKeyEvent(input)
+    }
+
     @Keep
     fun rustGetByName(name: String): String {
         return when (name) {

flutter/lib/desktop/screen/desktop_remote_screen.dart

@@ -12,9 +12,9 @@ class DesktopRemoteScreen extends StatelessWidget {
   final Map<String, dynamic> params;
 
   DesktopRemoteScreen({Key? key, required this.params}) : super(key: key) {
-    if (!bind.mainStartGrabKeyboard()) {
-      stateGlobal.grabKeyboard = true;
-    }
+      if (!bind.mainStartGrabKeyboard()) {
+        stateGlobal.grabKeyboard = true;
+      }
   }
 
   @override

flutter/lib/mobile/pages/remote_page.dart

@@ -364,6 +364,10 @@ class _RemotePageState extends State<RemotePage> {
                       ? []
                       : gFFI.ffiModel.isPeerAndroid
                           ? [
+                              IconButton(
+                                  color: Colors.white,
+                                  icon: Icon(Icons.keyboard),
+                                  onPressed: openKeyboard),
                               IconButton(
                                 color: Colors.white,
                                 icon: const Icon(Icons.build),

libs/scrap/src/android/ffi.rs

@@ -173,6 +173,26 @@ pub fn call_main_service_pointer_input(kind: &str, mask: i32, x: i32, y: i32) ->
     }
 }
 
+pub fn call_main_service_key_event(data: &[u8]) -> JniResult<()> {
+    if let (Some(jvm), Some(ctx)) = (
+        JVM.read().unwrap().as_ref(),
+        MAIN_SERVICE_CTX.read().unwrap().as_ref(),
+    ) {
+        let mut env = jvm.attach_current_thread_as_daemon()?;
+        let data = env.byte_array_from_slice(data)?;
+
+        env.call_method(
+            ctx,
+            "rustKeyEventInput",
+            "([B)V",
+            &[JValue::Object(&JObject::from(data))],
+        )?;
+        return Ok(());
+    } else {
+        return Err(JniError::ThrowFailed(-1));
+    }
+}
+
 pub fn call_main_service_get_by_name(name: &str) -> JniResult<String> {
     if let (Some(jvm), Some(ctx)) = (
         JVM.read().unwrap().as_ref(),