Merge pull request NetrisTV#57 from HardBoiledSmith/DEV-14440

DEV-14440 [라미엘] iOS 도입 - 라미엘 안드로이드와 화면과 동작 일치시키기 W10~ NetrisTV#2 - 키보드 입력 기능 구현
maxduke · May 2, 2022 · d8271ea · d8271ea
2 parents 04d4019 + 33bfc6c
commit d8271ea
Show file tree

Hide file tree

Showing 8 changed files with 274 additions and 11 deletions.
diff --git a/build.config.override.json b/build.config.override.json
@@ -1 +1,4 @@
-{}
+{
+  "INCLUDE_APPL": true,
+  "WDA_RUN_MJPEG_SERVER": true
+}
diff --git a/src/app/applDevice/KeyInputHandler.ts b/src/app/applDevice/KeyInputHandler.ts
@@ -0,0 +1,69 @@
+// TODO: HBsmith DEV-14440
+
+import { WindowsKeyCodeToKey } from '../googDevice/WindowsKeyCodeToKey';
+import { KeyToCodeMap, ToUpperCodeMap } from './KeyToCodeMap';
+
+export interface KeyEventListener {
+    onKeyEvent: (keyValue: string) => void;
+}
+
+export class KeyInputHandler {
+    private static readonly listeners: Set<KeyEventListener> = new Set();
+    private static handler = (e: Event): void => {
+        const event = e as KeyboardEvent;
+        if (event.type !== 'keyup') {
+            return;
+        }
+
+        let keyCode: string | undefined = event.code;
+        if (!keyCode) {
+            keyCode = WindowsKeyCodeToKey.get(event.keyCode);
+        }
+        if (!keyCode) {
+            return;
+        }
+
+        let keyValue = KeyToCodeMap.get(keyCode);
+        if (!keyValue) {
+            return;
+        }
+
+        const shift = event.getModifierState('Shift') ? 1 : 0;
+        const capsLock = event.getModifierState('CapsLock') ? 1 : 0;
+        if (shift || capsLock) {
+            if (keyValue.match(/[a-z]/i) && shift ^ capsLock) {
+                keyValue = keyValue.toUpperCase();
+            } else if (shift) {
+                const tt = ToUpperCodeMap.get(keyValue);
+                if (tt) {
+                    keyValue = tt;
+                }
+            }
+        }
+
+        KeyInputHandler.listeners.forEach((listener) => {
+            if (keyValue) {
+                listener.onKeyEvent(keyValue);
+            }
+        });
+        e.preventDefault();
+    };
+    private static attachListeners(): void {
+        document.body.addEventListener('keyup', this.handler);
+    }
+    private static detachListeners(): void {
+        document.body.removeEventListener('keyup', this.handler);
+    }
+    public static addEventListener(listener: KeyEventListener): void {
+        if (!this.listeners.size) {
+            this.attachListeners();
+        }
+        this.listeners.add(listener);
+    }
+    public static removeEventListener(listener: KeyEventListener): void {
+        this.listeners.delete(listener);
+        if (!this.listeners.size) {
+            this.detachListeners();
+        }
+    }
+}
diff --git a/src/app/applDevice/KeyToCodeMap.ts b/src/app/applDevice/KeyToCodeMap.ts
@@ -0,0 +1,141 @@
+import UIEventsCode from '../UIEventsCode';
+
+export const KeyToCodeMap = new Map([
+    [UIEventsCode.Backquote, '`'],
+    [UIEventsCode.Backslash, '\\'],
+    [UIEventsCode.BracketLeft, '('],
+    [UIEventsCode.BracketRight, ')'],
+    [UIEventsCode.Comma, ','],
+    [UIEventsCode.Digit0, '0'],
+    [UIEventsCode.Digit1, '1'],
+    [UIEventsCode.Digit2, '2'],
+    [UIEventsCode.Digit9, '9'],
+    [UIEventsCode.Digit3, '3'],
+    [UIEventsCode.Digit4, '4'],
+    [UIEventsCode.Digit5, '5'],
+    [UIEventsCode.Digit6, '6'],
+    [UIEventsCode.Digit7, '7'],
+    [UIEventsCode.Digit8, '8'],
+    [UIEventsCode.Equal, '='],
+    // [UIEventsCode.IntlRo, ''],
+    [UIEventsCode.IntlYen, '₩'],
+    [UIEventsCode.KeyA, 'a'],
+    [UIEventsCode.KeyB, 'b'],
+    [UIEventsCode.KeyC, 'c'],
+    [UIEventsCode.KeyD, 'd'],
+    [UIEventsCode.KeyE, 'e'],
+    [UIEventsCode.KeyF, 'f'],
+    [UIEventsCode.KeyG, 'g'],
+    [UIEventsCode.KeyH, 'h'],
+    [UIEventsCode.KeyI, 'i'],
+    [UIEventsCode.KeyJ, 'j'],
+    [UIEventsCode.KeyK, 'k'],
+    [UIEventsCode.KeyL, 'l'],
+    [UIEventsCode.KeyM, 'm'],
+    [UIEventsCode.KeyN, 'n'],
+    [UIEventsCode.KeyO, 'o'],
+    [UIEventsCode.KeyP, 'p'],
+    [UIEventsCode.KeyQ, 'q'],
+    [UIEventsCode.KeyR, 'r'],
+    [UIEventsCode.KeyS, 's'],
+    [UIEventsCode.KeyT, 't'],
+    [UIEventsCode.KeyU, 'u'],
+    [UIEventsCode.KeyV, 'v'],
+    [UIEventsCode.KeyW, 'w'],
+    [UIEventsCode.KeyX, 'x'],
+    [UIEventsCode.KeyY, 'y'],
+    [UIEventsCode.KeyZ, 'z'],
+    [UIEventsCode.Minus, '-'],
+    [UIEventsCode.Period, '.'],
+    [UIEventsCode.Quote, "'"],
+    [UIEventsCode.Semicolon, ';'],
+    [UIEventsCode.Slash, '/'],
+    // [UIEventsCode.KanaMode, ''],
+    // [UIEventsCode.Delete, ''],
+    // [UIEventsCode.End, ''],
+    // [UIEventsCode.Help, ''],
+    // [UIEventsCode.Home, ''],
+    // [UIEventsCode.Insert, ''],
+    // [UIEventsCode.PageDown, ''],
+    // [UIEventsCode.PageUp, ''],
+    // [UIEventsCode.AltLeft, ''],
+    // [UIEventsCode.AltRight, ''],
+    [UIEventsCode.Backspace, '\b'],
+    // [UIEventsCode.CapsLock, ''],
+    // [UIEventsCode.ControlLeft, ''],
+    // [UIEventsCode.ControlRight, ''],
+    [UIEventsCode.Enter, '\n'],
+    // [UIEventsCode.MetaLeft, KeyEvent.KEYCODE_META_LEFT],
+    // [UIEventsCode.MetaRight, KeyEvent.KEYCODE_META_RIGHT],
+    // [UIEventsCode.ShiftLeft, KeyEvent.KEYCODE_SHIFT_LEFT],
+    // [UIEventsCode.ShiftRight, KeyEvent.KEYCODE_SHIFT_RIGHT],
+    [UIEventsCode.Space, ' '],
+    [UIEventsCode.Tab, '\t'],
+    // [UIEventsCode.ArrowLeft, KeyEvent.KEYCODE_DPAD_LEFT],
+    // [UIEventsCode.ArrowUp, KeyEvent.KEYCODE_DPAD_UP],
+    // [UIEventsCode.ArrowRight, KeyEvent.KEYCODE_DPAD_RIGHT],
+    // [UIEventsCode.ArrowDown, KeyEvent.KEYCODE_DPAD_DOWN],
+
+    // [UIEventsCode.NumLock, KeyEvent.KEYCODE_NUM_LOCK],
+    [UIEventsCode.Numpad0, '0'],
+    [UIEventsCode.Numpad1, '1'],
+    [UIEventsCode.Numpad2, '2'],
+    [UIEventsCode.Numpad3, '3'],
+    [UIEventsCode.Numpad4, '4'],
+    [UIEventsCode.Numpad5, '5'],
+    [UIEventsCode.Numpad6, '6'],
+    [UIEventsCode.Numpad7, '7'],
+    [UIEventsCode.Numpad8, '8'],
+    [UIEventsCode.Numpad9, '9'],
+    // [UIEventsCode.NumpadAdd, KeyEvent.KEYCODE_NUMPAD_ADD],
+    [UIEventsCode.NumpadComma, ','],
+    [UIEventsCode.NumpadDecimal, '.'],
+    [UIEventsCode.NumpadDivide, '/'],
+    [UIEventsCode.NumpadEnter, '\n'],
+    [UIEventsCode.NumpadEqual, '='],
+    [UIEventsCode.NumpadMultiply, '*'],
+    [UIEventsCode.NumpadParenLeft, '('],
+    [UIEventsCode.NumpadParenRight, ')'],
+    [UIEventsCode.NumpadSubtract, '-'],
+
+    // [UIEventsCode.Escape, ''],
+    // [UIEventsCode.F1, ''],
+    // [UIEventsCode.F2, ''],
+    // [UIEventsCode.F3, ''],
+    // [UIEventsCode.F4, ''],
+    // [UIEventsCode.F5, ''],
+    // [UIEventsCode.F6, ''],
+    // [UIEventsCode.F7, ''],
+    // [UIEventsCode.F8, ''],
+    // [UIEventsCode.F9, ''],
+    // [UIEventsCode.F10, ''],
+    // [UIEventsCode.F11, ''],
+    // [UIEventsCode.F12, ''],
+    // [UIEventsCode.Fn, ''],
+    // [UIEventsCode.PrintScreen, ''],
+    // [UIEventsCode.Pause, ''],
+]);
+
+export const ToUpperCodeMap = new Map([
+    ['`', '~'],
+    ['1', '!'],
+    ['2', '@'],
+    ['3', '#'],
+    ['4', '$'],
+    ['5', '%'],
+    ['6', '^'],
+    ['7', '&'],
+    ['8', '*'],
+    ['9', '('],
+    ['0', ')'],
+    ['-', '_'],
+    ['+', '+'],
+    ['[', '{'],
+    [']', '}'],
+    ['\\', '|'],
+    [';', ':'],
+    ["'", '"'],
+    [',', '<'],
+    ['.', '>'],
+    ['/', '?'],
+]);
diff --git a/src/app/applDevice/client/StreamClientMJPEG.ts b/src/app/applDevice/client/StreamClientMJPEG.ts
@@ -5,15 +5,27 @@ import { StreamClient } from './StreamClient';
 import { BasePlayer, PlayerClass } from '../../player/BasePlayer';
 import { WdaStatus } from '../../../common/WdaStatus';
 import { ApplMjpegMoreBox } from '../toolbox/ApplMjpegMoreBox';
+// TODO: HBsmith
+import { KeyEventListener, KeyInputHandler } from '../KeyInputHandler';
+import { WDAMethod } from '../../../common/WDAMethod';
+//
 
 const TAG = '[StreamClientMJPEG]';
 
-export class StreamClientMJPEG extends StreamClient<ParamsStream> {
+export class StreamClientMJPEG
+    extends StreamClient<ParamsStream>
+    // TODO: HBsmith DEV-14440
+    implements KeyEventListener {
+    //
     public static ACTION = ACTION.STREAM_MJPEG;
     protected static players: Map<string, PlayerClass> = new Map<string, PlayerClass>();
 
     public static start(params: ParsedUrlQuery | ParamsStream): StreamClientMJPEG {
-        return new StreamClientMJPEG(params);
+        // TODO: HBsmith DEV-14440
+        const cc = new StreamClientMJPEG(params);
+        KeyInputHandler.addEventListener(cc);
+        return cc;
+        //
     }
 
     constructor(params: ParsedUrlQuery | ParamsStream) {
@@ -23,7 +35,7 @@ export class StreamClientMJPEG extends StreamClient<ParamsStream> {
         this.runWebDriverAgent().then(() => {
             this.startStream();
             this.player?.play();
-            // TODO: HBsmith DEV-14062
+            // TODO: HBsmith
             this.setBodyClass('stream');
 
             const headerText = document.getElementById('control-header-device-name-text');
@@ -39,6 +51,17 @@ export class StreamClientMJPEG extends StreamClient<ParamsStream> {
         });
     }
 
+    // TODO: HBsmith
+    public onKeyEvent(key: string): void {
+        this.wdaProxy.requestWebDriverAgent(WDAMethod.SEND_A_KEY, { key });
+    }
+
+    public onStop(ev?: string | Event): void {
+        KeyInputHandler.removeEventListener(this);
+        super.onStop(ev);
+    }
+    //
+
     public get action(): string {
         return StreamClientMJPEG.ACTION;
     }

diff --git a/src/app/googDevice/KeyInputHandler.ts b/src/app/googDevice/KeyInputHandler.ts
@@ -14,7 +14,7 @@ export class KeyInputHandler {
     private static readonly listeners: Set<KeyEventListener> = new Set();
     private static handler = (e: Event): void => {
         const event = e as KeyboardEvent;
-        // TODO: DEV-13327
+        // TODO: DEV-13471
         let keyCode = KeyToCodeMap.get(event.code);
         if (!keyCode) {
             const cc = WindowsKeyCodeToKey.get(event.keyCode);

diff --git a/src/app/googDevice/client/StreamClientScrcpy.ts b/src/app/googDevice/client/StreamClientScrcpy.ts
@@ -394,13 +394,10 @@ export class StreamClientScrcpy
         streamReceiver.on('disconnected', this.onDisconnected);
         // TODO: HBsmith DEV-12387
         streamReceiver.on('deviceDisconnected', this.onDeviceDisconnected);
-        //
-        console.log(TAG, player.getName(), udid);
 
-        /*
-            HBsmith tunning
-        */
         KeyInputHandler.addEventListener(this);
+        //
+        console.log(TAG, player.getName(), udid);
     }
 
     public sendMessage(e: ControlMessage): void {

diff --git a/src/common/WDAMethod.ts b/src/common/WDAMethod.ts
@@ -6,6 +6,7 @@ export enum WDAMethod {
     APPIUM_SETTINGS = 'APPIUM_SETTINGS',
     SEND_KEYS = 'SEND_KEYS',
     // TODO: HBsmith
+    SEND_A_KEY = 'SEND_A_KEY',
     UNLOCK = 'UNLOCK',
     TERMINATE_APP = 'TERMINATE_APP',
     //

diff --git a/src/server/appl-device/services/WDARunner.ts b/src/server/appl-device/services/WDARunner.ts
@@ -12,6 +12,7 @@ import { WdaStatus } from '../../../common/WdaStatus';
 import { Config } from '../../Config';
 import { Logger, Utils } from '../../Utils';
 import axios from 'axios';
+import { EventEmitter } from 'events';
 //
 
 const MJPEG_SERVER_PORT = 9100;
@@ -30,6 +31,10 @@ export class WdaRunner extends TypedEmitter<WdaRunnerEvents> {
     // TODO: HBsmith
     private appKey: string;
     private logger: Logger;
+
+    private keyEvents: Array<string> = [];
+    private keyEventInAction = false;
+    private keyEventEmitter: EventEmitter = new EventEmitter();
     private wdaProcessId: number | undefined;
     //
 
@@ -96,6 +101,27 @@ export class WdaRunner extends TypedEmitter<WdaRunnerEvents> {
         this.appKey = '';
         this.logger = new Logger(udid, 'iOS');
         this.wdaProcessId = undefined;
+        this.keyEventInAction = false;
+        this.keyEventEmitter.on('event', () => {
+            const driver = this.server?.driver;
+            if (!driver) {
+                return;
+            }
+            const key = this.keyEvents.shift();
+            if (!key) {
+                return;
+            }
+            this.keyEventInAction = true;
+            return driver.keys(key).finally(() => {
+                this.keyEventInAction = false;
+            });
+        });
+        setInterval(() => {
+            if (this.keyEventInAction || this.keyEvents.length === 0) {
+                return;
+            }
+            this.keyEventEmitter.emit('event');
+        }, 100);
         //
     }
 
@@ -156,13 +182,16 @@ export class WdaRunner extends TypedEmitter<WdaRunnerEvents> {
             case WDAMethod.SEND_KEYS:
                 return driver.keys(args.keys);
             // TODO: HBsmith
+            case WDAMethod.SEND_A_KEY:
+                this.keyEvents.push(args.key);
+                return;
             case WDAMethod.UNLOCK:
                 return driver.unlock();
             case WDAMethod.TERMINATE_APP:
                 return driver.mobileGetActiveAppInfo().then((appInfo) => {
                     const bundleId = appInfo['bundleId'];
                     if (bundleId === 'com.apple.springboard') {
-                        return true;
+                        return;
                     }
                     return driver.terminateApp(bundleId);
                 });