From eab17e918d0dca4e18c128d89824b5e74cf8049e Mon Sep 17 00:00:00 2001 From: Almas Baim Date: Sun, 28 Jan 2024 11:10:43 +0000 Subject: [PATCH] feat: added intial impl hand tracking service --- .../com/almasb/fxgl/gesturerecog/Hand.kt | 25 ++++ .../almasb/fxgl/gesturerecog/HandLandmark.kt | 42 ++++++ .../fxgl/gesturerecog/HandTrackingService.kt | 120 ++++++++++++++++++ 3 files changed, 187 insertions(+) create mode 100644 fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/gesturerecog/Hand.kt create mode 100644 fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/gesturerecog/HandLandmark.kt create mode 100644 fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/gesturerecog/HandTrackingService.kt diff --git a/fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/gesturerecog/Hand.kt b/fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/gesturerecog/Hand.kt new file mode 100644 index 000000000..643396b25 --- /dev/null +++ b/fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/gesturerecog/Hand.kt @@ -0,0 +1,25 @@ +/* + * FXGL - JavaFX Game Library. The MIT License (MIT). + * Copyright (c) AlmasB (almaslvl@gmail.com). + * See LICENSE for details. + */ + +package com.almasb.fxgl.gesturerecog + +import javafx.geometry.Point3D + +/** + * Each hand has an id and a list of 21 hand landmarks points. + * Each point can be mapped into 2D app (or screen where appropriate) space via: + * appX = (1 - pointX) * appWidth + * appY = pointY * appHeight + * + * @author Almas Baim (https://github.com/AlmasB) + */ +data class Hand( + val id: Int, + val points: List +) { + + fun getPoint(landmark: HandLandmark) = points[landmark.ordinal] +} diff --git a/fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/gesturerecog/HandLandmark.kt b/fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/gesturerecog/HandLandmark.kt new file mode 100644 index 000000000..2050eda8a --- /dev/null +++ b/fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/gesturerecog/HandLandmark.kt @@ -0,0 +1,42 @@ +/* + * FXGL - JavaFX Game Library. The MIT License (MIT). + * Copyright (c) AlmasB (almaslvl@gmail.com). + * See LICENSE for details. + */ + +package com.almasb.fxgl.gesturerecog + +/** + * The ordinal of each item matches the format defined at: + * https://developers.google.com/mediapipe/solutions/vision/hand_landmarker + * + * @author Almas Baim (https://github.com/AlmasB) + */ +enum class HandLandmark { + WRIST, + + THUMB_CMC, + THUMB_MCP, + THUMB_IP, + THUMB_TIP, + + INDEX_FINGER_MCP, + INDEX_FINGER_PIP, + INDEX_FINGER_DIP, + INDEX_FINGER_TIP, + + MIDDLE_FINGER_MCP, + MIDDLE_FINGER_PIP, + MIDDLE_FINGER_DIP, + MIDDLE_FINGER_TIP, + + RING_FINGER_MCP, + RING_FINGER_PIP, + RING_FINGER_DIP, + RING_FINGER_TIP, + + PINKY_MCP, + PINKY_PIP, + PINKY_DIP, + PINKY_TIP +} \ No newline at end of file diff --git a/fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/gesturerecog/HandTrackingService.kt b/fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/gesturerecog/HandTrackingService.kt new file mode 100644 index 000000000..e78b0c3cc --- /dev/null +++ b/fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/gesturerecog/HandTrackingService.kt @@ -0,0 +1,120 @@ +/* + * FXGL - JavaFX Game Library. The MIT License (MIT). + * Copyright (c) AlmasB (almaslvl@gmail.com). + * See LICENSE for details. + */ + +package com.almasb.fxgl.gesturerecog + +import com.almasb.fxgl.core.EngineService +import com.almasb.fxgl.core.concurrent.Async +import com.almasb.fxgl.intelligence.WebAPI +import com.almasb.fxgl.logging.Logger +import com.almasb.fxgl.speechrecog.SpeechRecognitionService +import com.almasb.fxgl.ws.LocalWebSocketServer +import javafx.geometry.Point3D +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.chrome.ChromeOptions +import java.util.function.Consumer + +/** + * TODO: remove duplicate code + * + * @author Almas Baim (https://github.com/AlmasB) + */ +class HandTrackingService : EngineService() { + + private val log = Logger.get(SpeechRecognitionService::class.java) + private val server = LocalWebSocketServer("HandTrackingServer", WebAPI.GESTURE_RECOGNITION_PORT) + + private var webDriver: WebDriver? = null + + private val handDataHandlers = arrayListOf>() + + override fun onInit() { + server.addMessageHandler { message -> + try { + val rawData = message.split(",").filter { it.isNotEmpty() } + + val id = rawData[0].toInt() + val points = ArrayList() + + var i = 1 + while (i < rawData.size) { + val x = rawData[i + 0].toDouble() + val y = rawData[i + 1].toDouble() + val z = rawData[i + 2].toDouble() + + points.add(Point3D(x, y, z)) + + i += 3 + } + + Async.startAsyncFX { + handDataHandlers.forEach { it.accept(Hand(id, points)) } + } + + } catch (e: Exception) { + log.warning("Failed to parse message.", e) + } + } + + server.start() + } + + /** + * Starts this service in a background thread. + * Can be called after stop() to restart the service. + * If the service has already started, then calls stop() and restarts it. + */ + fun start() { + Async.startAsync { + try { + if (webDriver != null) { + stop() + } + + val options = ChromeOptions() + options.addArguments("--headless=new") + options.addArguments("--use-fake-ui-for-media-stream") + + webDriver = ChromeDriver(options) + webDriver!!.get(WebAPI.GESTURE_RECOGNITION_API) + + // we are ready to use the web api service + } catch (e: Exception) { + log.warning("Failed to start Chrome web driver. Ensure Chrome is installed in default location") + log.warning("Error data", e) + } + } + } + + /** + * Stops this service. + * No-op if it has not started via start() before. + */ + fun stop() { + try { + if (webDriver != null) { + webDriver!!.quit() + webDriver = null + } + } catch (e: Exception) { + log.warning("Failed to quit web driver", e) + } + } + + fun addInputHandler(handler: Consumer) { + handDataHandlers += handler + } + + fun removeInputHandler(handler: Consumer) { + handDataHandlers -= handler + } + + override fun onExit() { + stop() + server.stop() + } +} \ No newline at end of file