feat: Hand Detection

package/example/ios/Frame Processor Plugins/Example Swift Plugin/ExampleSwiftFrameProcessor.swift

@@ -7,42 +7,58 @@
 
 #if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
 import VisionCamera
+import MediaPipeTasksVision
 
 // Example for a Swift Frame Processor plugin
 @objc(ExampleSwiftFrameProcessorPlugin)
 public class ExampleSwiftFrameProcessorPlugin: FrameProcessorPlugin {
+  private let handLandmarker: HandLandmarker
+
   public override init(proxy: VisionCameraProxyHolder, options: [AnyHashable: Any]! = [:]) {
-    super.init(proxy: proxy, options: options)
-
-    print("ExampleSwiftFrameProcessorPlugin initialized with options: \(String(describing: options))")
-  }
 
-  public override func callback(_ frame: Frame, withArguments arguments: [AnyHashable: Any]?) -> Any? {
-    let imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer)
+    guard let modelPath = Bundle.main.path(forResource: "hand_landmarker",
+                                           ofType: "task") else {
+      fatalError("Model not found!")
+    }
 
-    if let arguments, let imageBuffer {
-      let width = CVPixelBufferGetWidth(imageBuffer)
-      let height = CVPixelBufferGetHeight(imageBuffer)
-      let count = arguments.count
+    let landmarkerOptions = HandLandmarkerOptions()
+    landmarkerOptions.baseOptions.modelAssetPath = modelPath
+    landmarkerOptions.runningMode = .video
+    landmarkerOptions.minHandDetectionConfidence = 0.6
+    landmarkerOptions.minHandPresenceConfidence = 0.6
+    landmarkerOptions.minTrackingConfidence = 0.6
+    landmarkerOptions.numHands = 2
 
-      print(
-        "ExampleSwiftPlugin: \(width) x \(height) Image. Logging \(count) parameters:"
-      )
+    guard let handLandmarker = try? HandLandmarker(options: landmarkerOptions) else {
+      fatalError("Failed to init Hand Landmarker!")
+    }
+    self.handLandmarker = handLandmarker
+    super.init(proxy: proxy, options: options)
+  }
 
-      for key in arguments.keys {
-        let value = arguments[key]
-        let valueString = String(describing: value)
-        let valueClassString = String(describing: value.self)
-        print("ExampleSwiftPlugin:   -> \(valueString) (\(valueClassString))")
+  public override func callback(_ frame: Frame, withArguments arguments: [AnyHashable: Any]?) -> Any? {
+    do {
+      let image = try MPImage(sampleBuffer: frame.buffer)
+      let results = try handLandmarker.detect(videoFrame: image, timestampInMilliseconds: Int(frame.timestamp))
+
+      var hands: [[String: Any]] = []
+      for i in 0..<results.handedness.count {
+        hands.append([
+          "landmarks": results.landmarks[i].map({ landmark in
+            return [
+              "x": NSNumber(value: landmark.x),
+              "y": NSNumber(value: landmark.y),
+              "z": NSNumber(value: landmark.z),
+              "visibility": landmark.visibility
+            ]
+          }),
+        ])
       }
+      return hands
+    } catch (let error) {
+      print("Error: \(error.localizedDescription)")
+      return []
     }
-
-    return [
-      "example_str": "SwiftTest",
-      "example_bool": false,
-      "example_double": 6.7,
-      "example_array": ["Good bye", false, 21.37]
-    ]
   }
 }
 #endif

package/example/ios/Podfile

@@ -37,6 +37,7 @@ target 'VisionCameraExample' do
     :app_path => "#{Pod::Config.instance.installation_root}/.."
   )
 
+  pod 'MediaPipeTasksVision'
   pod 'VisionCamera', :path => '../..'
   require_relative './VisionCameraExampleCocoaPodUtils.rb'
 

package/example/ios/Podfile.lock

@@ -15,6 +15,9 @@ PODS:
     - hermes-engine/Pre-built (= 0.72.7)
   - hermes-engine/Pre-built (0.72.7)
   - libevent (2.1.12)
+  - MediaPipeTasksCommon (0.10.13)
+  - MediaPipeTasksVision (0.10.13):
+    - MediaPipeTasksCommon (= 0.10.13)
   - MMKV (1.3.4):
     - MMKVCore (~> 1.3.4)
   - MMKVCore (1.3.4)
@@ -489,6 +492,7 @@ DEPENDENCIES:
   - glog (from `../node_modules/react-native/third-party-podspecs/glog.podspec`)
   - hermes-engine (from `../node_modules/react-native/sdks/hermes-engine/hermes-engine.podspec`)
   - libevent (~> 2.1.12)
+  - MediaPipeTasksVision
   - RCT-Folly (from `../node_modules/react-native/third-party-podspecs/RCT-Folly.podspec`)
   - RCTRequired (from `../node_modules/react-native/Libraries/RCTRequired`)
   - RCTTypeSafety (from `../node_modules/react-native/Libraries/TypeSafety`)
@@ -541,6 +545,8 @@ SPEC REPOS:
   trunk:
     - fmt
     - libevent
+    - MediaPipeTasksCommon
+    - MediaPipeTasksVision
     - MMKV
     - MMKVCore
     - SocketRocket
@@ -661,6 +667,8 @@ SPEC CHECKSUMS:
   glog: 04b94705f318337d7ead9e6d17c019bd9b1f6b1b
   hermes-engine: 9180d43df05c1ed658a87cc733dc3044cf90c00a
   libevent: 4049cae6c81cdb3654a443be001fb9bdceff7913
+  MediaPipeTasksCommon: 5c86b477b18fa034db290aead83d1009be3dbcff
+  MediaPipeTasksVision: 4782191be198e124756e76e66bb3a8917850b1cb
   MMKV: ed58ad794b3f88c24d604a5b74f3fba17fcbaf74
   MMKVCore: a67a1cede26175c413176f404a7cedec43f96a0b
   RCT-Folly: 424b8c9a7a0b9ab2886ffe9c3b041ef628fd4fb1
@@ -711,6 +719,6 @@ SPEC CHECKSUMS:
   VisionCamera: b633f90960feab2669b7a1c51f8a201dd0a5bfc3
   Yoga: 4c3aa327e4a6a23eeacd71f61c81df1bcdf677d5
 
-PODFILE CHECKSUM: 66976ac26c778d788a06e6c1bab624e6a1233cdd
+PODFILE CHECKSUM: 39b1ed41d1db712916c95a20aa9c116fb1992588
 
-COCOAPODS: 1.11.3
+COCOAPODS: 1.14.3

package/example/ios/VisionCameraExample.xcodeproj/project.pbxproj

@@ -15,6 +15,7 @@
 		839E2C652ACB2E420037BC2B /* ExampleSwiftFrameProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 839E2C642ACB2E420037BC2B /* ExampleSwiftFrameProcessor.swift */; };
 		B8DB3BDC263DEA31004C18D7 /* ExampleFrameProcessorPlugin.m in Sources */ = {isa = PBXBuildFile; fileRef = B8DB3BD8263DEA31004C18D7 /* ExampleFrameProcessorPlugin.m */; };
 		B8F0E10825E0199F00586F16 /* File.swift in Sources */ = {isa = PBXBuildFile; fileRef = B8F0E10725E0199F00586F16 /* File.swift */; };
+		B8F0E24B2BEE236500E510DF /* hand_landmarker.task in Resources */ = {isa = PBXBuildFile; fileRef = B8F0E24A2BEE18C200E510DF /* hand_landmarker.task */; };
 		C0B129659921D2EA967280B2 /* libPods-VisionCameraExample.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 3CDCFE89C25C89320B98945E /* libPods-VisionCameraExample.a */; };
 /* End PBXBuildFile section */
 
@@ -33,6 +34,7 @@
 		B8DB3BD8263DEA31004C18D7 /* ExampleFrameProcessorPlugin.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ExampleFrameProcessorPlugin.m; sourceTree = "<group>"; };
 		B8F0E10625E0199F00586F16 /* VisionCameraExample-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "VisionCameraExample-Bridging-Header.h"; sourceTree = "<group>"; };
 		B8F0E10725E0199F00586F16 /* File.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = File.swift; sourceTree = "<group>"; };
+		B8F0E24A2BEE18C200E510DF /* hand_landmarker.task */ = {isa = PBXFileReference; lastKnownFileType = file; path = hand_landmarker.task; sourceTree = "<group>"; };
 		C1D342AD8210E7627A632602 /* Pods-VisionCameraExample.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-VisionCameraExample.debug.xcconfig"; path = "Target Support Files/Pods-VisionCameraExample/Pods-VisionCameraExample.debug.xcconfig"; sourceTree = "<group>"; };
 		ED297162215061F000B7C4FE /* JavaScriptCore.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = JavaScriptCore.framework; path = System/Library/Frameworks/JavaScriptCore.framework; sourceTree = SDKROOT; };
 		ED2971642150620600B7C4FE /* JavaScriptCore.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = JavaScriptCore.framework; path = Platforms/AppleTVOS.platform/Developer/SDKs/AppleTVOS12.0.sdk/System/Library/Frameworks/JavaScriptCore.framework; sourceTree = DEVELOPER_DIR; };
@@ -54,6 +56,7 @@
 		13B07FAE1A68108700A75B9A /* VisionCameraExample */ = {
 			isa = PBXGroup;
 			children = (
+				B8F0E24A2BEE18C200E510DF /* hand_landmarker.task */,
 				B8DB3BD6263DEA31004C18D7 /* Frame Processor Plugins */,
 				008F07F21AC5B25A0029DE68 /* main.jsbundle */,
 				13B07FAF1A68108700A75B9A /* AppDelegate.h */,
@@ -206,6 +209,7 @@
 			isa = PBXResourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				B8F0E24B2BEE236500E510DF /* hand_landmarker.task in Resources */,
 				81AB9BB82411601600AC10FF /* LaunchScreen.storyboard in Resources */,
 				13B07FBF1A68108700A75B9A /* Images.xcassets in Resources */,
 			);

package/example/src/CameraPage.tsx

@@ -6,10 +6,10 @@
 import { PinchGestureHandler, TapGestureHandler } from 'react-native-gesture-handler'
 import type { CameraProps, CameraRuntimeError, PhotoFile, VideoFile } from 'react-native-vision-camera'
 import {
   runAtTargetFps,
   useCameraDevice,
   useCameraFormat,
-  useFrameProcessor,
+  useSkiaFrameProcessor,
   useLocationPermission,
   useMicrophonePermission,
 } from 'react-native-vision-camera'
@@ -27,8 +27,9 @@
 import type { NativeStackScreenProps } from '@react-navigation/native-stack'
 import { useIsFocused } from '@react-navigation/core'
 import { usePreferredCameraDevice } from './hooks/usePreferredCameraDevice'
 import { examplePlugin } from './frame-processors/ExamplePlugin'
 import { exampleKotlinSwiftPlugin } from './frame-processors/ExampleKotlinSwiftPlugin'
+import { Paint, PointMode, Skia, StrokeJoin } from '@shopify/react-native-skia'
 
 const ReanimatedCamera = Reanimated.createAnimatedComponent(Camera)
 Reanimated.addWhitelistedNativeProps({
@@ -71,9 +72,7 @@
   const format = useCameraFormat(device, [
     { fps: targetFps },
     { videoAspectRatio: screenAspectRatio },
-    { videoResolution: 'max' },
-    { photoAspectRatio: screenAspectRatio },
-    { photoResolution: 'max' },
+    { videoResolution: { width: 720, height: 1080 } },
   ])
 
   const fps = Math.min(format?.maxFps ?? 1, targetFps)
@@ -178,16 +177,83 @@
     location.requestPermission()
   }, [location])
 
-  const frameProcessor = useFrameProcessor((frame) => {
-    'worklet'
+  interface Landmark {
+    x: number
+    y: number
+    z: number
+    visibility?: number
+  }
+  interface Hand {
+    landmarks: Landmark[]
+  }
+
+  const red = Skia.Paint()
+  red.setColor(Skia.Color('red'))
+
+  const green = Skia.Paint()
+  green.setColor(Skia.Color('#2c944b'))
+  green.setStrokeWidth(5)
 
-    runAtTargetFps(10, () => {
+  const lines = useMemo(
+    () =>
+      [
+        [0, 1],
+        [1, 2],
+        [2, 3],
+        [3, 4],
+        [0, 5],
+        [5, 6],
+        [6, 7],
+        [7, 8],
+        [5, 9],
+        [9, 10],
+        [10, 11],
+        [11, 12],
+        [9, 13],
+        [13, 14],
+        [14, 15],
+        [15, 16],
+        [13, 17],
+        [17, 18],
+        [18, 19],
+        [19, 20],
+        [0, 17],
+      ] as const,
+    [],
+  )
+
+  const frameProcessor = useSkiaFrameProcessor(
+    (frame) => {
       'worklet'
-      console.log(`${frame.timestamp}: ${frame.width}x${frame.height} ${frame.pixelFormat} Frame (${frame.orientation})`)
-      examplePlugin(frame)
-      exampleKotlinSwiftPlugin(frame)
-    })
-  }, [])
+
+      frame.render()
+      const hands = exampleKotlinSwiftPlugin(frame) as unknown as Hand[]
+
+      const width = frame.width
+      const height = frame.height
+
+      for (const hand of hands) {
+        const points = hand.landmarks.map((l) => ({
+          point: Skia.Point(l.x * width, l.y * height),
+          opacity: l.visibility,
+        }))
+
+        for (const line of lines) {
+          // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+          const from = points[line[0]]!
+          // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+          const to = points[line[1]]!
+          green.setAlphaf(Math.min(from.opacity ?? 1, to.opacity ?? 1))
+          frame.drawLine(from.point.x, from.point.y, to.point.x, to.point.y, green)
+        }
+        for (const { point, opacity } of points) {
+          red.setAlphaf(opacity ?? 1)
+          frame.drawCircle(point.x, point.y, 10, red)
+        }
+      }
+    },
+    [red, green, lines],
+  )
 
   const videoHdr = format?.supportsVideoHdr && enableHdr
   const photoHdr = format?.supportsPhotoHdr && enableHdr && !videoHdr
@@ -223,6 +289,7 @@
                 audio={microphone.hasPermission}
                 enableLocation={location.hasPermission}
                 frameProcessor={frameProcessor}
+                pixelFormat="rgb"
               />
             </TapGestureHandler>
           </Reanimated.View>