diff --git a/package/example/ios/Frame Processor Plugins/Example Swift Plugin/ExampleSwiftFrameProcessor.m b/package/example/ios/Frame Processor Plugins/Example Swift Plugin/ExampleSwiftFrameProcessor.m index ac36332183..19dfc034e5 100644 --- a/package/example/ios/Frame Processor Plugins/Example Swift Plugin/ExampleSwiftFrameProcessor.m +++ b/package/example/ios/Frame Processor Plugins/Example Swift Plugin/ExampleSwiftFrameProcessor.m @@ -13,5 +13,6 @@ // // Example for a Swift Frame Processor plugin automatic registration VISION_EXPORT_SWIFT_FRAME_PROCESSOR(ExampleSwiftFrameProcessorPlugin, example_kotlin_swift_plugin) +VISION_EXPORT_SWIFT_FRAME_PROCESSOR(ObjectDetectorPlugin, object_detector_plugin) #endif diff --git a/package/example/ios/Frame Processor Plugins/Example Swift Plugin/ExampleSwiftFrameProcessor.swift b/package/example/ios/Frame Processor Plugins/Example Swift Plugin/ExampleSwiftFrameProcessor.swift index f9403c2335..87318558ca 100644 --- a/package/example/ios/Frame Processor Plugins/Example Swift Plugin/ExampleSwiftFrameProcessor.swift +++ b/package/example/ios/Frame Processor Plugins/Example Swift Plugin/ExampleSwiftFrameProcessor.swift @@ -7,42 +7,58 @@ #if VISION_CAMERA_ENABLE_FRAME_PROCESSORS import VisionCamera +import MediaPipeTasksVision // Example for a Swift Frame Processor plugin @objc(ExampleSwiftFrameProcessorPlugin) public class ExampleSwiftFrameProcessorPlugin: FrameProcessorPlugin { + private let handLandmarker: HandLandmarker + public override init(proxy: VisionCameraProxyHolder, options: [AnyHashable: Any]! = [:]) { - super.init(proxy: proxy, options: options) - - print("ExampleSwiftFrameProcessorPlugin initialized with options: \(String(describing: options))") - } - public override func callback(_ frame: Frame, withArguments arguments: [AnyHashable: Any]?) -> Any? { - let imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer) + guard let modelPath = Bundle.main.path(forResource: "hand_landmarker", + ofType: "task") else { + fatalError("Model not found!") + } - if let arguments, let imageBuffer { - let width = CVPixelBufferGetWidth(imageBuffer) - let height = CVPixelBufferGetHeight(imageBuffer) - let count = arguments.count + let landmarkerOptions = HandLandmarkerOptions() + landmarkerOptions.baseOptions.modelAssetPath = modelPath + landmarkerOptions.runningMode = .video + landmarkerOptions.minHandDetectionConfidence = 0.6 + landmarkerOptions.minHandPresenceConfidence = 0.6 + landmarkerOptions.minTrackingConfidence = 0.6 + landmarkerOptions.numHands = 2 - print( - "ExampleSwiftPlugin: \(width) x \(height) Image. Logging \(count) parameters:" - ) + guard let handLandmarker = try? HandLandmarker(options: landmarkerOptions) else { + fatalError("Failed to init Hand Landmarker!") + } + self.handLandmarker = handLandmarker + super.init(proxy: proxy, options: options) + } - for key in arguments.keys { - let value = arguments[key] - let valueString = String(describing: value) - let valueClassString = String(describing: value.self) - print("ExampleSwiftPlugin: -> \(valueString) (\(valueClassString))") + public override func callback(_ frame: Frame, withArguments arguments: [AnyHashable: Any]?) -> Any? { + do { + let image = try MPImage(sampleBuffer: frame.buffer) + let results = try handLandmarker.detect(videoFrame: image, timestampInMilliseconds: Int(frame.timestamp)) + + var hands: [[String: Any]] = [] + for i in 0.. Any? { + do { + let image = VisionImage(buffer: frame.buffer) + let results = try detector.results(in: image) + return results.map { obj in + return [ + "x": obj.frame.origin.x, + "y": obj.frame.origin.y, + "width": obj.frame.size.width, + "height": obj.frame.size.height, + ] + } + } catch (let error) { + print("Error: \(error.localizedDescription)") + return [] + } + } +} diff --git a/package/example/ios/Podfile b/package/example/ios/Podfile index e4cb12da67..a2190feee6 100644 --- a/package/example/ios/Podfile +++ b/package/example/ios/Podfile @@ -37,6 +37,9 @@ target 'VisionCameraExample' do :app_path => "#{Pod::Config.instance.installation_root}/.." ) + pod 'MediaPipeTasksVision' + pod 'GoogleMLKit/ObjectDetection' + pod 'GoogleMLKit/FaceDetection' pod 'VisionCamera', :path => '../..' require_relative './VisionCameraExampleCocoaPodUtils.rb' diff --git a/package/example/ios/Podfile.lock b/package/example/ios/Podfile.lock index da2ed6afb2..ea227c6a50 100644 --- a/package/example/ios/Podfile.lock +++ b/package/example/ios/Podfile.lock @@ -11,13 +11,85 @@ PODS: - ReactCommon/turbomodule/core (= 0.72.7) - fmt (6.2.1) - glog (0.3.5) + - GoogleDataTransport (9.4.1): + - GoogleUtilities/Environment (~> 7.7) + - nanopb (< 2.30911.0, >= 2.30908.0) + - PromisesObjC (< 3.0, >= 1.2) + - GoogleMLKit/FaceDetection (6.0.0): + - GoogleMLKit/MLKitCore + - MLKitFaceDetection (~> 5.0.0) + - GoogleMLKit/MLKitCore (6.0.0): + - MLKitCommon (~> 11.0.0) + - GoogleMLKit/ObjectDetection (6.0.0): + - GoogleMLKit/MLKitCore + - MLKitObjectDetection (~> 5.0.0) + - GoogleToolboxForMac/Defines (4.2.1) + - GoogleToolboxForMac/Logger (4.2.1): + - GoogleToolboxForMac/Defines (= 4.2.1) + - "GoogleToolboxForMac/NSData+zlib (4.2.1)": + - GoogleToolboxForMac/Defines (= 4.2.1) + - GoogleUtilities/Environment (7.13.2): + - GoogleUtilities/Privacy + - PromisesObjC (< 3.0, >= 1.2) + - GoogleUtilities/Logger (7.13.2): + - GoogleUtilities/Environment + - GoogleUtilities/Privacy + - GoogleUtilities/Privacy (7.13.2) + - GoogleUtilities/UserDefaults (7.13.2): + - GoogleUtilities/Logger + - GoogleUtilities/Privacy + - GoogleUtilitiesComponents (1.1.0): + - GoogleUtilities/Logger + - GTMSessionFetcher/Core (3.4.1) - hermes-engine (0.72.7): - hermes-engine/Pre-built (= 0.72.7) - hermes-engine/Pre-built (0.72.7) - libevent (2.1.12) + - MediaPipeTasksCommon (0.10.13) + - MediaPipeTasksVision (0.10.13): + - MediaPipeTasksCommon (= 0.10.13) + - MLImage (1.0.0-beta5) + - MLKitCommon (11.0.0): + - GoogleDataTransport (< 10.0, >= 9.4.1) + - GoogleToolboxForMac/Logger (< 5.0, >= 4.2.1) + - "GoogleToolboxForMac/NSData+zlib (< 5.0, >= 4.2.1)" + - GoogleUtilities/UserDefaults (< 8.0, >= 7.13.0) + - GoogleUtilitiesComponents (~> 1.0) + - GTMSessionFetcher/Core (< 4.0, >= 3.3.2) + - MLKitFaceDetection (5.0.0): + - MLKitCommon (~> 11.0) + - MLKitVision (~> 7.0) + - MLKitImageLabelingCommon (7.0.0): + - MLKitCommon (~> 11.0) + - MLKitVision (~> 7.0) + - MLKitObjectDetection (5.0.0): + - MLKitCommon (~> 11.0) + - MLKitObjectDetectionCommon (~> 7.0) + - MLKitVision (~> 7.0) + - MLKitVisionKit (~> 8.0) + - MLKitObjectDetectionCommon (7.0.0): + - MLKitCommon (~> 11.0) + - MLKitVision (~> 7.0) + - MLKitVision (7.0.0): + - GoogleToolboxForMac/Logger (< 5.0, >= 4.2.1) + - "GoogleToolboxForMac/NSData+zlib (< 5.0, >= 4.2.1)" + - GTMSessionFetcher/Core (< 4.0, >= 3.3.2) + - MLImage (= 1.0.0-beta5) + - MLKitCommon (~> 11.0) + - MLKitVisionKit (8.0.0): + - MLKitCommon (~> 11.0) + - MLKitImageLabelingCommon (~> 7.0) + - MLKitObjectDetectionCommon (~> 7.0) + - MLKitVision (~> 7.0) - MMKV (1.3.4): - MMKVCore (~> 1.3.4) - MMKVCore (1.3.4) + - nanopb (2.30910.0): + - nanopb/decode (= 2.30910.0) + - nanopb/encode (= 2.30910.0) + - nanopb/decode (2.30910.0) + - nanopb/encode (2.30910.0) + - PromisesObjC (2.4.0) - RCT-Folly (2021.07.22.00): - boost - DoubleConversion @@ -487,8 +559,11 @@ DEPENDENCIES: - FBLazyVector (from `../node_modules/react-native/Libraries/FBLazyVector`) - FBReactNativeSpec (from `../node_modules/react-native/React/FBReactNativeSpec`) - glog (from `../node_modules/react-native/third-party-podspecs/glog.podspec`) + - GoogleMLKit/FaceDetection + - GoogleMLKit/ObjectDetection - hermes-engine (from `../node_modules/react-native/sdks/hermes-engine/hermes-engine.podspec`) - libevent (~> 2.1.12) + - MediaPipeTasksVision - RCT-Folly (from `../node_modules/react-native/third-party-podspecs/RCT-Folly.podspec`) - RCTRequired (from `../node_modules/react-native/Libraries/RCTRequired`) - RCTTypeSafety (from `../node_modules/react-native/Libraries/TypeSafety`) @@ -540,9 +615,27 @@ DEPENDENCIES: SPEC REPOS: trunk: - fmt + - GoogleDataTransport + - GoogleMLKit + - GoogleToolboxForMac + - GoogleUtilities + - GoogleUtilitiesComponents + - GTMSessionFetcher - libevent + - MediaPipeTasksCommon + - MediaPipeTasksVision + - MLImage + - MLKitCommon + - MLKitFaceDetection + - MLKitImageLabelingCommon + - MLKitObjectDetection + - MLKitObjectDetectionCommon + - MLKitVision + - MLKitVisionKit - MMKV - MMKVCore + - nanopb + - PromisesObjC - SocketRocket EXTERNAL SOURCES: @@ -659,10 +752,28 @@ SPEC CHECKSUMS: FBReactNativeSpec: 638095fe8a01506634d77b260ef8a322019ac671 fmt: ff9d55029c625d3757ed641535fd4a75fedc7ce9 glog: 04b94705f318337d7ead9e6d17c019bd9b1f6b1b + GoogleDataTransport: 6c09b596d841063d76d4288cc2d2f42cc36e1e2a + GoogleMLKit: 97ac7af399057e99182ee8edfa8249e3226a4065 + GoogleToolboxForMac: d1a2cbf009c453f4d6ded37c105e2f67a32206d8 + GoogleUtilities: c56430aef51a1aa57b25da78c3f8397e522c67b7 + GoogleUtilitiesComponents: 679b2c881db3b615a2777504623df6122dd20afe + GTMSessionFetcher: 8000756fc1c19d2e5697b90311f7832d2e33f6cd hermes-engine: 9180d43df05c1ed658a87cc733dc3044cf90c00a libevent: 4049cae6c81cdb3654a443be001fb9bdceff7913 + MediaPipeTasksCommon: 5c86b477b18fa034db290aead83d1009be3dbcff + MediaPipeTasksVision: 4782191be198e124756e76e66bb3a8917850b1cb + MLImage: 1824212150da33ef225fbd3dc49f184cf611046c + MLKitCommon: afec63980417d29ffbb4790529a1b0a2291699e1 + MLKitFaceDetection: 7c0e8bf09ddd27105da32d088fca978a99fc30cc + MLKitImageLabelingCommon: fa802f14c3a321121409f6cbe1df89ba9cf1a00c + MLKitObjectDetection: 9b40359730baceef23322c6d41070da7639ff100 + MLKitObjectDetectionCommon: f3ca8581a0741be6ce609920367316571a280efa + MLKitVision: e858c5f125ecc288e4a31127928301eaba9ae0c1 + MLKitVisionKit: 6d2ea9741b262ec0fed2a5a3de521cfa671c6e83 MMKV: ed58ad794b3f88c24d604a5b74f3fba17fcbaf74 MMKVCore: a67a1cede26175c413176f404a7cedec43f96a0b + nanopb: 438bc412db1928dac798aa6fd75726007be04262 + PromisesObjC: f5707f49cb48b9636751c5b2e7d227e43fba9f47 RCT-Folly: 424b8c9a7a0b9ab2886ffe9c3b041ef628fd4fb1 RCTRequired: 83bca1c184feb4d2e51c72c8369b83d641443f95 RCTTypeSafety: 13c4a87a16d7db6cd66006ce9759f073402ef85b @@ -711,6 +822,6 @@ SPEC CHECKSUMS: VisionCamera: b633f90960feab2669b7a1c51f8a201dd0a5bfc3 Yoga: 4c3aa327e4a6a23eeacd71f61c81df1bcdf677d5 -PODFILE CHECKSUM: 66976ac26c778d788a06e6c1bab624e6a1233cdd +PODFILE CHECKSUM: ed5841958979d8eedbe1edba1d1ed35b89429590 -COCOAPODS: 1.11.3 +COCOAPODS: 1.14.3 diff --git a/package/example/ios/VisionCameraExample.xcodeproj/project.pbxproj b/package/example/ios/VisionCameraExample.xcodeproj/project.pbxproj index db949901d2..d453f48ba7 100644 --- a/package/example/ios/VisionCameraExample.xcodeproj/project.pbxproj +++ b/package/example/ios/VisionCameraExample.xcodeproj/project.pbxproj @@ -15,6 +15,9 @@ 839E2C652ACB2E420037BC2B /* ExampleSwiftFrameProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 839E2C642ACB2E420037BC2B /* ExampleSwiftFrameProcessor.swift */; }; B8DB3BDC263DEA31004C18D7 /* ExampleFrameProcessorPlugin.m in Sources */ = {isa = PBXBuildFile; fileRef = B8DB3BD8263DEA31004C18D7 /* ExampleFrameProcessorPlugin.m */; }; B8F0E10825E0199F00586F16 /* File.swift in Sources */ = {isa = PBXBuildFile; fileRef = B8F0E10725E0199F00586F16 /* File.swift */; }; + B8F0E24B2BEE236500E510DF /* hand_landmarker.task in Resources */ = {isa = PBXBuildFile; fileRef = B8F0E24A2BEE18C200E510DF /* hand_landmarker.task */; }; + B8F0E24D2BEE307A00E510DF /* ObjectDetector.swift in Sources */ = {isa = PBXBuildFile; fileRef = B8F0E24C2BEE307A00E510DF /* ObjectDetector.swift */; }; + B8F0E24F2BEE319900E510DF /* FaceDetector.swift in Sources */ = {isa = PBXBuildFile; fileRef = B8F0E24E2BEE319900E510DF /* FaceDetector.swift */; }; C0B129659921D2EA967280B2 /* libPods-VisionCameraExample.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 3CDCFE89C25C89320B98945E /* libPods-VisionCameraExample.a */; }; /* End PBXBuildFile section */ @@ -33,6 +36,9 @@ B8DB3BD8263DEA31004C18D7 /* ExampleFrameProcessorPlugin.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ExampleFrameProcessorPlugin.m; sourceTree = ""; }; B8F0E10625E0199F00586F16 /* VisionCameraExample-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "VisionCameraExample-Bridging-Header.h"; sourceTree = ""; }; B8F0E10725E0199F00586F16 /* File.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = File.swift; sourceTree = ""; }; + B8F0E24A2BEE18C200E510DF /* hand_landmarker.task */ = {isa = PBXFileReference; lastKnownFileType = file; path = hand_landmarker.task; sourceTree = ""; }; + B8F0E24C2BEE307A00E510DF /* ObjectDetector.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ObjectDetector.swift; sourceTree = ""; }; + B8F0E24E2BEE319900E510DF /* FaceDetector.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FaceDetector.swift; sourceTree = ""; }; C1D342AD8210E7627A632602 /* Pods-VisionCameraExample.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-VisionCameraExample.debug.xcconfig"; path = "Target Support Files/Pods-VisionCameraExample/Pods-VisionCameraExample.debug.xcconfig"; sourceTree = ""; }; ED297162215061F000B7C4FE /* JavaScriptCore.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = JavaScriptCore.framework; path = System/Library/Frameworks/JavaScriptCore.framework; sourceTree = SDKROOT; }; ED2971642150620600B7C4FE /* JavaScriptCore.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = JavaScriptCore.framework; path = Platforms/AppleTVOS.platform/Developer/SDKs/AppleTVOS12.0.sdk/System/Library/Frameworks/JavaScriptCore.framework; sourceTree = DEVELOPER_DIR; }; @@ -54,6 +60,7 @@ 13B07FAE1A68108700A75B9A /* VisionCameraExample */ = { isa = PBXGroup; children = ( + B8F0E24A2BEE18C200E510DF /* hand_landmarker.task */, B8DB3BD6263DEA31004C18D7 /* Frame Processor Plugins */, 008F07F21AC5B25A0029DE68 /* main.jsbundle */, 13B07FAF1A68108700A75B9A /* AppDelegate.h */, @@ -99,6 +106,8 @@ children = ( 839E2C622ACB2E330037BC2B /* ExampleSwiftFrameProcessor.m */, 839E2C642ACB2E420037BC2B /* ExampleSwiftFrameProcessor.swift */, + B8F0E24C2BEE307A00E510DF /* ObjectDetector.swift */, + B8F0E24E2BEE319900E510DF /* FaceDetector.swift */, ); path = "Example Swift Plugin"; sourceTree = ""; @@ -206,6 +215,7 @@ isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; files = ( + B8F0E24B2BEE236500E510DF /* hand_landmarker.task in Resources */, 81AB9BB82411601600AC10FF /* LaunchScreen.storyboard in Resources */, 13B07FBF1A68108700A75B9A /* Images.xcassets in Resources */, ); @@ -235,6 +245,15 @@ ); inputPaths = ( "${PODS_ROOT}/Target Support Files/Pods-VisionCameraExample/Pods-VisionCameraExample-resources.sh", + "${PODS_CONFIGURATION_BUILD_DIR}/GTMSessionFetcher/GTMSessionFetcher_Core_Privacy.bundle", + "${PODS_CONFIGURATION_BUILD_DIR}/GoogleDataTransport/GoogleDataTransport_Privacy.bundle", + "${PODS_CONFIGURATION_BUILD_DIR}/GoogleToolboxForMac/GoogleToolboxForMac_Privacy.bundle", + "${PODS_CONFIGURATION_BUILD_DIR}/GoogleToolboxForMac/GoogleToolboxForMac_Logger_Privacy.bundle", + "${PODS_CONFIGURATION_BUILD_DIR}/GoogleUtilities/GoogleUtilities_Privacy.bundle", + "${PODS_CONFIGURATION_BUILD_DIR}/MLKitFaceDetection/GoogleMVFaceDetectorResources.bundle", + "${PODS_CONFIGURATION_BUILD_DIR}/MLKitObjectDetection/MLKitObjectDetectionResources.bundle", + "${PODS_CONFIGURATION_BUILD_DIR}/MLKitObjectDetectionCommon/MLKitObjectDetectionCommonResources.bundle", + "${PODS_CONFIGURATION_BUILD_DIR}/PromisesObjC/FBLPromises_Privacy.bundle", "${PODS_ROOT}/../../node_modules/react-native-vector-icons/Fonts/AntDesign.ttf", "${PODS_ROOT}/../../node_modules/react-native-vector-icons/Fonts/Entypo.ttf", "${PODS_ROOT}/../../node_modules/react-native-vector-icons/Fonts/EvilIcons.ttf", @@ -255,9 +274,19 @@ "${PODS_ROOT}/../../node_modules/react-native-vector-icons/Fonts/SimpleLineIcons.ttf", "${PODS_ROOT}/../../node_modules/react-native-vector-icons/Fonts/Zocial.ttf", "${PODS_CONFIGURATION_BUILD_DIR}/React-Core/AccessibilityResources.bundle", + "${PODS_CONFIGURATION_BUILD_DIR}/nanopb/nanopb_Privacy.bundle", ); name = "[CP] Copy Pods Resources"; outputPaths = ( + "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/GTMSessionFetcher_Core_Privacy.bundle", + "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/GoogleDataTransport_Privacy.bundle", + "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/GoogleToolboxForMac_Privacy.bundle", + "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/GoogleToolboxForMac_Logger_Privacy.bundle", + "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/GoogleUtilities_Privacy.bundle", + "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/GoogleMVFaceDetectorResources.bundle", + "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/MLKitObjectDetectionResources.bundle", + "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/MLKitObjectDetectionCommonResources.bundle", + "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/FBLPromises_Privacy.bundle", "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/AntDesign.ttf", "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/Entypo.ttf", "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/EvilIcons.ttf", @@ -278,6 +307,7 @@ "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/SimpleLineIcons.ttf", "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/Zocial.ttf", "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/AccessibilityResources.bundle", + "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/nanopb_Privacy.bundle", ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; @@ -391,6 +421,8 @@ 839E2C652ACB2E420037BC2B /* ExampleSwiftFrameProcessor.swift in Sources */, 839E2C632ACB2E330037BC2B /* ExampleSwiftFrameProcessor.m in Sources */, B8F0E10825E0199F00586F16 /* File.swift in Sources */, + B8F0E24D2BEE307A00E510DF /* ObjectDetector.swift in Sources */, + B8F0E24F2BEE319900E510DF /* FaceDetector.swift in Sources */, 13B07FC11A68108700A75B9A /* main.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/package/example/ios/hand_landmarker.task b/package/example/ios/hand_landmarker.task new file mode 100644 index 0000000000..0d53faf378 Binary files /dev/null and b/package/example/ios/hand_landmarker.task differ diff --git a/package/example/src/CameraPage.tsx b/package/example/src/CameraPage.tsx index 28ae31146c..130675c4d6 100644 --- a/package/example/src/CameraPage.tsx +++ b/package/example/src/CameraPage.tsx @@ -3,13 +3,15 @@ import { useRef, useState, useCallback, useMemo } from 'react' import type { GestureResponderEvent } from 'react-native' import { StyleSheet, Text, View } from 'react-native' import type { PinchGestureHandlerGestureEvent } from 'react-native-gesture-handler' +import { NativeViewGestureHandler } from 'react-native-gesture-handler' import { PinchGestureHandler, TapGestureHandler } from 'react-native-gesture-handler' import type { CameraProps, CameraRuntimeError, PhotoFile, VideoFile } from 'react-native-vision-camera' +import { VisionCameraProxy } from 'react-native-vision-camera' import { runAtTargetFps, useCameraDevice, useCameraFormat, - useFrameProcessor, + useSkiaFrameProcessor, useLocationPermission, useMicrophonePermission, } from 'react-native-vision-camera' @@ -29,6 +31,7 @@ import { useIsFocused } from '@react-navigation/core' import { usePreferredCameraDevice } from './hooks/usePreferredCameraDevice' import { examplePlugin } from './frame-processors/ExamplePlugin' import { exampleKotlinSwiftPlugin } from './frame-processors/ExampleKotlinSwiftPlugin' +import { ColorChannel, Paint, PaintStyle, PointMode, Skia, StrokeJoin, TileMode } from '@shopify/react-native-skia' const ReanimatedCamera = Reanimated.createAnimatedComponent(Camera) Reanimated.addWhitelistedNativeProps({ @@ -71,9 +74,7 @@ export function CameraPage({ navigation }: Props): React.ReactElement { const format = useCameraFormat(device, [ { fps: targetFps }, { videoAspectRatio: screenAspectRatio }, - { videoResolution: 'max' }, - { photoAspectRatio: screenAspectRatio }, - { photoResolution: 'max' }, + { videoResolution: { width: 720, height: 1080 } }, ]) const fps = Math.min(format?.maxFps ?? 1, targetFps) @@ -178,16 +179,181 @@ export function CameraPage({ navigation }: Props): React.ReactElement { location.requestPermission() }, [location]) - const frameProcessor = useFrameProcessor((frame) => { - 'worklet' + interface Pos { + x: number + y: number + width: number + height: number + } + interface Landmark { + x: number + y: number + z: number + visibility?: number + } + interface Hand { + landmarks: Landmark[] + } + + const rectPaint = Skia.Paint() + rectPaint.setColor(Skia.Color('red')) + rectPaint.setStyle(PaintStyle.Stroke) + rectPaint.setStrokeWidth(10) + rectPaint.setStrokeJoin(StrokeJoin.Round) + + const red = Skia.Paint() + red.setColor(Skia.Color('red')) + + const green = Skia.Paint() + green.setColor(Skia.Color('#2c944b')) + green.setStrokeWidth(5) + + const effect = Skia.RuntimeEffect.Make( + ` + uniform shader image; + +vec4 chromatic(vec2 pos, float offset) { + float r = image.eval(pos).r; + float g = image.eval(vec2(pos.x + offset, pos.y)).g; + float b = image.eval(vec2(pos.x + offset * 2.0, pos.y)).b; + return vec4(r, g, b, 1.0); +} + +half4 main(vec2 pos) { + float offset = 50.0; + return chromatic(pos, offset); +} + + `, + ) + if (effect == null) throw new Error('Failed to compile SKSL!') + const shader = Skia.RuntimeShaderBuilder(effect) + + const invert = Skia.RuntimeShaderBuilder( + Skia.RuntimeEffect.Make(` + uniform shader image; + +half4 main(vec2 pos) { + vec4 color = image.eval(pos); + return vec4(1.0 - color.rgb, 1.0); +} +`), + ) + + const p = Skia.Paint() - runAtTargetFps(10, () => { + const lines = useMemo( + () => + [ + [0, 1], + [1, 2], + [2, 3], + [3, 4], + [0, 5], + [5, 6], + [6, 7], + [7, 8], + [5, 9], + [9, 10], + [10, 11], + [11, 12], + [9, 13], + [13, 14], + [14, 15], + [15, 16], + [13, 17], + [17, 18], + [18, 19], + [19, 20], + [0, 17], + ] as const, + [], + ) + + type Samples = 'object-detection' | 'hand-detection' | 'vhs' | 'invert' + + const plugin = useMemo(() => VisionCameraProxy.initFrameProcessorPlugin('object_detector_plugin', {}), []) + + const demo = useMemo(() => Worklets.createSharedValue('hand-detection'), []) + + const frameProcessor = useSkiaFrameProcessor( + (frame) => { 'worklet' - console.log(`${frame.timestamp}: ${frame.width}x${frame.height} ${frame.pixelFormat} Frame (${frame.orientation})`) - examplePlugin(frame) - exampleKotlinSwiftPlugin(frame) - }) - }, []) + + switch (demo.value) { + case 'vhs': + p.setImageFilter(Skia.ImageFilter.MakeRuntimeShader(shader, null, null)) + frame.render(p) + break + case 'invert': + p.setImageFilter(Skia.ImageFilter.MakeRuntimeShader(invert, null, null)) + frame.render(p) + break + + case 'object-detection': { + frame.render() + + const results = plugin?.call(frame) as undefined | Pos[] + if (results == null) return + for (const r of results) { + const rect = Skia.XYWHRect(r.x, r.y, r.width, r.height) + frame.drawRect(rect, rectPaint) + } + break + } + case 'hand-detection': { + frame.render() + + const hands = exampleKotlinSwiftPlugin(frame) as unknown as Hand[] + + const width = frame.width + const height = frame.height + + for (const hand of hands) { + const points = hand.landmarks.map((l) => ({ + point: Skia.Point(l.x * width, l.y * height), + opacity: l.visibility, + })) + + for (const line of lines) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const from = points[line[0]]! + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const to = points[line[1]]! + green.setAlphaf(Math.min(from.opacity ?? 1, to.opacity ?? 1)) + frame.drawLine(from.point.x, from.point.y, to.point.x, to.point.y, green) + } + for (const { point, opacity } of points) { + red.setAlphaf(opacity ?? 1) + frame.drawCircle(point.x, point.y, 10, red) + } + } + break + } + } + }, + [demo, p, shader, invert, plugin, rectPaint, lines, green, red], + ) + + useEffect(() => { + const i = setInterval(() => { + switch (demo.value) { + case 'hand-detection': + demo.value = 'invert' + break + case 'invert': + demo.value = 'vhs' + break + case 'vhs': + demo.value = 'object-detection' + break + case 'object-detection': + demo.value = 'hand-detection' + break + } + }, 2000) + return () => clearInterval(i) + }, [demo]) const videoHdr = format?.supportsVideoHdr && enableHdr const photoHdr = format?.supportsPhotoHdr && enableHdr && !videoHdr @@ -223,6 +389,7 @@ export function CameraPage({ navigation }: Props): React.ReactElement { audio={microphone.hasPermission} enableLocation={location.hasPermission} frameProcessor={frameProcessor} + pixelFormat="rgb" />