feat: Check for input size mismatch (#17)

* feat: Add VisionCamera example * fix: Fix ts setup * Log inputs * feat: Check if input sizes mismatch in TFLite
mrousavy · Jan 19, 2024 · 0313eec · 0313eec
1 parent 252403e
commit 0313eec
Show file tree

Hide file tree

Showing 11 changed files with 2,170 additions and 2,031 deletions.
diff --git a/cpp/TensorHelpers.cpp b/cpp/TensorHelpers.cpp
@@ -142,7 +142,7 @@ void TensorHelpers::updateJSBufferFromTensor(jsi::Runtime& runtime, TypedArrayBa
   }
 
   // count of bytes, may be larger than count of numbers (e.g. for float32)
-  size_t size = getTensorTotalLength(tensor) * getTFLTensorDataTypeSize(dataType);
+  int size = getTensorTotalLength(tensor) * getTFLTensorDataTypeSize(dataType);
 
   switch (dataType) {
     case kTfLiteFloat32:
@@ -194,12 +194,17 @@ void TensorHelpers::updateJSBufferFromTensor(jsi::Runtime& runtime, TypedArrayBa
 void TensorHelpers::updateTensorFromJSBuffer(jsi::Runtime& runtime, TfLiteTensor* tensor,
                                              TypedArrayBase& jsBuffer) {
   auto name = std::string(TfLiteTensorName(tensor));
-  void* data = TfLiteTensorData(tensor);
-  if (data == nullptr) {
-    throw std::runtime_error("Failed to get data from tensor \"" + name + "\"!");
-  }
-
   auto buffer = jsBuffer.getBuffer(runtime);
+
+#if DEBUG
+  int inputBufferSize = buffer.size(runtime);
+  int tensorSize = getTensorTotalLength(tensor) * getTFLTensorDataTypeSize(tensor->type);
+  if (tensorSize != inputBufferSize) {
+    throw std::runtime_error("Input Buffer size (" + std::to_string(inputBufferSize) + ") does not "
+                             "match the Input Tensor's expected size (" + std::to_string(tensorSize) + ")! "
+                             "Make sure to resize the input values accordingly.");
+  }
+#endif
 
   TfLiteTensorCopyFromBuffer(tensor, buffer.data(runtime) + jsBuffer.byteOffset(runtime),
                              buffer.size(runtime));

diff --git a/example/babel.config.js b/example/babel.config.js
@@ -4,6 +4,7 @@ const pak = require('../package.json');
 module.exports = {
   presets: ['module:metro-react-native-babel-preset'],
   plugins: [
+    'react-native-worklets-core/plugin',
     [
       'module-resolver',
       {

diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock
@@ -374,6 +374,13 @@ PODS:
   - React-jsinspector (0.72.1)
   - React-logger (0.72.1):
     - glog
+  - react-native-fast-tflite (1.0.0):
+    - RCT-Folly (= 2021.07.22.00)
+    - React-Core
+  - react-native-worklets-core (0.2.4):
+    - React
+    - React-callinvoker
+    - React-Core
   - React-NativeModulesApple (0.72.1):
     - hermes-engine
     - React-callinvoker
@@ -485,9 +492,11 @@ PODS:
     - React-logger (= 0.72.1)
     - React-perflogger (= 0.72.1)
   - SocketRocket (0.6.1)
-  - react-native-fast-tflite (0.1.0):
-    - RCT-Folly (= 2021.07.22.00)
+  - VisionCamera (3.7.0):
+    - React
+    - React-callinvoker
     - React-Core
+    - react-native-worklets-core
   - Yoga (1.14.0)
   - YogaKit (1.18.1):
     - Yoga (~> 1.14)
@@ -538,6 +547,8 @@ DEPENDENCIES:
   - React-jsiexecutor (from `../node_modules/react-native/ReactCommon/jsiexecutor`)
   - React-jsinspector (from `../node_modules/react-native/ReactCommon/jsinspector`)
   - React-logger (from `../node_modules/react-native/ReactCommon/logger`)
+  - react-native-fast-tflite (from `../..`)
+  - react-native-worklets-core (from `../node_modules/react-native-worklets-core`)
   - React-NativeModulesApple (from `../node_modules/react-native/ReactCommon/react/nativemodule/core/platform/ios`)
   - React-perflogger (from `../node_modules/react-native/ReactCommon/reactperflogger`)
   - React-RCTActionSheet (from `../node_modules/react-native/Libraries/ActionSheetIOS`)
@@ -555,7 +566,7 @@ DEPENDENCIES:
   - React-runtimescheduler (from `../node_modules/react-native/ReactCommon/react/renderer/runtimescheduler`)
   - React-utils (from `../node_modules/react-native/ReactCommon/react/utils`)
   - ReactCommon/turbomodule/core (from `../node_modules/react-native/ReactCommon`)
-  - react-native-fast-tflite (from `../..`)
+  - VisionCamera (from `../node_modules/react-native-vision-camera`)
   - Yoga (from `../node_modules/react-native/ReactCommon/yoga`)
 
 SPEC REPOS:
@@ -619,6 +630,10 @@ EXTERNAL SOURCES:
     :path: "../node_modules/react-native/ReactCommon/jsinspector"
   React-logger:
     :path: "../node_modules/react-native/ReactCommon/logger"
+  react-native-fast-tflite:
+    :path: "../.."
+  react-native-worklets-core:
+    :path: "../node_modules/react-native-worklets-core"
   React-NativeModulesApple:
     :path: "../node_modules/react-native/ReactCommon/react/nativemodule/core/platform/ios"
   React-perflogger:
@@ -653,8 +668,8 @@ EXTERNAL SOURCES:
     :path: "../node_modules/react-native/ReactCommon/react/utils"
   ReactCommon:
     :path: "../node_modules/react-native/ReactCommon"
-  react-native-fast-tflite:
-    :path: "../.."
+  VisionCamera:
+    :path: "../node_modules/react-native-vision-camera"
   Yoga:
     :path: "../node_modules/react-native/ReactCommon/yoga"
 
@@ -692,6 +707,8 @@ SPEC CHECKSUMS:
   React-jsiexecutor: 184eae1ecdedc7a083194bd9ff809c93f08fd34c
   React-jsinspector: d0b5bfd1085599265f4212034321e829bdf83cc0
   React-logger: b8103c9b04e707b50cdd2b1aeb382483900cbb37
+  react-native-fast-tflite: 823498aab954834f709ddcaff82bc0e542fd08e3
+  react-native-worklets-core: b4094f51cb2bc649e297206425cb8956f4945e3e
   React-NativeModulesApple: 4f31a812364443cee6ef768d256c594ad3b20f53
   React-perflogger: 3d501f34c8d4b10cb75f348e43591765788525ad
   React-RCTActionSheet: f5335572c979198c0c3daff67b07bd1ad8370c1d
@@ -710,10 +727,10 @@ SPEC CHECKSUMS:
   React-utils: 0a70ea97d4e2749f336b450c082905be1d389435
   ReactCommon: e593d19c9e271a6da4d0bd7f13b28cfeae5d164b
   SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17
-  react-native-fast-tflite: 0966a28ed0d2e0bf927b91558bb39183a670f5be
+  VisionCamera: 45f8d46c0657ceac3f58eb1a3116fe9afa8f4277
   Yoga: 65286bb6a07edce5e4fe8c90774da977ae8fc009
   YogaKit: f782866e155069a2cca2517aafea43200b01fd5a
 
 PODFILE CHECKSUM: 9ac9e7462894d8a61cb346a87c5aeb9996b11140
 
-COCOAPODS: 1.12.1
+COCOAPODS: 1.14.3
diff --git a/example/ios/TfliteExample/Info.plist b/example/ios/TfliteExample/Info.plist
@@ -16,6 +16,8 @@
 	<string>$(PRODUCT_NAME)</string>
 	<key>CFBundlePackageType</key>
 	<string>APPL</string>
+	<key>NSCameraUsageDescription</key>
+	<string>$(PRODUCT_NAME) needs access to your Camera.</string>
 	<key>CFBundleShortVersionString</key>
 	<string>$(MARKETING_VERSION)</string>
 	<key>CFBundleSignature</key>

diff --git a/example/package.json b/example/package.json
@@ -10,7 +10,9 @@
   },
   "dependencies": {
     "react": "18.2.0",
-    "react-native": "0.72.1"
+    "react-native": "0.72.1",
+    "react-native-vision-camera": "^3.7.0",
+    "react-native-worklets-core": "^0.2.4"
   },
   "devDependencies": {
     "@babel/core": "^7.20.0",
@@ -19,10 +21,10 @@
     "@react-native/eslint-config": "^0.72.2",
     "@react-native/metro-config": "^0.72.7",
     "@types/metro-config": "^0.76.3",
-    "metro-react-native-babel-preset": "0.76.5",
-    "babel-plugin-module-resolver": "^5.0.0"
+    "babel-plugin-module-resolver": "^5.0.0",
+    "metro-react-native-babel-preset": "0.76.5"
   },
   "engines": {
     "node": ">=16"
   }
-}
+}
diff --git a/example/src/App.tsx b/example/src/App.tsx
@@ -1,32 +1,92 @@
 import * as React from 'react'
 
-import { StyleSheet, View, Text, Platform } from 'react-native'
 import {
+  StyleSheet,
+  View,
+  Text,
+  Platform,
+  ActivityIndicator,
+} from 'react-native'
+import {
+  Tensor,
+  TensorflowModel,
   useTensorflowModel,
 } from 'react-native-fast-tflite'
+import {
+  Camera,
+  useCameraDevice,
+  useCameraPermission,
+  useFrameProcessor,
+} from 'react-native-vision-camera'
+
+function tensorToString(tensor: Tensor): string {
+  return `\n  - ${tensor.dataType} ${tensor.name}[${tensor.shape}]`
+}
+function modelToString(model: TensorflowModel): string {
+  return (
+    `TFLite Model (${model.delegate}):\n` +
+    `- Inputs: ${model.inputs.map(tensorToString).join('')}\n` +
+    `- Outputs: ${model.outputs.map(tensorToString).join('')}`
+  )
+}
 
 export default function App() {
-  const [result, setResult] = React.useState('')
+  const { hasPermission, requestPermission } = useCameraPermission()
+  const device = useCameraDevice('back')
 
   const model = useTensorflowModel(
     require('../assets/object_detection_mobile_object_localizer_v1_1_default_1.tflite'),
     Platform.OS === 'ios' ? 'core-ml' : 'default'
   )
+  const actualModel = model.state === 'loaded' ? model.model : undefined
 
   React.useEffect(() => {
-    if (model.model == null) return
+    if (actualModel == null) return
+    console.log(`Model loaded! Shape:\n${modelToString(actualModel)}]`)
+  }, [actualModel])
 
-    console.log(`Running Model...`)
-    const r = model.model.run([new Uint8Array([5])])
-    r.then((output) => {
-      console.log(`Successfully ran Model!`, output)
-      setResult(`${output[0]}${output[1]}${output[2]}...`)
-    })
-  }, [model.model])
+  const frameProcessor = useFrameProcessor(
+    (frame) => {
+      'worklet'
+      if (actualModel == null) {
+        // model is still loading...
+        return
+      }
+
+      console.log(`Running inference on ${frame}`)
+      const data = frame.toArrayBuffer()
+      const result = actualModel.runSync([data])
+      console.log('Result: ' + result.length)
+    },
+    [actualModel]
+  )
+
+  React.useEffect(() => {
+    requestPermission()
+  }, [requestPermission])
+
+  console.log(`Model: ${model.state} (${model.model != null})`)
 
   return (
     <View style={styles.container}>
-      <Text>Result: {result}</Text>
+      {hasPermission && device != null ? (
+        <Camera
+          device={device}
+          style={StyleSheet.absoluteFill}
+          isActive={true}
+          frameProcessor={frameProcessor}
+        />
+      ) : (
+        <Text>No Camera available.</Text>
+      )}
+
+      {model.state === 'loading' && (
+        <ActivityIndicator size="small" color="white" />
+      )}
+
+      {model.state === 'error' && (
+        <Text>Failed to load model! {model.error.message}</Text>
+      )}
     </View>
   )
 }

diff --git a/example/yarn.lock b/example/yarn.lock
@@ -4603,6 +4603,18 @@ react-is@^17.0.1:
   resolved "https://registry.yarnpkg.com/react-is/-/react-is-17.0.2.tgz#e691d4a8e9c789365655539ab372762b0efb54f0"
   integrity sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==
 
+react-native-vision-camera@^3.7.0:
+  version "3.7.0"
+  resolved "https://registry.yarnpkg.com/react-native-vision-camera/-/react-native-vision-camera-3.7.0.tgz#acec4c0491b06b02e84749a20fc112aedc75e16d"
+  integrity sha512-+QvoNXc6QPBVhJjI+uBIWOUOBuno9n+jGLDjgoqtwBkG9GBEsysTwB8ajZCo48H+MQkACmBsnNvObzzpjK6JeA==
+
+react-native-worklets-core@^0.2.4:
+  version "0.2.4"
+  resolved "https://registry.yarnpkg.com/react-native-worklets-core/-/react-native-worklets-core-0.2.4.tgz#a4a79c04f2c6769ff03e96b7529c8638c88fa560"
+  integrity sha512-NKqxLRDOYfO7RJRZHHDA/1yztdiSadSTTILgHw2VwSrcQcmnWAZGXWr9EYpzKblpbk/vXfJa6QpWIKHjNrbe4w==
+  dependencies:
+    string-hash-64 "^1.0.3"
+
 react-native@0.72.1:
   version "0.72.1"
   resolved "https://registry.yarnpkg.com/react-native/-/react-native-0.72.1.tgz#a60f8494ec74cc201ed095aaf2ff2c912c708ec6"
@@ -5027,6 +5039,11 @@ statuses@~1.5.0:
   resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c"
   integrity sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==
 
+string-hash-64@^1.0.3:
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/string-hash-64/-/string-hash-64-1.0.3.tgz#0deb56df58678640db5c479ccbbb597aaa0de322"
+  integrity sha512-D5OKWKvDhyVWWn2x5Y9b+37NUllks34q1dCDhk/vYcso9fmhs+Tl3KR/gE4v5UNj2UA35cnX4KdVVGkG1deKqw==
+
 string-natural-compare@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/string-natural-compare/-/string-natural-compare-3.0.1.tgz#7a42d58474454963759e8e8b7ae63d71c1e7fdf4"

diff --git a/package.json b/package.json
@@ -60,21 +60,21 @@
   },
   "devDependencies": {
     "@react-native-community/eslint-config": "^3.0.2",
-    "@release-it/conventional-changelog": "^5.0.0",
-    "@types/jest": "^28.1.2",
-    "@types/react": "~17.0.21",
-    "@types/react-native": "0.70.0",
-    "eslint": "^8.4.1",
-    "eslint-config-prettier": "^8.5.0",
-    "eslint-plugin-prettier": "^4.0.0",
-    "jest": "^28.1.1",
-    "pod-install": "^0.1.0",
-    "prettier": "^2.0.5",
+    "@release-it/conventional-changelog": "^8.0.1",
+    "@types/jest": "^29.5.11",
+    "@types/react": "~18.2.47",
+    "@types/react-native": "0.73.0",
+    "eslint": "^8.56.0",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-plugin-prettier": "^5.1.3",
+    "jest": "^29.7.0",
+    "pod-install": "^0.1.39",
+    "prettier": "^3.1.1",
     "react": "18.2.0",
-    "react-native": "0.72.1",
-    "react-native-builder-bob": "^0.20.0",
-    "release-it": "^15.0.0",
-    "typescript": "^5.0.2"
+    "react-native": "0.73.2",
+    "react-native-builder-bob": "^0.23.2",
+    "release-it": "^17.0.1",
+    "typescript": "^5.3.3"
   },
   "resolutions": {
     "@types/react": "17.0.21"

diff --git a/src/TensorflowLite.ts b/src/TensorflowLite.ts
@@ -32,7 +32,7 @@ console.log('Successfully installed!')
 
 export type TensorflowModelDelegate = 'default' | 'metal' | 'core-ml'
 
-interface Tensor {
+export interface Tensor {
   /**
    * The name of the Tensor.
    */

diff --git a/tsconfig.json b/tsconfig.json
@@ -1,33 +1,35 @@
 {
-  "exclude": [
-    "tensorflow",
-    "node_modules",
-    "lib",
-    "example"
-  ],
-  "include": [
-    "src"
-  ],
   "compilerOptions": {
-    "baseUrl": "./",
+    "allowJs": false,
     "allowUnreachableCode": false,
     "allowUnusedLabels": false,
     "esModuleInterop": true,
     "forceConsistentCasingInFileNames": true,
-    "jsx": "react",
+    "jsx": "react-native",
     "lib": ["esnext"],
     "module": "esnext",
     "moduleResolution": "node",
     "noFallthroughCasesInSwitch": true,
     "noImplicitReturns": true,
-    "noImplicitUseStrict": false,
     "noStrictGenericChecks": false,
-    "noUncheckedIndexedAccess": true,
     "noUnusedLocals": true,
     "noUnusedParameters": true,
+    "noUncheckedIndexedAccess": true,
     "resolveJsonModule": true,
     "skipLibCheck": true,
     "strict": true,
-    "target": "esnext"
-  }
+    "target": "esnext",
+    "outDir": "lib"
+  },
+  "include": [
+    "src",
+    ".eslintrc.js",
+    "babel.config.js",
+  ],
+  "exclude": [
+    "node_modules",
+    "lib",
+    "docs",
+    "example"
+  ]
 }