feat: Reanimated v3 support in RNVisionCamera v2 (mrousavy#1759)

* Update Podfile.lock * Install REA v3 * Upgrade project to RN 0.71 * Update build.gradle * Update packages * Add VisionCamera pod again * Update dev deps * Extend TSConfig from RN * Fix Linting setup * fix tsconfig * Update project.pbxproj * Update Info.plist * Use `JSRuntimeHelper` (temporary workaround) * Wrap in GestureHandlerRootView * Use new console.log * fix: Update CameraRoll * It works! * Print _WORKLET in frameProcessor * Enforce shareable type * It works again! * console.log works * Setup console in Reanimated * Adapt to WorkletRuntime * Update Podfile.lock * Update CameraPage.tsx * Android works! * Fix iOS * Remove paths to Reanimated headers from node_modules from `target_include_directories` in CMakeLists.txt * Bump `com.android.tools.build:gradle` from 7.3.1 to 7.4.2 fix issue Gradle Sync issue * Use `@tomekzaw/prefab` branch instead of `link:` * Remove unnecessary header * Consume Reanimated from commit * Remove dependency on Reanimated Scheduler on iOS * Go back * Remove dependency on Reanimated Scheduler on Android * Pass job as runnable * Remove `makeJSIRuntime` * Use nightly * Remove react-native-reanimated from `watchFolders` in metro.config.js * Fix crash on reload * Fix import * Barcode detection works * Use latest commit * Use `runGuarded` instead of `callWithThis` * Remove private field * Remove unused type * fix: Optionally require Reanimated * Update .gitignore * chore: Revert example app changes * chore: Remove unused packages from example app --------- Co-authored-by: Marc Rousavy <me@mrousavy.com> Co-authored-by: Marc Rousavy <marcrousavy@hotmail.com>
j-piasecki · Feb 12, 2024 · 4532d70 · 4532d70
1 parent d454570
commit 4532d70
Show file tree

Hide file tree

Showing 44 changed files with 7,753 additions and 4,739 deletions.
diff --git a/.eslintrc.js b/.eslintrc.js
@@ -10,9 +10,9 @@ module.exports = {
     ecmaVersion: 2018,
     sourceType: 'module',
   },
-  ignorePatterns: ['scripts', 'lib', 'docs', 'example', 'app.plugin.js'],
+  ignorePatterns: ['scripts', 'lib', 'docs', 'example', 'app.plugin.js', '.eslintrc.js'],
   plugins: ['@typescript-eslint'],
-  extends: ['plugin:@typescript-eslint/recommended', '@react-native-community'],
+  extends: ['plugin:@typescript-eslint/recommended', '@react-native'],
   rules: {
     // eslint
     semi: 'off',

diff --git a/.github/workflows/validate-cpp.yml b/.github/workflows/validate-cpp.yml
@@ -24,7 +24,7 @@ jobs:
         with:
           github_token: ${{ secrets.github_token }}
           reporter: github-pr-review
-          flags: --linelength=230 --exclude "android/src/main/cpp/reanimated-headers"
+          flags: --linelength=230
           targets: --recursive cpp android/src/main/cpp
           filter: "-legal/copyright\
             ,-readability/todo\

diff --git a/.gitignore b/.gitignore
@@ -20,6 +20,8 @@ build/
 *.perspectivev3
 !default.perspectivev3
 xcuserdata
+ios/.xcode.env.local
+
 *.xccheckout
 *.moved-aside
 DerivedData
@@ -39,6 +41,10 @@ android.iml
 # Cocoapods
 #
 example/ios/Pods
+/vendor/bundle/
+
+# Temporary files created by Metro to check the health of the file watcher
+.metro-health-check*
 
 # node.js
 #
@@ -67,3 +73,7 @@ docs/docs/api
 # External native build folder generated in Android Studio 2.2 and later
 .externalNativeBuild
 .cxx/
+
+.cxx/
+*.keystore
+!debug.keystore
diff --git a/android/CMakeLists.txt b/android/CMakeLists.txt
@@ -2,7 +2,7 @@ project(VisionCamera)
 cmake_minimum_required(VERSION 3.4.1)
 
 set (CMAKE_VERBOSE_MAKEFILE ON)
-set (CMAKE_CXX_STANDARD 14)
+set (CMAKE_CXX_STANDARD 17)
 
 if(${REACT_NATIVE_VERSION} GREATER_EQUAL 71)
         include("${NODE_MODULES_DIR}/react-native/ReactAndroid/cmake-utils/folly-flags.cmake")
@@ -60,15 +60,6 @@ if(${REACT_NATIVE_VERSION} GREATER_EQUAL 71)
                 "${NODE_MODULES_DIR}/react-native/ReactCommon/react/renderer/graphics/platform/cxx"
                 "${NODE_MODULES_DIR}/react-native/ReactCommon/runtimeexecutor"
                 "${NODE_MODULES_DIR}/react-native/ReactCommon/yoga"
-                # --- Reanimated ---
-                # New
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/AnimatedSensor"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/Tools"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/SpecTools"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/SharedItems"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/Registries"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/LayoutAnimations"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/hidden_headers"
                 "src/main/cpp"
         )
 else()
@@ -95,22 +86,6 @@ else()
                 "${NODE_MODULES_DIR}/hermes-engine/android/include/"
                 ${INCLUDE_JSI_CPP} # only on older RN versions
                 ${INCLUDE_JSIDYNAMIC_CPP} # only on older RN versions
-                # --- Reanimated ---
-                # Old
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/headers/AnimatedSensor"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/headers/Tools"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/headers/SpecTools"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/headers/SharedItems"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/headers/Registries"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/headers/LayoutAnimations"
-                # New
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/AnimatedSensor"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/Tools"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/SpecTools"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/SharedItems"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/Registries"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/LayoutAnimations"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/hidden_headers"
                 "src/main/cpp"
         )
 endif()
@@ -154,7 +129,6 @@ if(${FOR_HERMES})
                         "${BUILD_DIR}/third-party-ndk/hermes/jni/${ANDROID_ABI}/libhermes.so"
                 )
         endif()
-        file (GLOB LIBREANIMATED_DIR "${BUILD_DIR}/react-native-reanimated-*-hermes.aar/jni/${ANDROID_ABI}")
 else()
         file (GLOB LIBJSC_DIR "${BUILD_DIR}/android-jsc*.aar/jni/${ANDROID_ABI}")
 
@@ -173,9 +147,6 @@ else()
                 ${PACKAGE_NAME}
                 ${JS_ENGINE_LIB}
         )
-
-        # Use Reanimated JSC
-        file (GLOB LIBREANIMATED_DIR "${BUILD_DIR}/react-native-reanimated-*-jsc.aar/jni/${ANDROID_ABI}")
 endif()
 
 if(${REACT_NATIVE_VERSION} LESS 71)
@@ -234,12 +205,7 @@ else()
         )
 endif()
 
-find_library(
-        REANIMATED_LIB
-        reanimated
-        PATHS ${LIBREANIMATED_DIR}
-        NO_CMAKE_FIND_ROOT_PATH
-)
+find_package(react-native-reanimated REQUIRED CONFIG)
 
 find_library(
         LOG_LIB
@@ -252,7 +218,7 @@ target_link_libraries(
         ${PACKAGE_NAME}
         ${LOG_LIB}
         ${JSI_LIB}
-        ${REANIMATED_LIB}
+        react-native-reanimated::reanimated
         ${REACT_NATIVE_JNI_LIB}
         ${FBJNI_LIB}
         ${FOLLY_LIB}

diff --git a/android/build.gradle b/android/build.gradle
@@ -109,9 +109,10 @@ def resolveBuildType() {
 }
 
 // plugin.js file only exists since REA v2.
-def hasReanimated2 = file("${nodeModules}/react-native-reanimated/plugin.js").exists()
-def disableFrameProcessors = true
-def ENABLE_FRAME_PROCESSORS = false
+// def hasReanimated2 = file("${nodeModules}/react-native-reanimated/plugin.js").exists()
+// def disableFrameProcessors = rootProject.ext.has("disableFrameProcessors") ? rootProject.ext.get("disableFrameProcessors").asBoolean() : false
+// def ENABLE_FRAME_PROCESSORS = hasReanimated2 && !disableFrameProcessors
+def ENABLE_FRAME_PROCESSORS = true
 
 if (ENABLE_FRAME_PROCESSORS) {
   logger.warn("VisionCamera: Frame Processors are enabled! Building C++ part...")
@@ -320,10 +321,6 @@ dependencies {
       def jscAAR = fileTree("${nodeModules}/jsc-android/dist/org/webkit/android-jsc").matching({ it.include "**/**/*.aar" }).singleFile
       extractJNI(files(rnAAR, jscAAR))
     }
-
-    def jsEngine = FOR_HERMES ? "hermes" : "jsc"
-    def reaAAR = "${nodeModules}/react-native-reanimated/android/react-native-reanimated-${REACT_NATIVE_VERSION}-${jsEngine}.aar"
-    extractJNI(files(reaAAR))
   }
 
   implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"

diff --git a/android/gradle.properties b/android/gradle.properties
@@ -13,7 +13,7 @@
 #Fri Feb 19 20:46:14 CET 2021
 VisionCamera_buildToolsVersion=30.0.0
 VisionCamera_compileSdkVersion=31
-VisionCamera_kotlinVersion=1.5.30
+VisionCamera_kotlinVersion=1.6.20
 VisionCamera_targetSdkVersion=31
 VisionCamera_ndkVersion=21.4.7075529
 android.enableJetifier=true

diff --git a/android/gradle/wrapper/gradle-wrapper.properties b/android/gradle/wrapper/gradle-wrapper.properties
@@ -1,6 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.0.1-all.zip
-networkTimeout=10000
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
diff --git a/android/src/main/cpp/FrameProcessorRuntimeManager.cpp b/android/src/main/cpp/FrameProcessorRuntimeManager.cpp
@@ -8,12 +8,6 @@
 #include <utility>
 #include <string>
 
-#include "RuntimeDecorator.h"
-#include "RuntimeManager.h"
-#include "reanimated-headers/AndroidScheduler.h"
-#include "reanimated-headers/AndroidErrorHandler.h"
-
-#include "MakeJSIRuntime.h"
 #include "CameraView.h"
 #include "FrameHostObject.h"
 #include "JSIJNIConversion.h"
@@ -55,7 +49,6 @@ TSelf vision::FrameProcessorRuntimeManager::initHybrid(
   auto runtime = reinterpret_cast<jsi::Runtime*>(jsRuntimePointer);
   auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker();
   auto scheduler = std::shared_ptr<VisionCameraScheduler>(androidScheduler->cthis());
-  scheduler->setJSCallInvoker(jsCallInvoker);
 
   return makeCxxInstance(jThis, runtime, jsCallInvoker, scheduler);
 }
@@ -65,16 +58,8 @@ void vision::FrameProcessorRuntimeManager::initializeRuntime() {
                       "Initializing Vision JS-Runtime...");
 
   // create JSI runtime and decorate it
-  auto runtime = makeJSIRuntime();
-  reanimated::RuntimeDecorator::decorateRuntime(*runtime, "FRAME_PROCESSOR");
-  runtime->global().setProperty(*runtime, "_FRAME_PROCESSOR",
-                                jsi::Value(true));
 
   // create REA runtime manager
-  auto errorHandler = std::make_shared<reanimated::AndroidErrorHandler>(scheduler_);
-  _runtimeManager = std::make_unique<reanimated::RuntimeManager>(std::move(runtime),
-                                                                 errorHandler,
-                                                                 scheduler_);
 
   __android_log_write(ANDROID_LOG_INFO, TAG,
                       "Initialized Vision JS-Runtime!");
@@ -86,6 +71,11 @@ global_ref<CameraView::javaobject> FrameProcessorRuntimeManager::findCameraViewB
   return make_global(weakCameraView);
 }
 
+void FrameProcessorRuntimeManager::registerPlugins() {
+  static const auto registerPluginsMethod = javaPart_->getClass()->getMethod<void()>("registerPlugins");
+  registerPluginsMethod(javaPart_.get());
+}
+
 void FrameProcessorRuntimeManager::logErrorToJS(const std::string& message) {
   if (!this->jsCallInvoker_) {
     return;
@@ -105,16 +95,18 @@ void FrameProcessorRuntimeManager::logErrorToJS(const std::string& message) {
   });
 }
 
-void FrameProcessorRuntimeManager::setFrameProcessor(jsi::Runtime& runtime,
+void FrameProcessorRuntimeManager::setFrameProcessor(jsi::Runtime& rnRuntime,
                                                      int viewTag,
-                                                     const jsi::Value& frameProcessor) {
+                                                     const jsi::Value& frameProcessor,
+                                                     const jsi::Value& workletRuntimeValue) {
   __android_log_write(ANDROID_LOG_INFO, TAG,
                       "Setting new Frame Processor...");
 
-  if (!_runtimeManager || !_runtimeManager->runtime) {
-    throw jsi::JSError(runtime,
-                       "setFrameProcessor(..): VisionCamera's RuntimeManager is not yet initialized!");
-  }
+  workletRuntime_ = reanimated::extractWorkletRuntime(rnRuntime, workletRuntimeValue);
+  jsi::Runtime &visionRuntime = workletRuntime_->getJSIRuntime();
+  visionRuntime.global().setProperty(visionRuntime, "_FRAME_PROCESSOR", jsi::Value(true));
+
+  registerPlugins();
 
   // find camera view
   auto cameraView = findCameraViewById(viewTag);
@@ -123,27 +115,18 @@ void FrameProcessorRuntimeManager::setFrameProcessor(jsi::Runtime& runtime,
   // convert jsi::Function to a ShareableValue (can be shared across runtimes)
   __android_log_write(ANDROID_LOG_INFO, TAG,
                       "Adapting Shareable value from function (conversion to worklet)...");
-  auto worklet = reanimated::ShareableValue::adapt(runtime,
-                                                   frameProcessor,
-                                                   _runtimeManager.get());
+  auto shareableWorklet = reanimated::extractShareableOrThrow<reanimated::ShareableWorklet>(rnRuntime, frameProcessor);
   __android_log_write(ANDROID_LOG_INFO, TAG, "Successfully created worklet!");
 
   scheduler_->scheduleOnUI([=]() {
       // cast worklet to a jsi::Function for the new runtime
-      auto& rt = *_runtimeManager->runtime;
-      auto function = std::make_shared<jsi::Function>(worklet->getValue(rt).asObject(rt).asFunction(rt));
-
       // assign lambda to frame processor
-      cameraView->cthis()->setFrameProcessor([this, &rt, function](jni::alias_ref<JImageProxy::javaobject> frame) {
-          try {
-            // create HostObject which holds the Frame (JImageProxy)
-            auto hostObject = std::make_shared<FrameHostObject>(frame);
-            function->callWithThis(rt, *function, jsi::Object::createFromHostObject(rt, hostObject));
-          } catch (jsi::JSError& jsError) {
-            auto message = "Frame Processor threw an error: " + jsError.getMessage();
-            __android_log_write(ANDROID_LOG_ERROR, TAG, message.c_str());
-            this->logErrorToJS(message);
-          }
+      cameraView->cthis()->setFrameProcessor([=](jni::alias_ref<JImageProxy::javaobject> frame) {
+          // create HostObject which holds the Frame (JImageProxy)
+          auto frameHostObject = std::make_shared<FrameHostObject>(frame);
+          jsi::Runtime &runtime = workletRuntime_->getJSIRuntime();
+          auto hostObject = jsi::Object::createFromHostObject(runtime, frameHostObject);
+          workletRuntime_->runGuarded(shareableWorklet, hostObject);
       });
 
       __android_log_write(ANDROID_LOG_INFO, TAG, "Frame Processor set!");
@@ -189,10 +172,15 @@ void FrameProcessorRuntimeManager::installJSIBindings() {
       throw jsi::JSError(runtime,
                          "Camera::setFrameProcessor: Second argument ('frameProcessor') must be a function!");
     }
+    if (!arguments[2].isObject()) {
+      throw jsi::JSError(runtime,
+                         "Camera::setFrameProcessor: Third argument ('workletRuntime') must be an object!");
+    }
 
     double viewTag = arguments[0].asNumber();
     const jsi::Value& frameProcessor = arguments[1];
-    this->setFrameProcessor(runtime, static_cast<int>(viewTag), frameProcessor);
+    const jsi::Value& workletRuntimeValue = arguments[2];
+    this->setFrameProcessor(runtime, static_cast<int>(viewTag), frameProcessor, workletRuntimeValue);
 
     return jsi::Value::undefined();
   };
@@ -235,11 +223,11 @@ void FrameProcessorRuntimeManager::installJSIBindings() {
 
 void FrameProcessorRuntimeManager::registerPlugin(alias_ref<JFrameProcessorPlugin::javaobject> plugin) {
   // _runtimeManager might never be null, but we can never be too sure.
-  if (!_runtimeManager || !_runtimeManager->runtime) {
+  if (!workletRuntime_) {
     throw std::runtime_error("Tried to register plugin before initializing JS runtime! Call `initializeRuntime()` first.");
   }
 
-  auto& runtime = *_runtimeManager->runtime;
+  auto& runtime = workletRuntime_->getJSIRuntime();
 
   // we need a strong reference on the plugin, make_global does that.
   auto pluginGlobal = make_global(plugin);

diff --git a/android/src/main/cpp/FrameProcessorRuntimeManager.h b/android/src/main/cpp/FrameProcessorRuntimeManager.h
@@ -10,8 +10,7 @@
 #include <memory>
 #include <string>
 
-#include "RuntimeManager.h"
-#include "reanimated-headers/AndroidScheduler.h"
+#include "WorkletRuntime.h"
 
 #include "CameraView.h"
 #include "VisionCameraScheduler.h"
@@ -46,18 +45,20 @@ class FrameProcessorRuntimeManager : public jni::HybridClass<FrameProcessorRunti
   jni::global_ref<FrameProcessorRuntimeManager::javaobject> javaPart_;
   jsi::Runtime* runtime_;
   std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker_;
-  std::shared_ptr<reanimated::RuntimeManager> _runtimeManager;
+  std::shared_ptr<reanimated::WorkletRuntime> workletRuntime_;
   std::shared_ptr<vision::VisionCameraScheduler> scheduler_;
 
   jni::global_ref<CameraView::javaobject> findCameraViewById(int viewId);
+  void registerPlugins();
   void initializeRuntime();
   void installJSIBindings();
   void registerPlugin(alias_ref<JFrameProcessorPlugin::javaobject> plugin);
   void logErrorToJS(const std::string& message);
 
   void setFrameProcessor(jsi::Runtime& runtime,                 // NOLINT(runtime/references)
                          int viewTag,
-                         const jsi::Value& frameProcessor);
+                         const jsi::Value& frameProcessor,
+                         const jsi::Value& workletRuntimeValue);
   void unsetFrameProcessor(int viewTag);
 };
 

diff --git a/android/src/main/cpp/MakeJSIRuntime.h b/android/src/main/cpp/MakeJSIRuntime.h