React Native Expo Crash CameraWithTensors

[Jersyfi]

System information

• React Native with Expo Development Build
• Running on iOS (iPhone 12 Pro)
• Yarn Package Manager (Also tried with npm)

// Versions:
// package.json
{
    "name": "demo-app",
    "version": "1.0.0",
    "private": true,
    "main": "node_modules/expo/AppEntry.js",
    "scripts": {
        "start": "npx expo start --dev-client --clear",
        "lint": "tsc && eslint --config .eslintrc.js .",
        "lint:fix": "tsc && eslint --config .eslintrc.js --fix .",
        "format:check": "prettier --config .prettierrc.js --check .",
        "format:write": "prettier --config .prettierrc.js --write .",
        "build:dev:android": "eas build --profile development --platform android",
        "build:dev:ios": "eas build --profile development --platform ios",
        "install:check": "npx expo install --check",
    },
    "dependencies": {
        "@react-native-async-storage/async-storage": "1.18.2",
        "@react-native-firebase/app": "^18.6.2",
        "@react-native-firebase/auth": "^18.6.2",
        "@react-native-firebase/firestore": "^18.7.1",
        "@react-native-firebase/functions": "^18.6.2",
        "@react-native-firebase/storage": "^18.7.1",
        "@react-navigation/bottom-tabs": "^6.5.8",
        "@react-navigation/native": "^6.1.7",
        "@react-navigation/native-stack": "^6.9.13",
        "@tensorflow-models/pose-detection": "^2.1.3",
        "@tensorflow/tfjs": "^4.15.0",
        "@tensorflow/tfjs-backend-webgl": "^4.15.0",
        "@tensorflow/tfjs-react-native": "^1.0.0",
        "expo": "^49.0.21",
        "expo-build-properties": "~0.8.3",
        "expo-camera": "~13.4.4",
        "expo-dev-client": "~2.4.12",
        "expo-file-system": "~15.4.5",
        "expo-gl": "~13.0.1",
        "expo-gl-cpp": "^11.4.0",
        "expo-image": "~1.3.5",
        "expo-linear-gradient": "~12.3.0",
        "expo-screen-orientation": "~6.0.6",
        "expo-splash-screen": "~0.20.5",
        "expo-status-bar": "~1.6.0",
        "expo-system-ui": "~2.4.0",
        "expo-web-browser": "~12.3.2",
        "install": "^0.13.0",
        "lucide-react-native": "^0.293.0",
        "npx": "^10.2.2",
        "react": "18.2.0",
        "react-hook-form": "^7.48.2",
        "react-native": "0.72.6",
        "react-native-fs": "^2.20.0",
        "react-native-reanimated": "~3.3.0",
        "react-native-safe-area-context": "4.6.3",
        "react-native-screens": "~3.22.0",
        "react-native-svg": "13.9.0"
    },
    "devDependencies": {
        "@babel/core": "^7.20.0",
        "@types/react": "^18.2.38",
        "@typescript-eslint/eslint-plugin": "^6.12.0",
        "@typescript-eslint/parser": "^6.12.0",
        "eslint": "^8.45.0",
        "eslint-config-prettier": "^9.0.0",
        "eslint-config-universe": "^12.0.0",
        "eslint-plugin-react-native": "^4.1.0",
        "prettier": "3.0.0",
        "typescript": "<5.3.0"
    }
}

Describe the current behavior
When I try to use TensorCamera as intended, the app crashes and I can't find the error. I have tested different versions, but I have no working result.

Describe the expected behavior
The implementation of TensorCamera should work as expected when using the documentations. The pose-detection example should be updated to newer versions.

Standalone code to reproduce the issue

// Playground.tsx
import { NativeStackScreenProps } from '@react-navigation/native-stack';
import * as tf from '@tensorflow/tfjs';
import { cameraWithTensors } from '@tensorflow/tfjs-react-native';
import { Camera, CameraType } from 'expo-camera';
import React, { useEffect, useRef, useState } from 'react';
import { StyleSheet, Text, TouchableOpacity, View } from 'react-native';
import { SafeAreaView } from 'react-native-safe-area-context';

import { TasksStackParamList } from './TasksNavigation';

import HomeLayout from '@/components/layout/home';
import { Badge } from '@/components/ui/badge';
import { Header } from '@/components/ui/typography';
import { ExpoWebGLRenderingContext } from 'expo-gl';

const TensorCamera = cameraWithTensors(Camera);
// Decides wether the front or back camera should be used
const CAMERA_TYPE = CameraType.front;

type Props = NativeStackScreenProps<TasksStackParamList, 'Playground'>;

const Playground = ({ navigation, route }: Props) => {
    const cameraRef = useRef<Camera>(null);
    const [tfReady, setTfReady] = useState(false);

    useEffect(() => {
        const prepare = async () => {
            await tf.ready();
            setTfReady(true);
        };

        prepare();
    }, []);

    const handleTensorCameraStream = async (
        images: IterableIterator<tf.Tensor3D>,
        updatePreview: () => void,
        gl: ExpoWebGLRenderingContext
    ) => {
        const loop = async () => {
            const imageTensor = images.next().value as tf.Tensor3D;
            console.log(imageTensor);
        };

        loop();
    };

    const handleCameraStream = () => {
        const loop = async () => {
            const image = await cameraRef.current?.takePictureAsync();
            console.log(image);
        };

        loop();
    };

    return (
        <HomeLayout>
            {tfReady ? (
                <TensorCamera
                    // Standard Camera props
                    style={styles.camera}
                    type={CAMERA_TYPE}
                    // tensor related props
                    useCustomShadersToResize={false}
                    cameraTextureHeight={1920}
                    cameraTextureWidth={1080}
                    resizeWidth={640}
                    resizeHeight={500}
                    resizeDepth={3}
                    rotation={undefined}
                    onReady={handleTensorCameraStream}
                    autorender={false}
                /> /*( <Camera
                    ref={cameraRef}
                    style={{ flex: 1 }}
                    type={CAMERA_TYPE}
                    onCameraReady={handleCameraStream}
                />
            )*/
            ) : (
                <View style={styles.loadingMsg}>
                    <Text>Loading...</Text>
                </View>
            )}
            <SafeAreaView
                style={{
                    backgroundColor: 'transparent',
                    position: 'absolute',
                    paddingHorizontal: 20,
                }}
            >
                <View style={{ flex: 1, flexDirection: 'row', gap: 5 }}>
                    <TouchableOpacity onPress={() => navigation.navigate('Home')}>
                        <Badge style={{}}>Back</Badge>
                    </TouchableOpacity>
                    <Badge>{`TensorFlow: ${tfReady ? 'Ready!' : 'Initializing...'}`}</Badge>
                </View>
            </SafeAreaView>
            <View style={{ height: 250, padding: 24 }}>
                <Header>Test</Header>
            </View>
        </HomeLayout>
    );
};

export default Playground;

const styles = StyleSheet.create({
    loadingMsg: {
        width: '100%',
        height: '100%',
        alignItems: 'center',
        justifyContent: 'center',
    },
    camera: {
        width: '100%',
        height: '100%',
        zIndex: 1,
    },
});

[gaikwadrahul8]

Hi, @Jersyfi

I apologize for the delayed response and thank you for bringing this issue to our attention. Could you please help me with your Github repo and complete steps to reproduce the same behavior from our end also. Thank you for your understanding and patience.

[Jersyfi]

Hi @gaikwadrahul8, it tried to build up a test app with minimal packages and i found out that i need to remove the packages "install": "^0.13.0" and "npx": "^10.2.2". I then did a rebuild and it worked.

Update for tfjsexamples/react-native/pose-detection

The package version can be updated and a small fix can be made in oder to work with "expo-camera": "~13.4.4".
Camera.Constants.Type.back/.front need to be changed to CameraType.back/.front and the iOS Flipped image need to be removed.

I created a pull request for these changes. tensorflow/tfjs-examples#1215

[gaikwadrahul8]

Hi, @Jersyfi

Good to hear that you're able to resolve your issue and thank you for submitting the PR to take care of this issue and our appropriate team will review your PR and will take appropriate action from their end and please note that due to the upcoming Christmas and New Year holidays, the review and merge process may take longer than usual.

I really appreciate your valuable efforts and time for bringing this issue to our attention and submission of PR to take care of this issue. Thank you for your understanding and patience.

[Jersyfi]

expo-gl-cpp needs to be removed. The hole tfjs-react-native package needs an update of the depencies. More updates on this topic in the issue #8292 (comment).

[google-ml-butler]

Are you satisfied with the resolution of your issue?
Yes
No