Tw/feat text recognition

.changeset/nine-ads-drum.md

@@ -0,0 +1,6 @@
+---
+"@infinitered/react-native-mlkit-text-recognition": major
+"example-app": major
+---
+
+Added first version of Text Recognition module

.gitignore

@@ -609,3 +609,6 @@ $RECYCLE.BIN/
 # .pnp.*
 
 # End of https://www.toptal.com/developers/gitignore/api/intellij,reactnative,turbo,yarn,react,macos,windows,swift,java,kotlin,objective-c
+
+# Do not version control .claude folders
+./claude

apps/ExampleApp/app.json

@@ -28,7 +28,10 @@
         "foregroundImage": "./assets/images/app-icon-android-adaptive-foreground.png",
         "backgroundImage": "./assets/images/app-icon-android-adaptive-background.png",
         "backgroundColor": "#F4F2F1"
-      }
+      },
+      "permissions": [
+        "android.permission.RECORD_AUDIO"
+      ]
     },
     "ios": {
       "icon": "./assets/images/app-icon-ios.png",
@@ -37,7 +40,8 @@
       "infoPlist": {
         "NSCameraUsageDescription": "This app uses the camera to take pictures to demo the machine learning algorithms. (Face detection, Object detection and Image Labeling).",
         "NSPhotoLibraryUsageDescription": "This app uses the photo library to select images for Machine Learning purposes. i.e. Object and Image detection."
-      }
+      },
+      "appleTeamId": "L7YNDPLSEB"
     },
     "web": {
       "favicon": "./assets/images/app-icon-web-favicon.png",
@@ -64,6 +68,12 @@
           "backgroundColor": "#F4F2F1",
           "imageWidth": 250
         }
+      ],
+      [
+        "expo-image-picker",
+        {
+          "photosPermission": "This app uses the photo library to select images for Machine Learning purposes. i.e. Object and Image detection."
+        }
       ]
     ],
     "experiments": {

apps/ExampleApp/app/navigators/AppNavigator.tsx

@@ -34,6 +34,7 @@ export type AppStackParamList = {
   ImageLabeling: Record<string, never>
   ObjectDetection: Record<string, never>
   DocumentScanner: Record<string, never>
+  TextRecognition: Record<string, never>
   // IGNITE_GENERATOR_ANCHOR_APP_STACK_PARAM_LIST
 }
 
@@ -61,6 +62,7 @@ const AppStack = observer(function AppStack() {
       <Stack.Screen name="ImageLabeling" component={Screens.ImageLabelingScreen} />
       <Stack.Screen name="ObjectDetection" component={Screens.ObjectDetectionScreen} />
       <Stack.Screen name="DocumentScanner" component={Screens.DocumentScannerScreen} />
+      <Stack.Screen name="TextRecognition" component={Screens.TextRecognitionScreen} />
       {/* IGNITE_GENERATOR_ANCHOR_APP_STACK_SCREENS */}
     </Stack.Navigator>
   )

apps/ExampleApp/app/screens/HomeScreen/demoInfo.ts

@@ -11,6 +11,7 @@ export interface DemoInfo {
 const FACE_DETECTION = require("../../../assets/images/face-detection.jpg")
 const FACE_HOLDER = require("../../../assets/images/welcome-face.png")
 const DOCUMENT_SCANNER = require("../../../assets/images/doc-scanner.png")
+const TEXT_RECOGNITION = require("../../../assets/images/text-recognition.png")
 
 const ANDROID_ONLY_DEMOS: DemoInfo[] = [
   {
@@ -46,5 +47,11 @@ export const DEMO_LIST: DemoInfo[] = [
     screen: "ImageLabeling",
     image: FACE_HOLDER,
   },
+  {
+    title: "Text recognition",
+    description: "Recognize text in an image",
+    screen: "TextRecognition",
+    image: TEXT_RECOGNITION,
+  },
   ...PLATFORM_SPECIFIC_DEMOS,
 ]

apps/ExampleApp/app/screens/TextRecognitionScreen.tsx

@@ -0,0 +1,130 @@
+import React, { FC, useState, useEffect, useCallback } from "react"
+import { observer } from "mobx-react-lite"
+import { ViewStyle, View, ImageStyle, TextStyle } from "react-native"
+import { NativeStackScreenProps } from "@react-navigation/native-stack"
+import { AppStackScreenProps } from "../navigators"
+import { Text, Icon, ImageSelector, Screen } from "../components"
+import { useTypedNavigation } from "../navigators/useTypedNavigation"
+
+import { recognizeText } from "@infinitered/react-native-mlkit-text-recognition"
+import { UseExampleImageStatus, SelectedImage } from "../utils/useExampleImage"
+
+type TextRecognitionScreenProps = NativeStackScreenProps<AppStackScreenProps<"TextRecognition">>
+
+export const TextRecognitionScreen: FC<TextRecognitionScreenProps> = observer(
+  function TextRecognitionScreen() {
+    const navigation = useTypedNavigation<"TextRecognition">()
+
+    const [image, setImage] = useState<SelectedImage | null>(null)
+
+    const handleImageChange = useCallback((nextImage: SelectedImage) => {
+      setImage(nextImage)
+    }, [])
+
+    const [result, setResult] = useState<string | null>(null)
+    const [status, setStatus] = useState<
+      "init" | "noPermissions" | "done" | "error" | "loading" | UseExampleImageStatus
+    >("init")
+
+    const onStatusChange = React.useCallback(
+      (status: "init" | "noPermissions" | "done" | "error" | "loading" | UseExampleImageStatus) => {
+        setStatus(status)
+      },
+      [],
+    )
+
+    useEffect(() => {
+      const recognizeImage = async () => {
+        if (!image?.uri) return
+        setStatus("recognizing")
+        try {
+          const recognitionResult = await recognizeText(image.uri)
+          setResult(recognitionResult.text)
+          setStatus("done")
+        } catch (error) {
+          console.error("Error recognizing image:", error)
+          setStatus("error")
+        }
+      }
+
+      recognizeImage().then(() => null)
+    }, [image])
+
+    const statusMessage = React.useMemo(() => {
+      if (!image && status !== "init") {
+        setStatus("init")
+      }
+      switch (status) {
+        case "init":
+          return "Take a photo or select one from your camera roll"
+        case "noPermissions":
+          return "You need to grant camera permissions to take a photo"
+        case "takingPhoto":
+          return "Taking photo..."
+        case "selectingPhoto":
+          return "Selecting photo..."
+        case "done":
+          return "Done!"
+        case "error":
+          return "Error during recognition!"
+        case "recognizing":
+          return "Recognizing Image..."
+        case "loading":
+          return "Loading Example Images..."
+        default:
+          throw new Error("Invalid status")
+      }
+    }, [result, image, status])
+
+    const clearResults = useCallback(() => {
+      setResult(null)
+    }, [])
+
+    return (
+      <Screen style={$root} preset="scroll" safeAreaEdges={["top", "bottom"]}>
+        <View>
+          <Icon icon={"back"} onPress={() => navigation.navigate("Home")} style={$backIcon} />
+          <Text preset={"heading"} text="Text Recognition" />
+          <Text style={$description}>Take a photo, and extract text from it.</Text>
+        </View>
+        <ImageSelector
+          onImageChange={handleImageChange}
+          onImageClear={clearResults}
+          onStatusChange={onStatusChange}
+          statusMessage={statusMessage}
+          status={status}
+          isLoading={false}
+          images={{
+            filter: "all",
+            groupBy: "label",
+          }}
+        />
+
+        {result && (
+          <View style={$resultContainer}>
+            <Text>{result}</Text>
+          </View>
+        )}
+      </Screen>
+    )
+  },
+)
+
+const $root: ViewStyle = {
+  flex: 1,
+  padding: 16,
+  display: "flex",
+  flexDirection: "column",
+}
+const $backIcon: ImageStyle = { marginVertical: 8 }
+
+const $description: TextStyle = {
+  marginVertical: 8,
+  color: "rgba(0,0,0,0.6)",
+}
+
+const $resultContainer: ViewStyle = {
+  width: "100%",
+  borderWidth: 1,
+  marginVertical: 24,
+}

apps/ExampleApp/app/screens/index.ts

@@ -7,3 +7,4 @@ export * from "./ImageLabelingScreen"
 export * from "./DocumentScannerScreen"
 export { BOX_COLORS } from "./FaceDetectionScreen"
 export * from "./ObjectDetectionScreen"
+export * from "./TextRecognitionScreen"

apps/ExampleApp/app/utils/useExampleImage/useExampleImage.ts

@@ -23,6 +23,7 @@ export type UseExampleImageStatus =
   | "takingPhoto"
   | "selectingPhoto"
   | "classifying"
+  | "recognizing"
   | "done"
   | "error"
   | "loading"
@@ -126,13 +127,13 @@ export function useExampleImage(predicates?: {
       return
     }
     setStatus("takingPhoto")
-    const result = await launchCameraAsync(IMAGE_PICKER_OPTIONS)
+    const result: ImagePickerResult = await launchCameraAsync(IMAGE_PICKER_OPTIONS)
     if (result.assets?.[0]) {
       setImage({ ...result.assets?.[0], localUri: result.assets?.[0].uri } as SelectedImage)
     } else {
       setImage(undefined)
     }
-  }, [checkPermissions, setStatus]) // Note: Removed parentheses from launchCameraAsync
+  }, [checkPermissions, setStatus])
 
   const [currentIndexes, setCurrentIndexes] = useState<Record<string, number>>(
     {} as Record<string, number>,

apps/ExampleApp/package.json

@@ -36,6 +36,7 @@
     "@infinitered/react-native-mlkit-face-detection": "workspace:^5.0.0",
     "@infinitered/react-native-mlkit-image-labeling": "workspace:^5.0.0",
     "@infinitered/react-native-mlkit-object-detection": "workspace:^5.0.0",
+    "@infinitered/react-native-mlkit-text-recognition": "workspace:^1.0.0",
     "@react-native-async-storage/async-storage": "2.2.0",
     "@react-navigation/native": "^6.0.8",
     "@react-navigation/native-stack": "^6.0.2",

docs/text-recognition/_category_.json

@@ -0,0 +1,8 @@
+{
+  "label": "Text Recognition",
+  "position": 500,
+  "link": {
+    "type": "generated-index",
+    "description": "Recognize text"
+  }
+}

docs/text-recognition/index.md

@@ -0,0 +1,98 @@
+---
+sidebar_position: 1
+title: Getting Started
+---
+
+# Object Detection
+
+## Getting Started
+
+This is an expo module that lets you use
+the [MLKit Text Recognition](https://developers.google.com/ml-kit/vision/text-recognition/v2) library in your Expo app.
+
+## Installation
+
+Install like any other npm package:
+
+```bash
+#yarn
+yarn add @infinitered/react-native-mlkit-text-recognition
+
+#npm
+npm install @infinitered/react-native-mlkit-text-recognition
+```
+
+## Basic Usage
+
+The models are made available through the context system. You can access them in your components using the same hook
+
+```tsx
+// MyComponent.tsx
+import { recognizeText } from "@infinitered/react-native-mlkit-text-recognition";
+import React, { useEffect, useState } from "react";
+import { View } from "react-native";
+import type { MyModelsConfig } from "./App";
+
+type Props = {
+  imagePath: string;
+};
+
+function MyComponent({ imagePath }: Props) {
+  const [recognizedText, setRecognizedText] = useState<string | null>(null);
+
+  useEffect(() => {
+    async function recognizeTextAsync(imagePath: string) {
+      try {
+        const { text } = await recognizeText(imagePath);
+        setRecognizedText(text);
+      } catch (error) {
+        console.error("Error recognizing text:", error);
+      }
+    }
+
+    if (imagePath) {
+      recognizeTextAsync(imagePath);
+    }
+  }, [imagePath]);
+
+  return (
+    <View>
+      <Text>{recognizedText}</Text>
+    </View>
+  );
+}
+```
+
+### Recognition Results
+
+The `recognizeText` method returns an `Text` object:
+
+```ts
+interface Rect {
+  left: number;
+  top: number;
+  right: number;
+  bottom: number;
+}
+
+interface TextBase {
+  text: string;
+  frame: Rect;
+  recognizedLanguage: string;
+}
+
+interface TextElement extends TextBase {}
+
+interface TextLine extends TextBase {
+  elements: TextElement[];
+}
+
+interface TextBlock extends TextBase {
+  lines: TextLine[];
+}
+
+interface Text {
+  text: string;
+  textBlocks: TextBlock[];
+}
+```

modules/react-native-mlkit-text-recognition/.eslintrc.js

@@ -0,0 +1,8 @@
+module.exports = {
+  root: true,
+  extends: ["universe/native", "universe/web"],
+  ignorePatterns: ["build"],
+  rules: {
+    "@typescript-eslint/array-type": "off",
+  },
+};