diff --git a/.changeset/nine-ads-drum.md b/.changeset/nine-ads-drum.md new file mode 100644 index 00000000..680479d8 --- /dev/null +++ b/.changeset/nine-ads-drum.md @@ -0,0 +1,6 @@ +--- +"@infinitered/react-native-mlkit-text-recognition": minor +"example-app": major +--- + +Added first version of Text Recognition module diff --git a/.gitignore b/.gitignore index 6fa6abf9..a9b1bb6d 100644 --- a/.gitignore +++ b/.gitignore @@ -609,3 +609,6 @@ $RECYCLE.BIN/ # .pnp.* # End of https://www.toptal.com/developers/gitignore/api/intellij,reactnative,turbo,yarn,react,macos,windows,swift,java,kotlin,objective-c + +# Do not version control .claude folders +./claude \ No newline at end of file diff --git a/apps/ExampleApp/app.json b/apps/ExampleApp/app.json index c2bffb28..ad770a10 100644 --- a/apps/ExampleApp/app.json +++ b/apps/ExampleApp/app.json @@ -28,7 +28,10 @@ "foregroundImage": "./assets/images/app-icon-android-adaptive-foreground.png", "backgroundImage": "./assets/images/app-icon-android-adaptive-background.png", "backgroundColor": "#F4F2F1" - } + }, + "permissions": [ + "android.permission.RECORD_AUDIO" + ] }, "ios": { "icon": "./assets/images/app-icon-ios.png", @@ -37,7 +40,8 @@ "infoPlist": { "NSCameraUsageDescription": "This app uses the camera to take pictures to demo the machine learning algorithms. (Face detection, Object detection and Image Labeling).", "NSPhotoLibraryUsageDescription": "This app uses the photo library to select images for Machine Learning purposes. i.e. Object and Image detection." - } + }, + "appleTeamId": "L7YNDPLSEB" }, "web": { "favicon": "./assets/images/app-icon-web-favicon.png", @@ -64,6 +68,12 @@ "backgroundColor": "#F4F2F1", "imageWidth": 250 } + ], + [ + "expo-image-picker", + { + "photosPermission": "This app uses the photo library to select images for Machine Learning purposes. i.e. Object and Image detection." + } ] ], "experiments": { diff --git a/apps/ExampleApp/app/navigators/AppNavigator.tsx b/apps/ExampleApp/app/navigators/AppNavigator.tsx index bed51bb4..1e355b3c 100644 --- a/apps/ExampleApp/app/navigators/AppNavigator.tsx +++ b/apps/ExampleApp/app/navigators/AppNavigator.tsx @@ -34,6 +34,7 @@ export type AppStackParamList = { ImageLabeling: Record ObjectDetection: Record DocumentScanner: Record + TextRecognition: Record // IGNITE_GENERATOR_ANCHOR_APP_STACK_PARAM_LIST } @@ -61,6 +62,7 @@ const AppStack = observer(function AppStack() { + {/* IGNITE_GENERATOR_ANCHOR_APP_STACK_SCREENS */} ) diff --git a/apps/ExampleApp/app/screens/HomeScreen/demoInfo.ts b/apps/ExampleApp/app/screens/HomeScreen/demoInfo.ts index 5a455057..331dc0f2 100644 --- a/apps/ExampleApp/app/screens/HomeScreen/demoInfo.ts +++ b/apps/ExampleApp/app/screens/HomeScreen/demoInfo.ts @@ -11,6 +11,7 @@ export interface DemoInfo { const FACE_DETECTION = require("../../../assets/images/face-detection.jpg") const FACE_HOLDER = require("../../../assets/images/welcome-face.png") const DOCUMENT_SCANNER = require("../../../assets/images/doc-scanner.png") +const TEXT_RECOGNITION = require("../../../assets/images/text-recognition.png") const ANDROID_ONLY_DEMOS: DemoInfo[] = [ { @@ -46,5 +47,11 @@ export const DEMO_LIST: DemoInfo[] = [ screen: "ImageLabeling", image: FACE_HOLDER, }, + { + title: "Text recognition", + description: "Recognize text in an image", + screen: "TextRecognition", + image: TEXT_RECOGNITION, + }, ...PLATFORM_SPECIFIC_DEMOS, ] diff --git a/apps/ExampleApp/app/screens/TextRecognitionScreen.tsx b/apps/ExampleApp/app/screens/TextRecognitionScreen.tsx new file mode 100644 index 00000000..df574a3e --- /dev/null +++ b/apps/ExampleApp/app/screens/TextRecognitionScreen.tsx @@ -0,0 +1,188 @@ +import React, { FC, useState, useEffect, useCallback } from "react" +import { observer } from "mobx-react-lite" +import { ViewStyle, View, ImageStyle, TextStyle, ScrollView, Pressable } from "react-native" +import { NativeStackScreenProps } from "@react-navigation/native-stack" +import { AppStackScreenProps } from "../navigators" +import { Text, Icon, ImageSelector, Screen } from "../components" +import { useTypedNavigation } from "../navigators/useTypedNavigation" + +import { recognizeText, Text as RecognizedText } from "@infinitered/react-native-mlkit-text-recognition" +import { UseExampleImageStatus, SelectedImage } from "../utils/useExampleImage" + +type TextRecognitionScreenProps = NativeStackScreenProps> + +function DebugOutput({ data }: { data: unknown }) { + const [expanded, setExpanded] = useState(false) + + return ( + + setExpanded(!expanded)} style={$debugHeader}> + {expanded ? "▼" : "▶"} Debug Output + + {expanded && ( + + + {JSON.stringify(data, null, 2)} + + + )} + + ) +} +export const TextRecognitionScreen: FC = observer( + function TextRecognitionScreen() { + const navigation = useTypedNavigation<"TextRecognition">() + + const [image, setImage] = useState(null) + + const handleImageChange = useCallback((nextImage: SelectedImage) => { + setImage(nextImage) + }, []) + + const [result, setResult] = useState(null) + const [status, setStatus] = useState< + "init" | "noPermissions" | "done" | "error" | "loading" | UseExampleImageStatus + >("init") + + const onStatusChange = React.useCallback( + (status: "init" | "noPermissions" | "done" | "error" | "loading" | UseExampleImageStatus) => { + setStatus(status) + }, + [], + ) + + useEffect(() => { + const recognizeImage = async () => { + if (!image?.uri) return + setStatus("recognizing") + try { + const recognitionResult = await recognizeText(image.uri) + setResult(recognitionResult) + setStatus("done") + } catch (error) { + console.error("Error recognizing image:", error) + setStatus("error") + } + } + + recognizeImage().then(() => null) + }, [image]) + + const statusMessage = React.useMemo(() => { + if (!image && status !== "init") { + setStatus("init") + } + switch (status) { + case "init": + return "Take a photo or select one from your camera roll" + case "noPermissions": + return "You need to grant camera permissions to take a photo" + case "takingPhoto": + return "Taking photo..." + case "selectingPhoto": + return "Selecting photo..." + case "done": + return "Done!" + case "error": + return "Error during recognition!" + case "recognizing": + return "Recognizing Image..." + case "loading": + return "Loading Example Images..." + default: + throw new Error("Invalid status") + } + }, [result, image, status]) + + const clearResults = useCallback(() => { + setResult(null) + }, []) + + return ( + + + navigation.navigate("Home")} style={$backIcon} /> + + Take a photo, and extract text from it. + + + + {result && ( + <> + + Recognized Text + {result.text} + + + + )} + + ) + }, +) + +const $root: ViewStyle = { + flex: 1, + padding: 16, + display: "flex", + flexDirection: "column", +} +const $backIcon: ImageStyle = { marginVertical: 8 } + +const $description: TextStyle = { + marginVertical: 8, + color: "rgba(0,0,0,0.6)", +} + +const $resultContainer: ViewStyle = { + width: "100%", + borderWidth: 1, + borderColor: "rgba(0,0,0,0.2)", + borderRadius: 8, + padding: 12, + marginVertical: 16, +} + +const $resultText: TextStyle = { + marginTop: 8, +} + +const $debugContainer: ViewStyle = { + width: "100%", + borderWidth: 1, + borderColor: "rgba(0,0,0,0.2)", + borderRadius: 8, + marginBottom: 24, + overflow: "hidden", +} + +const $debugHeader: ViewStyle = { + padding: 12, + backgroundColor: "rgba(0,0,0,0.05)", +} + +const $debugTitle: TextStyle = { + fontWeight: "bold", +} + +const $debugContent: ViewStyle = { + maxHeight: 300, + padding: 12, + backgroundColor: "rgba(0,0,0,0.02)", +} + +const $debugText: TextStyle = { + fontFamily: "monospace", + fontSize: 12, +} diff --git a/apps/ExampleApp/app/screens/index.ts b/apps/ExampleApp/app/screens/index.ts index f9588eb8..52f2fb39 100644 --- a/apps/ExampleApp/app/screens/index.ts +++ b/apps/ExampleApp/app/screens/index.ts @@ -7,3 +7,4 @@ export * from "./ImageLabelingScreen" export * from "./DocumentScannerScreen" export { BOX_COLORS } from "./FaceDetectionScreen" export * from "./ObjectDetectionScreen" +export * from "./TextRecognitionScreen" \ No newline at end of file diff --git a/apps/ExampleApp/app/utils/useExampleImage/useExampleImage.ts b/apps/ExampleApp/app/utils/useExampleImage/useExampleImage.ts index e02ec918..10e2c79e 100644 --- a/apps/ExampleApp/app/utils/useExampleImage/useExampleImage.ts +++ b/apps/ExampleApp/app/utils/useExampleImage/useExampleImage.ts @@ -23,6 +23,7 @@ export type UseExampleImageStatus = | "takingPhoto" | "selectingPhoto" | "classifying" + | "recognizing" | "done" | "error" | "loading" @@ -126,13 +127,13 @@ export function useExampleImage(predicates?: { return } setStatus("takingPhoto") - const result = await launchCameraAsync(IMAGE_PICKER_OPTIONS) + const result: ImagePickerResult = await launchCameraAsync(IMAGE_PICKER_OPTIONS) if (result.assets?.[0]) { setImage({ ...result.assets?.[0], localUri: result.assets?.[0].uri } as SelectedImage) } else { setImage(undefined) } - }, [checkPermissions, setStatus]) // Note: Removed parentheses from launchCameraAsync + }, [checkPermissions, setStatus]) const [currentIndexes, setCurrentIndexes] = useState>( {} as Record, diff --git a/apps/ExampleApp/assets/images/text-recognition.png b/apps/ExampleApp/assets/images/text-recognition.png new file mode 100644 index 00000000..c58cd834 Binary files /dev/null and b/apps/ExampleApp/assets/images/text-recognition.png differ diff --git a/apps/ExampleApp/package.json b/apps/ExampleApp/package.json index 5adbc678..34554af2 100644 --- a/apps/ExampleApp/package.json +++ b/apps/ExampleApp/package.json @@ -36,6 +36,7 @@ "@infinitered/react-native-mlkit-face-detection": "workspace:^5.0.0", "@infinitered/react-native-mlkit-image-labeling": "workspace:^5.0.0", "@infinitered/react-native-mlkit-object-detection": "workspace:^5.0.0", + "@infinitered/react-native-mlkit-text-recognition": "workspace:^1.0.0", "@react-native-async-storage/async-storage": "2.2.0", "@react-navigation/native": "^6.0.8", "@react-navigation/native-stack": "^6.0.2", diff --git a/docs/text-recognition/_category_.json b/docs/text-recognition/_category_.json new file mode 100644 index 00000000..e622a7e7 --- /dev/null +++ b/docs/text-recognition/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Text Recognition", + "position": 500, + "link": { + "type": "generated-index", + "description": "Recognize text" + } +} diff --git a/docs/text-recognition/index.md b/docs/text-recognition/index.md new file mode 100644 index 00000000..a73cd9f7 --- /dev/null +++ b/docs/text-recognition/index.md @@ -0,0 +1,102 @@ +--- +sidebar_position: 1 +title: Getting Started +--- + +# Object Detection + +## Getting Started + +This is an expo module that lets you use +the [MLKit Text Recognition](https://developers.google.com/ml-kit/vision/text-recognition/v2) library in your Expo app. + +## Installation + +Install like any other npm package: + +```bash +#yarn +yarn add @infinitered/react-native-mlkit-text-recognition + +#npm +npm install @infinitered/react-native-mlkit-text-recognition +``` + +## Basic Usage + +The models are made available through the context system. You can access them in your components using the same hook + +```tsx +// MyComponent.tsx +import { recognizeText } from "@infinitered/react-native-mlkit-text-recognition"; +import React, { useEffect, useState } from "react"; +import { View } from "react-native"; +import type { MyModelsConfig } from "./App"; + +type Props = { + imagePath: string; +}; + +function MyComponent({ imagePath }: Props) { + const [recognizedText, setRecognizedText] = useState(null); + + useEffect(() => { + async function recognizeTextAsync(imagePath: string) { + try { + const { text } = await recognizeText(imagePath); + setRecognizedText(text); + } catch (error) { + console.error("Error recognizing text:", error); + } + } + + if (imagePath) { + recognizeTextAsync(imagePath); + } + }, [imagePath]); + + return ( + + {recognizedText} + + ); +} +``` + +### Recognition Results + +The `recognizeText` method returns a `Text` object: + +```ts +interface Rect { + left: number; + top: number; + right: number; + bottom: number; +} + +interface TextElement { + text: string; + frame: Rect; + recognizedLanguages: string[]; +} + +interface TextLine { + text: string; + frame: Rect; + recognizedLanguages: string[]; + elements: TextElement[]; +} + +interface Block { + text: string; + frame: Rect; + recognizedLanguages: string[]; + lines: TextLine[]; +} + +interface Text { + text: string; + blocks: Block[]; +} +``` diff --git a/modules/react-native-mlkit-text-recognition/.eslintrc.js b/modules/react-native-mlkit-text-recognition/.eslintrc.js new file mode 100644 index 00000000..0dc2ae2a --- /dev/null +++ b/modules/react-native-mlkit-text-recognition/.eslintrc.js @@ -0,0 +1,8 @@ +module.exports = { + root: true, + extends: ["universe/native", "universe/web"], + ignorePatterns: ["build"], + rules: { + "@typescript-eslint/array-type": "off", + }, +}; diff --git a/modules/react-native-mlkit-text-recognition/.gitignore b/modules/react-native-mlkit-text-recognition/.gitignore new file mode 100644 index 00000000..e64b91c9 --- /dev/null +++ b/modules/react-native-mlkit-text-recognition/.gitignore @@ -0,0 +1,57 @@ +# OSX +# +.DS_Store + +# VSCode +.vscode/ +jsconfig.json + +# Xcode +# +build/ +*.pbxuser +!default.pbxuser +*.mode1v3 +!default.mode1v3 +*.mode2v3 +!default.mode2v3 +*.perspectivev3 +!default.perspectivev3 +xcuserdata +*.xccheckout +*.moved-aside +DerivedData +*.hmap +*.ipa +*.xcuserstate +project.xcworkspace + +# Android/IJ +# +.classpath +.cxx +.gradle +.idea +.project +.settings +local.properties +android.iml +android/app/libs +android/keystores/debug.keystore + +# Cocoapods +# +example/ios/Pods + +# Ruby +example/vendor/ + +# node.js +# +node_modules/ +npm-debug.log +yarn-debug.log +yarn-error.log + +# Expo +.expo/* diff --git a/modules/react-native-mlkit-text-recognition/.npmignore b/modules/react-native-mlkit-text-recognition/.npmignore new file mode 100644 index 00000000..fa6abe7c --- /dev/null +++ b/modules/react-native-mlkit-text-recognition/.npmignore @@ -0,0 +1,11 @@ +# Exclude all top-level hidden directories by convention +/.*/ + +__mocks__ +__tests__ + +/babel.config.js +/android/src/androidTest/ +/android/src/test/ +/android/build/ +/example/ diff --git a/modules/react-native-mlkit-text-recognition/android/build.gradle b/modules/react-native-mlkit-text-recognition/android/build.gradle new file mode 100644 index 00000000..679acd5e --- /dev/null +++ b/modules/react-native-mlkit-text-recognition/android/build.gradle @@ -0,0 +1,58 @@ +apply plugin: 'com.android.library' + +group = 'red.infinite.reactnativemlkit.textrecognition' +version = '1.0.0' + +def expoModulesCorePlugin = new File(project(":expo-modules-core").projectDir.absolutePath, "ExpoModulesCorePlugin.gradle") +apply from: expoModulesCorePlugin +applyKotlinExpoModulesCorePlugin() +useCoreDependencies() +useExpoPublishing() + +buildscript { + // Simple helper that allows the root project to override versions declared by this library. + ext.safeExtGet = { prop, fallback -> + rootProject.ext.has(prop) ? rootProject.ext.get(prop) : fallback + } + + // Ensures backward compatibility + ext.getKotlinVersion = { + if (ext.has("kotlinVersion")) { + ext.kotlinVersion() + } else { + ext.safeExtGet("kotlinVersion", "1.8.10") + } + } + + repositories { + mavenCentral() + } + + dependencies { + classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:${getKotlinVersion()}") + } +} +project.android { + compileSdkVersion safeExtGet("compileSdkVersion", 34) + defaultConfig { + minSdkVersion safeExtGet("minSdkVersion", 21) + targetSdkVersion safeExtGet("targetSdkVersion", 34) + } +} + +android { + namespace "red.infinite.reactnativemlkit.textrecognition" + defaultConfig { + versionCode 1 + versionName "1.0.0" + } + lintOptions { + abortOnError false + } +} + +dependencies { + implementation project(':expo-modules-core') + implementation project(path: ':infinitered-react-native-mlkit-core') + implementation 'com.google.mlkit:text-recognition:16.0.1' +} diff --git a/modules/react-native-mlkit-text-recognition/android/src/main/AndroidManifest.xml b/modules/react-native-mlkit-text-recognition/android/src/main/AndroidManifest.xml new file mode 100644 index 00000000..bdae66c8 --- /dev/null +++ b/modules/react-native-mlkit-text-recognition/android/src/main/AndroidManifest.xml @@ -0,0 +1,2 @@ + + diff --git a/modules/react-native-mlkit-text-recognition/android/src/main/java/red/infinite/reactnativemlkit/textrecognition/RNMLKitTextRecognitionModule.kt b/modules/react-native-mlkit-text-recognition/android/src/main/java/red/infinite/reactnativemlkit/textrecognition/RNMLKitTextRecognitionModule.kt new file mode 100644 index 00000000..44abbfc8 --- /dev/null +++ b/modules/react-native-mlkit-text-recognition/android/src/main/java/red/infinite/reactnativemlkit/textrecognition/RNMLKitTextRecognitionModule.kt @@ -0,0 +1,54 @@ +package red.infinite.reactnativemlkit.textrecognition + +import expo.modules.kotlin.Promise +import expo.modules.kotlin.exception.CodedException +import expo.modules.kotlin.modules.Module +import expo.modules.kotlin.modules.ModuleDefinition +import java.net.URL +import com.google.mlkit.vision.text.latin.TextRecognizerOptions +import com.google.mlkit.vision.text.TextRecognition +import com.google.mlkit.vision.common.InputImage +import android.graphics.BitmapFactory +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext + +suspend fun getInputImage( + imagePath: String +): InputImage { + try { + val bitmap = BitmapFactory.decodeStream(withContext(Dispatchers.IO) { + URL(imagePath).openStream() + }) + + return InputImage.fromBitmap(bitmap, 0) + } catch (e: Exception) { + throw Exception("RNMLKitTextRecognition: Could not load image from $imagePath", e) + } +} + +class RNMLKitTextRecognitionModule : Module() { + override fun definition() = ModuleDefinition { + Name("RNMLKitTextRecognition") + + AsyncFunction("recognizeText") { imagePath: String, promise: Promise -> + runBlocking { + val recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS) + + val image: InputImage = getInputImage(imagePath) + + recognizer.process(image) + .addOnSuccessListener { visionText -> + promise.resolve(mapTextToRecord(visionText)) + } + .addOnFailureListener { e -> + promise.reject( + CodedException( + "RNMLKitTextRecognitionModule - Error: ${e.message}", e + ) + ) + } + } + } + } +} diff --git a/modules/react-native-mlkit-text-recognition/android/src/main/java/red/infinite/reactnativemlkit/textrecognition/RNMLKitTextRecord.kt b/modules/react-native-mlkit-text-recognition/android/src/main/java/red/infinite/reactnativemlkit/textrecognition/RNMLKitTextRecord.kt new file mode 100644 index 00000000..ae2474da --- /dev/null +++ b/modules/react-native-mlkit-text-recognition/android/src/main/java/red/infinite/reactnativemlkit/textrecognition/RNMLKitTextRecord.kt @@ -0,0 +1,81 @@ +package red.infinite.reactnativemlkit.textrecognition + +import expo.modules.kotlin.records.Field +import expo.modules.kotlin.records.Record +import com.google.mlkit.vision.text.Text + +class RectRecord( + @Field val left: Double = 0.0, + @Field val top: Double = 0.0, + @Field val right: Double = 0.0, + @Field val bottom: Double = 0.0 +) : Record + +class TextElementRecord( + @Field val text: String = "", + @Field val frame: RectRecord = RectRecord(), + @Field val recognizedLanguages: List = emptyList() +) : Record + +class TextLineRecord( + @Field val text: String = "", + @Field val frame: RectRecord = RectRecord(), + @Field val recognizedLanguages: List = emptyList(), + @Field val elements: List = emptyList() +) : Record + +class BlockRecord( + @Field val text: String = "", + @Field val frame: RectRecord = RectRecord(), + @Field val recognizedLanguages: List = emptyList(), + @Field val lines: List = emptyList() +) : Record + +class TextRecord( + @Field val text: String = "", + @Field val blocks: List = emptyList() +) : Record + +fun mapRectToRecord(rect: android.graphics.Rect?): RectRecord { + if (rect == null) return RectRecord() + + return RectRecord( + left = rect.left.toDouble(), + top = rect.top.toDouble(), + right = rect.right.toDouble(), + bottom = rect.bottom.toDouble() + ) +} + +fun mapElementToRecord(element: Text.Element): TextElementRecord { + return TextElementRecord( + text = element.text, + frame = mapRectToRecord(element.boundingBox), + recognizedLanguages = listOfNotNull(element.recognizedLanguage) + ) +} + +fun mapLineToRecord(line: Text.Line): TextLineRecord { + return TextLineRecord( + text = line.text, + frame = mapRectToRecord(line.boundingBox), + recognizedLanguages = listOfNotNull(line.recognizedLanguage), + elements = line.elements.map { mapElementToRecord(it) } + ) +} + +fun mapTextBlockToRecord(textBlock: Text.TextBlock): BlockRecord { + return BlockRecord( + text = textBlock.text, + frame = mapRectToRecord(textBlock.boundingBox), + recognizedLanguages = listOfNotNull(textBlock.recognizedLanguage), + lines = textBlock.lines.map { mapLineToRecord(it) } + ) +} + +fun mapTextToRecord(text: Text): TextRecord { + return TextRecord( + text = text.text, + blocks = text.textBlocks.map { mapTextBlockToRecord(it) } + ) +} \ No newline at end of file diff --git a/modules/react-native-mlkit-text-recognition/expo-module.config.json b/modules/react-native-mlkit-text-recognition/expo-module.config.json new file mode 100644 index 00000000..b0badfb7 --- /dev/null +++ b/modules/react-native-mlkit-text-recognition/expo-module.config.json @@ -0,0 +1,17 @@ +{ + "platforms": [ + "apple", + "android", + "web" + ], + "apple": { + "modules": [ + "RNMLKitTextRecognitionModule" + ] + }, + "android": { + "modules": [ + "red.infinite.reactnativemlkit.textrecognition.RNMLKitTextRecognitionModule" + ] + } +} diff --git a/modules/react-native-mlkit-text-recognition/index.ts b/modules/react-native-mlkit-text-recognition/index.ts new file mode 100644 index 00000000..4fa27edf --- /dev/null +++ b/modules/react-native-mlkit-text-recognition/index.ts @@ -0,0 +1,4 @@ +// Reexport the native module. On web, it will be resolved to RNMLKitTextRecognitionModule.web.ts +// and on native platforms to RNMLKitTextRecognitionModule.ts +export { recognizeText } from './src/RNMLKitTextRecognitionModule'; + diff --git a/modules/react-native-mlkit-text-recognition/ios/RNMLKitTextRecognition.podspec b/modules/react-native-mlkit-text-recognition/ios/RNMLKitTextRecognition.podspec new file mode 100644 index 00000000..cb805928 --- /dev/null +++ b/modules/react-native-mlkit-text-recognition/ios/RNMLKitTextRecognition.podspec @@ -0,0 +1,29 @@ +require 'json' + +package = JSON.parse(File.read(File.join(__dir__, '..', 'package.json'))) + +Pod::Spec.new do |s| + s.name = 'RNMLKitTextRecognition' + s.version = package['version'] + s.summary = package['description'] + s.description = package['description'] + s.license = package['license'] + s.author = package['author'] + s.homepage = package['homepage'] + s.platform = :ios, '15.1' + s.swift_version = '5.4' + s.source = { git: 'http://github.com/infinitered/react-native-mlkit' } + s.static_framework = true + + s.dependency 'ExpoModulesCore' + s.dependency 'RNMLKitCore' + s.dependency 'GoogleMLKit/TextRecognition' + + # Swift/Objective-C compatibility + s.pod_target_xcconfig = { + 'DEFINES_MODULE' => 'YES', + 'SWIFT_COMPILATION_MODE' => 'wholemodule' + } + + s.source_files = "**/*.{h,m,swift}" +end diff --git a/modules/react-native-mlkit-text-recognition/ios/RNMLKitTextRecognitionModule.swift b/modules/react-native-mlkit-text-recognition/ios/RNMLKitTextRecognitionModule.swift new file mode 100644 index 00000000..0357141d --- /dev/null +++ b/modules/react-native-mlkit-text-recognition/ios/RNMLKitTextRecognitionModule.swift @@ -0,0 +1,34 @@ +import ExpoModulesCore +import MLKitCommon +import MLKitTextRecognition +import RNMLKitCore + +public class RNMLKitTextRecognitionModule: Module { + let ERROR_DOMAIN: String = "red.infinite.RNMLKit.RNMLKitTextRecognitionModuleErrorDomain" + + public func definition() -> ModuleDefinition { + Name("RNMLKitTextRecognition") + + AsyncFunction("recognizeText") { (imagePath: String, promise: Promise) in + let logger = Logger(logHandlers: [createOSLogHandler(category: Logger.EXPO_LOG_CATEGORY)]) + logger.info("RNMLKit", "recognize text: Recognizing text from image: \(imagePath) ") + + let options = TextRecognizerOptions() + let textRecognizer = TextRecognizer.textRecognizer(options:options) + + let image = try RNMLKitImage(imagePath: imagePath) + + Task { + do { + let result = try await textRecognizer.process(image.visionImage) + + promise.resolve(mapTextToRecord(result)) + } catch { + promise.reject( + NSError(domain: ERROR_DOMAIN, code: 1, userInfo: [NSLocalizedDescriptionKey: "Error occurred: \(error)"]) + ) + } + } + } + } +} diff --git a/modules/react-native-mlkit-text-recognition/ios/RNMLKitTextRecord.swift b/modules/react-native-mlkit-text-recognition/ios/RNMLKitTextRecord.swift new file mode 100644 index 00000000..7672854c --- /dev/null +++ b/modules/react-native-mlkit-text-recognition/ios/RNMLKitTextRecord.swift @@ -0,0 +1,78 @@ +import ExpoModulesCore +import MLKitCommon +import MLKitTextRecognition + +struct RectRecord: Record { + @Field var left: Double = 0.0 + @Field var top: Double = 0.0 + @Field var right: Double = 0.0 + @Field var bottom: Double = 0.0 +} + +struct TextElementRecord: Record { + @Field var text: String = "" + @Field var frame: RectRecord = RectRecord() + @Field var recognizedLanguages: [String] = [] +} + +struct TextLineRecord: Record { + @Field var text: String = "" + @Field var frame: RectRecord = RectRecord() + @Field var recognizedLanguages: [String] = [] + @Field var elements: [TextElementRecord] = [] +} + +struct BlockRecord: Record { + @Field var text: String = "" + @Field var frame: RectRecord = RectRecord() + @Field var recognizedLanguages: [String] = [] + @Field var lines: [TextLineRecord] = [] +} + +struct TextRecord: Record { + @Field var text: String = "" + @Field var blocks: [BlockRecord] = [] +} + +func mapRectToRecord(_ rect: CGRect?) -> RectRecord { + guard let rect = rect else { return RectRecord() } + return RectRecord( + left: Double(rect.minX), + top: Double(rect.minY), + right: Double(rect.maxX), + bottom: Double(rect.maxY) + ) +} + +func mapElementToRecord(_ element: TextElement) -> TextElementRecord { + return TextElementRecord( + text: element.text, + frame: mapRectToRecord(element.frame), + recognizedLanguages: element.recognizedLanguages.map { $0.languageCode ?? "" } + ) +} + +func mapLineToRecord(_ line: TextLine) -> TextLineRecord { + return TextLineRecord( + text: line.text, + frame: mapRectToRecord(line.frame), + recognizedLanguages: line.recognizedLanguages.map { $0.languageCode ?? "" }, + elements: line.elements.map(mapElementToRecord) + ) +} + +func mapTextBlockToRecord(_ textBlock: TextBlock) -> BlockRecord { + return BlockRecord( + text: textBlock.text, + frame: mapRectToRecord(textBlock.frame), + recognizedLanguages: textBlock.recognizedLanguages.map { $0.languageCode ?? "" }, + lines: textBlock.lines.map(mapLineToRecord) + ) +} + +func mapTextToRecord(_ text: Text) -> TextRecord { + return TextRecord( + text: text.text, + blocks: text.blocks.map(mapTextBlockToRecord) + ) +} diff --git a/modules/react-native-mlkit-text-recognition/package.json b/modules/react-native-mlkit-text-recognition/package.json new file mode 100644 index 00000000..3921c921 --- /dev/null +++ b/modules/react-native-mlkit-text-recognition/package.json @@ -0,0 +1,44 @@ +{ + "name": "@infinitered/react-native-mlkit-text-recognition", + "version": "1.0.0", + "description": "Expo module for MLKit Text Recognition", + "main": "build/index.js", + "types": "build/index.d.ts", + "scripts": { + "build": "expo-module build", + "ci:build": "tsc", + "clean": "expo-module clean", + "lint": "expo-module lint", + "test": "../../scripts/test-module.sh", + "prepublishOnly": "expo-module prepublishOnly", + "expo-module": "expo-module" + }, + "keywords": [ + "react-native", + "expo", + "react-native-mlkit-text-recognition", + "RNMLKitObjectDetection" + ], + "repository": "https://github.com/infinitered/react-native-mlkit", + "bugs": { + "url": "https://github.com/infinitered/react-native-mlkit/issues" + }, + "author": "Diogo Carmo (https://github.com/dccarmo)", + "license": "MIT", + "homepage": "https://docs.infinite.red/react-native-mlkit/text-recognition", + "dependencies": { + "@infinitered/react-native-mlkit-core": "3.1.0" + }, + "devDependencies": { + "@infinitered/react-native-mlkit-core": "3.1.0", + "@types/react": "~18.3.12", + "expo-module-scripts": "^3.4.1", + "expo-modules-core": "~2.2.0", + "typescript": "~5.1.6" + }, + "peerDependencies": { + "expo": "*", + "react": "*", + "react-native": "*" + } +} diff --git a/modules/react-native-mlkit-text-recognition/src/RNMLKitTextRecognitionModule.ts b/modules/react-native-mlkit-text-recognition/src/RNMLKitTextRecognitionModule.ts new file mode 100644 index 00000000..7335dcea --- /dev/null +++ b/modules/react-native-mlkit-text-recognition/src/RNMLKitTextRecognitionModule.ts @@ -0,0 +1,48 @@ +import { requireNativeModule } from "expo"; + +interface Rect { + left: number; + top: number; + right: number; + bottom: number; +} + +interface TextElement { + text: string; + frame: Rect; + recognizedLanguages: string[]; +} + +interface TextLine { + text: string; + frame: Rect; + recognizedLanguages: string[]; + elements: TextElement[]; +} + +interface Block { + text: string; + frame: Rect; + recognizedLanguages: string[]; + lines: TextLine[]; +} + +interface Text { + text: string; + blocks: Block[]; +} + +interface RNMLKitTextRecognitionModule { + recognizeText: (imagePath: string) => Promise; +} + +const textRecognitionModule = requireNativeModule( + "RNMLKitTextRecognition" +); + +async function recognizeText(imagePath: string): Promise { + return await textRecognitionModule.recognizeText(imagePath); +} + +export { recognizeText }; +export type { Text, Block, TextLine, TextElement, Rect }; diff --git a/modules/react-native-mlkit-text-recognition/src/index.test.ts b/modules/react-native-mlkit-text-recognition/src/index.test.ts new file mode 100644 index 00000000..dd21e350 --- /dev/null +++ b/modules/react-native-mlkit-text-recognition/src/index.test.ts @@ -0,0 +1,5 @@ +describe("placeholder test", () => { + it("should pass", () => { + expect(true).toBe(true); + }); +}); diff --git a/modules/react-native-mlkit-text-recognition/src/index.ts b/modules/react-native-mlkit-text-recognition/src/index.ts new file mode 100644 index 00000000..60ed681b --- /dev/null +++ b/modules/react-native-mlkit-text-recognition/src/index.ts @@ -0,0 +1 @@ +export * from "./RNMLKitTextRecognitionModule"; diff --git a/modules/react-native-mlkit-text-recognition/tsconfig.json b/modules/react-native-mlkit-text-recognition/tsconfig.json new file mode 100644 index 00000000..af4318d3 --- /dev/null +++ b/modules/react-native-mlkit-text-recognition/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "@infinitered/tsconfig/expo-module", + "compilerOptions": { + "outDir": "./build" + }, + "include": [ + "./src" + ], + "exclude": [ + "**/__mocks__/*", + "**/__tests__/*" + ] +} diff --git a/yarn.lock b/yarn.lock index 51188a69..29eba84e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4661,6 +4661,22 @@ __metadata: languageName: unknown linkType: soft +"@infinitered/react-native-mlkit-core@npm:3.1.0": + version: 3.1.0 + resolution: "@infinitered/react-native-mlkit-core@npm:3.1.0" + dependencies: + "@testing-library/jest-native": ^5.4.3 + "@testing-library/react-hooks": ^8.0.1 + "@types/jest": ^29.5.5 + peerDependencies: + expo: "*" + expo-image: "*" + react: "*" + react-native: "*" + checksum: 7d7febc935fa64adb9504b5f76d9500dc3875bad60709dcf7180e0e09ff82b4fa665e5019b8b4b2f31078926f80b2c16a8576b24f2a232befe8aafa011aec9c8 + languageName: node + linkType: hard + "@infinitered/react-native-mlkit-docs@workspace:packages/docusaurus": version: 0.0.0-use.local resolution: "@infinitered/react-native-mlkit-docs@workspace:packages/docusaurus" @@ -4741,6 +4757,22 @@ __metadata: languageName: unknown linkType: soft +"@infinitered/react-native-mlkit-text-recognition@workspace:^1.0.0, @infinitered/react-native-mlkit-text-recognition@workspace:modules/react-native-mlkit-text-recognition": + version: 0.0.0-use.local + resolution: "@infinitered/react-native-mlkit-text-recognition@workspace:modules/react-native-mlkit-text-recognition" + dependencies: + "@infinitered/react-native-mlkit-core": 3.1.0 + "@types/react": ~18.3.12 + expo-module-scripts: ^3.4.1 + expo-modules-core: ~2.2.0 + typescript: ~5.1.6 + peerDependencies: + expo: "*" + react: "*" + react-native: "*" + languageName: unknown + linkType: soft + "@infinitered/tsconfig@0.7.0, @infinitered/tsconfig@workspace:packages/tsconfig": version: 0.0.0-use.local resolution: "@infinitered/tsconfig@workspace:packages/tsconfig" @@ -6746,7 +6778,7 @@ __metadata: languageName: node linkType: hard -"@types/jest@npm:29.5.14": +"@types/jest@npm:29.5.14, @types/jest@npm:^29.5.5": version: 29.5.14 resolution: "@types/jest@npm:29.5.14" dependencies: @@ -6973,6 +7005,16 @@ __metadata: languageName: node linkType: hard +"@types/react@npm:~18.3.12": + version: 18.3.27 + resolution: "@types/react@npm:18.3.27" + dependencies: + "@types/prop-types": "*" + csstype: ^3.2.2 + checksum: c74d0ab5155226998a52b568f6280536205f8fe4317f77bd5d5258bc131cc9134a2db68dc818cb8e8402a2f229843c4a5bde339faf7e64d441630e569a9e5421 + languageName: node + linkType: hard + "@types/react@npm:~19.1.10": version: 19.1.17 resolution: "@types/react@npm:19.1.17" @@ -10197,6 +10239,13 @@ __metadata: languageName: node linkType: hard +"csstype@npm:^3.2.2": + version: 3.2.3 + resolution: "csstype@npm:3.2.3" + checksum: cb882521b3398958a1ce6ca98c011aec0bde1c77ecaf8a1dd4db3b112a189939beae3b1308243b2fe50fc27eb3edeb0f73a5a4d91d928765dc6d5ecc7bda92ee + languageName: node + linkType: hard + "csv-generate@npm:^3.4.3": version: 3.4.3 resolution: "csv-generate@npm:3.4.3" @@ -11991,6 +12040,7 @@ __metadata: "@infinitered/react-native-mlkit-face-detection": "workspace:^5.0.0" "@infinitered/react-native-mlkit-image-labeling": "workspace:^5.0.0" "@infinitered/react-native-mlkit-object-detection": "workspace:^5.0.0" + "@infinitered/react-native-mlkit-text-recognition": "workspace:^1.0.0" "@infinitered/tsconfig": 0.7.0 "@react-native-async-storage/async-storage": 2.2.0 "@react-native-community/cli": ^15.0.0 @@ -12398,6 +12448,15 @@ __metadata: languageName: node linkType: hard +"expo-modules-core@npm:~2.2.0": + version: 2.2.3 + resolution: "expo-modules-core@npm:2.2.3" + dependencies: + invariant: ^2.2.4 + checksum: 7b2952f1220b55eb03f395d1549525edeb5bff7bf805257d9652ea4ef85ea71e34ad13b5971f1b559e7aa080f41130846b24cbe3d754660c08196c3ce899143b + languageName: node + linkType: hard + "expo-server@npm:^1.0.4": version: 1.0.4 resolution: "expo-server@npm:1.0.4"