diff --git a/android/src/main/java/org/reactnative/camera/CameraViewManager.java b/android/src/main/java/org/reactnative/camera/CameraViewManager.java index 1e99da215..fd2da1b79 100644 --- a/android/src/main/java/org/reactnative/camera/CameraViewManager.java +++ b/android/src/main/java/org/reactnative/camera/CameraViewManager.java @@ -19,7 +19,8 @@ public enum Events { EVENT_ON_MOUNT_ERROR("onMountError"), EVENT_ON_BAR_CODE_READ("onBarCodeRead"), EVENT_ON_FACES_DETECTED("onFacesDetected"), - EVENT_ON_FACE_DETECTION_ERROR("onFaceDetectionError"); + EVENT_ON_FACE_DETECTION_ERROR("onFaceDetectionError"), + EVENT_ON_TEXT_RECOGNIZED("onTextRecognized"); private final String mName; @@ -138,4 +139,9 @@ public void setFaceDetectionLandmarks(RNCameraView view, int landmarks) { public void setFaceDetectionClassifications(RNCameraView view, int classifications) { view.setFaceDetectionClassifications(classifications); } + + @ReactProp(name = "textRecognizerEnabled") + public void setTextRecognizing(RNCameraView view, boolean textRecognizerEnabled) { + view.setShouldRecognizeText(textRecognizerEnabled); + } } diff --git a/android/src/main/java/org/reactnative/camera/RNCameraView.java b/android/src/main/java/org/reactnative/camera/RNCameraView.java index 0534bb2e8..8d52f4717 100644 --- a/android/src/main/java/org/reactnative/camera/RNCameraView.java +++ b/android/src/main/java/org/reactnative/camera/RNCameraView.java @@ -17,6 +17,9 @@ import com.facebook.react.uimanager.ThemedReactContext; import com.google.android.cameraview.CameraView; import com.google.android.gms.vision.face.Face; +import com.google.android.gms.vision.text.Text; +import com.google.android.gms.vision.text.TextBlock; +import com.google.android.gms.vision.text.TextRecognizer; import com.google.zxing.BarcodeFormat; import com.google.zxing.DecodeHintType; import com.google.zxing.MultiFormatReader; @@ -27,6 +30,8 @@ import org.reactnative.camera.tasks.FaceDetectorAsyncTask; import org.reactnative.camera.tasks.FaceDetectorAsyncTaskDelegate; import org.reactnative.camera.tasks.ResolveTakenPictureAsyncTask; +import org.reactnative.camera.tasks.TextRecognizerAsyncTask; +import org.reactnative.camera.tasks.TextRecognizerAsyncTaskDelegate; import org.reactnative.camera.utils.ImageDimensions; import org.reactnative.camera.utils.RNFileUtils; import org.reactnative.facedetector.RNFaceDetector; @@ -41,7 +46,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; -public class RNCameraView extends CameraView implements LifecycleEventListener, BarCodeScannerAsyncTaskDelegate, FaceDetectorAsyncTaskDelegate { +public class RNCameraView extends CameraView implements LifecycleEventListener, BarCodeScannerAsyncTaskDelegate, FaceDetectorAsyncTaskDelegate, + TextRecognizerAsyncTaskDelegate { private ThemedReactContext mThemedReactContext; private Queue mPictureTakenPromises = new ConcurrentLinkedQueue<>(); private Map mPictureTakenOptions = new ConcurrentHashMap<>(); @@ -55,12 +61,15 @@ public class RNCameraView extends CameraView implements LifecycleEventListener, // Concurrency lock for scanners to avoid flooding the runtime public volatile boolean barCodeScannerTaskLock = false; public volatile boolean faceDetectorTaskLock = false; + public volatile boolean textRecognizerTaskLock = false; // Scanning-related properties private final MultiFormatReader mMultiFormatReader = new MultiFormatReader(); private final RNFaceDetector mFaceDetector; + private final TextRecognizer mTextRecognizer; private boolean mShouldDetectFaces = false; private boolean mShouldScanBarCodes = false; + private boolean mShouldRecognizeText = false; private int mFaceDetectorMode = RNFaceDetector.FAST_MODE; private int mFaceDetectionLandmarks = RNFaceDetector.NO_LANDMARKS; private int mFaceDetectionClassifications = RNFaceDetector.NO_CLASSIFICATIONS; @@ -71,6 +80,7 @@ public RNCameraView(ThemedReactContext themedReactContext) { mThemedReactContext = themedReactContext; mFaceDetector = new RNFaceDetector(themedReactContext); setupFaceDetector(); + mTextRecognizer = new TextRecognizer.Builder(themedReactContext).build(); themedReactContext.addLifecycleEventListener(this); addCallback(new Callback() { @@ -121,6 +131,12 @@ public void onFramePreview(CameraView cameraView, byte[] data, int width, int he FaceDetectorAsyncTaskDelegate delegate = (FaceDetectorAsyncTaskDelegate) cameraView; new FaceDetectorAsyncTask(delegate, mFaceDetector, data, width, height, correctRotation).execute(); } + + if (mShouldRecognizeText && !textRecognizerTaskLock && cameraView instanceof TextRecognizerAsyncTaskDelegate) { + textRecognizerTaskLock = true; + TextRecognizerAsyncTaskDelegate delegate = (TextRecognizerAsyncTaskDelegate) cameraView; + new TextRecognizerAsyncTask(delegate, mTextRecognizer, data, width, height, correctRotation).execute(); + } } }); } @@ -145,7 +161,7 @@ public void requestLayout() { @Override public void onViewAdded(View child) { if (this.getView() == child || this.getView() == null) return; - // remove and readd view to make sure it is in the back. + // remove and read view to make sure it is in the back. // @TODO figure out why there was a z order issue in the first place and fix accordingly. this.removeView(this.getView()); this.addView(this.getView(), 0); @@ -210,7 +226,7 @@ private void initBarcodeReader() { public void setShouldScanBarCodes(boolean shouldScanBarCodes) { this.mShouldScanBarCodes = shouldScanBarCodes; - setScanning(mShouldDetectFaces || mShouldScanBarCodes); + setScanning(mShouldDetectFaces || mShouldScanBarCodes || mShouldRecognizeText); } public void onBarCodeRead(Result barCode) { @@ -260,7 +276,7 @@ public void setFaceDetectionMode(int mode) { public void setShouldDetectFaces(boolean shouldDetectFaces) { this.mShouldDetectFaces = shouldDetectFaces; - setScanning(mShouldDetectFaces || mShouldScanBarCodes); + setScanning(mShouldDetectFaces || mShouldScanBarCodes || mShouldRecognizeText); } public void onFacesDetected(SparseArray facesReported, int sourceWidth, int sourceHeight, int sourceRotation) { @@ -287,6 +303,28 @@ public void onFaceDetectingTaskCompleted() { faceDetectorTaskLock = false; } + public void setShouldRecognizeText(boolean shouldRecognizeText) { + this.mShouldRecognizeText = shouldRecognizeText; + setScanning(mShouldDetectFaces || mShouldScanBarCodes || mShouldRecognizeText); + } + + @Override + public void onTextRecognized(SparseArray textBlocks, int sourceWidth, int sourceHeight, int sourceRotation) { + if (!mShouldRecognizeText) { + return; + } + + SparseArray textBlocksDetected = textBlocks == null ? new SparseArray() : textBlocks; + ImageDimensions dimensions = new ImageDimensions(sourceWidth, sourceHeight, sourceRotation, getFacing()); + + RNCameraViewHelper.emitTextRecognizedEvent(this, textBlocksDetected, dimensions); + } + + @Override + public void onTextRecognizerTaskCompleted() { + textRecognizerTaskLock = false; + } + @Override public void onHostResume() { if (hasCameraPermissions()) { diff --git a/android/src/main/java/org/reactnative/camera/RNCameraViewHelper.java b/android/src/main/java/org/reactnative/camera/RNCameraViewHelper.java index 3dbccdb6b..93d963dd2 100644 --- a/android/src/main/java/org/reactnative/camera/RNCameraViewHelper.java +++ b/android/src/main/java/org/reactnative/camera/RNCameraViewHelper.java @@ -16,6 +16,7 @@ import com.facebook.react.uimanager.UIManagerModule; import com.google.android.cameraview.CameraView; import com.google.android.gms.vision.face.Face; +import com.google.android.gms.vision.text.TextBlock; import com.google.zxing.Result; import org.reactnative.camera.events.BarCodeReadEvent; @@ -23,6 +24,7 @@ import org.reactnative.camera.events.CameraReadyEvent; import org.reactnative.camera.events.FaceDetectionErrorEvent; import org.reactnative.camera.events.FacesDetectedEvent; +import org.reactnative.camera.events.TextRecognizedEvent; import org.reactnative.camera.utils.ImageDimensions; import org.reactnative.facedetector.RNFaceDetector; @@ -217,6 +219,29 @@ public static void emitBarCodeReadEvent(ViewGroup view, Result barCode) { reactContext.getNativeModule(UIManagerModule.class).getEventDispatcher().dispatchEvent(event); } + // Text recognition event + + public static void emitTextRecognizedEvent( + ViewGroup view, + SparseArray textBlocks, + ImageDimensions dimensions) { + float density = view.getResources().getDisplayMetrics().density; + + double scaleX = (double) view.getWidth() / (dimensions.getWidth() * density); + double scaleY = (double) view.getHeight() / (dimensions.getHeight() * density); + + TextRecognizedEvent event = TextRecognizedEvent.obtain( + view.getId(), + textBlocks, + dimensions, + scaleX, + scaleY + ); + + ReactContext reactContext = (ReactContext) view.getContext(); + reactContext.getNativeModule(UIManagerModule.class).getEventDispatcher().dispatchEvent(event); + } + // Utilities public static int getCorrectCameraRotation(int rotation, int facing) { diff --git a/android/src/main/java/org/reactnative/camera/events/TextRecognizedEvent.java b/android/src/main/java/org/reactnative/camera/events/TextRecognizedEvent.java new file mode 100644 index 000000000..96e4ecfdc --- /dev/null +++ b/android/src/main/java/org/reactnative/camera/events/TextRecognizedEvent.java @@ -0,0 +1,157 @@ +package org.reactnative.camera.events; + +import android.support.v4.util.Pools; +import android.util.SparseArray; + +import com.facebook.react.bridge.Arguments; +import com.facebook.react.bridge.ReadableArray; +import com.facebook.react.bridge.ReadableMap; +import com.facebook.react.bridge.WritableArray; +import com.facebook.react.bridge.WritableMap; +import com.facebook.react.uimanager.events.Event; +import com.facebook.react.uimanager.events.RCTEventEmitter; +import com.google.android.cameraview.CameraView; +import com.google.android.gms.vision.text.Line; +import com.google.android.gms.vision.text.Text; +import com.google.android.gms.vision.text.TextBlock; +import org.reactnative.camera.CameraViewManager; +import org.reactnative.camera.utils.ImageDimensions; +import org.reactnative.facedetector.FaceDetectorUtils; + + +public class TextRecognizedEvent extends Event { + + private static final Pools.SynchronizedPool EVENTS_POOL = + new Pools.SynchronizedPool<>(3); + + + private double mScaleX; + private double mScaleY; + private SparseArray mTextBlocks; + private ImageDimensions mImageDimensions; + + private TextRecognizedEvent() {} + + public static TextRecognizedEvent obtain( + int viewTag, + SparseArray textBlocks, + ImageDimensions dimensions, + double scaleX, + double scaleY) { + TextRecognizedEvent event = EVENTS_POOL.acquire(); + if (event == null) { + event = new TextRecognizedEvent(); + } + event.init(viewTag, textBlocks, dimensions, scaleX, scaleY); + return event; + } + + private void init( + int viewTag, + SparseArray textBlocks, + ImageDimensions dimensions, + double scaleX, + double scaleY) { + super.init(viewTag); + mTextBlocks = textBlocks; + mImageDimensions = dimensions; + mScaleX = scaleX; + mScaleY = scaleY; + } + + @Override + public String getEventName() { + return CameraViewManager.Events.EVENT_ON_TEXT_RECOGNIZED.toString(); + } + + @Override + public void dispatch(RCTEventEmitter rctEventEmitter) { + rctEventEmitter.receiveEvent(getViewTag(), getEventName(), serializeEventData()); + } + + private WritableMap serializeEventData() { + WritableArray textBlocksList = Arguments.createArray(); + for (int i = 0; i < mTextBlocks.size(); ++i) { + TextBlock textBlock = mTextBlocks.valueAt(i); + WritableMap serializedTextBlock = serializeText(textBlock); + if (mImageDimensions.getFacing() == CameraView.FACING_FRONT) { + serializedTextBlock = rotateTextX(serializedTextBlock); + } + textBlocksList.pushMap(serializedTextBlock); + } + + WritableMap event = Arguments.createMap(); + event.putString("type", "textBlock"); + event.putArray("textBlocks", textBlocksList); + event.putInt("target", getViewTag()); + return event; + } + + private WritableMap serializeText(Text text) { + WritableMap encodedText = Arguments.createMap(); + + WritableArray components = Arguments.createArray(); + for (Text component : text.getComponents()) { + components.pushMap(serializeText(component)); + } + encodedText.putArray("components", components); + + encodedText.putString("value", text.getValue()); + + WritableMap origin = Arguments.createMap(); + origin.putDouble("x", text.getBoundingBox().left * this.mScaleX); + origin.putDouble("y", text.getBoundingBox().top * this.mScaleY); + + WritableMap size = Arguments.createMap(); + size.putDouble("width", text.getBoundingBox().width() * this.mScaleX); + size.putDouble("height", text.getBoundingBox().width() * this.mScaleY); + + WritableMap bounds = Arguments.createMap(); + bounds.putMap("origin", origin); + bounds.putMap("size", size); + + encodedText.putMap("bounds", bounds); + + String type_; + if (text instanceof TextBlock) { + type_ = "block"; + } else if (text instanceof Line) { + type_ = "line"; + } else /*if (text instanceof Element)*/ { + type_ = "element"; + } + encodedText.putString("type", type_); + + return encodedText; + } + + private WritableMap rotateTextX(WritableMap text) { + ReadableMap faceBounds = text.getMap("bounds"); + + ReadableMap oldOrigin = faceBounds.getMap("origin"); + WritableMap mirroredOrigin = FaceDetectorUtils.positionMirroredHorizontally( + oldOrigin, mImageDimensions.getWidth(), mScaleX); + + double translateX = -faceBounds.getMap("size").getDouble("width"); + WritableMap translatedMirroredOrigin = FaceDetectorUtils.positionTranslatedHorizontally(mirroredOrigin, translateX); + + WritableMap newBounds = Arguments.createMap(); + newBounds.merge(faceBounds); + newBounds.putMap("origin", translatedMirroredOrigin); + + text.putMap("bounds", newBounds); + + ReadableArray oldComponents = text.getArray("components"); + WritableArray newComponents = Arguments.createArray(); + for (int i = 0; i < oldComponents.size(); ++i) { + WritableMap component = Arguments.createMap(); + component.merge(oldComponents.getMap(i)); + rotateTextX(component); + newComponents.pushMap(component); + } + text.putArray("components", newComponents); + + return text; + } + +} diff --git a/android/src/main/java/org/reactnative/camera/tasks/TextRecognizerAsyncTask.java b/android/src/main/java/org/reactnative/camera/tasks/TextRecognizerAsyncTask.java new file mode 100644 index 000000000..e1be5b3f2 --- /dev/null +++ b/android/src/main/java/org/reactnative/camera/tasks/TextRecognizerAsyncTask.java @@ -0,0 +1,55 @@ +package org.reactnative.camera.tasks; + +import android.util.SparseArray; + +import com.google.android.gms.vision.text.TextBlock; +import com.google.android.gms.vision.text.TextRecognizer; +import org.reactnative.facedetector.RNFrame; +import org.reactnative.facedetector.RNFrameFactory; + + +public class TextRecognizerAsyncTask extends android.os.AsyncTask> { + + private TextRecognizerAsyncTaskDelegate mDelegate; + private TextRecognizer mTextRecognizer; + private byte[] mImageData; + private int mWidth; + private int mHeight; + private int mRotation; + + public TextRecognizerAsyncTask( + TextRecognizerAsyncTaskDelegate delegate, + TextRecognizer textRecognizer, + byte[] imageData, + int width, + int height, + int rotation + ) { + mDelegate = delegate; + mTextRecognizer = textRecognizer; + mImageData = imageData; + mWidth = width; + mHeight = height; + mRotation = rotation; + } + + @Override + protected SparseArray doInBackground(Void... ignored) { + if (isCancelled() || mDelegate == null || mTextRecognizer == null || !mTextRecognizer.isOperational()) { + return null; + } + + RNFrame frame = RNFrameFactory.buildFrame(mImageData, mWidth, mHeight, mRotation); + return mTextRecognizer.detect(frame.getFrame()); + } + + @Override + protected void onPostExecute(SparseArray textBlocks) { + super.onPostExecute(textBlocks); + + if (textBlocks != null) { + mDelegate.onTextRecognized(textBlocks, mWidth, mHeight, mRotation); + } + mDelegate.onTextRecognizerTaskCompleted(); + } +} diff --git a/android/src/main/java/org/reactnative/camera/tasks/TextRecognizerAsyncTaskDelegate.java b/android/src/main/java/org/reactnative/camera/tasks/TextRecognizerAsyncTaskDelegate.java new file mode 100644 index 000000000..fad70a60e --- /dev/null +++ b/android/src/main/java/org/reactnative/camera/tasks/TextRecognizerAsyncTaskDelegate.java @@ -0,0 +1,10 @@ +package org.reactnative.camera.tasks; + +import android.util.SparseArray; + +import com.google.android.gms.vision.text.TextBlock; + +public interface TextRecognizerAsyncTaskDelegate { + void onTextRecognized(SparseArray textBlocks, int sourceWidth, int sourceHeight, int sourceRotation); + void onTextRecognizerTaskCompleted(); +} diff --git a/src/RNCamera.js b/src/RNCamera.js index a7aafb3fc..dfe91bf09 100644 --- a/src/RNCamera.js +++ b/src/RNCamera.js @@ -58,6 +58,7 @@ type PropsType = ViewPropTypes & { autoFocus?: string | boolean | number, faceDetectionClassifications?: number, onFacesDetected?: ({ faces: Array }) => void, + onTextRecognized?: Function, captureAudio?: boolean, useCamera2Api?: boolean, }; @@ -122,6 +123,7 @@ export default class Camera extends React.Component { onCameraReady: PropTypes.func, onBarCodeRead: PropTypes.func, onFacesDetected: PropTypes.func, + onTextRecognized: PropTypes.func, faceDetectionMode: PropTypes.number, faceDetectionLandmarks: PropTypes.number, faceDetectionClassifications: PropTypes.number, @@ -295,6 +297,7 @@ export default class Camera extends React.Component { onCameraReady={this._onCameraReady} onBarCodeRead={this._onObjectDetected(this.props.onBarCodeRead)} onFacesDetected={this._onObjectDetected(this.props.onFacesDetected)} + onTextRecognized={this._onObjectDetected(this.props.onTextRecognized)} /> ); } else if (!this.state.isAuthorizationChecked) { @@ -315,8 +318,13 @@ export default class Camera extends React.Component { newProps.faceDetectorEnabled = true; } + if (props.onTextRecognized) { + newProps.textRecognizerEnabled = true; + } + if (Platform.OS === 'ios') { delete newProps.ratio; + delete newProps.textRecognizerEnabled; } return newProps; @@ -340,6 +348,7 @@ const RNCamera = requireNativeComponent('RNCamera', Camera, { accessibilityLiveRegion: true, barCodeScannerEnabled: true, faceDetectorEnabled: true, + textRecognizerEnabled: true, importantForAccessibility: true, onBarCodeRead: true, onCameraReady: true,