diff --git a/android/src/main/java/com/lwansbrough/RCTCamera/DeepBelief.java b/android/src/main/java/com/lwansbrough/RCTCamera/DeepBelief.java index 794670e59..88d91089b 100644 --- a/android/src/main/java/com/lwansbrough/RCTCamera/DeepBelief.java +++ b/android/src/main/java/com/lwansbrough/RCTCamera/DeepBelief.java @@ -3,6 +3,8 @@ import android.app.Activity; import android.content.Context; import android.content.res.AssetManager; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; import android.os.Bundle; import java.io.File; @@ -10,14 +12,18 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; import com.jetpac.deepbelief.DeepBelief.JPCNNLibrary; import com.sun.jna.Pointer; +import com.sun.jna.ptr.IntByReference; +import com.sun.jna.ptr.PointerByReference; /** * Created by nsipplswezey on 12/19/17. */ + public class DeepBelief extends Activity { public static final String TAG = "DeepBelief"; @@ -53,6 +59,13 @@ static void initDeepBelief() { copyAsset(am, predictorFileName, predictorFile); predictorHandle = JPCNNLibrary.INSTANCE.jpcnn_load_predictor(predictorFile); + Bitmap lenaBitmap = getBitmapFromAsset(am,"lena.png"); + + if(lenaBitmap != null){ + android.util.Log.d("ReactNative", "Classifying lena.png"); + classifyBitmap(lenaBitmap); + } + } private static boolean copyAsset(AssetManager assetManager, @@ -84,6 +97,71 @@ private static void copyFile(InputStream in, OutputStream out) throws IOExceptio } } + public static Bitmap getBitmapFromAsset(AssetManager mgr, String path) { + InputStream is = null; + Bitmap bitmap = null; + try { + is = mgr.open(path); + bitmap = BitmapFactory.decodeStream(is); + } catch (final IOException e) { + bitmap = null; + android.util.Log.d("ReactNative", "error in creating bitmap from asset" + e.getMessage()); + android.util.Log.d("ReactNative", android.util.Log.getStackTraceString(e)); + + } finally { + if (is != null) { + try { + is.close(); + } catch (IOException ignored) { + + } + } + } + return bitmap; + } + + public static void classifyBitmap(Bitmap bitmap) { + final int width = bitmap.getWidth(); + final int height = bitmap.getHeight(); + final int pixelCount = (width * height); + final int bytesPerPixel = 4; + final int byteCount = (pixelCount * bytesPerPixel); + ByteBuffer buffer = ByteBuffer.allocate(byteCount); + bitmap.copyPixelsToBuffer(buffer); + byte[] pixels = buffer.array(); + Pointer imageHandle = JPCNNLibrary.INSTANCE.jpcnn_create_image_buffer_from_uint8_data(pixels, width, height, 4, (4 * width), 0, 1); + + PointerByReference predictionsValuesRef = new PointerByReference(); + IntByReference predictionsLengthRef = new IntByReference(); + PointerByReference predictionsNamesRef = new PointerByReference(); + IntByReference predictionsNamesLengthRef = new IntByReference(); + long startT = System.currentTimeMillis(); + JPCNNLibrary.INSTANCE.jpcnn_classify_image( + networkHandle, + imageHandle, + 0, + -2, + predictionsValuesRef, + predictionsLengthRef, + predictionsNamesRef, + predictionsNamesLengthRef); + + JPCNNLibrary.INSTANCE.jpcnn_destroy_image_buffer(imageHandle); + + Pointer predictionsValuesPointer = predictionsValuesRef.getValue(); + final int predictionsLength = predictionsLengthRef.getValue(); + + + //Start trained model prediction + float trainedPredictionValue = JPCNNLibrary.INSTANCE.jpcnn_predict(predictorHandle, predictionsValuesPointer, predictionsLength); + android.util.Log.d("ReactNative", "jpcnn_predict() value is " + trainedPredictionValue + "."); + //End trained model prediction + + long stopT = System.currentTimeMillis(); + float duration = (float) (stopT - startT) / 1000.0f; + android.util.Log.d("ReactNative", "jpcnn_classify_image() + predict() took " + duration + " seconds."); + } + } diff --git a/android/src/main/java/com/lwansbrough/RCTCamera/RCTCameraViewFinder.java b/android/src/main/java/com/lwansbrough/RCTCamera/RCTCameraViewFinder.java index 99f375b91..b69a38d32 100644 --- a/android/src/main/java/com/lwansbrough/RCTCamera/RCTCameraViewFinder.java +++ b/android/src/main/java/com/lwansbrough/RCTCamera/RCTCameraViewFinder.java @@ -5,8 +5,12 @@ package com.lwansbrough.RCTCamera; import android.content.Context; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.ImageFormat; import android.graphics.Rect; import android.graphics.SurfaceTexture; +import android.graphics.YuvImage; import android.hardware.Camera; import android.view.MotionEvent; import android.view.TextureView; @@ -20,6 +24,7 @@ import com.facebook.react.bridge.WritableMap; import com.facebook.react.modules.core.DeviceEventManagerModule; +import java.io.ByteArrayOutputStream; import java.util.ArrayList; import java.util.List; import java.util.EnumMap; @@ -51,6 +56,9 @@ class RCTCameraViewFinder extends TextureView implements TextureView.SurfaceText // reader instance for the barcode scanner private final MultiFormatReader _multiFormatReader = new MultiFormatReader(); + // CNN Setup: concurrency lock for CNN detector to avoid flooding the runtime + public static volatile boolean CNNDetectorTaskLock = false; + public RCTCameraViewFinder(Context context, int type) { super(context); this.setSurfaceTextureListener(this); @@ -291,12 +299,54 @@ private void initBarcodeReader(List barCodeTypes) { * See {Camera.PreviewCallback} */ public void onPreviewFrame(byte[] data, Camera camera) { + + if (!RCTCameraViewFinder.CNNDetectorTaskLock){ + RCTCameraViewFinder.CNNDetectorTaskLock = true; + new CNNAsyncTask(camera, data).execute(); + } + if (RCTCamera.getInstance().isBarcodeScannerEnabled() && !RCTCameraViewFinder.barcodeScannerTaskLock) { RCTCameraViewFinder.barcodeScannerTaskLock = true; new ReaderAsyncTask(camera, data).execute(); } } + private class CNNAsyncTask extends AsyncTask { + private byte[] imageData; + private final Camera camera; + + CNNAsyncTask(Camera camera, byte[] imageData) { + this.camera = camera; + this.imageData = imageData; + } + + @Override + protected Void doInBackground(Void... ignored) { + if (isCancelled()) { + return null; + } + + Camera.Size previewSize = camera.getParameters().getPreviewSize(); + YuvImage yuvimage=new YuvImage(imageData, ImageFormat.NV21, previewSize.width, previewSize.height, null); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + yuvimage.compressToJpeg(new Rect(0, 0, previewSize.width, previewSize.height), 80, baos); + byte[] jdata = baos.toByteArray(); + Bitmap bmp = BitmapFactory.decodeByteArray(jdata, 0, jdata.length); + + try { + DeepBelief.classifyBitmap(bmp); + } catch (Exception e) { + // meh + android.util.Log.d("ReactNative", "Error classifying bitmap"); + android.util.Log.d("ReactNative", android.util.Log.getStackTraceString(e)); + + } finally { + RCTCameraViewFinder.CNNDetectorTaskLock = false; + return null; + } + } + } + private class ReaderAsyncTask extends AsyncTask { private byte[] imageData; private final Camera camera;