Async CNN task, and running on live frames

android/src/main/java/com/lwansbrough/RCTCamera/DeepBelief.java

@@ -3,21 +3,27 @@
 import android.app.Activity;
 import android.content.Context;
 import android.content.res.AssetManager;
+import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
 import android.os.Bundle;
 
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.nio.ByteBuffer;
 
 import com.jetpac.deepbelief.DeepBelief.JPCNNLibrary;
 import com.sun.jna.Pointer;
+import com.sun.jna.ptr.IntByReference;
+import com.sun.jna.ptr.PointerByReference;
 
 /**
  * Created by nsipplswezey on 12/19/17.
  */
 
+
 public class DeepBelief extends Activity {
 
     public static final String TAG = "DeepBelief";
@@ -53,6 +59,13 @@ static void initDeepBelief() {
         copyAsset(am, predictorFileName, predictorFile);
         predictorHandle = JPCNNLibrary.INSTANCE.jpcnn_load_predictor(predictorFile);
 
+        Bitmap lenaBitmap = getBitmapFromAsset(am,"lena.png");
+
+        if(lenaBitmap != null){
+            android.util.Log.d("ReactNative", "Classifying lena.png");
+            classifyBitmap(lenaBitmap);
+        }
+
     }
 
     private static boolean copyAsset(AssetManager assetManager,
@@ -84,6 +97,71 @@ private static void copyFile(InputStream in, OutputStream out) throws IOExceptio
         }
     }
 
+    public static Bitmap getBitmapFromAsset(AssetManager mgr, String path) {
+        InputStream is = null;
+        Bitmap bitmap = null;
+        try {
+            is = mgr.open(path);
+            bitmap = BitmapFactory.decodeStream(is);
+        } catch (final IOException e) {
+            bitmap = null;
+            android.util.Log.d("ReactNative", "error in creating bitmap from asset" + e.getMessage());
+            android.util.Log.d("ReactNative",  android.util.Log.getStackTraceString(e));
+
+        } finally {
+            if (is != null) {
+                try {
+                    is.close();
+                } catch (IOException ignored) {
+
+                }
+            }
+        }
+        return bitmap;
+    }
+
+    public static void classifyBitmap(Bitmap bitmap) {
+        final int width = bitmap.getWidth();
+        final int height = bitmap.getHeight();
+        final int pixelCount = (width * height);
+        final int bytesPerPixel = 4;
+        final int byteCount = (pixelCount * bytesPerPixel);
+        ByteBuffer buffer = ByteBuffer.allocate(byteCount);
+        bitmap.copyPixelsToBuffer(buffer);
+        byte[] pixels = buffer.array();
+        Pointer imageHandle = JPCNNLibrary.INSTANCE.jpcnn_create_image_buffer_from_uint8_data(pixels, width, height, 4, (4 * width), 0, 1);
+
+        PointerByReference predictionsValuesRef = new PointerByReference();
+        IntByReference predictionsLengthRef = new IntByReference();
+        PointerByReference predictionsNamesRef = new PointerByReference();
+        IntByReference predictionsNamesLengthRef = new IntByReference();
+        long startT = System.currentTimeMillis();
+        JPCNNLibrary.INSTANCE.jpcnn_classify_image(
+                networkHandle,
+                imageHandle,
+                0,
+                -2,
+                predictionsValuesRef,
+                predictionsLengthRef,
+                predictionsNamesRef,
+                predictionsNamesLengthRef);
+
+        JPCNNLibrary.INSTANCE.jpcnn_destroy_image_buffer(imageHandle);
+
+        Pointer predictionsValuesPointer = predictionsValuesRef.getValue();
+        final int predictionsLength = predictionsLengthRef.getValue();
+
+
+        //Start trained model prediction
+        float trainedPredictionValue = JPCNNLibrary.INSTANCE.jpcnn_predict(predictorHandle, predictionsValuesPointer, predictionsLength);
+        android.util.Log.d("ReactNative", "jpcnn_predict() value is " + trainedPredictionValue + ".");
+        //End trained model prediction
+
+        long stopT = System.currentTimeMillis();
+        float duration = (float) (stopT - startT) / 1000.0f;
+        android.util.Log.d("ReactNative", "jpcnn_classify_image() + predict() took " + duration + " seconds.");
+    }
+
 
 
 }

android/src/main/java/com/lwansbrough/RCTCamera/RCTCameraViewFinder.java

@@ -5,8 +5,12 @@
 package com.lwansbrough.RCTCamera;
 
 import android.content.Context;
+import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
+import android.graphics.ImageFormat;
 import android.graphics.Rect;
 import android.graphics.SurfaceTexture;
+import android.graphics.YuvImage;
 import android.hardware.Camera;
 import android.view.MotionEvent;
 import android.view.TextureView;
@@ -20,6 +24,7 @@
 import com.facebook.react.bridge.WritableMap;
 import com.facebook.react.modules.core.DeviceEventManagerModule;
 
+import java.io.ByteArrayOutputStream;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.EnumMap;
@@ -51,6 +56,9 @@ class RCTCameraViewFinder extends TextureView implements TextureView.SurfaceText
     // reader instance for the barcode scanner
     private final MultiFormatReader _multiFormatReader = new MultiFormatReader();
 
+    // CNN Setup: concurrency lock for CNN detector to avoid flooding the runtime
+    public static volatile boolean CNNDetectorTaskLock = false;
+
     public RCTCameraViewFinder(Context context, int type) {
         super(context);
         this.setSurfaceTextureListener(this);
@@ -291,12 +299,54 @@ private void initBarcodeReader(List<String> barCodeTypes) {
      * See {Camera.PreviewCallback}
      */
     public void onPreviewFrame(byte[] data, Camera camera) {
+
+        if (!RCTCameraViewFinder.CNNDetectorTaskLock){
+            RCTCameraViewFinder.CNNDetectorTaskLock = true;
+            new CNNAsyncTask(camera, data).execute();
+        }
+
         if (RCTCamera.getInstance().isBarcodeScannerEnabled() && !RCTCameraViewFinder.barcodeScannerTaskLock) {
             RCTCameraViewFinder.barcodeScannerTaskLock = true;
             new ReaderAsyncTask(camera, data).execute();
         }
     }
 
+    private class CNNAsyncTask extends AsyncTask<Void, Void, Void> {
+        private byte[] imageData;
+        private final Camera camera;
+
+        CNNAsyncTask(Camera camera, byte[] imageData) {
+            this.camera = camera;
+            this.imageData = imageData;
+        }
+
+        @Override
+        protected Void doInBackground(Void... ignored) {
+            if (isCancelled()) {
+                return null;
+            }
+
+            Camera.Size previewSize = camera.getParameters().getPreviewSize();
+            YuvImage yuvimage=new YuvImage(imageData, ImageFormat.NV21, previewSize.width, previewSize.height, null);
+            ByteArrayOutputStream baos = new ByteArrayOutputStream();
+            yuvimage.compressToJpeg(new Rect(0, 0, previewSize.width, previewSize.height), 80, baos);
+            byte[] jdata = baos.toByteArray();
+            Bitmap bmp = BitmapFactory.decodeByteArray(jdata, 0, jdata.length);
+
+            try {
+                DeepBelief.classifyBitmap(bmp);
+            } catch (Exception e) {
+                // meh
+                android.util.Log.d("ReactNative", "Error classifying bitmap");
+                android.util.Log.d("ReactNative",  android.util.Log.getStackTraceString(e));
+
+            } finally {
+                RCTCameraViewFinder.CNNDetectorTaskLock = false;
+                return null;
+            }
+        }
+    }
+
     private class ReaderAsyncTask extends AsyncTask<Void, Void, Void> {
         private byte[] imageData;
         private final Camera camera;