Portrait Mode

app/build.gradle.kts

@@ -83,6 +83,11 @@ dependencies {
 
     // ML Kit for Barcode/QR scanning
     implementation(libs.barcode.scanning)
+
+    // TFLite
+    implementation(libs.tflite.task.vision)
+    implementation(libs.tflite.gpu)
+    implementation(libs.tflite.gpu.delegate)
 }
 
 

app/src/main/java/co/stonephone/stonecamera/StoneCameraApp.kt

@@ -27,10 +27,10 @@ import androidx.compose.ui.unit.dp
 import androidx.lifecycle.compose.LocalLifecycleOwner
 import androidx.lifecycle.viewmodel.compose.viewModel
 import co.stonephone.stonecamera.plugins.AspectRatioPlugin
-import co.stonephone.stonecamera.plugins.DebugPlugin
 import co.stonephone.stonecamera.plugins.FlashPlugin
 import co.stonephone.stonecamera.plugins.FocusBasePlugin
 import co.stonephone.stonecamera.plugins.PinchToZoomPlugin
+import co.stonephone.stonecamera.plugins.PortraitModePlugin
 import co.stonephone.stonecamera.plugins.QRScannerPlugin
 import co.stonephone.stonecamera.plugins.SettingLocation
 import co.stonephone.stonecamera.plugins.TapToFocusPlugin
@@ -48,6 +48,7 @@ val shootModes = arrayOf("Photo", "Video")
 // ZoomBar depends on ZoomBase, etc.
 val PLUGINS = listOf(
     QRScannerPlugin(),
+    PortraitModePlugin(),
     ZoomBasePlugin(),
     ZoomBarPlugin(),
     FocusBasePlugin(),

app/src/main/java/co/stonephone/stonecamera/plugins/PortraitModePlugin.kt

@@ -0,0 +1,89 @@
+package co.stonephone.stonecamera.plugins
+
+import android.annotation.SuppressLint
+import android.graphics.Bitmap
+import android.media.Image
+import android.os.Build
+import android.util.Log
+import androidx.annotation.RequiresApi
+import androidx.camera.core.ImageAnalysis
+import androidx.camera.core.ImageProxy
+import co.stonephone.stonecamera.MyApplication
+import co.stonephone.stonecamera.StoneCameraViewModel
+import co.stonephone.stonecamera.utils.ImageSegmentationUtils
+import co.stonephone.stonecamera.utils.ImageSegmentationUtils.SegmentationListener
+import kotlinx.coroutines.CompletableDeferred
+import org.tensorflow.lite.task.vision.segmenter.Segmentation
+
+class PortraitModePlugin: IPlugin, SegmentationListener {
+    override val id: String = "portraitModePlugin"
+    override val name: String = "Portrait Mode"
+
+    private lateinit var imageSegmentationUtils: ImageSegmentationUtils
+    private lateinit var bitmapBuffer: Bitmap
+    private var imageAnalyzer: ImageAnalysis? = null
+
+    override fun initialize(viewModel: StoneCameraViewModel) {
+        println("Test");
+        imageSegmentationUtils = ImageSegmentationUtils(
+            context = MyApplication.getAppContext(),
+            imageSegmentationListener = this
+        )
+    }
+
+
+    @RequiresApi(Build.VERSION_CODES.Q)
+    @SuppressLint("RestrictedApi")
+    override val onImageAnalysis: ((StoneCameraViewModel, ImageProxy, Image) -> CompletableDeferred<Unit>)? =
+        { _: StoneCameraViewModel,
+          imageProxy: ImageProxy,
+          image: Image
+            ->
+            println("Test")
+            Log.e("Test", "in here")
+            val deferred = CompletableDeferred<Unit>()
+
+            if (!::bitmapBuffer.isInitialized) {
+                // The image rotation and RGB image buffer are initialized only once
+                // the analyzer has started running
+                bitmapBuffer = Bitmap.createBitmap(
+                    image.width,
+                    image.height,
+                    Bitmap.Config.ARGB_8888
+                )
+            }
+
+            segmentImage(imageProxy)
+
+            deferred
+        }
+
+    //TODO Image vs imageproxy
+    @RequiresApi(Build.VERSION_CODES.Q)
+    private fun segmentImage(image: ImageProxy) {
+        // Copy out RGB bits to the shared bitmap buffer
+        image.use { bitmapBuffer.copyPixelsFromBuffer(image.planes[0].buffer) }
+
+        val imageRotation = image.imageInfo.rotationDegrees
+        // Pass Bitmap and rotation to the image segmentation helper for processing and segmentation
+        imageSegmentationUtils.segment(bitmapBuffer, imageRotation)
+    }
+
+    override val settings: List<PluginSetting> = emptyList()
+
+    override fun onError(error: String) {
+        Log.e("Segmentation", "Failed segmentation");
+    }
+
+    override fun onResults(
+        results: List<Segmentation>?,
+        inferenceTime: Long,
+        imageHeight: Int,
+        imageWidth: Int
+    ) {
+        Log.e("Segmentation", results.toString());
+        TODO("Not yet implemented")
+
+        //Segmentation has been complete. Can update UI to place segments on the screen.
+    }
+}

app/src/main/java/co/stonephone/stonecamera/utils/ImageSegmentationUtils.kt

@@ -0,0 +1,136 @@
+package co.stonephone.stonecamera.utils
+
+import android.content.Context
+import android.graphics.Bitmap
+import android.os.Build
+import android.os.SystemClock
+import android.util.Log
+import androidx.annotation.RequiresApi
+import org.tensorflow.lite.gpu.CompatibilityList
+import org.tensorflow.lite.support.image.ImageProcessor
+import org.tensorflow.lite.support.image.TensorImage
+import org.tensorflow.lite.support.image.ops.Rot90Op
+import org.tensorflow.lite.task.core.BaseOptions
+import org.tensorflow.lite.task.vision.segmenter.ImageSegmenter
+import org.tensorflow.lite.task.vision.segmenter.OutputType
+import org.tensorflow.lite.task.vision.segmenter.Segmentation
+
+class ImageSegmentationUtils(
+    var numThreads: Int = 2,
+    var currentDelegate: Int = 0,
+    val context: Context,
+    val imageSegmentationListener: SegmentationListener?
+) {
+    private var imageSegmenter: ImageSegmenter? = null
+
+    init {
+        setupImageSegmenter()
+    }
+
+    fun clearImageSegmenter() {
+        imageSegmenter = null
+    }
+
+    private fun setupImageSegmenter() {
+        // Create the base options for the segment
+        val optionsBuilder =
+            ImageSegmenter.ImageSegmenterOptions.builder()
+
+        // Set general segmentation options, including number of used threads
+        val baseOptionsBuilder = BaseOptions.builder().setNumThreads(numThreads)
+
+        // Use the specified hardware for running the model. Default to CPU
+        when (currentDelegate) {
+            DELEGATE_CPU -> {
+                // Default
+            }
+
+            DELEGATE_GPU -> {
+                if (CompatibilityList().isDelegateSupportedOnThisDevice) {
+                    baseOptionsBuilder.useGpu()
+                } else {
+                    imageSegmentationListener?.onError("GPU is not supported on this device")
+                }
+            }
+
+            DELEGATE_NNAPI -> {
+                baseOptionsBuilder.useNnapi()
+            }
+        }
+
+        optionsBuilder.setBaseOptions(baseOptionsBuilder.build())
+
+        /*
+        CATEGORY_MASK is being specifically used to predict the available objects
+        based on individual pixels in this sample. The other option available for
+        OutputType, CONFIDENCE_MAP, provides a gray scale mapping of the image
+        where each pixel has a confidence score applied to it from 0.0f to 1.0f
+         */
+        optionsBuilder.setOutputType(OutputType.CATEGORY_MASK)
+        try {
+            imageSegmenter =
+                ImageSegmenter.createFromFileAndOptions(
+                    context,
+                    MODEL_DEEPLABV3,
+                    optionsBuilder.build()
+                )
+        } catch (e: IllegalStateException) {
+            imageSegmentationListener?.onError(
+                "Image segmentation failed to initialize. See error logs for details"
+            )
+            Log.e(TAG, "TFLite failed to load model with error: " + e.message)
+        }
+    }
+
+    @RequiresApi(Build.VERSION_CODES.Q)
+    fun segment(image: Bitmap, imageRotation: Int) {
+
+        if (imageSegmenter == null) {
+            setupImageSegmenter()
+        }
+
+        // Inference time is the difference between the system time at the start and finish of the
+        // process
+        var inferenceTime = SystemClock.uptimeMillis()
+
+        // Create preprocessor for the image.
+        // See https://www.tensorflow.org/lite/inference_with_metadata/
+        //            lite_support#imageprocessor_architecture
+        val imageProcessor =
+            ImageProcessor.Builder()
+                .add(Rot90Op(-imageRotation / 90))
+                .build()
+
+        // Preprocess the image and convert it into a TensorImage for segmentation.
+        val tensorImage = imageProcessor.process(TensorImage.fromBitmap(image))
+
+        val segmentResult = imageSegmenter?.segment(tensorImage)
+        inferenceTime = SystemClock.uptimeMillis() - inferenceTime
+
+        imageSegmentationListener?.onResults(
+            segmentResult,
+            inferenceTime,
+            tensorImage.height,
+            tensorImage.width
+        )
+    }
+
+    interface SegmentationListener {
+        fun onError(error: String)
+        fun onResults(
+            results: List<Segmentation>?,
+            inferenceTime: Long,
+            imageHeight: Int,
+            imageWidth: Int
+        )
+    }
+
+    companion object {
+        const val DELEGATE_CPU = 0
+        const val DELEGATE_GPU = 1
+        const val DELEGATE_NNAPI = 2
+        const val MODEL_DEEPLABV3 = "deeplabv3.tflite"
+
+        private const val TAG = "Image Segmentation Helper"
+    }
+}

gradle/libs.versions.toml

@@ -17,6 +17,8 @@ material3version = "1.3.1"
 junitJunit = "4.12"
 monitor = "1.7.2"
 junitKtx = "1.2.1"
+tflite = "0.4.4"
+tflite-gpu = "2.16.1"
 
 [libraries]
 accompanist-permissions = { module = "com.google.accompanist:accompanist-permissions", version.ref = "accompanistPermissions" }
@@ -38,6 +40,9 @@ coil-compose = { module = "io.coil-kt:coil-compose", version.ref = "coilCompose"
 junit-junit = { group = "junit", name = "junit", version.ref = "junitJunit" }
 androidx-monitor = { group = "androidx.test", name = "monitor", version.ref = "monitor" }
 androidx-junit-ktx = { group = "androidx.test.ext", name = "junit-ktx", version.ref = "junitKtx" }
+tflite-task-vision = { group = "org.tensorflow", name = "tensorflow-lite-task-vision", version.ref = "tflite"}
+tflite-gpu-delegate = { group = "org.tensorflow", name = "tensorflow-lite-gpu-delegate-plugin", version.ref = "tflite"}
+tflite-gpu = { group = "org.tensorflow", name = "tensorflow-lite-gpu", version.ref = "tflite-gpu"}
 
 [plugins]
 android-application = { id = "com.android.application", version.ref = "agp" }