From be2751c71d550e357aa02bb8d7cb0e85f181456a Mon Sep 17 00:00:00 2001 From: izaak Date: Thu, 9 Jan 2025 22:04:48 +0000 Subject: [PATCH 1/2] Add tflite dependencies for segmentation and skeleton of PortraitMode --- app/build.gradle.kts | 5 +++++ .../stonecamera/plugins/PortraitMode.kt | 18 ++++++++++++++++++ gradle/libs.versions.toml | 5 +++++ 3 files changed, 28 insertions(+) create mode 100644 app/src/main/java/co/stonephone/stonecamera/plugins/PortraitMode.kt diff --git a/app/build.gradle.kts b/app/build.gradle.kts index f35d84e..7c3045c 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -83,6 +83,11 @@ dependencies { // ML Kit for Barcode/QR scanning implementation(libs.barcode.scanning) + + // TFLite + implementation(libs.tflite.task.vision) + implementation(libs.tflite.gpu) + implementation(libs.tflite.gpu.delegate) } diff --git a/app/src/main/java/co/stonephone/stonecamera/plugins/PortraitMode.kt b/app/src/main/java/co/stonephone/stonecamera/plugins/PortraitMode.kt new file mode 100644 index 0000000..30820c2 --- /dev/null +++ b/app/src/main/java/co/stonephone/stonecamera/plugins/PortraitMode.kt @@ -0,0 +1,18 @@ +package co.stonephone.stonecamera.plugins + +import co.stonephone.stonecamera.StoneCameraViewModel + +class PortraitMode: IPlugin { + + override val id: String = "portraitModePlugin" + override val name: String = "Portrait Mode" + + override fun initialize(viewModel: StoneCameraViewModel) { + TODO("Not yet implemented") + } + + override val settings: List + get() = TODO("Not yet implemented") + + +} \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a6c00e1..9382938 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -17,6 +17,8 @@ material3version = "1.3.1" junitJunit = "4.12" monitor = "1.7.2" junitKtx = "1.2.1" +tflite = "0.4.4" +tflite-gpu = "2.16.1" [libraries] accompanist-permissions = { module = "com.google.accompanist:accompanist-permissions", version.ref = "accompanistPermissions" } @@ -38,6 +40,9 @@ coil-compose = { module = "io.coil-kt:coil-compose", version.ref = "coilCompose" junit-junit = { group = "junit", name = "junit", version.ref = "junitJunit" } androidx-monitor = { group = "androidx.test", name = "monitor", version.ref = "monitor" } androidx-junit-ktx = { group = "androidx.test.ext", name = "junit-ktx", version.ref = "junitKtx" } +tflite-task-vision = { group = "org.tensorflow", name = "tensorflow-lite-task-vision", version.ref = "tflite"} +tflite-gpu-delegate = { group = "org.tensorflow", name = "tensorflow-lite-gpu-delegate-plugin", version.ref = "tflite"} +tflite-gpu = { group = "org.tensorflow", name = "tensorflow-lite-gpu", version.ref = "tflite-gpu"} [plugins] android-application = { id = "com.android.application", version.ref = "agp" } From fd6a3e6d0ca9c71199043c161a05ec598181b73d Mon Sep 17 00:00:00 2001 From: izaak Date: Fri, 10 Jan 2025 09:18:15 +0000 Subject: [PATCH 2/2] Init tflite runtime & attempt segmentation (it fails) --- .../stonephone/stonecamera/StoneCameraApp.kt | 3 +- .../stonecamera/plugins/PortraitMode.kt | 18 --- .../stonecamera/plugins/PortraitModePlugin.kt | 89 ++++++++++++ .../utils/ImageSegmentationUtils.kt | 136 ++++++++++++++++++ 4 files changed, 227 insertions(+), 19 deletions(-) delete mode 100644 app/src/main/java/co/stonephone/stonecamera/plugins/PortraitMode.kt create mode 100644 app/src/main/java/co/stonephone/stonecamera/plugins/PortraitModePlugin.kt create mode 100644 app/src/main/java/co/stonephone/stonecamera/utils/ImageSegmentationUtils.kt diff --git a/app/src/main/java/co/stonephone/stonecamera/StoneCameraApp.kt b/app/src/main/java/co/stonephone/stonecamera/StoneCameraApp.kt index cf5c000..ef29cc1 100644 --- a/app/src/main/java/co/stonephone/stonecamera/StoneCameraApp.kt +++ b/app/src/main/java/co/stonephone/stonecamera/StoneCameraApp.kt @@ -27,10 +27,10 @@ import androidx.compose.ui.unit.dp import androidx.lifecycle.compose.LocalLifecycleOwner import androidx.lifecycle.viewmodel.compose.viewModel import co.stonephone.stonecamera.plugins.AspectRatioPlugin -import co.stonephone.stonecamera.plugins.DebugPlugin import co.stonephone.stonecamera.plugins.FlashPlugin import co.stonephone.stonecamera.plugins.FocusBasePlugin import co.stonephone.stonecamera.plugins.PinchToZoomPlugin +import co.stonephone.stonecamera.plugins.PortraitModePlugin import co.stonephone.stonecamera.plugins.QRScannerPlugin import co.stonephone.stonecamera.plugins.SettingLocation import co.stonephone.stonecamera.plugins.TapToFocusPlugin @@ -48,6 +48,7 @@ val shootModes = arrayOf("Photo", "Video") // ZoomBar depends on ZoomBase, etc. val PLUGINS = listOf( QRScannerPlugin(), + PortraitModePlugin(), ZoomBasePlugin(), ZoomBarPlugin(), FocusBasePlugin(), diff --git a/app/src/main/java/co/stonephone/stonecamera/plugins/PortraitMode.kt b/app/src/main/java/co/stonephone/stonecamera/plugins/PortraitMode.kt deleted file mode 100644 index 30820c2..0000000 --- a/app/src/main/java/co/stonephone/stonecamera/plugins/PortraitMode.kt +++ /dev/null @@ -1,18 +0,0 @@ -package co.stonephone.stonecamera.plugins - -import co.stonephone.stonecamera.StoneCameraViewModel - -class PortraitMode: IPlugin { - - override val id: String = "portraitModePlugin" - override val name: String = "Portrait Mode" - - override fun initialize(viewModel: StoneCameraViewModel) { - TODO("Not yet implemented") - } - - override val settings: List - get() = TODO("Not yet implemented") - - -} \ No newline at end of file diff --git a/app/src/main/java/co/stonephone/stonecamera/plugins/PortraitModePlugin.kt b/app/src/main/java/co/stonephone/stonecamera/plugins/PortraitModePlugin.kt new file mode 100644 index 0000000..92b7c62 --- /dev/null +++ b/app/src/main/java/co/stonephone/stonecamera/plugins/PortraitModePlugin.kt @@ -0,0 +1,89 @@ +package co.stonephone.stonecamera.plugins + +import android.annotation.SuppressLint +import android.graphics.Bitmap +import android.media.Image +import android.os.Build +import android.util.Log +import androidx.annotation.RequiresApi +import androidx.camera.core.ImageAnalysis +import androidx.camera.core.ImageProxy +import co.stonephone.stonecamera.MyApplication +import co.stonephone.stonecamera.StoneCameraViewModel +import co.stonephone.stonecamera.utils.ImageSegmentationUtils +import co.stonephone.stonecamera.utils.ImageSegmentationUtils.SegmentationListener +import kotlinx.coroutines.CompletableDeferred +import org.tensorflow.lite.task.vision.segmenter.Segmentation + +class PortraitModePlugin: IPlugin, SegmentationListener { + override val id: String = "portraitModePlugin" + override val name: String = "Portrait Mode" + + private lateinit var imageSegmentationUtils: ImageSegmentationUtils + private lateinit var bitmapBuffer: Bitmap + private var imageAnalyzer: ImageAnalysis? = null + + override fun initialize(viewModel: StoneCameraViewModel) { + println("Test"); + imageSegmentationUtils = ImageSegmentationUtils( + context = MyApplication.getAppContext(), + imageSegmentationListener = this + ) + } + + + @RequiresApi(Build.VERSION_CODES.Q) + @SuppressLint("RestrictedApi") + override val onImageAnalysis: ((StoneCameraViewModel, ImageProxy, Image) -> CompletableDeferred)? = + { _: StoneCameraViewModel, + imageProxy: ImageProxy, + image: Image + -> + println("Test") + Log.e("Test", "in here") + val deferred = CompletableDeferred() + + if (!::bitmapBuffer.isInitialized) { + // The image rotation and RGB image buffer are initialized only once + // the analyzer has started running + bitmapBuffer = Bitmap.createBitmap( + image.width, + image.height, + Bitmap.Config.ARGB_8888 + ) + } + + segmentImage(imageProxy) + + deferred + } + + //TODO Image vs imageproxy + @RequiresApi(Build.VERSION_CODES.Q) + private fun segmentImage(image: ImageProxy) { + // Copy out RGB bits to the shared bitmap buffer + image.use { bitmapBuffer.copyPixelsFromBuffer(image.planes[0].buffer) } + + val imageRotation = image.imageInfo.rotationDegrees + // Pass Bitmap and rotation to the image segmentation helper for processing and segmentation + imageSegmentationUtils.segment(bitmapBuffer, imageRotation) + } + + override val settings: List = emptyList() + + override fun onError(error: String) { + Log.e("Segmentation", "Failed segmentation"); + } + + override fun onResults( + results: List?, + inferenceTime: Long, + imageHeight: Int, + imageWidth: Int + ) { + Log.e("Segmentation", results.toString()); + TODO("Not yet implemented") + + //Segmentation has been complete. Can update UI to place segments on the screen. + } +} \ No newline at end of file diff --git a/app/src/main/java/co/stonephone/stonecamera/utils/ImageSegmentationUtils.kt b/app/src/main/java/co/stonephone/stonecamera/utils/ImageSegmentationUtils.kt new file mode 100644 index 0000000..9ec158c --- /dev/null +++ b/app/src/main/java/co/stonephone/stonecamera/utils/ImageSegmentationUtils.kt @@ -0,0 +1,136 @@ +package co.stonephone.stonecamera.utils + +import android.content.Context +import android.graphics.Bitmap +import android.os.Build +import android.os.SystemClock +import android.util.Log +import androidx.annotation.RequiresApi +import org.tensorflow.lite.gpu.CompatibilityList +import org.tensorflow.lite.support.image.ImageProcessor +import org.tensorflow.lite.support.image.TensorImage +import org.tensorflow.lite.support.image.ops.Rot90Op +import org.tensorflow.lite.task.core.BaseOptions +import org.tensorflow.lite.task.vision.segmenter.ImageSegmenter +import org.tensorflow.lite.task.vision.segmenter.OutputType +import org.tensorflow.lite.task.vision.segmenter.Segmentation + +class ImageSegmentationUtils( + var numThreads: Int = 2, + var currentDelegate: Int = 0, + val context: Context, + val imageSegmentationListener: SegmentationListener? +) { + private var imageSegmenter: ImageSegmenter? = null + + init { + setupImageSegmenter() + } + + fun clearImageSegmenter() { + imageSegmenter = null + } + + private fun setupImageSegmenter() { + // Create the base options for the segment + val optionsBuilder = + ImageSegmenter.ImageSegmenterOptions.builder() + + // Set general segmentation options, including number of used threads + val baseOptionsBuilder = BaseOptions.builder().setNumThreads(numThreads) + + // Use the specified hardware for running the model. Default to CPU + when (currentDelegate) { + DELEGATE_CPU -> { + // Default + } + + DELEGATE_GPU -> { + if (CompatibilityList().isDelegateSupportedOnThisDevice) { + baseOptionsBuilder.useGpu() + } else { + imageSegmentationListener?.onError("GPU is not supported on this device") + } + } + + DELEGATE_NNAPI -> { + baseOptionsBuilder.useNnapi() + } + } + + optionsBuilder.setBaseOptions(baseOptionsBuilder.build()) + + /* + CATEGORY_MASK is being specifically used to predict the available objects + based on individual pixels in this sample. The other option available for + OutputType, CONFIDENCE_MAP, provides a gray scale mapping of the image + where each pixel has a confidence score applied to it from 0.0f to 1.0f + */ + optionsBuilder.setOutputType(OutputType.CATEGORY_MASK) + try { + imageSegmenter = + ImageSegmenter.createFromFileAndOptions( + context, + MODEL_DEEPLABV3, + optionsBuilder.build() + ) + } catch (e: IllegalStateException) { + imageSegmentationListener?.onError( + "Image segmentation failed to initialize. See error logs for details" + ) + Log.e(TAG, "TFLite failed to load model with error: " + e.message) + } + } + + @RequiresApi(Build.VERSION_CODES.Q) + fun segment(image: Bitmap, imageRotation: Int) { + + if (imageSegmenter == null) { + setupImageSegmenter() + } + + // Inference time is the difference between the system time at the start and finish of the + // process + var inferenceTime = SystemClock.uptimeMillis() + + // Create preprocessor for the image. + // See https://www.tensorflow.org/lite/inference_with_metadata/ + // lite_support#imageprocessor_architecture + val imageProcessor = + ImageProcessor.Builder() + .add(Rot90Op(-imageRotation / 90)) + .build() + + // Preprocess the image and convert it into a TensorImage for segmentation. + val tensorImage = imageProcessor.process(TensorImage.fromBitmap(image)) + + val segmentResult = imageSegmenter?.segment(tensorImage) + inferenceTime = SystemClock.uptimeMillis() - inferenceTime + + imageSegmentationListener?.onResults( + segmentResult, + inferenceTime, + tensorImage.height, + tensorImage.width + ) + } + + interface SegmentationListener { + fun onError(error: String) + fun onResults( + results: List?, + inferenceTime: Long, + imageHeight: Int, + imageWidth: Int + ) + } + + companion object { + const val DELEGATE_CPU = 0 + const val DELEGATE_GPU = 1 + const val DELEGATE_NNAPI = 2 + const val MODEL_DEEPLABV3 = "deeplabv3.tflite" + + private const val TAG = "Image Segmentation Helper" + } +} \ No newline at end of file