update: replace core + extensions modules with core + ml

core/build.gradle.kts

@@ -1,9 +1,9 @@
-import java.io.ByteArrayOutputStream
-
 plugins {
     id("com.android.library")
     id("org.jetbrains.kotlin.android")
     id("maven-publish")
+    id("com.google.devtools.ksp")
+
 }
 
 android {
@@ -13,15 +13,14 @@ android {
     defaultConfig {
         minSdk = 30
         testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
-
     }
 
     packaging {
         resources.excludes.addAll(
             listOf(
                 "META-INF/LICENSE.md",
                 "META-INF/LICENSE-notice.md",
-                )
+            )
         )
     }
 
@@ -40,13 +39,10 @@ android {
         jvmTarget = "17"
     }
 
-    buildFeatures {
-        // Add any enabled features here if needed
-    }
-
     lint {
         targetSdk = 34
     }
+
     testOptions {
         unitTests {
             isIncludeAndroidResources = true
@@ -55,6 +51,7 @@ android {
             }
         }
     }
+
 }
 
 java {
@@ -65,25 +62,28 @@ java {
 }
 
 dependencies {
-    implementation(libs.androidx.documentfile)
-    implementation(libs.onnxruntime.android)
-
-    // Expose core-ktx and onnxruntime to consumers of core or extensions
+    // Expose core-ktx to consumers of core or extensions
     api(libs.androidx.core.ktx)
 
     // JVM unit tests
-    testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.7.3")
-    testImplementation("org.junit.jupiter:junit-jupiter-api:5.10.0")
-    testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.10.0")
-    testImplementation("io.mockk:mockk:1.14.5")
+    testImplementation(libs.kotlinx.coroutines.test)
+    testImplementation(libs.junit.jupiter.api)
+    testRuntimeOnly(libs.junit.jupiter.engine)
+    testImplementation(libs.mockk)
     testImplementation(kotlin("test"))
 
     // Android instrumented tests
-    androidTestImplementation("androidx.test:core:1.7.0")
-    androidTestImplementation("androidx.test.ext:junit-ktx:1.3.0")
-    androidTestImplementation("androidx.test:runner:1.6.1")
-    androidTestImplementation("io.mockk:mockk-android:1.14.5")
-    androidTestImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.7.3")
+    androidTestImplementation(libs.androidx.core)
+    androidTestImplementation(libs.androidx.junit.ktx)
+    androidTestImplementation(libs.androidx.runner)
+    androidTestImplementation(libs.mockk.android)
+    androidTestImplementation(libs.kotlinx.coroutines.test)
+
+    androidTestImplementation(libs.androidx.room.runtime)
+    androidTestImplementation(libs.androidx.room.ktx)
+    androidTestImplementation(libs.androidx.room.testing)
+    ksp(libs.androidx.room.compiler)
+
 }
 
 val gitVersion: String by lazy {
@@ -96,6 +96,7 @@ val gitVersion: String by lazy {
     }.getOrDefault("1.0.0")
 }
 
+
 publishing {
     publications {
         register<MavenPublication>("release") {

extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions/data/images/Converters.kt → core/src/androidTest/kotlin/com/fpf/smartscansdk/core/data/images/Converters.kt

@@ -1,4 +1,4 @@
-package com.fpf.smartscansdk.extensions.data.images
+package com.fpf.smartscansdk.core.data.images
 
 import androidx.room.*
 

extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions/data/images/Dao.kt → core/src/androidTest/kotlin/com/fpf/smartscansdk/core/data/images/Dao.kt

@@ -1,4 +1,4 @@
-package com.fpf.smartscansdk.extensions.data.images
+package com.fpf.smartscansdk.core.data.images
 
 import androidx.room.*
 

extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions/data/images/Database.kt → core/src/androidTest/kotlin/com/fpf/smartscansdk/core/data/images/Database.kt

@@ -1,4 +1,4 @@
-package com.fpf.smartscansdk.extensions.data.images
+package com.fpf.smartscansdk.core.data.images
 
 import android.app.Application
 import androidx.room.*

extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions/data/images/Entity.kt → core/src/androidTest/kotlin/com/fpf/smartscansdk/core/data/images/Entity.kt

@@ -1,7 +1,7 @@
-package com.fpf.smartscansdk.extensions.data.images
+package com.fpf.smartscansdk.core.data.images
 
 import androidx.room.*
-import com.fpf.smartscansdk.core.ml.embeddings.Embedding
+import com.fpf.smartscansdk.core.data.Embedding
 
 @Entity(tableName = "image_embeddings")
 data class ImageEmbeddingEntity(

extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions/embeddings/EmbeddingLoadingBenchmarkTest.kt → core/src/androidTest/kotlin/com/fpf/smartscansdk/core/embeddings/EmbeddingLoadingBenchmarkTest.kt

@@ -1,11 +1,11 @@
-package com.fpf.smartscansdk.extensions.embeddings
+package com.fpf.smartscansdk.core.embeddings
 
 import android.app.Application
 import androidx.test.core.app.ApplicationProvider
 import androidx.test.ext.junit.runners.AndroidJUnit4
-import com.fpf.smartscansdk.extensions.data.images.ImageEmbeddingDatabase
-import com.fpf.smartscansdk.extensions.data.images.ImageEmbeddingEntity
-import com.fpf.smartscansdk.extensions.data.images.toEmbedding
+import com.fpf.smartscansdk.core.data.images.ImageEmbeddingDatabase
+import com.fpf.smartscansdk.core.data.images.ImageEmbeddingEntity
+import com.fpf.smartscansdk.core.data.images.toEmbedding
 import junit.framework.Assert.assertEquals
 import kotlinx.coroutines.runBlocking
 import org.junit.Test

core/src/main/java/com/fpf/smartscansdk/core/data/Classification.kt

@@ -0,0 +1,9 @@
+package com.fpf.smartscansdk.core.data
+
+sealed class ClassificationResult {
+    data class Success(val classId: String, val similarity: Float ): ClassificationResult()
+    data class Failure(val error: ClassificationError ): ClassificationResult()
+}
+
+enum class ClassificationError{MINIMUM_CLASS_SIZE, THRESHOLD, CONFIDENCE_MARGIN}
+

core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/EmbeddingTypes.kt → core/src/main/java/com/fpf/smartscansdk/core/data/Embedings.kt

@@ -1,4 +1,5 @@
-package com.fpf.smartscansdk.core.ml.embeddings
+package com.fpf.smartscansdk.core.data
+
 
 import android.graphics.Bitmap
 
@@ -39,6 +40,7 @@ interface IEmbeddingProvider<T> {
     val embeddingDim: Int? get() = null
     fun closeSession() = Unit
     suspend fun embed(data: T): FloatArray
+    suspend fun embedBatch(data: List<T>): List<FloatArray>
 }
 
 

core/src/main/java/com/fpf/smartscansdk/core/data/Processors.kt

@@ -0,0 +1,30 @@
+package com.fpf.smartscansdk.core.data
+
+import android.content.Context
+
+interface IProcessorListener<Input, Output> {
+    suspend fun onActive(context: Context) = Unit
+    suspend fun onBatchComplete(context: Context, batch: List<Output>) = Unit
+    suspend fun onComplete(context: Context, metrics: Metrics.Success) = Unit
+    suspend fun onProgress(context: Context, progress: Float) = Unit
+    fun onError(context: Context, error: Exception, item: Input) = Unit
+    suspend fun onFail(context: Context, failureMetrics: Metrics.Failure) = Unit
+}
+
+sealed class Metrics {
+    data class Success(val totalProcessed: Int = 0, val timeElapsed: Long = 0L) : Metrics()
+    data class Failure(val processedBeforeFailure: Int, val timeElapsed: Long, val error: Exception) : Metrics()
+}
+
+data class MemoryOptions(
+    val lowMemoryThreshold: Long = 800L * 1024 * 1024,
+    val highMemoryThreshold: Long = 1_600L * 1024 * 1024,
+    val minConcurrency: Int = 1,
+    val maxConcurrency: Int = 4
+)
+
+data class ProcessOptions(
+    val memory: MemoryOptions = MemoryOptions(),
+    val batchSize: Int = 10
+)
+

core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/EmbeddingUtils.kt → core/src/main/java/com/fpf/smartscansdk/core/embeddings/EmbeddingUtils.kt

@@ -1,6 +1,5 @@
-package com.fpf.smartscansdk.core.ml.embeddings
+package com.fpf.smartscansdk.core.embeddings
 
-import android.content.Context
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 import kotlin.math.sqrt
@@ -36,3 +35,24 @@ suspend fun generatePrototypeEmbedding(rawEmbeddings: List<FloatArray>): FloatAr
 
         normalizeL2(FloatArray(embeddingLength) { i -> sum[i] / rawEmbeddings.size })
     }
+
+
+fun flattenEmbeddings(embeddings: List<FloatArray>, embeddingDim: Int): FloatArray {
+    val batchSize = embeddings.size
+    val flattened = FloatArray(batchSize * embeddingDim)
+    for (i in embeddings.indices) {
+        System.arraycopy(embeddings[i], 0, flattened, i * embeddingDim, embeddingDim)
+    }
+    return flattened
+}
+
+fun unflattenEmbeddings(flattened: FloatArray, embeddingDim: Int): List<FloatArray> {
+    val batchSize = flattened.size / embeddingDim
+    val embeddings = mutableListOf<FloatArray>()
+    for (i in 0 until batchSize) {
+        val embedding = FloatArray(embeddingDim)
+        System.arraycopy(flattened, i * embeddingDim, embedding, 0, embeddingDim)
+        embeddings.add(embedding)
+    }
+    return embeddings
+}

extensions/src/main/java/com/fpf/smartscansdk/extensions/embeddings/FileEmbeddingRetriever.kt → core/src/main/java/com/fpf/smartscansdk/core/embeddings/FileEmbeddingRetriever.kt

@@ -1,21 +1,15 @@
-package com.fpf.smartscansdk.extensions.embeddings
+package com.fpf.smartscansdk.core.embeddings
 
-import com.fpf.smartscansdk.core.ml.embeddings.Embedding
-import com.fpf.smartscansdk.core.ml.embeddings.IRetriever
-import com.fpf.smartscansdk.core.ml.embeddings.getSimilarities
-import com.fpf.smartscansdk.core.ml.embeddings.getTopN
+import com.fpf.smartscansdk.core.data.Embedding
+import com.fpf.smartscansdk.core.data.IRetriever
 
 class FileEmbeddingRetriever(
     private val store: FileEmbeddingStore
 ): IRetriever {
 
     private var cachedIds: List<Long>? = null
 
-    override suspend fun query(
-        embedding: FloatArray,
-        topK: Int,
-        threshold: Float
-    ): List<Embedding> {
+    override suspend fun query(embedding: FloatArray, topK: Int, threshold: Float): List<Embedding> {
 
         cachedIds = null // clear on new search
 

extensions/src/main/java/com/fpf/smartscansdk/extensions/embeddings/FileEmbeddingStore.kt → core/src/main/java/com/fpf/smartscansdk/core/embeddings/FileEmbeddingStore.kt

@@ -1,8 +1,8 @@
-package com.fpf.smartscansdk.extensions.embeddings
+package com.fpf.smartscansdk.core.embeddings
 
 import android.util.Log
-import com.fpf.smartscansdk.core.ml.embeddings.Embedding
-import com.fpf.smartscansdk.core.ml.embeddings.IEmbeddingStore
+import com.fpf.smartscansdk.core.data.Embedding
+import com.fpf.smartscansdk.core.data.IEmbeddingStore
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 import java.io.File

extensions/src/main/java/com/fpf/smartscansdk/extensions/indexers/ImageIndexer.kt → core/src/main/java/com/fpf/smartscansdk/core/indexers/ImageIndexer.kt

@@ -1,17 +1,15 @@
-package com.fpf.smartscansdk.extensions.indexers
+package com.fpf.smartscansdk.core.indexers
 
-import android.app.Application
 import android.content.ContentUris
 import android.content.Context
 import android.provider.MediaStore
-import com.fpf.smartscansdk.core.utils.getBitmapFromUri
-import com.fpf.smartscansdk.core.ml.embeddings.Embedding
-import com.fpf.smartscansdk.core.ml.embeddings.IEmbeddingStore
-import com.fpf.smartscansdk.core.ml.embeddings.clip.ClipConfig
-import com.fpf.smartscansdk.core.ml.embeddings.clip.ClipImageEmbedder
+import com.fpf.smartscansdk.core.data.Embedding
+import com.fpf.smartscansdk.core.data.IEmbeddingStore
+import com.fpf.smartscansdk.core.data.IProcessorListener
+import com.fpf.smartscansdk.core.data.ImageEmbeddingProvider
+import com.fpf.smartscansdk.core.data.ProcessOptions
+import com.fpf.smartscansdk.core.media.getBitmapFromUri
 import com.fpf.smartscansdk.core.processors.BatchProcessor
-import com.fpf.smartscansdk.core.processors.IProcessorListener
-import com.fpf.smartscansdk.core.processors.ProcessOptions
 import kotlinx.coroutines.NonCancellable
 import kotlinx.coroutines.withContext
 
@@ -21,12 +19,13 @@ import kotlinx.coroutines.withContext
 // These benchmarks strongly favour the  FileEmbeddingStore for optimal on-device search functionality and UX.
 
 class ImageIndexer(
-    private val embedder: ClipImageEmbedder,
-    application: Application,
+    private val embedder: ImageEmbeddingProvider,
+    private val store: IEmbeddingStore,
+    private val maxImageSize: Int = 225,
+    context: Context,
     listener: IProcessorListener<Long, Embedding>? = null,
     options: ProcessOptions = ProcessOptions(),
-    private val store: IEmbeddingStore,
-    ): BatchProcessor<Long, Embedding>(application, listener, options){
+    ): BatchProcessor<Long, Embedding>(context, listener, options){
 
     companion object {
         const val INDEX_FILENAME = "image_index.bin"
@@ -40,7 +39,7 @@ class ImageIndexer(
         val contentUri = ContentUris.withAppendedId(
             MediaStore.Images.Media.EXTERNAL_CONTENT_URI, item
         )
-        val bitmap = getBitmapFromUri(context, contentUri, ClipConfig.IMAGE_SIZE_X)
+        val bitmap = getBitmapFromUri(context, contentUri, maxImageSize)
         val embedding = withContext(NonCancellable) {
             embedder.embed(bitmap)
         }

extensions/src/main/java/com/fpf/smartscansdk/extensions/indexers/VideoIndexer.kt → core/src/main/java/com/fpf/smartscansdk/core/indexers/VideoIndexer.kt

@@ -1,19 +1,17 @@
-package com.fpf.smartscansdk.extensions.indexers
+package com.fpf.smartscansdk.core.indexers
 
 import android.app.Application
 import android.content.ContentUris
 import android.content.Context
 import android.provider.MediaStore
-import com.fpf.smartscansdk.core.ml.embeddings.Embedding
-import com.fpf.smartscansdk.core.ml.embeddings.IEmbeddingStore
-import com.fpf.smartscansdk.core.ml.embeddings.clip.ClipConfig.IMAGE_SIZE_X
-import com.fpf.smartscansdk.core.ml.embeddings.clip.ClipConfig.IMAGE_SIZE_Y
-import com.fpf.smartscansdk.core.ml.embeddings.clip.ClipImageEmbedder
-import com.fpf.smartscansdk.core.ml.embeddings.generatePrototypeEmbedding
+import com.fpf.smartscansdk.core.data.Embedding
+import com.fpf.smartscansdk.core.data.IEmbeddingStore
+import com.fpf.smartscansdk.core.data.IProcessorListener
+import com.fpf.smartscansdk.core.data.ImageEmbeddingProvider
+import com.fpf.smartscansdk.core.data.ProcessOptions
+import com.fpf.smartscansdk.core.embeddings.generatePrototypeEmbedding
 import com.fpf.smartscansdk.core.processors.BatchProcessor
-import com.fpf.smartscansdk.core.processors.IProcessorListener
-import com.fpf.smartscansdk.core.processors.ProcessOptions
-import com.fpf.smartscansdk.core.utils.extractFramesFromVideo
+import com.fpf.smartscansdk.core.media.extractFramesFromVideo
 
 // ** Design Constraint**: For on-device vector search, the full index needs to be loaded in-memory (or make an Android native VectorDB)
 // File-based EmbeddingStore is used over a Room version due to significant faster index loading
@@ -22,15 +20,15 @@ import com.fpf.smartscansdk.core.utils.extractFramesFromVideo
 // **IMPORTANT**: Video frame extraction can fail due to codec incompatibility
 
 class VideoIndexer(
-    private val embedder: ClipImageEmbedder,
+    private val embedder: ImageEmbeddingProvider,
     private val frameCount: Int = 10,
-    private val width: Int = IMAGE_SIZE_X,
-    private val height: Int = IMAGE_SIZE_Y,
-    application: Application,
+    private val width: Int,
+    private val height: Int,
+    context: Context,
     listener: IProcessorListener<Long, Embedding>? = null,
     options: ProcessOptions = ProcessOptions(),
     private val store: IEmbeddingStore,
-    ): BatchProcessor<Long, Embedding>(application, listener, options){
+    ): BatchProcessor<Long, Embedding>(context, listener, options){
 
     companion object {
         const val INDEX_FILENAME = "video_index.bin"
@@ -48,7 +46,7 @@ class VideoIndexer(
 
         if(frameBitmaps == null) throw IllegalStateException("Invalid frames")
 
-        val rawEmbeddings = embedder.embedBatch(context.applicationContext, frameBitmaps)
+        val rawEmbeddings = embedder.embedBatch(frameBitmaps)
         val embedding: FloatArray = generatePrototypeEmbedding(rawEmbeddings)
 
         return Embedding(

core/src/main/java/com/fpf/smartscansdk/core/utils/ImageUtils.kt → core/src/main/java/com/fpf/smartscansdk/core/media/ImageUtils.kt

@@ -1,4 +1,4 @@
-package com.fpf.smartscansdk.core.utils
+package com.fpf.smartscansdk.core.media
 
 import android.content.Context
 import android.graphics.*

core/src/main/java/com/fpf/smartscansdk/core/utils/VideoUtils.kt → core/src/main/java/com/fpf/smartscansdk/core/media/VideoUtils.kt

@@ -1,4 +1,4 @@
-package com.fpf.smartscansdk.core.utils
+package com.fpf.smartscansdk.core.media
 
 import android.content.Context
 import android.graphics.Bitmap