From a8cf3db3427de328de9bb8d116926fd5f16ca721 Mon Sep 17 00:00:00 2001 From: d41dev Date: Wed, 29 Oct 2025 15:17:30 +0000 Subject: [PATCH 01/14] update: change architecture to core and ml modules --- core/build.gradle.kts | 27 ++++---- .../core}/data/images/Converters.kt | 2 +- .../fpf/smartscansdk/core}/data/images/Dao.kt | 2 +- .../core}/data/images/Database.kt | 2 +- .../smartscansdk/core}/data/images/Entity.kt | 2 +- .../EmbeddingLoadingBenchmarkTest.kt | 8 +-- .../EmbeddingTypes.kt => data/Embedings.kt} | 4 +- .../fpf/smartscansdk/core/data/Processors.kt | 30 +++++++++ .../{ml => }/embeddings/EmbeddingUtils.kt | 3 +- .../embeddings/FileEmbeddingRetriever.kt | 14 ++--- .../core}/embeddings/FileEmbeddingStore.kt | 6 +- .../core}/indexers/ImageIndexer.kt | 22 +++---- .../core}/indexers/VideoIndexer.kt | 26 ++++---- .../core/{utils => media}/ImageUtils.kt | 2 +- .../core/{utils => media}/VideoUtils.kt | 2 +- .../core/processors/BatchProcessor.kt | 3 + .../core/processors/IProcessorListener.kt | 12 ---- .../core/processors/MemoryUtils.kt | 1 + .../core/processors/ProcessorData.kt | 19 ------ .../embeddings/FileEmbeddingRetriever.kt | 2 +- .../embeddings/FileEmbeddingStoreTest.kt | 2 +- .../extensions/organisers/Organiser.kt | 62 ------------------- {extensions => ml}/build.gradle.kts | 25 ++++---- {extensions => ml}/proguard-rules.pro | 0 .../embeddings/clip/ClipImageEmbedderTest.kt | 12 ++-- .../embeddings/clip/ClipTextEmbedderTest.kt | 7 ++- .../core/ml/models/OnnxModelTest.kt | 3 + .../com/fpf/smartscansdk/ml/data/Loaders.kt | 11 ++++ .../com/fpf/smartscansdk/ml/data/Models.kt | 2 +- .../fpf/smartscansdk}/ml/models/BaseModel.kt | 4 +- .../fpf/smartscansdk}/ml/models/Loaders.kt | 13 +--- .../fpf/smartscansdk}/ml/models/OnnxModel.kt | 4 +- .../embeddings/FewShotClassifier.kt | 6 +- .../providers}/embeddings/clip/ByteEncoder.kt | 2 +- .../embeddings/clip/ClipImageEmbedder.kt | 35 +++++------ .../embeddings/clip/ClipTextEmbedder.kt | 36 ++++++----- .../providers}/embeddings/clip/Constants.kt | 3 +- .../providers}/embeddings/clip/PreProcess.kt | 4 +- .../providers}/embeddings/clip/Tokenizer.kt | 4 +- {core => ml}/src/main/res/raw/merges.txt | 0 {core => ml}/src/main/res/raw/vocab.json | 0 .../ml/embeddings/FewShotClassifierTest.kt | 4 ++ .../core/processors/BatchProcessorTest.kt | 0 settings.gradle.kts | 2 +- 44 files changed, 188 insertions(+), 242 deletions(-) rename {extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions => core/src/androidTest/kotlin/com/fpf/smartscansdk/core}/data/images/Converters.kt (87%) rename {extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions => core/src/androidTest/kotlin/com/fpf/smartscansdk/core}/data/images/Dao.kt (88%) rename {extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions => core/src/androidTest/kotlin/com/fpf/smartscansdk/core}/data/images/Database.kt (94%) rename {extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions => core/src/androidTest/kotlin/com/fpf/smartscansdk/core}/data/images/Entity.kt (88%) rename {extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions => core/src/androidTest/kotlin/com/fpf/smartscansdk/core}/embeddings/EmbeddingLoadingBenchmarkTest.kt (89%) rename core/src/main/java/com/fpf/smartscansdk/core/{ml/embeddings/EmbeddingTypes.kt => data/Embedings.kt} (91%) create mode 100644 core/src/main/java/com/fpf/smartscansdk/core/data/Processors.kt rename core/src/main/java/com/fpf/smartscansdk/core/{ml => }/embeddings/EmbeddingUtils.kt (94%) rename {extensions/src/main/java/com/fpf/smartscansdk/extensions => core/src/main/java/com/fpf/smartscansdk/core}/embeddings/FileEmbeddingRetriever.kt (72%) rename {extensions/src/main/java/com/fpf/smartscansdk/extensions => core/src/main/java/com/fpf/smartscansdk/core}/embeddings/FileEmbeddingStore.kt (97%) rename {extensions/src/main/java/com/fpf/smartscansdk/extensions => core/src/main/java/com/fpf/smartscansdk/core}/indexers/ImageIndexer.kt (72%) rename {extensions/src/main/java/com/fpf/smartscansdk/extensions => core/src/main/java/com/fpf/smartscansdk/core}/indexers/VideoIndexer.kt (68%) rename core/src/main/java/com/fpf/smartscansdk/core/{utils => media}/ImageUtils.kt (97%) rename core/src/main/java/com/fpf/smartscansdk/core/{utils => media}/VideoUtils.kt (97%) delete mode 100644 core/src/main/java/com/fpf/smartscansdk/core/processors/IProcessorListener.kt delete mode 100644 core/src/main/java/com/fpf/smartscansdk/core/processors/ProcessorData.kt rename {extensions/src/test/kotlin/com/fpf/smartscansdk/extensions => core/src/test/kotlin/com/fpf/smartscansdk/core}/embeddings/FileEmbeddingRetriever.kt (97%) rename {extensions/src/test/kotlin/com/fpf/smartscansdk/extensions => core/src/test/kotlin/com/fpf/smartscansdk/core}/embeddings/FileEmbeddingStoreTest.kt (99%) delete mode 100644 extensions/src/main/java/com/fpf/smartscansdk/extensions/organisers/Organiser.kt rename {extensions => ml}/build.gradle.kts (86%) rename {extensions => ml}/proguard-rules.pro (100%) rename {core => ml}/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipImageEmbedderTest.kt (92%) rename {core => ml}/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipTextEmbedderTest.kt (95%) rename {core => ml}/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/models/OnnxModelTest.kt (96%) create mode 100644 ml/src/main/java/com/fpf/smartscansdk/ml/data/Loaders.kt rename core/src/main/java/com/fpf/smartscansdk/core/ml/models/ModelTypes.kt => ml/src/main/java/com/fpf/smartscansdk/ml/data/Models.kt (95%) rename {core/src/main/java/com/fpf/smartscansdk/core => ml/src/main/java/com/fpf/smartscansdk}/ml/models/BaseModel.kt (76%) rename {core/src/main/java/com/fpf/smartscansdk/core => ml/src/main/java/com/fpf/smartscansdk}/ml/models/Loaders.kt (64%) rename {core/src/main/java/com/fpf/smartscansdk/core => ml/src/main/java/com/fpf/smartscansdk}/ml/models/OnnxModel.kt (94%) rename {core/src/main/java/com/fpf/smartscansdk/core/ml => ml/src/main/java/com/fpf/smartscansdk/ml/models/providers}/embeddings/FewShotClassifier.kt (86%) rename {core/src/main/java/com/fpf/smartscansdk/core/ml => ml/src/main/java/com/fpf/smartscansdk/ml/models/providers}/embeddings/clip/ByteEncoder.kt (98%) rename {core/src/main/java/com/fpf/smartscansdk/core/ml => ml/src/main/java/com/fpf/smartscansdk/ml/models/providers}/embeddings/clip/ClipImageEmbedder.kt (59%) rename {core/src/main/java/com/fpf/smartscansdk/core/ml => ml/src/main/java/com/fpf/smartscansdk/ml/models/providers}/embeddings/clip/ClipTextEmbedder.kt (77%) rename {core/src/main/java/com/fpf/smartscansdk/core/ml => ml/src/main/java/com/fpf/smartscansdk/ml/models/providers}/embeddings/clip/Constants.kt (66%) rename {core/src/main/java/com/fpf/smartscansdk/core/ml => ml/src/main/java/com/fpf/smartscansdk/ml/models/providers}/embeddings/clip/PreProcess.kt (91%) rename {core/src/main/java/com/fpf/smartscansdk/core/ml => ml/src/main/java/com/fpf/smartscansdk/ml/models/providers}/embeddings/clip/Tokenizer.kt (94%) rename {core => ml}/src/main/res/raw/merges.txt (100%) rename {core => ml}/src/main/res/raw/vocab.json (100%) rename {core => ml}/src/test/kotlin/com/fpf/smartscansdk/core/ml/embeddings/FewShotClassifierTest.kt (90%) rename {core => ml}/src/test/kotlin/com/fpf/smartscansdk/core/processors/BatchProcessorTest.kt (100%) diff --git a/core/build.gradle.kts b/core/build.gradle.kts index f38dfe7..dfa49d0 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -1,9 +1,9 @@ -import java.io.ByteArrayOutputStream - plugins { id("com.android.library") id("org.jetbrains.kotlin.android") id("maven-publish") + id("com.google.devtools.ksp") + } android { @@ -13,7 +13,6 @@ android { defaultConfig { minSdk = 30 testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" - } packaging { @@ -21,7 +20,7 @@ android { listOf( "META-INF/LICENSE.md", "META-INF/LICENSE-notice.md", - ) + ) ) } @@ -40,13 +39,10 @@ android { jvmTarget = "17" } - buildFeatures { - // Add any enabled features here if needed - } - lint { targetSdk = 34 } + testOptions { unitTests { isIncludeAndroidResources = true @@ -55,6 +51,7 @@ android { } } } + } java { @@ -65,12 +62,9 @@ java { } dependencies { - implementation(libs.androidx.documentfile) - implementation(libs.onnxruntime.android) - - // Expose core-ktx and onnxruntime to consumers of core or extensions + // Expose core-ktx to consumers of core or extensions api(libs.androidx.core.ktx) - + // JVM unit tests testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.7.3") testImplementation("org.junit.jupiter:junit-jupiter-api:5.10.0") @@ -84,6 +78,12 @@ dependencies { androidTestImplementation("androidx.test:runner:1.6.1") androidTestImplementation("io.mockk:mockk-android:1.14.5") androidTestImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.7.3") + + androidTestImplementation("androidx.room:room-runtime:2.7.2") + androidTestImplementation("androidx.room:room-ktx:2.7.2") + androidTestImplementation("androidx.room:room-testing:2.7.2") + ksp("androidx.room:room-compiler:2.7.2") + } val gitVersion: String by lazy { @@ -96,6 +96,7 @@ val gitVersion: String by lazy { }.getOrDefault("1.0.0") } + publishing { publications { register("release") { diff --git a/extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions/data/images/Converters.kt b/core/src/androidTest/kotlin/com/fpf/smartscansdk/core/data/images/Converters.kt similarity index 87% rename from extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions/data/images/Converters.kt rename to core/src/androidTest/kotlin/com/fpf/smartscansdk/core/data/images/Converters.kt index 9fd8509..13d4a77 100644 --- a/extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions/data/images/Converters.kt +++ b/core/src/androidTest/kotlin/com/fpf/smartscansdk/core/data/images/Converters.kt @@ -1,4 +1,4 @@ -package com.fpf.smartscansdk.extensions.data.images +package com.fpf.smartscansdk.core.data.images import androidx.room.* diff --git a/extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions/data/images/Dao.kt b/core/src/androidTest/kotlin/com/fpf/smartscansdk/core/data/images/Dao.kt similarity index 88% rename from extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions/data/images/Dao.kt rename to core/src/androidTest/kotlin/com/fpf/smartscansdk/core/data/images/Dao.kt index 0e91629..8485634 100644 --- a/extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions/data/images/Dao.kt +++ b/core/src/androidTest/kotlin/com/fpf/smartscansdk/core/data/images/Dao.kt @@ -1,4 +1,4 @@ -package com.fpf.smartscansdk.extensions.data.images +package com.fpf.smartscansdk.core.data.images import androidx.room.* diff --git a/extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions/data/images/Database.kt b/core/src/androidTest/kotlin/com/fpf/smartscansdk/core/data/images/Database.kt similarity index 94% rename from extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions/data/images/Database.kt rename to core/src/androidTest/kotlin/com/fpf/smartscansdk/core/data/images/Database.kt index 149dbe0..a6f3a8b 100644 --- a/extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions/data/images/Database.kt +++ b/core/src/androidTest/kotlin/com/fpf/smartscansdk/core/data/images/Database.kt @@ -1,4 +1,4 @@ -package com.fpf.smartscansdk.extensions.data.images +package com.fpf.smartscansdk.core.data.images import android.app.Application import androidx.room.* diff --git a/extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions/data/images/Entity.kt b/core/src/androidTest/kotlin/com/fpf/smartscansdk/core/data/images/Entity.kt similarity index 88% rename from extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions/data/images/Entity.kt rename to core/src/androidTest/kotlin/com/fpf/smartscansdk/core/data/images/Entity.kt index 087e3f7..76d04f4 100644 --- a/extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions/data/images/Entity.kt +++ b/core/src/androidTest/kotlin/com/fpf/smartscansdk/core/data/images/Entity.kt @@ -1,4 +1,4 @@ -package com.fpf.smartscansdk.extensions.data.images +package com.fpf.smartscansdk.core.data.images import androidx.room.* import com.fpf.smartscansdk.core.ml.embeddings.Embedding diff --git a/extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions/embeddings/EmbeddingLoadingBenchmarkTest.kt b/core/src/androidTest/kotlin/com/fpf/smartscansdk/core/embeddings/EmbeddingLoadingBenchmarkTest.kt similarity index 89% rename from extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions/embeddings/EmbeddingLoadingBenchmarkTest.kt rename to core/src/androidTest/kotlin/com/fpf/smartscansdk/core/embeddings/EmbeddingLoadingBenchmarkTest.kt index 6de4a4c..46b186d 100644 --- a/extensions/src/androidTest/kotlin/com/fpf/smartscansdk/extensions/embeddings/EmbeddingLoadingBenchmarkTest.kt +++ b/core/src/androidTest/kotlin/com/fpf/smartscansdk/core/embeddings/EmbeddingLoadingBenchmarkTest.kt @@ -1,11 +1,11 @@ -package com.fpf.smartscansdk.extensions.embeddings +package com.fpf.smartscansdk.core.embeddings import android.app.Application import androidx.test.core.app.ApplicationProvider import androidx.test.ext.junit.runners.AndroidJUnit4 -import com.fpf.smartscansdk.extensions.data.images.ImageEmbeddingDatabase -import com.fpf.smartscansdk.extensions.data.images.ImageEmbeddingEntity -import com.fpf.smartscansdk.extensions.data.images.toEmbedding +import com.fpf.smartscansdk.core.data.images.ImageEmbeddingDatabase +import com.fpf.smartscansdk.core.data.images.ImageEmbeddingEntity +import com.fpf.smartscansdk.core.data.images.toEmbedding import junit.framework.Assert.assertEquals import kotlinx.coroutines.runBlocking import org.junit.Test diff --git a/core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/EmbeddingTypes.kt b/core/src/main/java/com/fpf/smartscansdk/core/data/Embedings.kt similarity index 91% rename from core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/EmbeddingTypes.kt rename to core/src/main/java/com/fpf/smartscansdk/core/data/Embedings.kt index 487e5b4..93f3883 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/EmbeddingTypes.kt +++ b/core/src/main/java/com/fpf/smartscansdk/core/data/Embedings.kt @@ -1,4 +1,5 @@ -package com.fpf.smartscansdk.core.ml.embeddings +package com.fpf.smartscansdk.core.data + import android.graphics.Bitmap @@ -39,6 +40,7 @@ interface IEmbeddingProvider { val embeddingDim: Int? get() = null fun closeSession() = Unit suspend fun embed(data: T): FloatArray + suspend fun embedBatch(data: List): List } diff --git a/core/src/main/java/com/fpf/smartscansdk/core/data/Processors.kt b/core/src/main/java/com/fpf/smartscansdk/core/data/Processors.kt new file mode 100644 index 0000000..71519dd --- /dev/null +++ b/core/src/main/java/com/fpf/smartscansdk/core/data/Processors.kt @@ -0,0 +1,30 @@ +package com.fpf.smartscansdk.core.data + +import android.content.Context + +interface IProcessorListener { + suspend fun onActive(context: Context) = Unit + suspend fun onBatchComplete(context: Context, batch: List) = Unit + suspend fun onComplete(context: Context, metrics: Metrics.Success) = Unit + suspend fun onProgress(context: Context, progress: Float) = Unit + fun onError(context: Context, error: Exception, item: Input) = Unit + suspend fun onFail(context: Context, failureMetrics: Metrics.Failure) = Unit +} + +sealed class Metrics { + data class Success(val totalProcessed: Int = 0, val timeElapsed: Long = 0L) : Metrics() + data class Failure(val processedBeforeFailure: Int, val timeElapsed: Long, val error: Exception) : Metrics() +} + +data class MemoryOptions( + val lowMemoryThreshold: Long = 800L * 1024 * 1024, + val highMemoryThreshold: Long = 1_600L * 1024 * 1024, + val minConcurrency: Int = 1, + val maxConcurrency: Int = 4 +) + +data class ProcessOptions( + val memory: MemoryOptions = MemoryOptions(), + val batchSize: Int = 10 +) + diff --git a/core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/EmbeddingUtils.kt b/core/src/main/java/com/fpf/smartscansdk/core/embeddings/EmbeddingUtils.kt similarity index 94% rename from core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/EmbeddingUtils.kt rename to core/src/main/java/com/fpf/smartscansdk/core/embeddings/EmbeddingUtils.kt index a943a4c..79814e8 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/EmbeddingUtils.kt +++ b/core/src/main/java/com/fpf/smartscansdk/core/embeddings/EmbeddingUtils.kt @@ -1,6 +1,5 @@ -package com.fpf.smartscansdk.core.ml.embeddings +package com.fpf.smartscansdk.core.embeddings -import android.content.Context import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import kotlin.math.sqrt diff --git a/extensions/src/main/java/com/fpf/smartscansdk/extensions/embeddings/FileEmbeddingRetriever.kt b/core/src/main/java/com/fpf/smartscansdk/core/embeddings/FileEmbeddingRetriever.kt similarity index 72% rename from extensions/src/main/java/com/fpf/smartscansdk/extensions/embeddings/FileEmbeddingRetriever.kt rename to core/src/main/java/com/fpf/smartscansdk/core/embeddings/FileEmbeddingRetriever.kt index 2ce33ab..1db9de4 100644 --- a/extensions/src/main/java/com/fpf/smartscansdk/extensions/embeddings/FileEmbeddingRetriever.kt +++ b/core/src/main/java/com/fpf/smartscansdk/core/embeddings/FileEmbeddingRetriever.kt @@ -1,9 +1,7 @@ -package com.fpf.smartscansdk.extensions.embeddings +package com.fpf.smartscansdk.core.embeddings -import com.fpf.smartscansdk.core.ml.embeddings.Embedding -import com.fpf.smartscansdk.core.ml.embeddings.IRetriever -import com.fpf.smartscansdk.core.ml.embeddings.getSimilarities -import com.fpf.smartscansdk.core.ml.embeddings.getTopN +import com.fpf.smartscansdk.core.data.Embedding +import com.fpf.smartscansdk.core.data.IRetriever class FileEmbeddingRetriever( private val store: FileEmbeddingStore @@ -11,11 +9,7 @@ class FileEmbeddingRetriever( private var cachedIds: List? = null - override suspend fun query( - embedding: FloatArray, - topK: Int, - threshold: Float - ): List { + override suspend fun query(embedding: FloatArray, topK: Int, threshold: Float): List { cachedIds = null // clear on new search diff --git a/extensions/src/main/java/com/fpf/smartscansdk/extensions/embeddings/FileEmbeddingStore.kt b/core/src/main/java/com/fpf/smartscansdk/core/embeddings/FileEmbeddingStore.kt similarity index 97% rename from extensions/src/main/java/com/fpf/smartscansdk/extensions/embeddings/FileEmbeddingStore.kt rename to core/src/main/java/com/fpf/smartscansdk/core/embeddings/FileEmbeddingStore.kt index 11f79c8..2114a0c 100644 --- a/extensions/src/main/java/com/fpf/smartscansdk/extensions/embeddings/FileEmbeddingStore.kt +++ b/core/src/main/java/com/fpf/smartscansdk/core/embeddings/FileEmbeddingStore.kt @@ -1,8 +1,8 @@ -package com.fpf.smartscansdk.extensions.embeddings +package com.fpf.smartscansdk.core.embeddings import android.util.Log -import com.fpf.smartscansdk.core.ml.embeddings.Embedding -import com.fpf.smartscansdk.core.ml.embeddings.IEmbeddingStore +import com.fpf.smartscansdk.core.data.Embedding +import com.fpf.smartscansdk.core.data.IEmbeddingStore import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import java.io.File diff --git a/extensions/src/main/java/com/fpf/smartscansdk/extensions/indexers/ImageIndexer.kt b/core/src/main/java/com/fpf/smartscansdk/core/indexers/ImageIndexer.kt similarity index 72% rename from extensions/src/main/java/com/fpf/smartscansdk/extensions/indexers/ImageIndexer.kt rename to core/src/main/java/com/fpf/smartscansdk/core/indexers/ImageIndexer.kt index 9aaf3e9..7b3dc78 100644 --- a/extensions/src/main/java/com/fpf/smartscansdk/extensions/indexers/ImageIndexer.kt +++ b/core/src/main/java/com/fpf/smartscansdk/core/indexers/ImageIndexer.kt @@ -1,17 +1,16 @@ -package com.fpf.smartscansdk.extensions.indexers +package com.fpf.smartscansdk.core.indexers import android.app.Application import android.content.ContentUris import android.content.Context import android.provider.MediaStore -import com.fpf.smartscansdk.core.utils.getBitmapFromUri -import com.fpf.smartscansdk.core.ml.embeddings.Embedding -import com.fpf.smartscansdk.core.ml.embeddings.IEmbeddingStore -import com.fpf.smartscansdk.core.ml.embeddings.clip.ClipConfig -import com.fpf.smartscansdk.core.ml.embeddings.clip.ClipImageEmbedder +import com.fpf.smartscansdk.core.data.Embedding +import com.fpf.smartscansdk.core.data.IEmbeddingStore +import com.fpf.smartscansdk.core.data.IProcessorListener +import com.fpf.smartscansdk.core.data.ImageEmbeddingProvider +import com.fpf.smartscansdk.core.data.ProcessOptions +import com.fpf.smartscansdk.core.media.getBitmapFromUri import com.fpf.smartscansdk.core.processors.BatchProcessor -import com.fpf.smartscansdk.core.processors.IProcessorListener -import com.fpf.smartscansdk.core.processors.ProcessOptions import kotlinx.coroutines.NonCancellable import kotlinx.coroutines.withContext @@ -21,11 +20,12 @@ import kotlinx.coroutines.withContext // These benchmarks strongly favour the FileEmbeddingStore for optimal on-device search functionality and UX. class ImageIndexer( - private val embedder: ClipImageEmbedder, + private val embedder: ImageEmbeddingProvider, + private val store: IEmbeddingStore, + private val bitmapMaxSize: Int = 225, application: Application, listener: IProcessorListener? = null, options: ProcessOptions = ProcessOptions(), - private val store: IEmbeddingStore, ): BatchProcessor(application, listener, options){ companion object { @@ -40,7 +40,7 @@ class ImageIndexer( val contentUri = ContentUris.withAppendedId( MediaStore.Images.Media.EXTERNAL_CONTENT_URI, item ) - val bitmap = getBitmapFromUri(context, contentUri, ClipConfig.IMAGE_SIZE_X) + val bitmap = getBitmapFromUri(context, contentUri, bitmapMaxSize) val embedding = withContext(NonCancellable) { embedder.embed(bitmap) } diff --git a/extensions/src/main/java/com/fpf/smartscansdk/extensions/indexers/VideoIndexer.kt b/core/src/main/java/com/fpf/smartscansdk/core/indexers/VideoIndexer.kt similarity index 68% rename from extensions/src/main/java/com/fpf/smartscansdk/extensions/indexers/VideoIndexer.kt rename to core/src/main/java/com/fpf/smartscansdk/core/indexers/VideoIndexer.kt index 4f92a1f..ced4abc 100644 --- a/extensions/src/main/java/com/fpf/smartscansdk/extensions/indexers/VideoIndexer.kt +++ b/core/src/main/java/com/fpf/smartscansdk/core/indexers/VideoIndexer.kt @@ -1,19 +1,17 @@ -package com.fpf.smartscansdk.extensions.indexers +package com.fpf.smartscansdk.core.indexers import android.app.Application import android.content.ContentUris import android.content.Context import android.provider.MediaStore -import com.fpf.smartscansdk.core.ml.embeddings.Embedding -import com.fpf.smartscansdk.core.ml.embeddings.IEmbeddingStore -import com.fpf.smartscansdk.core.ml.embeddings.clip.ClipConfig.IMAGE_SIZE_X -import com.fpf.smartscansdk.core.ml.embeddings.clip.ClipConfig.IMAGE_SIZE_Y -import com.fpf.smartscansdk.core.ml.embeddings.clip.ClipImageEmbedder -import com.fpf.smartscansdk.core.ml.embeddings.generatePrototypeEmbedding +import com.fpf.smartscansdk.core.data.Embedding +import com.fpf.smartscansdk.core.data.IEmbeddingStore +import com.fpf.smartscansdk.core.data.IProcessorListener +import com.fpf.smartscansdk.core.data.ImageEmbeddingProvider +import com.fpf.smartscansdk.core.data.ProcessOptions +import com.fpf.smartscansdk.core.embeddings.generatePrototypeEmbedding import com.fpf.smartscansdk.core.processors.BatchProcessor -import com.fpf.smartscansdk.core.processors.IProcessorListener -import com.fpf.smartscansdk.core.processors.ProcessOptions -import com.fpf.smartscansdk.core.utils.extractFramesFromVideo +import com.fpf.smartscansdk.core.media.extractFramesFromVideo // ** Design Constraint**: For on-device vector search, the full index needs to be loaded in-memory (or make an Android native VectorDB) // File-based EmbeddingStore is used over a Room version due to significant faster index loading @@ -22,10 +20,10 @@ import com.fpf.smartscansdk.core.utils.extractFramesFromVideo // **IMPORTANT**: Video frame extraction can fail due to codec incompatibility class VideoIndexer( - private val embedder: ClipImageEmbedder, + private val embedder: ImageEmbeddingProvider, private val frameCount: Int = 10, - private val width: Int = IMAGE_SIZE_X, - private val height: Int = IMAGE_SIZE_Y, + private val width: Int, + private val height: Int, application: Application, listener: IProcessorListener? = null, options: ProcessOptions = ProcessOptions(), @@ -48,7 +46,7 @@ class VideoIndexer( if(frameBitmaps == null) throw IllegalStateException("Invalid frames") - val rawEmbeddings = embedder.embedBatch(context.applicationContext, frameBitmaps) + val rawEmbeddings = embedder.embedBatch(frameBitmaps) val embedding: FloatArray = generatePrototypeEmbedding(rawEmbeddings) return Embedding( diff --git a/core/src/main/java/com/fpf/smartscansdk/core/utils/ImageUtils.kt b/core/src/main/java/com/fpf/smartscansdk/core/media/ImageUtils.kt similarity index 97% rename from core/src/main/java/com/fpf/smartscansdk/core/utils/ImageUtils.kt rename to core/src/main/java/com/fpf/smartscansdk/core/media/ImageUtils.kt index 3def30c..d49a2e8 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/utils/ImageUtils.kt +++ b/core/src/main/java/com/fpf/smartscansdk/core/media/ImageUtils.kt @@ -1,4 +1,4 @@ -package com.fpf.smartscansdk.core.utils +package com.fpf.smartscansdk.core.media import android.content.Context import android.graphics.* diff --git a/core/src/main/java/com/fpf/smartscansdk/core/utils/VideoUtils.kt b/core/src/main/java/com/fpf/smartscansdk/core/media/VideoUtils.kt similarity index 97% rename from core/src/main/java/com/fpf/smartscansdk/core/utils/VideoUtils.kt rename to core/src/main/java/com/fpf/smartscansdk/core/media/VideoUtils.kt index 800aedd..be859b5 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/utils/VideoUtils.kt +++ b/core/src/main/java/com/fpf/smartscansdk/core/media/VideoUtils.kt @@ -1,4 +1,4 @@ -package com.fpf.smartscansdk.core.utils +package com.fpf.smartscansdk.core.media import android.content.Context import android.graphics.Bitmap diff --git a/core/src/main/java/com/fpf/smartscansdk/core/processors/BatchProcessor.kt b/core/src/main/java/com/fpf/smartscansdk/core/processors/BatchProcessor.kt index 10a282a..3ff561e 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/processors/BatchProcessor.kt +++ b/core/src/main/java/com/fpf/smartscansdk/core/processors/BatchProcessor.kt @@ -3,6 +3,9 @@ package com.fpf.smartscansdk.core.processors import android.app.Application import android.content.Context import android.util.Log +import com.fpf.smartscansdk.core.data.IProcessorListener +import com.fpf.smartscansdk.core.data.Metrics +import com.fpf.smartscansdk.core.data.ProcessOptions import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async diff --git a/core/src/main/java/com/fpf/smartscansdk/core/processors/IProcessorListener.kt b/core/src/main/java/com/fpf/smartscansdk/core/processors/IProcessorListener.kt deleted file mode 100644 index 340c583..0000000 --- a/core/src/main/java/com/fpf/smartscansdk/core/processors/IProcessorListener.kt +++ /dev/null @@ -1,12 +0,0 @@ -package com.fpf.smartscansdk.core.processors - -import android.content.Context - -interface IProcessorListener { - suspend fun onActive(context: Context) = Unit - suspend fun onBatchComplete(context: Context, batch: List) = Unit - suspend fun onComplete(context: Context, metrics: Metrics.Success) = Unit - suspend fun onProgress(context: Context, progress: Float) = Unit - fun onError(context: Context, error: Exception, item: Input) = Unit - suspend fun onFail(context: Context, failureMetrics: Metrics.Failure) = Unit -} diff --git a/core/src/main/java/com/fpf/smartscansdk/core/processors/MemoryUtils.kt b/core/src/main/java/com/fpf/smartscansdk/core/processors/MemoryUtils.kt index c8f4c9d..809b10b 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/processors/MemoryUtils.kt +++ b/core/src/main/java/com/fpf/smartscansdk/core/processors/MemoryUtils.kt @@ -2,6 +2,7 @@ package com.fpf.smartscansdk.core.processors import android.app.ActivityManager import android.content.Context +import com.fpf.smartscansdk.core.data.MemoryOptions class MemoryUtils( private val context: Context, diff --git a/core/src/main/java/com/fpf/smartscansdk/core/processors/ProcessorData.kt b/core/src/main/java/com/fpf/smartscansdk/core/processors/ProcessorData.kt deleted file mode 100644 index 6c15f06..0000000 --- a/core/src/main/java/com/fpf/smartscansdk/core/processors/ProcessorData.kt +++ /dev/null @@ -1,19 +0,0 @@ -package com.fpf.smartscansdk.core.processors - -sealed class Metrics { - data class Success(val totalProcessed: Int = 0, val timeElapsed: Long = 0L) : Metrics() - data class Failure(val processedBeforeFailure: Int, val timeElapsed: Long, val error: Exception) : Metrics() -} - -data class MemoryOptions( - val lowMemoryThreshold: Long = 800L * 1024 * 1024, - val highMemoryThreshold: Long = 1_600L * 1024 * 1024, - val minConcurrency: Int = 1, - val maxConcurrency: Int = 4 -) - -data class ProcessOptions( - val memory: MemoryOptions = MemoryOptions(), - val batchSize: Int = 10 -) - diff --git a/extensions/src/test/kotlin/com/fpf/smartscansdk/extensions/embeddings/FileEmbeddingRetriever.kt b/core/src/test/kotlin/com/fpf/smartscansdk/core/embeddings/FileEmbeddingRetriever.kt similarity index 97% rename from extensions/src/test/kotlin/com/fpf/smartscansdk/extensions/embeddings/FileEmbeddingRetriever.kt rename to core/src/test/kotlin/com/fpf/smartscansdk/core/embeddings/FileEmbeddingRetriever.kt index 027d9eb..a5919e7 100644 --- a/extensions/src/test/kotlin/com/fpf/smartscansdk/extensions/embeddings/FileEmbeddingRetriever.kt +++ b/core/src/test/kotlin/com/fpf/smartscansdk/core/embeddings/FileEmbeddingRetriever.kt @@ -1,4 +1,4 @@ -package com.fpf.smartscansdk.extensions.embeddings +package com.fpf.smartscansdk.core.embeddings import com.fpf.smartscansdk.core.ml.embeddings.Embedding import kotlinx.coroutines.test.runTest diff --git a/extensions/src/test/kotlin/com/fpf/smartscansdk/extensions/embeddings/FileEmbeddingStoreTest.kt b/core/src/test/kotlin/com/fpf/smartscansdk/core/embeddings/FileEmbeddingStoreTest.kt similarity index 99% rename from extensions/src/test/kotlin/com/fpf/smartscansdk/extensions/embeddings/FileEmbeddingStoreTest.kt rename to core/src/test/kotlin/com/fpf/smartscansdk/core/embeddings/FileEmbeddingStoreTest.kt index de930df..a9056ee 100644 --- a/extensions/src/test/kotlin/com/fpf/smartscansdk/extensions/embeddings/FileEmbeddingStoreTest.kt +++ b/core/src/test/kotlin/com/fpf/smartscansdk/core/embeddings/FileEmbeddingStoreTest.kt @@ -1,4 +1,4 @@ -package com.fpf.smartscansdk.extensions.embeddings +package com.fpf.smartscansdk.core.embeddings import android.util.Log import com.fpf.smartscansdk.core.ml.embeddings.Embedding diff --git a/extensions/src/main/java/com/fpf/smartscansdk/extensions/organisers/Organiser.kt b/extensions/src/main/java/com/fpf/smartscansdk/extensions/organisers/Organiser.kt deleted file mode 100644 index 11ddd18..0000000 --- a/extensions/src/main/java/com/fpf/smartscansdk/extensions/organisers/Organiser.kt +++ /dev/null @@ -1,62 +0,0 @@ -package com.fpf.smartscansdk.extensions.organisers - -import android.app.Application -import android.content.Context -import android.net.Uri -import androidx.core.net.toUri -import com.fpf.smartscansdk.core.ml.embeddings.ClassificationResult -import com.fpf.smartscansdk.core.ml.embeddings.PrototypeEmbedding -import com.fpf.smartscansdk.core.ml.embeddings.classify -import com.fpf.smartscansdk.core.ml.embeddings.clip.ClipImageEmbedder -import com.fpf.smartscansdk.core.ml.embeddings.clip.ClipConfig -import com.fpf.smartscansdk.core.processors.BatchProcessor -import com.fpf.smartscansdk.core.processors.IProcessorListener -import com.fpf.smartscansdk.core.processors.ProcessOptions -import com.fpf.smartscansdk.core.utils.getBitmapFromUri - -class Organiser( - private val application: Application, - private val embedder: ClipImageEmbedder, - private val prototypeList: List, - private val scanId: Int, - private val threshold: Float = 0.4f, - private val confidenceMargin: Float = 0.04f, - listener: IProcessorListener, - options: ProcessOptions = ProcessOptions(), - ): BatchProcessor(application, listener, options) { - - companion object { - private const val TAG = "Organiser" - const val PREF_KEY_LAST_USED_CLASSIFICATION_DIRS = "last_used_destinations" - } - - fun close() { - embedder.closeSession() - } - - // Delegate to listener (client app) to give control of how classified files are managed - override suspend fun onBatchComplete(context: Context, batch: List) { - listener?.onBatchComplete(context, batch) - } - - override suspend fun onProcess(context: Context, item: Uri): OrganiserResult { - val bitmap = getBitmapFromUri(application, item, ClipConfig.IMAGE_SIZE_X) - val embedding = embedder.embed(bitmap) - val result = classify(embedding, prototypeList, threshold=threshold, confidenceMargin=confidenceMargin) - return when(result){ - is ClassificationResult.Success -> { - OrganiserResult( source = item, destination = result.classId.toUri(), scanId=scanId) - } - is ClassificationResult.Failure -> { - OrganiserResult( source = item, destination = null, scanId=scanId) - } - - } - } -} - -data class OrganiserResult( - val destination: Uri?, - val source: Uri, - val scanId: Int, - ) diff --git a/extensions/build.gradle.kts b/ml/build.gradle.kts similarity index 86% rename from extensions/build.gradle.kts rename to ml/build.gradle.kts index f053d40..570d0d2 100644 --- a/extensions/build.gradle.kts +++ b/ml/build.gradle.kts @@ -1,18 +1,19 @@ +import java.io.ByteArrayOutputStream + plugins { id("com.android.library") id("org.jetbrains.kotlin.android") id("maven-publish") - id("com.google.devtools.ksp") - } android { - namespace = "com.fpf.smartscansdk.extensions" + namespace = "com.fpf.smartscansdk.core" compileSdk = 36 defaultConfig { minSdk = 30 testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + } packaging { @@ -20,7 +21,7 @@ android { listOf( "META-INF/LICENSE.md", "META-INF/LICENSE-notice.md", - ) + ) ) } @@ -39,10 +40,13 @@ android { jvmTarget = "17" } + buildFeatures { + // Add any enabled features here if needed + } + lint { targetSdk = 34 } - testOptions { unitTests { isIncludeAndroidResources = true @@ -51,7 +55,6 @@ android { } } } - } java { @@ -62,8 +65,9 @@ java { } dependencies { - // Pull in core transitively so consumers only need extensions api(project(":core")) + implementation(libs.androidx.documentfile) + implementation(libs.onnxruntime.android) // JVM unit tests testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.7.3") @@ -78,12 +82,6 @@ dependencies { androidTestImplementation("androidx.test:runner:1.6.1") androidTestImplementation("io.mockk:mockk-android:1.14.5") androidTestImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.7.3") - - androidTestImplementation("androidx.room:room-runtime:2.7.2") - androidTestImplementation("androidx.room:room-ktx:2.7.2") - androidTestImplementation("androidx.room:room-testing:2.7.2") - ksp("androidx.room:room-compiler:2.7.2") - } val gitVersion: String by lazy { @@ -96,7 +94,6 @@ val gitVersion: String by lazy { }.getOrDefault("1.0.0") } - publishing { publications { register("release") { diff --git a/extensions/proguard-rules.pro b/ml/proguard-rules.pro similarity index 100% rename from extensions/proguard-rules.pro rename to ml/proguard-rules.pro diff --git a/core/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipImageEmbedderTest.kt b/ml/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipImageEmbedderTest.kt similarity index 92% rename from core/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipImageEmbedderTest.kt rename to ml/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipImageEmbedderTest.kt index 92266c6..6cea0a7 100644 --- a/core/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipImageEmbedderTest.kt +++ b/ml/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipImageEmbedderTest.kt @@ -1,17 +1,17 @@ package com.fpf.smartscansdk.core.ml.embeddings.clip import ai.onnxruntime.OnnxTensor -import ai.onnxruntime.OnnxTensorLike import ai.onnxruntime.OrtEnvironment import android.content.Context import android.content.res.Resources import android.graphics.Bitmap import androidx.test.core.app.ApplicationProvider -import com.fpf.smartscansdk.core.ml.embeddings.clip.ClipConfig.IMAGE_SIZE_X -import com.fpf.smartscansdk.core.ml.embeddings.clip.ClipConfig.IMAGE_SIZE_Y -import com.fpf.smartscansdk.core.ml.models.OnnxModel -import com.fpf.smartscansdk.core.ml.models.ResourceId -import com.fpf.smartscansdk.core.ml.models.TensorData +import com.fpf.smartscansdk.ml.models.providers.embeddings.clip.ClipConfig.IMAGE_SIZE_X +import com.fpf.smartscansdk.ml.models.providers.embeddings.clip.ClipConfig.IMAGE_SIZE_Y +import com.fpf.smartscansdk.ml.models.OnnxModel +import com.fpf.smartscansdk.ml.models.ResourceId +import com.fpf.smartscansdk.ml.models.TensorData +import com.fpf.smartscansdk.ml.models.providers.embeddings.clip.ClipImageEmbedder import io.mockk.* import kotlinx.coroutines.runBlocking import org.junit.After diff --git a/core/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipTextEmbedderTest.kt b/ml/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipTextEmbedderTest.kt similarity index 95% rename from core/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipTextEmbedderTest.kt rename to ml/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipTextEmbedderTest.kt index 9181672..1a82f20 100644 --- a/core/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipTextEmbedderTest.kt +++ b/ml/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipTextEmbedderTest.kt @@ -5,9 +5,10 @@ import ai.onnxruntime.OrtEnvironment import android.content.Context import android.content.res.Resources import androidx.test.core.app.ApplicationProvider -import com.fpf.smartscansdk.core.ml.models.OnnxModel -import com.fpf.smartscansdk.core.ml.models.ResourceId -import com.fpf.smartscansdk.core.ml.models.TensorData +import com.fpf.smartscansdk.ml.models.providers.embeddings.clip.ClipTextEmbedder +import com.fpf.smartscansdk.ml.models.OnnxModel +import com.fpf.smartscansdk.ml.models.ResourceId +import com.fpf.smartscansdk.ml.models.TensorData import io.mockk.* import kotlinx.coroutines.runBlocking import org.junit.After diff --git a/core/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/models/OnnxModelTest.kt b/ml/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/models/OnnxModelTest.kt similarity index 96% rename from core/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/models/OnnxModelTest.kt rename to ml/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/models/OnnxModelTest.kt index 3a1ff46..53c91c7 100644 --- a/core/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/models/OnnxModelTest.kt +++ b/ml/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/models/OnnxModelTest.kt @@ -4,6 +4,9 @@ package com.fpf.smartscansdk.core.ml.models import android.content.Context import androidx.test.core.app.ApplicationProvider import ai.onnxruntime.* +import com.fpf.smartscansdk.ml.models.IModelLoader +import com.fpf.smartscansdk.ml.models.OnnxModel +import com.fpf.smartscansdk.ml.models.TensorData import io.mockk.Runs import io.mockk.coEvery import io.mockk.every diff --git a/ml/src/main/java/com/fpf/smartscansdk/ml/data/Loaders.kt b/ml/src/main/java/com/fpf/smartscansdk/ml/data/Loaders.kt new file mode 100644 index 0000000..c82cdee --- /dev/null +++ b/ml/src/main/java/com/fpf/smartscansdk/ml/data/Loaders.kt @@ -0,0 +1,11 @@ +package com.fpf.smartscansdk.ml.data + +import androidx.annotation.RawRes + +interface IModelLoader { + suspend fun load(): T +} + +sealed interface ModelSource +data class FilePath(val path: String) : ModelSource +data class ResourceId(@RawRes val resId: Int) : ModelSource diff --git a/core/src/main/java/com/fpf/smartscansdk/core/ml/models/ModelTypes.kt b/ml/src/main/java/com/fpf/smartscansdk/ml/data/Models.kt similarity index 95% rename from core/src/main/java/com/fpf/smartscansdk/core/ml/models/ModelTypes.kt rename to ml/src/main/java/com/fpf/smartscansdk/ml/data/Models.kt index 612dd48..a7ce848 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/ml/models/ModelTypes.kt +++ b/ml/src/main/java/com/fpf/smartscansdk/ml/data/Models.kt @@ -1,4 +1,4 @@ -package com.fpf.smartscansdk.core.ml.models +package com.fpf.smartscansdk.ml.data import ai.onnxruntime.OnnxJavaType import java.nio.ByteBuffer diff --git a/core/src/main/java/com/fpf/smartscansdk/core/ml/models/BaseModel.kt b/ml/src/main/java/com/fpf/smartscansdk/ml/models/BaseModel.kt similarity index 76% rename from core/src/main/java/com/fpf/smartscansdk/core/ml/models/BaseModel.kt rename to ml/src/main/java/com/fpf/smartscansdk/ml/models/BaseModel.kt index 66a4001..b13bc98 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/ml/models/BaseModel.kt +++ b/ml/src/main/java/com/fpf/smartscansdk/ml/models/BaseModel.kt @@ -1,4 +1,6 @@ -package com.fpf.smartscansdk.core.ml.models +package com.fpf.smartscansdk.ml.models + +import com.fpf.smartscansdk.ml.data.IModelLoader abstract class BaseModel : AutoCloseable { protected abstract val loader: IModelLoader<*> // hidden implementation detail diff --git a/core/src/main/java/com/fpf/smartscansdk/core/ml/models/Loaders.kt b/ml/src/main/java/com/fpf/smartscansdk/ml/models/Loaders.kt similarity index 64% rename from core/src/main/java/com/fpf/smartscansdk/core/ml/models/Loaders.kt rename to ml/src/main/java/com/fpf/smartscansdk/ml/models/Loaders.kt index 1e4c69c..df24be7 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/ml/models/Loaders.kt +++ b/ml/src/main/java/com/fpf/smartscansdk/ml/models/Loaders.kt @@ -1,19 +1,10 @@ -package com.fpf.smartscansdk.core.ml.models +package com.fpf.smartscansdk.ml.models import android.content.res.Resources import androidx.annotation.RawRes +import com.fpf.smartscansdk.ml.data.IModelLoader import java.io.File - -interface IModelLoader { - suspend fun load(): T -} - -sealed interface ModelSource -data class FilePath(val path: String) : ModelSource -data class ResourceId(@RawRes val resId: Int) : ModelSource - - class FileOnnxLoader(private val path: String) : IModelLoader { override suspend fun load(): ByteArray = File(path).readBytes() } diff --git a/core/src/main/java/com/fpf/smartscansdk/core/ml/models/OnnxModel.kt b/ml/src/main/java/com/fpf/smartscansdk/ml/models/OnnxModel.kt similarity index 94% rename from core/src/main/java/com/fpf/smartscansdk/core/ml/models/OnnxModel.kt rename to ml/src/main/java/com/fpf/smartscansdk/ml/models/OnnxModel.kt index a3169ec..95791a9 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/ml/models/OnnxModel.kt +++ b/ml/src/main/java/com/fpf/smartscansdk/ml/models/OnnxModel.kt @@ -1,8 +1,10 @@ -package com.fpf.smartscansdk.core.ml.models +package com.fpf.smartscansdk.ml.models import ai.onnxruntime.OnnxTensor import ai.onnxruntime.OrtEnvironment import ai.onnxruntime.OrtSession +import com.fpf.smartscansdk.ml.data.IModelLoader +import com.fpf.smartscansdk.ml.data.TensorData import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.withContext diff --git a/core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/FewShotClassifier.kt b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/FewShotClassifier.kt similarity index 86% rename from core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/FewShotClassifier.kt rename to ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/FewShotClassifier.kt index 54fedf0..738977a 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/FewShotClassifier.kt +++ b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/FewShotClassifier.kt @@ -1,4 +1,8 @@ -package com.fpf.smartscansdk.core.ml.embeddings +package com.fpf.smartscansdk.ml.models.providers.embeddings + +import com.fpf.smartscansdk.core.data.PrototypeEmbedding +import com.fpf.smartscansdk.core.embeddings.getSimilarities +import com.fpf.smartscansdk.core.embeddings.getTopN sealed class ClassificationResult { data class Success(val classId: String, val similarity: Float ): ClassificationResult() diff --git a/core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/clip/ByteEncoder.kt b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ByteEncoder.kt similarity index 98% rename from core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/clip/ByteEncoder.kt rename to ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ByteEncoder.kt index f90a14b..af72d4a 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/clip/ByteEncoder.kt +++ b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ByteEncoder.kt @@ -1,4 +1,4 @@ -package com.fpf.smartscansdk.core.ml.models.embeddings.clip +package com.fpf.smartscansdk.ml.models.providers.embeddings.clip internal val byteEncoder: Map by lazy { hashMapOf().apply { diff --git a/core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipImageEmbedder.kt b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipImageEmbedder.kt similarity index 59% rename from core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipImageEmbedder.kt rename to ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipImageEmbedder.kt index bddf414..d5c24f4 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipImageEmbedder.kt +++ b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipImageEmbedder.kt @@ -1,34 +1,29 @@ -package com.fpf.smartscansdk.core.ml.embeddings.clip +package com.fpf.smartscansdk.ml.models.providers.embeddings.clip import android.app.Application import android.content.Context -import android.content.res.Resources import android.graphics.Bitmap -import com.fpf.smartscansdk.core.ml.embeddings.ImageEmbeddingProvider -import com.fpf.smartscansdk.core.ml.models.OnnxModel -import com.fpf.smartscansdk.core.ml.embeddings.clip.ClipConfig.DIM_BATCH_SIZE -import com.fpf.smartscansdk.core.ml.embeddings.clip.ClipConfig.DIM_PIXEL_SIZE -import com.fpf.smartscansdk.core.ml.embeddings.clip.ClipConfig.IMAGE_SIZE_X -import com.fpf.smartscansdk.core.ml.embeddings.clip.ClipConfig.IMAGE_SIZE_Y -import com.fpf.smartscansdk.core.ml.embeddings.normalizeL2 -import com.fpf.smartscansdk.core.ml.models.FileOnnxLoader -import com.fpf.smartscansdk.core.ml.models.FilePath -import com.fpf.smartscansdk.core.ml.models.ModelSource -import com.fpf.smartscansdk.core.ml.models.ResourceId -import com.fpf.smartscansdk.core.ml.models.ResourceOnnxLoader -import com.fpf.smartscansdk.core.ml.models.TensorData +import com.fpf.smartscansdk.core.data.ImageEmbeddingProvider +import com.fpf.smartscansdk.core.embeddings.normalizeL2 import com.fpf.smartscansdk.core.processors.BatchProcessor +import com.fpf.smartscansdk.ml.data.FilePath +import com.fpf.smartscansdk.ml.data.ModelSource +import com.fpf.smartscansdk.ml.data.ResourceId +import com.fpf.smartscansdk.ml.data.TensorData +import com.fpf.smartscansdk.ml.models.OnnxModel +import com.fpf.smartscansdk.ml.models.FileOnnxLoader +import com.fpf.smartscansdk.ml.models.ResourceOnnxLoader import kotlinx.coroutines.* import java.nio.FloatBuffer // Using ModelSource enables using with bundle model or local model which has been downloaded class ClipImageEmbedder( - resources: Resources, + private val context: Context, modelSource: ModelSource, ) : ImageEmbeddingProvider { private val model: OnnxModel = when(modelSource){ is FilePath -> OnnxModel(FileOnnxLoader(modelSource.path)) - is ResourceId -> OnnxModel(ResourceOnnxLoader(resources, modelSource.resId)) + is ResourceId -> OnnxModel(ResourceOnnxLoader(context.resources, modelSource.resId)) } override val embeddingDim: Int = 512 @@ -41,14 +36,14 @@ class ClipImageEmbedder( override suspend fun embed(bitmap: Bitmap): FloatArray = withContext(Dispatchers.Default) { if (!isInitialized()) throw IllegalStateException("Model not initialized") - val inputShape = longArrayOf(DIM_BATCH_SIZE.toLong(), DIM_PIXEL_SIZE.toLong(), IMAGE_SIZE_X.toLong(), IMAGE_SIZE_Y.toLong()) + val inputShape = longArrayOf(ClipConfig.DIM_BATCH_SIZE.toLong(), ClipConfig.DIM_PIXEL_SIZE.toLong(), ClipConfig.IMAGE_SIZE_X.toLong(), ClipConfig.IMAGE_SIZE_Y.toLong()) val imgData: FloatBuffer = preProcess(bitmap) val inputName = model.getInputNames()?.firstOrNull() ?: throw IllegalStateException("Model inputs not available") val output = model.run(mapOf(inputName to TensorData.FloatBufferTensor(imgData, inputShape))) normalizeL2((output.values.first() as Array)[0]) } - suspend fun embedBatch(context: Context, bitmaps: List): List { + override suspend fun embedBatch(data: List): List { val allEmbeddings = mutableListOf() val processor = object : BatchProcessor(application = context.applicationContext as Application) { @@ -60,7 +55,7 @@ class ClipImageEmbedder( } } - processor.run(bitmaps) + processor.run(data) return allEmbeddings } diff --git a/core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipTextEmbedder.kt b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipTextEmbedder.kt similarity index 77% rename from core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipTextEmbedder.kt rename to ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipTextEmbedder.kt index b146a86..ee293c3 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipTextEmbedder.kt +++ b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipTextEmbedder.kt @@ -1,20 +1,20 @@ -package com.fpf.smartscansdk.core.ml.embeddings.clip +package com.fpf.smartscansdk.ml.models.providers.embeddings.clip import android.app.Application import android.content.Context import android.content.res.Resources import android.util.JsonReader import com.fpf.smartscansdk.core.R -import com.fpf.smartscansdk.core.ml.embeddings.TextEmbeddingProvider -import com.fpf.smartscansdk.core.ml.models.OnnxModel -import com.fpf.smartscansdk.core.ml.embeddings.normalizeL2 -import com.fpf.smartscansdk.core.ml.models.FileOnnxLoader -import com.fpf.smartscansdk.core.ml.models.FilePath -import com.fpf.smartscansdk.core.ml.models.ModelSource -import com.fpf.smartscansdk.core.ml.models.ResourceId -import com.fpf.smartscansdk.core.ml.models.ResourceOnnxLoader -import com.fpf.smartscansdk.core.ml.models.TensorData +import com.fpf.smartscansdk.core.data.TextEmbeddingProvider +import com.fpf.smartscansdk.core.embeddings.normalizeL2 +import com.fpf.smartscansdk.ml.models.OnnxModel +import com.fpf.smartscansdk.ml.models.FileOnnxLoader +import com.fpf.smartscansdk.ml.models.ResourceOnnxLoader import com.fpf.smartscansdk.core.processors.BatchProcessor +import com.fpf.smartscansdk.ml.data.FilePath +import com.fpf.smartscansdk.ml.data.ModelSource +import com.fpf.smartscansdk.ml.data.ResourceId +import com.fpf.smartscansdk.ml.data.TensorData import kotlinx.coroutines.* import java.io.BufferedReader import java.io.InputStreamReader @@ -24,17 +24,17 @@ import java.util.* // Using ModelSource enables using with bundle model or local model which has been downloaded class ClipTextEmbedder( - resources: Resources, + context: Context, modelSource: ModelSource ) : TextEmbeddingProvider { private val model: OnnxModel = when(modelSource){ is FilePath -> OnnxModel(FileOnnxLoader(modelSource.path)) - is ResourceId -> OnnxModel(ResourceOnnxLoader(resources, modelSource.resId)) + is ResourceId -> OnnxModel(ResourceOnnxLoader(context.resources, modelSource.resId)) } - private val tokenizerVocab: Map = getVocab(resources) - private val tokenizerMerges: HashMap, Int> = getMerges(resources) + private val tokenizerVocab: Map = getVocab(context.resources) + private val tokenizerMerges: HashMap, Int> = getMerges(context.resources) private val tokenizer = ClipTokenizer(tokenizerVocab, tokenizerMerges) private val tokenBOS = 49406 private val tokenEOS = 49407 @@ -46,10 +46,10 @@ class ClipTextEmbedder( fun isInitialized() = model.isLoaded() - override suspend fun embed(text: String): FloatArray = withContext(Dispatchers.Default) { + override suspend fun embed(data: String): FloatArray = withContext(Dispatchers.Default) { if (!isInitialized()) throw IllegalStateException("Model not initialized") - val clean = Regex("[^A-Za-z0-9 ]").replace(text, "").lowercase() + val clean = Regex("[^A-Za-z0-9 ]").replace(data, "").lowercase() var tokens = mutableListOf(tokenBOS) + tokenizer.encode(clean) + tokenEOS tokens = tokens.take(77) + List(77 - tokens.size) { 0 } @@ -63,6 +63,10 @@ class ClipTextEmbedder( normalizeL2((output.values.first() as Array)[0]) } + override suspend fun embedBatch(data: List): List { + TODO("Not yet implemented") + } + suspend fun embedBatch(context: Context, texts: List): List { val allEmbeddings = mutableListOf() diff --git a/core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/clip/Constants.kt b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/Constants.kt similarity index 66% rename from core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/clip/Constants.kt rename to ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/Constants.kt index f0d19c3..5cb2798 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/clip/Constants.kt +++ b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/Constants.kt @@ -1,4 +1,4 @@ -package com.fpf.smartscansdk.core.ml.embeddings.clip +package com.fpf.smartscansdk.ml.models.providers.embeddings.clip object ClipConfig { @@ -6,6 +6,5 @@ object ClipConfig { const val DIM_PIXEL_SIZE = 3 const val IMAGE_SIZE_X = 224 const val IMAGE_SIZE_Y = 224 - const val DEFAULT_IMAGE_DISPLAY_SIZE = 1024 const val CLIP_EMBEDDING_LENGTH = 512 } \ No newline at end of file diff --git a/core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/clip/PreProcess.kt b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/PreProcess.kt similarity index 91% rename from core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/clip/PreProcess.kt rename to ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/PreProcess.kt index 8d2f398..242357f 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/clip/PreProcess.kt +++ b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/PreProcess.kt @@ -1,11 +1,11 @@ -package com.fpf.smartscansdk.core.ml.embeddings.clip +package com.fpf.smartscansdk.ml.models.providers.embeddings.clip import android.graphics.Bitmap -import com.fpf.smartscansdk.core.utils.centerCrop import java.nio.ByteBuffer import java.nio.ByteOrder import java.nio.FloatBuffer import androidx.core.graphics.get +import com.fpf.smartscansdk.core.media.centerCrop fun preProcess(bitmap: Bitmap): FloatBuffer { val cropped = centerCrop(bitmap, ClipConfig.IMAGE_SIZE_X) diff --git a/core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/clip/Tokenizer.kt b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/Tokenizer.kt similarity index 94% rename from core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/clip/Tokenizer.kt rename to ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/Tokenizer.kt index 9938e7b..2c09b5a 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/ml/embeddings/clip/Tokenizer.kt +++ b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/Tokenizer.kt @@ -1,6 +1,4 @@ -package com.fpf.smartscansdk.core.ml.embeddings.clip - -import com.fpf.smartscansdk.core.ml.models.embeddings.clip.byteEncoder +package com.fpf.smartscansdk.ml.models.providers.embeddings.clip class ClipTokenizer( private val encoder: Map, diff --git a/core/src/main/res/raw/merges.txt b/ml/src/main/res/raw/merges.txt similarity index 100% rename from core/src/main/res/raw/merges.txt rename to ml/src/main/res/raw/merges.txt diff --git a/core/src/main/res/raw/vocab.json b/ml/src/main/res/raw/vocab.json similarity index 100% rename from core/src/main/res/raw/vocab.json rename to ml/src/main/res/raw/vocab.json diff --git a/core/src/test/kotlin/com/fpf/smartscansdk/core/ml/embeddings/FewShotClassifierTest.kt b/ml/src/test/kotlin/com/fpf/smartscansdk/core/ml/embeddings/FewShotClassifierTest.kt similarity index 90% rename from core/src/test/kotlin/com/fpf/smartscansdk/core/ml/embeddings/FewShotClassifierTest.kt rename to ml/src/test/kotlin/com/fpf/smartscansdk/core/ml/embeddings/FewShotClassifierTest.kt index b00ce26..b32055b 100644 --- a/core/src/test/kotlin/com/fpf/smartscansdk/core/ml/embeddings/FewShotClassifierTest.kt +++ b/ml/src/test/kotlin/com/fpf/smartscansdk/core/ml/embeddings/FewShotClassifierTest.kt @@ -1,5 +1,9 @@ package com.fpf.smartscansdk.core.ml.embeddings +import com.fpf.smartscansdk.ml.models.providers.embeddings.ClassificationError +import com.fpf.smartscansdk.ml.models.providers.embeddings.ClassificationResult +import com.fpf.smartscansdk.ml.models.providers.embeddings.PrototypeEmbedding +import com.fpf.smartscansdk.ml.models.providers.embeddings.classify import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test diff --git a/core/src/test/kotlin/com/fpf/smartscansdk/core/processors/BatchProcessorTest.kt b/ml/src/test/kotlin/com/fpf/smartscansdk/core/processors/BatchProcessorTest.kt similarity index 100% rename from core/src/test/kotlin/com/fpf/smartscansdk/core/processors/BatchProcessorTest.kt rename to ml/src/test/kotlin/com/fpf/smartscansdk/core/processors/BatchProcessorTest.kt diff --git a/settings.gradle.kts b/settings.gradle.kts index 5a09bd7..3cbe576 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -25,4 +25,4 @@ plugins{ } rootProject.name = "SmartScanSdk" -include(":core", ":extensions") +include(":ml", ":core") From dbffdecab5238032bfae179eafb32fb6ee82c854 Mon Sep 17 00:00:00 2001 From: d41dev Date: Wed, 29 Oct 2025 15:27:18 +0000 Subject: [PATCH 02/14] update: use library catalogue --- core/build.gradle.kts | 30 +++++++++++++++--------------- ml/build.gradle.kts | 18 +++++++++--------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/core/build.gradle.kts b/core/build.gradle.kts index dfa49d0..ce756be 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -64,25 +64,25 @@ java { dependencies { // Expose core-ktx to consumers of core or extensions api(libs.androidx.core.ktx) - + // JVM unit tests - testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.7.3") - testImplementation("org.junit.jupiter:junit-jupiter-api:5.10.0") - testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.10.0") - testImplementation("io.mockk:mockk:1.14.5") + testImplementation(libs.kotlinx.coroutines.test) + testImplementation(libs.junit.jupiter.api) + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.mockk) testImplementation(kotlin("test")) // Android instrumented tests - androidTestImplementation("androidx.test:core:1.7.0") - androidTestImplementation("androidx.test.ext:junit-ktx:1.3.0") - androidTestImplementation("androidx.test:runner:1.6.1") - androidTestImplementation("io.mockk:mockk-android:1.14.5") - androidTestImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.7.3") - - androidTestImplementation("androidx.room:room-runtime:2.7.2") - androidTestImplementation("androidx.room:room-ktx:2.7.2") - androidTestImplementation("androidx.room:room-testing:2.7.2") - ksp("androidx.room:room-compiler:2.7.2") + androidTestImplementation(libs.androidx.core) + androidTestImplementation(libs.androidx.junit.ktx) + androidTestImplementation(libs.androidx.runner) + androidTestImplementation(libs.mockk.android) + androidTestImplementation(libs.kotlinx.coroutines.test) + + androidTestImplementation(libs.androidx.room.runtime) + androidTestImplementation(libs.androidx.room.ktx) + androidTestImplementation(libs.androidx.room.testing) + ksp(libs.androidx.room.compiler) } diff --git a/ml/build.gradle.kts b/ml/build.gradle.kts index 570d0d2..aa870be 100644 --- a/ml/build.gradle.kts +++ b/ml/build.gradle.kts @@ -70,18 +70,18 @@ dependencies { implementation(libs.onnxruntime.android) // JVM unit tests - testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.7.3") - testImplementation("org.junit.jupiter:junit-jupiter-api:5.10.0") - testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.10.0") - testImplementation("io.mockk:mockk:1.14.5") + testImplementation(libs.kotlinx.coroutines.test) + testImplementation(libs.junit.jupiter.api) + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.mockk) testImplementation(kotlin("test")) // Android instrumented tests - androidTestImplementation("androidx.test:core:1.7.0") - androidTestImplementation("androidx.test.ext:junit-ktx:1.3.0") - androidTestImplementation("androidx.test:runner:1.6.1") - androidTestImplementation("io.mockk:mockk-android:1.14.5") - androidTestImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.7.3") + androidTestImplementation(libs.androidx.core) + androidTestImplementation(libs.androidx.junit.ktx) + androidTestImplementation(libs.androidx.runner) + androidTestImplementation(libs.mockk.android) + androidTestImplementation(libs.kotlinx.coroutines.test) } val gitVersion: String by lazy { From c9eb5ff281a1704e0127f61fa0bb64aba95186c5 Mon Sep 17 00:00:00 2001 From: d41dev Date: Wed, 29 Oct 2025 15:27:45 +0000 Subject: [PATCH 03/14] update: use library catalogue libs.versions --- gradle/libs.versions.toml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index f76616c..d399470 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,13 +1,32 @@ [versions] +core = "1.7.0" documentfile = "1.1.0" +junitJupiterApi = "5.10.0" +junitKtx = "1.3.0" kotlin = "2.0.21" coreKtx = "1.17.0" +kotlinxCoroutinesTest = "1.7.3" +mockk = "1.14.5" onnxruntimeAndroid = "1.22.0" +roomRuntime = "2.7.2" +runner = "1.6.1" [libraries] +androidx-core = { module = "androidx.test:core", version.ref = "core" } androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" } androidx-documentfile = { module = "androidx.documentfile:documentfile", version.ref = "documentfile" } +androidx-junit-ktx = { module = "androidx.test.ext:junit-ktx", version.ref = "junitKtx" } +androidx-room-compiler = { module = "androidx.room:room-compiler", version.ref = "roomRuntime" } +androidx-room-ktx = { module = "androidx.room:room-ktx", version.ref = "roomRuntime" } +androidx-room-runtime = { module = "androidx.room:room-runtime", version.ref = "roomRuntime" } +androidx-room-testing = { module = "androidx.room:room-testing", version.ref = "roomRuntime" } +androidx-runner = { module = "androidx.test:runner", version.ref = "runner" } +junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "junitJupiterApi" } +junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junitJupiterApi" } +kotlinx-coroutines-test = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-test", version.ref = "kotlinxCoroutinesTest" } +mockk = { module = "io.mockk:mockk", version.ref = "mockk" } +mockk-android = { module = "io.mockk:mockk-android", version.ref = "mockk" } onnxruntime-android = { module = "com.microsoft.onnxruntime:onnxruntime-android", version.ref = "onnxruntimeAndroid" } [plugins] From 7fdabef282649bca961aaa0ad96ee0f442439b13 Mon Sep 17 00:00:00 2001 From: d41dev Date: Wed, 29 Oct 2025 15:30:22 +0000 Subject: [PATCH 04/14] update: namespace --- ml/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ml/build.gradle.kts b/ml/build.gradle.kts index aa870be..1853e0d 100644 --- a/ml/build.gradle.kts +++ b/ml/build.gradle.kts @@ -7,7 +7,7 @@ plugins { } android { - namespace = "com.fpf.smartscansdk.core" + namespace = "com.fpf.smartscansdk.ml" compileSdk = 36 defaultConfig { From 92b838a093153decacab4346303f7ed84d96c269 Mon Sep 17 00:00:00 2001 From: d41dev Date: Wed, 29 Oct 2025 15:32:07 +0000 Subject: [PATCH 05/14] update: override embedbatch --- .../providers/embeddings/clip/ClipTextEmbedder.kt | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipTextEmbedder.kt b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipTextEmbedder.kt index ee293c3..5c797fc 100644 --- a/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipTextEmbedder.kt +++ b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipTextEmbedder.kt @@ -4,7 +4,7 @@ import android.app.Application import android.content.Context import android.content.res.Resources import android.util.JsonReader -import com.fpf.smartscansdk.core.R +import com.fpf.smartscansdk.ml.R import com.fpf.smartscansdk.core.data.TextEmbeddingProvider import com.fpf.smartscansdk.core.embeddings.normalizeL2 import com.fpf.smartscansdk.ml.models.OnnxModel @@ -24,7 +24,7 @@ import java.util.* // Using ModelSource enables using with bundle model or local model which has been downloaded class ClipTextEmbedder( - context: Context, + private val context: Context, modelSource: ModelSource ) : TextEmbeddingProvider { @@ -63,11 +63,8 @@ class ClipTextEmbedder( normalizeL2((output.values.first() as Array)[0]) } - override suspend fun embedBatch(data: List): List { - TODO("Not yet implemented") - } - suspend fun embedBatch(context: Context, texts: List): List { + override suspend fun embedBatch(data: List): List { val allEmbeddings = mutableListOf() val processor = object : BatchProcessor(application = context.applicationContext as Application) { @@ -79,7 +76,7 @@ class ClipTextEmbedder( } } - processor.run(texts) + processor.run(data) return allEmbeddings } From 6eade19226f7bc95f32b3d858063e41bd97724df Mon Sep 17 00:00:00 2001 From: d41dev Date: Wed, 29 Oct 2025 15:36:38 +0000 Subject: [PATCH 06/14] update: repalce application with context but always use context.applicationContext --- .../core/indexers/ImageIndexer.kt | 5 ++--- .../core/indexers/VideoIndexer.kt | 4 ++-- .../core/processors/BatchProcessor.kt | 20 +++++++++---------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/com/fpf/smartscansdk/core/indexers/ImageIndexer.kt b/core/src/main/java/com/fpf/smartscansdk/core/indexers/ImageIndexer.kt index 7b3dc78..c898840 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/indexers/ImageIndexer.kt +++ b/core/src/main/java/com/fpf/smartscansdk/core/indexers/ImageIndexer.kt @@ -1,6 +1,5 @@ package com.fpf.smartscansdk.core.indexers -import android.app.Application import android.content.ContentUris import android.content.Context import android.provider.MediaStore @@ -23,10 +22,10 @@ class ImageIndexer( private val embedder: ImageEmbeddingProvider, private val store: IEmbeddingStore, private val bitmapMaxSize: Int = 225, - application: Application, + context: Context, listener: IProcessorListener? = null, options: ProcessOptions = ProcessOptions(), - ): BatchProcessor(application, listener, options){ + ): BatchProcessor(context, listener, options){ companion object { const val INDEX_FILENAME = "image_index.bin" diff --git a/core/src/main/java/com/fpf/smartscansdk/core/indexers/VideoIndexer.kt b/core/src/main/java/com/fpf/smartscansdk/core/indexers/VideoIndexer.kt index ced4abc..b4b931f 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/indexers/VideoIndexer.kt +++ b/core/src/main/java/com/fpf/smartscansdk/core/indexers/VideoIndexer.kt @@ -24,11 +24,11 @@ class VideoIndexer( private val frameCount: Int = 10, private val width: Int, private val height: Int, - application: Application, + context: Context, listener: IProcessorListener? = null, options: ProcessOptions = ProcessOptions(), private val store: IEmbeddingStore, - ): BatchProcessor(application, listener, options){ + ): BatchProcessor(context, listener, options){ companion object { const val INDEX_FILENAME = "video_index.bin" diff --git a/core/src/main/java/com/fpf/smartscansdk/core/processors/BatchProcessor.kt b/core/src/main/java/com/fpf/smartscansdk/core/processors/BatchProcessor.kt index 3ff561e..1ac7d6d 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/processors/BatchProcessor.kt +++ b/core/src/main/java/com/fpf/smartscansdk/core/processors/BatchProcessor.kt @@ -16,7 +16,7 @@ import java.util.concurrent.atomic.AtomicInteger // For BatchProcessor’s use case—long-running, batched, asynchronous processing—the Application context should be used. abstract class BatchProcessor( - private val application: Application, + private val context: Context, protected val listener: IProcessorListener? = null, private val options: ProcessOptions = ProcessOptions(), ) { @@ -33,13 +33,13 @@ abstract class BatchProcessor( if (items.isEmpty()) { Log.w(TAG, "No items to process.") val metrics = Metrics.Success() - listener?.onComplete(application, metrics) + listener?.onComplete(context.applicationContext, metrics) return@withContext metrics } - val memoryUtils = MemoryUtils(application, options.memory) + val memoryUtils = MemoryUtils(context.applicationContext, options.memory) - listener?.onActive(application) + listener?.onActive(context.applicationContext) for (batch in items.chunked(options.batchSize)) { val currentConcurrency = memoryUtils.calculateConcurrencyLevel() @@ -49,15 +49,15 @@ abstract class BatchProcessor( async { semaphore.withPermit { try { - val output = onProcess(application, item) + val output = onProcess(context.applicationContext, item) output } catch (e: Exception) { - listener?.onError(application, e, item) + listener?.onError(context.applicationContext, e, item) null }finally { val current = processedCount.incrementAndGet() val progress = current.toFloat() / items.size - listener?.onProgress(application, progress) + listener?.onProgress(context.applicationContext, progress) } } } @@ -65,13 +65,13 @@ abstract class BatchProcessor( val outputBatch = deferredResults.mapNotNull { it.await() } totalSuccess += outputBatch.size - onBatchComplete(application, outputBatch) + onBatchComplete(context.applicationContext, outputBatch) } val endTime = System.currentTimeMillis() val metrics = Metrics.Success(totalSuccess, timeElapsed = endTime - startTime) - listener?.onComplete(application, metrics) + listener?.onComplete(context.applicationContext, metrics) metrics } catch (e: CancellationException) { @@ -83,7 +83,7 @@ abstract class BatchProcessor( timeElapsed = System.currentTimeMillis() - startTime, error = e ) - listener?.onFail(application, metrics) + listener?.onFail(context.applicationContext, metrics) metrics } } From 9a083d10bfdbb8a6d326a48bb649b1b1d1c0aeab Mon Sep 17 00:00:00 2001 From: d41dev Date: Wed, 29 Oct 2025 15:43:06 +0000 Subject: [PATCH 07/14] update: rename bitmapMaxSize to imageMaxSize for clarity --- .../java/com/fpf/smartscansdk/core/indexers/ImageIndexer.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/fpf/smartscansdk/core/indexers/ImageIndexer.kt b/core/src/main/java/com/fpf/smartscansdk/core/indexers/ImageIndexer.kt index c898840..a2eadc3 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/indexers/ImageIndexer.kt +++ b/core/src/main/java/com/fpf/smartscansdk/core/indexers/ImageIndexer.kt @@ -21,7 +21,7 @@ import kotlinx.coroutines.withContext class ImageIndexer( private val embedder: ImageEmbeddingProvider, private val store: IEmbeddingStore, - private val bitmapMaxSize: Int = 225, + private val maxImageSize: Int = 225, context: Context, listener: IProcessorListener? = null, options: ProcessOptions = ProcessOptions(), @@ -39,7 +39,7 @@ class ImageIndexer( val contentUri = ContentUris.withAppendedId( MediaStore.Images.Media.EXTERNAL_CONTENT_URI, item ) - val bitmap = getBitmapFromUri(context, contentUri, bitmapMaxSize) + val bitmap = getBitmapFromUri(context, contentUri, maxImageSize) val embedding = withContext(NonCancellable) { embedder.embed(bitmap) } From 8796a8c81bc0b151125c5e071ac80e2d63f9d4f2 Mon Sep 17 00:00:00 2001 From: d41dev Date: Wed, 29 Oct 2025 15:44:48 +0000 Subject: [PATCH 08/14] update: add utils to flatten and unflatten a list of embeddings --- .../core/embeddings/EmbeddingUtils.kt | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/core/src/main/java/com/fpf/smartscansdk/core/embeddings/EmbeddingUtils.kt b/core/src/main/java/com/fpf/smartscansdk/core/embeddings/EmbeddingUtils.kt index 79814e8..6faf337 100644 --- a/core/src/main/java/com/fpf/smartscansdk/core/embeddings/EmbeddingUtils.kt +++ b/core/src/main/java/com/fpf/smartscansdk/core/embeddings/EmbeddingUtils.kt @@ -35,3 +35,24 @@ suspend fun generatePrototypeEmbedding(rawEmbeddings: List): FloatAr normalizeL2(FloatArray(embeddingLength) { i -> sum[i] / rawEmbeddings.size }) } + + +fun flattenEmbeddings(embeddings: List, embeddingDim: Int): FloatArray { + val batchSize = embeddings.size + val flattened = FloatArray(batchSize * embeddingDim) + for (i in embeddings.indices) { + System.arraycopy(embeddings[i], 0, flattened, i * embeddingDim, embeddingDim) + } + return flattened +} + +fun unflattenEmbeddings(flattened: FloatArray, embeddingDim: Int): List { + val batchSize = flattened.size / embeddingDim + val embeddings = mutableListOf() + for (i in 0 until batchSize) { + val embedding = FloatArray(embeddingDim) + System.arraycopy(flattened, i * embeddingDim, embedding, 0, embeddingDim) + embeddings.add(embedding) + } + return embeddings +} From 9c0aca932eafd9b6c2cf1d4434ff4294171aa12c Mon Sep 17 00:00:00 2001 From: d41dev Date: Wed, 29 Oct 2025 15:49:15 +0000 Subject: [PATCH 09/14] update: imports for jvmTest --- .../kotlin/com/fpf/smartscansdk/core/data/images/Entity.kt | 2 +- .../fpf/smartscansdk/core/embeddings/FileEmbeddingRetriever.kt | 2 +- .../fpf/smartscansdk/core/embeddings/FileEmbeddingStoreTest.kt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/androidTest/kotlin/com/fpf/smartscansdk/core/data/images/Entity.kt b/core/src/androidTest/kotlin/com/fpf/smartscansdk/core/data/images/Entity.kt index 76d04f4..1e490c8 100644 --- a/core/src/androidTest/kotlin/com/fpf/smartscansdk/core/data/images/Entity.kt +++ b/core/src/androidTest/kotlin/com/fpf/smartscansdk/core/data/images/Entity.kt @@ -1,7 +1,7 @@ package com.fpf.smartscansdk.core.data.images import androidx.room.* -import com.fpf.smartscansdk.core.ml.embeddings.Embedding +import com.fpf.smartscansdk.core.data.Embedding @Entity(tableName = "image_embeddings") data class ImageEmbeddingEntity( diff --git a/core/src/test/kotlin/com/fpf/smartscansdk/core/embeddings/FileEmbeddingRetriever.kt b/core/src/test/kotlin/com/fpf/smartscansdk/core/embeddings/FileEmbeddingRetriever.kt index a5919e7..90e708c 100644 --- a/core/src/test/kotlin/com/fpf/smartscansdk/core/embeddings/FileEmbeddingRetriever.kt +++ b/core/src/test/kotlin/com/fpf/smartscansdk/core/embeddings/FileEmbeddingRetriever.kt @@ -1,6 +1,6 @@ package com.fpf.smartscansdk.core.embeddings -import com.fpf.smartscansdk.core.ml.embeddings.Embedding +import com.fpf.smartscansdk.core.data.Embedding import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue diff --git a/core/src/test/kotlin/com/fpf/smartscansdk/core/embeddings/FileEmbeddingStoreTest.kt b/core/src/test/kotlin/com/fpf/smartscansdk/core/embeddings/FileEmbeddingStoreTest.kt index a9056ee..b978b45 100644 --- a/core/src/test/kotlin/com/fpf/smartscansdk/core/embeddings/FileEmbeddingStoreTest.kt +++ b/core/src/test/kotlin/com/fpf/smartscansdk/core/embeddings/FileEmbeddingStoreTest.kt @@ -1,7 +1,7 @@ package com.fpf.smartscansdk.core.embeddings import android.util.Log -import com.fpf.smartscansdk.core.ml.embeddings.Embedding +import com.fpf.smartscansdk.core.data.Embedding import io.mockk.every import io.mockk.mockkStatic import kotlinx.coroutines.test.runTest From 170a2f7fc3b4f0e43b705fb0a2af8778e86d77d9 Mon Sep 17 00:00:00 2001 From: d41dev Date: Wed, 29 Oct 2025 16:16:14 +0000 Subject: [PATCH 10/14] update: add classificiton data --- .../com/fpf/smartscansdk/core/data/Classification.kt | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 core/src/main/java/com/fpf/smartscansdk/core/data/Classification.kt diff --git a/core/src/main/java/com/fpf/smartscansdk/core/data/Classification.kt b/core/src/main/java/com/fpf/smartscansdk/core/data/Classification.kt new file mode 100644 index 0000000..d0e098a --- /dev/null +++ b/core/src/main/java/com/fpf/smartscansdk/core/data/Classification.kt @@ -0,0 +1,9 @@ +package com.fpf.smartscansdk.core.data + +sealed class ClassificationResult { + data class Success(val classId: String, val similarity: Float ): ClassificationResult() + data class Failure(val error: ClassificationError ): ClassificationResult() +} + +enum class ClassificationError{MINIMUM_CLASS_SIZE, THRESHOLD, CONFIDENCE_MARGIN} + From e50e4da67e66a92f6d406c66c15e7de5b95680e3 Mon Sep 17 00:00:00 2001 From: d41dev Date: Wed, 29 Oct 2025 16:16:49 +0000 Subject: [PATCH 11/14] update: imports --- .../ml/models/providers/embeddings/FewShotClassifier.kt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/FewShotClassifier.kt b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/FewShotClassifier.kt index 738977a..d16bda1 100644 --- a/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/FewShotClassifier.kt +++ b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/FewShotClassifier.kt @@ -1,15 +1,11 @@ package com.fpf.smartscansdk.ml.models.providers.embeddings +import com.fpf.smartscansdk.core.data.ClassificationError +import com.fpf.smartscansdk.core.data.ClassificationResult import com.fpf.smartscansdk.core.data.PrototypeEmbedding import com.fpf.smartscansdk.core.embeddings.getSimilarities import com.fpf.smartscansdk.core.embeddings.getTopN -sealed class ClassificationResult { - data class Success(val classId: String, val similarity: Float ): ClassificationResult() - data class Failure(val error: ClassificationError ): ClassificationResult() -} - -enum class ClassificationError{MINIMUM_CLASS_SIZE, THRESHOLD, CONFIDENCE_MARGIN, LABELLED_BAD} fun classify(embedding: FloatArray, classPrototypes: List, threshold: Float = 0.4f, confidenceMargin: Float = 0.05f ): ClassificationResult{ if(classPrototypes.size < 2) return ClassificationResult.Failure(error= ClassificationError.MINIMUM_CLASS_SIZE) // Using a single class prototype leads to many false positives From 466b5475fde78341b492b7e2268387876416fb54 Mon Sep 17 00:00:00 2001 From: d41dev Date: Wed, 29 Oct 2025 16:19:15 +0000 Subject: [PATCH 12/14] update: reorganise tests and their imports --- .../core/processors/BatchProcessorTest.kt | 35 ++++++++++++++----- .../{core => }/ml/models/OnnxModelTest.kt | 7 ++-- .../embeddings/clip/ClipImageEmbedderTest.kt | 25 ++++++------- .../embeddings/clip/ClipTextEmbedderTest.kt | 20 +++++------ .../embeddings/FewShotClassifierTest.kt | 22 +++++++----- 5 files changed, 63 insertions(+), 46 deletions(-) rename {ml => core}/src/test/kotlin/com/fpf/smartscansdk/core/processors/BatchProcessorTest.kt (77%) rename ml/src/androidTest/kotlin/com/fpf/smartscansdk/{core => }/ml/models/OnnxModelTest.kt (95%) rename ml/src/androidTest/kotlin/com/fpf/smartscansdk/{core/ml => ml/models/providers}/embeddings/clip/ClipImageEmbedderTest.kt (85%) rename ml/src/androidTest/kotlin/com/fpf/smartscansdk/{core/ml => ml/models/providers}/embeddings/clip/ClipTextEmbedderTest.kt (85%) rename ml/src/test/kotlin/com/fpf/smartscansdk/{core/ml => ml/models/providers}/embeddings/FewShotClassifierTest.kt (81%) diff --git a/ml/src/test/kotlin/com/fpf/smartscansdk/core/processors/BatchProcessorTest.kt b/core/src/test/kotlin/com/fpf/smartscansdk/core/processors/BatchProcessorTest.kt similarity index 77% rename from ml/src/test/kotlin/com/fpf/smartscansdk/core/processors/BatchProcessorTest.kt rename to core/src/test/kotlin/com/fpf/smartscansdk/core/processors/BatchProcessorTest.kt index 19f2200..50af259 100644 --- a/ml/src/test/kotlin/com/fpf/smartscansdk/core/processors/BatchProcessorTest.kt +++ b/core/src/test/kotlin/com/fpf/smartscansdk/core/processors/BatchProcessorTest.kt @@ -1,8 +1,17 @@ package com.fpf.smartscansdk.core.processors import android.app.Application +import android.content.Context import android.util.Log -import io.mockk.* +import com.fpf.smartscansdk.core.data.IProcessorListener +import com.fpf.smartscansdk.core.data.Metrics +import com.fpf.smartscansdk.core.data.ProcessOptions +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import io.mockk.mockkConstructor +import io.mockk.mockkStatic +import io.mockk.verify import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -18,7 +27,7 @@ class BatchProcessorTest { fun setup() { mockApp = mockk(relaxed = true) mockListener = mockk(relaxed = true) - mockkStatic(android.util.Log::class) + mockkStatic(Log::class) every { Log.d(any(), any()) } returns 0 every { Log.e(any(), any()) } returns 0 every { Log.i(any(), any()) } returns 0 @@ -37,12 +46,12 @@ class BatchProcessorTest { options: ProcessOptions = ProcessOptions(batchSize = 2) ) : BatchProcessor(app, listener, options) { - override suspend fun onProcess(context: android.content.Context, item: Int): Int { + override suspend fun onProcess(context: Context, item: Int): Int { if (item in failOn) throw RuntimeException("Failed item $item") return item * 2 } - override suspend fun onBatchComplete(context: android.content.Context, batch: List) { + override suspend fun onBatchComplete(context: Context, batch: List) { // no-op for testing } } @@ -74,7 +83,7 @@ class BatchProcessorTest { assertEquals(0, metrics.totalProcessed) coVerify(exactly = 0) { mockListener.onProgress(any(), any()) } - coVerify ( exactly = 1){mockListener.onComplete(mockApp, any()) } + coVerify(exactly = 1) { mockListener.onComplete(mockApp, any()) } } @Test @@ -88,7 +97,11 @@ class BatchProcessorTest { assertEquals(2, metrics.totalProcessed) // only successful items counted verify { - mockListener.onError(mockApp, match { it.message?.contains("Failed item") == true }, any()) + mockListener.onError( + mockApp, + match { it.message?.contains("Failed item") == true }, + any() + ) } } @@ -102,6 +115,12 @@ class BatchProcessorTest { assertTrue(metrics is Metrics.Success) assertEquals(2, metrics.totalProcessed) - coVerify { mockListener.onError(mockApp, match { it.message?.contains("Failed item 2") == true }, 2) } + coVerify { + mockListener.onError( + mockApp, + match { it.message?.contains("Failed item 2") == true }, + 2 + ) + } } -} +} \ No newline at end of file diff --git a/ml/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/models/OnnxModelTest.kt b/ml/src/androidTest/kotlin/com/fpf/smartscansdk/ml/models/OnnxModelTest.kt similarity index 95% rename from ml/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/models/OnnxModelTest.kt rename to ml/src/androidTest/kotlin/com/fpf/smartscansdk/ml/models/OnnxModelTest.kt index 53c91c7..9bc9ba1 100644 --- a/ml/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/models/OnnxModelTest.kt +++ b/ml/src/androidTest/kotlin/com/fpf/smartscansdk/ml/models/OnnxModelTest.kt @@ -1,12 +1,11 @@ -package com.fpf.smartscansdk.core.ml.models +package com.fpf.smartscansdk.ml.models import android.content.Context import androidx.test.core.app.ApplicationProvider import ai.onnxruntime.* -import com.fpf.smartscansdk.ml.models.IModelLoader -import com.fpf.smartscansdk.ml.models.OnnxModel -import com.fpf.smartscansdk.ml.models.TensorData +import com.fpf.smartscansdk.ml.data.IModelLoader +import com.fpf.smartscansdk.ml.data.TensorData import io.mockk.Runs import io.mockk.coEvery import io.mockk.every diff --git a/ml/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipImageEmbedderTest.kt b/ml/src/androidTest/kotlin/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipImageEmbedderTest.kt similarity index 85% rename from ml/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipImageEmbedderTest.kt rename to ml/src/androidTest/kotlin/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipImageEmbedderTest.kt index 6cea0a7..fd0fe22 100644 --- a/ml/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipImageEmbedderTest.kt +++ b/ml/src/androidTest/kotlin/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipImageEmbedderTest.kt @@ -1,17 +1,15 @@ -package com.fpf.smartscansdk.core.ml.embeddings.clip +package com.fpf.smartscansdk.ml.models.providers.embeddings.clip import ai.onnxruntime.OnnxTensor import ai.onnxruntime.OrtEnvironment import android.content.Context -import android.content.res.Resources import android.graphics.Bitmap import androidx.test.core.app.ApplicationProvider +import com.fpf.smartscansdk.ml.data.ResourceId +import com.fpf.smartscansdk.ml.data.TensorData import com.fpf.smartscansdk.ml.models.providers.embeddings.clip.ClipConfig.IMAGE_SIZE_X import com.fpf.smartscansdk.ml.models.providers.embeddings.clip.ClipConfig.IMAGE_SIZE_Y import com.fpf.smartscansdk.ml.models.OnnxModel -import com.fpf.smartscansdk.ml.models.ResourceId -import com.fpf.smartscansdk.ml.models.TensorData -import com.fpf.smartscansdk.ml.models.providers.embeddings.clip.ClipImageEmbedder import io.mockk.* import kotlinx.coroutines.runBlocking import org.junit.After @@ -29,17 +27,16 @@ import io.mockk.unmockkAll import io.mockk.verify import org.junit.Assert.assertTrue import org.junit.Assert.assertEquals +import java.nio.FloatBuffer class ClipImageEmbedderInstrumentedTest { - private lateinit var resources: Resources private lateinit var context: Context @Before fun setup() { context = ApplicationProvider.getApplicationContext() - resources = context.resources mockkStatic(OnnxTensor::class) } @@ -50,7 +47,7 @@ class ClipImageEmbedderInstrumentedTest { @Test fun `initialize calls model loadModel and sets initialized`() = runBlocking { - val embedder = ClipImageEmbedder(resources, ResourceId(0)) + val embedder = ClipImageEmbedder(context, ResourceId(0)) // replace private model with a mock val mockModel = mockk(relaxed = true) @@ -69,7 +66,7 @@ class ClipImageEmbedderInstrumentedTest { @Test fun `embed returns normalized vector of expected dimension`() = runBlocking { - val embedder = ClipImageEmbedder(resources, ResourceId(0)) + val embedder = ClipImageEmbedder(context, ResourceId(0)) // mock internal model val mockModel = mockk(relaxed = true) @@ -83,7 +80,7 @@ class ClipImageEmbedderInstrumentedTest { // mock tensor creation and closing; specify types so mockk can infer overload val mockTensor = mockk(relaxed = true) - every { OnnxTensor.createTensor(any(), any(), any()) } returns mockTensor + every { OnnxTensor.createTensor(any(), any(), any()) } returns mockTensor every { mockTensor.close() } just Runs // inject mockModel @@ -103,7 +100,7 @@ class ClipImageEmbedderInstrumentedTest { @Test fun `embedBatch returns embeddings for all items`() = runBlocking { - val embedder = ClipImageEmbedder(resources, ResourceId(0)) + val embedder = ClipImageEmbedder(context, ResourceId(0)) val mockModel = mockk(relaxed = true) every { mockModel.isLoaded() } returns true @@ -114,7 +111,7 @@ class ClipImageEmbedderInstrumentedTest { every { mockModel.run(any>()) } returns mapOf("out" to raw) val mockTensor = mockk(relaxed = true) - every { OnnxTensor.createTensor(any(), any(), any()) } returns mockTensor + every { OnnxTensor.createTensor(any(), any(), any()) } returns mockTensor every { mockTensor.close() } just Runs val modelField = embedder::class.java.getDeclaredField("model") @@ -124,7 +121,7 @@ class ClipImageEmbedderInstrumentedTest { val bmp1 = Bitmap.createBitmap(IMAGE_SIZE_X, IMAGE_SIZE_Y, Bitmap.Config.ARGB_8888) val bmp2 = Bitmap.createBitmap(IMAGE_SIZE_X, IMAGE_SIZE_Y, Bitmap.Config.ARGB_8888) - val results = embedder.embedBatch(context, listOf(bmp1, bmp2)) + val results = embedder.embedBatch( listOf(bmp1, bmp2)) assertEquals(2, results.size) assertEquals(embedder.embeddingDim, results[0].size) @@ -132,7 +129,7 @@ class ClipImageEmbedderInstrumentedTest { @Test fun `closeSession closes model once`() { - val embedder = ClipImageEmbedder(resources, ResourceId(0)) + val embedder = ClipImageEmbedder(context, ResourceId(0)) val mockModel = mockk(relaxed = true) val modelField = embedder::class.java.getDeclaredField("model") modelField.isAccessible = true diff --git a/ml/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipTextEmbedderTest.kt b/ml/src/androidTest/kotlin/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipTextEmbedderTest.kt similarity index 85% rename from ml/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipTextEmbedderTest.kt rename to ml/src/androidTest/kotlin/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipTextEmbedderTest.kt index 1a82f20..cc11437 100644 --- a/ml/src/androidTest/kotlin/com/fpf/smartscansdk/core/ml/embeddings/clip/ClipTextEmbedderTest.kt +++ b/ml/src/androidTest/kotlin/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipTextEmbedderTest.kt @@ -1,14 +1,13 @@ -package com.fpf.smartscansdk.core.ml.embeddings.clip +package com.fpf.smartscansdk.ml.models.providers.embeddings.clip import ai.onnxruntime.OnnxTensor import ai.onnxruntime.OrtEnvironment import android.content.Context import android.content.res.Resources import androidx.test.core.app.ApplicationProvider -import com.fpf.smartscansdk.ml.models.providers.embeddings.clip.ClipTextEmbedder +import com.fpf.smartscansdk.ml.data.ResourceId +import com.fpf.smartscansdk.ml.data.TensorData import com.fpf.smartscansdk.ml.models.OnnxModel -import com.fpf.smartscansdk.ml.models.ResourceId -import com.fpf.smartscansdk.ml.models.TensorData import io.mockk.* import kotlinx.coroutines.runBlocking import org.junit.After @@ -21,14 +20,11 @@ import org.junit.Assert.assertTrue import java.nio.LongBuffer class ClipTextEmbedderInstrumentedTest { - - private lateinit var resources: Resources private lateinit var context: Context @Before fun setup() { context = ApplicationProvider.getApplicationContext() - resources = context.resources mockkStatic(OnnxTensor::class) } @@ -39,7 +35,7 @@ class ClipTextEmbedderInstrumentedTest { @Test fun `initialize calls model loadModel and sets initialized`() = runBlocking { - val embedder = ClipTextEmbedder(resources, ResourceId(0)) + val embedder = ClipTextEmbedder(context, ResourceId(0)) val mockModel = mockk(relaxed = true) coEvery { mockModel.loadModel() } answers { every { mockModel.isLoaded() } returns true } val field = embedder::class.java.getDeclaredField("model") @@ -54,7 +50,7 @@ class ClipTextEmbedderInstrumentedTest { @Test fun `embed returns normalized vector of expected dimension`() = runBlocking { - val embedder = ClipTextEmbedder(resources, ResourceId(0)) + val embedder = ClipTextEmbedder(context, ResourceId(0)) val mockModel = mockk(relaxed = true) every { mockModel.isLoaded() } returns true every { mockModel.getInputNames() } returns listOf("input") @@ -80,7 +76,7 @@ class ClipTextEmbedderInstrumentedTest { @Test fun `embedBatch returns embeddings for all items`() = runBlocking { - val embedder = ClipTextEmbedder(resources, ResourceId(0)) + val embedder = ClipTextEmbedder(context, ResourceId(0)) val mockModel = mockk(relaxed = true) every { mockModel.isLoaded() } returns true every { mockModel.getInputNames() } returns listOf("input") @@ -98,7 +94,7 @@ class ClipTextEmbedderInstrumentedTest { field.set(embedder, mockModel) val texts = listOf("Hello", "World") - val results = embedder.embedBatch(context, texts) + val results = embedder.embedBatch( texts) assertEquals(2, results.size) assertEquals(embedder.embeddingDim, results[0].size) @@ -106,7 +102,7 @@ class ClipTextEmbedderInstrumentedTest { @Test fun `closeSession closes model once`() { - val embedder = ClipTextEmbedder(resources, ResourceId(0)) + val embedder = ClipTextEmbedder(context, ResourceId(0)) val mockModel = mockk(relaxed = true) val field = embedder::class.java.getDeclaredField("model") field.isAccessible = true diff --git a/ml/src/test/kotlin/com/fpf/smartscansdk/core/ml/embeddings/FewShotClassifierTest.kt b/ml/src/test/kotlin/com/fpf/smartscansdk/ml/models/providers/embeddings/FewShotClassifierTest.kt similarity index 81% rename from ml/src/test/kotlin/com/fpf/smartscansdk/core/ml/embeddings/FewShotClassifierTest.kt rename to ml/src/test/kotlin/com/fpf/smartscansdk/ml/models/providers/embeddings/FewShotClassifierTest.kt index b32055b..08e230c 100644 --- a/ml/src/test/kotlin/com/fpf/smartscansdk/core/ml/embeddings/FewShotClassifierTest.kt +++ b/ml/src/test/kotlin/com/fpf/smartscansdk/ml/models/providers/embeddings/FewShotClassifierTest.kt @@ -1,9 +1,9 @@ -package com.fpf.smartscansdk.core.ml.embeddings +package com.fpf.smartscansdk.ml.models.providers.embeddings -import com.fpf.smartscansdk.ml.models.providers.embeddings.ClassificationError -import com.fpf.smartscansdk.ml.models.providers.embeddings.ClassificationResult -import com.fpf.smartscansdk.ml.models.providers.embeddings.PrototypeEmbedding -import com.fpf.smartscansdk.ml.models.providers.embeddings.classify +import com.fpf.smartscansdk.core.data.ClassificationError +import com.fpf.smartscansdk.core.data.ClassificationResult +import com.fpf.smartscansdk.core.data.PrototypeEmbedding +import com.fpf.smartscansdk.core.embeddings.dot import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test @@ -26,7 +26,10 @@ class FewShotClassifierTest { val result = classify(embedding, prototypes, threshold, minMargin) Assertions.assertTrue(result is ClassificationResult.Failure) - Assertions.assertEquals(ClassificationError.THRESHOLD, (result as ClassificationResult.Failure).error) + Assertions.assertEquals( + ClassificationError.THRESHOLD, + (result as ClassificationResult.Failure).error + ) } @Test @@ -39,7 +42,10 @@ class FewShotClassifierTest { val result = classify(embedding, prototypes, threshold, minMargin) Assertions.assertTrue(result is ClassificationResult.Failure) - Assertions.assertEquals(ClassificationError.CONFIDENCE_MARGIN, (result as ClassificationResult.Failure).error) + Assertions.assertEquals( + ClassificationError.CONFIDENCE_MARGIN, + (result as ClassificationResult.Failure).error + ) } @Test @@ -71,4 +77,4 @@ class FewShotClassifierTest { Assertions.assertTrue((result as ClassificationResult.Failure).error == ClassificationError.MINIMUM_CLASS_SIZE) } -} +} \ No newline at end of file From 95f7dfc2f99278beffe1511f6f34105bd6f1e1ea Mon Sep 17 00:00:00 2001 From: d41dev Date: Wed, 29 Oct 2025 16:22:32 +0000 Subject: [PATCH 13/14] update: replace application with context --- .../ml/models/providers/embeddings/clip/ClipImageEmbedder.kt | 2 +- .../ml/models/providers/embeddings/clip/ClipTextEmbedder.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipImageEmbedder.kt b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipImageEmbedder.kt index d5c24f4..ab8736e 100644 --- a/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipImageEmbedder.kt +++ b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipImageEmbedder.kt @@ -46,7 +46,7 @@ class ClipImageEmbedder( override suspend fun embedBatch(data: List): List { val allEmbeddings = mutableListOf() - val processor = object : BatchProcessor(application = context.applicationContext as Application) { + val processor = object : BatchProcessor(context = context.applicationContext as Application) { override suspend fun onProcess(context: Context, item: Bitmap): FloatArray { return embed(item) } diff --git a/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipTextEmbedder.kt b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipTextEmbedder.kt index 5c797fc..b1ac8de 100644 --- a/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipTextEmbedder.kt +++ b/ml/src/main/java/com/fpf/smartscansdk/ml/models/providers/embeddings/clip/ClipTextEmbedder.kt @@ -67,7 +67,7 @@ class ClipTextEmbedder( override suspend fun embedBatch(data: List): List { val allEmbeddings = mutableListOf() - val processor = object : BatchProcessor(application = context.applicationContext as Application) { + val processor = object : BatchProcessor(context = context.applicationContext as Application) { override suspend fun onProcess(context: Context, item: String): FloatArray { return embed(item) } From b81352c449dad0174cec44f4a22658a19571a987 Mon Sep 17 00:00:00 2001 From: d41dev Date: Wed, 29 Oct 2025 16:42:20 +0000 Subject: [PATCH 14/14] fix: test issue with context vs context.applicatonContexT --- .../core/processors/BatchProcessorTest.kt | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/core/src/test/kotlin/com/fpf/smartscansdk/core/processors/BatchProcessorTest.kt b/core/src/test/kotlin/com/fpf/smartscansdk/core/processors/BatchProcessorTest.kt index 50af259..eb69e8f 100644 --- a/core/src/test/kotlin/com/fpf/smartscansdk/core/processors/BatchProcessorTest.kt +++ b/core/src/test/kotlin/com/fpf/smartscansdk/core/processors/BatchProcessorTest.kt @@ -1,6 +1,5 @@ package com.fpf.smartscansdk.core.processors -import android.app.Application import android.content.Context import android.util.Log import com.fpf.smartscansdk.core.data.IProcessorListener @@ -20,12 +19,12 @@ import kotlin.test.assertTrue class BatchProcessorTest { - private lateinit var mockApp: Application + private lateinit var context: Context private lateinit var mockListener: IProcessorListener @BeforeEach fun setup() { - mockApp = mockk(relaxed = true) + context = mockk(relaxed = true) mockListener = mockk(relaxed = true) mockkStatic(Log::class) every { Log.d(any(), any()) } returns 0 @@ -40,11 +39,11 @@ class BatchProcessorTest { // Simple concrete subclass for testing class TestProcessor( - app: Application, + context: Context, listener: IProcessorListener, private val failOn: Set = emptySet(), options: ProcessOptions = ProcessOptions(batchSize = 2) - ) : BatchProcessor(app, listener, options) { + ) : BatchProcessor(context, listener, options) { override suspend fun onProcess(context: Context, item: Int): Int { if (item in failOn) throw RuntimeException("Failed item $item") @@ -58,7 +57,7 @@ class BatchProcessorTest { @Test fun `run processes all items successfully`() = runBlocking { - val processor = TestProcessor(mockApp, mockListener) + val processor = TestProcessor(context, mockListener) val items = listOf(1, 2, 3, 4) val metrics = processor.run(items) @@ -66,15 +65,15 @@ class BatchProcessorTest { assertTrue(metrics is Metrics.Success) assertEquals(4, metrics.totalProcessed) - coVerify { mockListener.onActive(mockApp) } - coVerify { mockListener.onProgress(mockApp, match { it in 0f..1f }) } - coVerify { mockListener.onComplete(mockApp, any()) } + coVerify { mockListener.onActive(context.applicationContext) } + coVerify { mockListener.onProgress(context.applicationContext, match { it in 0f..1f }) } + coVerify { mockListener.onComplete(context.applicationContext, any()) } coVerify(exactly = 0) { mockListener.onError(any(), any(), any()) } } @Test fun `run handles empty input`() = runBlocking { - val processor = TestProcessor(mockApp, mockListener) + val processor = TestProcessor(context, mockListener) val items = emptyList() val metrics = processor.run(items) @@ -82,13 +81,13 @@ class BatchProcessorTest { assertTrue(metrics is Metrics.Success) assertEquals(0, metrics.totalProcessed) - coVerify(exactly = 0) { mockListener.onProgress(any(), any()) } - coVerify(exactly = 1) { mockListener.onComplete(mockApp, any()) } + coVerify(exactly = 0) { mockListener.onProgress(context, any()) } + coVerify(exactly = 1) { mockListener.onComplete(context.applicationContext, any()) } } @Test fun `run handles item failures`() = runBlocking { - val processor = TestProcessor(mockApp, mockListener, failOn = setOf(2, 4)) + val processor = TestProcessor(context, mockListener, failOn = setOf(2, 4)) val items = listOf(1, 2, 3, 4) val metrics = processor.run(items) @@ -98,7 +97,7 @@ class BatchProcessorTest { verify { mockListener.onError( - mockApp, + context.applicationContext, match { it.message?.contains("Failed item") == true }, any() ) @@ -107,7 +106,7 @@ class BatchProcessorTest { @Test fun `run handles exceptions gracefully`() = runBlocking { - val processor = TestProcessor(mockApp, mockListener, failOn = setOf(2)) + val processor = TestProcessor(context, mockListener, failOn = setOf(2)) val items = listOf(1, 2, 3) val metrics = processor.run(items) @@ -117,7 +116,7 @@ class BatchProcessorTest { coVerify { mockListener.onError( - mockApp, + context.applicationContext, match { it.message?.contains("Failed item 2") == true }, 2 )