OCR Refactor

.github/workflows/build_unstable.yml

@@ -63,6 +63,14 @@ jobs:
           fileDir: ${{ github.workspace }}
           encodedString: ${{ secrets.LOCAL_PROPERTIES_FILE_BASE64 }}
 
+      - name: Ensure CRNN OCR model asset folder exists
+        run: mkdir -p app/src/main/assets/ocr
+
+      - name: Download CRNN OCR model to assets
+        run: |
+          curl -L -o app/src/main/assets/ocr/model_patched.onnx https://huggingface.co/ArcaeaOffline/crnn-pytorch/resolve/main/model-early_stop_patched.onnx
+          curl -L -o app/src/main/assets/ocr/model_info.json https://huggingface.co/ArcaeaOffline/crnn-pytorch/resolve/main/model_info.json
+
       - name: Build APK using gradle
         run: ./gradlew assembleUnstableRelease
 

.gitignore

@@ -1,5 +1,10 @@
 output-metadata.json
 
+# OCR ONNX models
+app/src/main/assets/ocr/model*.onnx
+app/src/main/assets/ocr/model*.ort
+app/src/main/assets/ocr/model_info.json
+
 # Android Studio added
 /.idea/caches
 /.idea/libraries

app/build.gradle.kts

@@ -3,8 +3,10 @@ import java.io.FileInputStream
 import java.util.Properties
 
 plugins {
+    alias(libs.plugins.ksp)
     alias(libs.plugins.android.application)
     alias(libs.plugins.kotlin.android)
+    alias(libs.plugins.kotlin.plugin.serialization)
     alias(libs.plugins.secrets.gradle.plugin)
     alias(libs.plugins.protobuf)
     alias(libs.plugins.androidGitVersion)
@@ -166,7 +168,9 @@ android {
 
 dependencies {
     // android & androidx
-    implementation(androidx.room.runtime)
+    api(androidx.room.runtime)
+    ksp(androidx.room.compiler)
+    implementation(androidx.room.ktx)
 
     implementation(androidx.datastore.core)
     implementation(androidx.datastore.preferences)
@@ -200,9 +204,16 @@ dependencies {
 
     implementation(androidx.documentfile)
 
+    implementation(androidx.work.workRuntime)
+    implementation(androidx.work.workRuntimeKtx)
+
     // 3rd party
+    implementation(libs.kotlinx.serialization)
+
     implementation(libs.opencv)
 
+    implementation(libs.onnxruntime.android)
+
     implementation(libs.apache.commons.io)
 
     implementation(libs.protobuf.protobufJavalite)

app/proguard-rules.pro

@@ -23,5 +23,8 @@
 -keep public class org.opencv.core.CvException
 # generated protobuf messages
 -keep public class * extends com.google.protobuf.GeneratedMessageLite { *; }
+# onnx runtime
+# @see https://onnxruntime.ai/docs/build/android.html#note-proguard-rules-for-r8-minimization-android-app-builds-to-work
+-keep class ai.onnxruntime.** { *; }
 
 -dontwarn javax.annotation.processing.*

app/schemas/xyz.sevive.arcaeaoffline.database.AppDatabase/1.json

@@ -0,0 +1,106 @@
+{
+    "formatVersion": 1,
+    "database": {
+        "version": 1,
+        "identityHash": "b733cbfb9112b42e22784ea6006f66f6",
+        "entities": [
+            {
+                "tableName": "ocr_history",
+                "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `source_package_name` TEXT, `store_date` INTEGER NOT NULL, `song_id` TEXT, `rating_class` TEXT NOT NULL, `playResult` INTEGER NOT NULL, `pure` INTEGER, `far` INTEGER, `lost` INTEGER, `date` INTEGER, `max_recall` INTEGER, `modifier` TEXT, `clear_type` TEXT)",
+                "fields": [
+                    {
+                        "fieldPath": "id",
+                        "columnName": "id",
+                        "affinity": "INTEGER",
+                        "notNull": true
+                    },
+                    {
+                        "fieldPath": "sourcePackageName",
+                        "columnName": "source_package_name",
+                        "affinity": "TEXT",
+                        "notNull": false
+                    },
+                    {
+                        "fieldPath": "storeDate",
+                        "columnName": "store_date",
+                        "affinity": "INTEGER",
+                        "notNull": true
+                    },
+                    {
+                        "fieldPath": "songId",
+                        "columnName": "song_id",
+                        "affinity": "TEXT",
+                        "notNull": false
+                    },
+                    {
+                        "fieldPath": "ratingClass",
+                        "columnName": "rating_class",
+                        "affinity": "TEXT",
+                        "notNull": true
+                    },
+                    {
+                        "fieldPath": "score",
+                        "columnName": "playResult",
+                        "affinity": "INTEGER",
+                        "notNull": true
+                    },
+                    {
+                        "fieldPath": "pure",
+                        "columnName": "pure",
+                        "affinity": "INTEGER",
+                        "notNull": false
+                    },
+                    {
+                        "fieldPath": "far",
+                        "columnName": "far",
+                        "affinity": "INTEGER",
+                        "notNull": false
+                    },
+                    {
+                        "fieldPath": "lost",
+                        "columnName": "lost",
+                        "affinity": "INTEGER",
+                        "notNull": false
+                    },
+                    {
+                        "fieldPath": "date",
+                        "columnName": "date",
+                        "affinity": "INTEGER",
+                        "notNull": false
+                    },
+                    {
+                        "fieldPath": "maxRecall",
+                        "columnName": "max_recall",
+                        "affinity": "INTEGER",
+                        "notNull": false
+                    },
+                    {
+                        "fieldPath": "modifier",
+                        "columnName": "modifier",
+                        "affinity": "TEXT",
+                        "notNull": false
+                    },
+                    {
+                        "fieldPath": "clearType",
+                        "columnName": "clear_type",
+                        "affinity": "TEXT",
+                        "notNull": false
+                    }
+                ],
+                "primaryKey": {
+                    "autoGenerate": true,
+                    "columnNames": [
+                        "id"
+                    ]
+                },
+                "indices": [],
+                "foreignKeys": []
+            }
+        ],
+        "views": [],
+        "setupQueries": [
+            "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
+            "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'b733cbfb9112b42e22784ea6006f66f6')"
+        ]
+    }
+}

app/schemas/xyz.sevive.arcaeaoffline.database.OcrQueueDatabase/1.json

@@ -0,0 +1,140 @@
+{
+    "formatVersion": 1,
+    "database": {
+        "version": 1,
+        "identityHash": "6c239fc091cf43980a8287de2452e5a5",
+        "entities": [
+            {
+                "tableName": "ocr_queue_tasks",
+                "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `inserted_at` INTEGER NOT NULL, `file_uri` TEXT NOT NULL, `file_name` TEXT, `status` INTEGER NOT NULL, `result` TEXT, `play_result` TEXT, `warnings` TEXT, `exception` BLOB)",
+                "fields": [
+                    {
+                        "fieldPath": "id",
+                        "columnName": "id",
+                        "affinity": "INTEGER",
+                        "notNull": true
+                    },
+                    {
+                        "fieldPath": "insertedAt",
+                        "columnName": "inserted_at",
+                        "affinity": "INTEGER",
+                        "notNull": true
+                    },
+                    {
+                        "fieldPath": "fileUri",
+                        "columnName": "file_uri",
+                        "affinity": "TEXT",
+                        "notNull": true
+                    },
+                    {
+                        "fieldPath": "fileName",
+                        "columnName": "file_name",
+                        "affinity": "TEXT",
+                        "notNull": false
+                    },
+                    {
+                        "fieldPath": "status",
+                        "columnName": "status",
+                        "affinity": "INTEGER",
+                        "notNull": true
+                    },
+                    {
+                        "fieldPath": "result",
+                        "columnName": "result",
+                        "affinity": "TEXT",
+                        "notNull": false
+                    },
+                    {
+                        "fieldPath": "playResult",
+                        "columnName": "play_result",
+                        "affinity": "TEXT",
+                        "notNull": false
+                    },
+                    {
+                        "fieldPath": "warnings",
+                        "columnName": "warnings",
+                        "affinity": "TEXT",
+                        "notNull": false
+                    },
+                    {
+                        "fieldPath": "exception",
+                        "columnName": "exception",
+                        "affinity": "BLOB",
+                        "notNull": false
+                    }
+                ],
+                "primaryKey": {
+                    "autoGenerate": true,
+                    "columnNames": [
+                        "id"
+                    ]
+                },
+                "indices": [
+                    {
+                        "name": "index_ocr_queue_tasks_file_uri",
+                        "unique": true,
+                        "columnNames": [
+                            "file_uri"
+                        ],
+                        "orders": [],
+                        "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_ocr_queue_tasks_file_uri` ON `${TABLE_NAME}` (`file_uri`)"
+                    }
+                ],
+                "foreignKeys": []
+            },
+            {
+                "tableName": "ocr_queue_enqueue_buffer",
+                "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `uri` TEXT NOT NULL, `checked` INTEGER NOT NULL, `should_insert` INTEGER NOT NULL)",
+                "fields": [
+                    {
+                        "fieldPath": "id",
+                        "columnName": "id",
+                        "affinity": "INTEGER",
+                        "notNull": true
+                    },
+                    {
+                        "fieldPath": "uri",
+                        "columnName": "uri",
+                        "affinity": "TEXT",
+                        "notNull": true
+                    },
+                    {
+                        "fieldPath": "checked",
+                        "columnName": "checked",
+                        "affinity": "INTEGER",
+                        "notNull": true
+                    },
+                    {
+                        "fieldPath": "shouldInsert",
+                        "columnName": "should_insert",
+                        "affinity": "INTEGER",
+                        "notNull": true
+                    }
+                ],
+                "primaryKey": {
+                    "autoGenerate": true,
+                    "columnNames": [
+                        "id"
+                    ]
+                },
+                "indices": [
+                    {
+                        "name": "index_ocr_queue_enqueue_buffer_uri",
+                        "unique": true,
+                        "columnNames": [
+                            "uri"
+                        ],
+                        "orders": [],
+                        "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_ocr_queue_enqueue_buffer_uri` ON `${TABLE_NAME}` (`uri`)"
+                    }
+                ],
+                "foreignKeys": []
+            }
+        ],
+        "views": [],
+        "setupQueries": [
+            "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
+            "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '6c239fc091cf43980a8287de2452e5a5')"
+        ]
+    }
+}

app/src/main/AndroidManifest.xml

@@ -9,6 +9,8 @@
     <uses-permission
         android:name="android.permission.READ_EXTERNAL_STORAGE"
         android:maxSdkVersion="33" />
+    <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
+    <uses-permission android:name="android.permission.FOREGROUND_SERVICE_SPECIAL_USE" />
 
     <uses-sdk tools:overrideLibrary="com.jvziyaoyao.scale.image, com.jvziyaoyao.scale.zoomable" />
 
@@ -65,10 +67,14 @@
 
         <activity
             android:name=".EmergencyModeActivity"
-            android:exported="true"
             android:excludeFromRecents="true"
+            android:exported="true"
             android:finishOnTaskLaunch="true"
             android:launchMode="singleInstance" />
-    </application>
 
+        <service
+            android:name="androidx.work.impl.foreground.SystemForegroundService"
+            android:foregroundServiceType="specialUse"
+            tools:node="merge" />
+    </application>
 </manifest>

app/src/main/java/xyz/sevive/arcaeaoffline/ArcaeaOfflineApplication.kt

@@ -8,20 +8,27 @@ import android.content.pm.ShortcutManager
 import android.graphics.drawable.Icon
 import android.os.Build
 import com.jakewharton.threetenabp.AndroidThreeTen
+import kotlinx.coroutines.MainScope
+import kotlinx.coroutines.launch
 import org.acra.config.dialog
 import org.acra.config.httpSender
 import org.acra.data.StringFormat
 import org.acra.ktx.initAcra
 import org.acra.sender.HttpSender
+import xyz.sevive.arcaeaoffline.core.ocr.device.DeviceOcrOnnxHelper
+import xyz.sevive.arcaeaoffline.data.notification.Notifications
+import xyz.sevive.arcaeaoffline.helpers.GlobalArcaeaButtonStateHelper
 import xyz.sevive.arcaeaoffline.ui.containers.AppDatabaseRepositoryContainer
 import xyz.sevive.arcaeaoffline.ui.containers.ArcaeaOfflineDatabaseRepositoryContainer
 import xyz.sevive.arcaeaoffline.ui.containers.ArcaeaOfflineDatabaseRepositoryContainerImpl
 import xyz.sevive.arcaeaoffline.ui.containers.DataStoreRepositoryContainerImpl
+import xyz.sevive.arcaeaoffline.ui.containers.OcrQueueDatabaseRepositoryContainer
 
 
 class ArcaeaOfflineApplication : Application() {
     lateinit var arcaeaOfflineDatabaseRepositoryContainer: ArcaeaOfflineDatabaseRepositoryContainer
     lateinit var appDatabaseRepositoryContainer: AppDatabaseRepositoryContainer
+    val ocrQueueDatabaseRepositoryContainer by lazy { OcrQueueDatabaseRepositoryContainer(this) }
     val dataStoreRepositoryContainer by lazy { DataStoreRepositoryContainerImpl(this) }
 
     override fun attachBaseContext(base: Context) {
@@ -70,6 +77,12 @@ class ArcaeaOfflineApplication : Application() {
         super.onCreate()
 
         AndroidThreeTen.init(this)
+        DeviceOcrOnnxHelper.loadModelInfo(this)
+        Notifications.createChannels(this)
+
+        MainScope().launch {
+            GlobalArcaeaButtonStateHelper.reload(this@ArcaeaOfflineApplication)
+        }
 
         addEmergencyModeShortcut()
         arcaeaOfflineDatabaseRepositoryContainer =