From 63136c2eefed9732b98a4223ed8e78267f166db9 Mon Sep 17 00:00:00 2001 From: Android PowerUser <88908510+Android-PowerUser@users.noreply.github.com> Date: Sun, 29 Mar 2026 18:10:58 +0200 Subject: [PATCH 1/4] Further decompose god classes with multi-concern extractions --- .../ai/sample/AccessibilityCommandQueue.kt | 27 +++ .../AccessibilityServiceStateChecker.kt | 35 +++ .../ai/sample/KeyboardVisibilityObserver.kt | 40 ++++ .../com/google/ai/sample/MainActivity.kt | 46 +--- .../sample/MediaProjectionServiceStarter.kt | 15 ++ .../ScreenOperatorAccessibilityService.kt | 47 +--- .../feature/multimodal/MainActivityBridge.kt | 13 ++ .../multimodal/PhotoReasoningSerialization.kt | 17 ++ .../multimodal/PhotoReasoningViewModel.kt | 37 ++-- .../com/google/ai/sample/util/AppMappings.kt | 190 +++++++--------- .../ai/sample/util/AppNamePackageMapper.kt | 117 ++-------- .../ai/sample/util/ChatHistoryPreferences.kt | 25 +-- .../com/google/ai/sample/util/Command.kt | 27 +++ .../google/ai/sample/util/CommandParser.kt | 203 +++++------------- .../google/ai/sample/util/CoordinateParser.kt | 13 +- .../util/GenerationSettingsPreferences.kt | 23 +- .../com/google/ai/sample/util/ImageUtils.kt | 48 ++--- .../google/ai/sample/util/StringSimilarity.kt | 35 +++ .../util/SystemMessageEntryPreferences.kt | 100 ++++----- .../sample/util/SystemMessagePreferences.kt | 20 +- .../ai/sample/util/UserInputPreferences.kt | 7 +- 21 files changed, 494 insertions(+), 591 deletions(-) create mode 100644 app/src/main/kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt create mode 100644 app/src/main/kotlin/com/google/ai/sample/AccessibilityServiceStateChecker.kt create mode 100644 app/src/main/kotlin/com/google/ai/sample/KeyboardVisibilityObserver.kt create mode 100644 app/src/main/kotlin/com/google/ai/sample/MediaProjectionServiceStarter.kt create mode 100644 app/src/main/kotlin/com/google/ai/sample/feature/multimodal/MainActivityBridge.kt create mode 100644 app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningSerialization.kt create mode 100644 app/src/main/kotlin/com/google/ai/sample/util/Command.kt create mode 100644 app/src/main/kotlin/com/google/ai/sample/util/StringSimilarity.kt diff --git a/app/src/main/kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt b/app/src/main/kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt new file mode 100644 index 00000000..2fb7c2ab --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt @@ -0,0 +1,27 @@ +package com.google.ai.sample + +import com.google.ai.sample.util.Command +import java.util.LinkedList +import java.util.concurrent.atomic.AtomicBoolean + +internal class AccessibilityCommandQueue { + private val queue = LinkedList() + private val processing = AtomicBoolean(false) + + fun clearAndUnlock() { + queue.clear() + processing.set(false) + } + + fun enqueue(command: Command): Int { + queue.add(command) + return queue.size + } + + fun peek(): Command? = queue.peek() + fun poll(): Command? = queue.poll() + fun size(): Int = queue.size + fun isEmpty(): Boolean = queue.isEmpty() + fun tryAcquireProcessing(): Boolean = processing.compareAndSet(false, true) + fun releaseProcessing() = processing.set(false) +} diff --git a/app/src/main/kotlin/com/google/ai/sample/AccessibilityServiceStateChecker.kt b/app/src/main/kotlin/com/google/ai/sample/AccessibilityServiceStateChecker.kt new file mode 100644 index 00000000..dd7981ed --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/AccessibilityServiceStateChecker.kt @@ -0,0 +1,35 @@ +package com.google.ai.sample + +import android.content.Context +import android.provider.Settings +import android.util.Log + +internal object AccessibilityServiceStateChecker { + fun isEnabled(context: Context, tag: String): Boolean { + val accessibilityEnabled = try { + Settings.Secure.getInt( + context.contentResolver, + Settings.Secure.ACCESSIBILITY_ENABLED + ) + } catch (e: Settings.SettingNotFoundException) { + Log.e(tag, "Error finding accessibility setting: ${e.message}") + return false + } + + if (accessibilityEnabled != 1) { + Log.d(tag, "Accessibility is not enabled") + return false + } + + val serviceString = + "${context.packageName}/${ScreenOperatorAccessibilityService::class.java.canonicalName}" + val enabledServices = Settings.Secure.getString( + context.contentResolver, + Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES + ) ?: "" + + val isEnabled = enabledServices.contains(serviceString) + Log.d(tag, "Service $serviceString is ${if (isEnabled) "enabled" else "not enabled"}") + return isEnabled + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/KeyboardVisibilityObserver.kt b/app/src/main/kotlin/com/google/ai/sample/KeyboardVisibilityObserver.kt new file mode 100644 index 00000000..9af65431 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/KeyboardVisibilityObserver.kt @@ -0,0 +1,40 @@ +package com.google.ai.sample + +import android.graphics.Rect +import android.util.Log +import android.view.View +import android.view.ViewTreeObserver +import kotlinx.coroutines.flow.MutableStateFlow + +internal class KeyboardVisibilityObserver( + private val tag: String +) { + private var onGlobalLayoutListener: ViewTreeObserver.OnGlobalLayoutListener? = null + + fun start(rootView: View, keyboardState: MutableStateFlow) { + stop(rootView) + onGlobalLayoutListener = ViewTreeObserver.OnGlobalLayoutListener { + val rect = Rect() + rootView.getWindowVisibleDisplayFrame(rect) + val screenHeight = rootView.rootView.height + val keypadHeight = screenHeight - rect.bottom + if (keypadHeight > screenHeight * 0.15) { + if (!keyboardState.value) { + keyboardState.value = true + Log.d(tag, "Keyboard visible") + } + } else if (keyboardState.value) { + keyboardState.value = false + Log.d(tag, "Keyboard hidden") + } + } + rootView.viewTreeObserver.addOnGlobalLayoutListener(onGlobalLayoutListener) + } + + fun stop(rootView: View) { + onGlobalLayoutListener?.let { + rootView.viewTreeObserver.removeOnGlobalLayoutListener(it) + } + onGlobalLayoutListener = null + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/MainActivity.kt b/app/src/main/kotlin/com/google/ai/sample/MainActivity.kt index 51be1126..9a0852f4 100644 --- a/app/src/main/kotlin/com/google/ai/sample/MainActivity.kt +++ b/app/src/main/kotlin/com/google/ai/sample/MainActivity.kt @@ -10,7 +10,6 @@ import android.content.SharedPreferences import android.content.pm.PackageManager import android.net.Uri import android.os.Build -import android.graphics.Rect import android.os.Bundle import android.provider.Settings import android.util.Log @@ -32,7 +31,6 @@ import java.text.SimpleDateFormat import java.util.Date import java.util.Locale import android.view.View -import android.view.ViewTreeObserver import android.widget.Toast import androidx.activity.ComponentActivity import androidx.activity.compose.setContent @@ -102,7 +100,7 @@ class MainActivity : ComponentActivity() { // Keyboard Visibility private val _isKeyboardOpen = MutableStateFlow(false) val isKeyboardOpen: StateFlow = _isKeyboardOpen.asStateFlow() - private var onGlobalLayoutListener: ViewTreeObserver.OnGlobalLayoutListener? = null + private val keyboardVisibilityObserver = KeyboardVisibilityObserver(TAG) private var photoReasoningViewModel: PhotoReasoningViewModel? = null private lateinit var apiKeyManager: ApiKeyManager @@ -131,6 +129,7 @@ class MainActivity : ComponentActivity() { private var isProcessingExplicitScreenshotRequest: Boolean = false private var onMediaProjectionPermissionGranted: (() -> Unit)? = null private var onWebRtcMediaProjectionResult: ((Int, Intent) -> Unit)? = null + private val mediaProjectionServiceStarter by lazy { MediaProjectionServiceStarter(this) } // Payment dialog state private var showPaymentMethodDialog by mutableStateOf(false) @@ -224,20 +223,12 @@ class MainActivity : ComponentActivity() { } if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { if (ContextCompat.checkSelfPermission(this, android.Manifest.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION) == PackageManager.PERMISSION_GRANTED) { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - startForegroundService(serviceIntent) - } else { - startService(serviceIntent) - } + mediaProjectionServiceStarter.start(serviceIntent) } else { requestForegroundServicePermissionLauncher.launch(android.Manifest.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION) } } else { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - startForegroundService(serviceIntent) - } else { - startService(serviceIntent) - } + mediaProjectionServiceStarter.start(serviceIntent) } if (isProcessingExplicitScreenshotRequest) { @@ -265,11 +256,7 @@ class MainActivity : ComponentActivity() { val serviceIntent = Intent(this, ScreenCaptureService::class.java).apply { action = ScreenCaptureService.ACTION_KEEP_ALIVE_FOR_WEBRTC } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - startForegroundService(serviceIntent) - } else { - startService(serviceIntent) - } + mediaProjectionServiceStarter.start(serviceIntent) onWebRtcMediaProjectionResult?.invoke(resultCode, resultData) onWebRtcMediaProjectionResult = null @@ -282,22 +269,7 @@ class MainActivity : ComponentActivity() { private fun setupKeyboardVisibilityListener() { val rootView = findViewById(android.R.id.content) - onGlobalLayoutListener = ViewTreeObserver.OnGlobalLayoutListener { - val rect = Rect() - rootView.getWindowVisibleDisplayFrame(rect) - val screenHeight = rootView.rootView.height - val keypadHeight = screenHeight - rect.bottom - if (keypadHeight > screenHeight * 0.15) { - if (!_isKeyboardOpen.value) { - _isKeyboardOpen.value = true - Log.d(TAG, "Keyboard visible") - } - } else if (_isKeyboardOpen.value) { - _isKeyboardOpen.value = false - Log.d(TAG, "Keyboard hidden") - } - } - rootView.viewTreeObserver.addOnGlobalLayoutListener(onGlobalLayoutListener) + keyboardVisibilityObserver.start(rootView, _isKeyboardOpen) } private fun registerScreenshotReceivers() { @@ -1141,10 +1113,8 @@ class MainActivity : ComponentActivity() { billingClient.endConnection() Log.d(TAG, "onDestroy: BillingClient connection ended.") } - onGlobalLayoutListener?.let { - findViewById(android.R.id.content).viewTreeObserver.removeOnGlobalLayoutListener(it) - Log.d(TAG, "onDestroy: Keyboard layout listener removed.") - } + keyboardVisibilityObserver.stop(findViewById(android.R.id.content)) + Log.d(TAG, "onDestroy: Keyboard layout listener removed.") if (this == instance) { instance = null Log.d(TAG, "onDestroy: MainActivity instance cleared.") diff --git a/app/src/main/kotlin/com/google/ai/sample/MediaProjectionServiceStarter.kt b/app/src/main/kotlin/com/google/ai/sample/MediaProjectionServiceStarter.kt new file mode 100644 index 00000000..4ef38be2 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/MediaProjectionServiceStarter.kt @@ -0,0 +1,15 @@ +package com.google.ai.sample + +import android.content.Context +import android.content.Intent +import android.os.Build + +internal class MediaProjectionServiceStarter(private val context: Context) { + fun start(intent: Intent) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + context.startForegroundService(intent) + } else { + context.startService(intent) + } + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt index a7580ffb..7af22d48 100644 --- a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt +++ b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt @@ -19,7 +19,6 @@ import android.os.Environment import android.os.Handler import android.os.Looper import android.provider.MediaStore -import android.provider.Settings import android.util.Log import android.view.KeyEvent import android.view.accessibility.AccessibilityEvent @@ -35,7 +34,6 @@ import java.util.Date import java.util.Locale import java.util.concurrent.atomic.AtomicBoolean import java.lang.NumberFormatException -import java.util.LinkedList class ScreenOperatorAccessibilityService : AccessibilityService() { private data class ResolvedPoint(val xPx: Float, val yPx: Float) @@ -48,8 +46,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { private enum class ScrollAxis { HORIZONTAL, VERTICAL } - private val commandQueue = LinkedList() - private val isProcessingQueue = AtomicBoolean(false) + private val commandQueue = AccessibilityCommandQueue() // private val handler = Handler(Looper.getMainLooper()) // Already exists at the class level companion object { @@ -68,30 +65,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { * Check if the accessibility service is enabled in system settings */ fun isAccessibilityServiceEnabled(context: Context): Boolean { - val accessibilityEnabled = try { - Settings.Secure.getInt( - context.contentResolver, - Settings.Secure.ACCESSIBILITY_ENABLED - ) - } catch (e: Settings.SettingNotFoundException) { - Log.e(TAG, "Error finding accessibility setting: ${e.message}") - return false - } - - if (accessibilityEnabled != 1) { - Log.d(TAG, "Accessibility is not enabled") - return false - } - - val serviceString = "${context.packageName}/${ScreenOperatorAccessibilityService::class.java.canonicalName}" - val enabledServices = Settings.Secure.getString( - context.contentResolver, - Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES - ) ?: "" - - val isEnabled = enabledServices.contains(serviceString) - Log.d(TAG, "Service $serviceString is ${if (isEnabled) "enabled" else "not enabled"}") - return isEnabled + return AccessibilityServiceStateChecker.isEnabled(context, TAG) } /** @@ -109,8 +83,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { fun clearCommandQueue() { val instance = serviceInstance if (instance != null) { - instance.commandQueue.clear() - instance.isProcessingQueue.set(false) + instance.commandQueue.clearAndUnlock() Log.d(TAG, "Command queue cleared and processing flag reset.") } else { Log.w(TAG, "clearCommandQueue: serviceInstance is null, nothing to clear.") @@ -143,8 +116,8 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { Log.e(TAG, "Service instance became null before queueing command") return } - instance.commandQueue.add(command) - Log.d(TAG, "Command $command added to queue. Queue size: ${instance.commandQueue.size}") + val queueSize = instance.commandQueue.enqueue(command) + Log.d(TAG, "Command $command added to queue. Queue size: $queueSize") // Ensure processCommandQueue is called on the service's handler thread instance.handler.post { @@ -234,7 +207,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { } handler.postDelayed({ - isProcessingQueue.set(false) // Release the lock before the next cycle + commandQueue.releaseProcessing() // Release the lock before the next cycle processCommandQueue() // Try to process the next command }, nextCommandDelay) } @@ -506,19 +479,19 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { } private fun processCommandQueue() { - if (!isProcessingQueue.compareAndSet(false, true)) { + if (!commandQueue.tryAcquireProcessing()) { Log.d(TAG, "Queue is already being processed.") return } if (commandQueue.isEmpty()) { Log.d(TAG, "Command queue is empty. Stopping processing.") - isProcessingQueue.set(false) + commandQueue.releaseProcessing() return } val command = commandQueue.poll() - Log.d(TAG, "Processing command: $command. Queue size after poll: ${commandQueue.size}") + Log.d(TAG, "Processing command: $command. Queue size after poll: ${commandQueue.size()}") if (command != null) { val commandWasAsync = executeSingleCommand(command) // executeSingleCommand now returns Boolean @@ -531,7 +504,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { // is responsible for calling scheduleNextCommandProcessing upon completion. } else { Log.d(TAG, "Polled null command from queue, stopping processing.") - isProcessingQueue.set(false) + commandQueue.releaseProcessing() } } diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/MainActivityBridge.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/MainActivityBridge.kt new file mode 100644 index 00000000..fb212493 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/MainActivityBridge.kt @@ -0,0 +1,13 @@ +package com.google.ai.sample.feature.multimodal + +import android.content.Context +import com.google.ai.sample.ApiProvider +import com.google.ai.sample.MainActivity + +internal object MainActivityBridge { + fun applicationContextOrNull(): Context? = + MainActivity.getInstance()?.applicationContext + + fun currentApiKeyOrEmpty(provider: ApiProvider): String = + MainActivity.getInstance()?.getCurrentApiKey(provider) ?: "" +} diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningSerialization.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningSerialization.kt new file mode 100644 index 00000000..639a61b4 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningSerialization.kt @@ -0,0 +1,17 @@ +package com.google.ai.sample.feature.multimodal + +import android.graphics.Bitmap +import android.util.Base64 +import kotlinx.serialization.json.Json +import java.io.ByteArrayOutputStream + +internal object PhotoReasoningSerialization { + fun createStreamingJsonParser(): Json = Json { ignoreUnknownKeys = true } + + fun bitmapToBase64(bitmap: Bitmap): String { + val outputStream = ByteArrayOutputStream() + bitmap.compress(Bitmap.CompressFormat.JPEG, 90, outputStream) + val bytes = outputStream.toByteArray() + return Base64.encodeToString(bytes, Base64.NO_WRAP) + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt index ce0c3218..b8ef663f 100644 --- a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt @@ -58,8 +58,6 @@ import java.util.concurrent.atomic.AtomicBoolean import com.google.mediapipe.tasks.genai.llminference.ProgressListener import android.graphics.Bitmap -import java.io.ByteArrayOutputStream -import android.util.Base64 import com.google.ai.sample.feature.live.LiveApiManager import com.google.ai.sample.ApiProvider import com.google.mediapipe.tasks.genai.llminference.LlmInference @@ -101,21 +99,6 @@ class PhotoReasoningViewModel( private var lastMediaProjectionResultCode: Int = 0 private var lastMediaProjectionResultData: Intent? = null - - - private fun Bitmap.toBase64(): String { - val outputStream = ByteArrayOutputStream() - this.compress(Bitmap.CompressFormat.JPEG, 90, outputStream) - val bytes = outputStream.toByteArray() - return Base64.encodeToString(bytes, Base64.NO_WRAP) - } - - private fun mainActivityApplicationContextOrNull(): Context? = - MainActivity.getInstance()?.applicationContext - - private fun currentApiKeyOrEmpty(provider: ApiProvider): String = - MainActivity.getInstance()?.getCurrentApiKey(provider) ?: "" - private val _uiState: MutableStateFlow = MutableStateFlow(PhotoReasoningUiState.Initial) val uiState: StateFlow = @@ -211,7 +194,7 @@ class PhotoReasoningViewModel( private var currentScreenInfoForPrompt: String? = null private var currentImageUrisForChat: List? = null - private val sseJson = Json { ignoreUnknownKeys = true } + private val sseJson = PhotoReasoningSerialization.createStreamingJsonParser() /** * Liest einen OpenAI-kompatiblen SSE-Stream zeilenweise. @@ -937,7 +920,7 @@ class PhotoReasoningViewModel( appendUserAndPendingModelMessages(userMessage) _uiState.value = PhotoReasoningUiState.Loading - val imageDataList = selectedImages.map { it.toBase64() } + val imageDataList = selectedImages.map { PhotoReasoningSerialization.bitmapToBase64(it) } val prompt = "FOLLOW THE INSTRUCTIONS STRICTLY: $combinedPromptText" liveApiManager?.sendMessage(prompt, imageDataList.ifEmpty { null }) } catch (e: Exception) { @@ -956,7 +939,7 @@ class PhotoReasoningViewModel( imageUrisForChat: List?, currentModel: ModelOption ) { - val context = mainActivityApplicationContextOrNull() + val context = MainActivityBridge.applicationContextOrNull() if (context == null) { Log.e(TAG, "Context not available, cannot proceed with reasoning") _uiState.value = PhotoReasoningUiState.Error("Application not ready") @@ -1183,7 +1166,13 @@ private fun reasonWithMistral( if (lastUserMsg.role == "user") { val updatedContent = lastUserMsg.content.toMutableList() for (bitmap in selectedImages) - updatedContent.add(MistralImageContent(imageUrl = MistralImageUrl(url = "data:image/jpeg;base64,${bitmap.toBase64()}"))) + updatedContent.add( + MistralImageContent( + imageUrl = MistralImageUrl( + url = "data:image/jpeg;base64,${PhotoReasoningSerialization.bitmapToBase64(bitmap)}" + ) + ) + ) apiMessages[apiMessages.lastIndex] = lastUserMsg.copy(content = updatedContent) } } @@ -1294,7 +1283,7 @@ private fun reasonWithMistral( screenInfoForPrompt: String?, imageUrisForChat: List? ) { - val apiKey = currentApiKeyOrEmpty(ApiProvider.PUTER) + val apiKey = MainActivityBridge.currentApiKeyOrEmpty(ApiProvider.PUTER) if (apiKey.isEmpty()) { _uiState.value = PhotoReasoningUiState.Error("Puter Authentication Token (API Key) is missing") return @@ -1652,7 +1641,7 @@ private fun reasonWithMistral( private suspend fun sendMessageWithRetry(inputContent: Content, retryCount: Int) { Log.d(TAG, "sendMessageWithRetry: Delegating AI call to ScreenCaptureService (retryCount=$retryCount).") - val context = mainActivityApplicationContextOrNull() + val context = MainActivityBridge.applicationContextOrNull() if (context == null) { Log.e(TAG, "sendMessageWithRetry: Context is null, cannot delegate AI call.") _uiState.value = PhotoReasoningUiState.Error("Application context not available for AI call.") @@ -1707,7 +1696,7 @@ private fun reasonWithMistral( Log.d(TAG, "Collected temporary file paths to send to service: $tempFilePaths") val currentModel = com.google.ai.sample.GenerativeAiViewModelFactory.getCurrentModel() - val apiKey = currentApiKeyOrEmpty(currentModel.apiProvider) + val apiKey = MainActivityBridge.currentApiKeyOrEmpty(currentModel.apiProvider) val serviceIntent = createExecuteAiCallIntent( context = context, inputContentJson = inputContentJson, diff --git a/app/src/main/kotlin/com/google/ai/sample/util/AppMappings.kt b/app/src/main/kotlin/com/google/ai/sample/util/AppMappings.kt index b83ed62c..f617b286 100644 --- a/app/src/main/kotlin/com/google/ai/sample/util/AppMappings.kt +++ b/app/src/main/kotlin/com/google/ai/sample/util/AppMappings.kt @@ -1,122 +1,82 @@ package com.google.ai.sample.util internal object AppMappings { - val appNameVariations: Map> = mapOf( - "whatsapp" to listOf("whats app", "whats", "wa"), - "facebook" to listOf("fb", "face book"), - "instagram" to listOf("insta", "ig"), - "youtube" to listOf("yt", "you tube"), - "twitter" to listOf("x", "tweet"), - "telegram" to listOf("tg"), - "tiktok" to listOf("tik tok"), - "snapchat" to listOf("snap"), - "netflix" to listOf("nflx"), - "spotify" to listOf("music"), - "chrome" to listOf("google chrome", "browser"), - "gmail" to listOf("google mail", "email", "mail"), - "maps" to listOf("google maps", "navigation"), - "calendar" to listOf("google calendar"), - "photos" to listOf("google photos", "gallery"), - "drive" to listOf("google drive"), - "docs" to listOf("google docs", "documents"), - "sheets" to listOf("google sheets", "spreadsheets"), - "slides" to listOf("google slides", "presentations"), - "keep" to listOf("google keep", "notes"), - "amazon" to listOf("amazon shopping", "shop"), - "uber" to listOf("uber driver"), - "lyft" to listOf("ride"), - "doordash" to listOf("food delivery"), - "ubereats" to listOf("uber eats", "food"), - "grubhub" to listOf("food delivery"), - "instacart" to listOf("grocery"), - "zoom" to listOf("zoom meeting"), - "teams" to listOf("microsoft teams"), - "skype" to listOf("microsoft skype"), - "outlook" to listOf("microsoft outlook", "email"), - "word" to listOf("microsoft word", "documents"), - "excel" to listOf("microsoft excel", "spreadsheets"), - "powerpoint" to listOf("microsoft powerpoint", "presentations"), - "onedrive" to listOf("microsoft onedrive", "cloud"), - "onenote" to listOf("microsoft onenote", "notes"), - "linkedin" to listOf("linked in"), - "pinterest" to listOf("pin"), - "reddit" to listOf("reddit app"), - "discord" to listOf("chat"), - "slack" to listOf("work chat"), - "twitch" to listOf("streaming"), - "venmo" to listOf("payment"), - "paypal" to listOf("payment"), - "cashapp" to listOf("cash app", "payment"), - "zelle" to listOf("payment"), - "robinhood" to listOf("stocks"), - "coinbase" to listOf("crypto"), - "binance" to listOf("crypto"), - "wechat" to listOf("we chat"), - "line" to listOf("line messenger"), - "viber" to listOf("viber messenger"), - "signal" to listOf("signal messenger"), - "threema" to listOf("threema messenger"), - "settings" to listOf("system settings", "preferences") + private data class AppDefinition( + val canonicalName: String, + val packageName: String, + val variations: List = emptyList(), + val aliasesForPackageLookup: List = emptyList() ) - val manualMappings: Map = mapOf( - "whatsapp" to "com.whatsapp", - "facebook" to "com.facebook.katana", - "messenger" to "com.facebook.orca", - "instagram" to "com.instagram.android", - "youtube" to "com.google.android.youtube", - "twitter" to "com.twitter.android", - "x" to "com.twitter.android", - "telegram" to "org.telegram.messenger", - "tiktok" to "com.zhiliaoapp.musically", - "snapchat" to "com.snapchat.android", - "netflix" to "com.netflix.mediaclient", - "spotify" to "com.spotify.music", - "chrome" to "com.android.chrome", - "gmail" to "com.google.android.gm", - "maps" to "com.google.android.apps.maps", - "calendar" to "com.google.android.calendar", - "photos" to "com.google.android.apps.photos", - "drive" to "com.google.android.apps.docs", - "docs" to "com.google.android.apps.docs.editors.docs", - "sheets" to "com.google.android.apps.docs.editors.sheets", - "slides" to "com.google.android.apps.docs.editors.slides", - "keep" to "com.google.android.keep", - "amazon" to "com.amazon.mShop.android.shopping", - "uber" to "com.ubercab", - "lyft" to "me.lyft.android", - "doordash" to "com.dd.doordash", - "ubereats" to "com.ubercab.eats", - "grubhub" to "com.grubhub.android", - "instacart" to "com.instacart.client", - "zoom" to "us.zoom.videomeetings", - "teams" to "com.microsoft.teams", - "skype" to "com.skype.raider", - "outlook" to "com.microsoft.office.outlook", - "word" to "com.microsoft.office.word", - "excel" to "com.microsoft.office.excel", - "powerpoint" to "com.microsoft.office.powerpoint", - "onedrive" to "com.microsoft.skydrive", - "onenote" to "com.microsoft.office.onenote", - "linkedin" to "com.linkedin.android", - "pinterest" to "com.pinterest", - "reddit" to "com.reddit.frontpage", - "discord" to "com.discord", - "slack" to "com.Slack", - "twitch" to "tv.twitch.android.app", - "venmo" to "com.venmo", - "paypal" to "com.paypal.android.p2pmobile", - "cashapp" to "com.squareup.cash", - "zelle" to "com.zellepay.zelle", - "robinhood" to "com.robinhood.android", - "coinbase" to "com.coinbase.android", - "binance" to "com.binance.dev", - "wechat" to "com.tencent.mm", - "line" to "jp.naver.line.android", - "viber" to "com.viber.voip", - "signal" to "org.thoughtcrime.securesms", - "threema" to "ch.threema.app", - "settings" to "com.android.settings" + private val appDefinitions: List = listOf( + AppDefinition("whatsapp", "com.whatsapp", listOf("whats app", "whats", "wa")), + AppDefinition("facebook", "com.facebook.katana", listOf("fb", "face book")), + AppDefinition("messenger", "com.facebook.orca"), + AppDefinition("instagram", "com.instagram.android", listOf("insta", "ig")), + AppDefinition("youtube", "com.google.android.youtube", listOf("yt", "you tube")), + AppDefinition("twitter", "com.twitter.android", listOf("x", "tweet"), aliasesForPackageLookup = listOf("x")), + AppDefinition("telegram", "org.telegram.messenger", listOf("tg")), + AppDefinition("tiktok", "com.zhiliaoapp.musically", listOf("tik tok")), + AppDefinition("snapchat", "com.snapchat.android", listOf("snap")), + AppDefinition("netflix", "com.netflix.mediaclient", listOf("nflx")), + AppDefinition("spotify", "com.spotify.music", listOf("music")), + AppDefinition("chrome", "com.android.chrome", listOf("google chrome", "browser")), + AppDefinition("gmail", "com.google.android.gm", listOf("google mail", "email", "mail")), + AppDefinition("maps", "com.google.android.apps.maps", listOf("google maps", "navigation")), + AppDefinition("calendar", "com.google.android.calendar", listOf("google calendar")), + AppDefinition("photos", "com.google.android.apps.photos", listOf("google photos", "gallery")), + AppDefinition("drive", "com.google.android.apps.docs", listOf("google drive")), + AppDefinition("docs", "com.google.android.apps.docs.editors.docs", listOf("google docs", "documents")), + AppDefinition("sheets", "com.google.android.apps.docs.editors.sheets", listOf("google sheets", "spreadsheets")), + AppDefinition("slides", "com.google.android.apps.docs.editors.slides", listOf("google slides", "presentations")), + AppDefinition("keep", "com.google.android.keep", listOf("google keep", "notes")), + AppDefinition("amazon", "com.amazon.mShop.android.shopping", listOf("amazon shopping", "shop")), + AppDefinition("uber", "com.ubercab", listOf("uber driver")), + AppDefinition("lyft", "me.lyft.android", listOf("ride")), + AppDefinition("doordash", "com.dd.doordash", listOf("food delivery")), + AppDefinition("ubereats", "com.ubercab.eats", listOf("uber eats", "food")), + AppDefinition("grubhub", "com.grubhub.android", listOf("food delivery")), + AppDefinition("instacart", "com.instacart.client", listOf("grocery")), + AppDefinition("zoom", "us.zoom.videomeetings", listOf("zoom meeting")), + AppDefinition("teams", "com.microsoft.teams", listOf("microsoft teams")), + AppDefinition("skype", "com.skype.raider", listOf("microsoft skype")), + AppDefinition("outlook", "com.microsoft.office.outlook", listOf("microsoft outlook", "email")), + AppDefinition("word", "com.microsoft.office.word", listOf("microsoft word", "documents")), + AppDefinition("excel", "com.microsoft.office.excel", listOf("microsoft excel", "spreadsheets")), + AppDefinition("powerpoint", "com.microsoft.office.powerpoint", listOf("microsoft powerpoint", "presentations")), + AppDefinition("onedrive", "com.microsoft.skydrive", listOf("microsoft onedrive", "cloud")), + AppDefinition("onenote", "com.microsoft.office.onenote", listOf("microsoft onenote", "notes")), + AppDefinition("linkedin", "com.linkedin.android", listOf("linked in")), + AppDefinition("pinterest", "com.pinterest", listOf("pin")), + AppDefinition("reddit", "com.reddit.frontpage", listOf("reddit app")), + AppDefinition("discord", "com.discord", listOf("chat")), + AppDefinition("slack", "com.Slack", listOf("work chat")), + AppDefinition("twitch", "tv.twitch.android.app", listOf("streaming")), + AppDefinition("venmo", "com.venmo", listOf("payment")), + AppDefinition("paypal", "com.paypal.android.p2pmobile", listOf("payment")), + AppDefinition("cashapp", "com.squareup.cash", listOf("cash app", "payment")), + AppDefinition("zelle", "com.zellepay.zelle", listOf("payment")), + AppDefinition("robinhood", "com.robinhood.android", listOf("stocks")), + AppDefinition("coinbase", "com.coinbase.android", listOf("crypto")), + AppDefinition("binance", "com.binance.dev", listOf("crypto")), + AppDefinition("wechat", "com.tencent.mm", listOf("we chat")), + AppDefinition("line", "jp.naver.line.android", listOf("line messenger")), + AppDefinition("viber", "com.viber.voip", listOf("viber messenger")), + AppDefinition("signal", "org.thoughtcrime.securesms", listOf("signal messenger")), + AppDefinition("threema", "ch.threema.app", listOf("threema messenger")), + AppDefinition("settings", "com.android.settings", listOf("system settings", "preferences")) ) -} + val appNameVariations: Map> = appDefinitions + .filter { it.variations.isNotEmpty() } + .associate { it.canonicalName to it.variations } + + val manualMappings: Map = buildMap { + appDefinitions.forEach { definition -> + put(definition.canonicalName, definition.packageName) + definition.aliasesForPackageLookup.forEach { alias -> + put(alias, definition.packageName) + } + } + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/util/AppNamePackageMapper.kt b/app/src/main/kotlin/com/google/ai/sample/util/AppNamePackageMapper.kt index b6fa80a5..9b2149d3 100644 --- a/app/src/main/kotlin/com/google/ai/sample/util/AppNamePackageMapper.kt +++ b/app/src/main/kotlin/com/google/ai/sample/util/AppNamePackageMapper.kt @@ -14,6 +14,8 @@ import java.util.concurrent.ConcurrentHashMap class AppNamePackageMapper(private val context: Context) { companion object { private const val TAG = "AppNamePackageMapper" + private const val MATCH_THRESHOLD = 70 + private val NON_ALPHANUMERIC_REGEX = Regex("[^a-z0-9]") } // Cache for app name to package name mappings @@ -39,9 +41,7 @@ class AppNamePackageMapper(private val context: Context) { val resolveInfoList = queryLauncherActivities(packageManager) // Add manual mappings once - for ((key, value) in manualMappings) { - appNameToPackageCache[key] = value - } + appNameToPackageCache.putAll(manualMappings) // Add all apps to the cache for (resolveInfo in resolveInfoList) { @@ -55,12 +55,10 @@ class AppNamePackageMapper(private val context: Context) { } // Add variations to the cache once - for ((baseAppName, variations) in appNameVariations) { - val variationPackageName = getPackageName(baseAppName) - if (variationPackageName != null) { - for (variation in variations) { - appNameToPackageCache[variation] = variationPackageName - } + appNameVariations.forEach { (baseAppName, variations) -> + val variationPackageName = getPackageName(baseAppName) ?: return@forEach + variations.forEach { variation -> + appNameToPackageCache[variation] = variationPackageName } } @@ -103,24 +101,17 @@ class AppNamePackageMapper(private val context: Context) { val resolveInfoList = queryLauncherActivities(packageManager) // Find the best match - var bestMatch: ResolveInfo? = null - var bestMatchScore = 0 - - for (resolveInfo in resolveInfoList) { - val currentAppName = resolveInfo.loadLabel(packageManager).toString() - val normalizedCurrentAppName = normalizeName(currentAppName) - - // Calculate match score - val score = calculateMatchScore(normalizedAppName, normalizedCurrentAppName) - - if (score > bestMatchScore) { - bestMatchScore = score - bestMatch = resolveInfo + val (bestMatch, bestMatchScore) = resolveInfoList + .map { resolveInfo -> + val currentAppName = resolveInfo.loadLabel(packageManager).toString() + val normalizedCurrentAppName = normalizeName(currentAppName) + resolveInfo to StringSimilarity.calculateMatchScore(normalizedAppName, normalizedCurrentAppName) } - } + .maxByOrNull { it.second } + ?: (null to 0) // If we found a good match, return its package name - if (bestMatchScore >= 70 && bestMatch != null) { // 70% match threshold + if (bestMatchScore >= MATCH_THRESHOLD && bestMatch != null) { val packageName = bestMatch.activityInfo.packageName Log.d(TAG, "Found package name for app name '$appName': $packageName (match score: $bestMatchScore%)") @@ -149,7 +140,7 @@ class AppNamePackageMapper(private val context: Context) { private fun normalizeName(value: String): String { return value.lowercase(Locale.getDefault()) .trim() - .replace(Regex("[^a-z0-9]"), "") + .replace(NON_ALPHANUMERIC_REGEX, "") } /** @@ -167,10 +158,7 @@ class AppNamePackageMapper(private val context: Context) { // Try to get the app name from the package manager try { - // Get the package manager val packageManager = context.packageManager - - // Try to get the app info val applicationInfo = packageManager.getApplicationInfo(packageName, 0) val appName = packageManager.getApplicationLabel(applicationInfo).toString() @@ -178,7 +166,7 @@ class AppNamePackageMapper(private val context: Context) { // Add to cache packageToAppNameCache[packageName] = appName - appNameToPackageCache[appName.lowercase(Locale.getDefault())] = packageName + appNameToPackageCache[normalizeName(appName)] = packageName return appName } catch (e: Exception) { @@ -186,75 +174,4 @@ class AppNamePackageMapper(private val context: Context) { return packageName } } - - /** - * Calculate a match score between two app names - * - * @param query The query app name - * @param target The target app name - * @return A score from 0 to 100 indicating how well the names match - */ - private fun calculateMatchScore(query: String, target: String): Int { - // Exact match - if (query == target) { - return 100 - } - - // Target contains query - if (target.contains(query)) { - return 90 - } - - // Query contains target - if (query.contains(target)) { - return 80 - } - - // Calculate Levenshtein distance - val distance = levenshteinDistance(query, target) - val maxLength = maxOf(query.length, target.length) - - // Convert distance to similarity percentage - val similarity = ((maxLength - distance) / maxLength.toFloat()) * 100 - - return similarity.toInt() - } - - /** - * Calculate the Levenshtein distance between two strings - * - * @param s1 The first string - * @param s2 The second string - * @return The Levenshtein distance - */ - private fun levenshteinDistance(s1: String, s2: String): Int { - val m = s1.length - val n = s2.length - - // Create a matrix of size (m+1) x (n+1) - val dp = Array(m + 1) { IntArray(n + 1) } - - // Initialize the matrix - for (i in 0..m) { - dp[i][0] = i - } - - for (j in 0..n) { - dp[0][j] = j - } - - // Fill the matrix - for (i in 1..m) { - for (j in 1..n) { - val cost = if (s1[i - 1] == s2[j - 1]) 0 else 1 - dp[i][j] = minOf( - dp[i - 1][j] + 1, // deletion - dp[i][j - 1] + 1, // insertion - dp[i - 1][j - 1] + cost // substitution - ) - } - } - - return dp[m][n] - } } diff --git a/app/src/main/kotlin/com/google/ai/sample/util/ChatHistoryPreferences.kt b/app/src/main/kotlin/com/google/ai/sample/util/ChatHistoryPreferences.kt index 8e18981f..db7bbb62 100644 --- a/app/src/main/kotlin/com/google/ai/sample/util/ChatHistoryPreferences.kt +++ b/app/src/main/kotlin/com/google/ai/sample/util/ChatHistoryPreferences.kt @@ -1,8 +1,6 @@ package com.google.ai.sample.util import android.content.Context -import android.content.SharedPreferences -import com.google.ai.sample.feature.multimodal.PhotoParticipant import com.google.ai.sample.feature.multimodal.PhotoReasoningMessage import com.google.gson.Gson import com.google.gson.reflect.TypeToken @@ -14,32 +12,26 @@ import java.lang.reflect.Type object ChatHistoryPreferences { private const val PREFS_NAME = "chat_history_prefs" private const val KEY_CHAT_MESSAGES = "chat_messages" - - // Initialize Gson instance private val gson = Gson() + private val messageListType: Type = object : TypeToken>() {}.type + + private fun prefs(context: Context) = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) /** * Save chat messages to SharedPreferences */ fun saveChatMessages(context: Context, messages: List) { - val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) - val editor = prefs.edit() - val json = gson.toJson(messages) - editor.putString(KEY_CHAT_MESSAGES, json) - editor.apply() + prefs(context).edit().putString(KEY_CHAT_MESSAGES, json).apply() } /** * Load chat messages from SharedPreferences */ fun loadChatMessages(context: Context): List { - val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) - val json = prefs.getString(KEY_CHAT_MESSAGES, null) ?: return emptyList() - - val listType: Type = object : TypeToken>() {}.type + val json = prefs(context).getString(KEY_CHAT_MESSAGES, null) ?: return emptyList() return try { - gson.fromJson(json, listType) + gson.fromJson(json, messageListType) } catch (e: Exception) { emptyList() } @@ -49,9 +41,6 @@ object ChatHistoryPreferences { * Clear all chat messages from SharedPreferences */ fun clearChatMessages(context: Context) { - val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) - val editor = prefs.edit() - editor.remove(KEY_CHAT_MESSAGES) - editor.apply() + prefs(context).edit().remove(KEY_CHAT_MESSAGES).apply() } } diff --git a/app/src/main/kotlin/com/google/ai/sample/util/Command.kt b/app/src/main/kotlin/com/google/ai/sample/util/Command.kt new file mode 100644 index 00000000..049d4bf3 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/util/Command.kt @@ -0,0 +1,27 @@ +package com.google.ai.sample.util + +/** + * Sealed class representing different types of commands. + */ +sealed class Command { + data class ClickButton(val buttonText: String) : Command() + data class LongClickButton(val buttonText: String) : Command() + data class TapCoordinates(val x: String, val y: String) : Command() + object TakeScreenshot : Command() + object PressHomeButton : Command() + object PressBackButton : Command() + object ShowRecentApps : Command() + object ScrollDown : Command() + object ScrollUp : Command() + object ScrollLeft : Command() + object PressEnterKey : Command() + object ScrollRight : Command() + data class ScrollDownFromCoordinates(val x: String, val y: String, val distance: String, val duration: Long) : Command() + data class ScrollUpFromCoordinates(val x: String, val y: String, val distance: String, val duration: Long) : Command() + data class ScrollLeftFromCoordinates(val x: String, val y: String, val distance: String, val duration: Long) : Command() + data class ScrollRightFromCoordinates(val x: String, val y: String, val distance: String, val duration: Long) : Command() + data class OpenApp(val packageName: String) : Command() + data class WriteText(val text: String) : Command() + object UseHighReasoningModel : Command() + object UseLowReasoningModel : Command() +} diff --git a/app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt b/app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt index a01d4d24..85978294 100644 --- a/app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt +++ b/app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt @@ -7,6 +7,7 @@ import android.util.Log */ object CommandParser { private const val TAG = "CommandParser" + private val SINGLE_INSTANCE_COMMAND_TYPES = setOf(CommandTypeEnum.TAKE_SCREENSHOT) // Enum to represent different command types private enum class CommandTypeEnum { @@ -25,6 +26,12 @@ object CommandParser { val commandBuilder: (MatchResult) -> Command, val commandType: CommandTypeEnum // Used for single-instance command check ) + private data class ProcessedMatch( + val startIndex: Int, + val endIndex: Int, + val command: Command, + val commandType: CommandTypeEnum + ) // Master list of all patterns private val ALL_PATTERNS: List = listOf( @@ -92,8 +99,7 @@ object CommandParser { */ @Synchronized fun parseCommands(text: String, clearBuffer: Boolean = false): List { - val commands = mutableListOf() - + var commands: List = emptyList() try { resetBufferIfNeeded(clearBuffer) @@ -107,7 +113,7 @@ object CommandParser { Log.d(TAG, "Current buffer for command parsing: $textBuffer") // Process each line and the combined buffer - processText(textBuffer, commands) + commands = processTextInternal(textBuffer) // If we found commands, clear the buffer for next time if (commands.isNotEmpty()) { @@ -163,57 +169,23 @@ object CommandParser { * Process text to find commands */ private fun processTextInternal(text: String): List { - data class ProcessedMatch(val startIndex: Int, val endIndex: Int, val command: Command, val type: CommandTypeEnum) - val foundRawMatches = mutableListOf() + val foundRawMatches = collectRawMatches(text) val finalCommands = mutableListOf() val addedSingleInstanceCommands = mutableSetOf() - for (patternInfo in ALL_PATTERNS) { - try { - patternInfo.regex.findAll(text).forEach { matchResult -> - try { - val command = patternInfo.commandBuilder(matchResult) - // Store the commandType from the patternInfo that generated this command - foundRawMatches.add(ProcessedMatch(matchResult.range.first, matchResult.range.last, command, patternInfo.commandType)) - Log.d(TAG, "Found raw match: Start=${matchResult.range.first}, End=${matchResult.range.last}, Command=${command}, Type=${patternInfo.commandType}, Pattern=${patternInfo.id}") - } catch (e: Exception) { - Log.e(TAG, "Error building command for pattern ${patternInfo.id} with match ${matchResult.value}: ${e.message}", e) - } - } - } catch (e: Exception) { - Log.e(TAG, "Error finding matches for pattern ${patternInfo.id}: ${e.message}", e) - } - } - // Sort matches by start index foundRawMatches.sortBy { it.startIndex } Log.d(TAG, "Sorted raw matches (${foundRawMatches.size}): $foundRawMatches") var currentPosition = 0 for (processedMatch in foundRawMatches) { - val (startIndex, endIndex, command, commandTypeFromMatch) = processedMatch // Destructure + val (startIndex, endIndex, command, commandType) = processedMatch if (startIndex >= currentPosition) { - var canAdd = true - // Use commandTypeFromMatch directly here - val isSingleInstanceType = when (commandTypeFromMatch) { - CommandTypeEnum.TAKE_SCREENSHOT -> true // Only TakeScreenshot is single-instance - else -> false - } - if (isSingleInstanceType) { - if (addedSingleInstanceCommands.contains(commandTypeFromMatch)) { - canAdd = false - Log.d(TAG, "Skipping duplicate single-instance command: $command (Type: $commandTypeFromMatch)") - } else { - addedSingleInstanceCommands.add(commandTypeFromMatch) - } - } - - if (canAdd) { - // Simplified duplicate check: if it's not a single instance type, allow it. - // More sophisticated duplicate checks for parameterized commands can be added here if needed. - // For now, only single-instance types are strictly controlled for duplication. - // The overlap filter (startIndex >= currentPosition) already prevents identical commands - // from the exact same text span. + val isSingleInstanceDuplicate = commandType in SINGLE_INSTANCE_COMMAND_TYPES && + !addedSingleInstanceCommands.add(commandType) + if (isSingleInstanceDuplicate) { + Log.d(TAG, "Skipping duplicate single-instance command: $command (Type: $commandType)") + } else { finalCommands.add(command) currentPosition = endIndex + 1 Log.d(TAG, "Added command: $command. New currentPosition: $currentPosition") @@ -226,12 +198,38 @@ object CommandParser { return finalCommands } - /** - * Process text to find commands - */ - private fun processText(text: String, commands: MutableList) { - val extractedCommands = processTextInternal(text) - commands.addAll(extractedCommands) + private fun collectRawMatches(text: String): MutableList { + val foundRawMatches = mutableListOf() + for (patternInfo in ALL_PATTERNS) { + try { + patternInfo.regex.findAll(text).forEach { matchResult -> + try { + val command = patternInfo.commandBuilder(matchResult) + foundRawMatches.add( + ProcessedMatch( + startIndex = matchResult.range.first, + endIndex = matchResult.range.last, + command = command, + commandType = patternInfo.commandType + ) + ) + Log.d( + TAG, + "Found raw match: Start=${matchResult.range.first}, End=${matchResult.range.last}, Command=$command, Type=${patternInfo.commandType}, Pattern=${patternInfo.id}" + ) + } catch (e: Exception) { + Log.e( + TAG, + "Error building command for pattern ${patternInfo.id} with match ${matchResult.value}: ${e.message}", + e + ) + } + } + } catch (e: Exception) { + Log.e(TAG, "Error finding matches for pattern ${patternInfo.id}: ${e.message}", e) + } + } + return foundRawMatches } /** @@ -279,108 +277,3 @@ object CommandParser { return textBuffer } } - -/** - * Sealed class representing different types of commands - */ -sealed class Command { - /** - * Command to click a button with the specified text - */ - data class ClickButton(val buttonText: String) : Command() - - /** - * Command to long click a button with the specified text - */ - data class LongClickButton(val buttonText: String) : Command() - - /** - * Command to tap at the specified coordinates - */ - data class TapCoordinates(val x: String, val y: String) : Command() - - /** - * Command to take a screenshot - */ - object TakeScreenshot : Command() - - /** - * Command to press the home button - */ - object PressHomeButton : Command() - - /** - * Command to press the back button - */ - object PressBackButton : Command() - - /** - * Command to show recent apps - */ - object ShowRecentApps : Command() - - /** - * Command to scroll down - */ - object ScrollDown : Command() - - /** - * Command to scroll up - */ - object ScrollUp : Command() - - /** - * Command to scroll left - */ - object ScrollLeft : Command() - - /** - * Command to press the Enter key - */ - object PressEnterKey : Command() - - /** - * Command to scroll right - */ - object ScrollRight : Command() - - /** - * Command to scroll down from specific coordinates with custom distance and duration - */ - data class ScrollDownFromCoordinates(val x: String, val y: String, val distance: String, val duration: Long) : Command() - - /** - * Command to scroll up from specific coordinates with custom distance and duration - */ - data class ScrollUpFromCoordinates(val x: String, val y: String, val distance: String, val duration: Long) : Command() - - /** - * Command to scroll left from specific coordinates with custom distance and duration - */ - data class ScrollLeftFromCoordinates(val x: String, val y: String, val distance: String, val duration: Long) : Command() - - /** - * Command to scroll right from specific coordinates with custom distance and duration - */ - data class ScrollRightFromCoordinates(val x: String, val y: String, val distance: String, val duration: Long) : Command() - - /** - * Command to open an app by package name - */ - data class OpenApp(val packageName: String) : Command() - - /** - * Command to write text into the currently focused text field - */ - data class WriteText(val text: String) : Command() - - /** - * Command to switch to high reasoning model (gemini-2.5-pro-preview-03-25) - */ - object UseHighReasoningModel : Command() - - /** - * Command to switch to low reasoning model (gemini-2.0-flash-lite) - */ - object UseLowReasoningModel : Command() -} diff --git a/app/src/main/kotlin/com/google/ai/sample/util/CoordinateParser.kt b/app/src/main/kotlin/com/google/ai/sample/util/CoordinateParser.kt index 0a5a9ecb..06b1940d 100644 --- a/app/src/main/kotlin/com/google/ai/sample/util/CoordinateParser.kt +++ b/app/src/main/kotlin/com/google/ai/sample/util/CoordinateParser.kt @@ -3,14 +3,15 @@ package com.google.ai.sample.util object CoordinateParser { fun parse(coordinateString: String, screenSize: Int): Float { return try { - if (coordinateString.endsWith("%")) { - val numericValue = coordinateString.removeSuffix("%").toFloat() - (numericValue / 100.0f) * screenSize - } else { - coordinateString.toFloat() - } + parseInternal(coordinateString, screenSize) } catch (e: NumberFormatException) { throw NumberFormatException("Invalid coordinate string: '$coordinateString'. Expected a number or percentage.") } } + + private fun parseInternal(coordinateString: String, screenSize: Int): Float { + if (!coordinateString.endsWith("%")) return coordinateString.toFloat() + val numericValue = coordinateString.removeSuffix("%").toFloat() + return (numericValue / 100.0f) * screenSize + } } diff --git a/app/src/main/kotlin/com/google/ai/sample/util/GenerationSettingsPreferences.kt b/app/src/main/kotlin/com/google/ai/sample/util/GenerationSettingsPreferences.kt index f16f0595..a13b8db2 100644 --- a/app/src/main/kotlin/com/google/ai/sample/util/GenerationSettingsPreferences.kt +++ b/app/src/main/kotlin/com/google/ai/sample/util/GenerationSettingsPreferences.kt @@ -9,6 +9,9 @@ import android.util.Log object GenerationSettingsPreferences { private const val TAG = "GenSettingsPrefs" private const val PREFS_NAME = "generation_settings" + private const val KEY_TEMPERATURE_SUFFIX = "_temperature" + private const val KEY_TOP_P_SUFFIX = "_topP" + private const val KEY_TOP_K_SUFFIX = "_topK" data class GenerationSettings( val temperature: Float = 0.0f, @@ -16,22 +19,24 @@ object GenerationSettingsPreferences { val topK: Int = 0 ) + private fun key(modelName: String, suffix: String) = "$modelName$suffix" + private fun prefs(context: Context) = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) + fun saveSettings(context: Context, modelName: String, settings: GenerationSettings) { - val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) - prefs.edit() - .putFloat("${modelName}_temperature", settings.temperature) - .putFloat("${modelName}_topP", settings.topP) - .putInt("${modelName}_topK", settings.topK) + prefs(context).edit() + .putFloat(key(modelName, KEY_TEMPERATURE_SUFFIX), settings.temperature) + .putFloat(key(modelName, KEY_TOP_P_SUFFIX), settings.topP) + .putInt(key(modelName, KEY_TOP_K_SUFFIX), settings.topK) .apply() Log.d(TAG, "Saved settings for $modelName: temp=${settings.temperature}, topP=${settings.topP}, topK=${settings.topK}") } fun loadSettings(context: Context, modelName: String): GenerationSettings { - val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) + val prefs = prefs(context) return GenerationSettings( - temperature = prefs.getFloat("${modelName}_temperature", 0.0f), - topP = prefs.getFloat("${modelName}_topP", 0.0f), - topK = prefs.getInt("${modelName}_topK", 0) + temperature = prefs.getFloat(key(modelName, KEY_TEMPERATURE_SUFFIX), 0.0f), + topP = prefs.getFloat(key(modelName, KEY_TOP_P_SUFFIX), 0.0f), + topK = prefs.getInt(key(modelName, KEY_TOP_K_SUFFIX), 0) ) } } diff --git a/app/src/main/kotlin/com/google/ai/sample/util/ImageUtils.kt b/app/src/main/kotlin/com/google/ai/sample/util/ImageUtils.kt index 3477fc8d..736dfe3c 100644 --- a/app/src/main/kotlin/com/google/ai/sample/util/ImageUtils.kt +++ b/app/src/main/kotlin/com/google/ai/sample/util/ImageUtils.kt @@ -1,4 +1,4 @@ -package com.google.ai.sample.util // Or your chosen utility package +package com.google.ai.sample.util import android.content.Context import android.graphics.Bitmap @@ -10,17 +10,17 @@ import java.io.File import java.io.FileOutputStream object ImageUtils { + private const val TAG = "ImageUtils" + private const val TEMP_IMAGE_DIR = "image_parts" + private const val TEMP_IMAGE_PREFIX = "temp_image_" + private const val IMAGE_EXTENSION = ".png" + private const val PNG_QUALITY = 100 fun bitmapToBase64(bitmap: Bitmap, @Suppress("UNUSED_PARAMETER") quality: Int = 80): String { val outputStream = ByteArrayOutputStream() - // Compress format can be PNG for lossless (but larger) or JPEG for lossy (smaller) - // PNG is generally safer for AI models if exact pixel data matters. - // Let's use PNG as it's lossless and doesn't require quality for compression itself in the same way JPEG does. - // For PNG, the 'quality' parameter is ignored by compress(). - // If JPEG was used: bitmap.compress(Bitmap.CompressFormat.JPEG, safeQuality, outputStream) - bitmap.compress(Bitmap.CompressFormat.PNG, 100, outputStream) // Quality is ignored for PNG + bitmap.compress(Bitmap.CompressFormat.PNG, PNG_QUALITY, outputStream) val byteArray = outputStream.toByteArray() - return Base64.encodeToString(byteArray, Base64.NO_WRAP) // NO_WRAP is important for compact string + return Base64.encodeToString(byteArray, Base64.NO_WRAP) } fun base64ToBitmap(base64String: String): Bitmap? { @@ -28,31 +28,29 @@ object ImageUtils { val decodedBytes = Base64.decode(base64String, Base64.DEFAULT) BitmapFactory.decodeByteArray(decodedBytes, 0, decodedBytes.size) } catch (e: IllegalArgumentException) { - // Log error or handle - Base64 string might be invalid - android.util.Log.e("ImageUtils", "Error decoding Base64 string to Bitmap: ${e.message}") + Log.e(TAG, "Error decoding Base64 string to Bitmap: ${e.message}") null } catch (e: Exception) { - android.util.Log.e("ImageUtils", "Unexpected error decoding Base64 string to Bitmap: ${e.message}") + Log.e(TAG, "Unexpected error decoding Base64 string to Bitmap: ${e.message}") null } } fun saveBitmapToTempFile(context: Context, bitmap: Bitmap): String? { return try { - val cacheDir = File(context.cacheDir, "image_parts") - cacheDir.mkdirs() // Ensure the directory exists + val cacheDir = File(context.cacheDir, TEMP_IMAGE_DIR) + cacheDir.mkdirs() - // Create a unique filename - val fileName = "temp_image_${System.currentTimeMillis()}.png" + val fileName = "$TEMP_IMAGE_PREFIX${System.currentTimeMillis()}$IMAGE_EXTENSION" val tempFile = File(cacheDir, fileName) FileOutputStream(tempFile).use { fos -> - bitmap.compress(Bitmap.CompressFormat.PNG, 100, fos) + bitmap.compress(Bitmap.CompressFormat.PNG, PNG_QUALITY, fos) } - Log.d("ImageUtils", "Bitmap saved to temp file: ${tempFile.absolutePath}") + Log.d(TAG, "Bitmap saved to temp file: ${tempFile.absolutePath}") tempFile.absolutePath } catch (e: Exception) { - Log.e("ImageUtils", "Error saving bitmap to temp file: ${e.message}", e) + Log.e(TAG, "Error saving bitmap to temp file: ${e.message}", e) null } } @@ -60,12 +58,12 @@ object ImageUtils { fun loadBitmapFromFile(filePath: String): Bitmap? { return try { if (!File(filePath).exists()) { - Log.e("ImageUtils", "File not found for loading bitmap: $filePath") + Log.e(TAG, "File not found for loading bitmap: $filePath") return null } BitmapFactory.decodeFile(filePath) } catch (e: Exception) { - Log.e("ImageUtils", "Error loading bitmap from file: $filePath", e) + Log.e(TAG, "Error loading bitmap from file: $filePath", e) null } } @@ -76,17 +74,17 @@ object ImageUtils { if (file.exists()) { val deleted = file.delete() if (deleted) { - Log.d("ImageUtils", "Successfully deleted file: $filePath") + Log.d(TAG, "Successfully deleted file: $filePath") } else { - Log.w("ImageUtils", "Failed to delete file: $filePath") + Log.w(TAG, "Failed to delete file: $filePath") } deleted } else { - Log.w("ImageUtils", "File not found for deletion: $filePath") - false // Or true if "not existing" means "successfully not there" + Log.w(TAG, "File not found for deletion: $filePath") + false } } catch (e: Exception) { - Log.e("ImageUtils", "Error deleting file: $filePath", e) + Log.e(TAG, "Error deleting file: $filePath", e) false } } diff --git a/app/src/main/kotlin/com/google/ai/sample/util/StringSimilarity.kt b/app/src/main/kotlin/com/google/ai/sample/util/StringSimilarity.kt new file mode 100644 index 00000000..09f0235d --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/util/StringSimilarity.kt @@ -0,0 +1,35 @@ +package com.google.ai.sample.util + +internal object StringSimilarity { + fun calculateMatchScore(query: String, target: String): Int { + if (query == target) return 100 + if (target.contains(query)) return 90 + if (query.contains(target)) return 80 + + val distance = levenshteinDistance(query, target) + val maxLength = maxOf(query.length, target.length) + val similarity = ((maxLength - distance) / maxLength.toFloat()) * 100 + return similarity.toInt() + } + + private fun levenshteinDistance(s1: String, s2: String): Int { + val m = s1.length + val n = s2.length + val dp = Array(m + 1) { IntArray(n + 1) } + + for (i in 0..m) dp[i][0] = i + for (j in 0..n) dp[0][j] = j + + for (i in 1..m) { + for (j in 1..n) { + val cost = if (s1[i - 1] == s2[j - 1]) 0 else 1 + dp[i][j] = minOf( + dp[i - 1][j] + 1, + dp[i][j - 1] + 1, + dp[i - 1][j - 1] + cost + ) + } + } + return dp[m][n] + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/util/SystemMessageEntryPreferences.kt b/app/src/main/kotlin/com/google/ai/sample/util/SystemMessageEntryPreferences.kt index acadfd82..93e32d54 100644 --- a/app/src/main/kotlin/com/google/ai/sample/util/SystemMessageEntryPreferences.kt +++ b/app/src/main/kotlin/com/google/ai/sample/util/SystemMessageEntryPreferences.kt @@ -3,28 +3,48 @@ package com.google.ai.sample.util import android.content.Context import android.content.SharedPreferences import android.util.Log +import androidx.core.content.edit import kotlinx.serialization.encodeToString -import kotlinx.serialization.json.Json import kotlinx.serialization.builtins.ListSerializer +import kotlinx.serialization.json.Json object SystemMessageEntryPreferences { private const val TAG = "SystemMessageEntryPrefs" private const val PREFS_NAME = "system_message_entry_prefs" private const val KEY_SYSTEM_MESSAGE_ENTRIES = "system_message_entries" - private const val KEY_DEFAULT_DB_ENTRIES_POPULATED = "default_db_entries_populated" // Added constant + private const val KEY_DEFAULT_DB_ENTRIES_POPULATED = "default_db_entries_populated" + private val entryListSerializer = ListSerializer(SystemMessageEntry.serializer()) private fun getSharedPreferences(context: Context): SharedPreferences { return context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) } + private val defaultEntries: List = listOf( + SystemMessageEntry( + title = "Termux", + guide = "To write something in Termux you must be sure the ESC HOME banner is away. If not: back() scrollRight(75%, 99%, 50%, 50) tapAtCoordinates(50%, 99%) this in one message. Check if the banner has disappeared also at the bottom. To show the keyboard in Termux if the banner is gone: tapAtCoordinates(50%, 99%) And you must always Enter() twice.\"" + ), + SystemMessageEntry( + title = "Chromium-based Browser", + guide = "To see more in a screenshot, you may want to consider zooming out. To do this, tap the three vertical dots, select the appropriate location in the menu, and then tap the 'minus' symbol (multiple times). It only works approximately in 10% increments. Press the button that many times in a message to zoom out until 50%. More isn't possible.\"" + ), + SystemMessageEntry( + title = "Miscellaneous", + guide = "Some fields will not be spelled out, such as \"Nach Apps und Spielen su...\". In such a case, you must always enter the full version: clickOnButton(\"Nach Apps und Spielen suchen\")\"" + ), + SystemMessageEntry( + title = "File operations", + guide = "As a VLM, Termux is the fastest way to perform file operations. If it's not installed, use existing file manager.\"" + ) + ) + fun saveEntries(context: Context, entries: List) { try { - val jsonString = Json.encodeToString(ListSerializer(SystemMessageEntry.serializer()), entries) + val jsonString = Json.encodeToString(entryListSerializer, entries) Log.d(TAG, "Saving ${entries.size} entries. First entry title if exists: ${entries.firstOrNull()?.title}.") - // Log.v(TAG, "Saving JSON: $jsonString") // Verbose, uncomment if needed for deep debugging - val editor = getSharedPreferences(context).edit() - editor.putString(KEY_SYSTEM_MESSAGE_ENTRIES, jsonString) - editor.apply() + getSharedPreferences(context).edit { + putString(KEY_SYSTEM_MESSAGE_ENTRIES, jsonString) + } } catch (e: Exception) { Log.e(TAG, "Error saving entries: ${e.message}", e) } @@ -33,44 +53,8 @@ object SystemMessageEntryPreferences { fun loadEntries(context: Context): List { try { val prefs = getSharedPreferences(context) - val defaultsPopulated = prefs.getBoolean(KEY_DEFAULT_DB_ENTRIES_POPULATED, false) - - if (!defaultsPopulated) { - Log.d(TAG, "Default entries not populated. Populating now.") - val defaultEntries = listOf( - SystemMessageEntry( - title = "Termux", - guide = "To write something in Termux you must be sure the ESC HOME banner is away. If not: back() scrollRight(75%, 99%, 50%, 50) tapAtCoordinates(50%, 99%) this in one message. Check if the banner has disappeared also at the bottom. To show the keyboard in Termux if the banner is gone: tapAtCoordinates(50%, 99%) And you must always Enter() twice.\"" - ), - SystemMessageEntry( - title = "Chromium-based Browser", - guide = "To see more in a screenshot, you may want to consider zooming out. To do this, tap the three vertical dots, select the appropriate location in the menu, and then tap the 'minus' symbol (multiple times). It only works approximately in 10% increments. Press the button that many times in a message to zoom out until 50%. More isn't possible.\"" - ), - SystemMessageEntry( - title = "Miscellaneous", - guide = "Some fields will not be spelled out, such as \"Nach Apps und Spielen su...\". In such a case, you must always enter the full version: clickOnButton(\"Nach Apps und Spielen suchen\")\"" - ), - SystemMessageEntry( - title = "File operations", - guide = "As a VLM, Termux is the fastest way to perform file operations. If it's not installed, use existing file manager.\"" - ) - ) - saveEntries(context, defaultEntries) // This saves them to KEY_SYSTEM_MESSAGE_ENTRIES - prefs.edit().putBoolean(KEY_DEFAULT_DB_ENTRIES_POPULATED, true).apply() - Log.d(TAG, "Populated and saved default database entries.") - // The logic will now fall through to load these just-saved entries. - } - - // Existing logic to load entries from KEY_SYSTEM_MESSAGE_ENTRIES: - val jsonString = prefs.getString(KEY_SYSTEM_MESSAGE_ENTRIES, null) - if (jsonString != null) { - // Log.v(TAG, "Loaded JSON: $jsonString") // Verbose - val loadedEntries = Json.decodeFromString(ListSerializer(SystemMessageEntry.serializer()), jsonString) - Log.d(TAG, "Loaded ${loadedEntries.size} entries. First entry title if exists: ${loadedEntries.firstOrNull()?.title}.") - return loadedEntries - } - Log.d(TAG, "No entries found, returning empty list.") - return emptyList() + ensureDefaultEntriesIfNeeded(context, prefs) + return loadPersistedEntries(prefs) } catch (e: Exception) { Log.e(TAG, "Error loading entries: ${e.message}", e) return emptyList() @@ -87,15 +71,13 @@ object SystemMessageEntryPreferences { fun updateEntry(context: Context, oldEntry: SystemMessageEntry, newEntry: SystemMessageEntry) { Log.d(TAG, "Updating entry: OldTitle='${oldEntry.title}', NewTitle='${newEntry.title}'") val entries = loadEntries(context).toMutableList() - val index = entries.indexOfFirst { it.title == oldEntry.title } + val index = entries.indexOfFirst { it.title == oldEntry.title } if (index != -1) { entries[index] = newEntry saveEntries(context, entries) Log.i(TAG, "Entry updated successfully: NewTitle='${newEntry.title}'") } else { Log.w(TAG, "Entry with old title '${oldEntry.title}' not found for update.") - // Optionally, add the new entry if the old one is not found - // addEntry(context, newEntry) } } @@ -110,4 +92,26 @@ object SystemMessageEntryPreferences { Log.w(TAG, "Entry not found for deletion: ${entryToDelete.title}") } } + + private fun ensureDefaultEntriesIfNeeded(context: Context, prefs: SharedPreferences) { + val defaultsPopulated = prefs.getBoolean(KEY_DEFAULT_DB_ENTRIES_POPULATED, false) + if (defaultsPopulated) return + + Log.d(TAG, "Default entries not populated. Populating now.") + saveEntries(context, defaultEntries) + prefs.edit { putBoolean(KEY_DEFAULT_DB_ENTRIES_POPULATED, true) } + Log.d(TAG, "Populated and saved default database entries.") + } + + private fun loadPersistedEntries(prefs: SharedPreferences): List { + val jsonString = prefs.getString(KEY_SYSTEM_MESSAGE_ENTRIES, null) + if (jsonString == null) { + Log.d(TAG, "No entries found, returning empty list.") + return emptyList() + } + + val loadedEntries = Json.decodeFromString(entryListSerializer, jsonString) + Log.d(TAG, "Loaded ${loadedEntries.size} entries. First entry title if exists: ${loadedEntries.firstOrNull()?.title}.") + return loadedEntries + } } diff --git a/app/src/main/kotlin/com/google/ai/sample/util/SystemMessagePreferences.kt b/app/src/main/kotlin/com/google/ai/sample/util/SystemMessagePreferences.kt index 0b81c359..71b87cef 100644 --- a/app/src/main/kotlin/com/google/ai/sample/util/SystemMessagePreferences.kt +++ b/app/src/main/kotlin/com/google/ai/sample/util/SystemMessagePreferences.kt @@ -1,8 +1,8 @@ package com.google.ai.sample.util import android.content.Context -import android.content.SharedPreferences import android.util.Log +import androidx.core.content.edit /** * Utility class to manage system message persistence @@ -15,6 +15,7 @@ object SystemMessagePreferences { // Content from pasted_content.txt private const val DEFAULT_SYSTEM_MESSAGE_ON_FIRST_START = """You are on an App on a Smartphone. Your app is called Screen Operator. You start from this app. Proceed step by step! DON'T USE TOOL CODE! You must operate the screen with exactly following commands: "home()" "back()" "recentApps()" "openApp("sample")" for buttons and words: "click("sample")" "longClick("sample")" "tapAtCoordinates(x, y)" "tapAtCoordinates(x percent of screen%, y percent of screen%)" "scrollDown()" "scrollUp()" "scrollLeft()" "scrollRight()" "scrollDown(x, y, how much pixel to scroll, duration in milliseconds)" "scrollUp(x, y, how much pixel to scroll, duration in milliseconds)" "scrollLeft(x, y, how much pixel to scroll, duration in milliseconds)" "scrollRight(x, y, how much pixel to scroll, duration in milliseconds)" "scrollDown(x percent of screen%, y percent of screen%, how much percent to scroll%, duration in milliseconds)" "scrollUp(x percent of screen%, y percent of screen%, how much percent to scroll, duration in milliseconds)" "scrollLeft(x percent of screen%, y percent of screen%, how much percent to scroll, duration in milliseconds)" "scrollRight(x percent of screen%, y percent of screen%, how much percent to scroll, duration in milliseconds)" scroll status bar down: "scrollUp(540, 0, 1100, 50)" "takeScreenshot()" To write text, search and click the textfield thereafter: "writeText("sample text")" You need to write the already existing text, if it should continue exist. If the keyboard is displayed, you can press "Enter()". Otherwise, you have to open the keyboard by clicking on the text field. You can see the screen and get additional Informations about them with: "takeScreenshot()" You need this command at the end of every message until you are finish. When you're done don't say "takeScreenshot()" Your task is:""" + private fun prefs(context: Context) = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) /** * Save system message to SharedPreferences @@ -22,10 +23,7 @@ object SystemMessagePreferences { fun saveSystemMessage(context: Context, message: String) { try { Log.d(TAG, "Saving system message: $message") - val sharedPreferences = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) - val editor = sharedPreferences.edit() - editor.putString(KEY_SYSTEM_MESSAGE, message) - editor.apply() + prefs(context).edit { putString(KEY_SYSTEM_MESSAGE, message) } } catch (e: Exception) { Log.e(TAG, "Error saving system message: ${e.message}", e) } @@ -37,16 +35,15 @@ object SystemMessagePreferences { */ fun loadSystemMessage(context: Context): String { try { - val sharedPreferences = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) + val sharedPreferences = prefs(context) val isFirstStartCompleted = sharedPreferences.getBoolean(KEY_FIRST_START_COMPLETED, false) if (!isFirstStartCompleted) { Log.d(TAG, "First start detected. Loading and saving default system message.") - // Save the default message and the flag - val editor = sharedPreferences.edit() - editor.putString(KEY_SYSTEM_MESSAGE, DEFAULT_SYSTEM_MESSAGE_ON_FIRST_START) - editor.putBoolean(KEY_FIRST_START_COMPLETED, true) - editor.apply() + sharedPreferences.edit { + putString(KEY_SYSTEM_MESSAGE, DEFAULT_SYSTEM_MESSAGE_ON_FIRST_START) + putBoolean(KEY_FIRST_START_COMPLETED, true) + } Log.d(TAG, "Loaded default system message: $DEFAULT_SYSTEM_MESSAGE_ON_FIRST_START") return DEFAULT_SYSTEM_MESSAGE_ON_FIRST_START } else { @@ -67,4 +64,3 @@ object SystemMessagePreferences { return DEFAULT_SYSTEM_MESSAGE_ON_FIRST_START } } - diff --git a/app/src/main/kotlin/com/google/ai/sample/util/UserInputPreferences.kt b/app/src/main/kotlin/com/google/ai/sample/util/UserInputPreferences.kt index 57950dcf..fcc39033 100644 --- a/app/src/main/kotlin/com/google/ai/sample/util/UserInputPreferences.kt +++ b/app/src/main/kotlin/com/google/ai/sample/util/UserInputPreferences.kt @@ -5,14 +5,13 @@ import android.content.Context object UserInputPreferences { private const val PREFS_NAME = "UserInputPrefs" private const val KEY_USER_INPUT = "user_input" + private fun prefs(context: Context) = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) fun saveUserInput(context: Context, text: String) { - val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) - prefs.edit().putString(KEY_USER_INPUT, text).apply() + prefs(context).edit().putString(KEY_USER_INPUT, text).apply() } fun loadUserInput(context: Context): String { - val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) - return prefs.getString(KEY_USER_INPUT, "") ?: "" + return prefs(context).getString(KEY_USER_INPUT, "") ?: "" } } From 5d0cadf85ef9fcee058dc80e1775f3066ea4fc43 Mon Sep 17 00:00:00 2001 From: Android PowerUser <88908510+Android-PowerUser@users.noreply.github.com> Date: Sun, 29 Mar 2026 19:15:37 +0200 Subject: [PATCH 2/4] Update app/src/main/kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt Co-authored-by: amazon-q-developer[bot] <208079219+amazon-q-developer[bot]@users.noreply.github.com> --- .../kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/app/src/main/kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt b/app/src/main/kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt index 2fb7c2ab..6c7aec7e 100644 --- a/app/src/main/kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt +++ b/app/src/main/kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt @@ -8,6 +8,7 @@ internal class AccessibilityCommandQueue { private val queue = LinkedList() private val processing = AtomicBoolean(false) + @Synchronized fun clearAndUnlock() { queue.clear() processing.set(false) From 8c80c2ff2c7a11c0fa5a9ccf116cd21a1e2467a5 Mon Sep 17 00:00:00 2001 From: Android PowerUser <88908510+Android-PowerUser@users.noreply.github.com> Date: Sun, 29 Mar 2026 19:16:02 +0200 Subject: [PATCH 3/4] Update app/src/main/kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt Co-authored-by: amazon-q-developer[bot] <208079219+amazon-q-developer[bot]@users.noreply.github.com> --- .../kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/app/src/main/kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt b/app/src/main/kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt index 6c7aec7e..6cba38b6 100644 --- a/app/src/main/kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt +++ b/app/src/main/kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt @@ -14,6 +14,7 @@ internal class AccessibilityCommandQueue { processing.set(false) } + @Synchronized fun enqueue(command: Command): Int { queue.add(command) return queue.size From 5e70206918310b377e68de91b1635feb569dadd9 Mon Sep 17 00:00:00 2001 From: Android PowerUser <88908510+Android-PowerUser@users.noreply.github.com> Date: Sun, 29 Mar 2026 19:16:16 +0200 Subject: [PATCH 4/4] Update app/src/main/kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt Co-authored-by: amazon-q-developer[bot] <208079219+amazon-q-developer[bot]@users.noreply.github.com> --- .../com/google/ai/sample/AccessibilityCommandQueue.kt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/src/main/kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt b/app/src/main/kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt index 6cba38b6..9a42b29a 100644 --- a/app/src/main/kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt +++ b/app/src/main/kotlin/com/google/ai/sample/AccessibilityCommandQueue.kt @@ -20,9 +20,16 @@ internal class AccessibilityCommandQueue { return queue.size } + @Synchronized fun peek(): Command? = queue.peek() + + @Synchronized fun poll(): Command? = queue.poll() + + @Synchronized fun size(): Int = queue.size + + @Synchronized fun isEmpty(): Boolean = queue.isEmpty() fun tryAcquireProcessing(): Boolean = processing.compareAndSet(false, true) fun releaseProcessing() = processing.set(false)