diff --git a/app/src/main/kotlin/com/google/ai/sample/AccessibilityServiceStatusResolver.kt b/app/src/main/kotlin/com/google/ai/sample/AccessibilityServiceStatusResolver.kt new file mode 100644 index 0000000..ecff0dc --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/AccessibilityServiceStatusResolver.kt @@ -0,0 +1,15 @@ +package com.google.ai.sample + +import android.content.ContentResolver +import android.provider.Settings + +internal object AccessibilityServiceStatusResolver { + fun isServiceEnabled(contentResolver: ContentResolver, packageName: String): Boolean { + val service = "$packageName/${ScreenOperatorAccessibilityService::class.java.canonicalName}" + val enabledServices = Settings.Secure.getString( + contentResolver, + Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES + ) + return enabledServices?.contains(service, ignoreCase = true) == true + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/MainActivity.kt b/app/src/main/kotlin/com/google/ai/sample/MainActivity.kt index 9a0852f..d05dbfa 100644 --- a/app/src/main/kotlin/com/google/ai/sample/MainActivity.kt +++ b/app/src/main/kotlin/com/google/ai/sample/MainActivity.kt @@ -155,24 +155,28 @@ class MainActivity : ComponentActivity() { // Permission Launchers private lateinit var requestNotificationPermissionLauncher: ActivityResultLauncher private lateinit var requestForegroundServicePermissionLauncher: ActivityResultLauncher + private val foregroundMediaProjectionPermission = android.Manifest.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION fun requestMediaProjectionPermission(onGranted: (() -> Unit)? = null) { Log.d(TAG, "Requesting MediaProjection permission") onMediaProjectionPermissionGranted = onGranted - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - if (ContextCompat.checkSelfPermission(this, android.Manifest.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION) != PackageManager.PERMISSION_GRANTED) { - requestForegroundServicePermissionLauncher.launch(android.Manifest.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION) - } - } + requestForegroundMediaProjectionPermissionIfMissing() - // Ensure mediaProjectionManager is initialized before using it. - // This should be guaranteed by its placement in onCreate. - if (!isMediaProjectionManagerInitialized("requestMediaProjectionPermission")) { - return + launchCaptureIntent(mediaProjectionLauncher, "requestMediaProjectionPermission") + } + + private fun hasForegroundMediaProjectionPermission(): Boolean { + return Build.VERSION.SDK_INT < Build.VERSION_CODES.Q || + ContextCompat.checkSelfPermission(this, foregroundMediaProjectionPermission) == PackageManager.PERMISSION_GRANTED + } + + private fun requestForegroundMediaProjectionPermissionIfMissing(): Boolean { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && !hasForegroundMediaProjectionPermission()) { + requestForegroundServicePermissionLauncher.launch(foregroundMediaProjectionPermission) + return true } - val intent = mediaProjectionManager.createScreenCaptureIntent() - mediaProjectionLauncher.launch(intent) + return false } /** @@ -183,11 +187,7 @@ class MainActivity : ComponentActivity() { Log.d(TAG, "Requesting MediaProjection permission for WebRTC") onWebRtcMediaProjectionResult = onResult - if (!isMediaProjectionManagerInitialized("requestMediaProjectionForWebRTC")) { - return - } - val intent = mediaProjectionManager.createScreenCaptureIntent() - webRtcMediaProjectionLauncher.launch(intent) + launchCaptureIntent(webRtcMediaProjectionLauncher, "requestMediaProjectionForWebRTC") } private fun initializeMediaProjection() { @@ -210,63 +210,65 @@ class MainActivity : ComponentActivity() { private fun handleMediaProjectionResult(resultCode: Int, resultData: Intent?) { if (resultCode == Activity.RESULT_OK && resultData != null) { - val shouldTakeScreenshotOnThisStart = isProcessingExplicitScreenshotRequest - Log.i(TAG, "MediaProjection permission granted. Starting ScreenCaptureService. Explicit request: $shouldTakeScreenshotOnThisStart") + handleMediaProjectionPermissionGranted(resultCode, resultData) + } else { + handleMediaProjectionPermissionDenied() + } + } - photoReasoningViewModel?.onMediaProjectionPermissionGranted(resultCode, resultData) + private fun handleMediaProjectionPermissionGranted(resultCode: Int, resultData: Intent) { + val shouldTakeScreenshotOnThisStart = isProcessingExplicitScreenshotRequest + Log.i(TAG, "MediaProjection permission granted. Starting ScreenCaptureService. Explicit request: $shouldTakeScreenshotOnThisStart") - val serviceIntent = Intent(this, ScreenCaptureService::class.java).apply { - action = ScreenCaptureService.ACTION_START_CAPTURE - putExtra(ScreenCaptureService.EXTRA_RESULT_CODE, resultCode) - putExtra(ScreenCaptureService.EXTRA_RESULT_DATA, resultData) - putExtra(ScreenCaptureService.EXTRA_TAKE_SCREENSHOT_ON_START, shouldTakeScreenshotOnThisStart) - } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - if (ContextCompat.checkSelfPermission(this, android.Manifest.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION) == PackageManager.PERMISSION_GRANTED) { - mediaProjectionServiceStarter.start(serviceIntent) - } else { - requestForegroundServicePermissionLauncher.launch(android.Manifest.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION) - } - } else { - mediaProjectionServiceStarter.start(serviceIntent) - } + photoReasoningViewModel?.onMediaProjectionPermissionGranted(resultCode, resultData) - if (isProcessingExplicitScreenshotRequest) { - Log.d(TAG, "Resetting isProcessingExplicitScreenshotRequest flag after successful explicit grant.") - isProcessingExplicitScreenshotRequest = false - } - _isMediaProjectionPermissionGranted.value = true - onMediaProjectionPermissionGranted?.invoke() - onMediaProjectionPermissionGranted = null - } else { - Log.w(TAG, "MediaProjection permission denied or cancelled by user.") - Toast.makeText(this, "Screen capture permission denied", Toast.LENGTH_SHORT).show() - if (isProcessingExplicitScreenshotRequest) { - Log.d(TAG, "Resetting isProcessingExplicitScreenshotRequest flag after explicit denial.") - isProcessingExplicitScreenshotRequest = false - } - _isMediaProjectionPermissionGranted.value = false + val serviceIntent = MainActivityMediaProjectionIntents.startCapture( + context = this, + resultCode = resultCode, + resultData = resultData, + takeScreenshotOnStart = shouldTakeScreenshotOnThisStart + ) + if (!requestForegroundMediaProjectionPermissionIfMissing()) { + mediaProjectionServiceStarter.start(serviceIntent) } + + resetExplicitScreenshotRequestFlagIfNeeded("successful explicit grant") + _isMediaProjectionPermissionGranted.value = true + onMediaProjectionPermissionGranted?.invoke() + onMediaProjectionPermissionGranted = null + } + + private fun handleMediaProjectionPermissionDenied() { + Log.w(TAG, "MediaProjection permission denied or cancelled by user.") + Toast.makeText(this, "Screen capture permission denied", Toast.LENGTH_SHORT).show() + resetExplicitScreenshotRequestFlagIfNeeded("explicit denial") + _isMediaProjectionPermissionGranted.value = false } private fun handleWebRtcMediaProjectionResult(resultCode: Int, resultData: Intent?) { if (resultCode == Activity.RESULT_OK && resultData != null) { - Log.i(TAG, "WebRTC MediaProjection permission granted. Starting keep-alive service.") - - val serviceIntent = Intent(this, ScreenCaptureService::class.java).apply { - action = ScreenCaptureService.ACTION_KEEP_ALIVE_FOR_WEBRTC - } - mediaProjectionServiceStarter.start(serviceIntent) - - onWebRtcMediaProjectionResult?.invoke(resultCode, resultData) - onWebRtcMediaProjectionResult = null + handleWebRtcMediaProjectionPermissionGranted(resultCode, resultData) } else { - Log.w(TAG, "WebRTC MediaProjection permission denied.") - Toast.makeText(this, "Screen capture permission denied", Toast.LENGTH_SHORT).show() - onWebRtcMediaProjectionResult = null + handleWebRtcMediaProjectionPermissionDenied() } } + private fun handleWebRtcMediaProjectionPermissionGranted(resultCode: Int, resultData: Intent) { + Log.i(TAG, "WebRTC MediaProjection permission granted. Starting keep-alive service.") + + val serviceIntent = MainActivityMediaProjectionIntents.keepAliveForWebRtc(this) + mediaProjectionServiceStarter.start(serviceIntent) + + onWebRtcMediaProjectionResult?.invoke(resultCode, resultData) + onWebRtcMediaProjectionResult = null + } + + private fun handleWebRtcMediaProjectionPermissionDenied() { + Log.w(TAG, "WebRTC MediaProjection permission denied.") + Toast.makeText(this, "Screen capture permission denied", Toast.LENGTH_SHORT).show() + onWebRtcMediaProjectionResult = null + } + private fun setupKeyboardVisibilityListener() { val rootView = findViewById(android.R.id.content) keyboardVisibilityObserver.start(rootView, _isKeyboardOpen) @@ -294,8 +296,7 @@ class MainActivity : ComponentActivity() { requestForegroundServicePermissionLauncher = registerForActivityResult(ActivityResultContracts.RequestPermission()) { isGranted: Boolean -> if (isGranted) { Toast.makeText(this, "Foreground service permission granted.", Toast.LENGTH_SHORT).show() - val intent = mediaProjectionManager.createScreenCaptureIntent() - mediaProjectionLauncher.launch(intent) + launchCaptureIntent(mediaProjectionLauncher, "requestForegroundServicePermissionLauncher") } else { Toast.makeText(this, "Foreground service permission denied. The app may not function correctly.", Toast.LENGTH_LONG).show() } @@ -324,23 +325,25 @@ class MainActivity : ComponentActivity() { private fun handleScreenshotRequest(intent: Intent) { Log.d(TAG, "Received request for screenshot via broadcast.") - currentScreenInfoForScreenshot = intent.getStringExtra(EXTRA_SCREEN_INFO) + currentScreenInfoForScreenshot = MainActivityScreenshotIntents.extractScreenInfo(intent) Log.d(TAG, "Stored screenInfo for upcoming screenshot.") - if (ScreenCaptureService.isRunning()) { - Log.d(TAG, "ScreenCaptureService is running. Calling takeAdditionalScreenshot().") - takeAdditionalScreenshot() - } else { - Log.d(TAG, "ScreenCaptureService not running. Calling requestMediaProjectionPermission() to start it.") - isProcessingExplicitScreenshotRequest = true - requestMediaProjectionPermission() + when (MainActivityScreenshotFlowDecider.decide(ScreenCaptureService.isRunning())) { + MainActivityScreenshotFlowDecider.Action.TAKE_ADDITIONAL_SCREENSHOT -> { + Log.d(TAG, "ScreenCaptureService is running. Calling takeAdditionalScreenshot().") + takeAdditionalScreenshot() + } + MainActivityScreenshotFlowDecider.Action.REQUEST_PERMISSION -> { + Log.d(TAG, "ScreenCaptureService not running. Calling requestMediaProjectionPermission() to start it.") + isProcessingExplicitScreenshotRequest = true + requestMediaProjectionPermission() + } } } private fun handleScreenshotResult(intent: Intent) { - val screenshotUriString = intent.getStringExtra(EXTRA_SCREENSHOT_URI) - if (screenshotUriString != null) { - val screenshotUri = Uri.parse(screenshotUriString) + val screenshotUri = MainActivityScreenshotIntents.extractScreenshotUri(intent) + if (screenshotUri != null) { Log.d(TAG, "Received screenshot captured broadcast. URI: $screenshotUri") Log.d(TAG, "Using screenInfo: ${currentScreenInfoForScreenshot?.substring(0, minOf(100, currentScreenInfoForScreenshot?.length ?: 0))}...") @@ -363,12 +366,28 @@ class MainActivity : ComponentActivity() { return true } + private fun launchCaptureIntent( + launcher: ActivityResultLauncher, + caller: String + ) { + if (!isMediaProjectionManagerInitialized(caller)) { + return + } + val intent = mediaProjectionManager.createScreenCaptureIntent() + launcher.launch(intent) + } + + private fun resetExplicitScreenshotRequestFlagIfNeeded(reason: String) { + if (isProcessingExplicitScreenshotRequest) { + Log.d(TAG, "Resetting isProcessingExplicitScreenshotRequest flag after $reason.") + isProcessingExplicitScreenshotRequest = false + } + } + fun takeAdditionalScreenshot() { if (ScreenCaptureService.isRunning()) { Log.d(TAG, "MainActivity: Instructing ScreenCaptureService to take an additional screenshot.") - val intent = Intent(this, ScreenCaptureService::class.java).apply { - action = ScreenCaptureService.ACTION_TAKE_SCREENSHOT - } + val intent = MainActivityMediaProjectionIntents.takeScreenshot(this) // Use startService as the service is already foreground if running. // If it somehow wasn't foreground but running, this still works. startService(intent) @@ -388,9 +407,7 @@ class MainActivity : ComponentActivity() { fun stopScreenCaptureService() { if (ScreenCaptureService.isRunning()) { // Check if it's actually running to avoid errors Log.d(TAG, "MainActivity: Instructing ScreenCaptureService to stop.") - val intent = Intent(this, ScreenCaptureService::class.java).apply { - action = ScreenCaptureService.ACTION_STOP_CAPTURE - } + val intent = MainActivityMediaProjectionIntents.stopCapture(this) startService(intent) } else { Log.d(TAG, "MainActivity: stopScreenCaptureService called, but service was not running.") @@ -446,22 +463,13 @@ class MainActivity : ComponentActivity() { val oldState = currentTrialState currentTrialState = newState Log.i(TAG, "updateTrialState: Trial state updated from $oldState to $currentTrialState") - - when (currentTrialState) { - TrialManager.TrialState.EXPIRED_INTERNET_TIME_CONFIRMED -> { - trialInfoMessage = "Please support the development of the app so that you can continue using it \uD83C\uDF89" - showTrialInfoDialog = true - Log.d(TAG, "updateTrialState: Set message to \'$trialInfoMessage\', showTrialInfoDialog = true (EXPIRED)") - } - TrialManager.TrialState.ACTIVE_INTERNET_TIME_CONFIRMED, - TrialManager.TrialState.PURCHASED, - TrialManager.TrialState.NOT_YET_STARTED_AWAITING_INTERNET, - TrialManager.TrialState.INTERNET_UNAVAILABLE_CANNOT_VERIFY -> { - trialInfoMessage = "" - showTrialInfoDialog = false - Log.d(TAG, "updateTrialState: Cleared message, showTrialInfoDialog = false (ACTIVE, PURCHASED, AWAITING, OR UNAVAILABLE)") - } - } + val uiModel = TrialStateUiModelResolver.resolve(currentTrialState) + trialInfoMessage = uiModel.infoMessage + showTrialInfoDialog = uiModel.shouldShowInfoDialog + Log.d( + TAG, + "updateTrialState: trialInfoMessage='${uiModel.infoMessage}', showTrialInfoDialog=${uiModel.shouldShowInfoDialog}" + ) } private val purchasesUpdatedListener = PurchasesUpdatedListener { billingResult, purchases -> @@ -493,11 +501,12 @@ class MainActivity : ComponentActivity() { internal fun refreshAccessibilityServiceStatus() { Log.d(TAG, "refreshAccessibilityServiceStatus called.") - val service = packageName + "/" + ScreenOperatorAccessibilityService::class.java.canonicalName - val enabledServices = Settings.Secure.getString(contentResolver, Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES) - val isEnabled = enabledServices?.contains(service, ignoreCase = true) == true + val isEnabled = AccessibilityServiceStatusResolver.isServiceEnabled( + contentResolver = contentResolver, + packageName = packageName + ) _isAccessibilityServiceEnabled.value = isEnabled // Update the flow - Log.d(TAG, "Accessibility Service $service isEnabled: $isEnabled. Flow updated.") + Log.d(TAG, "Accessibility Service isEnabled: $isEnabled. Flow updated.") if (!isEnabled) { Log.d(TAG, "Accessibility Service not enabled.") } @@ -508,12 +517,12 @@ class MainActivity : ComponentActivity() { } fun updateStatusMessage(message: String, isError: Boolean = false) { - Toast.makeText(this, message, if (isError) Toast.LENGTH_LONG else Toast.LENGTH_SHORT).show() - if (isError) { - Log.e(TAG, "updateStatusMessage (Error): $message") - } else { - Log.d(TAG, "updateStatusMessage (Info): $message") - } + MainActivityStatusNotifier.showStatusMessage( + context = this, + tag = TAG, + message = message, + isError = isError + ) } fun getPhotoReasoningViewModel(): PhotoReasoningViewModel? { @@ -718,16 +727,11 @@ class MainActivity : ComponentActivity() { } fun hasShownNotificationRationale(): Boolean { - val prefs = getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) - return prefs.getBoolean(KEY_NOTIFICATION_RATIONALE_SHOWN, false) + return NotificationPermissionPreferences.hasShownNotificationRationale(this) } fun setNotificationRationaleShown(shown: Boolean) { - val prefs = getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) - with(prefs.edit()) { - putBoolean(KEY_NOTIFICATION_RATIONALE_SHOWN, shown) - apply() - } + NotificationPermissionPreferences.setNotificationRationaleShown(this, shown) } @Composable @@ -785,7 +789,7 @@ class MainActivity : ComponentActivity() { private fun startTrialServiceIfNeeded() { Log.d(TAG, "startTrialServiceIfNeeded called. Current state: $currentTrialState") - if (currentTrialState != TrialManager.TrialState.PURCHASED && currentTrialState != TrialManager.TrialState.EXPIRED_INTERNET_TIME_CONFIRMED) { + if (MainActivityBillingStateEvaluator.shouldStartTrialService(currentTrialState)) { Log.i(TAG, "Starting TrialTimerService because current state is: $currentTrialState") val serviceIntent = Intent(this, TrialTimerService::class.java) serviceIntent.action = TrialTimerService.ACTION_START_TIMER @@ -802,11 +806,11 @@ class MainActivity : ComponentActivity() { private fun setupBillingClient() { Log.d(TAG, "setupBillingClient called.") - if (::billingClient.isInitialized && billingClient.isReady) { + if (MainActivityBillingClientState.isInitializedAndReady(::billingClient.isInitialized, if (::billingClient.isInitialized) billingClient.isReady else false)) { Log.d(TAG, "setupBillingClient: BillingClient already initialized and ready.") return } - if (::billingClient.isInitialized && billingClient.connectionState == BillingClient.ConnectionState.CONNECTING) { + if (MainActivityBillingClientState.isConnecting(::billingClient.isInitialized, if (::billingClient.isInitialized) billingClient.connectionState else BillingClient.ConnectionState.DISCONNECTED)) { Log.d(TAG, "setupBillingClient: BillingClient already connecting.") return } @@ -842,7 +846,7 @@ class MainActivity : ComponentActivity() { private fun queryProductDetails() { Log.d(TAG, "queryProductDetails called.") - if (!billingClient.isReady) { + if (!MainActivityBillingClientState.isInitializedAndReady(::billingClient.isInitialized, if (::billingClient.isInitialized) billingClient.isReady else false)) { Log.w(TAG, "queryProductDetails: BillingClient not ready. Cannot query.") return } @@ -884,7 +888,7 @@ class MainActivity : ComponentActivity() { if (!billingClient.isReady) { Log.e(TAG, "launchGooglePlayBilling: BillingClient not ready. Connection state: ${billingClient.connectionState}") updateStatusMessage("Payment service not ready. Please try again later.", true) - if (billingClient.connectionState == BillingClient.ConnectionState.CLOSED || billingClient.connectionState == BillingClient.ConnectionState.DISCONNECTED){ + if (MainActivityBillingClientState.shouldReconnect(billingClient.connectionState)) { Log.d(TAG, "launchGooglePlayBilling: BillingClient disconnected, attempting to reconnect.") billingClient.startConnection(object : BillingClientStateListener { override fun onBillingSetupFinished(setupResult: BillingResult) { @@ -944,7 +948,7 @@ class MainActivity : ComponentActivity() { Log.i(TAG, "handlePurchase called for purchase: OrderId: ${purchase.orderId}, Products: ${purchase.products}, State: ${purchase.purchaseState}, Token: ${purchase.purchaseToken}, Ack: ${purchase.isAcknowledged}") if (purchase.purchaseState == Purchase.PurchaseState.PURCHASED) { Log.d(TAG, "handlePurchase: Purchase state is PURCHASED.") - if (purchase.products.any { it == subscriptionProductId }) { + if (MainActivityBillingStateEvaluator.containsSubscriptionProduct(purchase, subscriptionProductId)) { Log.d(TAG, "handlePurchase: Purchase contains target product ID: $subscriptionProductId") if (!purchase.isAcknowledged) { Log.i(TAG, "handlePurchase: Purchase not acknowledged. Acknowledging now.") @@ -986,7 +990,7 @@ class MainActivity : ComponentActivity() { private fun queryActiveSubscriptions() { Log.d(TAG, "queryActiveSubscriptions called.") - if (!::billingClient.isInitialized || !billingClient.isReady) { + if (!MainActivityBillingClientState.isInitializedAndReady(::billingClient.isInitialized, if (::billingClient.isInitialized) billingClient.isReady else false)) { Log.w(TAG, "queryActiveSubscriptions: BillingClient not initialized or not ready. Cannot query. isInitialized: ${::billingClient.isInitialized}, isReady: ${if(::billingClient.isInitialized) billingClient.isReady else "N/A"}") return } @@ -999,7 +1003,7 @@ class MainActivity : ComponentActivity() { if (billingResult.responseCode == BillingClient.BillingResponseCode.OK) { purchases.forEach { purchase -> Log.d(TAG, "queryActiveSubscriptions: Checking purchase - Products: ${purchase.products}, State: ${purchase.purchaseState}") - if (purchase.products.any { it == subscriptionProductId } && purchase.purchaseState == Purchase.PurchaseState.PURCHASED) { + if (MainActivityBillingStateEvaluator.isPurchasedSubscription(purchase, subscriptionProductId)) { Log.i(TAG, "queryActiveSubscriptions: Active subscription found for $subscriptionProductId.") isSubscribedLocally = true if (!purchase.isAcknowledged) { @@ -1132,7 +1136,6 @@ class MainActivity : ComponentActivity() { } private const val PREFS_NAME = "AppPrefs" private const val PREF_KEY_FIRST_LAUNCH_INFO_SHOWN = "firstLaunchInfoShown" - private const val KEY_NOTIFICATION_RATIONALE_SHOWN = "notification_rationale_shown" // New Broadcast Actions for MediaProjection Screenshot Flow const val ACTION_REQUEST_MEDIAPROJECTION_SCREENSHOT = "com.google.ai.sample.REQUEST_MEDIAPROJECTION_SCREENSHOT" @@ -1153,219 +1156,3 @@ class MainActivity : ComponentActivity() { } } } - -@Composable -private fun TrialStateDialogs( - trialState: TrialManager.TrialState, - showTrialInfoDialog: Boolean, - trialInfoMessage: String, - onDismissTrialInfo: () -> Unit, - onPurchaseClick: () -> Unit -) { - when (trialState) { - TrialManager.TrialState.EXPIRED_INTERNET_TIME_CONFIRMED -> { - TrialExpiredDialog( - onPurchaseClick = onPurchaseClick, - onDismiss = {} - ) - } - - TrialManager.TrialState.NOT_YET_STARTED_AWAITING_INTERNET, - TrialManager.TrialState.INTERNET_UNAVAILABLE_CANNOT_VERIFY -> { - if (showTrialInfoDialog) { - InfoDialog( - message = trialInfoMessage, - onDismiss = onDismissTrialInfo - ) - } - } - - TrialManager.TrialState.ACTIVE_INTERNET_TIME_CONFIRMED, - TrialManager.TrialState.PURCHASED -> Unit - } -} - -@Composable -private fun PaymentMethodDialog( - onDismiss: () -> Unit, - onPayPalClick: () -> Unit, - onGooglePlayClick: () -> Unit -) { - androidx.compose.material3.AlertDialog( - onDismissRequest = onDismiss, - title = { Text("Choose Payment Method") }, - text = { - Column { - Button( - onClick = onPayPalClick, - modifier = Modifier.fillMaxWidth().padding(bottom = 8.dp) - ) { - Text("PayPal (2,60 €/Month)") - } - Button( - onClick = onGooglePlayClick, - modifier = Modifier.fillMaxWidth() - ) { - Text("Google Play (2,90 €/Month)") - } - } - }, - confirmButton = {}, - dismissButton = { - TextButton(onClick = onDismiss) { - Text("Cancel") - } - } - ) -} - -@Composable -private fun ApiKeyDialogSection( - apiKeyManager: ApiKeyManager, - isFirstLaunch: Boolean, - initialProvider: ApiProvider?, - onDismiss: () -> Unit -) { - ApiKeyDialog( - apiKeyManager = apiKeyManager, - isFirstLaunch = isFirstLaunch, - initialProvider = initialProvider, - onDismiss = onDismiss - ) -} - -@Composable -fun FirstLaunchInfoDialog(onDismiss: () -> Unit) { - Log.d("FirstLaunchInfoDialog", "Composing FirstLaunchInfoDialog") - Dialog(onDismissRequest = { - Log.d("FirstLaunchInfoDialog", "onDismissRequest called") - onDismiss() - }) { - Card( - modifier = Modifier - .fillMaxWidth() - .padding(16.dp), - ) { - Column( - modifier = Modifier - .padding(16.dp) - .fillMaxWidth(), - verticalArrangement = Arrangement.Center, - horizontalAlignment = Alignment.CenterHorizontally - ) { - Text( - text = "Trial Information", - style = MaterialTheme.typography.titleLarge - ) - Spacer(modifier = Modifier.height(16.dp)) - Text( - text = "You can try Screen Operator for 7 days before you have to subscribe to support the development of more features.", - style = MaterialTheme.typography.bodyMedium, - modifier = Modifier.align(Alignment.CenterHorizontally) - ) - Spacer(modifier = Modifier.height(24.dp)) - TextButton( - onClick = { - Log.d("FirstLaunchInfoDialog", "OK button clicked") - onDismiss() - }, - modifier = Modifier.fillMaxWidth() - ) { - Text("OK") - } - } - } - } -} - - - -@Composable -fun TrialExpiredDialog( - onPurchaseClick: () -> Unit, - @Suppress("UNUSED_PARAMETER") onDismiss: () -> Unit -) { - Log.d("TrialExpiredDialog", "Composing TrialExpiredDialog") - Dialog(onDismissRequest = { - Log.d("TrialExpiredDialog", "onDismissRequest called (persistent dialog - user tried to dismiss)") - }) { - Card( - modifier = Modifier - .fillMaxWidth() - .padding(16.dp), - ) { - Column( - modifier = Modifier - .padding(16.dp) - .fillMaxWidth(), - verticalArrangement = Arrangement.Center, - horizontalAlignment = Alignment.CenterHorizontally - ) { - Text( - text = "Trial period expired", - style = MaterialTheme.typography.titleLarge - ) - Spacer(modifier = Modifier.height(16.dp)) - Text( - text = "Please support the development of the app so that you can continue using it \uD83C\uDF89", - style = MaterialTheme.typography.bodyMedium, - modifier = Modifier.align(Alignment.CenterHorizontally) - ) - Spacer(modifier = Modifier.height(24.dp)) - Button( - onClick = { - Log.d("TrialExpiredDialog", "Purchase button clicked") - onPurchaseClick() - }, - modifier = Modifier.fillMaxWidth() - ) { - Text("Subscribe") - } - } - } - } -} - -@Composable -fun InfoDialog( - message: String, - onDismiss: () -> Unit -) { - Log.d("InfoDialog", "Composing InfoDialog with message: $message") - Dialog(onDismissRequest = { - Log.d("InfoDialog", "onDismissRequest called") - onDismiss() - }) { - Card( - modifier = Modifier - .fillMaxWidth() - .padding(16.dp) - ) { - Column( - modifier = Modifier - .padding(16.dp) - .fillMaxWidth(), - verticalArrangement = Arrangement.Center, - horizontalAlignment = Alignment.CenterHorizontally - ) { - Text( - text = "Information", - style = MaterialTheme.typography.titleMedium - ) - Spacer(modifier = Modifier.height(16.dp)) - Text( - text = message, - style = MaterialTheme.typography.bodyMedium, - modifier = Modifier.align(Alignment.CenterHorizontally) - ) - Spacer(modifier = Modifier.height(24.dp)) - TextButton(onClick = { - Log.d("InfoDialog", "OK button clicked") - onDismiss() - }) { - Text("OK") - } - } - } - } -} diff --git a/app/src/main/kotlin/com/google/ai/sample/MainActivityBillingClientState.kt b/app/src/main/kotlin/com/google/ai/sample/MainActivityBillingClientState.kt new file mode 100644 index 0000000..d779761 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/MainActivityBillingClientState.kt @@ -0,0 +1,18 @@ +package com.google.ai.sample + +import com.android.billingclient.api.BillingClient + +internal object MainActivityBillingClientState { + fun isInitializedAndReady(isInitialized: Boolean, isReady: Boolean): Boolean { + return isInitialized && isReady + } + + fun isConnecting(isInitialized: Boolean, connectionState: Int): Boolean { + return isInitialized && connectionState == BillingClient.ConnectionState.CONNECTING + } + + fun shouldReconnect(connectionState: Int): Boolean { + return connectionState == BillingClient.ConnectionState.CLOSED || + connectionState == BillingClient.ConnectionState.DISCONNECTED + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/MainActivityBillingStateEvaluator.kt b/app/src/main/kotlin/com/google/ai/sample/MainActivityBillingStateEvaluator.kt new file mode 100644 index 0000000..e8b35a0 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/MainActivityBillingStateEvaluator.kt @@ -0,0 +1,19 @@ +package com.google.ai.sample + +import com.android.billingclient.api.Purchase + +internal object MainActivityBillingStateEvaluator { + fun shouldStartTrialService(state: TrialManager.TrialState): Boolean { + return state != TrialManager.TrialState.PURCHASED && + state != TrialManager.TrialState.EXPIRED_INTERNET_TIME_CONFIRMED + } + + fun containsSubscriptionProduct(purchase: Purchase, subscriptionProductId: String): Boolean { + return purchase.products.any { it == subscriptionProductId } + } + + fun isPurchasedSubscription(purchase: Purchase, subscriptionProductId: String): Boolean { + return containsSubscriptionProduct(purchase, subscriptionProductId) && + purchase.purchaseState == Purchase.PurchaseState.PURCHASED + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/MainActivityDialogs.kt b/app/src/main/kotlin/com/google/ai/sample/MainActivityDialogs.kt new file mode 100644 index 0000000..92a9434 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/MainActivityDialogs.kt @@ -0,0 +1,235 @@ +package com.google.ai.sample + +import android.util.Log +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.material3.Button +import androidx.compose.material3.Card +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.material3.TextButton +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.unit.dp +import androidx.compose.ui.window.Dialog + +@Composable +internal fun TrialStateDialogs( + trialState: TrialManager.TrialState, + showTrialInfoDialog: Boolean, + trialInfoMessage: String, + onDismissTrialInfo: () -> Unit, + onPurchaseClick: () -> Unit +) { + when (trialState) { + TrialManager.TrialState.EXPIRED_INTERNET_TIME_CONFIRMED -> { + TrialExpiredDialog( + onPurchaseClick = onPurchaseClick, + onDismiss = {} + ) + } + + TrialManager.TrialState.NOT_YET_STARTED_AWAITING_INTERNET, + TrialManager.TrialState.INTERNET_UNAVAILABLE_CANNOT_VERIFY -> { + if (showTrialInfoDialog) { + InfoDialog( + message = trialInfoMessage, + onDismiss = onDismissTrialInfo + ) + } + } + + TrialManager.TrialState.ACTIVE_INTERNET_TIME_CONFIRMED, + TrialManager.TrialState.PURCHASED -> Unit + } +} + +@Composable +internal fun PaymentMethodDialog( + onDismiss: () -> Unit, + onPayPalClick: () -> Unit, + onGooglePlayClick: () -> Unit +) { + androidx.compose.material3.AlertDialog( + onDismissRequest = onDismiss, + title = { Text("Choose Payment Method") }, + text = { + Column { + Button( + onClick = onPayPalClick, + modifier = Modifier.fillMaxWidth().padding(bottom = 8.dp) + ) { + Text("PayPal (2,60 €/Month)") + } + Button( + onClick = onGooglePlayClick, + modifier = Modifier.fillMaxWidth() + ) { + Text("Google Play (2,90 €/Month)") + } + } + }, + confirmButton = {}, + dismissButton = { + TextButton(onClick = onDismiss) { + Text("Cancel") + } + } + ) +} + +@Composable +internal fun ApiKeyDialogSection( + apiKeyManager: ApiKeyManager, + isFirstLaunch: Boolean, + initialProvider: ApiProvider?, + onDismiss: () -> Unit +) { + ApiKeyDialog( + apiKeyManager = apiKeyManager, + isFirstLaunch = isFirstLaunch, + initialProvider = initialProvider, + onDismiss = onDismiss + ) +} + +@Composable +fun FirstLaunchInfoDialog(onDismiss: () -> Unit) { + Log.d("FirstLaunchInfoDialog", "Composing FirstLaunchInfoDialog") + Dialog(onDismissRequest = { + Log.d("FirstLaunchInfoDialog", "onDismissRequest called") + onDismiss() + }) { + Card( + modifier = Modifier + .fillMaxWidth() + .padding(16.dp), + ) { + Column( + modifier = Modifier + .padding(16.dp) + .fillMaxWidth(), + verticalArrangement = Arrangement.Center, + horizontalAlignment = Alignment.CenterHorizontally + ) { + Text( + text = "Trial Information", + style = MaterialTheme.typography.titleLarge + ) + Spacer(modifier = Modifier.height(16.dp)) + Text( + text = "You can try Screen Operator for 7 days before you have to subscribe to support the development of more features.", + style = MaterialTheme.typography.bodyMedium, + modifier = Modifier.align(Alignment.CenterHorizontally) + ) + Spacer(modifier = Modifier.height(24.dp)) + TextButton( + onClick = { + Log.d("FirstLaunchInfoDialog", "OK button clicked") + onDismiss() + }, + modifier = Modifier.fillMaxWidth() + ) { + Text("OK") + } + } + } + } +} + + + +@Composable +fun TrialExpiredDialog( + onPurchaseClick: () -> Unit, + @Suppress("UNUSED_PARAMETER") onDismiss: () -> Unit +) { + Log.d("TrialExpiredDialog", "Composing TrialExpiredDialog") + Dialog(onDismissRequest = { + Log.d("TrialExpiredDialog", "onDismissRequest called (persistent dialog - user tried to dismiss)") + }) { + Card( + modifier = Modifier + .fillMaxWidth() + .padding(16.dp), + ) { + Column( + modifier = Modifier + .padding(16.dp) + .fillMaxWidth(), + verticalArrangement = Arrangement.Center, + horizontalAlignment = Alignment.CenterHorizontally + ) { + Text( + text = "Trial period expired", + style = MaterialTheme.typography.titleLarge + ) + Spacer(modifier = Modifier.height(16.dp)) + Text( + text = "Please support the development of the app so that you can continue using it \uD83C\uDF89", + style = MaterialTheme.typography.bodyMedium, + modifier = Modifier.align(Alignment.CenterHorizontally) + ) + Spacer(modifier = Modifier.height(24.dp)) + Button( + onClick = { + Log.d("TrialExpiredDialog", "Purchase button clicked") + onPurchaseClick() + }, + modifier = Modifier.fillMaxWidth() + ) { + Text("Subscribe") + } + } + } + } +} + +@Composable +fun InfoDialog( + message: String, + onDismiss: () -> Unit +) { + Log.d("InfoDialog", "Composing InfoDialog with message: $message") + Dialog(onDismissRequest = { + Log.d("InfoDialog", "onDismissRequest called") + onDismiss() + }) { + Card( + modifier = Modifier + .fillMaxWidth() + .padding(16.dp) + ) { + Column( + modifier = Modifier + .padding(16.dp) + .fillMaxWidth(), + verticalArrangement = Arrangement.Center, + horizontalAlignment = Alignment.CenterHorizontally + ) { + Text( + text = "Information", + style = MaterialTheme.typography.titleMedium + ) + Spacer(modifier = Modifier.height(16.dp)) + Text( + text = message, + style = MaterialTheme.typography.bodyMedium, + modifier = Modifier.align(Alignment.CenterHorizontally) + ) + Spacer(modifier = Modifier.height(24.dp)) + TextButton(onClick = { + Log.d("InfoDialog", "OK button clicked") + onDismiss() + }) { + Text("OK") + } + } + } + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/MainActivityMediaProjectionIntents.kt b/app/src/main/kotlin/com/google/ai/sample/MainActivityMediaProjectionIntents.kt new file mode 100644 index 0000000..749cfe8 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/MainActivityMediaProjectionIntents.kt @@ -0,0 +1,38 @@ +package com.google.ai.sample + +import android.content.Context +import android.content.Intent + +internal object MainActivityMediaProjectionIntents { + fun startCapture( + context: Context, + resultCode: Int, + resultData: Intent, + takeScreenshotOnStart: Boolean + ): Intent { + return Intent(context, ScreenCaptureService::class.java).apply { + action = ScreenCaptureService.ACTION_START_CAPTURE + putExtra(ScreenCaptureService.EXTRA_RESULT_CODE, resultCode) + putExtra(ScreenCaptureService.EXTRA_RESULT_DATA, resultData) + putExtra(ScreenCaptureService.EXTRA_TAKE_SCREENSHOT_ON_START, takeScreenshotOnStart) + } + } + + fun keepAliveForWebRtc(context: Context): Intent { + return Intent(context, ScreenCaptureService::class.java).apply { + action = ScreenCaptureService.ACTION_KEEP_ALIVE_FOR_WEBRTC + } + } + + fun takeScreenshot(context: Context): Intent { + return Intent(context, ScreenCaptureService::class.java).apply { + action = ScreenCaptureService.ACTION_TAKE_SCREENSHOT + } + } + + fun stopCapture(context: Context): Intent { + return Intent(context, ScreenCaptureService::class.java).apply { + action = ScreenCaptureService.ACTION_STOP_CAPTURE + } + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/MainActivityScreenshotFlowDecider.kt b/app/src/main/kotlin/com/google/ai/sample/MainActivityScreenshotFlowDecider.kt new file mode 100644 index 0000000..f9e2968 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/MainActivityScreenshotFlowDecider.kt @@ -0,0 +1,16 @@ +package com.google.ai.sample + +internal object MainActivityScreenshotFlowDecider { + enum class Action { + TAKE_ADDITIONAL_SCREENSHOT, + REQUEST_PERMISSION + } + + fun decide(isScreenCaptureServiceRunning: Boolean): Action { + return if (isScreenCaptureServiceRunning) { + Action.TAKE_ADDITIONAL_SCREENSHOT + } else { + Action.REQUEST_PERMISSION + } + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/MainActivityScreenshotIntents.kt b/app/src/main/kotlin/com/google/ai/sample/MainActivityScreenshotIntents.kt new file mode 100644 index 0000000..c21683f --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/MainActivityScreenshotIntents.kt @@ -0,0 +1,15 @@ +package com.google.ai.sample + +import android.content.Intent +import android.net.Uri + +internal object MainActivityScreenshotIntents { + fun extractScreenInfo(intent: Intent): String? { + return intent.getStringExtra(MainActivity.EXTRA_SCREEN_INFO) + } + + fun extractScreenshotUri(intent: Intent): Uri? { + val uriString = intent.getStringExtra(MainActivity.EXTRA_SCREENSHOT_URI) + return uriString?.let(Uri::parse) + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/MainActivityStatusNotifier.kt b/app/src/main/kotlin/com/google/ai/sample/MainActivityStatusNotifier.kt new file mode 100644 index 0000000..fe0b40b --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/MainActivityStatusNotifier.kt @@ -0,0 +1,16 @@ +package com.google.ai.sample + +import android.content.Context +import android.util.Log +import android.widget.Toast + +internal object MainActivityStatusNotifier { + fun showStatusMessage(context: Context, tag: String, message: String, isError: Boolean) { + Toast.makeText(context, message, if (isError) Toast.LENGTH_LONG else Toast.LENGTH_SHORT).show() + if (isError) { + Log.e(tag, "updateStatusMessage (Error): $message") + } else { + Log.d(tag, "updateStatusMessage (Info): $message") + } + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/NotificationPermissionPreferences.kt b/app/src/main/kotlin/com/google/ai/sample/NotificationPermissionPreferences.kt new file mode 100644 index 0000000..028583e --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/NotificationPermissionPreferences.kt @@ -0,0 +1,21 @@ +package com.google.ai.sample + +import android.content.Context + +internal object NotificationPermissionPreferences { + private const val PREFS_NAME = "AppPrefs" + private const val KEY_NOTIFICATION_RATIONALE_SHOWN = "notification_rationale_shown" + + fun hasShownNotificationRationale(context: Context): Boolean { + val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) + return prefs.getBoolean(KEY_NOTIFICATION_RATIONALE_SHOWN, false) + } + + fun setNotificationRationaleShown(context: Context, shown: Boolean) { + val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) + with(prefs.edit()) { + putBoolean(KEY_NOTIFICATION_RATIONALE_SHOWN, shown) + apply() + } + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/ScreenCaptureApiClients.kt b/app/src/main/kotlin/com/google/ai/sample/ScreenCaptureApiClients.kt new file mode 100644 index 0000000..00489d4 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/ScreenCaptureApiClients.kt @@ -0,0 +1,210 @@ +package com.google.ai.sample + +import android.util.Log +import com.google.ai.client.generativeai.type.Content +import com.google.ai.client.generativeai.type.ImagePart +import com.google.ai.client.generativeai.type.TextPart +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.SerializationException +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonClassDiscriminator +import kotlinx.serialization.modules.SerializersModule +import kotlinx.serialization.modules.polymorphic +import kotlinx.serialization.modules.subclass +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import java.io.IOException + +// Data classes for Mistral API in Service +@Serializable +data class ServiceMistralRequest( + val model: String, + val messages: List, + val max_tokens: Int = 4096, + val temperature: Double = 0.7, + val top_p: Double = 1.0, + val stream: Boolean = false +) + +@Serializable +data class ServiceMistralMessage( + val role: String, + val content: List +) + +@Serializable +@OptIn(ExperimentalSerializationApi::class) +@JsonClassDiscriminator("type") +sealed class ServiceMistralContent + +@Serializable +@SerialName("text") +data class ServiceMistralTextContent(@SerialName("text") val text: String) : ServiceMistralContent() + +@Serializable +@SerialName("image_url") +data class ServiceMistralImageContent(@SerialName("image_url") val imageUrl: ServiceMistralImageUrl) : ServiceMistralContent() + +@Serializable +data class ServiceMistralImageUrl(val url: String) + +@Serializable +data class ServiceMistralResponse( + val choices: List +) + +@Serializable +data class ServiceMistralChoice( + val message: ServiceMistralResponseMessage +) + +@Serializable +data class ServiceMistralResponseMessage( + val role: String, + val content: String +) + +internal suspend fun callMistralApi(modelName: String, apiKey: String, chatHistory: List, inputContent: Content): Pair { + var responseText: String? = null + var errorMessage: String? = null + + val json = Json { + serializersModule = SerializersModule { + polymorphic(ServiceMistralContent::class) { + subclass(ServiceMistralTextContent::class, ServiceMistralTextContent.serializer()) + subclass(ServiceMistralImageContent::class, ServiceMistralImageContent.serializer()) + } + } + ignoreUnknownKeys = true + } + + val currentModelOption = com.google.ai.sample.ModelOption.values().find { it.modelName == modelName } + val supportsScreenshot = currentModelOption?.supportsScreenshot ?: true + + try { + val apiMessages = mutableListOf() + + // Combine history and input, but handle system role if needed + (chatHistory + inputContent).forEach { content -> + val parts = content.parts.mapNotNull { part -> + when (part) { + is TextPart -> if (part.text.isNotBlank()) ServiceMistralTextContent(text = part.text) else null + is ImagePart -> { + if (supportsScreenshot) { + ServiceMistralImageContent(imageUrl = ServiceMistralImageUrl(url = "data:image/jpeg;base64,${com.google.ai.sample.util.ImageUtils.bitmapToBase64(part.image)}")) + } else null + } + else -> null + } + } + if (parts.isNotEmpty()) { + val role = when (content.role) { + "user" -> "user" + "system" -> "system" + else -> "assistant" + } + apiMessages.add(ServiceMistralMessage(role = role, content = parts)) + } + } + + val requestBody = ServiceMistralRequest( + model = modelName, + messages = apiMessages + ) + + val client = OkHttpClient() + val mediaType = "application/json".toMediaType() + val jsonBody = json.encodeToString(ServiceMistralRequest.serializer(), requestBody) + + val request = Request.Builder() + .url("https://api.mistral.ai/v1/chat/completions") + .post(jsonBody.toRequestBody(mediaType)) + .addHeader("Content-Type", "application/json") + .addHeader("Authorization", "Bearer $apiKey") + .build() + + client.newCall(request).execute().use { response -> + val responseBody = response.body?.string() + if (!response.isSuccessful) { + Log.e("ScreenCaptureService", "Mistral API Error ($response.code): $responseBody") + errorMessage = "Mistral Error ${response.code}: $responseBody" + } else { + if (responseBody != null) { + val mistralResponse = json.decodeFromString(ServiceMistralResponse.serializer(), responseBody) + responseText = mistralResponse.choices.firstOrNull()?.message?.content ?: "No response from model" + } else { + errorMessage = "Empty response body from Mistral" + } + } + } + } catch (e: IOException) { + errorMessage = e.localizedMessage ?: "Mistral API network call failed" + Log.e("ScreenCaptureService", "Mistral API network failure", e) + } catch (e: SerializationException) { + errorMessage = e.localizedMessage ?: "Mistral API response parse failed" + Log.e("ScreenCaptureService", "Mistral API parse failure", e) + } catch (e: IllegalStateException) { + errorMessage = e.localizedMessage ?: "Mistral API call failed" + Log.e("ScreenCaptureService", "Mistral API state failure", e) + } + + return Pair(responseText, errorMessage) +} + +internal suspend fun callPuterApi(modelName: String, apiKey: String, chatHistory: List, inputContent: Content): Pair { + var responseText: String? = null + var errorMessage: String? = null + + val currentModelOption = com.google.ai.sample.ModelOption.values().find { it.modelName == modelName } + val supportsScreenshot = currentModelOption?.supportsScreenshot ?: true + + try { + val apiMessages = mutableListOf() + + // Combine history and input, but handle system role if needed + (chatHistory + inputContent).forEach { content -> + val parts = content.parts.mapNotNull { part -> + when (part) { + is TextPart -> if (part.text.isNotBlank()) com.google.ai.sample.network.PuterTextContent(text = part.text) else null + is ImagePart -> { + if (supportsScreenshot) { + val base64Uri = com.google.ai.sample.network.PuterApiClient.bitmapToBase64DataUri(part.image) + com.google.ai.sample.network.PuterImageContent(image_url = com.google.ai.sample.network.PuterImageUrl(url = base64Uri)) + } else null + } + else -> null + } + } + if (parts.isNotEmpty()) { + val role = when (content.role) { + "user" -> "user" + "system" -> "system" + else -> "assistant" + } + apiMessages.add(com.google.ai.sample.network.PuterMessage(role = role, content = parts)) + } + } + + val requestBody = com.google.ai.sample.network.PuterRequest( + model = modelName, + messages = apiMessages + ) + + responseText = com.google.ai.sample.network.PuterApiClient.call(apiKey, requestBody) + + } catch (e: IOException) { + errorMessage = e.localizedMessage ?: "Puter API network call failed" + Log.e("ScreenCaptureService", "Puter API network failure", e) + } catch (e: IllegalStateException) { + errorMessage = e.localizedMessage ?: "Puter API call failed" + Log.e("ScreenCaptureService", "Puter API state failure", e) + } + + return Pair(responseText, errorMessage) +} diff --git a/app/src/main/kotlin/com/google/ai/sample/ScreenCaptureNotificationFactory.kt b/app/src/main/kotlin/com/google/ai/sample/ScreenCaptureNotificationFactory.kt new file mode 100644 index 0000000..c1611e4 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/ScreenCaptureNotificationFactory.kt @@ -0,0 +1,47 @@ +package com.google.ai.sample + +import android.app.Notification +import android.app.NotificationChannel +import android.app.NotificationManager +import android.content.Context +import android.os.Build +import androidx.core.app.NotificationCompat + +internal class ScreenCaptureNotificationFactory( + private val context: Context, + private val channelId: String +) { + fun createAiOperationNotification(): Notification { + return NotificationCompat.Builder(context, channelId) + .setContentTitle("Screen Operator") + .setContentText("Processing AI request...") + .setSmallIcon(android.R.drawable.ic_dialog_info) + .setPriority(NotificationCompat.PRIORITY_LOW) + .setOngoing(false) + .build() + } + + fun createNotification(): Notification { + return NotificationCompat.Builder(context, channelId) + .setContentTitle("Screen Capture Active") + .setContentText("Ready to take screenshots") + .setSmallIcon(android.R.drawable.ic_menu_camera) + .setPriority(NotificationCompat.PRIORITY_LOW) + .setOngoing(true) + .build() + } + + fun createNotificationChannel(channelName: String) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val serviceChannel = NotificationChannel( + channelId, + channelName, + NotificationManager.IMPORTANCE_LOW + ) + serviceChannel.description = "Notifications for screen capture service" + serviceChannel.setShowBadge(false) + val manager = context.getSystemService(NotificationManager::class.java) + manager.createNotificationChannel(serviceChannel) + } + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/ScreenCaptureService.kt b/app/src/main/kotlin/com/google/ai/sample/ScreenCaptureService.kt index 3f9d7e7..4551070 100644 --- a/app/src/main/kotlin/com/google/ai/sample/ScreenCaptureService.kt +++ b/app/src/main/kotlin/com/google/ai/sample/ScreenCaptureService.kt @@ -2,8 +2,6 @@ package com.google.ai.sample import android.app.Activity import android.app.Notification -import android.app.NotificationChannel -import android.app.NotificationManager import android.app.Service import android.content.Context import android.content.Intent @@ -17,7 +15,6 @@ import android.media.projection.MediaProjection import android.media.projection.MediaProjectionManager import android.net.Uri import android.os.Build -import android.os.Environment import android.os.Handler import android.os.IBinder import android.os.Looper @@ -51,20 +48,19 @@ import kotlinx.serialization.json.JsonClassDiscriminator import kotlinx.serialization.modules.SerializersModule import kotlinx.serialization.modules.polymorphic import kotlinx.serialization.modules.subclass -import androidx.core.app.NotificationCompat import androidx.localbroadcastmanager.content.LocalBroadcastManager import okhttp3.MediaType.Companion.toMediaType import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody.Companion.toRequestBody -import java.io.File import java.io.IOException -import java.io.FileOutputStream -import java.text.SimpleDateFormat -import java.util.Date -import java.util.Locale class ScreenCaptureService : Service() { + private val screenCaptureStorage by lazy { ScreenCaptureStorage(applicationContext, TAG) } + private val notificationFactory by lazy { + ScreenCaptureNotificationFactory(applicationContext, CHANNEL_ID) + } + companion object { private const val TAG = "ScreenCaptureService" private const val CHANNEL_ID = "ScreenCaptureChannel" @@ -144,48 +140,6 @@ class ScreenCaptureService : Service() { Log.d(TAG, "Sent broadcast ACTION_MEDIAPROJECTION_SCREENSHOT_CAPTURED with URI: $screenshotUri") } - private fun ensureScreenshotDirectory(): File { - val picturesDir = File(getExternalFilesDir(Environment.DIRECTORY_PICTURES), "Screenshots") - if (!picturesDir.exists()) { - picturesDir.mkdirs() - } - return picturesDir - } - - private fun pruneOldScreenshots(picturesDir: File, maxScreenshots: Int) { - val screenshotFiles = picturesDir.listFiles { _, name -> - name.startsWith("screenshot_") && name.endsWith(".png") - }?.toMutableList() ?: mutableListOf() - - screenshotFiles.sortBy { it.name } - - val screenshotsToDelete = screenshotFiles.size - (maxScreenshots - 1) - if (screenshotsToDelete <= 0) return - - Log.i( - TAG, - "Max screenshots reached. Current count: ${screenshotFiles.size}. Attempting to delete $screenshotsToDelete oldest screenshot(s)." - ) - - for (i in 0 until screenshotsToDelete) { - if (i < screenshotFiles.size) { - val oldestFile = screenshotFiles[i] - if (oldestFile.delete()) { - Log.i(TAG, "Deleted oldest screenshot: ${oldestFile.absolutePath}") - } else { - Log.e(TAG, "Failed to delete oldest screenshot: ${oldestFile.absolutePath}") - } - } - } - } - - private fun writeBitmapToPngFile(bitmap: Bitmap, file: File) { - FileOutputStream(file).use { outputStream -> - bitmap.compress(Bitmap.CompressFormat.PNG, 100, outputStream) - outputStream.flush() - } - } - override fun onCreate() { super.onCreate() instance = this @@ -480,39 +434,16 @@ class ScreenCaptureService : Service() { } private fun createAiOperationNotification(): Notification { - return NotificationCompat.Builder(this, CHANNEL_ID) // Reuse existing channel - .setContentTitle("Screen Operator") - .setContentText("Processing AI request...") - .setSmallIcon(android.R.drawable.ic_dialog_info) // Replace with a proper app icon - .setPriority(NotificationCompat.PRIORITY_LOW) - .setOngoing(false) // AI operation is not typically as long as screen capture - .build() + return notificationFactory.createAiOperationNotification() } private fun createNotification(): Notification { - return NotificationCompat.Builder(this, CHANNEL_ID) - .setContentTitle("Screen Capture Active") - .setContentText("Ready to take screenshots") - .setSmallIcon(android.R.drawable.ic_menu_camera) // Replace with a proper app icon - .setPriority(NotificationCompat.PRIORITY_LOW) - .setOngoing(true) - .build() + return notificationFactory.createNotification() } private fun createNotificationChannel() { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - val channel = NotificationChannel( - CHANNEL_ID, - "Screen Capture Service", - NotificationManager.IMPORTANCE_LOW - ).apply { - description = "Notifications for screen capture service" - setShowBadge(false) - } - val notificationManager = getSystemService(NotificationManager::class.java) - notificationManager.createNotificationChannel(channel) - Log.d(TAG, "Notification channel created") - } + notificationFactory.createNotificationChannel("Screen Capture Service") + Log.d(TAG, "Notification channel created") } private fun startCapture(resultCode: Int, data: Intent, takeScreenshotOnStart: Boolean) { @@ -681,32 +612,12 @@ class ScreenCaptureService : Service() { } private fun saveScreenshot(bitmap: Bitmap) { - try { - val picturesDir = ensureScreenshotDirectory() - val maxScreenshots = 100 - pruneOldScreenshots(picturesDir, maxScreenshots) - - val timestamp = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.getDefault()).format(Date()) - val file = File(picturesDir, "screenshot_$timestamp.png") - writeBitmapToPngFile(bitmap, file) - - Log.i(TAG, "Screenshot saved to: ${file.absolutePath}") - - showLongToast("Screenshot saved to: Android/data/com.google.ai.sample/files/Pictures/Screenshots/") - - val screenshotUri = Uri.fromFile(file) - broadcastScreenshotCaptured(screenshotUri) - - } catch (e: IOException) { - Log.e(TAG, "Failed to save screenshot (I/O)", e) - showLongToast("Failed to save screenshot: ${e.message}") - } catch (e: SecurityException) { - Log.e(TAG, "Failed to save screenshot (security)", e) - showLongToast("Failed to save screenshot: ${e.message}") - } catch (e: IllegalStateException) { - Log.e(TAG, "Failed to save screenshot", e) - showLongToast("Failed to save screenshot: ${e.message}") - } + screenCaptureStorage.saveScreenshot( + bitmap = bitmap, + onSaved = { screenshotUri -> broadcastScreenshotCaptured(screenshotUri) }, + onSuccessMessage = { message -> showLongToast(message) }, + onErrorMessage = { message -> showLongToast(message) } + ) } private fun cleanup() { @@ -745,299 +656,3 @@ class ScreenCaptureService : Service() { override fun onBind(intent: Intent?): IBinder? = null } - -// Data classes for Vercel API -@Serializable -data class VercelRequest( - val model: String, - val messages: List, - val stream: Boolean = true -) - -@Serializable -data class VercelStreamChunk( - val choices: List -) - -@Serializable -data class VercelStreamChoice( - val delta: VercelStreamDelta -) - -@Serializable -data class VercelStreamDelta( - val content: String? = null -) - -@Serializable -data class VercelMessage( - val role: String, - val content: String -) - -private suspend fun callVercelApi( - context: android.content.Context, - modelName: String, - apiKey: String, - chatHistory: List, - inputContent: ContentDto -): String { - val messages = mutableListOf() - - // Add Chat History - chatHistory.forEach { contentDto -> - val role = if (contentDto.role == "user") "user" else "assistant" - val text = contentDto.parts.filterIsInstance() - .joinToString("\n") { it.text } - if (text.isNotBlank()) messages.add(VercelMessage(role = role, content = text)) - } - - // Add current input - val inputText = inputContent.parts.filterIsInstance() - .joinToString("\n") { it.text } - if (inputText.isNotBlank()) messages.add(VercelMessage(role = "user", content = inputText)) - - val requestBodyJson = Json.encodeToString(VercelRequest.serializer(), VercelRequest(model = modelName, messages = messages, stream = true)) - val mediaType = "application/json".toMediaType() - - val httpRequest = Request.Builder() - .url("https://v0-screen-operator-clon-pi.vercel.app/api/chat") - .post(requestBodyJson.toRequestBody(mediaType)) - .addHeader("Content-Type", "application/json") - .addHeader("x-api-key", apiKey) - .build() - - val client = OkHttpClient() - val response = client.newCall(httpRequest).execute() - - if (!response.isSuccessful) { - val err = response.body?.string() - response.close() - throw IOException("Vercel API error ${response.code}: $err") - } - - val body = response.body ?: throw IOException("Empty response from Vercel") - val reader = body.charStream().buffered() - val accumulated = StringBuilder() - val sseJson = Json { ignoreUnknownKeys = true } - - try { - var line: String? - while (reader.readLine().also { line = it } != null) { - val l = line ?: break - if (l.startsWith("data: ")) { - val data = l.removePrefix("data: ").trim() - if (data == "[DONE]") break - if (data.isEmpty()) continue - - try { - val chunk = sseJson.decodeFromString(data) - val delta = chunk.choices.firstOrNull()?.delta?.content - if (!delta.isNullOrEmpty()) { - accumulated.append(delta) - // Broadcast update to ViewModel - val intent = Intent(ScreenCaptureService.ACTION_AI_STREAM_UPDATE).apply { - putExtra(ScreenCaptureService.EXTRA_AI_STREAM_CHUNK, accumulated.toString()) - } - androidx.localbroadcastmanager.content.LocalBroadcastManager.getInstance(context).sendBroadcast(intent) - } - } catch (e: SerializationException) { - // Skip malformed chunks - } - } - } - } finally { - reader.close() - response.close() - } - - return accumulated.toString() -} - -// Data classes for Mistral API in Service -@Serializable -data class ServiceMistralRequest( - val model: String, - val messages: List, - val max_tokens: Int = 4096, - val temperature: Double = 0.7, - val top_p: Double = 1.0, - val stream: Boolean = false -) - -@Serializable -data class ServiceMistralMessage( - val role: String, - val content: List -) - -@Serializable -@OptIn(ExperimentalSerializationApi::class) -@JsonClassDiscriminator("type") -sealed class ServiceMistralContent - -@Serializable -@SerialName("text") -data class ServiceMistralTextContent(@SerialName("text") val text: String) : ServiceMistralContent() - -@Serializable -@SerialName("image_url") -data class ServiceMistralImageContent(@SerialName("image_url") val imageUrl: ServiceMistralImageUrl) : ServiceMistralContent() - -@Serializable -data class ServiceMistralImageUrl(val url: String) - -@Serializable -data class ServiceMistralResponse( - val choices: List -) - -@Serializable -data class ServiceMistralChoice( - val message: ServiceMistralResponseMessage -) - -@Serializable -data class ServiceMistralResponseMessage( - val role: String, - val content: String -) - -private suspend fun callMistralApi(modelName: String, apiKey: String, chatHistory: List, inputContent: Content): Pair { - var responseText: String? = null - var errorMessage: String? = null - - val json = Json { - serializersModule = SerializersModule { - polymorphic(ServiceMistralContent::class) { - subclass(ServiceMistralTextContent::class, ServiceMistralTextContent.serializer()) - subclass(ServiceMistralImageContent::class, ServiceMistralImageContent.serializer()) - } - } - ignoreUnknownKeys = true - } - - val currentModelOption = com.google.ai.sample.ModelOption.values().find { it.modelName == modelName } - val supportsScreenshot = currentModelOption?.supportsScreenshot ?: true - - try { - val apiMessages = mutableListOf() - - // Combine history and input, but handle system role if needed - (chatHistory + inputContent).forEach { content -> - val parts = content.parts.mapNotNull { part -> - when (part) { - is TextPart -> if (part.text.isNotBlank()) ServiceMistralTextContent(text = part.text) else null - is ImagePart -> { - if (supportsScreenshot) { - ServiceMistralImageContent(imageUrl = ServiceMistralImageUrl(url = "data:image/jpeg;base64,${com.google.ai.sample.util.ImageUtils.bitmapToBase64(part.image)}")) - } else null - } - else -> null - } - } - if (parts.isNotEmpty()) { - val role = when (content.role) { - "user" -> "user" - "system" -> "system" - else -> "assistant" - } - apiMessages.add(ServiceMistralMessage(role = role, content = parts)) - } - } - - val requestBody = ServiceMistralRequest( - model = modelName, - messages = apiMessages - ) - - val client = OkHttpClient() - val mediaType = "application/json".toMediaType() - val jsonBody = json.encodeToString(ServiceMistralRequest.serializer(), requestBody) - - val request = Request.Builder() - .url("https://api.mistral.ai/v1/chat/completions") - .post(jsonBody.toRequestBody(mediaType)) - .addHeader("Content-Type", "application/json") - .addHeader("Authorization", "Bearer $apiKey") - .build() - - client.newCall(request).execute().use { response -> - val responseBody = response.body?.string() - if (!response.isSuccessful) { - Log.e("ScreenCaptureService", "Mistral API Error ($response.code): $responseBody") - errorMessage = "Mistral Error ${response.code}: $responseBody" - } else { - if (responseBody != null) { - val mistralResponse = json.decodeFromString(ServiceMistralResponse.serializer(), responseBody) - responseText = mistralResponse.choices.firstOrNull()?.message?.content ?: "No response from model" - } else { - errorMessage = "Empty response body from Mistral" - } - } - } - } catch (e: IOException) { - errorMessage = e.localizedMessage ?: "Mistral API network call failed" - Log.e("ScreenCaptureService", "Mistral API network failure", e) - } catch (e: SerializationException) { - errorMessage = e.localizedMessage ?: "Mistral API response parse failed" - Log.e("ScreenCaptureService", "Mistral API parse failure", e) - } catch (e: IllegalStateException) { - errorMessage = e.localizedMessage ?: "Mistral API call failed" - Log.e("ScreenCaptureService", "Mistral API state failure", e) - } - - return Pair(responseText, errorMessage) -} - -private suspend fun callPuterApi(modelName: String, apiKey: String, chatHistory: List, inputContent: Content): Pair { - var responseText: String? = null - var errorMessage: String? = null - - val currentModelOption = com.google.ai.sample.ModelOption.values().find { it.modelName == modelName } - val supportsScreenshot = currentModelOption?.supportsScreenshot ?: true - - try { - val apiMessages = mutableListOf() - - // Combine history and input, but handle system role if needed - (chatHistory + inputContent).forEach { content -> - val parts = content.parts.mapNotNull { part -> - when (part) { - is TextPart -> if (part.text.isNotBlank()) com.google.ai.sample.network.PuterTextContent(text = part.text) else null - is ImagePart -> { - if (supportsScreenshot) { - val base64Uri = com.google.ai.sample.network.PuterApiClient.bitmapToBase64DataUri(part.image) - com.google.ai.sample.network.PuterImageContent(image_url = com.google.ai.sample.network.PuterImageUrl(url = base64Uri)) - } else null - } - else -> null - } - } - if (parts.isNotEmpty()) { - val role = when (content.role) { - "user" -> "user" - "system" -> "system" - else -> "assistant" - } - apiMessages.add(com.google.ai.sample.network.PuterMessage(role = role, content = parts)) - } - } - - val requestBody = com.google.ai.sample.network.PuterRequest( - model = modelName, - messages = apiMessages - ) - - responseText = com.google.ai.sample.network.PuterApiClient.call(apiKey, requestBody) - - } catch (e: IOException) { - errorMessage = e.localizedMessage ?: "Puter API network call failed" - Log.e("ScreenCaptureService", "Puter API network failure", e) - } catch (e: IllegalStateException) { - errorMessage = e.localizedMessage ?: "Puter API call failed" - Log.e("ScreenCaptureService", "Puter API state failure", e) - } - - return Pair(responseText, errorMessage) -} diff --git a/app/src/main/kotlin/com/google/ai/sample/ScreenCaptureStorage.kt b/app/src/main/kotlin/com/google/ai/sample/ScreenCaptureStorage.kt new file mode 100644 index 0000000..240cbfc --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/ScreenCaptureStorage.kt @@ -0,0 +1,90 @@ +package com.google.ai.sample + +import android.content.Context +import android.graphics.Bitmap +import android.net.Uri +import android.os.Environment +import android.util.Log +import java.io.File +import java.io.FileOutputStream +import java.io.IOException +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale + +internal class ScreenCaptureStorage( + private val context: Context, + private val loggerTag: String +) { + fun saveScreenshot( + bitmap: Bitmap, + onSaved: (Uri) -> Unit, + onSuccessMessage: (String) -> Unit, + onErrorMessage: (String) -> Unit + ) { + try { + val picturesDir = ensureScreenshotDirectory() + val maxScreenshots = 100 + pruneOldScreenshots(picturesDir, maxScreenshots) + + val timestamp = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.getDefault()).format(Date()) + val file = File(picturesDir, "screenshot_$timestamp.png") + writeBitmapToPngFile(bitmap, file) + + Log.i(loggerTag, "Screenshot saved to: ${file.absolutePath}") + onSuccessMessage("Screenshot saved to: Android/data/com.google.ai.sample/files/Pictures/Screenshots/") + onSaved(Uri.fromFile(file)) + } catch (e: IOException) { + Log.e(loggerTag, "Failed to save screenshot (I/O)", e) + onErrorMessage("Failed to save screenshot: ${e.message}") + } catch (e: SecurityException) { + Log.e(loggerTag, "Failed to save screenshot (security)", e) + onErrorMessage("Failed to save screenshot: ${e.message}") + } catch (e: IllegalStateException) { + Log.e(loggerTag, "Failed to save screenshot", e) + onErrorMessage("Failed to save screenshot: ${e.message}") + } + } + + private fun ensureScreenshotDirectory(): File { + val picturesDir = File(context.getExternalFilesDir(Environment.DIRECTORY_PICTURES), "Screenshots") + if (!picturesDir.exists()) { + picturesDir.mkdirs() + } + return picturesDir + } + + private fun pruneOldScreenshots(picturesDir: File, maxScreenshots: Int) { + val screenshotFiles = picturesDir.listFiles { _, name -> + name.startsWith("screenshot_") && name.endsWith(".png") + }?.toMutableList() ?: mutableListOf() + + screenshotFiles.sortBy { it.name } + + val screenshotsToDelete = screenshotFiles.size - (maxScreenshots - 1) + if (screenshotsToDelete <= 0) return + + Log.i( + loggerTag, + "Max screenshots reached. Current count: ${screenshotFiles.size}. Attempting to delete $screenshotsToDelete oldest screenshot(s)." + ) + + for (i in 0 until screenshotsToDelete) { + if (i < screenshotFiles.size) { + val oldestFile = screenshotFiles[i] + if (oldestFile.delete()) { + Log.i(loggerTag, "Deleted oldest screenshot: ${oldestFile.absolutePath}") + } else { + Log.e(loggerTag, "Failed to delete oldest screenshot: ${oldestFile.absolutePath}") + } + } + } + } + + private fun writeBitmapToPngFile(bitmap: Bitmap, file: File) { + FileOutputStream(file).use { outputStream -> + bitmap.compress(Bitmap.CompressFormat.PNG, 100, outputStream) + outputStream.flush() + } + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/ScreenCaptureVercelClient.kt b/app/src/main/kotlin/com/google/ai/sample/ScreenCaptureVercelClient.kt new file mode 100644 index 0000000..8c408e0 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/ScreenCaptureVercelClient.kt @@ -0,0 +1,123 @@ +package com.google.ai.sample + +import android.content.Intent +import androidx.localbroadcastmanager.content.LocalBroadcastManager +import com.google.ai.sample.feature.multimodal.dtos.ContentDto +import kotlinx.serialization.Serializable +import kotlinx.serialization.SerializationException +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import java.io.IOException + +// Data classes for Vercel API +@Serializable +data class VercelRequest( + val model: String, + val messages: List, + val stream: Boolean = true +) + +@Serializable +data class VercelStreamChunk( + val choices: List +) + +@Serializable +data class VercelStreamChoice( + val delta: VercelStreamDelta +) + +@Serializable +data class VercelStreamDelta( + val content: String? = null +) + +@Serializable +data class VercelMessage( + val role: String, + val content: String +) + +internal suspend fun callVercelApi( + context: android.content.Context, + modelName: String, + apiKey: String, + chatHistory: List, + inputContent: ContentDto +): String { + val messages = mutableListOf() + + // Add Chat History + chatHistory.forEach { contentDto -> + val role = if (contentDto.role == "user") "user" else "assistant" + val text = contentDto.parts.filterIsInstance() + .joinToString("\n") { it.text } + if (text.isNotBlank()) messages.add(VercelMessage(role = role, content = text)) + } + + // Add current input + val inputText = inputContent.parts.filterIsInstance() + .joinToString("\n") { it.text } + if (inputText.isNotBlank()) messages.add(VercelMessage(role = "user", content = inputText)) + + val requestBodyJson = Json.encodeToString(VercelRequest.serializer(), VercelRequest(model = modelName, messages = messages, stream = true)) + val mediaType = "application/json".toMediaType() + + val httpRequest = Request.Builder() + .url("https://v0-screen-operator-clon-pi.vercel.app/api/chat") + .post(requestBodyJson.toRequestBody(mediaType)) + .addHeader("Content-Type", "application/json") + .addHeader("x-api-key", apiKey) + .build() + + val client = OkHttpClient() + val response = client.newCall(httpRequest).execute() + + if (!response.isSuccessful) { + val err = response.body?.string() + response.close() + throw IOException("Vercel API error ${response.code}: $err") + } + + val body = response.body ?: throw IOException("Empty response from Vercel") + val reader = body.charStream().buffered() + val accumulated = StringBuilder() + val sseJson = Json { ignoreUnknownKeys = true } + + try { + var line: String? + while (reader.readLine().also { line = it } != null) { + val l = line ?: break + if (l.startsWith("data: ")) { + val data = l.removePrefix("data: ").trim() + if (data == "[DONE]") break + if (data.isEmpty()) continue + + try { + val chunk = sseJson.decodeFromString(data) + val delta = chunk.choices.firstOrNull()?.delta?.content + if (!delta.isNullOrEmpty()) { + accumulated.append(delta) + // Broadcast update to ViewModel + val intent = Intent(ScreenCaptureService.ACTION_AI_STREAM_UPDATE).apply { + putExtra(ScreenCaptureService.EXTRA_AI_STREAM_CHUNK, accumulated.toString()) + } + LocalBroadcastManager.getInstance(context).sendBroadcast(intent) + } + } catch (e: SerializationException) { + // Skip malformed chunks + } + } + } + } finally { + reader.close() + response.close() + } + + return accumulated.toString() +} diff --git a/app/src/main/kotlin/com/google/ai/sample/ScreenCommandGeometryResolver.kt b/app/src/main/kotlin/com/google/ai/sample/ScreenCommandGeometryResolver.kt new file mode 100644 index 0000000..36f06b5 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/ScreenCommandGeometryResolver.kt @@ -0,0 +1,49 @@ +package com.google.ai.sample + +internal object ScreenCommandGeometryResolver { + data class ResolvedPoint(val xPx: Float, val yPx: Float) + + data class ResolvedScrollGesture( + val xPx: Float, + val yPx: Float, + val distancePx: Float, + val durationMs: Long + ) + + enum class ScrollAxis { HORIZONTAL, VERTICAL } + + fun resolvePoint( + x: String, + y: String, + screenWidth: Int, + screenHeight: Int, + coordinateResolver: (String, Int) -> Float + ): ResolvedPoint { + return ResolvedPoint( + xPx = coordinateResolver(x, screenWidth), + yPx = coordinateResolver(y, screenHeight) + ) + } + + fun resolveScrollGesture( + x: String, + y: String, + distance: String, + durationMs: Long, + axis: ScrollAxis, + screenWidth: Int, + screenHeight: Int, + coordinateResolver: (String, Int) -> Float + ): ResolvedScrollGesture { + val point = resolvePoint( + x = x, + y = y, + screenWidth = screenWidth, + screenHeight = screenHeight, + coordinateResolver = coordinateResolver + ) + val distanceBasis = if (axis == ScrollAxis.HORIZONTAL) screenWidth else screenHeight + val distancePx = coordinateResolver(distance, distanceBasis) + return ResolvedScrollGesture(point.xPx, point.yPx, distancePx, durationMs) + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt index 7af22d4..7342c53 100644 --- a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt +++ b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt @@ -36,16 +36,6 @@ import java.util.concurrent.atomic.AtomicBoolean import java.lang.NumberFormatException class ScreenOperatorAccessibilityService : AccessibilityService() { - private data class ResolvedPoint(val xPx: Float, val yPx: Float) - private data class ResolvedScrollGesture( - val xPx: Float, - val yPx: Float, - val distancePx: Float, - val durationMs: Long - ) - - private enum class ScrollAxis { HORIZONTAL, VERTICAL } - private val commandQueue = AccessibilityCommandQueue() // private val handler = Handler(Looper.getMainLooper()) // Already exists at the class level @@ -212,28 +202,6 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { }, nextCommandDelay) } - private fun resolvePoint(x: String, y: String, screenWidth: Int, screenHeight: Int): ResolvedPoint { - return ResolvedPoint( - xPx = convertCoordinate(x, screenWidth), - yPx = convertCoordinate(y, screenHeight) - ) - } - - private fun resolveScrollGesture( - x: String, - y: String, - distance: String, - durationMs: Long, - axis: ScrollAxis, - screenWidth: Int, - screenHeight: Int - ): ResolvedScrollGesture { - val point = resolvePoint(x, y, screenWidth, screenHeight) - val distanceBasis = if (axis == ScrollAxis.HORIZONTAL) screenWidth else screenHeight - val distancePx = convertCoordinate(distance, distanceBasis) - return ResolvedScrollGesture(point.xPx, point.yPx, distancePx, durationMs) - } - private fun executeSingleCommand(command: Command): Boolean { val displayMetrics = this.resources.displayMetrics val screenWidth = displayMetrics.widthPixels @@ -258,7 +226,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { } } is Command.TapCoordinates -> { - val point = resolvePoint(command.x, command.y, screenWidth, screenHeight) + val point = ScreenCommandGeometryResolver.resolvePoint(command.x, command.y, screenWidth, screenHeight, ::convertCoordinate) Log.d(TAG, "Tapping at coordinates: (${command.x} -> ${point.xPx}, ${command.y} -> ${point.yPx})") this.showToast("Trying to tap coordinates: (${point.xPx}, ${point.yPx})", false) this.tapAtCoordinates(point.xPx, point.yPx) @@ -354,64 +322,64 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { } } is Command.ScrollDownFromCoordinates -> { - Log.d(TAG, "ScrollDownFromCoordinates: Original inputs x='${command.x}', y='${command.y}', distance='${command.distance}', duration='${command.duration}'") - val resolved = resolveScrollGesture( + executeScrollFromCoordinatesCommand( + commandName = "ScrollDownFromCoordinates", + directionLabel = "down", x = command.x, y = command.y, distance = command.distance, - durationMs = command.duration, - axis = ScrollAxis.VERTICAL, + duration = command.duration, + axis = ScreenCommandGeometryResolver.ScrollAxis.VERTICAL, screenWidth = screenWidth, screenHeight = screenHeight - ) - this.showToast("Trying to scroll down from position (${resolved.xPx}, ${resolved.yPx})", false) - this.scrollDown(resolved.xPx, resolved.yPx, resolved.distancePx, resolved.durationMs) - true // Asynchronous + ) { resolved -> + scrollDown(resolved.xPx, resolved.yPx, resolved.distancePx, resolved.durationMs) + } } is Command.ScrollUpFromCoordinates -> { - Log.d(TAG, "ScrollUpFromCoordinates: Original inputs x='${command.x}', y='${command.y}', distance='${command.distance}', duration='${command.duration}'") - val resolved = resolveScrollGesture( + executeScrollFromCoordinatesCommand( + commandName = "ScrollUpFromCoordinates", + directionLabel = "up", x = command.x, y = command.y, distance = command.distance, - durationMs = command.duration, - axis = ScrollAxis.VERTICAL, + duration = command.duration, + axis = ScreenCommandGeometryResolver.ScrollAxis.VERTICAL, screenWidth = screenWidth, screenHeight = screenHeight - ) - this.showToast("Trying to scroll up from position (${resolved.xPx}, ${resolved.yPx})", false) - this.scrollUp(resolved.xPx, resolved.yPx, resolved.distancePx, resolved.durationMs) - true // Asynchronous + ) { resolved -> + scrollUp(resolved.xPx, resolved.yPx, resolved.distancePx, resolved.durationMs) + } } is Command.ScrollLeftFromCoordinates -> { - Log.d(TAG, "ScrollLeftFromCoordinates: Original inputs x='${command.x}', y='${command.y}', distance='${command.distance}', duration='${command.duration}'") - val resolved = resolveScrollGesture( + executeScrollFromCoordinatesCommand( + commandName = "ScrollLeftFromCoordinates", + directionLabel = "left", x = command.x, y = command.y, distance = command.distance, - durationMs = command.duration, - axis = ScrollAxis.HORIZONTAL, + duration = command.duration, + axis = ScreenCommandGeometryResolver.ScrollAxis.HORIZONTAL, screenWidth = screenWidth, screenHeight = screenHeight - ) - this.showToast("Trying to scroll left from position (${resolved.xPx}, ${resolved.yPx})", false) - this.scrollLeft(resolved.xPx, resolved.yPx, resolved.distancePx, resolved.durationMs) - true // Asynchronous + ) { resolved -> + scrollLeft(resolved.xPx, resolved.yPx, resolved.distancePx, resolved.durationMs) + } } is Command.ScrollRightFromCoordinates -> { - Log.d(TAG, "ScrollRightFromCoordinates: Original inputs x='${command.x}', y='${command.y}', distance='${command.distance}', duration='${command.duration}'") - val resolved = resolveScrollGesture( + executeScrollFromCoordinatesCommand( + commandName = "ScrollRightFromCoordinates", + directionLabel = "right", x = command.x, y = command.y, distance = command.distance, - durationMs = command.duration, - axis = ScrollAxis.HORIZONTAL, + duration = command.duration, + axis = ScreenCommandGeometryResolver.ScrollAxis.HORIZONTAL, screenWidth = screenWidth, screenHeight = screenHeight - ) - this.showToast("Trying to scroll right from position (${resolved.xPx}, ${resolved.yPx})", false) - this.scrollRight(resolved.xPx, resolved.yPx, resolved.distancePx, resolved.durationMs) - true // Asynchronous + ) { resolved -> + scrollRight(resolved.xPx, resolved.yPx, resolved.distancePx, resolved.durationMs) + } } is Command.OpenApp -> { executeSyncCommandAction( @@ -478,6 +446,38 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { return true } + private fun executeScrollFromCoordinatesCommand( + commandName: String, + directionLabel: String, + x: String, + y: String, + distance: String, + duration: Long, + axis: ScreenCommandGeometryResolver.ScrollAxis, + screenWidth: Int, + screenHeight: Int, + execute: (ScreenCommandGeometryResolver.ResolvedScrollGesture) -> Unit + ): Boolean { + Log.d( + TAG, + "$commandName: Original inputs x='$x', y='$y', distance='$distance', duration='$duration'" + ) + val resolved = ScreenCommandGeometryResolver.resolveScrollGesture( + x = x, + y = y, + distance = distance, + durationMs = duration, + axis = axis, + screenWidth = screenWidth, + screenHeight = screenHeight, + coordinateResolver = ::convertCoordinate + ) + showToast("Trying to scroll $directionLabel from position (${resolved.xPx}, ${resolved.yPx})", false) + execute(resolved) + return true + } + + private fun processCommandQueue() { if (!commandQueue.tryAcquireProcessing()) { Log.d(TAG, "Queue is already being processed.") @@ -1237,50 +1237,79 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { /** * Tap at the specified coordinates */ + private fun dispatchGestureWithCallbacks( + gesture: GestureDescription, + onCompleted: () -> Unit, + onCancelled: () -> Unit, + onDispatchFailed: () -> Unit + ) { + val dispatchResult = dispatchGesture(gesture, object : GestureResultCallback() { + override fun onCompleted(gestureDescription: GestureDescription) { + super.onCompleted(gestureDescription) + onCompleted() + } + + override fun onCancelled(gestureDescription: GestureDescription) { + super.onCancelled(gestureDescription) + onCancelled() + } + }, null) + + if (!dispatchResult) { + onDispatchFailed() + } + } + + private fun ensureGestureApiAvailable(actionDescription: String): Boolean { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) { + Log.e(TAG, "Gesture API is not available on this Android version") + showToast("$actionDescription not available: Gesture API is not available on this Android version", true) + scheduleNextCommandProcessing() + return false + } + return true + } + + private fun buildTapGesture(x: Float, y: Float, durationMs: Long): GestureDescription { + val path = Path().apply { + moveTo(x, y) + } + return GestureDescription.Builder() + .addStroke(GestureDescription.StrokeDescription(path, 0, durationMs)) + .build() + } + fun tapAtCoordinates(x: Float, y: Float) { Log.d(TAG, "Tapping at coordinates: ($x, $y)") showToast("Tapping at coordinates: ($x, $y)", false) - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) { - Log.e(TAG, "Gesture API is not available on this Android version") - showToast("Gesture API is not available on this Android version", true) - scheduleNextCommandProcessing() // Continue queue if API not available + if (!ensureGestureApiAvailable("Tap")) { return } try { - // Create a tap gesture - val path = Path() - path.moveTo(x, y) + val gesture = buildTapGesture(x = x, y = y, durationMs = 100) - val gesture = GestureDescription.Builder() - .addStroke(GestureDescription.StrokeDescription(path, 0, 100)) - .build() - - // Dispatch the gesture - val dispatchResult = dispatchGesture(gesture, object : GestureResultCallback() { - override fun onCompleted(gestureDescription: GestureDescription) { - super.onCompleted(gestureDescription) + dispatchGestureWithCallbacks( + gesture = gesture, + onCompleted = { Log.d(TAG, "Tap gesture completed") showToast("Tapped coordinates ($x, $y) successfully", false) scheduleNextCommandProcessing() - } - - override fun onCancelled(gestureDescription: GestureDescription) { - super.onCancelled(gestureDescription) + }, + onCancelled = { Log.e(TAG, "Tap gesture cancelled") showToast("Tap at coordinates ($x, $y) cancelled, trying longer duration", true) // Try with longer duration, which will then call scheduleNextCommandProcessing tapAtCoordinatesWithLongerDuration(x, y) + }, + onDispatchFailed = { + Log.e(TAG, "Failed to dispatch tap gesture") + showToast("Error dispatching tap gesture, trying longer duration", true) + // Try with longer duration, which will then call scheduleNextCommandProcessing + tapAtCoordinatesWithLongerDuration(x, y) } - }, null) - - if (!dispatchResult) { - Log.e(TAG, "Failed to dispatch tap gesture") - showToast("Error dispatching tap gesture, trying longer duration", true) - // Try with longer duration, which will then call scheduleNextCommandProcessing - tapAtCoordinatesWithLongerDuration(x, y) - } + ) } catch (e: Exception) { Log.e(TAG, "Error tapping at coordinates: ${e.message}") showToast("Error tapping at coordinates: ${e.message}", true) @@ -1295,44 +1324,31 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { Log.d(TAG, "Tapping at coordinates with longer duration: ($x, $y)") showToast("Trying to tap with longer duration at: ($x, $y)", false) - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) { - Log.e(TAG, "Gesture API is not available on this Android version") - showToast("Gesture API is not available on this Android version", true) - scheduleNextCommandProcessing() // Continue queue + if (!ensureGestureApiAvailable("Long tap")) { return } try { - // Create a tap gesture with longer duration - val path = Path() - path.moveTo(x, y) + val gesture = buildTapGesture(x = x, y = y, durationMs = 300) - val gesture = GestureDescription.Builder() - .addStroke(GestureDescription.StrokeDescription(path, 0, 300)) // 300ms duration - .build() - - // Dispatch the gesture - val dispatchResult = dispatchGesture(gesture, object : GestureResultCallback() { - override fun onCompleted(gestureDescription: GestureDescription) { - super.onCompleted(gestureDescription) + dispatchGestureWithCallbacks( + gesture = gesture, + onCompleted = { Log.d(TAG, "Long tap gesture completed") showToast("Tapped with longer duration at coordinates ($x, $y) successfully", false) scheduleNextCommandProcessing() - } - - override fun onCancelled(gestureDescription: GestureDescription) { - super.onCancelled(gestureDescription) + }, + onCancelled = { Log.e(TAG, "Long tap gesture cancelled") showToast("Tap with longer duration at coordinates ($x, $y) cancelled", true) scheduleNextCommandProcessing() + }, + onDispatchFailed = { + Log.e(TAG, "Failed to dispatch long tap gesture") + showToast("Error dispatching tap gesture with longer duration", true) + scheduleNextCommandProcessing() } - }, null) - - if (!dispatchResult) { - Log.e(TAG, "Failed to dispatch long tap gesture") - showToast("Error dispatching tap gesture with longer duration", true) - scheduleNextCommandProcessing() - } + ) } catch (e: Exception) { Log.e(TAG, "Error tapping at coordinates with longer duration: ${e.message}") showToast("Error tapping with longer duration at coordinates: ${e.message}", true) @@ -1931,32 +1947,24 @@ private fun openAppUsingLaunchIntent(packageName: String, appName: String): Bool ) gestureBuilder.addStroke(gesture) - // Dispatch the gesture - val result = dispatchGesture( - gestureBuilder.build(), - object : GestureResultCallback() { - override fun onCompleted(gestureDescription: GestureDescription) { - super.onCompleted(gestureDescription) - Log.d(TAG, "scrollDown method: Gesture completed for path from ($startX, $startY) to ($endX, $endY)") - showToast("Successfully scrolled down from position ($startX, $startY)", false) - scheduleNextCommandProcessing() - } - - override fun onCancelled(gestureDescription: GestureDescription) { - super.onCancelled(gestureDescription) - Log.e(TAG, "scrollDown method: Gesture CANCELLED for path from ($startX, $startY) to ($endX, $endY). GestureDescription: $gestureDescription") - showToast("Scroll down from position ($startX, $startY) cancelled", true) - scheduleNextCommandProcessing() - } + dispatchGestureWithCallbacks( + gesture = gestureBuilder.build(), + onCompleted = { + Log.d(TAG, "scrollDown method: Gesture completed for path from ($startX, $startY) to ($endX, $endY)") + showToast("Successfully scrolled down from position ($startX, $startY)", false) + scheduleNextCommandProcessing() }, - null // handler + onCancelled = { + Log.e(TAG, "scrollDown method: Gesture CANCELLED for path from ($startX, $startY) to ($endX, $endY)") + showToast("Scroll down from position ($startX, $startY) cancelled", true) + scheduleNextCommandProcessing() + }, + onDispatchFailed = { + Log.e(TAG, "Failed to dispatch coordinate-based scroll down gesture for path from ($startX, $startY) to ($endX, $endY)") + showToast("Error scrolling down from position ($startX, $startY)", true) + scheduleNextCommandProcessing() + } ) - - if (!result) { - Log.e(TAG, "Failed to dispatch coordinate-based scroll down gesture for path from ($startX, $startY) to ($endX, $endY)") - showToast("Error scrolling down from position ($startX, $startY)", true) - scheduleNextCommandProcessing() - } } catch (e: Exception) { Log.e(TAG, "Error scrolling down from coordinates: ${e.message}") showToast("Error scrolling down from position ($x, $y): ${e.message}", true) diff --git a/app/src/main/kotlin/com/google/ai/sample/TrialStateUiModel.kt b/app/src/main/kotlin/com/google/ai/sample/TrialStateUiModel.kt new file mode 100644 index 0000000..2ff7d6e --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/TrialStateUiModel.kt @@ -0,0 +1,24 @@ +package com.google.ai.sample + +internal data class TrialStateUiModel( + val infoMessage: String, + val shouldShowInfoDialog: Boolean +) + +internal object TrialStateUiModelResolver { + fun resolve(state: TrialManager.TrialState): TrialStateUiModel { + return when (state) { + TrialManager.TrialState.EXPIRED_INTERNET_TIME_CONFIRMED -> TrialStateUiModel( + infoMessage = "Please support the development of the app so that you can continue using it \uD83C\uDF89", + shouldShowInfoDialog = true + ) + TrialManager.TrialState.ACTIVE_INTERNET_TIME_CONFIRMED, + TrialManager.TrialState.PURCHASED, + TrialManager.TrialState.NOT_YET_STARTED_AWAITING_INTERNET, + TrialManager.TrialState.INTERNET_UNAVAILABLE_CANNOT_VERIFY -> TrialStateUiModel( + infoMessage = "", + shouldShowInfoDialog = false + ) + } + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningChatBubbles.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningChatBubbles.kt new file mode 100644 index 0000000..e690c2a --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningChatBubbles.kt @@ -0,0 +1,191 @@ +package com.google.ai.sample.feature.multimodal + +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.requiredSize +import androidx.compose.foundation.lazy.LazyRow +import androidx.compose.foundation.lazy.items +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.outlined.Person +import androidx.compose.material.icons.rounded.Refresh +import androidx.compose.material3.Button +import androidx.compose.material3.ButtonDefaults +import androidx.compose.material3.Card +import androidx.compose.material3.CardDefaults +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.drawBehind +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.unit.dp +import coil.compose.AsyncImage + +@Composable +fun StopButton(onClick: () -> Unit) { + Button( + onClick = onClick, + colors = ButtonDefaults.buttonColors(containerColor = Color.Red), + modifier = Modifier + .fillMaxWidth() + .padding(8.dp) + ) { + Text("Stop", color = Color.White) + } +} + +@Composable +fun UserChatBubble( + text: String, + isPending: Boolean, + imageUris: List = emptyList(), + showUndo: Boolean = false, + onUndoClicked: () -> Unit = {} +) { + Row( + modifier = Modifier + .padding(vertical = 8.dp, horizontal = 8.dp) + .fillMaxWidth(), + verticalAlignment = Alignment.Top + ) { + if (showUndo) { + IconButton( + onClick = onUndoClicked, + modifier = Modifier + .padding(end = 8.dp) + .align(Alignment.CenterVertically) + ) { + Icon( + imageVector = Icons.Rounded.Refresh, + contentDescription = "Undo", + tint = Color.Gray + ) + } + } + Spacer(modifier = Modifier.weight(1f)) + Card( + shape = MaterialTheme.shapes.medium, + colors = CardDefaults.cardColors( + containerColor = MaterialTheme.colorScheme.primaryContainer + ), + modifier = Modifier.weight(4f) + ) { + Column( + modifier = Modifier.padding(all = 16.dp) + ) { + Text( + text = text, + color = MaterialTheme.colorScheme.onPrimaryContainer + ) + if (imageUris.isNotEmpty()) { + LazyRow( + modifier = Modifier.padding(top = 8.dp) + ) { + items(imageUris) { uri -> + AsyncImage( + model = uri, + contentDescription = null, + modifier = Modifier + .padding(4.dp) + .requiredSize(100.dp) + ) + } + } + } + if (isPending) { + CircularProgressIndicator( + modifier = Modifier + .padding(top = 8.dp) + .requiredSize(16.dp), + strokeWidth = 2.dp + ) + } + } + } + } +} + +@Composable +fun ModelChatBubble( + text: String, + isPending: Boolean +) { + Row( + modifier = Modifier + .padding(vertical = 8.dp, horizontal = 8.dp) + .fillMaxWidth(), + verticalAlignment = Alignment.Top + ) { + Card( + shape = MaterialTheme.shapes.medium, + colors = CardDefaults.cardColors( + containerColor = MaterialTheme.colorScheme.secondaryContainer + ), + modifier = Modifier.weight(4f) + ) { + Row( + modifier = Modifier.padding(all = 16.dp) + ) { + Icon( + Icons.Outlined.Person, + contentDescription = "AI Assistant", + tint = MaterialTheme.colorScheme.onSecondaryContainer, + modifier = Modifier + .requiredSize(24.dp) + .drawBehind { + drawCircle(color = Color.White) + } + .padding(end = 8.dp) + ) + Column { + Text( + text = text, + color = MaterialTheme.colorScheme.onSecondaryContainer + ) + if (isPending) { + CircularProgressIndicator( + modifier = Modifier + .padding(top = 8.dp) + .requiredSize(16.dp), + strokeWidth = 2.dp + ) + } + } + } + } + Spacer(modifier = Modifier.weight(1f)) + } +} + +@Composable +fun ErrorChatBubble( + text: String +) { + Box( + modifier = Modifier + .padding(vertical = 8.dp, horizontal = 8.dp) + .fillMaxWidth() + ) { + Card( + shape = MaterialTheme.shapes.medium, + colors = CardDefaults.cardColors( + containerColor = MaterialTheme.colorScheme.errorContainer + ), + modifier = Modifier.fillMaxWidth() + ) { + Text( + text = text, + color = MaterialTheme.colorScheme.error, + modifier = Modifier.padding(all = 16.dp) + ) + } + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningChatState.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningChatState.kt index f6615c1..3b1ff4f 100644 --- a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningChatState.kt +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningChatState.kt @@ -1,29 +1,40 @@ package com.google.ai.sample.feature.multimodal -import androidx.compose.runtime.toMutableStateList - class PhotoReasoningChatState( messages: List = emptyList() ) { - private val _messages: MutableList = messages.toMutableStateList() - val messages: List = _messages + private val _messages: MutableList = messages.toMutableList() + val messages: List + get() = _messages.toList() fun addMessage(msg: PhotoReasoningMessage) { _messages.add(msg) } fun replaceLastPendingMessage() { - val lastMessage = _messages.lastOrNull() - lastMessage?.let { - val newMessage = lastMessage.apply { isPending = false } - if (_messages.isNotEmpty()) { - _messages.removeAt(_messages.lastIndex) - } - _messages.add(newMessage) + val lastPendingIndex = _messages.indexOfLast { it.isPending } + if (lastPendingIndex >= 0) { + _messages.removeAt(lastPendingIndex) } } fun clearMessages() { _messages.clear() } + + fun updateLastMessageText(newText: String) { + if (_messages.isNotEmpty()) { + val lastMessage = _messages.last() + _messages[_messages.size - 1] = lastMessage.copy(text = newText, isPending = false) + } + } + + fun getAllMessages(): List { + return _messages.toList() + } + + fun setAllMessages(messages: List) { + _messages.clear() + _messages.addAll(messages) + } } diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningCommandExecutionGuard.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningCommandExecutionGuard.kt new file mode 100644 index 0000000..5052887 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningCommandExecutionGuard.kt @@ -0,0 +1,7 @@ +package com.google.ai.sample.feature.multimodal + +internal object PhotoReasoningCommandExecutionGuard { + fun shouldAbort(isJobActive: Boolean, isStopRequested: Boolean): Boolean { + return !isJobActive || isStopRequested + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningCommandProcessing.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningCommandProcessing.kt new file mode 100644 index 0000000..1ae5fd2 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningCommandProcessing.kt @@ -0,0 +1,25 @@ +package com.google.ai.sample.feature.multimodal + +import com.google.ai.sample.util.Command +import com.google.ai.sample.util.CommandParser + +internal data class ParsedCommandBatch( + val commands: List, + val hasTakeScreenshotCommand: Boolean, + val commandDescriptions: String +) + +internal object PhotoReasoningCommandProcessing { + fun parseForStreaming(accumulatedText: String): List { + return CommandParser.parseCommands(accumulatedText, clearBuffer = true) + } + + fun parseForFinalExecution(text: String): ParsedCommandBatch { + val commands = CommandParser.parseCommands(text) + return ParsedCommandBatch( + commands = commands, + hasTakeScreenshotCommand = commands.any { it is Command.TakeScreenshot }, + commandDescriptions = commands.joinToString("; ") { it.toString() } + ) + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningCommandStateUpdater.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningCommandStateUpdater.kt new file mode 100644 index 0000000..f53712b --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningCommandStateUpdater.kt @@ -0,0 +1,17 @@ +package com.google.ai.sample.feature.multimodal + +import com.google.ai.sample.util.Command + +internal object PhotoReasoningCommandStateUpdater { + fun appendCommand(existing: List, command: Command): List { + return existing.toMutableList().also { it.add(command) } + } + + fun appendCommands(existing: List, commands: List): List { + return existing.toMutableList().also { it.addAll(commands) } + } + + fun buildDetectedStatus(commandDescriptions: String): String { + return "Commands detected: $commandDescriptions" + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningCommandUiNotifier.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningCommandUiNotifier.kt new file mode 100644 index 0000000..3402b6a --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningCommandUiNotifier.kt @@ -0,0 +1,10 @@ +package com.google.ai.sample.feature.multimodal + +import android.content.Context +import android.widget.Toast + +internal object PhotoReasoningCommandUiNotifier { + fun showStoppedByAi(context: Context) { + Toast.makeText(context, "The AI stopped Screen Operator", Toast.LENGTH_SHORT).show() + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningDatabasePopup.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningDatabasePopup.kt new file mode 100644 index 0000000..a67bb7d --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningDatabasePopup.kt @@ -0,0 +1,463 @@ +package com.google.ai.sample.feature.multimodal + +import android.net.Uri +import android.util.Log +import android.widget.Toast +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.foundation.BorderStroke +import androidx.compose.foundation.background +import androidx.compose.foundation.border +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxHeight +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.MoreVert +import androidx.compose.material3.AlertDialog +import androidx.compose.material3.Button +import androidx.compose.material3.ButtonDefaults +import androidx.compose.material3.Card +import androidx.compose.material3.CardDefaults +import androidx.compose.material3.Checkbox +import androidx.compose.material3.CheckboxDefaults +import androidx.compose.material3.DropdownMenu +import androidx.compose.material3.DropdownMenuItem +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.material3.TextButton +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.runtime.saveable.rememberSaveable +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.unit.dp +import androidx.compose.ui.window.Dialog +import androidx.compose.ui.window.DialogProperties +import com.google.ai.sample.util.SystemMessageEntry +import com.google.ai.sample.util.SystemMessageEntryPreferences +import com.google.ai.sample.util.shareTextFile +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import kotlinx.serialization.SerializationException +import kotlinx.serialization.builtins.ListSerializer +import kotlinx.serialization.json.Json + +// Reuse shared colors from PhotoReasoningScreen.kt + +@Composable +fun DatabaseListPopup( + onDismissRequest: () -> Unit, + entries: List, + onNewClicked: () -> Unit, + onEntryClicked: (SystemMessageEntry) -> Unit, + onDeleteClicked: (SystemMessageEntry) -> Unit, + onImportCompleted: () -> Unit +) { + val TAG_IMPORT_PROCESS = "ImportProcess" + val scope = rememberCoroutineScope() + var entryMenuToShow: SystemMessageEntry? by remember { mutableStateOf(null) } + var selectionModeActive by rememberSaveable { mutableStateOf(false) } + var selectedEntryTitles by rememberSaveable { mutableStateOf(emptySet()) } + var selectAllChecked by rememberSaveable { mutableStateOf(false) } + val context = LocalContext.current + + var entryToConfirmOverwrite by remember { mutableStateOf?>(null) } + var remainingEntriesToImport by remember { mutableStateOf>(emptyList()) } + var skipAllDuplicates by remember { mutableStateOf(false) } + + // processImportedEntries is defined within DatabaseListPopup, so it has access to context, onImportCompleted, etc. + fun processImportedEntries( + imported: List, + currentSystemEntries: List + ) { + val TAG_IMPORT_PROCESS_FUNCTION = "ImportProcessFunction" + Log.d(TAG_IMPORT_PROCESS_FUNCTION, "Starting processImportedEntries. Imported: ${imported.size}, Current: ${currentSystemEntries.size}, SkipAll: $skipAllDuplicates") + + var newCount = 0 + var updatedCount = 0 + var skippedCount = 0 + val entriesToProcess = imported.toMutableList() + + while (entriesToProcess.isNotEmpty()) { + val newEntry = entriesToProcess.removeAt(0) + Log.d(TAG_IMPORT_PROCESS_FUNCTION, "Processing entry: Title='${newEntry.title}'. Remaining in batch: ${entriesToProcess.size}") + val existingEntry = currentSystemEntries.find { it.title.equals(newEntry.title, ignoreCase = true) } + + if (existingEntry != null) { + Log.d(TAG_IMPORT_PROCESS_FUNCTION, "Duplicate found for title: '${newEntry.title}'. Existing guide: '${existingEntry.guide.take(50)}', New guide: '${newEntry.guide.take(50)}'") + if (skipAllDuplicates) { + Log.d(TAG_IMPORT_PROCESS_FUNCTION, "Skipping duplicate '${newEntry.title}' due to skipAllDuplicates flag.") + skippedCount++ + continue + } + Log.d(TAG_IMPORT_PROCESS_FUNCTION, "Calling askForOverwrite for '${newEntry.title}'.") + entryToConfirmOverwrite = Pair(existingEntry, newEntry) + remainingEntriesToImport = entriesToProcess.toList() + return + } else { + Log.i(TAG_IMPORT_PROCESS_FUNCTION, "Adding new entry: Title='${newEntry.title}'") + SystemMessageEntryPreferences.addEntry(context, newEntry) + newCount++ + } + } + Log.i(TAG_IMPORT_PROCESS_FUNCTION, "Finished processing batch. newCount=$newCount, updatedCount=$updatedCount, skippedCount=$skippedCount") + val summary = "Import finished: $newCount added, $updatedCount updated, $skippedCount skipped." + Toast.makeText(context, summary as CharSequence, Toast.LENGTH_LONG).show() + onImportCompleted() + skipAllDuplicates = false + } + + + val filePickerLauncher = rememberLauncherForActivityResult( + contract = ActivityResultContracts.GetContent(), + onResult = { uri: Uri? -> + Log.d(TAG_IMPORT_PROCESS, "FilePickerLauncher onResult triggered.") + if (uri == null) { + Log.w(TAG_IMPORT_PROCESS, "URI is null, no file selected or operation cancelled.") + scope.launch(Dispatchers.Main) { + Toast.makeText(context, "No file selected." as CharSequence, Toast.LENGTH_SHORT).show() + } + return@rememberLauncherForActivityResult + } + + Log.i(TAG_IMPORT_PROCESS, "Selected file URI: $uri") + scope.launch(Dispatchers.Main) { + Toast.makeText(context, "File selected: $uri. Starting import..." as CharSequence, Toast.LENGTH_SHORT).show() + } + + scope.launch(Dispatchers.IO) { + try { + Log.d(TAG_IMPORT_PROCESS, "Attempting to open InputStream for URI: $uri on thread: ${Thread.currentThread().name}") + + var fileSize = -1L + try { + context.contentResolver.openFileDescriptor(uri, "r")?.use { pfd -> + fileSize = pfd.statSize + } + Log.i(TAG_IMPORT_PROCESS, "Estimated file size: $fileSize bytes.") + } catch (e: Exception) { + Log.w(TAG_IMPORT_PROCESS, "Could not determine file size for URI: $uri. Will proceed without size check.", e) + } + + val MAX_FILE_SIZE_BYTES = 10 * 1024 * 1024 + if (fileSize != -1L && fileSize > MAX_FILE_SIZE_BYTES) { + Log.e(TAG_IMPORT_PROCESS, "File size ($fileSize bytes) exceeds limit of $MAX_FILE_SIZE_BYTES bytes.") + withContext(Dispatchers.Main) { + Toast.makeText(context, "File is too large (max 10MB)." as CharSequence, Toast.LENGTH_LONG).show() + } + return@launch + } + if (fileSize == 0L) { + Log.w(TAG_IMPORT_PROCESS, "Imported file is empty (0 bytes).") + withContext(Dispatchers.Main) { + Toast.makeText(context, "Imported file is empty." as CharSequence, Toast.LENGTH_LONG).show() + } + return@launch + } + + context.contentResolver.openInputStream(uri)?.use { inputStream -> + Log.i(TAG_IMPORT_PROCESS, "InputStream opened. Reading text on thread: ${Thread.currentThread().name}") + val jsonString = inputStream.bufferedReader().readText() + Log.i(TAG_IMPORT_PROCESS, "File content read. Size: ${jsonString.length} chars.") + Log.v(TAG_IMPORT_PROCESS, "File content snippet: ${jsonString.take(500)}") + + if (jsonString.isBlank()) { + Log.w(TAG_IMPORT_PROCESS, "Imported file content is blank.") + withContext(Dispatchers.Main) { + Toast.makeText(context, "Imported file content is blank." as CharSequence, Toast.LENGTH_LONG).show() + } + return@use + } + + Log.d(TAG_IMPORT_PROCESS, "Attempting to parse JSON string on thread: ${Thread.currentThread().name}") + val parsedEntries = Json.decodeFromString(ListSerializer(SystemMessageEntry.serializer()), jsonString) + Log.i(TAG_IMPORT_PROCESS, "JSON parsed. Found ${parsedEntries.size} entries.") + + val currentSystemEntries = SystemMessageEntryPreferences.loadEntries(context) + Log.d(TAG_IMPORT_PROCESS, "Current system entries loaded: ${currentSystemEntries.size} entries.") + + withContext(Dispatchers.Main) { + Log.d(TAG_IMPORT_PROCESS, "Switching to Main thread for processImportedEntries: ${Thread.currentThread().name}") + skipAllDuplicates = false + processImportedEntries( + imported = parsedEntries, + currentSystemEntries = currentSystemEntries + ) + } + } ?: Log.w(TAG_IMPORT_PROCESS, "ContentResolver.openInputStream returned null for URI: $uri (second check).") + } catch (oom: OutOfMemoryError) { + Log.e(TAG_IMPORT_PROCESS, "Out of memory during file import for URI: $uri on thread: ${Thread.currentThread().name}", oom) + withContext(Dispatchers.Main) { + Toast.makeText( + context, + "Error importing file: Out of memory. File may be too large or contain too many entries." as CharSequence, + Toast.LENGTH_LONG + ).show() + } + } catch (e: Exception) { + Log.e(TAG_IMPORT_PROCESS, "Error during file import for URI: $uri on thread: ${Thread.currentThread().name}", e) + withContext(Dispatchers.Main) { + val errorMessage = e.message ?: "Unknown error during import." + Toast.makeText(context, "Error importing file: $errorMessage" as CharSequence, Toast.LENGTH_LONG).show() + } + } + } + } + ) + + if (entryToConfirmOverwrite != null) { + val (existingEntry, newEntry) = checkNotNull(entryToConfirmOverwrite) + OverwriteConfirmationDialog( + entryTitle = newEntry.title, + onConfirm = { + Log.d(TAG_IMPORT_PROCESS, "Overwrite confirmed for title: '${newEntry.title}'") + SystemMessageEntryPreferences.updateEntry(context, existingEntry, newEntry) + Toast.makeText(context, "Entry '${newEntry.title}' overwritten." as CharSequence, Toast.LENGTH_SHORT).show() + entryToConfirmOverwrite = null + val currentSystemEntriesAfterUpdate = SystemMessageEntryPreferences.loadEntries(context) + Log.d(TAG_IMPORT_PROCESS, "Continuing with remaining ${remainingEntriesToImport.size} entries after dialog (Confirm).") + processImportedEntries( + imported = remainingEntriesToImport, + currentSystemEntries = currentSystemEntriesAfterUpdate + ) + }, + onDeny = { + Log.d(TAG_IMPORT_PROCESS, "Overwrite denied for title: '${newEntry.title}'") + entryToConfirmOverwrite = null + val currentSystemEntriesAfterDeny = SystemMessageEntryPreferences.loadEntries(context) + Log.d(TAG_IMPORT_PROCESS, "Continuing with remaining ${remainingEntriesToImport.size} entries after dialog (Deny).") + processImportedEntries( + imported = remainingEntriesToImport, + currentSystemEntries = currentSystemEntriesAfterDeny + ) + }, + onSkipAll = { + Log.d(TAG_IMPORT_PROCESS, "Skip All selected for title: '${newEntry.title}'") + skipAllDuplicates = true + entryToConfirmOverwrite = null + val currentSystemEntriesAfterSkipAll = SystemMessageEntryPreferences.loadEntries(context) + Log.d(TAG_IMPORT_PROCESS, "Continuing with remaining ${remainingEntriesToImport.size} entries after dialog (SkipAll).") + processImportedEntries( + imported = remainingEntriesToImport, + currentSystemEntries = currentSystemEntriesAfterSkipAll + ) + }, + onDismiss = { + Log.d(TAG_IMPORT_PROCESS, "Overwrite dialog dismissed for title: '${entryToConfirmOverwrite?.second?.title}'. Import process for this batch might halt.") + entryToConfirmOverwrite = null + remainingEntriesToImport = emptyList() + skipAllDuplicates = false + Toast.makeText(context, "Import cancelled for remaining items." as CharSequence, Toast.LENGTH_SHORT).show() + onImportCompleted() + } + ) + } + + + Dialog( + onDismissRequest = onDismissRequest, + properties = DialogProperties(usePlatformDefaultWidth = false) + ) { + Card( + modifier = Modifier + .fillMaxWidth(0.95f) + .fillMaxHeight(0.85f), + shape = RoundedCornerShape(16.dp), + colors = CardDefaults.cardColors(containerColor = DarkYellow1) + ) { + Column( + modifier = Modifier + .padding(16.dp) + .fillMaxSize() + ) { + val displayRowCount = 15 + val newButtonRowIndex = entries.size + + LazyColumn(modifier = Modifier.weight(1f)) { + items(displayRowCount) { rowIndex -> + val currentAlternatingColor = if (rowIndex % 2 == 0) DarkYellow1 else DarkYellow2 + + when { + rowIndex < entries.size -> { + val entry = entries[rowIndex] + val isSelected = selectedEntryTitles.contains(entry.title) + + Row( + modifier = Modifier + .fillMaxWidth() + .background(currentAlternatingColor) + .clickable { + if (selectionModeActive) { + val entryTitle = entry.title + val isCurrentlySelected = selectedEntryTitles.contains(entryTitle) + selectedEntryTitles = if (isCurrentlySelected) { + selectedEntryTitles - entryTitle + } else { + selectedEntryTitles + entryTitle + } + selectAllChecked = if (selectedEntryTitles.size == entries.size && entries.isNotEmpty()) true else if (selectedEntryTitles.isEmpty()) false else false + } else { + onEntryClicked(entry) + } + } + .padding(horizontal = 8.dp, vertical = 4.dp), + verticalAlignment = Alignment.CenterVertically + ) { + if (selectionModeActive) { + Checkbox( + checked = isSelected, + onCheckedChange = { isChecked -> + val entryTitle = entry.title + selectedEntryTitles = if (isChecked) { + selectedEntryTitles + entryTitle + } else { + selectedEntryTitles - entryTitle + } + selectAllChecked = if (selectedEntryTitles.size == entries.size && entries.isNotEmpty()) true else if (selectedEntryTitles.isEmpty()) false else false + }, + modifier = Modifier.padding(end = 8.dp), + colors = CheckboxDefaults.colors(checkedColor = MaterialTheme.colorScheme.primary) + ) + } + Box( + modifier = Modifier + .weight(1f) + .border(BorderStroke(1.dp, Color.Black), shape = RoundedCornerShape(12.dp)) + .padding(horizontal = 12.dp, vertical = 8.dp), + contentAlignment = Alignment.CenterStart + ) { + Text(entry.title, color = Color.Black, style = MaterialTheme.typography.bodyLarge) + } + if (!selectionModeActive) { + Box { + IconButton(onClick = { entryMenuToShow = entry }) { + Icon(Icons.Filled.MoreVert, "More options", tint = Color.Black) + } + DropdownMenu( + expanded = entryMenuToShow == entry, + onDismissRequest = { entryMenuToShow = null } + ) { + DropdownMenuItem( + text = { Text("Delete") }, + onClick = { + onDeleteClicked(entry) + entryMenuToShow = null + } + ) + } + } + } else { + Spacer(modifier = Modifier.width(48.dp)) + } + } + } + rowIndex == newButtonRowIndex -> { + Row( + modifier = Modifier.fillMaxWidth().background(currentAlternatingColor) + .padding(8.dp).clickable(onClick = onNewClicked), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.End + ) { + Text("This is also sent to the AI", color = Color.Black.copy(alpha = 0.6f), style = MaterialTheme.typography.bodyMedium, modifier = Modifier.weight(1f)) + Button(onClick = onNewClicked, colors = ButtonDefaults.buttonColors(containerColor = MaterialTheme.colorScheme.primary), modifier = Modifier.padding(start = 8.dp)) { + Text("New") + } + } + } + else -> { + Box(modifier = Modifier.fillMaxWidth().height(56.dp).background(currentAlternatingColor).padding(16.dp)) {} + } + } + } + } + Spacer(modifier = Modifier.height(8.dp)) + Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.SpaceBetween, verticalAlignment = Alignment.CenterVertically) { + if (selectionModeActive) { + Row(verticalAlignment = Alignment.CenterVertically) { + Checkbox( + checked = selectAllChecked, + onCheckedChange = { isChecked -> + selectAllChecked = isChecked + selectedEntryTitles = if (isChecked) entries.map { it.title }.toSet() else emptySet() + }, + colors = CheckboxDefaults.colors(checkedColor = MaterialTheme.colorScheme.primary) + ) + Text("All", color = Color.Black, style = MaterialTheme.typography.bodyMedium) + } + } else { + Spacer(modifier = Modifier.width(80.dp)) // Placeholder for alignment + } + + Row(verticalAlignment = Alignment.CenterVertically) { + Button(onClick = { filePickerLauncher.launch("*/*") }, colors = ButtonDefaults.buttonColors(containerColor = MaterialTheme.colorScheme.primary), modifier = Modifier.padding(end = 8.dp)) { Text("Import") } + Button( + onClick = { + if (selectionModeActive) { + if (selectedEntryTitles.isEmpty()) { + Toast.makeText(context, "No entries selected for export." as CharSequence, Toast.LENGTH_SHORT).show() + } else { + val entriesToExport = entries.filter { selectedEntryTitles.contains(it.title) } + val jsonString = Json.encodeToString(ListSerializer(SystemMessageEntry.serializer()), entriesToExport) + shareTextFile(context, "system_messages_export.txt", jsonString) + } + selectionModeActive = false + selectedEntryTitles = emptySet() + selectAllChecked = false + } else { + selectionModeActive = true + } + }, + colors = ButtonDefaults.buttonColors(containerColor = MaterialTheme.colorScheme.primary) + ) { Text("Export") } // Text is now always "Export" + } + } + } + } + } +} + + +@Composable +fun OverwriteConfirmationDialog( + entryTitle: String, + onConfirm: () -> Unit, + onDeny: () -> Unit, + onSkipAll: () -> Unit, + onDismiss: () -> Unit +) { + AlertDialog( + onDismissRequest = onDismiss, + title = { Text("Confirm Overwrite") }, + text = { Text("An entry with the title \"$entryTitle\" already exists. Do you want to overwrite its guide?") }, + confirmButton = { + TextButton(onClick = onConfirm) { Text("Yes") } + }, + dismissButton = { + Row { + TextButton(onClick = onSkipAll) { Text("Skip All") } + TextButton(onClick = onDeny) { Text("No") } + } + } + ) +} diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningEntryEditor.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningEntryEditor.kt new file mode 100644 index 0000000..e292cee --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningEntryEditor.kt @@ -0,0 +1,124 @@ +package com.google.ai.sample.feature.multimodal + +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxHeight +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.Button +import androidx.compose.material3.ButtonDefaults +import androidx.compose.material3.Card +import androidx.compose.material3.CardDefaults +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedTextField +import androidx.compose.material3.Text +import androidx.compose.material3.TextFieldDefaults +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.saveable.Saver +import androidx.compose.runtime.saveable.rememberSaveable +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.unit.dp +import androidx.compose.ui.window.Dialog +import androidx.compose.ui.window.DialogProperties +import com.google.ai.sample.util.SystemMessageEntry + +// Reuse shared colors from PhotoReasoningScreen.kt + +@Composable +fun EditEntryPopup( + entry: SystemMessageEntry?, + onDismissRequest: () -> Unit, + onSaveClicked: (title: String, guide: String, originalEntry: SystemMessageEntry?) -> Unit +) { + Dialog( + onDismissRequest = onDismissRequest, + properties = DialogProperties(usePlatformDefaultWidth = false) + ) { + Card( + modifier = Modifier.fillMaxWidth(0.9f).fillMaxHeight(0.7f).padding(16.dp), + shape = RoundedCornerShape(16.dp), + colors = CardDefaults.cardColors(containerColor = DarkYellow1) + ) { + Column(modifier = Modifier.padding(16.dp).fillMaxSize()) { + var titleInput by rememberSaveable { mutableStateOf(entry?.title ?: "") } + var guideInput by rememberSaveable { mutableStateOf(entry?.guide ?: "") } + + Text("Title", style = MaterialTheme.typography.labelMedium, color = Color.Black.copy(alpha = 0.7f)) + Spacer(modifier = Modifier.height(4.dp)) + OutlinedTextField( + value = titleInput, + onValueChange = { titleInput = it }, + placeholder = { Text("App/Task", color = Color.Gray) }, + modifier = Modifier.fillMaxWidth(), + colors = TextFieldDefaults.colors( + focusedContainerColor = Color.White, + unfocusedContainerColor = Color.White, + disabledContainerColor = Color.White, + cursorColor = Color.Black, + focusedTextColor = Color.Black, + unfocusedTextColor = Color.Black, + ), + singleLine = true + ) + Spacer(modifier = Modifier.height(8.dp)) + Text("Guide", style = MaterialTheme.typography.labelMedium, color = Color.Black.copy(alpha = 0.7f)) + Spacer(modifier = Modifier.height(4.dp)) + OutlinedTextField( + value = guideInput, + onValueChange = { guideInput = it }, + placeholder = { Text("Write a guide for an LLM on how it should perform certain tasks to be successful", color = Color.Gray) }, + modifier = Modifier.fillMaxWidth().weight(1f), + colors = TextFieldDefaults.colors( + focusedContainerColor = Color.White, + unfocusedContainerColor = Color.White, + disabledContainerColor = Color.White, + cursorColor = Color.Black, + focusedTextColor = Color.Black, + unfocusedTextColor = Color.Black, + ), + minLines = 5 + ) + Spacer(modifier = Modifier.height(16.dp)) + Button( + onClick = { onSaveClicked(titleInput, guideInput, entry) }, + modifier = Modifier.align(Alignment.End), + colors = ButtonDefaults.buttonColors(containerColor = MaterialTheme.colorScheme.primary) + ) { Text("Save") } + } + } + } +} + +@Preview(showBackground = true, name = "EditEntryPopup New") +@Composable +fun EditEntryPopupNewPreview() { + MaterialTheme { + EditEntryPopup(entry = null, onDismissRequest = {}, onSaveClicked = { _, _, _ -> }) + } +} + +@Preview(showBackground = true, name = "EditEntryPopup Edit") +@Composable +fun EditEntryPopupEditPreview() { + MaterialTheme { + EditEntryPopup(entry = SystemMessageEntry("Existing Title", "Existing Guide"), onDismissRequest = {}, onSaveClicked = { _, _, _ -> }) + } +} + +val SystemMessageEntrySaver = Saver>( + save = { entry -> if (entry == null) listOf(null, null) else listOf(entry.title, entry.guide) }, + restore = { list -> + val title = list[0] + val guide = list[1] + if (title != null && guide != null) SystemMessageEntry(title, guide) else null + } +) diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningHistoryBuilder.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningHistoryBuilder.kt new file mode 100644 index 0000000..7e76089 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningHistoryBuilder.kt @@ -0,0 +1,70 @@ +package com.google.ai.sample.feature.multimodal + +import com.google.ai.client.generativeai.type.Content +import com.google.ai.client.generativeai.type.content + +internal object PhotoReasoningHistoryBuilder { + fun buildInitialHistory( + systemMessage: String, + formattedDbEntries: String + ): List { + val initialHistory = mutableListOf() + if (systemMessage.isNotBlank()) { + initialHistory.add(content(role = "user") { text(systemMessage) }) + } + if (formattedDbEntries.isNotBlank()) { + initialHistory.add(content(role = "user") { text(formattedDbEntries) }) + } + return initialHistory + } + + fun buildHistoryFromMessages( + messages: List, + systemMessage: String, + formattedDbEntries: String + ): List { + val history = buildInitialHistory(systemMessage, formattedDbEntries).toMutableList() + + var currentUserContent = "" + var currentModelContent = "" + + for (message in messages) { + when (message.participant) { + PhotoParticipant.USER -> { + if (currentModelContent.isNotEmpty()) { + history.add(content(role = "model") { text(currentModelContent) }) + currentModelContent = "" + } + if (currentUserContent.isNotEmpty()) { + currentUserContent += "\n\n" + } + currentUserContent += message.text + } + + PhotoParticipant.MODEL -> { + if (currentUserContent.isNotEmpty()) { + history.add(content(role = "user") { text(currentUserContent) }) + currentUserContent = "" + } + if (currentModelContent.isNotEmpty()) { + currentModelContent += "\n\n" + } + currentModelContent += message.text + } + + PhotoParticipant.ERROR -> { + continue + } + } + } + + if (currentUserContent.isNotEmpty()) { + history.add(content(role = "user") { text(currentUserContent) }) + } + if (currentModelContent.isNotEmpty()) { + history.add(content(role = "model") { text(currentModelContent) }) + } + + return history + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningMessageMutations.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningMessageMutations.kt new file mode 100644 index 0000000..b7f2e18 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningMessageMutations.kt @@ -0,0 +1,95 @@ +package com.google.ai.sample.feature.multimodal + +internal object PhotoReasoningMessageMutations { + fun appendUserAndPendingModelMessages( + chatState: PhotoReasoningChatState, + userMessage: PhotoReasoningMessage + ): List { + chatState.addMessage(userMessage) + chatState.addMessage( + PhotoReasoningMessage( + text = "", + participant = PhotoParticipant.MODEL, + isPending = true + ) + ) + return chatState.getAllMessages() + } + + fun appendErrorMessage( + chatState: PhotoReasoningChatState, + errorText: String + ): List { + chatState.replaceLastPendingMessage() + chatState.addMessage( + PhotoReasoningMessage( + text = errorText, + participant = PhotoParticipant.ERROR + ) + ) + return chatState.getAllMessages() + } + + fun finalizeAiMessage( + chatState: PhotoReasoningChatState, + finalText: String + ): List { + val messages = chatState.getAllMessages().toMutableList() + val lastMessageIndex = messages.indexOfLast { + it.participant == PhotoParticipant.MODEL && it.isPending + } + + if (lastMessageIndex != -1) { + messages[lastMessageIndex] = messages[lastMessageIndex].copy(text = finalText, isPending = false) + chatState.setAllMessages(messages) + } else { + chatState.addMessage( + PhotoReasoningMessage( + text = finalText, + participant = PhotoParticipant.MODEL, + isPending = false + ) + ) + } + return chatState.getAllMessages() + } + + fun updateAiMessage( + chatState: PhotoReasoningChatState, + text: String, + isPending: Boolean + ): List { + val messages = chatState.getAllMessages().toMutableList() + val lastAiMessageIndex = messages.indexOfLast { it.participant == PhotoParticipant.MODEL } + + if (lastAiMessageIndex != -1 && messages[lastAiMessageIndex].isPending) { + val updatedMessage = messages[lastAiMessageIndex].let { + it.copy(text = it.text + text, isPending = isPending) + } + messages[lastAiMessageIndex] = updatedMessage + } else { + messages.add(PhotoReasoningMessage(text = text, participant = PhotoParticipant.MODEL, isPending = isPending)) + } + + chatState.setAllMessages(messages) + return chatState.getAllMessages() + } + + fun replaceAiMessageText( + chatState: PhotoReasoningChatState, + text: String, + isPending: Boolean + ): List { + val messages = chatState.getAllMessages().toMutableList() + val lastAiMessageIndex = messages.indexOfLast { it.participant == PhotoParticipant.MODEL } + + if (lastAiMessageIndex != -1 && messages[lastAiMessageIndex].isPending) { + messages[lastAiMessageIndex] = messages[lastAiMessageIndex].copy(text = text, isPending = isPending) + } else { + messages.add(PhotoReasoningMessage(text = text, participant = PhotoParticipant.MODEL, isPending = isPending)) + } + + chatState.setAllMessages(messages) + return chatState.getAllMessages() + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningOpenAiStreamParser.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningOpenAiStreamParser.kt new file mode 100644 index 0000000..4cffee5 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningOpenAiStreamParser.kt @@ -0,0 +1,66 @@ +package com.google.ai.sample.feature.multimodal + +import kotlinx.serialization.Serializable +import kotlinx.serialization.SerializationException +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.json.Json +import java.io.IOException + +internal class PhotoReasoningOpenAiStreamParser( + private val json: Json +) { + /** + * Liest einen OpenAI-kompatiblen SSE-Stream zeilenweise. + * Ruft [onChunk] mit dem jeweils akkumulierten Text auf. + * Gibt den vollständigen Text zurück. + */ + suspend fun parse( + body: okhttp3.ResponseBody, + onChunk: suspend (accumulatedText: String) -> Unit + ): String { + val accumulated = StringBuilder() + val reader = body.charStream().buffered() + try { + var line: String? + while (reader.readLine().also { line = it } != null) { + val currentLine = line ?: break + if (currentLine.startsWith("data: ")) { + val data = currentLine.removePrefix("data: ").trim() + if (data == "[DONE]") break + if (data.isEmpty()) continue + try { + val chunk = json.decodeFromString(data) + val delta = chunk.choices.firstOrNull()?.delta?.content + if (!delta.isNullOrEmpty()) { + accumulated.append(delta) + onChunk(accumulated.toString()) + } + } catch (_: SerializationException) { + // Fehlerhafte Chunk überspringen + } + } + } + } finally { + reader.close() + } + return accumulated.toString() + } +} + +/** Thrown when Mistral returns HTTP 429 (rate limit). */ +internal class MistralRateLimitException(message: String) : IOException(message) + +@Serializable +internal data class OpenAIStreamChunk( + val choices: List +) + +@Serializable +internal data class OpenAIStreamChoice( + val delta: OpenAIStreamDelta +) + +@Serializable +internal data class OpenAIStreamDelta( + val content: String? = null +) diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningPreviews.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningPreviews.kt new file mode 100644 index 0000000..e0b8da7 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningPreviews.kt @@ -0,0 +1,82 @@ +package com.google.ai.sample.feature.multimodal + +import androidx.compose.foundation.layout.PaddingValues +import androidx.compose.material3.MaterialTheme +import androidx.compose.runtime.Composable +import androidx.compose.ui.tooling.preview.Preview +import kotlinx.coroutines.flow.MutableStateFlow +import com.google.ai.sample.util.Command +import com.google.ai.sample.util.SystemMessageEntry + +@Composable +fun PhotoReasoningScreenPreviewWithContent() { + MaterialTheme { + PhotoReasoningScreen( + innerPadding = PaddingValues(), + uiState = PhotoReasoningUiState.Success("This is a preview of the photo reasoning screen."), + commandExecutionStatus = "Command executed: Take screenshot", + detectedCommands = listOf( + Command.TakeScreenshot, + Command.ClickButton("OK") + ), + systemMessage = "This is a system message for the AI", + chatMessages = MutableStateFlow(listOf( + PhotoReasoningMessage(text = "Hello, how can I help you?", participant = PhotoParticipant.USER), + PhotoReasoningMessage(text = "I am here to help you. What do you want to know?", participant = PhotoParticipant.MODEL) + )), + isKeyboardOpen = false, + onStopClicked = {}, + isInitialized = true + ) + } +} + +@Composable +@Preview(showSystemUi = true) +fun PhotoReasoningScreenPreviewEmpty() { + MaterialTheme { + PhotoReasoningScreen( + innerPadding = PaddingValues(), + chatMessages = MutableStateFlow>(emptyList()), + isKeyboardOpen = false, + onStopClicked = {}, + isInitialized = true + ) + } +} + +@Preview(showBackground = true) +@Composable +fun DatabaseListPopupPreview() { + MaterialTheme { + DatabaseListPopup( + onDismissRequest = {}, + entries = listOf( + SystemMessageEntry("Title 1", "Guide for prompt 1"), + SystemMessageEntry("Title 2", "Another guide for prompt 2"), + SystemMessageEntry("Title 3", "Yet another guide for prompt 3.") + ), + onNewClicked = {}, + onEntryClicked = {}, + onDeleteClicked = {}, + onImportCompleted = {} + ) + } +} + +@Preview(showBackground = true, name = "DatabaseListPopup Empty") +@Composable +fun DatabaseListPopupEmptyPreview() { + MaterialTheme { + DatabaseListPopup(onDismissRequest = {}, entries = emptyList(), onNewClicked = {}, onEntryClicked = {}, onDeleteClicked = {}, onImportCompleted = {}) + } +} + +@Preview(showBackground = true, name = "Stop Button Preview") +@Composable +fun StopButtonPreview() { + MaterialTheme { + StopButton {} + } +} + diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningProviderDtos.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningProviderDtos.kt new file mode 100644 index 0000000..8ade575 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningProviderDtos.kt @@ -0,0 +1,89 @@ +package com.google.ai.sample.feature.multimodal + +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.JsonClassDiscriminator + +// Data classes for Cerebras API +@Serializable +internal data class CerebrasRequest( + val model: String, + val messages: List, + val max_completion_tokens: Int = 1024, + val temperature: Double = 0.2, + val top_p: Double = 1.0, + val stream: Boolean = false +) + +@Serializable +internal data class CerebrasMessage( + val role: String, + val content: String +) + +@Serializable +internal data class CerebrasResponse( + val choices: List +) + +@Serializable +internal data class CerebrasChoice( + val message: CerebrasResponseMessage +) + +@Serializable +internal data class CerebrasResponseMessage( + val role: String, + val content: String +) + +// Data classes for Mistral API +@Serializable +internal data class MistralRequest( + val model: String, + val messages: List, + val max_tokens: Int = 4096, + val temperature: Double = 0.7, + val top_p: Double = 1.0, + val stream: Boolean = false +) + +@Serializable +internal data class MistralMessage( + val role: String, + val content: List +) + +@Serializable +@OptIn(ExperimentalSerializationApi::class) +@JsonClassDiscriminator("type") +internal sealed class MistralContent + +@Serializable +@kotlinx.serialization.SerialName("text") +internal data class MistralTextContent(val text: String) : MistralContent() + +@Serializable +@kotlinx.serialization.SerialName("image_url") +internal data class MistralImageContent( + @kotlinx.serialization.SerialName("image_url") val imageUrl: MistralImageUrl +) : MistralContent() + +@Serializable +internal data class MistralImageUrl(val url: String) + +@Serializable +internal data class MistralResponse( + val choices: List +) + +@Serializable +internal data class MistralChoice( + val message: MistralResponseMessage +) + +@Serializable +internal data class MistralResponseMessage( + val role: String, + val content: String +) diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningRoute.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningRoute.kt new file mode 100644 index 0000000..bed5c8c --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningRoute.kt @@ -0,0 +1,142 @@ +package com.google.ai.sample.feature.multimodal + +import android.content.Intent +import android.graphics.drawable.BitmapDrawable +import android.provider.Settings +import android.net.Uri +import android.widget.Toast +import androidx.activity.compose.BackHandler +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.foundation.layout.PaddingValues +import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.runtime.saveable.rememberSaveable +import androidx.compose.runtime.setValue +import androidx.compose.ui.platform.LocalContext +import androidx.lifecycle.viewmodel.compose.viewModel +import coil.ImageLoader +import coil.request.ImageRequest +import coil.request.SuccessResult +import coil.size.Precision +import com.google.ai.sample.ApiProvider +import com.google.ai.sample.GenerativeViewModelFactory +import com.google.ai.sample.MainActivity +import com.google.ai.sample.ModelOption +import kotlinx.coroutines.launch + +@Composable +internal fun PhotoReasoningRoute( + innerPadding: PaddingValues, // Füge Parameter hinzu + viewModelStoreOwner: androidx.lifecycle.ViewModelStoreOwner = checkNotNull( + androidx.lifecycle.viewmodel.compose.LocalViewModelStoreOwner.current + ) { "ViewModelStoreOwner is required" } +) { + val context = LocalContext.current + val mainActivity = context as? MainActivity + + // Scoped to MainActivity so it survives navigation and avoids duplicate initialization. + val owner = mainActivity ?: viewModelStoreOwner + val viewModel: PhotoReasoningViewModel = androidx.lifecycle.viewmodel.compose.viewModel( + viewModelStoreOwner = owner, + factory = GenerativeViewModelFactory + ) + val photoReasoningUiState by viewModel.uiState.collectAsState() + val commandExecutionStatus by viewModel.commandExecutionStatus.collectAsState() + val detectedCommands by viewModel.detectedCommands.collectAsState() + val systemMessage by viewModel.systemMessage.collectAsState() + val isInitialized by viewModel.isInitialized.collectAsState() + val modelName by viewModel.modelNameState.collectAsState() + val userInput by viewModel.userInput.collectAsState() + val isGenerationRunning by viewModel.isGenerationRunningFlow.collectAsState() + val isOfflineGpuModelLoaded by viewModel.isOfflineGpuModelLoadedFlow.collectAsState() + val isInitializingOfflineModel by viewModel.isInitializingOfflineModelFlow.collectAsState() + + // Hoisted: var showNotificationRationaleDialog by rememberSaveable { mutableStateOf(false) } + // This state will now be managed in PhotoReasoningRoute and passed down. + + + val coroutineScope = rememberCoroutineScope() + val imageRequestBuilder = ImageRequest.Builder(LocalContext.current) + val imageLoader = ImageLoader.Builder(LocalContext.current).build() + + val isAccessibilityServiceEffectivelyEnabled by mainActivity?.isAccessibilityServiceEnabledFlow?.collectAsState() ?: mutableStateOf(false) + val isMediaProjectionPermissionGranted by mainActivity?.isMediaProjectionPermissionGrantedFlow?.collectAsState() ?: mutableStateOf(false) + val isKeyboardOpen by mainActivity?.isKeyboardOpen?.collectAsState() ?: mutableStateOf(false) + + val accessibilitySettingsLauncher = rememberLauncherForActivityResult( + contract = ActivityResultContracts.StartActivityForResult() + ) { + mainActivity?.refreshAccessibilityServiceStatus() + } + + DisposableEffect(viewModel, mainActivity) { + mainActivity?.setPhotoReasoningViewModel(viewModel) + mainActivity?.refreshAccessibilityServiceStatus() + viewModel.loadSystemMessage(context) + viewModel.collectLiveApiMessages() + onDispose { } + } + + PhotoReasoningScreen( + innerPadding = innerPadding, // Übergebe an PhotoReasoningScreen + uiState = photoReasoningUiState, + commandExecutionStatus = commandExecutionStatus, + detectedCommands = detectedCommands, + systemMessage = systemMessage, + chatMessages = viewModel.chatMessagesFlow, + onSystemMessageChanged = { message -> + viewModel.updateSystemMessage(message, context) + }, + onRestoreSystemMessageClicked = { + viewModel.restoreSystemMessage(context) + }, + onReasonClicked = { inputText, selectedItems -> + coroutineScope.launch { + val bitmaps = selectedItems.mapNotNull { + val imageRequest = imageRequestBuilder.data(it).precision(Precision.EXACT).build() + try { + val result = imageLoader.execute(imageRequest) + if (result is SuccessResult) (result.drawable as BitmapDrawable).bitmap else null + } catch (e: Exception) { null } + } + viewModel.reason( + userInput = inputText, + selectedImages = bitmaps, + screenInfoForPrompt = null, // User-initiated messages don't have prior screen context here + imageUrisForChat = selectedItems.map { it.toString() } + ) + } + }, + isAccessibilityServiceEnabled = isAccessibilityServiceEffectivelyEnabled, + isMediaProjectionPermissionGranted = isMediaProjectionPermissionGranted, + onEnableAccessibilityService = { + try { + // val intent = Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS) // Corrected + // accessibilitySettingsLauncher.launch(intent) // Corrected below + val intent = Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS) + accessibilitySettingsLauncher.launch(intent) + } catch (e: Exception) { + Toast.makeText(context, "Error opening Accessibility Settings.", Toast.LENGTH_LONG).show() + } + }, + onClearChatHistory = { + mainActivity?.getPhotoReasoningViewModel()?.clearChatHistory(context) + }, + isKeyboardOpen = isKeyboardOpen, + onStopClicked = { viewModel.onStopClicked() }, + isInitialized = isInitialized, + modelName = modelName, + userQuestion = userInput, + onUserQuestionChanged = { viewModel.updateUserInput(it) }, + isGenerationRunning = isGenerationRunning, + isOfflineGpuModelLoaded = isOfflineGpuModelLoaded, + isInitializingOfflineModel = isInitializingOfflineModel + ) +} diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt index 1d2c926..db0f122 100644 --- a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt @@ -122,133 +122,6 @@ import kotlinx.serialization.json.Json import android.util.Log import kotlinx.serialization.SerializationException -// Define Colors -val DarkYellow1 = Color(0xFFF0A500) // A darker yellow -val DarkYellow2 = Color(0xFFF3C100) // A slightly lighter dark yellow - -@Composable -fun StopButton(onClick: () -> Unit) { - Button( - onClick = onClick, - colors = ButtonDefaults.buttonColors(containerColor = Color.Red), - modifier = Modifier - .fillMaxWidth() - .padding(8.dp) - ) { - Text("Stop", color = Color.White) - } -} - -@Composable -internal fun PhotoReasoningRoute( - innerPadding: PaddingValues, // Füge Parameter hinzu - viewModelStoreOwner: androidx.lifecycle.ViewModelStoreOwner = checkNotNull( - androidx.lifecycle.viewmodel.compose.LocalViewModelStoreOwner.current - ) { "ViewModelStoreOwner is required" } -) { - val context = LocalContext.current - val mainActivity = context as? MainActivity - - // Scoped to MainActivity so it survives navigation and avoids duplicate initialization. - val owner = mainActivity ?: viewModelStoreOwner - val viewModel: PhotoReasoningViewModel = androidx.lifecycle.viewmodel.compose.viewModel( - viewModelStoreOwner = owner, - factory = GenerativeViewModelFactory - ) - val photoReasoningUiState by viewModel.uiState.collectAsState() - val commandExecutionStatus by viewModel.commandExecutionStatus.collectAsState() - val detectedCommands by viewModel.detectedCommands.collectAsState() - val systemMessage by viewModel.systemMessage.collectAsState() - val isInitialized by viewModel.isInitialized.collectAsState() - val modelName by viewModel.modelNameState.collectAsState() - val userInput by viewModel.userInput.collectAsState() - val isGenerationRunning by viewModel.isGenerationRunningFlow.collectAsState() - val isOfflineGpuModelLoaded by viewModel.isOfflineGpuModelLoadedFlow.collectAsState() - val isInitializingOfflineModel by viewModel.isInitializingOfflineModelFlow.collectAsState() - - // Hoisted: var showNotificationRationaleDialog by rememberSaveable { mutableStateOf(false) } - // This state will now be managed in PhotoReasoningRoute and passed down. - - - val coroutineScope = rememberCoroutineScope() - val imageRequestBuilder = ImageRequest.Builder(LocalContext.current) - val imageLoader = ImageLoader.Builder(LocalContext.current).build() - - val isAccessibilityServiceEffectivelyEnabled by mainActivity?.isAccessibilityServiceEnabledFlow?.collectAsState() ?: mutableStateOf(false) - val isMediaProjectionPermissionGranted by mainActivity?.isMediaProjectionPermissionGrantedFlow?.collectAsState() ?: mutableStateOf(false) - val isKeyboardOpen by mainActivity?.isKeyboardOpen?.collectAsState() ?: mutableStateOf(false) - - val accessibilitySettingsLauncher = rememberLauncherForActivityResult( - contract = ActivityResultContracts.StartActivityForResult() - ) { - mainActivity?.refreshAccessibilityServiceStatus() - } - - DisposableEffect(viewModel, mainActivity) { - mainActivity?.setPhotoReasoningViewModel(viewModel) - mainActivity?.refreshAccessibilityServiceStatus() - viewModel.loadSystemMessage(context) - viewModel.collectLiveApiMessages() - onDispose { } - } - - PhotoReasoningScreen( - innerPadding = innerPadding, // Übergebe an PhotoReasoningScreen - uiState = photoReasoningUiState, - commandExecutionStatus = commandExecutionStatus, - detectedCommands = detectedCommands, - systemMessage = systemMessage, - chatMessages = viewModel.chatMessagesFlow, - onSystemMessageChanged = { message -> - viewModel.updateSystemMessage(message, context) - }, - onRestoreSystemMessageClicked = { - viewModel.restoreSystemMessage(context) - }, - onReasonClicked = { inputText, selectedItems -> - coroutineScope.launch { - val bitmaps = selectedItems.mapNotNull { - val imageRequest = imageRequestBuilder.data(it).precision(Precision.EXACT).build() - try { - val result = imageLoader.execute(imageRequest) - if (result is SuccessResult) (result.drawable as BitmapDrawable).bitmap else null - } catch (e: Exception) { null } - } - viewModel.reason( - userInput = inputText, - selectedImages = bitmaps, - screenInfoForPrompt = null, // User-initiated messages don't have prior screen context here - imageUrisForChat = selectedItems.map { it.toString() } - ) - } - }, - isAccessibilityServiceEnabled = isAccessibilityServiceEffectivelyEnabled, - isMediaProjectionPermissionGranted = isMediaProjectionPermissionGranted, - onEnableAccessibilityService = { - try { - // val intent = Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS) // Corrected - // accessibilitySettingsLauncher.launch(intent) // Corrected below - val intent = Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS) - accessibilitySettingsLauncher.launch(intent) - } catch (e: Exception) { - Toast.makeText(context, "Error opening Accessibility Settings.", Toast.LENGTH_LONG).show() - } - }, - onClearChatHistory = { - mainActivity?.getPhotoReasoningViewModel()?.clearChatHistory(context) - }, - isKeyboardOpen = isKeyboardOpen, - onStopClicked = { viewModel.onStopClicked() }, - isInitialized = isInitialized, - modelName = modelName, - userQuestion = userInput, - onUserQuestionChanged = { viewModel.updateUserInput(it) }, - isGenerationRunning = isGenerationRunning, - isOfflineGpuModelLoaded = isOfflineGpuModelLoaded, - isInitializingOfflineModel = isInitializingOfflineModel - ) -} - @Composable fun PhotoReasoningScreen( innerPadding: PaddingValues, // Füge Parameter hinzu @@ -644,763 +517,3 @@ fun PhotoReasoningScreen( ) } } - - -@Composable -fun DatabaseListPopup( - onDismissRequest: () -> Unit, - entries: List, - onNewClicked: () -> Unit, - onEntryClicked: (SystemMessageEntry) -> Unit, - onDeleteClicked: (SystemMessageEntry) -> Unit, - onImportCompleted: () -> Unit -) { - val TAG_IMPORT_PROCESS = "ImportProcess" - val scope = rememberCoroutineScope() - var entryMenuToShow: SystemMessageEntry? by remember { mutableStateOf(null) } - var selectionModeActive by rememberSaveable { mutableStateOf(false) } - var selectedEntryTitles by rememberSaveable { mutableStateOf(emptySet()) } - var selectAllChecked by rememberSaveable { mutableStateOf(false) } - val context = LocalContext.current - - var entryToConfirmOverwrite by remember { mutableStateOf?>(null) } - var remainingEntriesToImport by remember { mutableStateOf>(emptyList()) } - var skipAllDuplicates by remember { mutableStateOf(false) } - - // processImportedEntries is defined within DatabaseListPopup, so it has access to context, onImportCompleted, etc. - fun processImportedEntries( - imported: List, - currentSystemEntries: List - ) { - val TAG_IMPORT_PROCESS_FUNCTION = "ImportProcessFunction" - Log.d(TAG_IMPORT_PROCESS_FUNCTION, "Starting processImportedEntries. Imported: ${imported.size}, Current: ${currentSystemEntries.size}, SkipAll: $skipAllDuplicates") - - var newCount = 0 - var updatedCount = 0 - var skippedCount = 0 - val entriesToProcess = imported.toMutableList() - - while (entriesToProcess.isNotEmpty()) { - val newEntry = entriesToProcess.removeAt(0) - Log.d(TAG_IMPORT_PROCESS_FUNCTION, "Processing entry: Title='${newEntry.title}'. Remaining in batch: ${entriesToProcess.size}") - val existingEntry = currentSystemEntries.find { it.title.equals(newEntry.title, ignoreCase = true) } - - if (existingEntry != null) { - Log.d(TAG_IMPORT_PROCESS_FUNCTION, "Duplicate found for title: '${newEntry.title}'. Existing guide: '${existingEntry.guide.take(50)}', New guide: '${newEntry.guide.take(50)}'") - if (skipAllDuplicates) { - Log.d(TAG_IMPORT_PROCESS_FUNCTION, "Skipping duplicate '${newEntry.title}' due to skipAllDuplicates flag.") - skippedCount++ - continue - } - Log.d(TAG_IMPORT_PROCESS_FUNCTION, "Calling askForOverwrite for '${newEntry.title}'.") - entryToConfirmOverwrite = Pair(existingEntry, newEntry) - remainingEntriesToImport = entriesToProcess.toList() - return - } else { - Log.i(TAG_IMPORT_PROCESS_FUNCTION, "Adding new entry: Title='${newEntry.title}'") - SystemMessageEntryPreferences.addEntry(context, newEntry) - newCount++ - } - } - Log.i(TAG_IMPORT_PROCESS_FUNCTION, "Finished processing batch. newCount=$newCount, updatedCount=$updatedCount, skippedCount=$skippedCount") - val summary = "Import finished: $newCount added, $updatedCount updated, $skippedCount skipped." - Toast.makeText(context, summary as CharSequence, Toast.LENGTH_LONG).show() - onImportCompleted() - skipAllDuplicates = false - } - - - val filePickerLauncher = rememberLauncherForActivityResult( - contract = ActivityResultContracts.GetContent(), - onResult = { uri: Uri? -> - Log.d(TAG_IMPORT_PROCESS, "FilePickerLauncher onResult triggered.") - if (uri == null) { - Log.w(TAG_IMPORT_PROCESS, "URI is null, no file selected or operation cancelled.") - scope.launch(Dispatchers.Main) { - Toast.makeText(context, "No file selected." as CharSequence, Toast.LENGTH_SHORT).show() - } - return@rememberLauncherForActivityResult - } - - Log.i(TAG_IMPORT_PROCESS, "Selected file URI: $uri") - scope.launch(Dispatchers.Main) { - Toast.makeText(context, "File selected: $uri. Starting import..." as CharSequence, Toast.LENGTH_SHORT).show() - } - - scope.launch(Dispatchers.IO) { - try { - Log.d(TAG_IMPORT_PROCESS, "Attempting to open InputStream for URI: $uri on thread: ${Thread.currentThread().name}") - - var fileSize = -1L - try { - context.contentResolver.openFileDescriptor(uri, "r")?.use { pfd -> - fileSize = pfd.statSize - } - Log.i(TAG_IMPORT_PROCESS, "Estimated file size: $fileSize bytes.") - } catch (e: Exception) { - Log.w(TAG_IMPORT_PROCESS, "Could not determine file size for URI: $uri. Will proceed without size check.", e) - } - - val MAX_FILE_SIZE_BYTES = 10 * 1024 * 1024 - if (fileSize != -1L && fileSize > MAX_FILE_SIZE_BYTES) { - Log.e(TAG_IMPORT_PROCESS, "File size ($fileSize bytes) exceeds limit of $MAX_FILE_SIZE_BYTES bytes.") - withContext(Dispatchers.Main) { - Toast.makeText(context, "File is too large (max 10MB)." as CharSequence, Toast.LENGTH_LONG).show() - } - return@launch - } - if (fileSize == 0L) { - Log.w(TAG_IMPORT_PROCESS, "Imported file is empty (0 bytes).") - withContext(Dispatchers.Main) { - Toast.makeText(context, "Imported file is empty." as CharSequence, Toast.LENGTH_LONG).show() - } - return@launch - } - - context.contentResolver.openInputStream(uri)?.use { inputStream -> - Log.i(TAG_IMPORT_PROCESS, "InputStream opened. Reading text on thread: ${Thread.currentThread().name}") - val jsonString = inputStream.bufferedReader().readText() - Log.i(TAG_IMPORT_PROCESS, "File content read. Size: ${jsonString.length} chars.") - Log.v(TAG_IMPORT_PROCESS, "File content snippet: ${jsonString.take(500)}") - - if (jsonString.isBlank()) { - Log.w(TAG_IMPORT_PROCESS, "Imported file content is blank.") - withContext(Dispatchers.Main) { - Toast.makeText(context, "Imported file content is blank." as CharSequence, Toast.LENGTH_LONG).show() - } - return@use - } - - Log.d(TAG_IMPORT_PROCESS, "Attempting to parse JSON string on thread: ${Thread.currentThread().name}") - val parsedEntries = Json.decodeFromString(ListSerializer(SystemMessageEntry.serializer()), jsonString) - Log.i(TAG_IMPORT_PROCESS, "JSON parsed. Found ${parsedEntries.size} entries.") - - val currentSystemEntries = SystemMessageEntryPreferences.loadEntries(context) - Log.d(TAG_IMPORT_PROCESS, "Current system entries loaded: ${currentSystemEntries.size} entries.") - - withContext(Dispatchers.Main) { - Log.d(TAG_IMPORT_PROCESS, "Switching to Main thread for processImportedEntries: ${Thread.currentThread().name}") - skipAllDuplicates = false - processImportedEntries( - imported = parsedEntries, - currentSystemEntries = currentSystemEntries - ) - } - } ?: Log.w(TAG_IMPORT_PROCESS, "ContentResolver.openInputStream returned null for URI: $uri (second check).") - } catch (oom: OutOfMemoryError) { - Log.e(TAG_IMPORT_PROCESS, "Out of memory during file import for URI: $uri on thread: ${Thread.currentThread().name}", oom) - withContext(Dispatchers.Main) { - Toast.makeText( - context, - "Error importing file: Out of memory. File may be too large or contain too many entries." as CharSequence, - Toast.LENGTH_LONG - ).show() - } - } catch (e: Exception) { - Log.e(TAG_IMPORT_PROCESS, "Error during file import for URI: $uri on thread: ${Thread.currentThread().name}", e) - withContext(Dispatchers.Main) { - val errorMessage = e.message ?: "Unknown error during import." - Toast.makeText(context, "Error importing file: $errorMessage" as CharSequence, Toast.LENGTH_LONG).show() - } - } - } - } - ) - - if (entryToConfirmOverwrite != null) { - val (existingEntry, newEntry) = checkNotNull(entryToConfirmOverwrite) - OverwriteConfirmationDialog( - entryTitle = newEntry.title, - onConfirm = { - Log.d(TAG_IMPORT_PROCESS, "Overwrite confirmed for title: '${newEntry.title}'") - SystemMessageEntryPreferences.updateEntry(context, existingEntry, newEntry) - Toast.makeText(context, "Entry '${newEntry.title}' overwritten." as CharSequence, Toast.LENGTH_SHORT).show() - entryToConfirmOverwrite = null - val currentSystemEntriesAfterUpdate = SystemMessageEntryPreferences.loadEntries(context) - Log.d(TAG_IMPORT_PROCESS, "Continuing with remaining ${remainingEntriesToImport.size} entries after dialog (Confirm).") - processImportedEntries( - imported = remainingEntriesToImport, - currentSystemEntries = currentSystemEntriesAfterUpdate - ) - }, - onDeny = { - Log.d(TAG_IMPORT_PROCESS, "Overwrite denied for title: '${newEntry.title}'") - entryToConfirmOverwrite = null - val currentSystemEntriesAfterDeny = SystemMessageEntryPreferences.loadEntries(context) - Log.d(TAG_IMPORT_PROCESS, "Continuing with remaining ${remainingEntriesToImport.size} entries after dialog (Deny).") - processImportedEntries( - imported = remainingEntriesToImport, - currentSystemEntries = currentSystemEntriesAfterDeny - ) - }, - onSkipAll = { - Log.d(TAG_IMPORT_PROCESS, "Skip All selected for title: '${newEntry.title}'") - skipAllDuplicates = true - entryToConfirmOverwrite = null - val currentSystemEntriesAfterSkipAll = SystemMessageEntryPreferences.loadEntries(context) - Log.d(TAG_IMPORT_PROCESS, "Continuing with remaining ${remainingEntriesToImport.size} entries after dialog (SkipAll).") - processImportedEntries( - imported = remainingEntriesToImport, - currentSystemEntries = currentSystemEntriesAfterSkipAll - ) - }, - onDismiss = { - Log.d(TAG_IMPORT_PROCESS, "Overwrite dialog dismissed for title: '${entryToConfirmOverwrite?.second?.title}'. Import process for this batch might halt.") - entryToConfirmOverwrite = null - remainingEntriesToImport = emptyList() - skipAllDuplicates = false - Toast.makeText(context, "Import cancelled for remaining items." as CharSequence, Toast.LENGTH_SHORT).show() - onImportCompleted() - } - ) - } - - - Dialog( - onDismissRequest = onDismissRequest, - properties = DialogProperties(usePlatformDefaultWidth = false) - ) { - Card( - modifier = Modifier - .fillMaxWidth(0.95f) - .fillMaxHeight(0.85f), - shape = RoundedCornerShape(16.dp), - colors = CardDefaults.cardColors(containerColor = DarkYellow1) - ) { - Column( - modifier = Modifier - .padding(16.dp) - .fillMaxSize() - ) { - val displayRowCount = 15 - val newButtonRowIndex = entries.size - - LazyColumn(modifier = Modifier.weight(1f)) { - items(displayRowCount) { rowIndex -> - val currentAlternatingColor = if (rowIndex % 2 == 0) DarkYellow1 else DarkYellow2 - - when { - rowIndex < entries.size -> { - val entry = entries[rowIndex] - val isSelected = selectedEntryTitles.contains(entry.title) - - Row( - modifier = Modifier - .fillMaxWidth() - .background(currentAlternatingColor) - .clickable { - if (selectionModeActive) { - val entryTitle = entry.title - val isCurrentlySelected = selectedEntryTitles.contains(entryTitle) - selectedEntryTitles = if (isCurrentlySelected) { - selectedEntryTitles - entryTitle - } else { - selectedEntryTitles + entryTitle - } - selectAllChecked = if (selectedEntryTitles.size == entries.size && entries.isNotEmpty()) true else if (selectedEntryTitles.isEmpty()) false else false - } else { - onEntryClicked(entry) - } - } - .padding(horizontal = 8.dp, vertical = 4.dp), - verticalAlignment = Alignment.CenterVertically - ) { - if (selectionModeActive) { - Checkbox( - checked = isSelected, - onCheckedChange = { isChecked -> - val entryTitle = entry.title - selectedEntryTitles = if (isChecked) { - selectedEntryTitles + entryTitle - } else { - selectedEntryTitles - entryTitle - } - selectAllChecked = if (selectedEntryTitles.size == entries.size && entries.isNotEmpty()) true else if (selectedEntryTitles.isEmpty()) false else false - }, - modifier = Modifier.padding(end = 8.dp), - colors = CheckboxDefaults.colors(checkedColor = MaterialTheme.colorScheme.primary) - ) - } - Box( - modifier = Modifier - .weight(1f) - .border(BorderStroke(1.dp, Color.Black), shape = RoundedCornerShape(12.dp)) - .padding(horizontal = 12.dp, vertical = 8.dp), - contentAlignment = Alignment.CenterStart - ) { - Text(entry.title, color = Color.Black, style = MaterialTheme.typography.bodyLarge) - } - if (!selectionModeActive) { - Box { - IconButton(onClick = { entryMenuToShow = entry }) { - Icon(Icons.Filled.MoreVert, "More options", tint = Color.Black) - } - DropdownMenu( - expanded = entryMenuToShow == entry, - onDismissRequest = { entryMenuToShow = null } - ) { - DropdownMenuItem( - text = { Text("Delete") }, - onClick = { - onDeleteClicked(entry) - entryMenuToShow = null - } - ) - } - } - } else { - Spacer(modifier = Modifier.width(48.dp)) - } - } - } - rowIndex == newButtonRowIndex -> { - Row( - modifier = Modifier.fillMaxWidth().background(currentAlternatingColor) - .padding(8.dp).clickable(onClick = onNewClicked), - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.End - ) { - Text("This is also sent to the AI", color = Color.Black.copy(alpha = 0.6f), style = MaterialTheme.typography.bodyMedium, modifier = Modifier.weight(1f)) - Button(onClick = onNewClicked, colors = ButtonDefaults.buttonColors(containerColor = MaterialTheme.colorScheme.primary), modifier = Modifier.padding(start = 8.dp)) { - Text("New") - } - } - } - else -> { - Box(modifier = Modifier.fillMaxWidth().height(56.dp).background(currentAlternatingColor).padding(16.dp)) {} - } - } - } - } - Spacer(modifier = Modifier.height(8.dp)) - Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.SpaceBetween, verticalAlignment = Alignment.CenterVertically) { - if (selectionModeActive) { - Row(verticalAlignment = Alignment.CenterVertically) { - Checkbox( - checked = selectAllChecked, - onCheckedChange = { isChecked -> - selectAllChecked = isChecked - selectedEntryTitles = if (isChecked) entries.map { it.title }.toSet() else emptySet() - }, - colors = CheckboxDefaults.colors(checkedColor = MaterialTheme.colorScheme.primary) - ) - Text("All", color = Color.Black, style = MaterialTheme.typography.bodyMedium) - } - } else { - Spacer(modifier = Modifier.width(80.dp)) // Placeholder for alignment - } - - Row(verticalAlignment = Alignment.CenterVertically) { - Button(onClick = { filePickerLauncher.launch("*/*") }, colors = ButtonDefaults.buttonColors(containerColor = MaterialTheme.colorScheme.primary), modifier = Modifier.padding(end = 8.dp)) { Text("Import") } - Button( - onClick = { - if (selectionModeActive) { - if (selectedEntryTitles.isEmpty()) { - Toast.makeText(context, "No entries selected for export." as CharSequence, Toast.LENGTH_SHORT).show() - } else { - val entriesToExport = entries.filter { selectedEntryTitles.contains(it.title) } - val jsonString = Json.encodeToString(ListSerializer(SystemMessageEntry.serializer()), entriesToExport) - shareTextFile(context, "system_messages_export.txt", jsonString) - } - selectionModeActive = false - selectedEntryTitles = emptySet() - selectAllChecked = false - } else { - selectionModeActive = true - } - }, - colors = ButtonDefaults.buttonColors(containerColor = MaterialTheme.colorScheme.primary) - ) { Text("Export") } // Text is now always "Export" - } - } - } - } - } -} - - -@Composable -fun OverwriteConfirmationDialog( - entryTitle: String, - onConfirm: () -> Unit, - onDeny: () -> Unit, - onSkipAll: () -> Unit, - onDismiss: () -> Unit -) { - AlertDialog( - onDismissRequest = onDismiss, - title = { Text("Confirm Overwrite") }, - text = { Text("An entry with the title \"$entryTitle\" already exists. Do you want to overwrite its guide?") }, - confirmButton = { - TextButton(onClick = onConfirm) { Text("Yes") } - }, - dismissButton = { - Row { - TextButton(onClick = onSkipAll) { Text("Skip All") } - TextButton(onClick = onDeny) { Text("No") } - } - } - ) -} - - -@Composable -fun UserChatBubble( - text: String, - isPending: Boolean, - imageUris: List = emptyList(), - showUndo: Boolean = false, - onUndoClicked: () -> Unit = {} -) { - Row( - modifier = Modifier - .padding(vertical = 8.dp, horizontal = 8.dp) - .fillMaxWidth(), - verticalAlignment = Alignment.Top - ) { - if (showUndo) { - IconButton( - onClick = onUndoClicked, - modifier = Modifier - .padding(end = 8.dp) - .align(Alignment.CenterVertically) - ) { - Icon( - imageVector = Icons.Rounded.Refresh, - contentDescription = "Undo", - tint = Color.Gray - ) - } - } - Spacer(modifier = Modifier.weight(1f)) - Card( - shape = MaterialTheme.shapes.medium, - colors = CardDefaults.cardColors( - containerColor = MaterialTheme.colorScheme.primaryContainer - ), - modifier = Modifier.weight(4f) - ) { - Column( - modifier = Modifier.padding(all = 16.dp) - ) { - Text( - text = text, - color = MaterialTheme.colorScheme.onPrimaryContainer - ) - if (imageUris.isNotEmpty()) { - LazyRow( - modifier = Modifier.padding(top = 8.dp) - ) { - items(imageUris) { uri -> - AsyncImage( - model = uri, - contentDescription = null, - modifier = Modifier - .padding(4.dp) - .requiredSize(100.dp) - ) - } - } - } - if (isPending) { - CircularProgressIndicator( - modifier = Modifier - .padding(top = 8.dp) - .requiredSize(16.dp), - strokeWidth = 2.dp - ) - } - } - } - } -} - -@Composable -fun ModelChatBubble( - text: String, - isPending: Boolean -) { - Row( - modifier = Modifier - .padding(vertical = 8.dp, horizontal = 8.dp) - .fillMaxWidth(), - verticalAlignment = Alignment.Top - ) { - Card( - shape = MaterialTheme.shapes.medium, - colors = CardDefaults.cardColors( - containerColor = MaterialTheme.colorScheme.secondaryContainer - ), - modifier = Modifier.weight(4f) - ) { - Row( - modifier = Modifier.padding(all = 16.dp) - ) { - Icon( - Icons.Outlined.Person, - contentDescription = "AI Assistant", - tint = MaterialTheme.colorScheme.onSecondaryContainer, - modifier = Modifier - .requiredSize(24.dp) - .drawBehind { - drawCircle(color = Color.White) - } - .padding(end = 8.dp) - ) - Column { - Text( - text = text, - color = MaterialTheme.colorScheme.onSecondaryContainer - ) - if (isPending) { - CircularProgressIndicator( - modifier = Modifier - .padding(top = 8.dp) - .requiredSize(16.dp), - strokeWidth = 2.dp - ) - } - } - } - } - Spacer(modifier = Modifier.weight(1f)) - } -} - -@Composable -fun ErrorChatBubble( - text: String -) { - Box( - modifier = Modifier - .padding(vertical = 8.dp, horizontal = 8.dp) - .fillMaxWidth() - ) { - Card( - shape = MaterialTheme.shapes.medium, - colors = CardDefaults.cardColors( - containerColor = MaterialTheme.colorScheme.errorContainer - ), - modifier = Modifier.fillMaxWidth() - ) { - Text( - text = text, - color = MaterialTheme.colorScheme.error, - modifier = Modifier.padding(all = 16.dp) - ) - } - } -} - -@Preview -@Composable -fun PhotoReasoningScreenPreviewWithContent() { - MaterialTheme { - PhotoReasoningScreen( - innerPadding = PaddingValues(), - uiState = PhotoReasoningUiState.Success("This is a preview of the photo reasoning screen."), - commandExecutionStatus = "Command executed: Take screenshot", - detectedCommands = listOf( - Command.TakeScreenshot, - Command.ClickButton("OK") - ), - systemMessage = "This is a system message for the AI", - chatMessages = MutableStateFlow(listOf( - PhotoReasoningMessage(text = "Hello, how can I help you?", participant = PhotoParticipant.USER), - PhotoReasoningMessage(text = "I am here to help you. What do you want to know?", participant = PhotoParticipant.MODEL) - )), - isKeyboardOpen = false, - onStopClicked = {}, - isInitialized = true - ) - } -} - -@Composable -fun EditEntryPopup( - entry: SystemMessageEntry?, - onDismissRequest: () -> Unit, - onSaveClicked: (title: String, guide: String, originalEntry: SystemMessageEntry?) -> Unit -) { - Dialog( - onDismissRequest = onDismissRequest, - properties = DialogProperties(usePlatformDefaultWidth = false) - ) { - Card( - modifier = Modifier.fillMaxWidth(0.9f).fillMaxHeight(0.7f).padding(16.dp), - shape = RoundedCornerShape(16.dp), - colors = CardDefaults.cardColors(containerColor = DarkYellow1) - ) { - Column(modifier = Modifier.padding(16.dp).fillMaxSize()) { - var titleInput by rememberSaveable { mutableStateOf(entry?.title ?: "") } - var guideInput by rememberSaveable { mutableStateOf(entry?.guide ?: "") } - - Text("Title", style = MaterialTheme.typography.labelMedium, color = Color.Black.copy(alpha = 0.7f)) - Spacer(modifier = Modifier.height(4.dp)) - OutlinedTextField( - value = titleInput, - onValueChange = { titleInput = it }, - placeholder = { Text("App/Task", color = Color.Gray) }, - modifier = Modifier.fillMaxWidth(), - colors = TextFieldDefaults.colors( - focusedContainerColor = Color.White, - unfocusedContainerColor = Color.White, - disabledContainerColor = Color.White, - cursorColor = Color.Black, - focusedTextColor = Color.Black, - unfocusedTextColor = Color.Black, - ), - singleLine = true - ) - Spacer(modifier = Modifier.height(8.dp)) - Text("Guide", style = MaterialTheme.typography.labelMedium, color = Color.Black.copy(alpha = 0.7f)) - Spacer(modifier = Modifier.height(4.dp)) - OutlinedTextField( - value = guideInput, - onValueChange = { guideInput = it }, - placeholder = { Text("Write a guide for an LLM on how it should perform certain tasks to be successful", color = Color.Gray) }, - modifier = Modifier.fillMaxWidth().weight(1f), - colors = TextFieldDefaults.colors( - focusedContainerColor = Color.White, - unfocusedContainerColor = Color.White, - disabledContainerColor = Color.White, - cursorColor = Color.Black, - focusedTextColor = Color.Black, - unfocusedTextColor = Color.Black, - ), - minLines = 5 - ) - Spacer(modifier = Modifier.height(16.dp)) - Button( - onClick = { onSaveClicked(titleInput, guideInput, entry) }, - modifier = Modifier.align(Alignment.End), - colors = ButtonDefaults.buttonColors(containerColor = MaterialTheme.colorScheme.primary) - ) { Text("Save") } - } - } - } -} - -@Preview(showBackground = true, name = "EditEntryPopup New") -@Composable -fun EditEntryPopupNewPreview() { - MaterialTheme { - EditEntryPopup(entry = null, onDismissRequest = {}, onSaveClicked = { _, _, _ -> }) - } -} - -@Preview(showBackground = true, name = "EditEntryPopup Edit") -@Composable -fun EditEntryPopupEditPreview() { - MaterialTheme { - EditEntryPopup(entry = SystemMessageEntry("Existing Title", "Existing Guide"), onDismissRequest = {}, onSaveClicked = { _, _, _ -> }) - } -} - -val SystemMessageEntrySaver = Saver>( - save = { entry -> if (entry == null) listOf(null, null) else listOf(entry.title, entry.guide) }, - restore = { list -> - val title = list[0] - val guide = list[1] - if (title != null && guide != null) SystemMessageEntry(title, guide) else null - } -) - -@Composable -@Preview(showSystemUi = true) -fun PhotoReasoningScreenPreviewEmpty() { - MaterialTheme { - PhotoReasoningScreen( - innerPadding = PaddingValues(), - chatMessages = MutableStateFlow>(emptyList()), - isKeyboardOpen = false, - onStopClicked = {}, - isInitialized = true - ) - } -} - -@Preview(showBackground = true) -@Composable -fun DatabaseListPopupPreview() { - MaterialTheme { - DatabaseListPopup( - onDismissRequest = {}, - entries = listOf( - SystemMessageEntry("Title 1", "Guide for prompt 1"), - SystemMessageEntry("Title 2", "Another guide for prompt 2"), - SystemMessageEntry("Title 3", "Yet another guide for prompt 3.") - ), - onNewClicked = {}, - onEntryClicked = {}, - onDeleteClicked = {}, - onImportCompleted = {} - ) - } -} - -@Preview(showBackground = true, name = "DatabaseListPopup Empty") -@Composable -fun DatabaseListPopupEmptyPreview() { - MaterialTheme { - DatabaseListPopup(onDismissRequest = {}, entries = emptyList(), onNewClicked = {}, onEntryClicked = {}, onDeleteClicked = {}, onImportCompleted = {}) - } -} - -@Preview(showBackground = true, name = "Stop Button Preview") -@Composable -fun StopButtonPreview() { - MaterialTheme { - StopButton {} - } -} - -@Composable -fun VerticalScrollbar( - modifier: Modifier = Modifier, - listState: LazyListState -) { - val scrollbarState by remember { - derivedStateOf { - val layoutInfo = listState.layoutInfo - val totalItems = layoutInfo.totalItemsCount - if (totalItems == 0) return@derivedStateOf null - - val viewportHeight = layoutInfo.viewportSize.height.toFloat() - val firstVisibleItemIndex = listState.firstVisibleItemIndex - val visibleItemCount = layoutInfo.visibleItemsInfo.size - - if (visibleItemCount >= totalItems) return@derivedStateOf null // All items visible, no scrollbar - - val thumbHeight = (visibleItemCount.toFloat() / totalItems * viewportHeight) - .coerceAtLeast(20f) - .coerceAtMost(viewportHeight) - - val maxScrollOffset = (viewportHeight - thumbHeight).coerceAtLeast(0f) - if (maxScrollOffset == 0f) return@derivedStateOf null - - val firstItemOffset = if (layoutInfo.visibleItemsInfo.isNotEmpty()) { - val firstItem = layoutInfo.visibleItemsInfo.first() - if (firstItem.size > 0) listState.firstVisibleItemScrollOffset.toFloat() / firstItem.size else 0f - } else 0f - - val scrollProgress = (firstVisibleItemIndex + firstItemOffset) / - (totalItems - visibleItemCount).coerceAtLeast(1) - val scrollOffset = (scrollProgress * maxScrollOffset).coerceIn(0f, maxScrollOffset) - - Pair(scrollOffset, thumbHeight) - } - } - - if (scrollbarState != null) { - val (offset, height) = checkNotNull(scrollbarState) - Canvas(modifier = modifier.width(4.dp)) { - drawRoundRect( - color = Color.Gray, - topLeft = Offset(0f, offset), - size = Size(size.width, height), - cornerRadius = CornerRadius(4.dp.toPx()) - ) - } - } -} diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreenshotDebouncer.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreenshotDebouncer.kt new file mode 100644 index 0000000..6580e52 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreenshotDebouncer.kt @@ -0,0 +1,22 @@ +package com.google.ai.sample.feature.multimodal + +import android.net.Uri + +internal class PhotoReasoningScreenshotDebouncer( + private val debounceWindowMs: Long = 2000L +) { + private var lastProcessedScreenshotUri: Uri? = null + private var lastProcessedScreenshotTimeMs: Long = 0L + + fun shouldProcess(screenshotUri: Uri, nowMs: Long): Boolean { + val isDuplicateWithinWindow = screenshotUri == lastProcessedScreenshotUri && + (nowMs - lastProcessedScreenshotTimeMs) < debounceWindowMs + if (isDuplicateWithinWindow) { + return false + } + + lastProcessedScreenshotUri = screenshotUri + lastProcessedScreenshotTimeMs = nowMs + return true + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreenshotProcessor.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreenshotProcessor.kt new file mode 100644 index 0000000..53c134f --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreenshotProcessor.kt @@ -0,0 +1,30 @@ +package com.google.ai.sample.feature.multimodal + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.drawable.BitmapDrawable +import android.net.Uri +import coil.ImageLoader +import coil.request.ImageRequest +import coil.request.SuccessResult +import coil.size.Precision + +internal class PhotoReasoningScreenshotProcessor { + private var imageLoader: ImageLoader? = null + private var imageRequestBuilder: ImageRequest.Builder? = null + + suspend fun loadBitmap(context: Context, screenshotUri: Uri): Bitmap? { + val loader = imageLoader ?: ImageLoader.Builder(context).build().also { imageLoader = it } + val builder = imageRequestBuilder ?: ImageRequest.Builder(context).also { imageRequestBuilder = it } + + val imageRequest = builder + .data(screenshotUri) + .precision(Precision.EXACT) + .build() + + val result = loader.execute(imageRequest) + val successResult = result as? SuccessResult ?: return null + val bitmapDrawable = successResult.drawable as? BitmapDrawable ?: return null + return bitmapDrawable.bitmap + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreenshotUiNotifier.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreenshotUiNotifier.kt new file mode 100644 index 0000000..c4ffe82 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreenshotUiNotifier.kt @@ -0,0 +1,20 @@ +package com.google.ai.sample.feature.multimodal + +import android.content.Context +import android.widget.Toast + +internal object PhotoReasoningScreenshotUiNotifier { + fun showProcessing(context: Context, onStatus: (String) -> Unit) { + onStatus("Processing screenshot...") + Toast.makeText(context, "Processing screenshot...", Toast.LENGTH_SHORT).show() + } + + fun showSendingToAi(context: Context, onStatus: (String) -> Unit) { + onStatus("Screenshot added, sending to AI...") + Toast.makeText(context, "Screenshot added, sending to AI...", Toast.LENGTH_SHORT).show() + } + + fun showAddedToConversation(context: Context) { + Toast.makeText(context, "Screenshot added to conversation", Toast.LENGTH_SHORT).show() + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScrollbar.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScrollbar.kt new file mode 100644 index 0000000..ebc69d8 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScrollbar.kt @@ -0,0 +1,65 @@ +package com.google.ai.sample.feature.multimodal + +import androidx.compose.foundation.Canvas +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.lazy.LazyListState +import androidx.compose.runtime.Composable +import androidx.compose.runtime.derivedStateOf +import androidx.compose.runtime.getValue +import androidx.compose.runtime.remember +import androidx.compose.ui.Modifier +import androidx.compose.ui.geometry.CornerRadius +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.geometry.Size +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.unit.dp + +@Composable +fun VerticalScrollbar( + modifier: Modifier = Modifier, + listState: LazyListState +) { + val scrollbarState by remember { + derivedStateOf { + val layoutInfo = listState.layoutInfo + val totalItems = layoutInfo.totalItemsCount + if (totalItems == 0) return@derivedStateOf null + + val viewportHeight = layoutInfo.viewportSize.height.toFloat() + val firstVisibleItemIndex = listState.firstVisibleItemIndex + val visibleItemCount = layoutInfo.visibleItemsInfo.size + + if (visibleItemCount >= totalItems) return@derivedStateOf null // All items visible, no scrollbar + + val thumbHeight = (visibleItemCount.toFloat() / totalItems * viewportHeight) + .coerceAtLeast(20f) + .coerceAtMost(viewportHeight) + + val maxScrollOffset = (viewportHeight - thumbHeight).coerceAtLeast(0f) + if (maxScrollOffset == 0f) return@derivedStateOf null + + val firstItemOffset = if (layoutInfo.visibleItemsInfo.isNotEmpty()) { + val firstItem = layoutInfo.visibleItemsInfo.first() + if (firstItem.size > 0) listState.firstVisibleItemScrollOffset.toFloat() / firstItem.size else 0f + } else 0f + + val scrollProgress = (firstVisibleItemIndex + firstItemOffset) / + (totalItems - visibleItemCount).coerceAtLeast(1) + val scrollOffset = (scrollProgress * maxScrollOffset).coerceIn(0f, maxScrollOffset) + + Pair(scrollOffset, thumbHeight) + } + } + + if (scrollbarState != null) { + val (offset, height) = checkNotNull(scrollbarState) + Canvas(modifier = modifier.width(4.dp)) { + drawRoundRect( + color = Color.Gray, + topLeft = Offset(0f, offset), + size = Size(size.width, height), + cornerRadius = CornerRadius(4.dp.toPx()) + ) + } + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningTextPolicies.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningTextPolicies.kt new file mode 100644 index 0000000..30fc466 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningTextPolicies.kt @@ -0,0 +1,50 @@ +package com.google.ai.sample.feature.multimodal + +import android.content.Context +import com.google.ai.sample.util.SystemMessageEntryPreferences + +internal object PhotoReasoningTextPolicies { + fun buildPromptWithScreenInfo(userInput: String, screenInfoForPrompt: String?): String { + return if (screenInfoForPrompt != null && screenInfoForPrompt.isNotBlank()) { + "$userInput\n\n$screenInfoForPrompt" + } else { + userInput + } + } + + fun isQuotaExceededError(message: String): Boolean { + return message.contains("exceeded your current quota") || + message.contains("code 429") || + message.contains("Too Many Requests", ignoreCase = true) || + message.contains("rate_limit", ignoreCase = true) + } + + /** + * Point 14: Check if error is a high-demand 503 (UNAVAILABLE) error. + * These should NOT trigger API key switching. + */ + fun isHighDemandError(message: String): Boolean { + return message.contains("Service Unavailable (503)") || + message.contains("UNAVAILABLE") || + message.contains("high demand") || + message.contains("overloaded") + } + + /** + * Helper function to format database entries as text. + */ + fun formatDatabaseEntriesAsText(context: Context): String { + val entries = SystemMessageEntryPreferences.loadEntries(context) + if (entries.isEmpty()) { + return "" + } + val builder = StringBuilder() + builder.append("Available System Guides:\n---\n") + for (entry in entries) { + builder.append("Title: ${entry.title}\n") + builder.append("Guide: ${entry.guide}\n") + builder.append("---\n") + } + return builder.toString() + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningUiConstants.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningUiConstants.kt new file mode 100644 index 0000000..845f817 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningUiConstants.kt @@ -0,0 +1,8 @@ +package com.google.ai.sample.feature.multimodal + +import androidx.compose.ui.graphics.Color + +// Define Colors +val DarkYellow1 = Color(0xFFF0A500) // A darker yellow +val DarkYellow2 = Color(0xFFF3C100) // A slightly lighter dark yellow + diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt index b8ef663..0f5cf0e 100644 --- a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt @@ -7,17 +7,12 @@ import android.content.Context import android.content.BroadcastReceiver import android.content.Intent import android.content.IntentFilter -import android.graphics.drawable.BitmapDrawable import android.net.Uri import android.os.Build import android.util.Log import android.widget.Toast import androidx.lifecycle.AndroidViewModel import androidx.lifecycle.viewModelScope -import coil.ImageLoader -import coil.request.ImageRequest -import coil.request.SuccessResult -import coil.size.Precision import com.google.ai.client.generativeai.Chat import com.google.ai.client.generativeai.GenerativeModel import com.google.ai.client.generativeai.type.content @@ -33,7 +28,6 @@ import com.google.ai.sample.util.ChatHistoryPreferences import com.google.ai.sample.util.Command import com.google.ai.sample.util.CommandParser import com.google.ai.sample.util.SystemMessagePreferences -import com.google.ai.sample.util.SystemMessageEntryPreferences import com.google.ai.sample.util.SystemMessageEntry import com.google.ai.sample.util.UserInputPreferences import com.google.ai.sample.feature.multimodal.ModelDownloadManager @@ -65,14 +59,11 @@ import okhttp3.MediaType.Companion.toMediaType import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody.Companion.toRequestBody -import kotlinx.serialization.Serializable -import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.SerializationException import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonElement import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive -import kotlinx.serialization.json.JsonClassDiscriminator import kotlinx.serialization.modules.SerializersModule import kotlinx.serialization.modules.polymorphic import kotlinx.serialization.modules.subclass @@ -118,11 +109,20 @@ class PhotoReasoningViewModel( private val _isInitializingOfflineModelFlow = MutableStateFlow(false) val isInitializingOfflineModelFlow: StateFlow = _isInitializingOfflineModelFlow.asStateFlow() + + private val app: Application + get() = getApplication() + + private val appContext: Context + get() = app.applicationContext + + private fun saveChatHistoryForApplication() { + saveChatHistory(app) + } // Keep track of the latest screenshot URI private var latestScreenshotUri: Uri? = null - private var lastProcessedScreenshotUri: Uri? = null - private var lastProcessedScreenshotTime: Long = 0L + private val screenshotDebouncer = PhotoReasoningScreenshotDebouncer() // Keep track of the current selected images private var currentSelectedImages: List = emptyList() @@ -136,7 +136,7 @@ class PhotoReasoningViewModel( fun updateUserInput(text: String) { _userInput.value = text - val context = getApplication().applicationContext + val context = appContext UserInputPreferences.saveUserInput(context, text) } @@ -156,7 +156,7 @@ class PhotoReasoningViewModel( val modelNameState: StateFlow = _modelNameState.asStateFlow() // Chat history state - private val _chatState = ChatState() + private val _chatState = PhotoReasoningChatState() val chatMessages: List get() = _chatState.messages @@ -164,9 +164,8 @@ class PhotoReasoningViewModel( private val _chatMessagesFlow = MutableStateFlow>(emptyList()) val chatMessagesFlow: StateFlow> = _chatMessagesFlow.asStateFlow() - // ImageLoader and ImageRequestBuilder for processing images - private var imageLoader: ImageLoader? = null - private var imageRequestBuilder: ImageRequest.Builder? = null + // Screenshot decoding/pipeline + private val screenshotProcessor = PhotoReasoningScreenshotProcessor() // Chat instance for maintaining conversation context private var chat = generativeModel.startChat( @@ -195,43 +194,7 @@ class PhotoReasoningViewModel( private var currentImageUrisForChat: List? = null private val sseJson = PhotoReasoningSerialization.createStreamingJsonParser() - - /** - * Liest einen OpenAI-kompatiblen SSE-Stream zeilenweise. - * Ruft [onChunk] mit dem jeweils akkumulierten Text auf. - * Gibt den vollständigen Text zurück. - */ - private suspend fun streamOpenAIResponse( - body: okhttp3.ResponseBody, - onChunk: suspend (accumulatedText: String) -> Unit - ): String { - val accumulated = StringBuilder() - val reader = body.charStream().buffered() - try { - var line: String? - while (reader.readLine().also { line = it } != null) { - val l = line ?: break - if (l.startsWith("data: ")) { - val data = l.removePrefix("data: ").trim() - if (data == "[DONE]") break - if (data.isEmpty()) continue - try { - val chunk = sseJson.decodeFromString(data) - val delta = chunk.choices.firstOrNull()?.delta?.content - if (!delta.isNullOrEmpty()) { - accumulated.append(delta) - onChunk(accumulated.toString()) - } - } catch (e: SerializationException) { - // Fehlerhafte Chunk überspringen - } - } - } - } finally { - reader.close() - } - return accumulated.toString() - } + private val openAiStreamParser = PhotoReasoningOpenAiStreamParser(sseJson) private val aiResultStreamReceiver = object : BroadcastReceiver() { override fun onReceive(context: Context?, intent: Intent?) { @@ -259,30 +222,23 @@ class PhotoReasoningViewModel( _uiState.value = PhotoReasoningUiState.Success(responseText) finalizeAiMessage(responseText) processCommands(responseText) - saveChatHistory(getApplication()) + saveChatHistoryForApplication() } else if (errorMessage != null) { Log.e(TAG, "AI Call Error via Broadcast: $errorMessage") - val receiverContext = context ?: getApplication() + val receiverContext = context ?: app _uiState.value = PhotoReasoningUiState.Error(errorMessage) _commandExecutionStatus.value = "Error during AI generation: $errorMessage" _chatState.replaceLastPendingMessage() val apiKeyManager = ApiKeyManager.getInstance(receiverContext) - val isQuotaError = isQuotaExceededError(errorMessage) - val isHighDemand = isHighDemandError(errorMessage) + val isQuotaError = PhotoReasoningTextPolicies.isQuotaExceededError(errorMessage) + val isHighDemand = PhotoReasoningTextPolicies.isHighDemandError(errorMessage) val currentModel = com.google.ai.sample.GenerativeAiViewModelFactory.getCurrentModel() // Point 14: Don't switch keys for high-demand 503 errors if (isHighDemand) { Log.d(TAG, "High demand error detected - not switching API keys") - _chatState.addMessage( - PhotoReasoningMessage( - text = "This model is currently experiencing high demand. Please try again later.", - participant = PhotoParticipant.ERROR - ) - ) - _chatMessagesFlow.value = chatMessages - saveChatHistory(getApplication()) + appendAndPublishErrorMessage("This model is currently experiencing high demand. Please try again later.") return } @@ -313,14 +269,7 @@ class PhotoReasoningViewModel( } // Normal error handling if not quota or max retries reached - _chatState.addMessage( - PhotoReasoningMessage( - text = errorMessage, - participant = PhotoParticipant.ERROR - ) - ) - _chatMessagesFlow.value = chatMessages - saveChatHistory(getApplication()) + appendAndPublishErrorMessage(errorMessage) } // Reset pending AI message if any (assuming updateAiMessage or error handling does this) } @@ -330,7 +279,7 @@ class PhotoReasoningViewModel( init { // Initialize model if it's the offline one and already downloaded val currentModel = com.google.ai.sample.GenerativeAiViewModelFactory.getCurrentModel() - val context = getApplication().applicationContext + val context = appContext if (currentModel == ModelOption.GEMMA_3N_E4B_IT) { if (ModelDownloadManager.isModelDownloaded(context)) { // Point 7 & 16: Initialize model asynchronously to not block UI @@ -461,6 +410,14 @@ class PhotoReasoningViewModel( _isOfflineGpuModelLoadedFlow.value = isOfflineGpuModelLoaded() } + private fun resetStreamingCommandState() { + incrementalCommandCount = 0 + streamingAccumulatedText.clear() + CommandParser.clearBuffer() + _detectedCommands.value = emptyList() + _commandExecutionStatus.value = "" + } + fun closeOfflineModel() { try { llmInference?.close() @@ -476,7 +433,7 @@ class PhotoReasoningViewModel( override fun onCleared() { super.onCleared() liveApiManager?.close() - val context = getApplication().applicationContext + val context = appContext LocalBroadcastManager.getInstance(context).unregisterReceiver(aiResultReceiver) Log.d(TAG, "AIResultReceiver unregistered with LocalBroadcastManager.") LocalBroadcastManager.getInstance(context).unregisterReceiver(aiResultStreamReceiver) @@ -499,14 +456,11 @@ class PhotoReasoningViewModel( private fun createChatWithSystemMessage(context: Context? = null): Chat { val ctx = context ?: getApplication() - val history = mutableListOf() - if (_systemMessage.value.isNotBlank()) { - history.add(content(role = "user") { text(_systemMessage.value) }) - } - val formattedDbEntries = formatDatabaseEntriesAsText(ctx) - if (formattedDbEntries.isNotBlank()) { - history.add(content(role = "user") { text(formattedDbEntries) }) - } + val formattedDbEntries = PhotoReasoningTextPolicies.formatDatabaseEntriesAsText(ctx) + val history = PhotoReasoningHistoryBuilder.buildInitialHistory( + systemMessage = _systemMessage.value, + formattedDbEntries = formattedDbEntries + ) return generativeModel.startChat(history = history) } @@ -523,7 +477,7 @@ class PhotoReasoningViewModel( imageUrisForChat: List? = null ) { // Get context for rebuildChatHistory - val context = getApplication().applicationContext + val context = appContext // Update the generative model with the current API key if retrying if (currentRetryAttempt > 0) { @@ -571,11 +525,7 @@ class PhotoReasoningViewModel( currentImageUrisForChat = imageUrisForChat // Clear previous commands and reset incremental tracking - _detectedCommands.value = emptyList() - _commandExecutionStatus.value = "" - incrementalCommandCount = 0 - streamingAccumulatedText.clear() - CommandParser.clearBuffer() + resetStreamingCommandState() // Add user message to chat history val userMessage = PhotoReasoningMessage( @@ -693,7 +643,7 @@ class PhotoReasoningViewModel( // Check for offline model (Gemma) if (currentModel == ModelOption.GEMMA_3N_E4B_IT) { - val context = getApplication().applicationContext + val context = appContext if (!ModelDownloadManager.isModelDownloaded(context)) { _uiState.value = PhotoReasoningUiState.Error("Model not downloaded.") @@ -704,15 +654,11 @@ class PhotoReasoningViewModel( ensureInitialized(context) // Reset incremental command tracking for this new reasoning - incrementalCommandCount = 0 - streamingAccumulatedText.clear() - CommandParser.clearBuffer() - _detectedCommands.value = emptyList() - _commandExecutionStatus.value = "" + resetStreamingCommandState() // Build the combined prompt with system message + DB entries + user input val systemMsg = _systemMessage.value - val dbEntries = formatDatabaseEntriesAsText(context) + val dbEntries = PhotoReasoningTextPolicies.formatDatabaseEntriesAsText(context) val combinedPromptParts = mutableListOf() if (systemMsg.isNotBlank()) { combinedPromptParts.add(systemMsg) @@ -888,14 +834,6 @@ class PhotoReasoningViewModel( } } - private fun buildPromptWithScreenInfo(userInput: String, screenInfoForPrompt: String?): String { - return if (screenInfoForPrompt != null && screenInfoForPrompt.isNotBlank()) { - "$userInput\n\n$screenInfoForPrompt" - } else { - userInput - } - } - private fun reasonInLiveMode( userInput: String, selectedImages: List, @@ -910,7 +848,7 @@ class PhotoReasoningViewModel( delay(2000) } - val combinedPromptText = buildPromptWithScreenInfo(userInput, screenInfoForPrompt) + val combinedPromptText = PhotoReasoningTextPolicies.buildPromptWithScreenInfo(userInput, screenInfoForPrompt) val userMessage = PhotoReasoningMessage( text = combinedPromptText, participant = PhotoParticipant.USER, @@ -927,7 +865,7 @@ class PhotoReasoningViewModel( Log.e(TAG, "Error in live mode reasoning", e) _uiState.value = PhotoReasoningUiState.Error(e.message ?: "Unknown error") appendErrorMessage("Error: ${e.message}") - saveChatHistory(getApplication()) + saveChatHistoryForApplication() } } } @@ -974,7 +912,7 @@ class PhotoReasoningViewModel( screenInfoForPrompt: String? = null ) { _uiState.value = PhotoReasoningUiState.Loading - val context = getApplication().applicationContext + val context = appContext val apiKeyManager = ApiKeyManager.getInstance(context) val initialApiKey = apiKeyManager.getCurrentApiKey(ApiProvider.CEREBRAS) @@ -993,18 +931,14 @@ class PhotoReasoningViewModel( ) ) - incrementalCommandCount = 0 - streamingAccumulatedText.clear() - CommandParser.clearBuffer() - _detectedCommands.value = emptyList() - _commandExecutionStatus.value = "" + resetStreamingCommandState() viewModelScope.launch(Dispatchers.IO) { try { val apiMessages = mutableListOf() if (_systemMessage.value.isNotBlank()) apiMessages.add(CerebrasMessage(role = "user", content = _systemMessage.value)) - val formattedDbEntries = formatDatabaseEntriesAsText(context) + val formattedDbEntries = PhotoReasoningTextPolicies.formatDatabaseEntriesAsText(context) if (formattedDbEntries.isNotBlank()) apiMessages.add(CerebrasMessage(role = "user", content = formattedDbEntries)) _chatState.getAllMessages() @@ -1048,7 +982,7 @@ class PhotoReasoningViewModel( } val body = response.body ?: throw IOException("Empty response body from Cerebras") - val aiResponseText = streamOpenAIResponse(body) { accText -> + val aiResponseText = openAiStreamParser.parse(body) { accText -> withContext(Dispatchers.Main) { replaceAiMessageText(accText, isPending = true) processCommandsIncrementally(accText) @@ -1074,26 +1008,17 @@ class PhotoReasoningViewModel( } private fun appendUserAndPendingModelMessages(userMessage: PhotoReasoningMessage) { - _chatState.addMessage(userMessage) - _chatState.addMessage( - PhotoReasoningMessage( - text = "", - participant = PhotoParticipant.MODEL, - isPending = true - ) + _chatMessagesFlow.value = PhotoReasoningMessageMutations.appendUserAndPendingModelMessages( + chatState = _chatState, + userMessage = userMessage ) - _chatMessagesFlow.value = _chatState.getAllMessages() } private fun appendErrorMessage(errorText: String) { - _chatState.replaceLastPendingMessage() - _chatState.addMessage( - PhotoReasoningMessage( - text = errorText, - participant = PhotoParticipant.ERROR - ) + _chatMessagesFlow.value = PhotoReasoningMessageMutations.appendErrorMessage( + chatState = _chatState, + errorText = errorText ) - _chatMessagesFlow.value = _chatState.getAllMessages() } private fun reasonWithMistral( @@ -1103,7 +1028,7 @@ private fun reasonWithMistral( imageUrisForChat: List? = null ) { _uiState.value = PhotoReasoningUiState.Loading - val context = getApplication().applicationContext + val context = appContext val apiKeyManager = ApiKeyManager.getInstance(context) val initialApiKey = apiKeyManager.getCurrentApiKey(ApiProvider.MISTRAL) @@ -1114,21 +1039,17 @@ private fun reasonWithMistral( val combinedPromptText = (userInput + "\n\n" + (screenInfoForPrompt ?: "")).trim() - _chatState.addMessage(PhotoReasoningMessage( - text = combinedPromptText, - participant = PhotoParticipant.USER, - imageUris = if (com.google.ai.sample.GenerativeAiViewModelFactory.getCurrentModel().supportsScreenshot) - (imageUrisForChat ?: emptyList()) else emptyList(), - isPending = false - )) - _chatState.addMessage(PhotoReasoningMessage(text = "", participant = PhotoParticipant.MODEL, isPending = true)) - _chatMessagesFlow.value = _chatState.getAllMessages() - - incrementalCommandCount = 0 - streamingAccumulatedText.clear() - CommandParser.clearBuffer() - _detectedCommands.value = emptyList() - _commandExecutionStatus.value = "" + appendUserAndPendingModelMessages( + PhotoReasoningMessage( + text = combinedPromptText, + participant = PhotoParticipant.USER, + imageUris = if (com.google.ai.sample.GenerativeAiViewModelFactory.getCurrentModel().supportsScreenshot) + (imageUrisForChat ?: emptyList()) else emptyList(), + isPending = false + ) + ) + + resetStreamingCommandState() viewModelScope.launch(Dispatchers.IO) { // Rate limiting: nur die verbleibende Zeit warten @@ -1146,7 +1067,7 @@ private fun reasonWithMistral( val systemContent = mutableListOf() if (_systemMessage.value.isNotBlank()) systemContent.add(MistralTextContent(text = _systemMessage.value)) - val formattedDbEntries = formatDatabaseEntriesAsText(context) + val formattedDbEntries = PhotoReasoningTextPolicies.formatDatabaseEntriesAsText(context) if (formattedDbEntries.isNotBlank()) systemContent.add(MistralTextContent(text = "Additional context from database:\n$formattedDbEntries")) if (systemContent.isNotEmpty()) @@ -1252,7 +1173,7 @@ private fun reasonWithMistral( } val body = response.body ?: throw IOException("Empty response body from Mistral") - val aiResponseText = streamOpenAIResponse(body) { accText -> + val aiResponseText = openAiStreamParser.parse(body) { accText -> withContext(Dispatchers.Main) { replaceAiMessageText(accText, isPending = true) processCommandsIncrementally(accText) @@ -1289,7 +1210,7 @@ private fun reasonWithMistral( return } - val context = getApplication().applicationContext + val context = appContext val currentModel = com.google.ai.sample.GenerativeAiViewModelFactory.getCurrentModel() val genSettings = com.google.ai.sample.util.GenerationSettingsPreferences.loadSettings(context, currentModel.modelName) @@ -1318,11 +1239,7 @@ private fun reasonWithMistral( _uiState.value = PhotoReasoningUiState.Loading // Reset tracking vars - incrementalCommandCount = 0 - streamingAccumulatedText.clear() - CommandParser.clearBuffer() - _detectedCommands.value = emptyList() - _commandExecutionStatus.value = "" + resetStreamingCommandState() viewModelScope.launch(Dispatchers.IO) { try { @@ -1333,7 +1250,7 @@ private fun reasonWithMistral( if (_systemMessage.value.isNotBlank()) { systemContent.add(com.google.ai.sample.network.PuterTextContent(text = _systemMessage.value)) } - val formattedDbEntries = formatDatabaseEntriesAsText(context) + val formattedDbEntries = PhotoReasoningTextPolicies.formatDatabaseEntriesAsText(context) if (formattedDbEntries.isNotBlank()) { systemContent.add(com.google.ai.sample.network.PuterTextContent(text = "Additional context from database:\n$formattedDbEntries")) } @@ -1415,7 +1332,7 @@ private fun reasonWithMistral( } val body = httpResponse.body ?: throw java.io.IOException("Empty response from Puter") - val aiResponseText = streamOpenAIResponse(body) { accText -> + val aiResponseText = openAiStreamParser.parse(body) { accText -> withContext(Dispatchers.Main) { replaceAiMessageText(accText, isPending = true) processCommandsIncrementally(accText) @@ -1453,7 +1370,7 @@ private fun reasonWithMistral( if (liveApiManager != null) { // Set system message and history when connecting viewModelScope.launch { - val context = getApplication().applicationContext + val context = appContext ensureInitialized(context) // Convert chat history to format for Live API @@ -1464,7 +1381,7 @@ private fun reasonWithMistral( historyPairs.add(Pair("user", _systemMessage.value)) } - val formattedDbEntries = formatDatabaseEntriesAsText(context) + val formattedDbEntries = PhotoReasoningTextPolicies.formatDatabaseEntriesAsText(context) if (formattedDbEntries.isNotBlank()) { historyPairs.add(Pair("user", formattedDbEntries)) } @@ -1535,7 +1452,7 @@ private fun reasonWithMistral( processCommands(finalMessage.text) // Save chat history - saveChatHistory(getApplication()) + saveChatHistoryForApplication() } } } @@ -1737,14 +1654,7 @@ private fun reasonWithMistral( _uiState.value = PhotoReasoningUiState.Error(uiMessage) _showStopNotificationFlow.value = false _chatState.replaceLastPendingMessage() - _chatState.addMessage( - PhotoReasoningMessage( - text = uiMessage, - participant = PhotoParticipant.ERROR - ) - ) - _chatMessagesFlow.value = chatMessages - saveChatHistory(context) + appendAndPublishErrorMessage(uiMessage, context) } private fun createExecuteAiCallIntent( @@ -1925,7 +1835,7 @@ private fun reasonWithMistral( } private fun postTaskToHumanExpert(text: String) { - val context = getApplication().applicationContext + val context = appContext val prefs = context.getSharedPreferences("AppPrefs", Context.MODE_PRIVATE) val supportId = prefs.getString("payment_support_id", null) @@ -1959,57 +1869,27 @@ private fun reasonWithMistral( private fun finalizeAiMessage(finalText: String) { Log.d(TAG, "finalizeAiMessage: Finalizing AI message.") - val messages = _chatState.getAllMessages().toMutableList() - val lastMessageIndex = messages.indexOfLast { it.participant == PhotoParticipant.MODEL && it.isPending } - - if (lastMessageIndex != -1) { - messages[lastMessageIndex] = messages[lastMessageIndex].copy(text = finalText, isPending = false) - _chatState.setAllMessages(messages) - _chatMessagesFlow.value = _chatState.getAllMessages() - } else { - // This case should ideally not happen if streaming is working correctly - // but as a fallback, we can just add the final message. - _chatState.addMessage( - PhotoReasoningMessage( - text = finalText, - participant = PhotoParticipant.MODEL, - isPending = false - ) - ) - _chatMessagesFlow.value = _chatState.getAllMessages() - } - saveChatHistory(getApplication()) + _chatMessagesFlow.value = PhotoReasoningMessageMutations.finalizeAiMessage( + chatState = _chatState, + finalText = finalText + ) + saveChatHistoryForApplication() refreshStopButtonState() } private fun updateAiMessage(text: String, isPending: Boolean = false) { Log.d(TAG, "updateAiMessage: Adding to _chatState. Text: \"${text.take(100)}...\", Participant: MODEL, IsPending: $isPending") - // Get a copy of current messages - val messages = _chatState.getAllMessages().toMutableList() - val lastAiMessageIndex = messages.indexOfLast { it.participant == PhotoParticipant.MODEL } - - if (lastAiMessageIndex != -1 && messages[lastAiMessageIndex].isPending) { - // If the last AI message is pending, append the new text - val updatedMessage = messages[lastAiMessageIndex].let { - it.copy(text = it.text + text, isPending = isPending) - } - messages[lastAiMessageIndex] = updatedMessage - } else { - // Otherwise, add a new AI message - messages.add(PhotoReasoningMessage(text = text, participant = PhotoParticipant.MODEL, isPending = isPending)) - } - - // Set all messages atomically - _chatState.setAllMessages(messages) - - // Update the flow - _chatMessagesFlow.value = _chatState.getAllMessages() + _chatMessagesFlow.value = PhotoReasoningMessageMutations.updateAiMessage( + chatState = _chatState, + text = text, + isPending = isPending + ) Log.d(TAG, "updateAiMessage: _chatState now has ${_chatState.messages.size} messages.") // Save chat history after updating message if (!stopExecutionFlag.get() || text.contains("stopped by user", ignoreCase = true)) { - saveChatHistory(getApplication()) + saveChatHistoryForApplication() } } @@ -2018,21 +1898,34 @@ private fun reasonWithMistral( * Unlike updateAiMessage which appends, this sets the full text directly. */ private fun replaceAiMessageText(text: String, isPending: Boolean = true) { - val messages = _chatState.getAllMessages().toMutableList() - val lastAiMessageIndex = messages.indexOfLast { it.participant == PhotoParticipant.MODEL } + _chatMessagesFlow.value = PhotoReasoningMessageMutations.replaceAiMessageText( + chatState = _chatState, + text = text, + isPending = isPending + ) + } - if (lastAiMessageIndex != -1 && messages[lastAiMessageIndex].isPending) { - // Replace the full text (not append) - messages[lastAiMessageIndex] = messages[lastAiMessageIndex].copy(text = text, isPending = isPending) + private fun handleScreenshotProcessingError( + context: Context, + message: String, + throwable: Throwable? = null + ) { + if (throwable != null) { + Log.e(TAG, message, throwable) } else { - // No pending message found, add a new one - messages.add(PhotoReasoningMessage(text = text, participant = PhotoParticipant.MODEL, isPending = isPending)) + Log.e(TAG, message) } - - _chatState.setAllMessages(messages) - _chatMessagesFlow.value = _chatState.getAllMessages() + _commandExecutionStatus.value = message + Toast.makeText(context, message, Toast.LENGTH_SHORT).show() + _chatMessagesFlow.value = PhotoReasoningMessageMutations.appendErrorMessage( + chatState = _chatState, + errorText = message + ) + saveChatHistory(context) } + private fun createGenericScreenshotPrompt(): String = "" + /** * Update the system message */ @@ -2073,42 +1966,6 @@ private fun reasonWithMistral( updateSystemMessage(defaultMessage, context) } - private fun isQuotaExceededError(message: String): Boolean { - return message.contains("exceeded your current quota") || - message.contains("code 429") || - message.contains("Too Many Requests", ignoreCase = true) || - message.contains("rate_limit", ignoreCase = true) - } - - /** - * Point 14: Check if error is a high-demand 503 (UNAVAILABLE) error. - * These should NOT trigger API key switching. - */ - private fun isHighDemandError(message: String): Boolean { - return message.contains("Service Unavailable (503)") || - message.contains("UNAVAILABLE") || - message.contains("high demand") || - message.contains("overloaded") - } - - /** - * Helper function to format database entries as text. - */ - private fun formatDatabaseEntriesAsText(context: Context): String { - val entries = SystemMessageEntryPreferences.loadEntries(context) - if (entries.isEmpty()) { - return "" - } - val builder = StringBuilder() - builder.append("Available System Guides:\n---\n") - for (entry in entries) { - builder.append("Title: ${entry.title}\n") - builder.append("Guide: ${entry.guide}\n") - builder.append("---\n") - } - return builder.toString() - } - /** * Incrementally process commands during streaming. * Parses the full accumulated text but only executes NEW commands @@ -2124,7 +1981,7 @@ private fun reasonWithMistral( try { // Parse all commands from the full accumulated text // Use a fresh parse (not the buffer-based one) to get all commands in order - val allCommands = CommandParser.parseCommands(accumulatedText, clearBuffer = true) + val allCommands = PhotoReasoningCommandProcessing.parseForStreaming(accumulatedText) if (allCommands.size > incrementalCommandCount) { // There are new commands to execute @@ -2144,12 +2001,7 @@ private fun reasonWithMistral( try { Log.d(TAG, "Incremental: Executing command: $command") _commandExecutionStatus.value = "Executing: $command" - ScreenOperatorAccessibilityService.executeCommand(command) - - // Track as executed - val currentCommands = _detectedCommands.value.toMutableList() - currentCommands.add(command) - _detectedCommands.value = currentCommands + executeAccessibilityCommand(command, shouldTrackCommand = true) } catch (e: Exception) { Log.e(TAG, "Incremental: Error executing command: ${e.message}", e) } @@ -2168,39 +2020,37 @@ private fun reasonWithMistral( private fun processCommands(text: String) { commandProcessingJob?.cancel() // Cancel any previous command processing commandProcessingJob = PhotoReasoningApplication.applicationScope.launch(Dispatchers.Main) { - if (commandProcessingJob?.isActive != true || stopExecutionFlag.get()) return@launch // Check for cancellation + if (PhotoReasoningCommandExecutionGuard.shouldAbort(commandProcessingJob?.isActive == true, stopExecutionFlag.get())) return@launch // Check for cancellation try { - // Parse commands from the text - val commands = CommandParser.parseCommands(text) - - // Check if takeScreenshot command is present - val hasTakeScreenshotCommand = commands.any { it is Command.TakeScreenshot } + val commandBatch = PhotoReasoningCommandProcessing.parseForFinalExecution(text) + val commands = commandBatch.commands + val hasTakeScreenshotCommand = commandBatch.hasTakeScreenshotCommand if (commands.isNotEmpty()) { - if (commandProcessingJob?.isActive != true || stopExecutionFlag.get()) return@launch + if (PhotoReasoningCommandExecutionGuard.shouldAbort(commandProcessingJob?.isActive == true, stopExecutionFlag.get())) return@launch Log.d(TAG, "Found ${commands.size} commands in response") // Update the detected commands - val currentCommands = _detectedCommands.value.toMutableList() - currentCommands.addAll(commands) - _detectedCommands.value = currentCommands + _detectedCommands.value = PhotoReasoningCommandStateUpdater.appendCommands( + existing = _detectedCommands.value, + commands = commands + ) // Update status to show commands were detected - val commandDescriptions = commands.joinToString("; ") { command -> - command.toString() - } - _commandExecutionStatus.value = "Commands detected: $commandDescriptions" + _commandExecutionStatus.value = PhotoReasoningCommandStateUpdater.buildDetectedStatus( + commandBatch.commandDescriptions + ) // Execute the commands for (command in commands) { - if (commandProcessingJob?.isActive != true || stopExecutionFlag.get()) { // Check for cancellation before executing each command + if (PhotoReasoningCommandExecutionGuard.shouldAbort(commandProcessingJob?.isActive == true, stopExecutionFlag.get())) { // Check for cancellation before executing each command Log.d(TAG, "Command execution stopped before executing: $command") _commandExecutionStatus.value = "Command execution stopped." break // Exit loop if cancelled } try { Log.d(TAG, "Executing command: $command") - ScreenOperatorAccessibilityService.executeCommand(command) + executeAccessibilityCommand(command, shouldTrackCommand = false) // Check immediately after execution attempt if a stop was requested if (stopExecutionFlag.get()) { Log.d(TAG, "Command execution stopped after attempting: $command") @@ -2208,7 +2058,7 @@ private fun processCommands(text: String) { break } } catch (e: Exception) { - if (commandProcessingJob?.isActive != true || stopExecutionFlag.get()) break // Exit loop if cancelled during error handling + if (PhotoReasoningCommandExecutionGuard.shouldAbort(commandProcessingJob?.isActive == true, stopExecutionFlag.get())) break // Exit loop if cancelled during error handling Log.e(TAG, "Error executing command: ${e.message}", e) _commandExecutionStatus.value = "Error during command execution: ${e.message}" } @@ -2222,12 +2072,12 @@ private fun processCommands(text: String) { if (!hasTakeScreenshotCommand && !text.contains("takeScreenshot()", ignoreCase = true)) { val context = MainActivity.getInstance() if (context != null) { - Toast.makeText(context, "The AI stopped Screen Operator", Toast.LENGTH_SHORT).show() + PhotoReasoningCommandUiNotifier.showStoppedByAi(context) } } } catch (e: Exception) { - if (commandProcessingJob?.isActive != true || stopExecutionFlag.get()) return@launch + if (PhotoReasoningCommandExecutionGuard.shouldAbort(commandProcessingJob?.isActive == true, stopExecutionFlag.get())) return@launch Log.e(TAG, "Error processing commands: ${e.message}", e) _commandExecutionStatus.value = "Error during command processing: ${e.message}" } finally { @@ -2239,90 +2089,26 @@ private fun processCommands(text: String) { } } -// Data classes for Cerebras API -@Serializable -data class CerebrasRequest( - val model: String, - val messages: List, - val max_completion_tokens: Int = 1024, - val temperature: Double = 0.2, - val top_p: Double = 1.0, - val stream: Boolean = false -) - -@Serializable -data class CerebrasMessage( - val role: String, - val content: String -) - -@Serializable -data class CerebrasResponse( - val choices: List -) - -@Serializable -data class CerebrasChoice( - val message: CerebrasResponseMessage -) - -@Serializable -data class CerebrasResponseMessage( - val role: String, - val content: String -) - -// Data classes for Mistral API -@Serializable -data class MistralRequest( - val model: String, - val messages: List, - val max_tokens: Int = 4096, - val temperature: Double = 0.7, - val top_p: Double = 1.0, - val stream: Boolean = false -) - -@Serializable -data class MistralMessage( - val role: String, - val content: List -) - -@Serializable -@OptIn(ExperimentalSerializationApi::class) -@JsonClassDiscriminator("type") -sealed class MistralContent - -@Serializable -@kotlinx.serialization.SerialName("text") -data class MistralTextContent(val text: String) : MistralContent() - -@Serializable -@kotlinx.serialization.SerialName("image_url") -data class MistralImageContent(@kotlinx.serialization.SerialName("image_url") val imageUrl: MistralImageUrl) : MistralContent() - -@Serializable -data class MistralImageUrl(val url: String) - -@Serializable -data class MistralResponse( - val choices: List -) - -@Serializable -data class MistralChoice( - val message: MistralResponseMessage -) - -@Serializable -data class MistralResponseMessage( - val role: String, - val content: String -) - - + private fun executeAccessibilityCommand(command: Command, shouldTrackCommand: Boolean) { + ScreenOperatorAccessibilityService.executeCommand(command) + if (shouldTrackCommand) { + _detectedCommands.value = PhotoReasoningCommandStateUpdater.appendCommand( + existing = _detectedCommands.value, + command = command + ) + } + } + private fun appendAndPublishErrorMessage(errorText: String, context: Context = appContext) { + _chatState.addMessage( + PhotoReasoningMessage( + text = errorText, + participant = PhotoParticipant.ERROR + ) + ) + _chatMessagesFlow.value = chatMessages + saveChatHistory(context) + } /** * Save chat history to SharedPreferences */ @@ -2358,69 +2144,12 @@ data class MistralResponseMessage( */ private fun rebuildChatHistory(context: Context) { Log.d(TAG, "rebuildChatHistory: Starting. Input _chatState.messages size: ${_chatState.messages.size}") - // Convert the current chat messages to Content objects for the chat history - val history = mutableListOf() - - // 1. Active System Message - if (_systemMessage.value.isNotBlank()) { - history.add(content(role = "user") { text(_systemMessage.value) }) - } - - // 2. Formatted Database Entries - val formattedDbEntries = formatDatabaseEntriesAsText(context) - if (formattedDbEntries.isNotBlank()) { - history.add(content(role = "user") { text(formattedDbEntries) }) - } - - // 3. Group messages by participant to create proper conversation turns - var currentUserContent = "" - var currentModelContent = "" - - for (message in chatMessages) { - when (message.participant) { - PhotoParticipant.USER -> { - // If we have model content and are now seeing a user message, - // add the model content to history and reset - if (currentModelContent.isNotEmpty()) { - history.add(content(role = "model") { text(currentModelContent) }) - currentModelContent = "" - } - - // Append to current user content - if (currentUserContent.isNotEmpty()) { - currentUserContent += "\n\n" - } - currentUserContent += message.text - } - PhotoParticipant.MODEL -> { - // If we have user content and are now seeing a model message, - // add the user content to history and reset - if (currentUserContent.isNotEmpty()) { - history.add(content(role = "user") { text(currentUserContent) }) - currentUserContent = "" - } - - // Append to current model content - if (currentModelContent.isNotEmpty()) { - currentModelContent += "\n\n" - } - currentModelContent += message.text - } - PhotoParticipant.ERROR -> { - // Errors are not included in the AI history - continue - } - } - Log.d(TAG, " Processed PhotoReasoningMessage: role=${message.participant}, text collected for current SDK Content part: \"${ (if (message.participant == PhotoParticipant.USER) currentUserContent else currentModelContent).take(100) }...\"") - } - - // Add any remaining content - if (currentUserContent.isNotEmpty()) { - history.add(content(role = "user") { text(currentUserContent) }) - } - if (currentModelContent.isNotEmpty()) { - history.add(content(role = "model") { text(currentModelContent) }) - } + val formattedDbEntries = PhotoReasoningTextPolicies.formatDatabaseEntriesAsText(context) + val history = PhotoReasoningHistoryBuilder.buildHistoryFromMessages( + messages = chatMessages, + systemMessage = _systemMessage.value, + formattedDbEntries = formattedDbEntries + ) Log.d(TAG, "rebuildChatHistory: Finished processing. Generated SDK history size: ${history.size}") history.forEachIndexed { index, content -> @@ -2455,16 +2184,13 @@ data class MistralResponseMessage( _chatState.setAllMessages(emptyList()) // Create new chat with system message and DB entries in history (for AI context only, not visible in UI) - val initialHistory = mutableListOf() - if (_systemMessage.value.isNotBlank()) { - initialHistory.add(content(role = "user") { text(_systemMessage.value) }) - } - context?.let { ctx -> - val formattedDbEntries = formatDatabaseEntriesAsText(ctx) - if (formattedDbEntries.isNotBlank()) { - initialHistory.add(content(role = "user") { text(formattedDbEntries) }) - } - } + val formattedDbEntries = context?.let { ctx -> + PhotoReasoningTextPolicies.formatDatabaseEntriesAsText(ctx) + } ?: "" + val initialHistory = PhotoReasoningHistoryBuilder.buildInitialHistory( + systemMessage = _systemMessage.value, + formattedDbEntries = formattedDbEntries + ) chat = generativeModel.startChat(history = initialHistory.toList()) // Update the flow with empty messages @@ -2485,7 +2211,7 @@ data class MistralResponseMessage( } context?.let { ctx -> - val formattedDbEntries = formatDatabaseEntriesAsText(ctx) + val formattedDbEntries = PhotoReasoningTextPolicies.formatDatabaseEntriesAsText(ctx) if (formattedDbEntries.isNotBlank()) { historyPairs.add(Pair("user", formattedDbEntries)) } @@ -2522,7 +2248,7 @@ data class MistralResponseMessage( if (screenshotUri == Uri.EMPTY) { // This case is for gemma-3n-e4b-it, where we don't have a screenshot. // We just want to send the screen info. - val genericAnalysisPrompt = "" + val genericAnalysisPrompt = createGenericScreenshotPrompt() reason( userInput = genericAnalysisPrompt, selectedImages = emptyList(), @@ -2532,12 +2258,10 @@ data class MistralResponseMessage( return } val currentTime = System.currentTimeMillis() - if (screenshotUri == lastProcessedScreenshotUri && (currentTime - lastProcessedScreenshotTime) < 2000) { // 2-second debounce window + if (!screenshotDebouncer.shouldProcess(screenshotUri, currentTime)) { Log.w(TAG, "addScreenshotToConversation: Debouncing duplicate/rapid call for URI $screenshotUri") return // Exit the function early if it's a duplicate call within the window } - lastProcessedScreenshotUri = screenshotUri - lastProcessedScreenshotTime = currentTime PhotoReasoningApplication.applicationScope.launch(Dispatchers.Main) { try { @@ -2546,44 +2270,14 @@ data class MistralResponseMessage( // Store the latest screenshot URI latestScreenshotUri = screenshotUri - // Initialize ImageLoader and ImageRequestBuilder if needed - if (imageLoader == null) { - imageLoader = ImageLoader.Builder(context).build() + PhotoReasoningScreenshotUiNotifier.showProcessing(context) { status -> + _commandExecutionStatus.value = status } - if (imageRequestBuilder == null) { - imageRequestBuilder = ImageRequest.Builder(context) - } - - // Update status - _commandExecutionStatus.value = "Processing screenshot..." - - // Show toast - Toast.makeText(context, "Processing screenshot...", Toast.LENGTH_SHORT).show() - - // Process the screenshot - val requestBuilder = imageRequestBuilder - if (requestBuilder == null) { - Log.e(TAG, "ImageRequest.Builder is not initialized") - _commandExecutionStatus.value = "Failed to process screenshot: request builder unavailable" - return@launch - } - val currentImageLoader = imageLoader - if (currentImageLoader == null) { - Log.e(TAG, "ImageLoader is not initialized") - _commandExecutionStatus.value = "Failed to process screenshot: image loader unavailable" - return@launch - } - - val imageRequest = requestBuilder - .data(screenshotUri) - .precision(Precision.EXACT) - .build() try { - val result = currentImageLoader.execute(imageRequest) - if (result is SuccessResult) { + val bitmap = screenshotProcessor.loadBitmap(context, screenshotUri) + if (bitmap != null) { Log.d(TAG, "Successfully processed screenshot") - val bitmap = (result.drawable as BitmapDrawable).bitmap // Add the screenshot to the current images val updatedImages = currentSelectedImages.toMutableList() @@ -2592,14 +2286,12 @@ data class MistralResponseMessage( // Update the current selected images - only keep the latest screenshot currentSelectedImages = listOf(bitmap) - // Update status - _commandExecutionStatus.value = "Screenshot added, sending to AI..." - - // Show toast - Toast.makeText(context, "Screenshot added, sending to AI...", Toast.LENGTH_SHORT).show() + PhotoReasoningScreenshotUiNotifier.showSendingToAi(context) { status -> + _commandExecutionStatus.value = status + } // Create prompt with screen information if available - val genericAnalysisPrompt = "" + val genericAnalysisPrompt = createGenericScreenshotPrompt() // Re-send the query with only the latest screenshot reason( @@ -2609,41 +2301,19 @@ data class MistralResponseMessage( imageUrisForChat = listOf(screenshotUri.toString()) // Add this argument ) - // Show a toast to indicate the screenshot was added - Toast.makeText(context, "Screenshot added to conversation", Toast.LENGTH_SHORT).show() + PhotoReasoningScreenshotUiNotifier.showAddedToConversation(context) } else { - Log.e(TAG, "Failed to process screenshot: result is not SuccessResult") - _commandExecutionStatus.value = "Error processing screenshot" - Toast.makeText(context, "Error processing screenshot", Toast.LENGTH_SHORT).show() - - // Add error message to chat - _chatState.addMessage( - PhotoReasoningMessage( - text = "Error processing screenshot", - participant = PhotoParticipant.ERROR - ) + handleScreenshotProcessingError( + context = context, + message = "Error processing screenshot" ) - _chatMessagesFlow.value = chatMessages - - // Save chat history after adding error message - saveChatHistory(context) } } catch (e: Exception) { - Log.e(TAG, "Error processing screenshot: ${e.message}", e) - _commandExecutionStatus.value = "Error processing screenshot: ${e.message}" - Toast.makeText(context, "Error processing screenshot: ${e.message}", Toast.LENGTH_SHORT).show() - - // Add error message to chat - _chatState.addMessage( - PhotoReasoningMessage( - text = "Error processing screenshot: ${e.message}", - participant = PhotoParticipant.ERROR - ) + handleScreenshotProcessingError( + context = context, + message = "Error processing screenshot: ${e.message}", + throwable = e ) - _chatMessagesFlow.value = chatMessages - - // Save chat history after adding error message - saveChatHistory(context) } } catch (e: Exception) { Log.e(TAG, "Error adding screenshot to conversation: ${e.message}", e) @@ -2653,63 +2323,4 @@ data class MistralResponseMessage( } } - /** - * Chat state management class - */ - private class ChatState { - private val _messages = mutableListOf() - val messages: List - get() = _messages.toList() // Return a copy to prevent concurrent modification - - fun addMessage(message: PhotoReasoningMessage) { - _messages.add(message) - } - - fun clearMessages() { - _messages.clear() - } - - fun replaceLastPendingMessage() { - val lastPendingIndex = _messages.indexOfLast { it.isPending } - if (lastPendingIndex >= 0) { - _messages.removeAt(lastPendingIndex) - } - } - - fun updateLastMessageText(newText: String) { - if (_messages.isNotEmpty()) { - val lastMessage = _messages.last() - _messages[_messages.size - 1] = lastMessage.copy(text = newText, isPending = false) - } - } - - // Add this method to get all messages atomically - fun getAllMessages(): List { - return _messages.toList() - } - - // Add this method to set all messages atomically - fun setAllMessages(messages: List) { - _messages.clear() - _messages.addAll(messages) - } - } } - -/** Thrown when Mistral returns HTTP 429 (rate limit). */ -private class MistralRateLimitException(message: String) : IOException(message) - -@Serializable -data class OpenAIStreamChunk( - val choices: List -) - -@Serializable -data class OpenAIStreamChoice( - val delta: OpenAIStreamDelta -) - -@Serializable -data class OpenAIStreamDelta( - val content: String? = null -) diff --git a/gradle.properties b/gradle.properties index c87abea..dc68f3c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,12 +2,12 @@ # http://www.gradle.org/docs/current/userguide/build_environment.html # # Specifies the JVM arguments used for the daemon process. -org.gradle.jvmargs=-Xmx768m -XX:MaxMetaspaceSize=256m -XX:+UseSerialGC -XX:CICompilerCount=2 -kotlin.daemon.jvmargs=-Xmx512m -XX:MaxMetaspaceSize=256m -XX:+UseSerialGC +org.gradle.jvmargs=-Xmx2048m -XX:MaxMetaspaceSize=1024m -XX:+UseSerialGC -XX:CICompilerCount=2 +kotlin.daemon.jvmargs=-Xmx1024m -XX:MaxMetaspaceSize=512m -XX:+UseSerialGC kotlin.compiler.execution.strategy=in-process org.gradle.workers.max=1 -# JDK 17 path for AGP 8.1.3 compatibility -# org.gradle.java.home=C:/Program Files/Microsoft/jdk-17.0.18.8-hotspot +# JDK 17 path for AGP/Kotlin compatibility (CI/Linux) +org.gradle.java.home=/root/.local/share/mise/installs/java/17.0.2 # # When configured, Gradle will run in incubating parallel mode. # This option should only be used with decoupled projects. More details, visit