Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package com.google.ai.sample

import com.google.ai.sample.util.Command
import java.util.LinkedList
import java.util.concurrent.atomic.AtomicBoolean

internal class AccessibilityCommandQueue {
private val queue = LinkedList<Command>()
private val processing = AtomicBoolean(false)

@Synchronized
fun clearAndUnlock() {
queue.clear()
processing.set(false)
}

@Synchronized
fun enqueue(command: Command): Int {
queue.add(command)
return queue.size
}

@Synchronized
fun peek(): Command? = queue.peek()

@Synchronized
fun poll(): Command? = queue.poll()

@Synchronized
fun size(): Int = queue.size

@Synchronized
fun isEmpty(): Boolean = queue.isEmpty()
fun tryAcquireProcessing(): Boolean = processing.compareAndSet(false, true)
fun releaseProcessing() = processing.set(false)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package com.google.ai.sample

import android.content.Context
import android.provider.Settings
import android.util.Log

internal object AccessibilityServiceStateChecker {
fun isEnabled(context: Context, tag: String): Boolean {
val accessibilityEnabled = try {
Settings.Secure.getInt(
context.contentResolver,
Settings.Secure.ACCESSIBILITY_ENABLED
)
} catch (e: Settings.SettingNotFoundException) {
Log.e(tag, "Error finding accessibility setting: ${e.message}")
return false
}

if (accessibilityEnabled != 1) {
Log.d(tag, "Accessibility is not enabled")
return false
}

val serviceString =
"${context.packageName}/${ScreenOperatorAccessibilityService::class.java.canonicalName}"
val enabledServices = Settings.Secure.getString(
context.contentResolver,
Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES
) ?: ""

val isEnabled = enabledServices.contains(serviceString)
Log.d(tag, "Service $serviceString is ${if (isEnabled) "enabled" else "not enabled"}")
return isEnabled
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package com.google.ai.sample

import android.graphics.Rect
import android.util.Log
import android.view.View
import android.view.ViewTreeObserver
import kotlinx.coroutines.flow.MutableStateFlow

internal class KeyboardVisibilityObserver(
private val tag: String
) {
private var onGlobalLayoutListener: ViewTreeObserver.OnGlobalLayoutListener? = null

fun start(rootView: View, keyboardState: MutableStateFlow<Boolean>) {
stop(rootView)
onGlobalLayoutListener = ViewTreeObserver.OnGlobalLayoutListener {
val rect = Rect()
rootView.getWindowVisibleDisplayFrame(rect)
val screenHeight = rootView.rootView.height
val keypadHeight = screenHeight - rect.bottom
if (keypadHeight > screenHeight * 0.15) {
if (!keyboardState.value) {
keyboardState.value = true
Log.d(tag, "Keyboard visible")
}
} else if (keyboardState.value) {
keyboardState.value = false
Log.d(tag, "Keyboard hidden")
}
}
rootView.viewTreeObserver.addOnGlobalLayoutListener(onGlobalLayoutListener)
}

fun stop(rootView: View) {
onGlobalLayoutListener?.let {
rootView.viewTreeObserver.removeOnGlobalLayoutListener(it)
}
onGlobalLayoutListener = null
}
}
46 changes: 8 additions & 38 deletions app/src/main/kotlin/com/google/ai/sample/MainActivity.kt
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import android.content.SharedPreferences
import android.content.pm.PackageManager
import android.net.Uri
import android.os.Build
import android.graphics.Rect
import android.os.Bundle
import android.provider.Settings
import android.util.Log
Expand All @@ -32,7 +31,6 @@ import java.text.SimpleDateFormat
import java.util.Date
import java.util.Locale
import android.view.View
import android.view.ViewTreeObserver
import android.widget.Toast
import androidx.activity.ComponentActivity
import androidx.activity.compose.setContent
Expand Down Expand Up @@ -102,7 +100,7 @@ class MainActivity : ComponentActivity() {
// Keyboard Visibility
private val _isKeyboardOpen = MutableStateFlow(false)
val isKeyboardOpen: StateFlow<Boolean> = _isKeyboardOpen.asStateFlow()
private var onGlobalLayoutListener: ViewTreeObserver.OnGlobalLayoutListener? = null
private val keyboardVisibilityObserver = KeyboardVisibilityObserver(TAG)

private var photoReasoningViewModel: PhotoReasoningViewModel? = null
private lateinit var apiKeyManager: ApiKeyManager
Expand Down Expand Up @@ -131,6 +129,7 @@ class MainActivity : ComponentActivity() {
private var isProcessingExplicitScreenshotRequest: Boolean = false
private var onMediaProjectionPermissionGranted: (() -> Unit)? = null
private var onWebRtcMediaProjectionResult: ((Int, Intent) -> Unit)? = null
private val mediaProjectionServiceStarter by lazy { MediaProjectionServiceStarter(this) }

// Payment dialog state
private var showPaymentMethodDialog by mutableStateOf(false)
Expand Down Expand Up @@ -224,20 +223,12 @@ class MainActivity : ComponentActivity() {
}
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
if (ContextCompat.checkSelfPermission(this, android.Manifest.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION) == PackageManager.PERMISSION_GRANTED) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
startForegroundService(serviceIntent)
} else {
startService(serviceIntent)
}
mediaProjectionServiceStarter.start(serviceIntent)
} else {
requestForegroundServicePermissionLauncher.launch(android.Manifest.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION)
}
} else {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
startForegroundService(serviceIntent)
} else {
startService(serviceIntent)
}
mediaProjectionServiceStarter.start(serviceIntent)
}

if (isProcessingExplicitScreenshotRequest) {
Expand Down Expand Up @@ -265,11 +256,7 @@ class MainActivity : ComponentActivity() {
val serviceIntent = Intent(this, ScreenCaptureService::class.java).apply {
action = ScreenCaptureService.ACTION_KEEP_ALIVE_FOR_WEBRTC
}
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
startForegroundService(serviceIntent)
} else {
startService(serviceIntent)
}
mediaProjectionServiceStarter.start(serviceIntent)

onWebRtcMediaProjectionResult?.invoke(resultCode, resultData)
onWebRtcMediaProjectionResult = null
Expand All @@ -282,22 +269,7 @@ class MainActivity : ComponentActivity() {

private fun setupKeyboardVisibilityListener() {
val rootView = findViewById<View>(android.R.id.content)
onGlobalLayoutListener = ViewTreeObserver.OnGlobalLayoutListener {
val rect = Rect()
rootView.getWindowVisibleDisplayFrame(rect)
val screenHeight = rootView.rootView.height
val keypadHeight = screenHeight - rect.bottom
if (keypadHeight > screenHeight * 0.15) {
if (!_isKeyboardOpen.value) {
_isKeyboardOpen.value = true
Log.d(TAG, "Keyboard visible")
}
} else if (_isKeyboardOpen.value) {
_isKeyboardOpen.value = false
Log.d(TAG, "Keyboard hidden")
}
}
rootView.viewTreeObserver.addOnGlobalLayoutListener(onGlobalLayoutListener)
keyboardVisibilityObserver.start(rootView, _isKeyboardOpen)
}

private fun registerScreenshotReceivers() {
Expand Down Expand Up @@ -1141,10 +1113,8 @@ class MainActivity : ComponentActivity() {
billingClient.endConnection()
Log.d(TAG, "onDestroy: BillingClient connection ended.")
}
onGlobalLayoutListener?.let {
findViewById<View>(android.R.id.content).viewTreeObserver.removeOnGlobalLayoutListener(it)
Log.d(TAG, "onDestroy: Keyboard layout listener removed.")
}
keyboardVisibilityObserver.stop(findViewById(android.R.id.content))
Log.d(TAG, "onDestroy: Keyboard layout listener removed.")
if (this == instance) {
instance = null
Log.d(TAG, "onDestroy: MainActivity instance cleared.")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package com.google.ai.sample

import android.content.Context
import android.content.Intent
import android.os.Build

internal class MediaProjectionServiceStarter(private val context: Context) {
fun start(intent: Intent) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
context.startForegroundService(intent)
} else {
context.startService(intent)
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import android.os.Environment
import android.os.Handler
import android.os.Looper
import android.provider.MediaStore
import android.provider.Settings
import android.util.Log
import android.view.KeyEvent
import android.view.accessibility.AccessibilityEvent
Expand All @@ -35,7 +34,6 @@ import java.util.Date
import java.util.Locale
import java.util.concurrent.atomic.AtomicBoolean
import java.lang.NumberFormatException
import java.util.LinkedList

class ScreenOperatorAccessibilityService : AccessibilityService() {
private data class ResolvedPoint(val xPx: Float, val yPx: Float)
Expand All @@ -48,8 +46,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {

private enum class ScrollAxis { HORIZONTAL, VERTICAL }

private val commandQueue = LinkedList<Command>()
private val isProcessingQueue = AtomicBoolean(false)
private val commandQueue = AccessibilityCommandQueue()
// private val handler = Handler(Looper.getMainLooper()) // Already exists at the class level

companion object {
Expand All @@ -68,30 +65,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
* Check if the accessibility service is enabled in system settings
*/
fun isAccessibilityServiceEnabled(context: Context): Boolean {
val accessibilityEnabled = try {
Settings.Secure.getInt(
context.contentResolver,
Settings.Secure.ACCESSIBILITY_ENABLED
)
} catch (e: Settings.SettingNotFoundException) {
Log.e(TAG, "Error finding accessibility setting: ${e.message}")
return false
}

if (accessibilityEnabled != 1) {
Log.d(TAG, "Accessibility is not enabled")
return false
}

val serviceString = "${context.packageName}/${ScreenOperatorAccessibilityService::class.java.canonicalName}"
val enabledServices = Settings.Secure.getString(
context.contentResolver,
Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES
) ?: ""

val isEnabled = enabledServices.contains(serviceString)
Log.d(TAG, "Service $serviceString is ${if (isEnabled) "enabled" else "not enabled"}")
return isEnabled
return AccessibilityServiceStateChecker.isEnabled(context, TAG)
}

/**
Expand All @@ -109,8 +83,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
fun clearCommandQueue() {
val instance = serviceInstance
if (instance != null) {
instance.commandQueue.clear()
instance.isProcessingQueue.set(false)
instance.commandQueue.clearAndUnlock()
Log.d(TAG, "Command queue cleared and processing flag reset.")
} else {
Log.w(TAG, "clearCommandQueue: serviceInstance is null, nothing to clear.")
Expand Down Expand Up @@ -143,8 +116,8 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
Log.e(TAG, "Service instance became null before queueing command")
return
}
instance.commandQueue.add(command)
Log.d(TAG, "Command $command added to queue. Queue size: ${instance.commandQueue.size}")
val queueSize = instance.commandQueue.enqueue(command)
Log.d(TAG, "Command $command added to queue. Queue size: $queueSize")

// Ensure processCommandQueue is called on the service's handler thread
instance.handler.post {
Expand Down Expand Up @@ -234,7 +207,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
}

handler.postDelayed({
isProcessingQueue.set(false) // Release the lock before the next cycle
commandQueue.releaseProcessing() // Release the lock before the next cycle
processCommandQueue() // Try to process the next command
}, nextCommandDelay)
}
Expand Down Expand Up @@ -506,19 +479,19 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
}

private fun processCommandQueue() {
if (!isProcessingQueue.compareAndSet(false, true)) {
if (!commandQueue.tryAcquireProcessing()) {
Log.d(TAG, "Queue is already being processed.")
return
}

if (commandQueue.isEmpty()) {
Log.d(TAG, "Command queue is empty. Stopping processing.")
isProcessingQueue.set(false)
commandQueue.releaseProcessing()
return
}

val command = commandQueue.poll()
Log.d(TAG, "Processing command: $command. Queue size after poll: ${commandQueue.size}")
Log.d(TAG, "Processing command: $command. Queue size after poll: ${commandQueue.size()}")

if (command != null) {
val commandWasAsync = executeSingleCommand(command) // executeSingleCommand now returns Boolean
Expand All @@ -531,7 +504,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
// is responsible for calling scheduleNextCommandProcessing upon completion.
} else {
Log.d(TAG, "Polled null command from queue, stopping processing.")
isProcessingQueue.set(false)
commandQueue.releaseProcessing()
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package com.google.ai.sample.feature.multimodal

import android.content.Context
import com.google.ai.sample.ApiProvider
import com.google.ai.sample.MainActivity

internal object MainActivityBridge {
fun applicationContextOrNull(): Context? =
MainActivity.getInstance()?.applicationContext

fun currentApiKeyOrEmpty(provider: ApiProvider): String =
MainActivity.getInstance()?.getCurrentApiKey(provider) ?: ""
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package com.google.ai.sample.feature.multimodal

import android.graphics.Bitmap
import android.util.Base64
import kotlinx.serialization.json.Json
import java.io.ByteArrayOutputStream

internal object PhotoReasoningSerialization {
fun createStreamingJsonParser(): Json = Json { ignoreUnknownKeys = true }

fun bitmapToBase64(bitmap: Bitmap): String {
val outputStream = ByteArrayOutputStream()
bitmap.compress(Bitmap.CompressFormat.JPEG, 90, outputStream)
val bytes = outputStream.toByteArray()
return Base64.encodeToString(bytes, Base64.NO_WRAP)
}
}
Loading