diff --git a/README.md b/README.md index b83677228..ea65e697a 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ Flags: - `--device ` - `--udid ` (iOS) - `--serial ` (Android) -- `--activity ` (Android; package/Activity or package/.Activity) +- `--activity ` (Android app launch only; package/Activity or package/.Activity; not for URL opens) - `--session ` - `--verbose` for daemon and runner logs - `--json` for structured output @@ -117,7 +117,7 @@ npx skills add https://github.com/callstackincubator/agent-device --skill agent- Sessions: - `open` starts a session. Without args boots/activates the target device/simulator without launching an app. - All interaction commands require an open session. -- If a session is already open, `open ` switches the active app and updates the session app bundle. +- If a session is already open, `open ` switches the active app or opens a deep link URL. - `close` stops the session and releases device resources. Pass an app to close it explicitly, or omit to just close the session. - Use `--session ` to manage multiple sessions. - Session scripts are written to `~/.agent-device/sessions/-.ad` when recording is enabled with `--save-script`. @@ -126,10 +126,21 @@ Sessions: Navigation helpers: - `boot --platform ios|android` ensures the target is ready without launching an app. - Use `boot` mainly when starting a new session and `open` fails because no booted simulator/emulator is available. -- `open [app]` already boots/activates the selected target when needed. +- `open [app|url]` already boots/activates the selected target when needed. - `reinstall ` uninstalls and installs the app binary in one command (Android + iOS simulator in v1). - `reinstall` accepts package/bundle id style app names and supports `~` in paths. +Deep links: +- `open ` supports deep links with `scheme://...`. +- Android opens deep links via `VIEW` intent. +- iOS deep link open is simulator-only in v1. +- `--activity` cannot be combined with URL opens. + +```bash +agent-device open "myapp://home" --platform android +agent-device open "https://example.com" --platform ios +``` + Find (semantic): - `find [value]` finds by any text (label/value/identifier) using a scoped snapshot. - `find text|label|value|role|id [value]` for specific locators. diff --git a/skills/agent-device/SKILL.md b/skills/agent-device/SKILL.md index 2b12dd52d..06ead2e47 100644 --- a/skills/agent-device/SKILL.md +++ b/skills/agent-device/SKILL.md @@ -27,7 +27,7 @@ npx -y agent-device ## Core workflow -1. Open app: `open [app]` (`open` handles target selection + boot/activation in the normal flow) +1. Open app or deep link: `open [app|url]` (`open` handles target selection + boot/activation in the normal flow) 2. Snapshot: `snapshot` to get refs from accessibility tree 3. Interact using refs (`click @ref`, `fill @ref "text"`) 4. Re-snapshot after navigation/UI changes @@ -41,8 +41,10 @@ npx -y agent-device agent-device boot # Ensure target is booted/ready without opening app agent-device boot --platform ios # Boot iOS simulator agent-device boot --platform android # Boot Android emulator/device target -agent-device open [app] # Boot device/simulator; optionally launch app -agent-device open [app] --activity com.example/.MainActivity # Android: open specific activity +agent-device open [app|url] # Boot device/simulator; optionally launch app or deep link URL +agent-device open [app] --activity com.example/.MainActivity # Android: open specific activity (app targets only) +agent-device open "myapp://home" --platform android # Android deep link +agent-device open "https://example.com" --platform ios # iOS simulator deep link agent-device close [app] # Close app or just end session agent-device reinstall # Uninstall + install app in one command agent-device session list # List active sessions @@ -168,10 +170,12 @@ agent-device apps --platform android --user-installed - Prefer `snapshot -i` to reduce output size. - On iOS, `xctest` is the default and does not require Accessibility permission. - If XCTest returns 0 nodes (foreground app changed), agent-device falls back to AX when available. -- `open ` can be used within an existing session to switch apps and update the session bundle id. +- `open ` can be used within an existing session to switch apps or open deep links. +- `open ` updates session app bundle context; URL opens do not set an app bundle id. - If AX returns the Simulator window or empty tree, restart Simulator or use `--backend xctest`. - Use `--session ` for parallel sessions; avoid device contention. -- Use `--activity ` on Android to launch a specific activity (e.g. TV apps with LEANBACK). +- Use `--activity ` on Android to launch a specific activity (e.g. TV apps with LEANBACK); do not combine with URL opens. +- iOS deep-link opens are simulator-only in v1. - Use `fill` when you want clear-then-type semantics. - Use `type` when you want to append/enter text without clearing. - On Android, prefer `fill` for important fields; it verifies entered text and retries once when IME reorders characters. diff --git a/src/core/__tests__/open-target.test.ts b/src/core/__tests__/open-target.test.ts new file mode 100644 index 000000000..a0b62426e --- /dev/null +++ b/src/core/__tests__/open-target.test.ts @@ -0,0 +1,14 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { isDeepLinkTarget } from '../open-target.ts'; + +test('isDeepLinkTarget accepts URL-style deep links', () => { + assert.equal(isDeepLinkTarget('myapp://home'), true); + assert.equal(isDeepLinkTarget('https://example.com'), true); +}); + +test('isDeepLinkTarget rejects app identifiers and malformed URLs', () => { + assert.equal(isDeepLinkTarget('com.example.app'), false); + assert.equal(isDeepLinkTarget('settings'), false); + assert.equal(isDeepLinkTarget('http:/x'), false); +}); diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index 33e2b872c..f2112579f 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -106,7 +106,7 @@ export async function dispatchCommand( await interactor.openDevice(); return { app: null }; } - await interactor.open(app, { activity: context?.activity }); + await interactor.open(app, { activity: context?.activity, appBundleId: context?.appBundleId }); return { app }; } case 'close': { diff --git a/src/core/open-target.ts b/src/core/open-target.ts new file mode 100644 index 000000000..b680017f4 --- /dev/null +++ b/src/core/open-target.ts @@ -0,0 +1,5 @@ +export function isDeepLinkTarget(input: string): boolean { + const value = input.trim(); + if (!value) return false; + return /^[A-Za-z][A-Za-z0-9+.-]*:\/\/.+/.test(value); +} diff --git a/src/daemon/handlers/__tests__/session.test.ts b/src/daemon/handlers/__tests__/session.test.ts index e9ebe5ac4..332b477d0 100644 --- a/src/daemon/handlers/__tests__/session.test.ts +++ b/src/daemon/handlers/__tests__/session.test.ts @@ -120,3 +120,95 @@ test('boot succeeds for supported device in session', async () => { assert.equal(response.data?.booted, true); } }); + +test('open URL on existing iOS session clears stale app bundle id', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'ios-session'; + sessionStore.set( + sessionName, + { + ...makeSession(sessionName, { + platform: 'ios', + id: 'sim-1', + name: 'iPhone 15', + kind: 'simulator', + booted: true, + }), + appBundleId: 'com.example.old', + appName: 'Old App', + }, + ); + + let dispatchedContext: Record | undefined; + const response = await handleSessionCommands({ + req: { + token: 't', + session: sessionName, + command: 'open', + positionals: ['https://example.com/path'], + flags: {}, + }, + sessionName, + logPath: path.join(os.tmpdir(), 'daemon.log'), + sessionStore, + invoke: noopInvoke, + dispatch: async (_device, _command, _positionals, _out, context) => { + dispatchedContext = context as Record | undefined; + return {}; + }, + ensureReady: async () => {}, + }); + + assert.ok(response); + assert.equal(response?.ok, true); + const updated = sessionStore.get(sessionName); + assert.equal(updated?.appBundleId, undefined); + assert.equal(updated?.appName, 'https://example.com/path'); + assert.equal(dispatchedContext?.appBundleId, undefined); +}); + +test('open app on existing iOS session resolves and stores bundle id', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'ios-session'; + sessionStore.set( + sessionName, + { + ...makeSession(sessionName, { + platform: 'ios', + id: 'sim-1', + name: 'iPhone 15', + kind: 'simulator', + booted: true, + }), + appBundleId: 'com.example.old', + appName: 'Old App', + }, + ); + + let dispatchedContext: Record | undefined; + const response = await handleSessionCommands({ + req: { + token: 't', + session: sessionName, + command: 'open', + positionals: ['settings'], + flags: {}, + }, + sessionName, + logPath: path.join(os.tmpdir(), 'daemon.log'), + sessionStore, + invoke: noopInvoke, + dispatch: async (_device, _command, _positionals, _out, context) => { + dispatchedContext = context as Record | undefined; + return {}; + }, + ensureReady: async () => {}, + }); + + assert.ok(response); + assert.equal(response?.ok, true); + const updated = sessionStore.get(sessionName); + assert.equal(updated?.appBundleId, 'com.apple.Preferences'); + assert.equal(updated?.appName, 'settings'); + assert.equal(dispatchedContext?.appBundleId, 'com.apple.Preferences'); +}); diff --git a/src/daemon/handlers/session.ts b/src/daemon/handlers/session.ts index baf000a47..a33282171 100644 --- a/src/daemon/handlers/session.ts +++ b/src/daemon/handlers/session.ts @@ -1,6 +1,7 @@ import fs from 'node:fs'; import { dispatchCommand, resolveTargetDevice } from '../../core/dispatch.ts'; import { isCommandSupportedOnDevice } from '../../core/capabilities.ts'; +import { isDeepLinkTarget } from '../../core/open-target.ts'; import { AppError, asAppError } from '../../utils/errors.ts'; import type { DeviceInfo } from '../../utils/device.ts'; import type { DaemonRequest, DaemonResponse, SessionAction, SessionState } from '../types.ts'; @@ -299,7 +300,7 @@ export async function handleSessionCommands(params: { }; } let appBundleId: string | undefined; - if (session.device.platform === 'ios') { + if (session.device.platform === 'ios' && !isDeepLinkTarget(appName)) { try { const { resolveIosApp } = await import('../../platforms/ios/index.ts'); appBundleId = await resolveIosApp(session.device, appName); @@ -340,10 +341,10 @@ export async function handleSessionCommands(params: { } let appBundleId: string | undefined; const appName = req.positionals?.[0]; - if (device.platform === 'ios') { + if (device.platform === 'ios' && appName && !isDeepLinkTarget(appName)) { try { const { resolveIosApp } = await import('../../platforms/ios/index.ts'); - appBundleId = await resolveIosApp(device, req.positionals?.[0] ?? ''); + appBundleId = await resolveIosApp(device, appName); } catch { appBundleId = undefined; } diff --git a/src/platforms/android/__tests__/index.test.ts b/src/platforms/android/__tests__/index.test.ts index f553837c7..ea4bf9416 100644 --- a/src/platforms/android/__tests__/index.test.ts +++ b/src/platforms/android/__tests__/index.test.ts @@ -1,6 +1,8 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { parseAndroidLaunchComponent } from '../index.ts'; +import { openAndroidApp, parseAndroidLaunchComponent } from '../index.ts'; +import type { DeviceInfo } from '../../../utils/device.ts'; +import { AppError } from '../../../utils/errors.ts'; import { findBounds, parseUiHierarchy } from '../ui-hierarchy.ts'; test('parseUiHierarchy reads double-quoted Android node attributes', () => { @@ -89,3 +91,22 @@ test('parseAndroidLaunchComponent returns null when no component is present', () const stdout = 'No activity found'; assert.equal(parseAndroidLaunchComponent(stdout), null); }); + +test('openAndroidApp rejects activity override for deep link URLs', async () => { + const device: DeviceInfo = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, + }; + + await assert.rejects( + () => openAndroidApp(device, ' https://example.com/path ', '.MainActivity'), + (error: unknown) => { + assert.equal(error instanceof AppError, true); + assert.equal((error as AppError).code, 'INVALID_ARGS'); + return true; + }, + ); +}); diff --git a/src/platforms/android/index.ts b/src/platforms/android/index.ts index 412057d83..33463167a 100644 --- a/src/platforms/android/index.ts +++ b/src/platforms/android/index.ts @@ -4,6 +4,7 @@ import { withRetry } from '../../utils/retry.ts'; import { AppError } from '../../utils/errors.ts'; import type { DeviceInfo } from '../../utils/device.ts'; import type { RawSnapshotNode, SnapshotOptions } from '../../utils/snapshot.ts'; +import { isDeepLinkTarget } from '../../core/open-target.ts'; import { waitForAndroidBoot } from './devices.ts'; import { findBounds, parseBounds, parseUiHierarchy, readNodeAttributes } from './ui-hierarchy.ts'; @@ -157,6 +158,23 @@ export async function openAndroidApp( if (!device.booted) { await waitForAndroidBoot(device.id); } + const deepLinkTarget = app.trim(); + if (isDeepLinkTarget(deepLinkTarget)) { + if (activity) { + throw new AppError('INVALID_ARGS', 'Activity override is not supported when opening a deep link URL'); + } + await runCmd('adb', adbArgs(device, [ + 'shell', + 'am', + 'start', + '-W', + '-a', + 'android.intent.action.VIEW', + '-d', + deepLinkTarget, + ])); + return; + } const resolved = await resolveAndroidApp(device, app); if (resolved.type === 'intent') { if (activity) { diff --git a/src/platforms/ios/__tests__/index.test.ts b/src/platforms/ios/__tests__/index.test.ts new file mode 100644 index 000000000..d96927468 --- /dev/null +++ b/src/platforms/ios/__tests__/index.test.ts @@ -0,0 +1,24 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { openIosApp } from '../index.ts'; +import type { DeviceInfo } from '../../../utils/device.ts'; +import { AppError } from '../../../utils/errors.ts'; + +test('openIosApp rejects deep links on iOS physical devices', async () => { + const device: DeviceInfo = { + platform: 'ios', + id: 'ios-device-1', + name: 'iPhone Device', + kind: 'device', + booted: true, + }; + + await assert.rejects( + () => openIosApp(device, 'https://example.com/path'), + (error: unknown) => { + assert.equal(error instanceof AppError, true); + assert.equal((error as AppError).code, 'UNSUPPORTED_OPERATION'); + return true; + }, + ); +}); diff --git a/src/platforms/ios/index.ts b/src/platforms/ios/index.ts index 9dbf8fd13..c937158e6 100644 --- a/src/platforms/ios/index.ts +++ b/src/platforms/ios/index.ts @@ -3,6 +3,7 @@ import type { ExecResult } from '../../utils/exec.ts'; import { AppError } from '../../utils/errors.ts'; import type { DeviceInfo } from '../../utils/device.ts'; import { Deadline, isEnvTruthy, retryWithPolicy, TIMEOUT_PROFILES, type RetryTelemetryEvent } from '../../utils/retry.ts'; +import { isDeepLinkTarget } from '../../core/open-target.ts'; import { bootFailureHint, classifyBootFailure } from '../boot-diagnostics.ts'; const ALIASES: Record = { @@ -35,8 +36,22 @@ export async function resolveIosApp(device: DeviceInfo, app: string): Promise { - const bundleId = await resolveIosApp(device, app); +export async function openIosApp( + device: DeviceInfo, + app: string, + options?: { appBundleId?: string }, +): Promise { + const deepLinkTarget = app.trim(); + if (isDeepLinkTarget(deepLinkTarget)) { + if (device.kind !== 'simulator') { + throw new AppError('UNSUPPORTED_OPERATION', 'Deep link open is only supported on iOS simulators in v1'); + } + await ensureBootedSimulator(device); + await runCmd('open', ['-a', 'Simulator'], { allowFailure: true }); + await runCmd('xcrun', ['simctl', 'openurl', device.id, deepLinkTarget]); + return; + } + const bundleId = options?.appBundleId ?? (await resolveIosApp(device, app)); if (device.kind === 'simulator') { await ensureBootedSimulator(device); await runCmd('open', ['-a', 'Simulator'], { allowFailure: true }); diff --git a/src/utils/args.ts b/src/utils/args.ts index cb1968c0d..97fdabdac 100644 --- a/src/utils/args.ts +++ b/src/utils/args.ts @@ -174,7 +174,7 @@ CLI to control iOS and Android devices for AI agents. Commands: boot Ensure target device/simulator is booted and ready - open [app] Boot device/simulator; optionally launch app + open [app|url] Boot device/simulator; optionally launch app or deep link URL close [app] Close app or just end session reinstall Uninstall + install app from binary path snapshot [-i] [-c] [-d ] [-s ] [--raw] [--backend ax|xctest] @@ -227,7 +227,7 @@ Flags: --device Device name to target --udid iOS device UDID --serial Android device serial - --activity Android activity to launch (package/Activity) + --activity Android app launch activity (package/Activity); not for URL opens --session Named session --verbose Stream daemon/runner logs --json JSON output diff --git a/src/utils/interactors.ts b/src/utils/interactors.ts index 2d9e71eeb..30abf4460 100644 --- a/src/utils/interactors.ts +++ b/src/utils/interactors.ts @@ -29,7 +29,7 @@ export type RunnerContext = { }; export type Interactor = { - open(app: string, options?: { activity?: string }): Promise; + open(app: string, options?: { activity?: string; appBundleId?: string }): Promise; openDevice(): Promise; close(app: string): Promise; tap(x: number, y: number): Promise; @@ -60,7 +60,7 @@ export function getInteractor(device: DeviceInfo, runnerContext: RunnerContext): }; case 'ios': return { - open: (app) => openIosApp(device, app), + open: (app, options) => openIosApp(device, app, { appBundleId: options?.appBundleId }), openDevice: () => openIosDevice(device), close: (app) => closeIosApp(device, app), screenshot: (outPath) => screenshotIos(device, outPath),