Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CONTEXT.md
Original file line number Diff line number Diff line change
Expand Up @@ -431,6 +431,11 @@ the observable freshness and failure semantics below before any runtime refactor
- iOS sparse and AX failures are not proof of empty UI. Regular visible snapshots can recover through
the capture plan; raw and strict paths preserve failure. `runnerFatal` invalidates the cached target
and must never refresh healthy mutation recency.
- An `XCTEST_RECORDED_FAILURE` after an iOS tap is an ambiguous outcome, not proof that the tap missed.
The daemon may take one same-presentation post-action capture against a usable retained snapshot;
only a changed accessibility digest converts the result to success with a warning. Capture failure,
sparse or mismatched presentation, and an unchanged digest remain failures so corroboration cannot
turn an unknown tap into a false success.
- Android helper reuse must not become snapshot result caching. Freshness is short lived, marked only
after navigation-sensitive actions, compared against broad route-safe baselines, and not learned
from scoped, depth-limited, interactive, or ref-refresh snapshots.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,16 @@ extension RunnerTests {
}
#endif

func testInjectedTapRecordedFailureGateIsTapOnlyAndCountGated() {
// The seam's recording side cannot run in-bundle (a real XCTIssue would
// fail this very test run — same constraint the record(_:) suppression
// tests document); the live daemon proof covers it. This pins the gate.
XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .tap, remaining: 0))
XCTAssertTrue(RunnerTests.shouldInjectTapRecordedFailure(command: .tap, remaining: 1))
XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .type, remaining: 1))
XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .snapshot, remaining: 1))
}

func testXCTestRecordedFailureResponseFailsMutatingSuccesses() throws {
let command = try runnerCommandFixture(#"{"command":"tap","commandId":"tap-1"}"#)
let response = Response(ok: true, data: DataPayload(message: "tapped"))
Expand Down Expand Up @@ -1227,6 +1237,20 @@ extension RunnerTests {
userInfo: [NSLocalizedDescriptionKey: "command returned no response"]
)
}
#if AGENT_DEVICE_RUNNER_UNIT_TESTS
// #1605 merge gate: the REAL gesture already executed above; recording a
// production-shaped issue here makes the per-command failure-count
// conversion below fire exactly as in the field (bsky-24: activation
// lands, bookkeeping records a failure). Compiled out of production.
if consumeInjectedTapRecordedFailureForTesting(command: command.command) {
record(
XCTIssue(
type: .assertionFailure,
compactDescription: "Injected tap recorded-failure (#1605 corroboration merge gate)"
)
)
}
#endif
if didRecordXCTestFailure(since: failureCountBefore),
let failureResponse = xctestRecordedFailureResponse(command: command, response: response)
{
Expand Down Expand Up @@ -2481,7 +2505,7 @@ extension RunnerTests {
error: ErrorPayload(
code: "XCTEST_RECORDED_FAILURE",
message: "XCTest recorded a failure while executing \(command.command.rawValue); the action may not have been performed.",
hint: "The iOS runner session will be restarted. Retry after a fresh snapshot, or use screenshot plus coordinate commands when the accessibility tree is unavailable."
hint: "The iOS runner session was invalidated. Re-observe with a fresh snapshot before retrying; if the accessibility tree is unavailable, use screenshot plus coordinate commands instead of retrying the tap blindly."
)
)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,11 @@ extension RunnerTests {

func resetTargetAfterExternalRelaunch() -> Response {
invalidateCachedTarget(reason: "external_app_relaunch")
// The app process is replaced, but the retained runner survives. Clear
// process-bound capture state explicitly because invalidation drops the
// old PID before refreshCachedTargetIfProcessChanged can observe it.
clearSnapshotXCTestChannelPenalty(reason: "external_app_relaunch")
clearPrivateAXAcceptedDepth(reason: "external_app_relaunch")
needsFirstInteractionDelay = true
return Response(ok: true, data: DataPayload(message: "target reset"))
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,8 @@ extension RunnerTests {
currentAppProcessIdentifier = 42
snapshotXCTestPenaltyWarmupExemptionPending = true
needsFirstInteractionDelay = false
penalizeSnapshotXCTestChannel(bundleId: "com.example.app", reason: "test")
XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app"))

let response = resetTargetAfterExternalRelaunch()

Expand All @@ -142,6 +144,7 @@ extension RunnerTests {
XCTAssertNil(currentBundleId)
XCTAssertNil(currentAppProcessIdentifier)
XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending)
XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app"))
XCTAssertTrue(needsFirstInteractionDelay)
}
#endif
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,42 @@ final class RunnerTests: XCTestCase {
// seconds on remote-hosted consent dialogs and bypass the plan budget (#1244).
let systemModalProbeBudget: TimeInterval = 4
#if AGENT_DEVICE_RUNNER_UNIT_TESTS
// #1605 merge gate: deterministic live reproduction of the field ambiguity —
// a tap whose coordinate activation LANDS while XCTest bookkeeping records a
// failure. Armed by writing a decrementing count to the flag file below
// (the daemon regenerates tampered xctestrun templates, so env plumbing
// cannot reach a daemon-spawned runner); consumed one injection per tap.
// The injection records a real XCTIssue AFTER the real gesture, so
// `xctestRecordedFailureResponse` and target invalidation fire byte-for-byte
// like a field failure. Production builds compile none of this.
static let injectedTapFailureFlagPathForTesting =
"/tmp/agent-device-inject-tap-recorded-failure-for-testing"

static func shouldInjectTapRecordedFailure(command: CommandType, remaining: Int) -> Bool {
command == .tap && remaining > 0
}

func consumeInjectedTapRecordedFailureForTesting(command: CommandType) -> Bool {
guard
let raw = try? String(
contentsOfFile: Self.injectedTapFailureFlagPathForTesting,
encoding: .utf8
),
let remaining = Int(raw.trimmingCharacters(in: .whitespacesAndNewlines))
else {
return false
}
guard Self.shouldInjectTapRecordedFailure(command: command, remaining: remaining) else {
return false
}
try? String(remaining - 1).write(
toFile: Self.injectedTapFailureFlagPathForTesting,
atomically: true,
encoding: .utf8
)
return true
}

// Unit-test-only injectable override for the system-modal probe (see
// `boundedBlockingSystemAlertSnapshot` in RunnerTests+Snapshot.swift): when set, a test's probe
// body runs in place of `blockingSystemAlertSnapshot` so it can force a real timeout without a
Expand Down
12 changes: 10 additions & 2 deletions docs/adr/0005-ios-runner-interaction-lifecycle.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@ normal activation path.

An external iOS simulator relaunch also invalidates process-bound target state. After replacing the
app process, the daemon sends a lifecycle reset to the retained runner so the next command reacquires
`XCUIApplication`; if that reset cannot be confirmed, the daemon discards the runner session.
`XCUIApplication`; the reset also clears process-bound snapshot penalty and private-AX depth state. If
that reset cannot be confirmed, the daemon discards the runner session.

The snapshot surface intentionally has two AX-failure shapes. Interactive fast snapshots return a
truncated success payload with `runnerFatal` so agents can still see that AX state is unavailable
Expand Down Expand Up @@ -93,8 +94,15 @@ Apps with broken accessibility trees may still be impossible for XCTest to inspe
failed snapshot no longer teaches the runner to keep using a suspect cached app target or to amplify
the failure by walking every interactive element query.

An `XCTEST_RECORDED_FAILURE` returned after a tap is treated as an ambiguous outcome at the daemon
boundary. When a usable retained snapshot exists, the daemon takes one same-presentation post-action
capture. A changed accessibility digest is reported as success with an explicit warning so agents do
not blindly repeat a tap that may already have navigated; unchanged, sparse, mismatched, or unavailable
evidence remains the original failure.

Simulator relaunch keeps the healthy XCTest process warm without carrying an app target across
process identity. The reset adds one local runner request instead of paying for a runner restart.
process identity. The reset adds one local runner request instead of paying for a runner restart and
clears the old process's hostile-screen capture penalty before the replacement is reacquired.

Future optimization work should only reduce these preflights after the runner exposes status in a
way that survives command-induced XCTest teardown and can prove the session is still serving new
Expand Down
26 changes: 18 additions & 8 deletions src/commands/interaction/runtime/interactions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { isFillableType } from '@agent-device/contracts/snapshot';
import { successText } from '../../../utils/success-text.ts';
import { findMistargetedTypeRefToken } from '../../../utils/type-target-warning.ts';
import { requireIntInRange } from '../../../utils/validation.ts';
import { attachResolvedInteractionTarget } from '../../../contracts/interaction-outcome.ts';
import type { RepeatedInput } from '../../command-input.ts';
import { toBackendContext } from '../../runtime-common.ts';
import {
Expand Down Expand Up @@ -196,14 +197,23 @@ async function tapCommand(
throw new AppError('UNSUPPORTED_OPERATION', 'tap is not supported by this backend');
}
const point = requireResolvedPoint(resolved);
const backendResult = await runtime.backend.tap(toBackendContext(runtime, options), point, {
button: options.button,
count: options.count,
intervalMs: options.intervalMs,
holdMs: options.holdMs,
jitterPx: options.jitterPx,
doubleTap: options.doubleTap,
});
let backendResult;
try {
backendResult = await runtime.backend.tap(toBackendContext(runtime, options), point, {
button: options.button,
count: options.count,
intervalMs: options.intervalMs,
holdMs: options.holdMs,
jitterPx: options.jitterPx,
doubleTap: options.doubleTap,
});
} catch (error) {
// Resolution is complete before the backend call. Preserve it out of
// band so a daemon-level failure corroboration can still record the same
// target identity if the backend reports an ambiguous tap outcome.
attachResolvedInteractionTarget(error, resolved);
throw error;
}
const formattedBackendResult = toBackendResult(backendResult);
return await applyPostActionObservation(
runtime,
Expand Down
25 changes: 25 additions & 0 deletions src/contracts/interaction-outcome.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
import type { ResolvedInteractionTarget } from '@agent-device/contracts/interaction';

// The runtime can resolve an interaction before the backend reports a failure.
// Keep that resolution out of serialized error details while making it
// available to the daemon's failure-corroboration boundary.
const resolvedInteractionTargets = new WeakMap<object, ResolvedInteractionTarget>();

/** Preserve pre-dispatch target identity across a backend rejection. */
export function attachResolvedInteractionTarget(
error: unknown,
target: ResolvedInteractionTarget,
): void {
if (isObjectLike(error)) resolvedInteractionTargets.set(error, target);
}

/** Read the target captured before a backend interaction rejection. */
export function readResolvedInteractionTarget(
error: unknown,
): ResolvedInteractionTarget | undefined {
return isObjectLike(error) ? resolvedInteractionTargets.get(error) : undefined;
}

function isObjectLike(value: unknown): value is object {
return (typeof value === 'object' && value !== null) || typeof value === 'function';
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
import { buildSnapshotState } from '../snapshot-capture.ts';

export const profileNodes: RawSnapshotNode[] = [
{
index: 0,
type: 'Application',
label: 'Profile',
rect: { x: 0, y: 0, width: 390, height: 844 },
},
{
index: 1,
parentIndex: 0,
type: 'Button',
identifier: 'unfollow',
label: 'Unfollow',
rect: { x: 24, y: 200, width: 160, height: 44 },
hittable: true,
},
];

export const imageViewerNodes: RawSnapshotNode[] = [
{
index: 0,
type: 'Application',
label: 'Image viewer',
rect: { x: 0, y: 0, width: 390, height: 844 },
},
{
index: 1,
parentIndex: 0,
type: 'Button',
identifier: 'close-image',
label: 'Close image',
rect: { x: 24, y: 40, width: 120, height: 44 },
hittable: true,
},
];

export function snapshot(nodes: RawSnapshotNode[]) {
return buildSnapshotState(
{
nodes,
backend: 'xctest',
quality: { state: 'healthy', backend: 'tree' },
},
{ snapshotInteractiveOnly: false },
);
}

export function snapshotPayload(
nodes: RawSnapshotNode[],
backend: 'tree' | 'queries' | 'private-ax' = 'tree',
) {
return {
backend: 'xctest' as const,
nodes,
quality: { state: 'healthy' as const, backend },
};
}
Loading
Loading