From d856e8e83173a354a04851035c83d2bf35129628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 4 Aug 2026 20:19:27 +0200 Subject: [PATCH 1/7] fix(ios): corroborate recorded tap outcomes --- CONTEXT.md | 5 + .../RunnerTests+CommandExecution.swift | 2 +- .../RunnerTests+Lifecycle.swift | 5 + .../RunnerTests+LifecycleCacheTests.swift | 3 + .../0005-ios-runner-interaction-lifecycle.md | 12 +- .../interaction-ios-tap-outcome.test.ts | 183 +++++++++++++++ .../handlers/interaction-ios-tap-outcome.ts | 220 ++++++++++++++++++ .../handlers/interaction-touch-response.ts | 54 ++++- src/daemon/handlers/interaction-touch.ts | 132 ++++++++++- .../core/__tests__/runner-client.test.ts | 3 +- 10 files changed, 605 insertions(+), 14 deletions(-) create mode 100644 src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts create mode 100644 src/daemon/handlers/interaction-ios-tap-outcome.ts diff --git a/CONTEXT.md b/CONTEXT.md index df30e1fed0..3f998a7b70 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -431,6 +431,11 @@ the observable freshness and failure semantics below before any runtime refactor - iOS sparse and AX failures are not proof of empty UI. Regular visible snapshots can recover through the capture plan; raw and strict paths preserve failure. `runnerFatal` invalidates the cached target and must never refresh healthy mutation recency. +- An `XCTEST_RECORDED_FAILURE` after an iOS tap is an ambiguous outcome, not proof that the tap missed. + The daemon may take one same-presentation post-action capture against a usable retained snapshot; + only a changed accessibility digest converts the result to success with a warning. Capture failure, + sparse or mismatched presentation, and an unchanged digest remain failures so corroboration cannot + turn an unknown tap into a false success. - Android helper reuse must not become snapshot result caching. Freshness is short lived, marked only after navigation-sensitive actions, compared against broad route-safe baselines, and not learned from scoped, depth-limited, interactive, or ref-refresh snapshots. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 3446c5e10c..a9636b5620 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -2481,7 +2481,7 @@ extension RunnerTests { error: ErrorPayload( code: "XCTEST_RECORDED_FAILURE", message: "XCTest recorded a failure while executing \(command.command.rawValue); the action may not have been performed.", - hint: "The iOS runner session will be restarted. Retry after a fresh snapshot, or use screenshot plus coordinate commands when the accessibility tree is unavailable." + hint: "The iOS runner session was invalidated. Re-observe with a fresh snapshot before retrying; if the accessibility tree is unavailable, use screenshot plus coordinate commands instead of retrying the tap blindly." ) ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift index 389e3300aa..c5e669f532 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift @@ -103,6 +103,11 @@ extension RunnerTests { func resetTargetAfterExternalRelaunch() -> Response { invalidateCachedTarget(reason: "external_app_relaunch") + // The app process is replaced, but the retained runner survives. Clear + // process-bound capture state explicitly because invalidation drops the + // old PID before refreshCachedTargetIfProcessChanged can observe it. + clearSnapshotXCTestChannelPenalty(reason: "external_app_relaunch") + clearPrivateAXAcceptedDepth(reason: "external_app_relaunch") needsFirstInteractionDelay = true return Response(ok: true, data: DataPayload(message: "target reset")) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift index 2c2aee314b..6d3216bce2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift @@ -134,6 +134,8 @@ extension RunnerTests { currentAppProcessIdentifier = 42 snapshotXCTestPenaltyWarmupExemptionPending = true needsFirstInteractionDelay = false + penalizeSnapshotXCTestChannel(bundleId: "com.example.app", reason: "test") + XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app")) let response = resetTargetAfterExternalRelaunch() @@ -142,6 +144,7 @@ extension RunnerTests { XCTAssertNil(currentBundleId) XCTAssertNil(currentAppProcessIdentifier) XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending) + XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app")) XCTAssertTrue(needsFirstInteractionDelay) } #endif diff --git a/docs/adr/0005-ios-runner-interaction-lifecycle.md b/docs/adr/0005-ios-runner-interaction-lifecycle.md index 17875d3f9b..f645e0643d 100644 --- a/docs/adr/0005-ios-runner-interaction-lifecycle.md +++ b/docs/adr/0005-ios-runner-interaction-lifecycle.md @@ -64,7 +64,8 @@ normal activation path. An external iOS simulator relaunch also invalidates process-bound target state. After replacing the app process, the daemon sends a lifecycle reset to the retained runner so the next command reacquires -`XCUIApplication`; if that reset cannot be confirmed, the daemon discards the runner session. +`XCUIApplication`; the reset also clears process-bound snapshot penalty and private-AX depth state. If +that reset cannot be confirmed, the daemon discards the runner session. The snapshot surface intentionally has two AX-failure shapes. Interactive fast snapshots return a truncated success payload with `runnerFatal` so agents can still see that AX state is unavailable @@ -93,8 +94,15 @@ Apps with broken accessibility trees may still be impossible for XCTest to inspe failed snapshot no longer teaches the runner to keep using a suspect cached app target or to amplify the failure by walking every interactive element query. +An `XCTEST_RECORDED_FAILURE` returned after a tap is treated as an ambiguous outcome at the daemon +boundary. When a usable retained snapshot exists, the daemon takes one same-presentation post-action +capture. A changed accessibility digest is reported as success with an explicit warning so agents do +not blindly repeat a tap that may already have navigated; unchanged, sparse, mismatched, or unavailable +evidence remains the original failure. + Simulator relaunch keeps the healthy XCTest process warm without carrying an app target across -process identity. The reset adds one local runner request instead of paying for a runner restart. +process identity. The reset adds one local runner request instead of paying for a runner restart and +clears the old process's hostile-screen capture penalty before the replacement is reacquired. Future optimization work should only reduce these preflights after the runner exposes status in a way that survives command-induced XCTest teardown and can prove the session is still serving new diff --git a/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts b/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts new file mode 100644 index 0000000000..83cd14fc83 --- /dev/null +++ b/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts @@ -0,0 +1,183 @@ +import { beforeEach, expect, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { buildSnapshotState } from '../snapshot-capture.ts'; +import { handleInteractionCommands } from '../interaction.ts'; +import { dispatchCommand } from '../../../core/dispatch.ts'; +import { makeIosSession } from '../../../__tests__/test-utils/session-factories.ts'; +import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; +import type { CommandFlags } from '../../../core/dispatch.ts'; + +vi.mock('../../../core/dispatch.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + dispatchCommand: vi.fn(async () => ({})), + }; +}); + +const mockDispatch = vi.mocked(dispatchCommand); + +const contextFromFlags = (flags: CommandFlags | undefined) => ({ + count: flags?.count, + intervalMs: flags?.intervalMs, + delayMs: flags?.delayMs, + holdMs: flags?.holdMs, + jitterPx: flags?.jitterPx, + doubleTap: flags?.doubleTap, + clickButton: flags?.clickButton, +}); + +const profileNodes = [ + { + index: 0, + type: 'Application', + label: 'Profile', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + parentIndex: 0, + type: 'Button', + identifier: 'unfollow', + label: 'Unfollow', + rect: { x: 24, y: 200, width: 160, height: 44 }, + hittable: true, + }, +]; + +const imageViewerNodes = [ + { + index: 0, + type: 'Application', + label: 'Image viewer', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + parentIndex: 0, + type: 'Button', + identifier: 'close-image', + label: 'Close image', + rect: { x: 24, y: 40, width: 120, height: 44 }, + hittable: true, + }, +]; + +function snapshot(nodes: typeof profileNodes) { + return buildSnapshotState({ nodes, backend: 'xctest' }, { snapshotInteractiveOnly: false }); +} + +async function runClick( + sessionStore: ReturnType, + sessionName: string, +): Promise>> { + return await handleInteractionCommands({ + req: { + token: 'test', + session: sessionName, + command: 'click', + positionals: ['id="unfollow"'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + }); +} + +beforeEach(() => { + mockDispatch.mockReset(); +}); + +test('a changed post-action capture corroborates a direct iOS tap reported as failed', async () => { + const sessionName = 'ios-direct-tap-corroboration'; + const sessionStore = makeSessionStore(); + const session = makeIosSession(sessionName, { + appBundleId: 'com.example.app', + snapshot: snapshot(profileNodes), + }); + sessionStore.set(sessionName, session); + mockDispatch.mockImplementation(async (_device, command) => { + if (command === 'press') { + throw new AppError( + 'XCTEST_RECORDED_FAILURE', + 'XCTest recorded a failure while executing tap; the action may not have been performed.', + ); + } + if (command === 'snapshot') return { backend: 'xctest', nodes: imageViewerNodes }; + return {}; + }); + + const response = await runClick(sessionStore, sessionName); + + expect(response?.ok).toBe(true); + if (response?.ok) { + expect(response.data?.warning).toMatch(/post-action accessibility capture changed/); + expect(response.data?.selector).toBe('id="unfollow"'); + } + expect(mockDispatch.mock.calls.filter((call) => call[1] === 'press')).toHaveLength(1); + expect(sessionStore.get(sessionName)?.actions).toHaveLength(1); +}); + +test('an unchanged post-action capture keeps a failed iOS tap failed', async () => { + const sessionName = 'ios-unchanged-tap-corroboration'; + const sessionStore = makeSessionStore(); + const session = makeIosSession(sessionName, { + appBundleId: 'com.example.app', + snapshot: snapshot(profileNodes), + }); + sessionStore.set(sessionName, session); + mockDispatch.mockImplementation(async (_device, command) => { + if (command === 'press') { + throw new AppError( + 'XCTEST_RECORDED_FAILURE', + 'XCTest recorded a failure while executing tap; the action may not have been performed.', + ); + } + if (command === 'snapshot') return { backend: 'xctest', nodes: profileNodes }; + return {}; + }); + + const response = await runClick(sessionStore, sessionName); + + expect(response?.ok).toBe(false); + if (response && !response.ok) expect(response.error.code).toBe('XCTEST_RECORDED_FAILURE'); + expect(sessionStore.get(sessionName)?.actions).toHaveLength(0); +}); + +test('runtime-resolved taps use the same corroboration boundary', async () => { + const sessionName = 'ios-runtime-tap-corroboration'; + const sessionStore = makeSessionStore(); + sessionStore.set( + sessionName, + makeIosSession(sessionName, { + appBundleId: 'com.example.app', + recordSession: true, + }), + ); + let snapshotCount = 0; + mockDispatch.mockImplementation(async (_device, command) => { + if (command === 'press') { + throw new AppError( + 'XCTEST_RECORDED_FAILURE', + 'XCTest recorded a failure while executing tap; the action may not have been performed.', + ); + } + if (command === 'snapshot') { + snapshotCount += 1; + return { + backend: 'xctest', + nodes: snapshotCount === 1 ? profileNodes : imageViewerNodes, + }; + } + return {}; + }); + + const response = await runClick(sessionStore, sessionName); + + expect(response?.ok).toBe(true); + if (response?.ok) + expect(response.data?.warning).toMatch(/post-action accessibility capture changed/); + expect(snapshotCount).toBe(2); + expect(sessionStore.get(sessionName)?.actions).toHaveLength(1); +}); diff --git a/src/daemon/handlers/interaction-ios-tap-outcome.ts b/src/daemon/handlers/interaction-ios-tap-outcome.ts new file mode 100644 index 0000000000..dc11eeb373 --- /dev/null +++ b/src/daemon/handlers/interaction-ios-tap-outcome.ts @@ -0,0 +1,220 @@ +import type { InteractionTarget } from '@agent-device/contracts/interaction'; +import type { SnapshotState } from '@agent-device/kernel/snapshot'; +import { asAppError } from '@agent-device/kernel/errors'; +import type { CommandFlags } from '../../core/dispatch.ts'; +import { isSparseSnapshotQualityVerdict } from '../../snapshot/snapshot-quality.ts'; +import { summarizeAxEvidence } from '../../utils/ax-digest.ts'; +import { getRequestSignal } from '../../request/cancel.ts'; +import { isLocalIosRunnerSession } from '../direct-ios-selector.ts'; +import { emitDiagnostic } from '../../utils/diagnostics.ts'; +import type { SessionStore } from '../session-store.ts'; +import type { SessionState } from '../types.ts'; +import type { ContextFromFlags } from './interaction-common.ts'; +import type { CaptureSnapshotForSession } from './interaction-snapshot.ts'; + +const XCTEST_RECORDED_FAILURE = 'XCTEST_RECORDED_FAILURE'; + +const IOS_TAP_CORROBORATION_WARNING = + 'XCTest reported the tap as failed, but a same-scope post-action accessibility capture changed; treating the tap as landed. Observe the current screen before issuing another tap.'; + +type SnapshotPresentation = { + interactiveOnly: boolean; + depth?: number; + scope?: string; + raw: boolean; +}; + +export type IosTapCorroborationParams = { + error: unknown; + command: string; + requestId: string | undefined; + flags: CommandFlags | undefined; + session: SessionState; + sessionStore: SessionStore; + contextFromFlags: ContextFromFlags; + captureSnapshotForSession: CaptureSnapshotForSession; +}; + +export type IosTapCorroboration = { + warning: string; +}; + +/** + * A recorded XCTest failure is ambiguous for a tap: the runner can report the + * XCTest bookkeeping failure after the coordinate activation already reached + * the app. This helper spends exactly one matching capture to distinguish a + * changed surface from an unchanged/unknown outcome. Unknown stays failure. + */ +export async function corroborateIosTapFailure( + params: IosTapCorroborationParams, +): Promise { + if (!canCorroborateIosTapFailure(params)) return undefined; + const baseline = readCorroborationBaseline(params.session.snapshot); + if (!baseline) return undefined; + + const after = await captureCorroborationSnapshot(params, baseline.presentation); + if (!after || !hasMatchingPresentation(baseline.snapshot, after, params.command)) { + return undefined; + } + return compareCorroborationEvidence(baseline.snapshot, after, params.command); +} + +function canCorroborateIosTapFailure(params: IosTapCorroborationParams): boolean { + return ( + isTapCommand(params.command) && + asAppError(params.error).code === XCTEST_RECORDED_FAILURE && + isLocalIosRunnerSession(params.session, { skipPendingPostGestureStabilization: false }) + ); +} + +function isTapCommand(command: string): boolean { + return command === 'click' || command === 'press'; +} + +function readCorroborationBaseline( + snapshot: SnapshotState | undefined, +): { snapshot: SnapshotState; presentation: SnapshotPresentation | undefined } | undefined { + if (!hasUsableBaseline(snapshot)) return undefined; + return { snapshot, presentation: readSnapshotPresentation(snapshot.presentationKey) }; +} + +async function captureCorroborationSnapshot( + params: IosTapCorroborationParams, + presentation: SnapshotPresentation | undefined, +): Promise { + try { + return await params.captureSnapshotForSession( + params.session, + matchingCaptureFlags(params.flags, presentation), + params.sessionStore, + params.contextFromFlags, + { + interactiveOnly: presentation?.interactiveOnly ?? true, + signal: getRequestSignal(params.requestId), + }, + ); + } catch (captureError) { + emitDiagnostic({ + level: 'debug', + phase: 'ios_tap_failure_corroboration_capture_failed', + data: { + command: params.command, + error: captureError instanceof Error ? captureError.message : String(captureError), + }, + }); + return undefined; + } +} + +function hasMatchingPresentation( + baseline: SnapshotState, + after: SnapshotState, + command: string, +): boolean { + if (!baseline.presentationKey || baseline.presentationKey === after.presentationKey) return true; + emitDiagnostic({ + level: 'debug', + phase: 'ios_tap_failure_corroboration_scope_mismatch', + data: { + command, + baselinePresentationKey: baseline.presentationKey, + afterPresentationKey: after.presentationKey, + }, + }); + return false; +} + +function compareCorroborationEvidence( + baseline: SnapshotState, + after: SnapshotState, + command: string, +): IosTapCorroboration | undefined { + if (!hasUsableBaseline(after)) return undefined; + const beforeEvidence = summarizeAxEvidence(baseline.nodes); + const afterEvidence = summarizeAxEvidence(after.nodes); + if (beforeEvidence.digest === afterEvidence.digest) return undefined; + + emitDiagnostic({ + level: 'warn', + phase: 'ios_tap_failure_corroborated', + data: { + command, + beforeNodeCount: beforeEvidence.nodeCount, + afterNodeCount: afterEvidence.nodeCount, + }, + }); + return { warning: IOS_TAP_CORROBORATION_WARNING }; +} + +function hasUsableBaseline(snapshot: SnapshotState | undefined): snapshot is SnapshotState { + return Boolean( + snapshot && + snapshot.nodes.length > 0 && + !isSparseSnapshotQualityVerdict(snapshot.snapshotQuality), + ); +} + +function readSnapshotPresentation(value: string | undefined): SnapshotPresentation | undefined { + if (!value) return undefined; + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + return undefined; + } + return parseSnapshotPresentation(parsed); +} + +function parseSnapshotPresentation(value: unknown): SnapshotPresentation | undefined { + const raw = asRecord(value); + if (!raw || typeof raw.interactiveOnly !== 'boolean' || typeof raw.raw !== 'boolean') { + return undefined; + } + if (!isOptionalNumber(raw.depth) || !isOptionalString(raw.scope)) return undefined; + return { + interactiveOnly: raw.interactiveOnly, + ...(typeof raw.depth === 'number' ? { depth: raw.depth } : {}), + ...(typeof raw.scope === 'string' ? { scope: raw.scope } : {}), + raw: raw.raw, + }; +} + +function asRecord(value: unknown): Record | undefined { + return typeof value === 'object' && value !== null + ? (value as Record) + : undefined; +} + +function isOptionalNumber(value: unknown): value is number | null | undefined { + return value === null || value === undefined || typeof value === 'number'; +} + +function isOptionalString(value: unknown): value is string | null | undefined { + return value === null || value === undefined || typeof value === 'string'; +} + +function matchingCaptureFlags( + flags: CommandFlags | undefined, + presentation: SnapshotPresentation | undefined, +): CommandFlags | undefined { + if (!flags) return undefined; + return { + ...flags, + out: undefined, + ...(presentation + ? { + snapshotDepth: presentation.depth, + snapshotScope: presentation.scope, + snapshotRaw: presentation.raw, + } + : {}), + }; +} + +export function interactionTargetExtra(target: InteractionTarget): Record { + if (target.kind === 'selector') return { selector: target.selector }; + if (target.kind === 'ref') { + return { ref: target.ref.startsWith('@') ? target.ref.slice(1) : target.ref }; + } + return {}; +} diff --git a/src/daemon/handlers/interaction-touch-response.ts b/src/daemon/handlers/interaction-touch-response.ts index c3e8569180..79e18d6608 100644 --- a/src/daemon/handlers/interaction-touch-response.ts +++ b/src/daemon/handlers/interaction-touch-response.ts @@ -47,6 +47,17 @@ export type InteractionResponseSource = targetKind: InteractionRuntimeResult['kind']; data: Record; point: { x: number; y: number }; + }) + | (InteractionResponseSourceBase & { + // An XCTest mutation failure was corroborated by a changed same-scope + // post-action capture. The target was resolved before the runner error, + // but no runtime result survived the thrown backend failure. + kind: 'corroborated-tap'; + targetKind: InteractionRuntimeResult['kind']; + data: Record; + publicData?: Record; + point?: { x: number; y: number }; + resolution?: ResolutionDisclosure; }); // ADR 0012 decision 2: the XCTest fast path has no daemon tree, so it can only @@ -92,6 +103,28 @@ export type InteractionResponsePayloads = { recordedTarget?: RecordedTargetCapture; }; +export function buildCorroboratedTapResponseData(params: { + targetKind: InteractionRuntimeResult['kind']; + point?: { x: number; y: number }; + warning: string; + resolution?: ResolutionDisclosure; + referenceFrame?: GestureReferenceFrame; + extra?: Record; +}): InteractionResponsePayloads { + return buildInteractionResponseData({ + source: { + kind: 'corroborated-tap', + targetKind: params.targetKind, + data: { warning: params.warning }, + publicData: { warning: params.warning }, + point: params.point, + resolution: params.resolution, + }, + referenceFrame: params.referenceFrame, + extra: params.extra, + }); +} + export function buildInteractionResponseData(params: { source: InteractionResponseSource; referenceFrame: GestureReferenceFrame | undefined; @@ -120,25 +153,30 @@ export function buildInteractionResponseData(params: { settleRefsGeneration?: number; }): InteractionResponsePayloads { const { source, referenceFrame, extra } = params; - if (source.kind === 'runner-payload') { + if (source.kind === 'runner-payload' || source.kind === 'corroborated-tap') { const commonExtra = { targetKind: source.targetKind, - ...applyResolutionDisclosurePolicy(source, { - resolution: DIRECT_IOS_NOT_OBSERVED_RESOLUTION, - }), + ...applyResolutionDisclosurePolicy( + source, + source.kind === 'runner-payload' + ? { resolution: DIRECT_IOS_NOT_OBSERVED_RESOLUTION } + : source.resolution + ? { resolution: source.resolution } + : {}, + ), ...(extra ?? {}), }; const result = buildTouchPayload({ data: source.data, - fallbackX: source.point.x, - fallbackY: source.point.y, + fallbackX: source.point?.x, + fallbackY: source.point?.y, referenceFrame, extra: commonExtra, }); const responseData = buildTouchPayload({ data: source.publicData, - fallbackX: source.point.x, - fallbackY: source.point.y, + fallbackX: source.point?.x, + fallbackY: source.point?.y, referenceFrame, extra: commonExtra, }); diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index 7433b1e0bb..66069b9715 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -58,6 +58,7 @@ import { } from './interaction-touch-reference-frame.ts'; import { buildInteractionResponseData, + buildCorroboratedTapResponseData, type InteractionResponsePayloads, } from './interaction-touch-response.ts'; import { @@ -66,6 +67,7 @@ import { parseLongPressTarget, parseTouchTarget, } from './interaction-touch-targets.ts'; +import { corroborateIosTapFailure, interactionTargetExtra } from './interaction-ios-tap-outcome.ts'; import { errorResponse, noActiveSessionError, requireCommandSupported } from './response.ts'; export async function handleTouchInteractionCommands( @@ -196,6 +198,16 @@ async function dispatchTargetedTouchViaRuntime( staleRefsWarning, } : undefined, + iosTapCorroboration: { + target: parsedTarget.target, + extra: + command === 'longpress' + ? { + ...(durationMs !== undefined ? { durationMs } : {}), + gesture: 'longpress', + } + : resultButtonTag, + }, run: async (runtime) => await runTargetedTouchInteraction({ runtime, @@ -394,7 +406,7 @@ function hasNonDefaultClickOptions(flags: CommandFlags | undefined): boolean { } async function dispatchDirectIosSelectorTap( - params: InteractionHandlerParams, + params: InteractionHandlerParams & { captureSnapshotForSession: CaptureSnapshotForSession }, session: SessionState, selector: DirectIosSelectorTarget, ): Promise { @@ -410,7 +422,7 @@ async function dispatchDirectIosSelectorTap( } async function dispatchDirectIosSelectorInteraction(params: { - params: InteractionHandlerParams; + params: InteractionHandlerParams & { captureSnapshotForSession: CaptureSnapshotForSession }; session: SessionState; selector: DirectIosSelectorTarget; command: 'press' | 'fill'; @@ -484,6 +496,15 @@ async function dispatchDirectIosSelectorInteraction(params: { actionFinishedAt, }); } catch (error) { + const corroboratedResponse = await buildDirectIosCorroboratedResponse({ + error, + handlerParams, + session, + extra, + positionals, + actionStartedAt, + }); + if (corroboratedResponse) return corroboratedResponse; // ADR 0011 delegation-on-error: semantic runner failures fall back to the // tree-based runtime path — except for Maestro replay dispatches, whose // runner-native error shapes must be preserved. @@ -505,6 +526,48 @@ async function dispatchDirectIosSelectorInteraction(params: { } } +async function buildDirectIosCorroboratedResponse(params: { + error: unknown; + handlerParams: InteractionHandlerParams & { + captureSnapshotForSession: CaptureSnapshotForSession; + }; + session: SessionState; + extra: Record; + positionals: string[]; + actionStartedAt: number; +}): Promise { + const { error, handlerParams, session, extra, positionals, actionStartedAt } = params; + const corroboration = await corroborateIosTapFailure({ + error, + command: handlerParams.req.command, + requestId: handlerParams.req.meta?.requestId, + flags: handlerParams.req.flags, + session, + sessionStore: handlerParams.sessionStore, + contextFromFlags: handlerParams.contextFromFlags, + captureSnapshotForSession: handlerParams.captureSnapshotForSession, + }); + if (!corroboration) return undefined; + + const { result, responseData } = buildCorroboratedTapResponseData({ + targetKind: 'selector', + warning: corroboration.warning, + resolution: { source: 'direct-ios', kind: 'not-observed' }, + extra, + }); + return finalizeTouchInteraction({ + session, + sessionStore: handlerParams.sessionStore, + command: handlerParams.req.command, + positionals: handlerParams.req.positionals ?? positionals, + flags: handlerParams.req.flags, + result, + responseData, + actionStartedAt, + actionFinishedAt: Date.now(), + }); +} + function transformTouchResponseData(params: { session: SessionState; command?: InteractionResponseDataTransformCommand; @@ -738,6 +801,11 @@ async function dispatchRuntimeInteraction< * admission rejection built from this context. */ refContext?: RefAdmissionContext; + /** A failed local iOS tap may be corroborated against one same-scope capture. */ + iosTapCorroboration?: { + target: InteractionTarget; + extra: Record; + }; run(runtime: ReturnType): Promise; /** May return a warning to append to the successful response (e.g. a pending Android permission dialog). */ afterRun?(result: TResult): Promise; @@ -797,10 +865,70 @@ async function dispatchRuntimeInteraction< } catch (error) { const appError = asAppError(error); if (isAndroidEscapeError(appError)) throw appError; + const corroboratedResponse = await buildRuntimeIosCorroboratedResponse({ + error, + handlerParams: params, + session, + target: options.iosTapCorroboration?.target, + extra: options.iosTapCorroboration?.extra, + actionStartedAt, + androidFreshnessBaseline: options.androidFreshnessBaseline, + }); + if (corroboratedResponse) return corroboratedResponse; return appErrorResponse(error); } } +async function buildRuntimeIosCorroboratedResponse(params: { + error: unknown; + handlerParams: InteractionHandlerParams & { + captureSnapshotForSession: CaptureSnapshotForSession; + }; + session: SessionState; + target: InteractionTarget | undefined; + extra: Record | undefined; + actionStartedAt: number; + androidFreshnessBaseline: SessionState['snapshot'] | undefined; +}): Promise { + if (!params.target) return undefined; + const corroboration = await corroborateIosTapFailure({ + error: params.error, + command: params.handlerParams.req.command, + requestId: params.handlerParams.req.meta?.requestId, + flags: params.handlerParams.req.flags, + session: params.session, + sessionStore: params.handlerParams.sessionStore, + contextFromFlags: params.handlerParams.contextFromFlags, + captureSnapshotForSession: params.handlerParams.captureSnapshotForSession, + }); + if (!corroboration) return undefined; + + const target = params.target; + const point = target.kind === 'point' ? { x: target.x, y: target.y } : undefined; + const { result, responseData } = buildCorroboratedTapResponseData({ + targetKind: target.kind, + point, + warning: corroboration.warning, + referenceFrame: readSnapshotNodesReferenceFrame(params.session.snapshot?.nodes ?? []), + extra: { + ...interactionTargetExtra(target), + ...(params.extra ?? {}), + }, + }); + return finalizeTouchInteraction({ + session: params.session, + sessionStore: params.handlerParams.sessionStore, + command: params.handlerParams.req.command, + positionals: params.handlerParams.req.positionals ?? [], + flags: params.handlerParams.req.flags, + result, + responseData, + actionStartedAt: params.actionStartedAt, + actionFinishedAt: Date.now(), + androidFreshnessBaseline: params.androidFreshnessBaseline, + }); +} + type RefAdmissionContext = { ref: string; mintedGeneration: number | undefined; diff --git a/src/platforms/apple/core/__tests__/runner-client.test.ts b/src/platforms/apple/core/__tests__/runner-client.test.ts index 22743bdce0..8143119e89 100644 --- a/src/platforms/apple/core/__tests__/runner-client.test.ts +++ b/src/platforms/apple/core/__tests__/runner-client.test.ts @@ -727,7 +727,8 @@ test('parseRunnerResponse preserves iOS AX snapshot failure code and hint', asyn }); test('parseRunnerResponse preserves XCTest recorded failure code and hint', async () => { - const hint = 'The iOS runner session will be restarted.'; + const hint = + 'The iOS runner session was invalidated. Re-observe with a fresh snapshot before retrying; if the accessibility tree is unavailable, use screenshot plus coordinate commands instead of retrying the tap blindly.'; const response = new Response( JSON.stringify({ ok: false, From b10b895a8098a5cedb5238c04d9e28cd83a5e3a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 4 Aug 2026 21:36:15 +0200 Subject: [PATCH 2/7] fix(ios): preserve corroborated tap target identity --- .../interaction/runtime/interactions.ts | 26 +++-- src/contracts/interaction-outcome.ts | 25 +++++ .../interaction-ios-tap-outcome.test.ts | 100 ++++++++++++++++++ src/daemon/handlers/interaction-touch.ts | 57 ++++++++-- 4 files changed, 189 insertions(+), 19 deletions(-) create mode 100644 src/contracts/interaction-outcome.ts diff --git a/src/commands/interaction/runtime/interactions.ts b/src/commands/interaction/runtime/interactions.ts index bba43a117c..e988b74d21 100644 --- a/src/commands/interaction/runtime/interactions.ts +++ b/src/commands/interaction/runtime/interactions.ts @@ -11,6 +11,7 @@ import { isFillableType } from '@agent-device/contracts/snapshot'; import { successText } from '../../../utils/success-text.ts'; import { findMistargetedTypeRefToken } from '../../../utils/type-target-warning.ts'; import { requireIntInRange } from '../../../utils/validation.ts'; +import { attachResolvedInteractionTarget } from '../../../contracts/interaction-outcome.ts'; import type { RepeatedInput } from '../../command-input.ts'; import { toBackendContext } from '../../runtime-common.ts'; import { @@ -196,14 +197,23 @@ async function tapCommand( throw new AppError('UNSUPPORTED_OPERATION', 'tap is not supported by this backend'); } const point = requireResolvedPoint(resolved); - const backendResult = await runtime.backend.tap(toBackendContext(runtime, options), point, { - button: options.button, - count: options.count, - intervalMs: options.intervalMs, - holdMs: options.holdMs, - jitterPx: options.jitterPx, - doubleTap: options.doubleTap, - }); + let backendResult; + try { + backendResult = await runtime.backend.tap(toBackendContext(runtime, options), point, { + button: options.button, + count: options.count, + intervalMs: options.intervalMs, + holdMs: options.holdMs, + jitterPx: options.jitterPx, + doubleTap: options.doubleTap, + }); + } catch (error) { + // Resolution is complete before the backend call. Preserve it out of + // band so a daemon-level failure corroboration can still record the same + // target identity if the backend reports an ambiguous tap outcome. + attachResolvedInteractionTarget(error, resolved); + throw error; + } const formattedBackendResult = toBackendResult(backendResult); return await applyPostActionObservation( runtime, diff --git a/src/contracts/interaction-outcome.ts b/src/contracts/interaction-outcome.ts new file mode 100644 index 0000000000..1d9845523e --- /dev/null +++ b/src/contracts/interaction-outcome.ts @@ -0,0 +1,25 @@ +import type { ResolvedInteractionTarget } from '@agent-device/contracts/interaction'; + +// The runtime can resolve an interaction before the backend reports a failure. +// Keep that resolution out of serialized error details while making it +// available to the daemon's failure-corroboration boundary. +const resolvedInteractionTargets = new WeakMap(); + +/** Preserve pre-dispatch target identity across a backend rejection. */ +export function attachResolvedInteractionTarget( + error: unknown, + target: ResolvedInteractionTarget, +): void { + if (isObjectLike(error)) resolvedInteractionTargets.set(error, target); +} + +/** Read the target captured before a backend interaction rejection. */ +export function readResolvedInteractionTarget( + error: unknown, +): ResolvedInteractionTarget | undefined { + return isObjectLike(error) ? resolvedInteractionTargets.get(error) : undefined; +} + +function isObjectLike(value: unknown): value is object { + return (typeof value === 'object' && value !== null) || typeof value === 'function'; +} diff --git a/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts b/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts index 83cd14fc83..6d45332c86 100644 --- a/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts +++ b/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts @@ -1,4 +1,7 @@ import { beforeEach, expect, test, vi } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; import { buildSnapshotState } from '../snapshot-capture.ts'; import { handleInteractionCommands } from '../interaction.ts'; @@ -6,6 +9,8 @@ import { dispatchCommand } from '../../../core/dispatch.ts'; import { makeIosSession } from '../../../__tests__/test-utils/session-factories.ts'; import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; import type { CommandFlags } from '../../../core/dispatch.ts'; +import { SessionScriptWriter } from '../../session-script-writer.ts'; +import { runReplayScriptFile } from '../session-replay-runtime.ts'; vi.mock('../../../core/dispatch.ts', async (importOriginal) => { const actual = await importOriginal(); @@ -181,3 +186,98 @@ test('runtime-resolved taps use the same corroboration boundary', async () => { expect(snapshotCount).toBe(2); expect(sessionStore.get(sessionName)?.actions).toHaveLength(1); }); + +test('corroborated runtime taps retain target evidence through save and replay', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-ios-tap-replay-')); + const sessionName = 'ios-recorded-tap'; + const sessionStore = makeSessionStore(); + sessionStore.set( + sessionName, + makeIosSession(sessionName, { + appBundleId: 'com.example.app', + recordSession: true, + snapshot: snapshot(profileNodes), + }), + ); + let recording = true; + let snapshotCount = 0; + let pressCount = 0; + mockDispatch.mockImplementation(async (_device, command) => { + if (command === 'press') { + pressCount += 1; + if (recording) { + throw new AppError( + 'XCTEST_RECORDED_FAILURE', + 'XCTest recorded a failure while executing tap; the action may not have been performed.', + ); + } + return {}; + } + if (command === 'snapshot') { + snapshotCount += 1; + return { + backend: 'xctest', + nodes: recording && snapshotCount === 2 ? imageViewerNodes : profileNodes, + }; + } + return {}; + }); + + const recordedResponse = await runClick(sessionStore, sessionName); + expect(recordedResponse?.ok).toBe(true); + const recordedAction = sessionStore.get(sessionName)?.actions[0]; + expect(recordedAction?.result?.selectorChain).toEqual([ + 'id="unfollow"', + 'role="button" label="Unfollow"', + 'label="Unfollow"', + ]); + expect(recordedAction?.targetEvidence).toBeDefined(); + + const written = new SessionScriptWriter(path.join(root, 'sessions')).write( + sessionStore.get(sessionName)!, + ); + expect(written.written).toBe(true); + if (!written.written) return; + const savedScript = fs.readFileSync(written.path, 'utf8'); + expect(savedScript).toContain('# agent-device:target-v1'); + expect(savedScript).toContain( + 'click "id=\\"unfollow\\" || role=\\"button\\" label=\\"Unfollow\\" || label=\\"Unfollow\\""', + ); + + recording = false; + const replaySessionName = 'ios-replayed-tap'; + const replayStore = makeSessionStore(); + replayStore.set( + replaySessionName, + makeIosSession(replaySessionName, { + appBundleId: 'com.example.app', + snapshot: snapshot(profileNodes), + }), + ); + const replayResponse = await runReplayScriptFile({ + req: { + token: 'test', + session: replaySessionName, + command: 'replay', + positionals: [written.path], + flags: { replayKeepSession: true }, + }, + sessionName: replaySessionName, + logPath: path.join(root, 'replay-daemon.log'), + sessionStore: replayStore, + invoke: async (req) => { + const response = await handleInteractionCommands({ + req, + sessionName: replaySessionName, + sessionStore: replayStore, + contextFromFlags, + }); + if (!response) throw new Error(`unexpected empty response for ${req.command}`); + return response; + }, + }); + + expect(replayResponse.ok).toBe(true); + expect(pressCount).toBe(2); + expect(snapshotCount).toBeGreaterThanOrEqual(4); +}); diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index 66069b9715..59e6aecc97 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -4,6 +4,7 @@ import type { InteractionTarget, LongPressCommandResult, PressCommandResult, + ResolvedInteractionTarget, } from '@agent-device/contracts/interaction'; import { buttonTag, @@ -25,6 +26,7 @@ import { normalizeAppleRunnerResultForResponse } from '../../platforms/apple/cor import type { ReplayTargetGuardDenotation } from '@agent-device/contracts/replay'; import { emitDiagnostic } from '../../utils/diagnostics.ts'; import { getActiveAndroidSnapshotFreshness } from '../android-snapshot-freshness.ts'; +import { readResolvedInteractionTarget } from '../../contracts/interaction-outcome.ts'; import { ensureAndroidBlockingSystemDialogReady, type AndroidBlockingDialogReadinessResult, @@ -891,6 +893,8 @@ async function buildRuntimeIosCorroboratedResponse(params: { androidFreshnessBaseline: SessionState['snapshot'] | undefined; }): Promise { if (!params.target) return undefined; + const resolvedTarget = readResolvedInteractionTarget(params.error); + if (!resolvedTarget && params.session.recordSession) return undefined; const corroboration = await corroborateIosTapFailure({ error: params.error, command: params.handlerParams.req.command, @@ -903,17 +907,12 @@ async function buildRuntimeIosCorroboratedResponse(params: { }); if (!corroboration) return undefined; - const target = params.target; - const point = target.kind === 'point' ? { x: target.x, y: target.y } : undefined; - const { result, responseData } = buildCorroboratedTapResponseData({ - targetKind: target.kind, - point, + const payloads = buildIosCorroboratedPayloads({ + target: params.target, + resolvedTarget, warning: corroboration.warning, referenceFrame: readSnapshotNodesReferenceFrame(params.session.snapshot?.nodes ?? []), - extra: { - ...interactionTargetExtra(target), - ...(params.extra ?? {}), - }, + extra: params.extra, }); return finalizeTouchInteraction({ session: params.session, @@ -921,14 +920,50 @@ async function buildRuntimeIosCorroboratedResponse(params: { command: params.handlerParams.req.command, positionals: params.handlerParams.req.positionals ?? [], flags: params.handlerParams.req.flags, - result, - responseData, + result: payloads.result, + responseData: payloads.responseData, + recordedTarget: payloads.recordedTarget, actionStartedAt: params.actionStartedAt, actionFinishedAt: Date.now(), androidFreshnessBaseline: params.androidFreshnessBaseline, }); } +function buildIosCorroboratedPayloads(params: { + target: InteractionTarget; + resolvedTarget: ResolvedInteractionTarget | undefined; + warning: string; + referenceFrame: GestureReferenceFrame | undefined; + extra: Record | undefined; +}): InteractionResponsePayloads { + if (params.resolvedTarget) { + return buildInteractionResponseData({ + source: { + kind: 'runtime', + result: { ...params.resolvedTarget, warning: params.warning }, + }, + referenceFrame: params.referenceFrame, + extra: params.extra, + }); + } + return buildCorroboratedTapResponseData({ + targetKind: params.target.kind, + point: pointFromInteractionTarget(params.target), + warning: params.warning, + referenceFrame: params.referenceFrame, + extra: { + ...interactionTargetExtra(params.target), + ...(params.extra ?? {}), + }, + }); +} + +function pointFromInteractionTarget( + target: InteractionTarget, +): { x: number; y: number } | undefined { + return target.kind === 'point' ? { x: target.x, y: target.y } : undefined; +} + type RefAdmissionContext = { ref: string; mintedGeneration: number | undefined; From c09a2f00da116871929c9046bb6e836bed9b1deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 5 Aug 2026 08:21:43 +0200 Subject: [PATCH 3/7] fix(ios): suppress corroborated tap retries --- .../interaction-ios-tap-outcome-fixtures.ts | 42 +++++++ .../interaction-ios-tap-outcome.test.ts | 109 ++++++++++-------- src/daemon/handlers/interaction-common.ts | 19 +-- src/daemon/handlers/interaction-touch.ts | 2 + 4 files changed, 119 insertions(+), 53 deletions(-) create mode 100644 src/daemon/handlers/__tests__/interaction-ios-tap-outcome-fixtures.ts diff --git a/src/daemon/handlers/__tests__/interaction-ios-tap-outcome-fixtures.ts b/src/daemon/handlers/__tests__/interaction-ios-tap-outcome-fixtures.ts new file mode 100644 index 0000000000..ef275a3036 --- /dev/null +++ b/src/daemon/handlers/__tests__/interaction-ios-tap-outcome-fixtures.ts @@ -0,0 +1,42 @@ +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { buildSnapshotState } from '../snapshot-capture.ts'; + +export const profileNodes: RawSnapshotNode[] = [ + { + index: 0, + type: 'Application', + label: 'Profile', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + parentIndex: 0, + type: 'Button', + identifier: 'unfollow', + label: 'Unfollow', + rect: { x: 24, y: 200, width: 160, height: 44 }, + hittable: true, + }, +]; + +export const imageViewerNodes: RawSnapshotNode[] = [ + { + index: 0, + type: 'Application', + label: 'Image viewer', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + parentIndex: 0, + type: 'Button', + identifier: 'close-image', + label: 'Close image', + rect: { x: 24, y: 40, width: 120, height: 44 }, + hittable: true, + }, +]; + +export function snapshot(nodes: RawSnapshotNode[]) { + return buildSnapshotState({ nodes, backend: 'xctest' }, { snapshotInteractiveOnly: false }); +} diff --git a/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts b/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts index 6d45332c86..7741e5693a 100644 --- a/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts +++ b/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts @@ -3,14 +3,19 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; -import { buildSnapshotState } from '../snapshot-capture.ts'; import { handleInteractionCommands } from '../interaction.ts'; +import { handleSnapshotCommands } from '../snapshot.ts'; import { dispatchCommand } from '../../../core/dispatch.ts'; import { makeIosSession } from '../../../__tests__/test-utils/session-factories.ts'; import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; import type { CommandFlags } from '../../../core/dispatch.ts'; import { SessionScriptWriter } from '../../session-script-writer.ts'; import { runReplayScriptFile } from '../session-replay-runtime.ts'; +import { + imageViewerNodes, + profileNodes, + snapshot, +} from './interaction-ios-tap-outcome-fixtures.ts'; vi.mock('../../../core/dispatch.ts', async (importOriginal) => { const actual = await importOriginal(); @@ -32,57 +37,18 @@ const contextFromFlags = (flags: CommandFlags | undefined) => ({ clickButton: flags?.clickButton, }); -const profileNodes = [ - { - index: 0, - type: 'Application', - label: 'Profile', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - parentIndex: 0, - type: 'Button', - identifier: 'unfollow', - label: 'Unfollow', - rect: { x: 24, y: 200, width: 160, height: 44 }, - hittable: true, - }, -]; - -const imageViewerNodes = [ - { - index: 0, - type: 'Application', - label: 'Image viewer', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - parentIndex: 0, - type: 'Button', - identifier: 'close-image', - label: 'Close image', - rect: { x: 24, y: 40, width: 120, height: 44 }, - hittable: true, - }, -]; - -function snapshot(nodes: typeof profileNodes) { - return buildSnapshotState({ nodes, backend: 'xctest' }, { snapshotInteractiveOnly: false }); -} - async function runClick( sessionStore: ReturnType, sessionName: string, + options: { positionals?: string[]; flags?: CommandFlags } = {}, ): Promise>> { return await handleInteractionCommands({ req: { token: 'test', session: sessionName, command: 'click', - positionals: ['id="unfollow"'], - flags: {}, + positionals: options.positionals ?? ['id="unfollow"'], + flags: options.flags ?? {}, }, sessionName, sessionStore, @@ -90,9 +56,7 @@ async function runClick( }); } -beforeEach(() => { - mockDispatch.mockReset(); -}); +beforeEach(() => mockDispatch.mockReset()); test('a changed post-action capture corroborates a direct iOS tap reported as failed', async () => { const sessionName = 'ios-direct-tap-corroboration'; @@ -187,6 +151,59 @@ test('runtime-resolved taps use the same corroboration boundary', async () => { expect(sessionStore.get(sessionName)?.actions).toHaveLength(1); }); +test('a corroborated runtime coordinate tap does not schedule a no-change retry', async () => { + const sessionName = 'ios-runtime-coordinate-corroboration'; + const sessionStore = makeSessionStore(); + sessionStore.set( + sessionName, + makeIosSession(sessionName, { + appBundleId: 'com.example.app', + snapshot: snapshot(profileNodes), + }), + ); + let pressCount = 0; + mockDispatch.mockImplementation(async (_device, command) => { + if (command === 'press') { + pressCount += 1; + if (pressCount === 1) { + throw new AppError( + 'XCTEST_RECORDED_FAILURE', + 'XCTest recorded a failure while executing tap; the action may not have been performed.', + ); + } + return {}; + } + if (command === 'snapshot') return { backend: 'xctest', nodes: imageViewerNodes }; + return {}; + }); + + const clickResponse = await runClick(sessionStore, sessionName, { + positionals: ['104', '222'], + flags: { interactionOutcome: { retryOnNoChange: true } }, + }); + expect(clickResponse?.ok).toBe(true); + if (clickResponse?.ok) { + expect(clickResponse.data?.warning).toMatch(/post-action accessibility capture changed/); + } + + const snapshotResponse = await handleSnapshotCommands({ + req: { + token: 'test', + session: sessionName, + command: 'snapshot', + positionals: [], + flags: {}, + }, + sessionName, + logPath: '/tmp/daemon.log', + sessionStore, + }); + + expect(snapshotResponse?.ok).toBe(true); + expect(pressCount).toBe(1); + expect(sessionStore.get(sessionName)?.pendingInteractionOutcome).toBeUndefined(); +}); + test('corroborated runtime taps retain target evidence through save and replay', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-ios-tap-replay-')); const sessionName = 'ios-recorded-tap'; diff --git a/src/daemon/handlers/interaction-common.ts b/src/daemon/handlers/interaction-common.ts index dbbbfd9b62..f1b95e368f 100644 --- a/src/daemon/handlers/interaction-common.ts +++ b/src/daemon/handlers/interaction-common.ts @@ -46,6 +46,8 @@ export function finalizeTouchInteraction(params: { /** ADR 0012 decision 3: record-time input for the `target-v1` annotation. */ recordedTarget?: RecordedTargetCapture; recordedTargets?: { source: RecordedTargetCapture; destination: RecordedTargetCapture }; + /** False when a post-action observation already proved the interaction landed. */ + scheduleInteractionOutcomeRetry?: boolean; actionStartedAt: number; actionFinishedAt: number; androidFreshnessBaseline?: SnapshotState | undefined; @@ -62,6 +64,7 @@ export function finalizeTouchInteraction(params: { responseData, recordedTarget, recordedTargets, + scheduleInteractionOutcomeRetry = true, actionStartedAt, actionFinishedAt, androidFreshnessBaseline, @@ -88,13 +91,15 @@ export function finalizeTouchInteraction(params: { ...(targetEvidence ? { targetEvidence } : {}), ...(targetEvidences ? { targetEvidences } : {}), }); - markPendingInteractionOutcome({ - session, - command, - positionals: retryPositionals ?? positionals, - flags, - preSnapshot: session.snapshot, - }); + if (scheduleInteractionOutcomeRetry) { + markPendingInteractionOutcome({ + session, + command, + positionals: retryPositionals ?? positionals, + flags, + preSnapshot: session.snapshot, + }); + } if (isNavigationSensitiveAction(actionCommand)) { markAndroidSnapshotFreshness( session, diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index 59e6aecc97..eda1cdfa1c 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -565,6 +565,7 @@ async function buildDirectIosCorroboratedResponse(params: { flags: handlerParams.req.flags, result, responseData, + scheduleInteractionOutcomeRetry: false, actionStartedAt, actionFinishedAt: Date.now(), }); @@ -923,6 +924,7 @@ async function buildRuntimeIosCorroboratedResponse(params: { result: payloads.result, responseData: payloads.responseData, recordedTarget: payloads.recordedTarget, + scheduleInteractionOutcomeRetry: false, actionStartedAt: params.actionStartedAt, actionFinishedAt: Date.now(), androidFreshnessBaseline: params.androidFreshnessBaseline, From d3af332c2f3ab3b2e91d702936d4ff7e80a91b4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 5 Aug 2026 09:37:55 +0200 Subject: [PATCH 4/7] fix: require comparable iOS tap evidence --- .../interaction-ios-tap-outcome-fixtures.ts | 20 ++- .../interaction-ios-tap-outcome.test.ts | 136 ++++++++++++++++-- .../handlers/interaction-ios-tap-outcome.ts | 14 ++ 3 files changed, 158 insertions(+), 12 deletions(-) diff --git a/src/daemon/handlers/__tests__/interaction-ios-tap-outcome-fixtures.ts b/src/daemon/handlers/__tests__/interaction-ios-tap-outcome-fixtures.ts index ef275a3036..59da672c38 100644 --- a/src/daemon/handlers/__tests__/interaction-ios-tap-outcome-fixtures.ts +++ b/src/daemon/handlers/__tests__/interaction-ios-tap-outcome-fixtures.ts @@ -38,5 +38,23 @@ export const imageViewerNodes: RawSnapshotNode[] = [ ]; export function snapshot(nodes: RawSnapshotNode[]) { - return buildSnapshotState({ nodes, backend: 'xctest' }, { snapshotInteractiveOnly: false }); + return buildSnapshotState( + { + nodes, + backend: 'xctest', + quality: { state: 'healthy', backend: 'tree' }, + }, + { snapshotInteractiveOnly: false }, + ); +} + +export function snapshotPayload( + nodes: RawSnapshotNode[], + backend: 'tree' | 'queries' | 'private-ax' = 'tree', +) { + return { + backend: 'xctest' as const, + nodes, + quality: { state: 'healthy' as const, backend }, + }; } diff --git a/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts b/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts index 7741e5693a..ce0c6b9cea 100644 --- a/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts +++ b/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts @@ -15,6 +15,7 @@ import { imageViewerNodes, profileNodes, snapshot, + snapshotPayload, } from './interaction-ios-tap-outcome-fixtures.ts'; vi.mock('../../../core/dispatch.ts', async (importOriginal) => { @@ -73,7 +74,7 @@ test('a changed post-action capture corroborates a direct iOS tap reported as fa 'XCTest recorded a failure while executing tap; the action may not have been performed.', ); } - if (command === 'snapshot') return { backend: 'xctest', nodes: imageViewerNodes }; + if (command === 'snapshot') return snapshotPayload(imageViewerNodes); return {}; }); @@ -103,7 +104,126 @@ test('an unchanged post-action capture keeps a failed iOS tap failed', async () 'XCTest recorded a failure while executing tap; the action may not have been performed.', ); } - if (command === 'snapshot') return { backend: 'xctest', nodes: profileNodes }; + if (command === 'snapshot') return snapshotPayload(profileNodes); + return {}; + }); + + const response = await runClick(sessionStore, sessionName); + + expect(response?.ok).toBe(false); + if (response && !response.ok) expect(response.error.code).toBe('XCTEST_RECORDED_FAILURE'); + expect(sessionStore.get(sessionName)?.actions).toHaveLength(0); +}); + +test('a changed capture from a different iOS backend keeps the tap failure', async () => { + const sessionName = 'ios-cross-backend-tap-corroboration'; + const sessionStore = makeSessionStore(); + sessionStore.set( + sessionName, + makeIosSession(sessionName, { + appBundleId: 'com.example.app', + snapshot: snapshot(profileNodes), + }), + ); + mockDispatch.mockImplementation(async (_device, command) => { + if (command === 'press') { + throw new AppError( + 'XCTEST_RECORDED_FAILURE', + 'XCTest recorded a failure while executing tap; the action may not have been performed.', + ); + } + if (command === 'snapshot') return snapshotPayload(imageViewerNodes, 'private-ax'); + return {}; + }); + + const response = await runClick(sessionStore, sessionName); + + expect(response?.ok).toBe(false); + if (response && !response.ok) expect(response.error.code).toBe('XCTEST_RECORDED_FAILURE'); + expect(sessionStore.get(sessionName)?.actions).toHaveLength(0); +}); + +test('a sparse changed capture keeps the tap failure', async () => { + const sessionName = 'ios-sparse-tap-corroboration'; + const sessionStore = makeSessionStore(); + sessionStore.set( + sessionName, + makeIosSession(sessionName, { + appBundleId: 'com.example.app', + snapshot: snapshot(profileNodes), + }), + ); + mockDispatch.mockImplementation(async (_device, command) => { + if (command === 'press') { + throw new AppError( + 'XCTEST_RECORDED_FAILURE', + 'XCTest recorded a failure while executing tap; the action may not have been performed.', + ); + } + if (command === 'snapshot') { + return { + ...snapshotPayload(imageViewerNodes), + quality: { state: 'sparse', backend: 'tree', reason: 'capture incomplete' }, + }; + } + return {}; + }); + + const response = await runClick(sessionStore, sessionName); + + expect(response?.ok).toBe(false); + if (response && !response.ok) expect(response.error.code).toBe('XCTEST_RECORDED_FAILURE'); + expect(sessionStore.get(sessionName)?.actions).toHaveLength(0); +}); + +test('a corroboration capture failure keeps the tap failure', async () => { + const sessionName = 'ios-capture-failed-tap-corroboration'; + const sessionStore = makeSessionStore(); + sessionStore.set( + sessionName, + makeIosSession(sessionName, { + appBundleId: 'com.example.app', + snapshot: snapshot(profileNodes), + }), + ); + mockDispatch.mockImplementation(async (_device, command) => { + if (command === 'press') { + throw new AppError( + 'XCTEST_RECORDED_FAILURE', + 'XCTest recorded a failure while executing tap; the action may not have been performed.', + ); + } + if (command === 'snapshot') throw new Error('forced corroboration capture failure'); + return {}; + }); + + const response = await runClick(sessionStore, sessionName); + + expect(response?.ok).toBe(false); + if (response && !response.ok) expect(response.error.code).toBe('XCTEST_RECORDED_FAILURE'); + expect(sessionStore.get(sessionName)?.actions).toHaveLength(0); +}); + +test('a changed capture with a different presentation keeps the tap failure', async () => { + const sessionName = 'ios-presentation-mismatch-tap-corroboration'; + const sessionStore = makeSessionStore(); + const baseline = snapshot(profileNodes); + baseline.presentationKey = 'unreadable-presentation'; + sessionStore.set( + sessionName, + makeIosSession(sessionName, { + appBundleId: 'com.example.app', + snapshot: baseline, + }), + ); + mockDispatch.mockImplementation(async (_device, command) => { + if (command === 'press') { + throw new AppError( + 'XCTEST_RECORDED_FAILURE', + 'XCTest recorded a failure while executing tap; the action may not have been performed.', + ); + } + if (command === 'snapshot') return snapshotPayload(imageViewerNodes); return {}; }); @@ -134,10 +254,7 @@ test('runtime-resolved taps use the same corroboration boundary', async () => { } if (command === 'snapshot') { snapshotCount += 1; - return { - backend: 'xctest', - nodes: snapshotCount === 1 ? profileNodes : imageViewerNodes, - }; + return snapshotPayload(snapshotCount === 1 ? profileNodes : imageViewerNodes); } return {}; }); @@ -173,7 +290,7 @@ test('a corroborated runtime coordinate tap does not schedule a no-change retry' } return {}; } - if (command === 'snapshot') return { backend: 'xctest', nodes: imageViewerNodes }; + if (command === 'snapshot') return snapshotPayload(imageViewerNodes); return {}; }); @@ -232,10 +349,7 @@ test('corroborated runtime taps retain target evidence through save and replay', } if (command === 'snapshot') { snapshotCount += 1; - return { - backend: 'xctest', - nodes: recording && snapshotCount === 2 ? imageViewerNodes : profileNodes, - }; + return snapshotPayload(recording && snapshotCount === 2 ? imageViewerNodes : profileNodes); } return {}; }); diff --git a/src/daemon/handlers/interaction-ios-tap-outcome.ts b/src/daemon/handlers/interaction-ios-tap-outcome.ts index dc11eeb373..bdc9a66a55 100644 --- a/src/daemon/handlers/interaction-ios-tap-outcome.ts +++ b/src/daemon/handlers/interaction-ios-tap-outcome.ts @@ -111,6 +111,20 @@ function hasMatchingPresentation( after: SnapshotState, command: string, ): boolean { + const baselineBackend = baseline.snapshotQuality?.backend; + const afterBackend = after.snapshotQuality?.backend; + if (!baselineBackend || baselineBackend !== afterBackend) { + emitDiagnostic({ + level: 'debug', + phase: 'ios_tap_failure_corroboration_backend_mismatch', + data: { + command, + baselineBackend, + afterBackend, + }, + }); + return false; + } if (!baseline.presentationKey || baseline.presentationKey === after.presentationKey) return true; emitDiagnostic({ level: 'debug', From 8b27f3d18dacce1d3509266be1d27f781249f49f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 5 Aug 2026 09:52:49 +0200 Subject: [PATCH 5/7] fix: bound iOS tap corroboration baseline --- .../interaction-ios-tap-outcome.test.ts | 62 +++++++++++++++++++ .../handlers/interaction-ios-tap-outcome.ts | 18 +++++- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts b/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts index ce0c6b9cea..250c418fc7 100644 --- a/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts +++ b/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts @@ -234,6 +234,68 @@ test('a changed capture with a different presentation keeps the tap failure', as expect(sessionStore.get(sessionName)?.actions).toHaveLength(0); }); +test('a changed capture against a stale baseline keeps the tap failure', async () => { + const sessionName = 'ios-stale-baseline-tap-corroboration'; + const sessionStore = makeSessionStore(); + const baseline = snapshot(profileNodes); + baseline.createdAt = Date.now() - 5_001; + sessionStore.set( + sessionName, + makeIosSession(sessionName, { + appBundleId: 'com.example.app', + snapshot: baseline, + }), + ); + mockDispatch.mockImplementation(async (_device, command) => { + if (command === 'press') { + throw new AppError( + 'XCTEST_RECORDED_FAILURE', + 'XCTest recorded a failure while executing tap; the action may not have been performed.', + ); + } + if (command === 'snapshot') return snapshotPayload(imageViewerNodes); + return {}; + }); + + const response = await runClick(sessionStore, sessionName); + + expect(response?.ok).toBe(false); + if (response && !response.ok) expect(response.error.code).toBe('XCTEST_RECORDED_FAILURE'); + expect(mockDispatch.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(0); + expect(sessionStore.get(sessionName)?.actions).toHaveLength(0); +}); + +test('a changed capture against a keyless baseline keeps the tap failure', async () => { + const sessionName = 'ios-keyless-baseline-tap-corroboration'; + const sessionStore = makeSessionStore(); + const baseline = snapshot(profileNodes); + baseline.presentationKey = undefined; + sessionStore.set( + sessionName, + makeIosSession(sessionName, { + appBundleId: 'com.example.app', + snapshot: baseline, + }), + ); + mockDispatch.mockImplementation(async (_device, command) => { + if (command === 'press') { + throw new AppError( + 'XCTEST_RECORDED_FAILURE', + 'XCTest recorded a failure while executing tap; the action may not have been performed.', + ); + } + if (command === 'snapshot') return snapshotPayload(imageViewerNodes); + return {}; + }); + + const response = await runClick(sessionStore, sessionName); + + expect(response?.ok).toBe(false); + if (response && !response.ok) expect(response.error.code).toBe('XCTEST_RECORDED_FAILURE'); + expect(mockDispatch.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(0); + expect(sessionStore.get(sessionName)?.actions).toHaveLength(0); +}); + test('runtime-resolved taps use the same corroboration boundary', async () => { const sessionName = 'ios-runtime-tap-corroboration'; const sessionStore = makeSessionStore(); diff --git a/src/daemon/handlers/interaction-ios-tap-outcome.ts b/src/daemon/handlers/interaction-ios-tap-outcome.ts index bdc9a66a55..526b755c99 100644 --- a/src/daemon/handlers/interaction-ios-tap-outcome.ts +++ b/src/daemon/handlers/interaction-ios-tap-outcome.ts @@ -13,6 +13,7 @@ import type { ContextFromFlags } from './interaction-common.ts'; import type { CaptureSnapshotForSession } from './interaction-snapshot.ts'; const XCTEST_RECORDED_FAILURE = 'XCTEST_RECORDED_FAILURE'; +const IOS_TAP_CORROBORATION_BASELINE_MAX_AGE_MS = 5_000; const IOS_TAP_CORROBORATION_WARNING = 'XCTest reported the tap as failed, but a same-scope post-action accessibility capture changed; treating the tap as landed. Observe the current screen before issuing another tap.'; @@ -73,9 +74,20 @@ function isTapCommand(command: string): boolean { function readCorroborationBaseline( snapshot: SnapshotState | undefined, -): { snapshot: SnapshotState; presentation: SnapshotPresentation | undefined } | undefined { +): { snapshot: SnapshotState; presentation: SnapshotPresentation } | undefined { if (!hasUsableBaseline(snapshot)) return undefined; - return { snapshot, presentation: readSnapshotPresentation(snapshot.presentationKey) }; + const ageMs = Date.now() - snapshot.createdAt; + if (ageMs < 0 || ageMs > IOS_TAP_CORROBORATION_BASELINE_MAX_AGE_MS) { + emitDiagnostic({ + level: 'debug', + phase: 'ios_tap_failure_corroboration_stale_baseline', + data: { ageMs }, + }); + return undefined; + } + const presentation = readSnapshotPresentation(snapshot.presentationKey); + if (!presentation) return undefined; + return { snapshot, presentation }; } async function captureCorroborationSnapshot( @@ -125,7 +137,7 @@ function hasMatchingPresentation( }); return false; } - if (!baseline.presentationKey || baseline.presentationKey === after.presentationKey) return true; + if (baseline.presentationKey === after.presentationKey) return true; emitDiagnostic({ level: 'debug', phase: 'ios_tap_failure_corroboration_scope_mismatch', From 35a806b7df7febae4b14ec17095cc4abec075c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 5 Aug 2026 12:42:34 +0200 Subject: [PATCH 6/7] test(ios): deterministic injection seam for recorded-tap-failure corroboration (#1605 merge gate) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The field failure cannot be reproduced on this head: the tap false-failures were a downstream symptom of XCTest-channel saturation, which the #1587 capture fixes removed. The seam records a real XCTIssue AFTER the real gesture inside the per-command failure-count window, so xctestRecordedFailureResponse and target invalidation fire byte-for-byte like the field failure. Armed via a decrementing /tmp flag file (the daemon regenerates tampered xctestrun templates, so env plumbing cannot reach a daemon-spawned runner); compiled only under AGENT_DEVICE_RUNNER_UNIT_TESTS. Live evidence on a daemon-spawned runner (Bluesky, ad-bsky-repro sim): - landed case: injected failure on a real Search-tab tap -> success with the corroboration warning, screen verifiably on Search, no redispatch, runner serving next commands; flag consumed exactly once. - unchanged case: injected failure on a dead-coordinate tap -> capture unchanged -> XCTEST_RECORDED_FAILURE preserved with the new honest hint; runner still usable. - field-shape race (relaunch -> full snapshot -> immediate press, 5 attempts): no natural recorded failure occurs on this head — the hostile tree needed for channel saturation is gone, corroborating the causal story. --- .../RunnerTests+CommandExecution.swift | 24 +++++++++++++ .../RunnerTests.swift | 36 +++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index a9636b5620..6bc440949c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -439,6 +439,16 @@ extension RunnerTests { } #endif + func testInjectedTapRecordedFailureGateIsTapOnlyAndCountGated() { + // The seam's recording side cannot run in-bundle (a real XCTIssue would + // fail this very test run — same constraint the record(_:) suppression + // tests document); the live daemon proof covers it. This pins the gate. + XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .tap, remaining: 0)) + XCTAssertTrue(RunnerTests.shouldInjectTapRecordedFailure(command: .tap, remaining: 1)) + XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .type, remaining: 1)) + XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .snapshot, remaining: 1)) + } + func testXCTestRecordedFailureResponseFailsMutatingSuccesses() throws { let command = try runnerCommandFixture(#"{"command":"tap","commandId":"tap-1"}"#) let response = Response(ok: true, data: DataPayload(message: "tapped")) @@ -1227,6 +1237,20 @@ extension RunnerTests { userInfo: [NSLocalizedDescriptionKey: "command returned no response"] ) } +#if AGENT_DEVICE_RUNNER_UNIT_TESTS + // #1605 merge gate: the REAL gesture already executed above; recording a + // production-shaped issue here makes the per-command failure-count + // conversion below fire exactly as in the field (bsky-24: activation + // lands, bookkeeping records a failure). Compiled out of production. + if consumeInjectedTapRecordedFailureForTesting(command: command.command) { + record( + XCTIssue( + type: .assertionFailure, + compactDescription: "Injected tap recorded-failure (#1605 corroboration merge gate)" + ) + ) + } +#endif if didRecordXCTestFailure(since: failureCountBefore), let failureResponse = xctestRecordedFailureResponse(command: command, response: response) { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index 56eeaf5526..babe904ce2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -120,6 +120,42 @@ final class RunnerTests: XCTestCase { // seconds on remote-hosted consent dialogs and bypass the plan budget (#1244). let systemModalProbeBudget: TimeInterval = 4 #if AGENT_DEVICE_RUNNER_UNIT_TESTS + // #1605 merge gate: deterministic live reproduction of the field ambiguity — + // a tap whose coordinate activation LANDS while XCTest bookkeeping records a + // failure. Armed by writing a decrementing count to the flag file below + // (the daemon regenerates tampered xctestrun templates, so env plumbing + // cannot reach a daemon-spawned runner); consumed one injection per tap. + // The injection records a real XCTIssue AFTER the real gesture, so + // `xctestRecordedFailureResponse` and target invalidation fire byte-for-byte + // like a field failure. Production builds compile none of this. + static let injectedTapFailureFlagPathForTesting = + "/tmp/agent-device-inject-tap-recorded-failure-for-testing" + + static func shouldInjectTapRecordedFailure(command: CommandType, remaining: Int) -> Bool { + command == .tap && remaining > 0 + } + + func consumeInjectedTapRecordedFailureForTesting(command: CommandType) -> Bool { + guard + let raw = try? String( + contentsOfFile: Self.injectedTapFailureFlagPathForTesting, + encoding: .utf8 + ), + let remaining = Int(raw.trimmingCharacters(in: .whitespacesAndNewlines)) + else { + return false + } + guard Self.shouldInjectTapRecordedFailure(command: command, remaining: remaining) else { + return false + } + try? String(remaining - 1).write( + toFile: Self.injectedTapFailureFlagPathForTesting, + atomically: true, + encoding: .utf8 + ) + return true + } + // Unit-test-only injectable override for the system-modal probe (see // `boundedBlockingSystemAlertSnapshot` in RunnerTests+Snapshot.swift): when set, a test's probe // body runs in place of `blockingSystemAlertSnapshot` so it can force a real timeout without a From b25e3a0e0ef79bb54d028a111198ccb1d3e70083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 5 Aug 2026 18:09:00 +0200 Subject: [PATCH 7/7] fix: reconcile tap corroboration with current interaction semantics --- src/daemon/handlers/interaction-touch-response.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/daemon/handlers/interaction-touch-response.ts b/src/daemon/handlers/interaction-touch-response.ts index 79e18d6608..823cd438dc 100644 --- a/src/daemon/handlers/interaction-touch-response.ts +++ b/src/daemon/handlers/interaction-touch-response.ts @@ -156,14 +156,10 @@ export function buildInteractionResponseData(params: { if (source.kind === 'runner-payload' || source.kind === 'corroborated-tap') { const commonExtra = { targetKind: source.targetKind, - ...applyResolutionDisclosurePolicy( - source, - source.kind === 'runner-payload' - ? { resolution: DIRECT_IOS_NOT_OBSERVED_RESOLUTION } - : source.resolution - ? { resolution: source.resolution } - : {}, - ), + ...applyResolutionDisclosurePolicy(source, { + resolution: + source.kind === 'runner-payload' ? DIRECT_IOS_NOT_OBSERVED_RESOLUTION : source.resolution, + }), ...(extra ?? {}), }; const result = buildTouchPayload({