Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1229,7 +1229,8 @@ extension RunnerTests {
interactiveOnly: command.interactiveOnly ?? false,
depth: command.depth,
scope: command.scope,
raw: command.raw ?? false
raw: command.raw ?? false,
preferIndependentBackend: command.snapshotCapturePreference == "independent"
)
do {
let payload: DataPayload
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ struct Command: Codable {
let depth: Int?
let scope: String?
let raw: Bool?
let snapshotCapturePreference: String?
let fullscreen: Bool?
let inlineScreenshot: Bool?
let synthesized: Bool?
Expand Down Expand Up @@ -390,4 +391,19 @@ struct SnapshotOptions {
let depth: Int?
let scope: String?
let raw: Bool
let preferIndependentBackend: Bool

init(
interactiveOnly: Bool,
depth: Int?,
scope: String?,
raw: Bool,
preferIndependentBackend: Bool = false
) {
self.interactiveOnly = interactiveOnly
self.depth = depth
self.scope = scope
self.raw = raw
self.preferIndependentBackend = preferIndependentBackend
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ enum SnapshotBackendKind: String, CaseIterable {

enum SnapshotXCTestChannelPlanState: Equatable {
case normal
case preferredIndependentBackend
case deferredToIndependentBackend
case boundedXCTestProbe
}
Expand Down Expand Up @@ -150,16 +151,34 @@ extension RunnerTests {
static func effectiveSnapshotCapturePlan(
_ plan: [SnapshotBackendKind],
xCTestChannelPenalized: Bool,
availableBackends: Set<SnapshotBackendKind> = Set(SnapshotBackendKind.allCases)
availableBackends: Set<SnapshotBackendKind> = Set(SnapshotBackendKind.allCases),
preferIndependentBackend: Bool = false
) -> EffectiveSnapshotCapturePlan {
guard xCTestChannelPenalized, plan == Self.regularVisiblePlan else {
guard plan == Self.regularVisiblePlan else {
return EffectiveSnapshotCapturePlan(
plan: plan,
xCTestChannelState: .normal,
treeCaptureSliceBudgetOverride: nil
)
}
let availablePlan = plan.filter { availableBackends.contains($0) }
if preferIndependentBackend && !xCTestChannelPenalized {
let independentPlan = availablePlan.filter { !$0.usesXCTestAccessibilityChannel }
if !independentPlan.isEmpty {
return EffectiveSnapshotCapturePlan(
plan: independentPlan + availablePlan.filter(\.usesXCTestAccessibilityChannel),
xCTestChannelState: .preferredIndependentBackend,
treeCaptureSliceBudgetOverride: nil
)
}
}
guard xCTestChannelPenalized else {
return EffectiveSnapshotCapturePlan(
plan: availablePlan,
xCTestChannelState: .normal,
treeCaptureSliceBudgetOverride: nil
)
}
let recoveryPlan = availablePlan.filter { !$0.usesXCTestAccessibilityChannel }
if !recoveryPlan.isEmpty {
return EffectiveSnapshotCapturePlan(
Expand Down Expand Up @@ -205,12 +224,19 @@ extension RunnerTests {
let effective = Self.effectiveSnapshotCapturePlan(
plan,
xCTestChannelPenalized: xCTestChannelPenalized,
availableBackends: Set(SnapshotBackendKind.allCases.filter(\.isAvailableOnCurrentPlatform))
availableBackends: Set(SnapshotBackendKind.allCases.filter(\.isAvailableOnCurrentPlatform)),
preferIndependentBackend: options.preferIndependentBackend
)
let effectivePlan = effective.plan
switch effective.xCTestChannelState {
case .normal:
break
case .preferredIndependentBackend:
firstFailure = (
"an independent accessibility backend was preferred to verify post-gesture freshness",
"freshness"
)
NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_INDEPENDENT_BACKEND_PREFERRED bundle=%@", currentBundleId ?? "")
case .deferredToIndependentBackend:
firstFailure = (
"XCTest-backed snapshot tiers were deferred after recent slow accessibility work on this screen",
Expand Down Expand Up @@ -301,7 +327,10 @@ extension RunnerTests {
continue
}

let recovered = kind != effectivePlan.first || effective.xCTestChannelState != .normal
let recovered =
kind != effectivePlan.first
|| effective.xCTestChannelState == .deferredToIndependentBackend
|| effective.xCTestChannelState == .boundedXCTestProbe
if recovered {
NSLog(
"AGENT_DEVICE_RUNNER_SNAPSHOT_RECOVERED backend=%@ reason=%@",
Expand Down Expand Up @@ -687,6 +716,28 @@ extension RunnerTests {
)
}

func testEffectiveSnapshotCapturePlanPrefersIndependentBackendForFreshnessProbe() {
let simulatorPlan = Self.effectiveSnapshotCapturePlan(
Self.regularVisiblePlan,
xCTestChannelPenalized: false,
preferIndependentBackend: true
)

XCTAssertEqual(simulatorPlan.plan, [.privateAX, .recursiveTree, .querySweep])
XCTAssertEqual(simulatorPlan.xCTestChannelState, .preferredIndependentBackend)
XCTAssertNil(simulatorPlan.treeCaptureSliceBudgetOverride)

let physicalDevicePlan = Self.effectiveSnapshotCapturePlan(
Self.regularVisiblePlan,
xCTestChannelPenalized: false,
availableBackends: [.recursiveTree, .querySweep],
preferIndependentBackend: true
)

XCTAssertEqual(physicalDevicePlan.plan, [.recursiveTree, .querySweep])
XCTAssertEqual(physicalDevicePlan.xCTestChannelState, .normal)
}

func testSnapshotXCTestChannelPenaltyMatchesBundleAndExpires() {
defer {
snapshotXCTestChannelPenaltyBundleId = nil
Expand Down
6 changes: 6 additions & 0 deletions docs/adr/0004-ios-snapshot-backend-strategy.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@ strategies:
carry the response, fail explicitly instead of silently truncating the tree at a hard node count.
If XCTest reports a real AX serialization failure, preserve that error instead of pretending the
UI is empty.
- **Post-gesture freshness probe**: after the daemon observes two quiet captures that still match
the pre-gesture surface, it may ask the regular plan to prefer an independent backend for the
next observations. On iOS simulators this puts private AX first while retaining the normal plan
as recovery; ordinary snapshots remain tree-first. The daemon also uses a fixed distrust
observation count, with a separate hard safety ceiling, so slow XCTest reads cannot turn a
contention-shaped delay into premature stale geometry acceptance.
- **Future AX-service strategy**: treat Bluesky-class failures as evidence that XCTest is
not a complete semantic snapshot backend. A robust semantic fix should add a host-side simulator
accessibility backend, similar in role to existing simulator accessibility inspection tools,
Expand Down
3 changes: 3 additions & 0 deletions packages/contracts/src/interactor-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ export type ElementSelectorTapOptions = {
expectedPoint?: Point;
};

export type SnapshotCapturePreference = 'independent';

/**
* Legacy success text retained for compatibility when the XCTest runner used
* the Maestro non-hittable coordinate fallback. Usage itself is carried by
Expand All @@ -76,6 +78,7 @@ export type SnapshotOptions = BaseSnapshotOptions & {
signal?: AbortSignal;
includeRects?: boolean;
includeHiddenContentHints?: boolean;
snapshotCapturePreference?: SnapshotCapturePreference;
surface?: SessionSurface;
};

Expand Down
2 changes: 2 additions & 0 deletions src/core/dispatch-context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import type {
BackMode,
ClickButton,
ElementSelectorKey,
SnapshotCapturePreference,
SwipePattern,
} from '@agent-device/contracts/interaction';
import type { RunnerLogicalLeaseContext } from '@agent-device/contracts/platform';
Expand Down Expand Up @@ -36,6 +37,7 @@ export type DispatchContext = ScreenshotDispatchFlags & {
snapshotDepth?: number;
snapshotScope?: string;
snapshotRaw?: boolean;
snapshotCapturePreference?: SnapshotCapturePreference;
snapshotIncludeRects?: boolean;
snapshotIncludeHiddenContentHints?: boolean;
skipIosSimulatorBootCheck?: boolean;
Expand Down
1 change: 1 addition & 0 deletions src/core/dispatch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -639,6 +639,7 @@ async function handleSnapshotCommand(
depth: snapshotContext.snapshotDepth,
scope: snapshotContext.snapshotScope,
raw: snapshotContext.snapshotRaw,
snapshotCapturePreference: snapshotContext.snapshotCapturePreference,
includeRects: snapshotContext.snapshotIncludeRects,
includeHiddenContentHints: snapshotContext.snapshotIncludeHiddenContentHints,
surface: snapshotContext.surface,
Expand Down
29 changes: 29 additions & 0 deletions src/daemon/__tests__/post-gesture-stabilization-verdict.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,35 @@ test('decidePostGestureStabilityVerdict accepts a baseline-matching signature on
);
});

test('decidePostGestureStabilityVerdict uses the observation budget when capture latency exceeds the time cap', () => {
const signature = buildInteractionSurfaceSignature(pickupSnapshot().nodes);

assert.equal(
decidePostGestureStabilityVerdict({
needsBaselineDistrust: true,
baselineSignature: signature,
quietSignature: signature,
elapsedMs: 9_000,
distrustCapMs: 3_500,
distrustAttemptCount: 11,
distrustAttemptLimit: 12,
}),
'distrust',
);
assert.equal(
decidePostGestureStabilityVerdict({
needsBaselineDistrust: true,
baselineSignature: signature,
quietSignature: signature,
elapsedMs: 2_000,
distrustCapMs: 3_500,
distrustAttemptCount: 12,
distrustAttemptLimit: 12,
}),
'accept-stale',
);
});

// --- #1563 review, finding 1: a root-only shared overlap must trust immediately, not tax the cap ---

test('decidePostGestureStabilityVerdict trusts immediately when a real scroll leaves only the application root shared (no cap tax)', () => {
Expand Down
74 changes: 74 additions & 0 deletions src/daemon/__tests__/post-gesture-stabilization.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,80 @@ test('capturePostGestureStabilizedResult keeps polling past the normal deadline
assert.ok(captureCount > 8, `expected sustained polling, saw ${captureCount} captures`);
});

test('capturePostGestureStabilizedResult preserves its distrust observation count when captures are slow (iOS)', async () => {
vi.useFakeTimers();
vi.setSystemTime(0);
const session = makeSession('ios');
session.snapshot = pickupSnapshot(500);
markPostGestureStabilization(session, 'scroll');

const capture = vi.fn(async () => {
// Simulate the XCTest channel taking 500ms per read without making the test
// itself wait. The current wall-clock cap accepts after roughly six reads,
// before it has collected the normal distrust sample count.
vi.setSystemTime(Date.now() + 500);
return pickupSnapshot(500);
});

const resultPromise = withDiagnosticsScope({}, async () => {
const result = await capturePostGestureStabilizedResult({
session,
capture,
readSnapshot: (snapshot) => snapshot,
});
return {
result,
staleAccepts: countDiagnosticEventsByPhase(['post_gesture_snapshot_stale_accept']),
};
});

await vi.advanceTimersByTimeAsync(20_000);
const { staleAccepts } = await resultPromise;

assert.equal(staleAccepts, 1);
assert.ok(
capture.mock.calls.length >= 12,
`expected the distrust sample count to survive slow captures, saw ${capture.mock.calls.length}`,
);
});

test('capturePostGestureStabilizedResult switches to independent evidence after a baseline match (iOS)', async () => {
vi.useFakeTimers();
const session = makeSession('ios');
session.snapshot = pickupSnapshot(500);
markPostGestureStabilization(session, 'scroll');

const capture = vi.fn(async (options?: { snapshotCapturePreference?: 'independent' }) =>
options?.snapshotCapturePreference === 'independent'
? pickupSnapshot(120)
: pickupSnapshot(500),
);

const resultPromise = withDiagnosticsScope({}, async () => {
const result = await capturePostGestureStabilizedResult({
session,
capture,
readSnapshot: (snapshot) => snapshot,
});
return {
result,
staleAccepts: countDiagnosticEventsByPhase(['post_gesture_snapshot_stale_accept']),
settled: countDiagnosticEventsByPhase(['post_gesture_snapshot_stabilized']),
};
});

await vi.advanceTimersByTimeAsync(2_000);
const { staleAccepts, settled } = await resultPromise;

assert.equal(staleAccepts, 0);
assert.equal(settled, 1);
assert.equal(capture.mock.calls.length, 4);
assert.equal(capture.mock.calls[0]?.[0], undefined);
assert.equal(capture.mock.calls[1]?.[0], undefined);
assert.deepEqual(capture.mock.calls[2], [{ snapshotCapturePreference: 'independent' }]);
assert.deepEqual(capture.mock.calls[3], [{ snapshotCapturePreference: 'independent' }]);
});

test('capturePostGestureStabilizedResult trusts a quiet signature once content genuinely differs from the baseline (iOS)', async () => {
vi.useFakeTimers();
const session = makeSession('ios');
Expand Down
14 changes: 11 additions & 3 deletions src/daemon/handlers/snapshot-capture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
type SnapshotBackend,
type SnapshotState,
} from '@agent-device/kernel/snapshot';
import type { SnapshotCapturePreference } from '@agent-device/contracts/interaction';
import { dispatchCommand, type CommandFlags } from '../../core/dispatch.ts';
import { runMacOsSnapshotAction } from '../../platforms/apple/os/macos/helper.ts';
import { snapshotLinux } from '../../platforms/linux/snapshot.ts';
Expand Down Expand Up @@ -58,6 +59,7 @@ type CaptureSnapshotParams = {
outPath?: string;
logPath: string;
snapshotScope?: string;
snapshotCapturePreference?: SnapshotCapturePreference;
androidFreshnessMode?: AndroidFreshnessMode;
signal?: AbortSignal;
};
Expand Down Expand Up @@ -156,7 +158,7 @@ async function captureInteractionOutcomeAwareSnapshot(
latest = await capturePostGestureStabilizedResult({
session,
initial: latest,
capture: async () => await capturePostActionSnapshotAttempt(params),
capture: async (options) => await capturePostActionSnapshotAttempt(params, options),
readSnapshot: (attempt) => attempt.snapshot,
});
if (outcome.change !== 'ambiguous' && latest.annotations.freshness?.staleAfterRetries !== true) {
Expand Down Expand Up @@ -229,6 +231,7 @@ export async function captureSnapshotData(params: CaptureSnapshotParams): Promis
session?.trace?.outPath,
),
snapshotIncludeRects: params.includeRects,
snapshotCapturePreference: params.snapshotCapturePreference,
signal: params.signal,
})) as SnapshotData;
}
Expand Down Expand Up @@ -290,7 +293,7 @@ async function capturePostGestureAwareSnapshot(
): Promise<CaptureSnapshotResult> {
const latest = await capturePostGestureStabilizedResult({
session: params.session,
capture: async () => await capturePostActionSnapshotAttempt(params),
capture: async (options) => await capturePostActionSnapshotAttempt(params, options),
readSnapshot: (attempt) => attempt.snapshot,
});
return {
Expand All @@ -301,12 +304,17 @@ async function capturePostGestureAwareSnapshot(

async function capturePostActionSnapshotAttempt(
params: CaptureSnapshotParams & { session: SessionState },
options?: { snapshotCapturePreference?: SnapshotCapturePreference },
): Promise<SnapshotAttempt> {
const freshness = getActiveAndroidSnapshotFreshness(params.session);
if (freshness && params.device.platform === 'android') {
return await captureAndroidFreshnessAwareAttempt(params, freshness);
}
return await captureSnapshotAttempt(params);
return await captureSnapshotAttempt(
options?.snapshotCapturePreference
? { ...params, snapshotCapturePreference: options.snapshotCapturePreference }
: params,
);
}

async function captureSnapshotAttempt(params: CaptureSnapshotParams): Promise<SnapshotAttempt> {
Expand Down
Loading
Loading