From b113410d179193620356e1fe655c7167ba9c41ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 6 Aug 2026 16:51:42 +0200 Subject: [PATCH] feat: support --settle on scroll and back (generic-route settle observation) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scroll-then-observe and back-then-observe are legitimate combos that the post-action observation registry never grew past the touch commands, so AppControlBench models kept burning a tool call on an INVALID_ARGS refusal. Registry: POST_ACTION_OBSERVATION_BY_COMMAND gains `scroll: 'settle'` and `back: 'settle'` — settle without verify, because neither resolves a target and there is no pre-action node to digest into evidence. Every downstream seam already derives from that map: CLI flags, MCP/SDK input fields, the daemon flag guard, and now the timeout policy (a new postActionObservationTimeoutPolicy keeps the map the enumerator, so granting a command --settle cannot leave its envelope too narrow for the wait it accepts). Engine: settleAfterInteraction is split into a target-less settleAfterAction plus a thin adapter, and exposed as a runtime command (`runtime.interactions.observeSettle`) so the generic route reaches settle through the same runtime seam every other settle-carrying command uses. Wiring: request-generic-dispatch reads the pre-action tree before dispatch and, after markDeferredInteractionOutcome (so settle's first capture folds the #1542 stabilization) and after Android's blocking-dialog postflight, attaches the observation as `data.settle` with the ADR 0014 refs-generation dance. The whole path sits behind a lazy import, like every daemon route: home/screenshot/orientation must not pay the interaction runtime's import cost, and the static edge would have folded that cluster into the daemon's largest type cycle. The diff baseline is the STORED pre-action snapshot, which can be several commands older than the action. That is the honest #1101 contract ("settled diff vs the pre-action tree") but is weaker than press's freshly-resolved baseline — a `back` right after an unsnapshotted `press` diffs root-to-root. Also extracts the shared settle CLI rendering so scroll/back print the same verdict/diff/tail block as the touch commands, and lifts the ADR 0014 settle ref-issuance rule into one function shared by both routes. Closes #1638 --- packages/contracts/src/client-gesture.ts | 13 +- .../parser/__tests__/cli-help-topics.test.ts | 2 +- src/cli/parser/cli-help.ts | 2 +- src/commands/interaction/index.ts | 24 +- src/commands/interaction/interactions.ts | 1 + src/commands/interaction/metadata.ts | 41 +-- src/commands/interaction/output.test.ts | 39 +++ src/commands/interaction/output.ts | 89 +---- src/commands/interaction/runtime/index.ts | 11 + src/commands/interaction/runtime/settle.ts | 143 +++++--- .../post-action-observation-surface.ts | 58 ++++ src/commands/settle-output.ts | 95 ++++++ src/commands/system/index.test.ts | 45 +++ src/commands/system/index.ts | 11 +- src/commands/system/navigation-projection.ts | 12 +- src/commands/system/output.ts | 10 +- .../__tests__/post-action-observation.test.ts | 8 + .../__tests__/timeout-policy.test.ts | 5 + .../post-action-observation.ts | 13 + src/core/command-descriptor/registry.ts | 32 +- src/daemon/__tests__/generic-settle.test.ts | 306 ++++++++++++++++++ src/daemon/generic-settle-observation.ts | 85 +++++ src/daemon/handlers/interaction-touch.ts | 37 +-- src/daemon/request-generic-dispatch.ts | 47 ++- src/daemon/session-event-action.ts | 19 +- src/daemon/session-snapshot.ts | 35 ++ 26 files changed, 951 insertions(+), 232 deletions(-) create mode 100644 src/commands/post-action-observation-surface.ts create mode 100644 src/commands/settle-output.ts create mode 100644 src/daemon/__tests__/generic-settle.test.ts create mode 100644 src/daemon/generic-settle-observation.ts diff --git a/packages/contracts/src/client-gesture.ts b/packages/contracts/src/client-gesture.ts index 823f318e4f..faa3d5ca35 100644 --- a/packages/contracts/src/client-gesture.ts +++ b/packages/contracts/src/client-gesture.ts @@ -135,9 +135,10 @@ export type RotateGestureOptions = DeviceCommandBaseOptions & { export type TransformGestureOptions = DeviceCommandBaseOptions & TransformGestureParams; -export type ScrollOptions = DeviceCommandBaseOptions & { - direction: ScrollInputDirection; - amount?: number; - pixels?: number; - durationMs?: number; -}; +export type ScrollOptions = DeviceCommandBaseOptions & + SettleCommandOptions & { + direction: ScrollInputDirection; + amount?: number; + pixels?: number; + durationMs?: number; + }; diff --git a/src/cli/parser/__tests__/cli-help-topics.test.ts b/src/cli/parser/__tests__/cli-help-topics.test.ts index 74ffaebb8d..79edd59163 100644 --- a/src/cli/parser/__tests__/cli-help-topics.test.ts +++ b/src/cli/parser/__tests__/cli-help-topics.test.ts @@ -91,7 +91,7 @@ test('usage includes agent workflows, config, environment, and examples footers' assert.match(usageText, /Default app loop: agent-device open /); assert.match( usageText, - /Use --settle only on planned press, click, fill, or longpress commands; never add it to open, snapshot, or close/, + /Use --settle only on planned press, click, fill, longpress, scroll, or back commands; never add it to open, snapshot, or close/, ); assert.match(usageText, /type never accepts --settle/); assert.match(usageText, /explicit success confirmation is visible, stop/); diff --git a/src/cli/parser/cli-help.ts b/src/cli/parser/cli-help.ts index cecf158a2c..ed24aebfc2 100644 --- a/src/cli/parser/cli-help.ts +++ b/src/cli/parser/cli-help.ts @@ -79,7 +79,7 @@ const AGENT_START_LINES = [ // Haiku from 0/2 baseline to 4/4; generic structured-hint recovery passed 8/8 // uncoached output cases versus 7/8 with the longer special-case prose. 'Default app loop: agent-device open -> agent-device snapshot -i -> mutate a current target with --settle -> continue from that settled diff -> agent-device close.', - 'Use --settle only on planned press, click, fill, or longpress commands; never add it to open, snapshot, or close. type never accepts --settle: run agent-device type "text", then diff snapshot if verification is needed. Once the task\'s requested end state or an explicit success confirmation is visible, stop; do not tap transient follow-up controls or navigate away only to re-verify.', + 'Use --settle only on planned press, click, fill, longpress, scroll, or back commands; never add it to open, snapshot, or close. type never accepts --settle: run agent-device type "text", then diff snapshot if verification is needed. Once the task\'s requested end state or an explicit success confirmation is visible, stop; do not tap transient follow-up controls or navigate away only to re-verify.', 'Follow structured command hints before choosing a recovery action.', 'Targets are concrete refs or selectors: @e12, label="Query", role=button label="Submit".', 'Selector keys are only: id, role, text, label, value, appname, windowtitle, visible, hidden, editable, selected, focused, enabled, hittable. placeholder, index, and key are not selector keys.', diff --git a/src/commands/interaction/index.ts b/src/commands/interaction/index.ts index 80b91799b2..e0d5557e95 100644 --- a/src/commands/interaction/index.ts +++ b/src/commands/interaction/index.ts @@ -19,17 +19,9 @@ import type { TypeTextOptions, } from '@agent-device/contracts/client'; import type { CommandSchemaOverride } from '../../cli-schema/types.ts'; -import { - commandSupportsSettleObservation, - commandSupportsVerifyEvidence, -} from '../../core/command-descriptor/registry.ts'; -import { - REPEATED_TOUCH_FLAGS, - SELECTOR_SNAPSHOT_FLAGS, - SETTLE_FLAGS, -} from '../cli-grammar/flag-groups.ts'; -import { type FlagKey } from '../cli-grammar/flag-types.ts'; +import { REPEATED_TOUCH_FLAGS, SELECTOR_SNAPSHOT_FLAGS } from '../cli-grammar/flag-groups.ts'; import { defineExecutableCommand } from '../command-contract.ts'; +import { postActionObservationCliFlags } from '../post-action-observation-surface.ts'; import { commonToClientOptions, toClientElementTarget, @@ -137,20 +129,15 @@ const interactionCliSchemas = { ], }, scroll: { - usageOverride: 'scroll [amount] [--pixels ] [--duration-ms ]', + usageOverride: + 'scroll [amount] [--pixels ] [--duration-ms ] [--settle]', positionalArgs: ['directionOrEdge', 'amount?'], - allowedFlags: ['pixels', 'durationMs'], + allowedFlags: ['pixels', 'durationMs', ...postActionObservationCliFlags('scroll')], }, } as const satisfies Record; type InteractionCommandMetadata = (typeof interactionCommandMetadata)[number]; type InteractionCommandName = InteractionCommandMetadata['name']; -function postActionObservationCliFlags(command: InteractionCommandName): readonly FlagKey[] { - const flags: FlagKey[] = []; - if (commandSupportsVerifyEvidence(command)) flags.push('verify'); - if (commandSupportsSettleObservation(command)) flags.push(...SETTLE_FLAGS); - return flags; -} const clickCommandDefinition = defineExecutableCommand(metadata('click'), (client, input) => client.interactions.click(toClickOptions(input)), @@ -318,6 +305,7 @@ const scrollCommandFacet = defineCommandFacet({ cliSchema: interactionCliSchemas.scroll, cliReader: interactionCliReaders.scroll, daemonWriter: interactionDaemonWriters.scroll, + cliOutputFormatter: interactionCliOutputFormatters.scroll, }); const getCommandFacet = defineCommandFacet({ diff --git a/src/commands/interaction/interactions.ts b/src/commands/interaction/interactions.ts index eaf41ba68a..110eb85fff 100644 --- a/src/commands/interaction/interactions.ts +++ b/src/commands/interaction/interactions.ts @@ -97,6 +97,7 @@ export const interactionCliReaders = { }, scroll: (positionals, flags) => ({ ...commonInputFromFlags(flags), + ...settleInputFromFlags(flags), direction: readScrollDirection(positionals[0]), amount: optionalCliNumber(positionals[1]), pixels: flags.pixels, diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index b80746fdd9..8351960de8 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -16,13 +16,9 @@ import { type SwipeGesturePayload, type TransformGesturePayload, } from '@agent-device/contracts/interaction'; -import type { PostActionObservationSupportFor } from '../../core/command-descriptor/post-action-observation.ts'; -import { - commandSupportsSettleObservation, - commandSupportsVerifyEvidence, -} from '../../core/command-descriptor/registry.ts'; import { FIND_LOCATORS } from '@agent-device/selectors'; import { defineCommandMetadata } from '../command-contract.ts'; +import { postActionObservationFields } from '../post-action-observation-surface.ts'; import { booleanField, elementTargetField, @@ -70,7 +66,8 @@ const interactionCommandDescriptions = { focus: 'Move input focus to explicit screen coordinates without entering text. Prefer semantic interactions when a snapshot ref or selector is available; use type or fill after focus.', type: 'Append text to the currently focused input. Use fill when the existing field value should be replaced, and focus first when no input is active.', - scroll: 'Scroll in a direction, or toward the top/bottom edge of scrollable content.', + scroll: + 'Scroll in a direction, or toward the top/bottom edge of scrollable content. Use settle to get the scrolled-into-view diff without a follow-up snapshot.', get: 'Read text or accessibility attributes from a snapshot ref or selector without changing the app. Use format text for visible content or attrs for the element attribute map.', is: 'Check whether a selector satisfies a UI predicate such as visible, hidden, editable, selected, focused, or text. Use wait when the condition may appear asynchronously.', find: 'Find by text/label/value/role/id and run action', @@ -80,37 +77,6 @@ const interactionCommandDescriptions = { type InteractionCommandName = keyof typeof interactionCommandDescriptions; -const verifyField = () => - booleanField( - 'Capture cheap post-action evidence (AX digest, node counts, changedFromBefore) instead of a follow-up snapshot.', - ); - -const settleFields = () => ({ - settle: booleanField( - 'After the action, wait for the UI to go quiet and return the settled diff vs the pre-action tree in the same response. Best-effort; never fails the action.', - ), - settleQuietMs: integerField('Settle: quiet window in milliseconds (default 500).', { min: 0 }), - timeoutMs: integerField('Settle: wait deadline in milliseconds (default 10000).', { min: 1 }), -}); - -type VerifyFieldMap = { verify: ReturnType }; -type SettleFieldMap = ReturnType; -type PostActionObservationFields = - PostActionObservationSupportFor extends 'settle-and-verify' - ? VerifyFieldMap & SettleFieldMap - : PostActionObservationSupportFor extends 'settle' - ? SettleFieldMap - : {}; - -function postActionObservationFields( - command: TName, -): PostActionObservationFields { - return { - ...(commandSupportsVerifyEvidence(command) ? { verify: verifyField() } : {}), - ...(commandSupportsSettleObservation(command) ? settleFields() : {}), - } as PostActionObservationFields; -} - const clickFields = { target: requiredField(interactionTargetField()), button: enumField(CLICK_BUTTONS, 'Pointer button for platforms that support mouse buttons.'), @@ -170,6 +136,7 @@ const scrollFields = { min: 0, max: SCROLL_DURATION_MAX_MS, }), + ...postActionObservationFields('scroll'), }; // #1271 stage 2 (ADR 0012 amendment): `get`/`is`/`find` are observation-only, diff --git a/src/commands/interaction/output.test.ts b/src/commands/interaction/output.test.ts index 4c9ad2ef29..db2799f606 100644 --- a/src/commands/interaction/output.test.ts +++ b/src/commands/interaction/output.test.ts @@ -13,6 +13,9 @@ const formatFill = (result: Record) => const formatLongPress = (result: Record) => interactionCliOutputFormatters.longpress({ input: {}, result }); +const formatScroll = (result: Record) => + interactionCliOutputFormatters.scroll({ input: {}, result }); + describe('find CLI output', () => { test('click prints the same success line as a direct press', () => { const output = formatFind({ @@ -275,3 +278,39 @@ describe('longpress CLI output', () => { ); }); }); + +describe('scroll CLI output', () => { + test('prints the scroll message alone without a settle observation', () => { + const output = formatScroll({ message: 'Scrolled down' }); + + expect(output.text).toBe('Scrolled down'); + }); + + // #1638: the generic-route settle renders exactly like the touch commands. + test('appends settle verdict and diff lines when present', () => { + const output = formatScroll({ + message: 'Scrolled down', + settle: { + settled: true, + waitedMs: 400, + refsGeneration: 7, + diff: { + summary: { additions: 1, removals: 1, unchanged: 9 }, + lines: [ + { kind: 'removed', text: '@e2 [button] "Load more"' }, + { kind: 'added', text: '@e2 [button] "Next page"', ref: 'e2' }, + ], + }, + }, + }); + + expect(output.text).toBe( + [ + 'Scrolled down', + 'settled after 400ms: +1 -1 (~9 unchanged)', + '- @e2 [button] "Load more"', + '+ @e2 [button] "Next page"', + ].join('\n'), + ); + }); +}); diff --git a/src/commands/interaction/output.ts b/src/commands/interaction/output.ts index 0ab316db8f..28b9360c56 100644 --- a/src/commands/interaction/output.ts +++ b/src/commands/interaction/output.ts @@ -4,6 +4,7 @@ import type { CliOutput } from '../command-contract.ts'; import { displayLabel, formatRole } from '../../snapshot/snapshot-lines.ts'; import { readCommandMessage } from '../../utils/success-text.ts'; import { messageCliOutput, resultOutput, type CliOutputFormatter } from '../output-common.ts'; +import { appendResponseNotes, pinnedRefText } from '../settle-output.ts'; function getCliOutput(params: { result: CommandRequestResult; format?: string }): CliOutput { const data = params.result as Record; @@ -16,17 +17,6 @@ function getCliOutput(params: { result: CommandRequestResult; format?: string }) return defaultCommandCliOutput(data); } -// ADR 0014: a reusable ref in a PARTIAL result renders in ready-to-copy -// `@eN~s` form so a human CLI caller can paste it into the next -// mutation without a separate pin step. A mutating result carries no -// `refsGeneration`, so its acted ref is never pinned. -function pinnedRefText(ref: unknown, refsGeneration: unknown): string | undefined { - if (typeof ref !== 'string' || ref.length === 0) return undefined; - if (typeof refsGeneration !== 'number') return undefined; - const body = ref.startsWith('@') ? ref.slice(1) : ref; - return `@${body}~s${refsGeneration}`; -} - function findCliOutput(result: CommandRequestResult): CliOutput { const data = result as Record; // Interactive find actions (click/fill/focus/type) carry the same success message as @@ -90,85 +80,14 @@ function messageWithSettleCliOutput(result: CommandRequestResult): CliOutput { return { data: output.data, text: appendResponseNotes(output.text, data) }; } -function appendResponseNotes( - text: string | null | undefined, - data: Record, -): string { - const warning = typeof data.warning === 'string' ? `\nWarning: ${data.warning}` : ''; - return `${text ?? ''}${warning}${formatSettleText(data.settle)}`; -} - -type SettleTextView = { - settled?: boolean; - waitedMs?: number; - hint?: string; - diff?: { - summary?: { additions?: number; removals?: number; unchanged?: number }; - lines?: Array<{ kind?: string; text?: string }>; - truncated?: boolean; - }; - tail?: Array<{ ref?: string; role?: string; label?: string }>; - tailTruncated?: boolean; - refsGeneration?: number; -}; - -/** - * Compact `--settle` (#1101) rendering appended to the tap line: the verdict, - * the changed-count summary, and the changed lines themselves (the payload the - * agent acts on). Empty for non-settle responses. - */ -function formatSettleText(settle: unknown): string { - if (!settle || typeof settle !== 'object') return ''; - const view = settle as SettleTextView; - const parts = [ - formatSettleVerdict(view), - ...formatSettleDiffLines(view.diff), - ...formatSettleTailLines(view), - ]; - if (view.hint) parts.push(`hint: ${view.hint}`); - return `\n${parts.join('\n')}`; -} - -function formatSettleDiffLines(diff: SettleTextView['diff']): string[] { - const lines = (diff?.lines ?? []).map( - (line) => `${line.kind === 'removed' ? '-' : '+'} ${line.text ?? ''}`, - ); - if (diff?.truncated) lines.push('… changed lines truncated'); - return lines; -} - -// Unchanged interactive tail: only present when the diff's added lines -// carried zero refs (modal-dismiss/toast-only diff), so the settled tree's -// remaining actionable elements would otherwise be invisible. -function formatSettleTailLines(view: SettleTextView): string[] { - const tail = view.tail ?? []; - if (tail.length === 0) return []; - const lines = [`unchanged interactive (${tail.length}):`]; - for (const entry of tail) { - const label = entry.label ? ` "${entry.label}"` : ''; - // ADR 0014: the settled tail refs are reusable, so render them pinned when - // the settle response carried its generation. - const ref = pinnedRefText(entry.ref, view.refsGeneration) ?? `@${entry.ref ?? ''}`; - lines.push(`= ${ref} [${entry.role ?? ''}]${label}`); - } - if (view.tailTruncated) { - lines.push('… more interactive elements not shown, use snapshot -i'); - } - return lines; -} - -function formatSettleVerdict(view: SettleTextView): string { - const verdict = view.settled === true ? 'settled' : 'not settled'; - const summary = view.diff?.summary; - if (!summary) return `${verdict} after ${view.waitedMs ?? 0}ms`; - return `${verdict} after ${view.waitedMs ?? 0}ms: +${summary.additions ?? 0} -${summary.removals ?? 0} (~${summary.unchanged ?? 0} unchanged)`; -} - export const interactionCliOutputFormatters = { click: resultOutput(tapCliOutput), press: resultOutput(tapCliOutput), fill: resultOutput(messageWithSettleCliOutput), longpress: resultOutput(messageWithSettleCliOutput), + // #1638: scroll takes --settle on the generic route, so its line renders the + // settled diff exactly like the touch commands. + scroll: resultOutput(messageWithSettleCliOutput), get: ({ input, result }) => getCliOutput({ result: result as CommandRequestResult, diff --git a/src/commands/interaction/runtime/index.ts b/src/commands/interaction/runtime/index.ts index f46bbc532d..17a3c2b745 100644 --- a/src/commands/interaction/runtime/index.ts +++ b/src/commands/interaction/runtime/index.ts @@ -53,6 +53,8 @@ import { type GestureCommandOptions, type GestureCommandResult, } from './gesture-command.ts'; +import { settleObservationCommand, type SettleObservationCommandOptions } from './settle.ts'; +import type { SettleObservation } from '@agent-device/contracts/interaction'; export type SelectorCommands = { find: RuntimeCommand; @@ -78,6 +80,12 @@ export type InteractionCommands = { longPress: RuntimeCommand; scroll: RuntimeCommand; gesture: RuntimeCommand; + /** + * `--settle` (#1101) for a caller that resolved no target — the daemon's + * generic scroll/back route (#1638). Every other settle-carrying command gets + * its observation folded into its own result. + */ + observeSettle: RuntimeCommand; }; export type BoundSelectorCommands = { @@ -135,6 +143,7 @@ export type BoundInteractionCommands = { ) => Promise; scroll: BoundRuntimeCommand; gesture: BoundRuntimeCommand; + observeSettle: BoundRuntimeCommand; }; export const selectorCommands: SelectorCommands = { @@ -158,6 +167,7 @@ export const interactionCommands: InteractionCommands = { longPress: longPressCommand, scroll: scrollCommand, gesture: gestureCommand, + observeSettle: settleObservationCommand, }; export function bindSelectorCommands(runtime: AgentDeviceRuntime): BoundSelectorCommands { @@ -188,6 +198,7 @@ export function bindInteractionCommands(runtime: AgentDeviceRuntime): BoundInter interactionCommands.longPress(runtime, { ...options, target }), scroll: (options) => interactionCommands.scroll(runtime, options), gesture: (options) => interactionCommands.gesture(runtime, options), + observeSettle: (options) => interactionCommands.observeSettle(runtime, options), }; } diff --git a/src/commands/interaction/runtime/settle.ts b/src/commands/interaction/runtime/settle.ts index 0d2abaa275..d6e7f18f58 100644 --- a/src/commands/interaction/runtime/settle.ts +++ b/src/commands/interaction/runtime/settle.ts @@ -16,6 +16,7 @@ import type { SettleParams, SettleTailEntry, } from '@agent-device/contracts/interaction'; +import type { RuntimeCommand } from '../../runtime-types.ts'; import type { CapturedSnapshot } from './selector-read-shared.ts'; import { DEFAULT_STABLE_QUIET_MS, @@ -62,6 +63,55 @@ export async function settleAfterInteraction( runtime: AgentDeviceRuntime, options: CommandContext, params: SettleParams & { resolved: ResolvedInteractionTarget }, +): Promise { + const { resolved, ...settleParams } = params; + return await settleAfterAction(runtime, options, { + ...settleParams, + baselineNodes: resolveBaselineNodes(resolved), + ...(resolved.point ? { actionPoint: resolved.point } : {}), + }); +} + +export type SettleObservationCommandOptions = CommandContext & + SettleParams & { + /** The pre-action tree the settled capture is diffed against. */ + baselineNodes: SnapshotNode[]; + }; + +/** + * The target-less settle as a runtime command (#1638), so a caller that never + * resolved a target — the daemon's generic scroll/back route — reaches the same + * observation through the same runtime seam every other settle-carrying command + * uses, instead of reaching into the engine directly. + * + * Returns only the observation: with no resolved target there is no `--verify` + * evidence to pair the settled nodes with. + */ +export const settleObservationCommand: RuntimeCommand< + SettleObservationCommandOptions, + SettleObservation +> = async (runtime, options) => { + const { baselineNodes, ...settle } = options; + const outcome = await settleAfterAction(runtime, options, { ...settle, baselineNodes }); + return outcome.observation; +}; + +/** + * The target-less settle (#1638): same stable-capture loop, storage rules and + * hints as {@link settleAfterInteraction}, but the caller supplies the diff + * baseline instead of it coming from a freshly resolved target, and there is + * no action point to recognize self-echo added lines by. + * + * The generic route (scroll/back) passes the session's STORED pre-action + * snapshot nodes as the baseline. That tree can be several commands older than + * the action — which is the honest reading of the #1101 contract ("the settled + * diff vs the pre-action tree"), but is a weaker baseline than press's + * freshly-resolved one. + */ +async function settleAfterAction( + runtime: AgentDeviceRuntime, + options: CommandContext, + params: SettleParams & { baselineNodes: SnapshotNode[]; actionPoint?: Point }, ): Promise { const quietMs = params.quietMs ?? DEFAULT_STABLE_QUIET_MS; const timeoutMs = params.timeoutMs ?? DEFAULT_STABLE_TIMEOUT_MS; @@ -72,44 +122,7 @@ export async function settleAfterInteraction( timeoutMs, resetBudgetOnPrivateAxRecovery: true, }); - const observation: SettleObservation = { - ...base, - settled: outcome.settled, - waitedMs: outcome.waitedMs, - captures: outcome.captures, - }; - if (!outcome.lastCapture) { - return { - observation: { - ...observation, - hint: outcome.stalled ? SETTLE_CAPTURE_STALLED_HINT : NEVER_SETTLED_HINT, - }, - }; - } - const { stored, session } = await storeSettledSnapshot(runtime, options, outcome.lastCapture); - const settledNodes = outcome.lastCapture.snapshot.nodes; - return { - observation: { - ...observation, - // The diff (with its added-line refs) is only attached when the settled - // tree actually became the stored session snapshot: those refs must be - // valid against the tree the next @ref command resolves on. The daemon - // treats `diff` presence as "this response issues refs". Unsettled - // captures are intentionally diff-less: they are not a stable - // observation, so surfacing refs would invite agents to act on - // advisory state. - ...(outcome.settled && stored - ? buildSettleDiffAndTail( - resolveBaselineNodes(params.resolved), - settledNodes, - params.resolved.point, - session?.appBundleId, - ) - : {}), - ...resolveSettleHint(outcome, stored, settledNodes.length), - }, - settledNodes, - }; + return await buildSettleOutcome(runtime, options, { ...params, base, outcome }); } catch (error) { // Never fail the action over the observation: report that settling itself // broke and let the caller fall back to an explicit snapshot. @@ -122,6 +135,60 @@ export async function settleAfterInteraction( } } +type StableCaptureOutcome = Awaited>; + +/** Turns a completed stable-capture loop into the settle observation it reports. */ +async function buildSettleOutcome( + runtime: AgentDeviceRuntime, + options: CommandContext, + params: { + baselineNodes: SnapshotNode[]; + actionPoint?: Point; + base: SettleObservation; + outcome: StableCaptureOutcome; + }, +): Promise { + const { base, outcome } = params; + const observation: SettleObservation = { + ...base, + settled: outcome.settled, + waitedMs: outcome.waitedMs, + captures: outcome.captures, + }; + if (!outcome.lastCapture) { + return { + observation: { + ...observation, + hint: outcome.stalled ? SETTLE_CAPTURE_STALLED_HINT : NEVER_SETTLED_HINT, + }, + }; + } + const { stored, session } = await storeSettledSnapshot(runtime, options, outcome.lastCapture); + const settledNodes = outcome.lastCapture.snapshot.nodes; + return { + observation: { + ...observation, + // The diff (with its added-line refs) is only attached when the settled + // tree actually became the stored session snapshot: those refs must be + // valid against the tree the next @ref command resolves on. The daemon + // treats `diff` presence as "this response issues refs". Unsettled + // captures are intentionally diff-less: they are not a stable + // observation, so surfacing refs would invite agents to act on + // advisory state. + ...(outcome.settled && stored + ? buildSettleDiffAndTail( + params.baselineNodes, + settledNodes, + params.actionPoint, + session?.appBundleId, + ) + : {}), + ...resolveSettleHint(outcome, stored, settledNodes.length), + }, + settledNodes, + }; +} + /** * `--settle --verify` composition: the settle loop's final capture doubles as * the verify evidence source, so the pair costs zero extra captures. Without a diff --git a/src/commands/post-action-observation-surface.ts b/src/commands/post-action-observation-surface.ts new file mode 100644 index 0000000000..73e78ce5f1 --- /dev/null +++ b/src/commands/post-action-observation-surface.ts @@ -0,0 +1,58 @@ +import type { PostActionObservationSupportFor } from '../core/command-descriptor/post-action-observation.ts'; +import { + commandSupportsSettleObservation, + commandSupportsVerifyEvidence, +} from '../core/command-descriptor/registry.ts'; +import { SETTLE_FLAGS } from './cli-grammar/flag-groups.ts'; +import type { FlagKey } from './cli-grammar/flag-types.ts'; +import { booleanField, integerField } from './command-input.ts'; + +/** + * The command-surface projection of the descriptor post-action observation + * trait (src/core/command-descriptor/post-action-observation.ts): the CLI flags + * a settle/verify-capable command accepts, and the input fields its MCP tool + * and SDK options expose. Both derive from the trait map, so granting a command + * `--settle` is one registry edit rather than a hunt through the per-family + * grammar files. + * + * It lives outside `interaction/` because the trait is not an interaction-family + * property: `back` (src/commands/system/index.ts) carries it too (#1638). + */ + +const verifyField = () => + booleanField( + 'Capture cheap post-action evidence (AX digest, node counts, changedFromBefore) instead of a follow-up snapshot.', + ); + +const settleFields = () => ({ + settle: booleanField( + 'After the action, wait for the UI to go quiet and return the settled diff vs the pre-action tree in the same response. Best-effort; never fails the action.', + ), + settleQuietMs: integerField('Settle: quiet window in milliseconds (default 500).', { min: 0 }), + timeoutMs: integerField('Settle: wait deadline in milliseconds (default 10000).', { min: 1 }), +}); + +type VerifyFieldMap = { verify: ReturnType }; +type SettleFieldMap = ReturnType; +export type PostActionObservationFields = + PostActionObservationSupportFor extends 'settle-and-verify' + ? VerifyFieldMap & SettleFieldMap + : PostActionObservationSupportFor extends 'settle' + ? SettleFieldMap + : {}; + +export function postActionObservationFields( + command: TName, +): PostActionObservationFields { + return { + ...(commandSupportsVerifyEvidence(command) ? { verify: verifyField() } : {}), + ...(commandSupportsSettleObservation(command) ? settleFields() : {}), + } as PostActionObservationFields; +} + +export function postActionObservationCliFlags(command: string): readonly FlagKey[] { + const flags: FlagKey[] = []; + if (commandSupportsVerifyEvidence(command)) flags.push('verify'); + if (commandSupportsSettleObservation(command)) flags.push(...SETTLE_FLAGS); + return flags; +} diff --git a/src/commands/settle-output.ts b/src/commands/settle-output.ts new file mode 100644 index 0000000000..cfe04048da --- /dev/null +++ b/src/commands/settle-output.ts @@ -0,0 +1,95 @@ +/** + * CLI rendering for the `--settle` (#1101) observation. Every command that can + * attach one renders it identically — the touch commands, and the generic-route + * `scroll`/`back` (#1638) — so the verdict/diff/tail shape an agent learns from + * one command is the shape it gets from all of them. + */ + +// ADR 0014: a reusable ref in a PARTIAL result renders in ready-to-copy +// `@eN~s` form so a human CLI caller can paste it into the next +// mutation without a separate pin step. A mutating result carries no +// `refsGeneration`, so its acted ref is never pinned. +export function pinnedRefText(ref: unknown, refsGeneration: unknown): string | undefined { + if (typeof ref !== 'string' || ref.length === 0) return undefined; + if (typeof refsGeneration !== 'number') return undefined; + const body = ref.startsWith('@') ? ref.slice(1) : ref; + return `@${body}~s${refsGeneration}`; +} + +type SettleTextView = { + settled?: boolean; + waitedMs?: number; + hint?: string; + diff?: { + summary?: { additions?: number; removals?: number; unchanged?: number }; + lines?: Array<{ kind?: string; text?: string }>; + truncated?: boolean; + }; + tail?: Array<{ ref?: string; role?: string; label?: string }>; + tailTruncated?: boolean; + refsGeneration?: number; +}; + +/** + * Appends the response's advisory notes — the warning line, then the compact + * settle rendering — to a command's own success text. + */ +export function appendResponseNotes( + text: string | null | undefined, + data: Record, +): string { + const warning = typeof data.warning === 'string' ? `\nWarning: ${data.warning}` : ''; + return `${text ?? ''}${warning}${formatSettleText(data.settle)}`; +} + +/** + * Compact `--settle` (#1101) rendering appended to the action line: the verdict, + * the changed-count summary, and the changed lines themselves (the payload the + * agent acts on). Empty for non-settle responses. + */ +function formatSettleText(settle: unknown): string { + if (!settle || typeof settle !== 'object') return ''; + const view = settle as SettleTextView; + const parts = [ + formatSettleVerdict(view), + ...formatSettleDiffLines(view.diff), + ...formatSettleTailLines(view), + ]; + if (view.hint) parts.push(`hint: ${view.hint}`); + return `\n${parts.join('\n')}`; +} + +function formatSettleDiffLines(diff: SettleTextView['diff']): string[] { + const lines = (diff?.lines ?? []).map( + (line) => `${line.kind === 'removed' ? '-' : '+'} ${line.text ?? ''}`, + ); + if (diff?.truncated) lines.push('… changed lines truncated'); + return lines; +} + +// Unchanged interactive tail: only present when the diff's added lines +// carried zero refs (modal-dismiss/toast-only diff), so the settled tree's +// remaining actionable elements would otherwise be invisible. +function formatSettleTailLines(view: SettleTextView): string[] { + const tail = view.tail ?? []; + if (tail.length === 0) return []; + const lines = [`unchanged interactive (${tail.length}):`]; + for (const entry of tail) { + const label = entry.label ? ` "${entry.label}"` : ''; + // ADR 0014: the settled tail refs are reusable, so render them pinned when + // the settle response carried its generation. + const ref = pinnedRefText(entry.ref, view.refsGeneration) ?? `@${entry.ref ?? ''}`; + lines.push(`= ${ref} [${entry.role ?? ''}]${label}`); + } + if (view.tailTruncated) { + lines.push('… more interactive elements not shown, use snapshot -i'); + } + return lines; +} + +function formatSettleVerdict(view: SettleTextView): string { + const verdict = view.settled === true ? 'settled' : 'not settled'; + const summary = view.diff?.summary; + if (!summary) return `${verdict} after ${view.waitedMs ?? 0}ms`; + return `${verdict} after ${view.waitedMs ?? 0}ms: +${summary.additions ?? 0} -${summary.removals ?? 0} (~${summary.unchanged ?? 0} unchanged)`; +} diff --git a/src/commands/system/index.test.ts b/src/commands/system/index.test.ts index 14cc3b7cc7..7f3a1160d1 100644 --- a/src/commands/system/index.test.ts +++ b/src/commands/system/index.test.ts @@ -27,6 +27,7 @@ import { tvRemoteDaemonWriter, systemCommandFamily, } from './index.ts'; +import { systemCliOutputFormatters } from './output.ts'; function flags(overrides: Partial = {}): CliFlags { return overrides as CliFlags; @@ -118,6 +119,50 @@ describe('system command interface', () => { ).toBeUndefined(); }); + // #1638: --settle has to survive BOTH back seams — the reader that builds the + // input and the writer that turns it into daemon request options. + test('back reader and writer carry the settle request through to daemon flags', () => { + const input = backCliReader([], flags({ settle: true, settleQuietMs: 250, timeoutMs: 8_000 })); + expect(input).toMatchObject({ settle: true, settleQuietMs: 250, timeoutMs: 8_000 }); + expect(backDaemonWriter(input).options).toMatchObject({ + settle: true, + settleQuietMs: 250, + timeoutMs: 8_000, + }); + expect(backCliReader([], flags()).settle).toBeUndefined(); + }); + + test('back CLI output renders the settled observation', () => { + const output = systemCliOutputFormatters.back({ + input: {}, + result: { + action: 'back', + mode: 'in-app', + message: 'Back', + settle: { + settled: true, + waitedMs: 300, + diff: { + summary: { additions: 1, removals: 2, unchanged: 5 }, + lines: [ + { kind: 'removed', text: '@e9 [button] "Save"' }, + { kind: 'added', text: '@e3 [button] "Edit"', ref: 'e3' }, + ], + }, + }, + }, + }); + + expect(output.text).toBe( + [ + 'Back', + 'settled after 300ms: +1 -2 (~5 unchanged)', + '- @e9 [button] "Save"', + '+ @e3 [button] "Edit"', + ].join('\n'), + ); + }); + test('orientation reader and writer normalize orientation', () => { expect(orientationCliReader(['left'], flags())).toMatchObject({ orientation: 'landscape-left', diff --git a/src/commands/system/index.ts b/src/commands/system/index.ts index 29bfa38f62..a920185430 100644 --- a/src/commands/system/index.ts +++ b/src/commands/system/index.ts @@ -16,6 +16,7 @@ import { optionalString, request, requiredDaemonString, + settleInputFromFlags, } from '../cli-grammar/common.ts'; import type { CliReader, DaemonWriter } from '../cli-grammar/types.ts'; import { defineExecutableCommand } from '../command-contract.ts'; @@ -32,6 +33,10 @@ import { projectCommandOutputSchemas, } from '../family/types.ts'; import { defineFieldCommandMetadata } from '../field-command-contract.ts'; +import { + postActionObservationCliFlags, + postActionObservationFields, +} from '../post-action-observation-surface.ts'; import { NAVIGATION_COMMAND_PROJECTIONS } from './navigation-projection.ts'; import { systemCliOutputFormatters } from './output.ts'; @@ -71,6 +76,7 @@ const appStateCommandMetadata = defineFieldCommandMetadata( const backCommandMetadata = defineFieldCommandMetadata(BACK_COMMAND_NAME, backCommandDescription, { mode: enumField(BACK_MODES), + ...postActionObservationFields(BACK_COMMAND_NAME), }); const homeCommandMetadata = defineFieldCommandMetadata( @@ -172,8 +178,8 @@ const tvRemoteCommandDefinition = defineExecutableCommand( const appStateCliSchema = {} as const satisfies CommandSchemaOverride; const backCliSchema = { - usageOverride: 'back [--in-app|--system]', - allowedFlags: ['backMode'], + usageOverride: 'back [--in-app|--system] [--settle]', + allowedFlags: ['backMode', ...postActionObservationCliFlags(BACK_COMMAND_NAME)], } as const satisfies CommandSchemaOverride; const homeCliSchema = {} as const satisfies CommandSchemaOverride; @@ -210,6 +216,7 @@ export const appSwitcherCliReader: CliReader = (_positionals, flags) => commonIn export const backCliReader: CliReader = (_positionals, flags) => ({ ...commonInputFromFlags(flags), + ...settleInputFromFlags(flags), mode: flags.backMode, }); diff --git a/src/commands/system/navigation-projection.ts b/src/commands/system/navigation-projection.ts index 9102093b17..5de98e7e4f 100644 --- a/src/commands/system/navigation-projection.ts +++ b/src/commands/system/navigation-projection.ts @@ -1,3 +1,4 @@ +import type { SettleCommandOptions } from '@agent-device/contracts/client'; import { DEVICE_ROTATIONS, type DeviceRotation } from '@agent-device/contracts/device'; import { BACK_MODES, @@ -39,7 +40,16 @@ function defineNavigationCommandProjection< } export const NAVIGATION_COMMAND_PROJECTIONS = { - back: defineNavigationCommandProjection<{ mode?: BackMode }, BackCommandResult, false, 'back'>({ + // #1638: `back` carries the descriptor post-action observation trait, so its + // options accept `--settle`. The settled observation itself rides the payload + // additively — these schemas are never strict, exactly so an opt-in field can + // (like `cost`) validate without every projection re-declaring it. + back: defineNavigationCommandProjection< + { mode?: BackMode } & SettleCommandOptions, + BackCommandResult, + false, + 'back' + >({ clientMethod: 'back', outputSchema: { type: 'object', diff --git a/src/commands/system/output.ts b/src/commands/system/output.ts index 6ba5306409..79731e83cd 100644 --- a/src/commands/system/output.ts +++ b/src/commands/system/output.ts @@ -10,6 +10,7 @@ import { resultOutput, type CliOutputFormatter, } from '../output-common.ts'; +import { appendResponseNotes } from '../settle-output.ts'; function appStateCliOutput(result: AppStateCommandResult): CliOutput { return { @@ -40,9 +41,16 @@ function clipboardCliOutput(result: ClipboardCommandResult): CliOutput { return messageCliOutput(result); } +// #1638: back takes --settle on the generic route, so its line renders the +// settled diff exactly like the touch commands. +function backCliOutput(result: Record): CliOutput { + const output = messageCliOutput(result); + return { data: output.data, text: appendResponseNotes(output.text, result) }; +} + export const systemCliOutputFormatters = { appstate: resultOutput(appStateCliOutput), - back: messageOutput, + back: resultOutput(backCliOutput), home: messageOutput, orientation: messageOutput, 'app-switcher': messageOutput, diff --git a/src/core/command-descriptor/__tests__/post-action-observation.test.ts b/src/core/command-descriptor/__tests__/post-action-observation.test.ts index ee0521204c..60c2bf511d 100644 --- a/src/core/command-descriptor/__tests__/post-action-observation.test.ts +++ b/src/core/command-descriptor/__tests__/post-action-observation.test.ts @@ -11,10 +11,12 @@ import { } from '../registry.ts'; const SETTLE_OBSERVATION_COMMANDS = [ + PUBLIC_COMMANDS.back, PUBLIC_COMMANDS.click, PUBLIC_COMMANDS.fill, PUBLIC_COMMANDS.longPress, PUBLIC_COMMANDS.press, + PUBLIC_COMMANDS.scroll, ] as const; test('post-action observation descriptor traits are the source for settle command support', () => { @@ -31,6 +33,12 @@ test('post-action observation descriptor traits are the source for settle comman assert.equal(resolveCommandPostActionObservationSupport('fill'), 'settle-and-verify'); assert.equal(resolveCommandPostActionObservationSupport('longpress'), 'settle'); assert.equal(commandSupportsVerifyEvidence('longpress'), false); + // #1638: the generic-route pair resolves no target, so there is no pre-action + // node to digest into evidence — settle without verify. + assert.equal(resolveCommandPostActionObservationSupport('scroll'), 'settle'); + assert.equal(commandSupportsVerifyEvidence('scroll'), false); + assert.equal(resolveCommandPostActionObservationSupport('back'), 'settle'); + assert.equal(commandSupportsVerifyEvidence('back'), false); }); test('post-action observation CLI flags follow descriptor traits', () => { diff --git a/src/core/command-descriptor/__tests__/timeout-policy.test.ts b/src/core/command-descriptor/__tests__/timeout-policy.test.ts index 3d9bba776d..959f48aa7c 100644 --- a/src/core/command-descriptor/__tests__/timeout-policy.test.ts +++ b/src/core/command-descriptor/__tests__/timeout-policy.test.ts @@ -68,10 +68,14 @@ test('daemon-preserving timeout commands are a bounded, reviewed set', () => { // Interaction commands joined in #1105: their target resolution runs the // same capture as snapshot, and resetting the daemon on a wedged capture // destroyed healthy app sessions. + // `back`/`scroll` joined in #1638: `--settle` gives them the same capture + // hang mode, and the policy cannot be flag-conditional — preserving is the + // safe side of that choice (a reset loses every session on the daemon). const preserving = commandDescriptors .filter((descriptor) => descriptor.timeoutPolicy.onTimeout === 'preserve-daemon') .map((descriptor) => descriptor.name); assert.deepEqual(preserving.sort(), [ + 'back', 'click', 'fill', 'find', @@ -79,6 +83,7 @@ test('daemon-preserving timeout commands are a bounded, reviewed set', () => { 'is', 'longpress', 'press', + 'scroll', 'snapshot', 'type', 'wait', diff --git a/src/core/command-descriptor/post-action-observation.ts b/src/core/command-descriptor/post-action-observation.ts index 355d9a7415..48d2598c94 100644 --- a/src/core/command-descriptor/post-action-observation.ts +++ b/src/core/command-descriptor/post-action-observation.ts @@ -1,10 +1,23 @@ export type PostActionObservationSupport = 'settle' | 'settle-and-verify'; +/** + * The single source for which commands accept `--settle`/`--verify` (#1101, + * #1047). Every downstream seam — CLI flag grammar, MCP/SDK input fields, + * daemon flag guards, timeout policy, dispatch wiring — derives its answer + * from this map rather than repeating a command list. + * + * `scroll` and `back` (#1638) carry `settle` only: they resolve no target, so + * there is no pre-action node to digest into `--verify` evidence. Their + * settled diff is taken against the STORED pre-action snapshot, which the + * generic route holds instead of a freshly resolved baseline. + */ const POST_ACTION_OBSERVATION_BY_COMMAND = { click: 'settle-and-verify', press: 'settle-and-verify', fill: 'settle-and-verify', longpress: 'settle', + scroll: 'settle', + back: 'settle', } as const satisfies Record; export type PostActionObservationCommandName = keyof typeof POST_ACTION_OBSERVATION_BY_COMMAND; diff --git a/src/core/command-descriptor/registry.ts b/src/core/command-descriptor/registry.ts index 1889db6636..7bfe70fc67 100644 --- a/src/core/command-descriptor/registry.ts +++ b/src/core/command-descriptor/registry.ts @@ -287,10 +287,24 @@ const FILL_INTERACTION_RESPONSE_DATA_TRANSFORM = { }, } as const satisfies CommandResponseDataTransform; -function interactionTimeoutPolicy(command: string): CommandTimeoutPolicy { +/** + * Timeout policy for a command that MAY carry the post-action observation + * trait: the settle policy when the observation map lists it, the caller's + * baseline otherwise. Keeping the map the enumerator means adding `--settle` + * to a command cannot leave its request envelope too narrow for the settle + * wait it now accepts. + */ +function postActionObservationTimeoutPolicy( + command: string, + withoutObservation: CommandTimeoutPolicy, +): CommandTimeoutPolicy { return resolvePostActionObservationSupport(command) !== undefined ? SETTLE_FLAG_PRESERVE_DAEMON_TIMEOUT_POLICY - : PRESERVE_DAEMON_TIMEOUT_POLICY; + : withoutObservation; +} + +function interactionTimeoutPolicy(command: string): CommandTimeoutPolicy { + return postActionObservationTimeoutPolicy(command, PRESERVE_DAEMON_TIMEOUT_POLICY); } function postActionObservation(command: string): PostActionObservationSupport { @@ -1030,6 +1044,14 @@ export const RAW_COMMAND_DESCRIPTORS = [ ...GENERIC_MUTATING_LINUX_DEVICE_COMMAND_TRAITS.capability, vega: VEGA_VVD, }, + // #1638: back-then-observe in one round trip. The settle wait runs on the + // generic route (src/daemon/request-generic-dispatch.ts), after Android's + // blocking-dialog postflight. + timeoutPolicy: postActionObservationTimeoutPolicy( + 'back', + GENERIC_MUTATING_LINUX_DEVICE_COMMAND_TRAITS.timeoutPolicy, + ), + postActionObservation: postActionObservation('back'), }, { name: 'gesture', @@ -1103,6 +1125,12 @@ export const RAW_COMMAND_DESCRIPTORS = [ ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/interaction/index.ts'] as const } : {}), catalog: { group: 'public' }, ...GENERIC_MUTATING_LINUX_DEVICE_COMMAND_TRAITS, + // #1638: scroll-then-observe in one round trip; see `back` above. + timeoutPolicy: postActionObservationTimeoutPolicy( + 'scroll', + GENERIC_MUTATING_LINUX_DEVICE_COMMAND_TRAITS.timeoutPolicy, + ), + postActionObservation: postActionObservation('scroll'), }, { name: 'swipe', diff --git a/src/daemon/__tests__/generic-settle.test.ts b/src/daemon/__tests__/generic-settle.test.ts new file mode 100644 index 0000000000..484ce9fe04 --- /dev/null +++ b/src/daemon/__tests__/generic-settle.test.ts @@ -0,0 +1,306 @@ +import type { CommandFlags } from '@agent-device/contracts/command'; +import type { SnapshotBackend } from '@agent-device/kernel/snapshot'; +import { beforeEach, expect, test, vi } from 'vitest'; +import { makeIosSession } from '../../__tests__/test-utils/session-factories.ts'; +import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; +import type { SessionStore } from '../session-store.ts'; +import type { DaemonResponse, SessionState } from '../types.ts'; +import { activateCompleteRefFrame } from '../ref-frame.ts'; +import { isPostGestureStabilizationPending } from '../deferred-interaction-outcome.ts'; +import { buildSnapshotState } from '../handlers/snapshot-capture.ts'; +import { setSessionSnapshot } from '../session-snapshot.ts'; + +// #1638 `--settle` on the generic route (scroll/back): the settled diff is +// taken against the session's STORED pre-action tree, rides the response as +// `data.settle`, and is ref-issuing exactly like the touch route. Quiet windows +// are tuned down (settleQuietMs 25) so no test waits real time. + +vi.mock('../../core/dispatch.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, dispatchCommand: vi.fn(async () => ({})) }; +}); + +vi.mock('../handlers/interaction-snapshot.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, captureSnapshotForSession: vi.fn() }; +}); + +import { dispatchCommand } from '../../core/dispatch.ts'; +import { captureSnapshotForSession } from '../handlers/interaction-snapshot.ts'; +import { dispatchGenericCommand } from '../request-generic-dispatch.ts'; + +const mockDispatch = vi.mocked(dispatchCommand); +const mockCapture = vi.mocked(captureSnapshotForSession); + +const BEFORE_NODES = [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Load more', + rect: { x: 10, y: 700, width: 120, height: 44 }, + hittable: true, + }, +]; + +const AFTER_NODES = [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Next page', + rect: { x: 10, y: 700, width: 120, height: 44 }, + hittable: true, + }, +]; + +type SettlePayload = { + settled: boolean; + captures: number; + quietMs: number; + timeoutMs: number; + refsGeneration?: number; + diff?: { + summary: { additions: number; removals: number; unchanged: number }; + lines: Array<{ kind: string; text: string; ref?: string }>; + }; + hint?: string; +}; + +const contextFromFlags = () => ({}) as never; + +/** Every settle capture observes the post-action tree. */ +function captureNodes(nodes: typeof BEFORE_NODES, onCapture?: (session: SessionState) => void) { + mockCapture.mockImplementation(async (session, flags, sessionStore) => { + onCapture?.(session); + const snapshot = buildSnapshotState( + { nodes, backend: 'xctest' as SnapshotBackend }, + (flags ?? {}) as CommandFlags, + ); + setSessionSnapshot(session, snapshot); + sessionStore.set(session.name, session); + return snapshot; + }); +} + +function seedSession(sessionName: string, sessionStore: SessionStore): SessionState { + const session = makeIosSession(sessionName); + setSessionSnapshot(session, buildSnapshotState({ nodes: BEFORE_NODES, backend: 'xctest' }, {})); + // The seed emulates a snapshot response that issued these refs (ADR 0014). + activateCompleteRefFrame(session); + sessionStore.set(sessionName, session); + return session; +} + +async function runGeneric(params: { + sessionName: string; + sessionStore: SessionStore; + command: string; + positionals?: string[]; + flags?: CommandFlags; +}): Promise { + const session = params.sessionStore.get(params.sessionName) as SessionState; + return await dispatchGenericCommand({ + req: { + token: 't', + session: params.sessionName, + command: params.command, + positionals: params.positionals ?? [], + ...(params.flags ? { flags: params.flags } : {}), + }, + session, + sessionName: params.sessionName, + logPath: '', + sessionStore: params.sessionStore, + contextFromFlags, + }); +} + +function expectOkData(response: DaemonResponse): Record { + expect(response.ok).toBe(true); + if (!response.ok) throw new Error('expected an ok daemon response'); + return (response.data ?? {}) as Record; +} + +const SETTLE_FLAGS = { settle: true, settleQuietMs: 25, timeoutMs: 2_000 } satisfies CommandFlags; + +beforeEach(() => { + mockDispatch.mockReset(); + mockDispatch.mockResolvedValue({}); + mockCapture.mockReset(); +}); + +test('scroll --settle diffs the settled tree against the stored pre-action snapshot', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-scroll'; + seedSession(sessionName, sessionStore); + captureNodes(AFTER_NODES); + + const response = await runGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + flags: { ...SETTLE_FLAGS }, + }); + + const data = expectOkData(response); + const settle = data.settle as SettlePayload; + expect(settle.settled).toBe(true); + expect(settle.quietMs).toBe(25); + expect(settle.timeoutMs).toBe(2_000); + // Baseline is the STORED pre-action tree, not a fresh resolution capture. + expect(settle.diff?.summary).toEqual({ additions: 1, removals: 1, unchanged: 1 }); + expect(settle.diff?.lines.find((line) => line.kind === 'added')).toEqual({ + kind: 'added', + text: expect.stringContaining('Next page'), + ref: 'e2', + }); + // The device action itself still ran, once. + expect(mockDispatch.mock.calls.filter(([, command]) => command === 'scroll')).toHaveLength(1); +}); + +test('scroll --settle is ref-issuing: partial frame plus the stored tree generation', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-refs'; + seedSession(sessionName, sessionStore); + captureNodes(AFTER_NODES); + + const response = await runGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + flags: { ...SETTLE_FLAGS }, + }); + + const settle = expectOkData(response).settle as SettlePayload; + const session = sessionStore.get(sessionName) as SessionState; + expect(session.refFrameState).toBe('active'); + expect(settle.refsGeneration).toBe(session.snapshotGeneration); + expect(session.snapshot?.nodes.some((node) => node.label === 'Next page')).toBe(true); +}); + +test('back --settle carries the settled observation on the navigation payload', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-back'; + seedSession(sessionName, sessionStore); + mockDispatch.mockImplementation(async (_device, command) => + command === 'back' ? { action: 'back', mode: 'in-app', message: 'Back' } : {}, + ); + captureNodes(AFTER_NODES); + + const data = expectOkData( + await runGeneric({ + sessionName, + sessionStore, + command: 'back', + flags: { ...SETTLE_FLAGS, backMode: 'in-app' }, + }), + ); + + // The command's own payload survives alongside the additive observation. + expect(data.action).toBe('back'); + expect(data.message).toBe('Back'); + expect((data.settle as SettlePayload).settled).toBe(true); +}); + +test('the settle capture runs after the post-gesture stabilization marker is placed', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-order'; + seedSession(sessionName, sessionStore); + // #1542 ordering: settle's FIRST capture must be the one that folds the + // pending stabilization, so the marker has to exist by the time it runs. + const pendingAtCapture: boolean[] = []; + captureNodes(AFTER_NODES, (session) => { + pendingAtCapture.push(isPostGestureStabilizationPending(session)); + }); + + await runGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + flags: { ...SETTLE_FLAGS }, + }); + + expect(pendingAtCapture[0]).toBe(true); +}); + +test('scroll without --settle observes nothing', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-no-settle'; + seedSession(sessionName, sessionStore); + captureNodes(AFTER_NODES); + + const data = expectOkData( + await runGeneric({ sessionName, sessionStore, command: 'scroll', positionals: ['down'] }), + ); + + expect(data.settle).toBeUndefined(); + expect(mockCapture).not.toHaveBeenCalled(); +}); + +test('a generic command without the observation trait ignores a stray settle flag', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-untraited'; + seedSession(sessionName, sessionStore); + captureNodes(AFTER_NODES); + + // `home` is a generic leaf with no post-action observation trait. The CLI + // schema already refuses --settle for it; the daemon must not observe either. + const data = expectOkData( + await runGeneric({ sessionName, sessionStore, command: 'home', flags: { ...SETTLE_FLAGS } }), + ); + + expect(data.settle).toBeUndefined(); + expect(mockCapture).not.toHaveBeenCalled(); +}); + +test('scroll --settle-quiet without --settle is rejected before the device action', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-orphan'; + seedSession(sessionName, sessionStore); + + const response = await runGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + flags: { settleQuietMs: 25 }, + }); + + expect(response.ok).toBe(false); + if (response.ok) throw new Error('expected a rejected daemon response'); + expect(response.error.code).toBe('INVALID_ARGS'); + expect(response.error.message).toMatch(/--settle-quiet requires --settle/); + expect(mockDispatch).not.toHaveBeenCalled(); +}); + +test('a failed settle observation never fails the action', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-broken'; + seedSession(sessionName, sessionStore); + mockCapture.mockRejectedValue(new Error('AX bridge crashed')); + + const data = expectOkData( + await runGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + flags: { ...SETTLE_FLAGS }, + }), + ); + + const settle = data.settle as SettlePayload; + expect(settle.settled).toBe(false); + expect(settle.diff).toBeUndefined(); + expect(settle.hint).toMatch(/Settle observation unavailable/); + // The scroll itself was dispatched and reported success. + expect(mockDispatch.mock.calls.filter(([, command]) => command === 'scroll')).toHaveLength(1); + // Nothing was published, so the scroll's own leaf seam still owns the frame. + expect(sessionStore.get(sessionName)?.refFrameState).toBe('expired'); +}); diff --git a/src/daemon/generic-settle-observation.ts b/src/daemon/generic-settle-observation.ts new file mode 100644 index 0000000000..6d8cc7b89b --- /dev/null +++ b/src/daemon/generic-settle-observation.ts @@ -0,0 +1,85 @@ +import type { CommandFlags } from '@agent-device/contracts/command'; +import type { SettleObservation, SettleParams } from '@agent-device/contracts/interaction'; +import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { PostActionObservationCommandName } from '../core/command-descriptor/post-action-observation.ts'; +import type { ContextFromFlags } from './handlers/interaction-common.ts'; +import { readSettleRequest, settleFlagGuardResponse } from './handlers/interaction-flags.ts'; +import { createInteractionRuntime } from './handlers/interaction-runtime.ts'; +import { captureSnapshotForSession } from './handlers/interaction-snapshot.ts'; +import { issueSettleObservationRefs } from './session-snapshot.ts'; +import type { SessionStore } from './session-store.ts'; +import type { DaemonRequest, DaemonResponse, SessionState } from './types.ts'; + +/** + * `--settle` (#1101) for the generic daemon route (#1638) — today `scroll` and + * `back`, the two generic leaves carrying the descriptor post-action + * observation trait. + * + * It lives in its own module, loaded lazily by request-generic-dispatch.ts only + * when a settle-capable command actually asks to settle, for the same reason + * every daemon ROUTE is a lazy seam (src/daemon/request-handler-chain.ts): + * observing pulls in the whole interaction runtime, and no other generic leaf — + * home, screenshot, orientation, tv-remote — should pay for it in import cost + * or in comprehension (the static edge would fold that runtime cluster into the + * daemon's largest type cycle). + */ + +type GenericSettleReadiness = { settle: SettleParams | undefined } | { response: DaemonResponse }; + +/** + * Reads the settle request off the flags, refusing an orphaned + * `--settle-quiet`. Called before the device action so a grammar mistake costs + * no device work — and only for a command the caller already checked carries + * the post-action observation trait, which is what makes the name cast sound. + */ +export function readGenericSettleRequest( + command: string, + flags: CommandFlags | undefined, +): GenericSettleReadiness { + const invalid = settleFlagGuardResponse(command as PostActionObservationCommandName, flags); + if (invalid) return { response: invalid }; + return { settle: readSettleRequest(flags) }; +} + +/** + * Runs the target-less settle through the interaction runtime and attaches the + * observation to the command's response payload. + * + * Best-effort by contract (the settle engine never throws): the device action + * already succeeded, so a broken observation degrades to a hint rather than + * failing the command. + */ +export async function observeGenericSettle(params: { + req: DaemonRequest; + session: SessionState; + sessionName: string; + logPath: string; + sessionStore: SessionStore; + contextFromFlags: ContextFromFlags; + settle: SettleParams; + /** The session's STORED pre-action tree, read before the action dispatched. */ + baselineNodes: SnapshotNode[]; +}): Promise<{ settle: SettleObservation }> { + const { req, session, sessionName, settle, baselineNodes } = params; + const runtime = createInteractionRuntime({ + req, + sessionName, + logPath: params.logPath, + sessionStore: params.sessionStore, + contextFromFlags: params.contextFromFlags, + captureSnapshotForSession, + }); + const observation = await runtime.interactions.observeSettle({ + ...settle, + session: sessionName, + requestId: req.meta?.requestId, + baselineNodes, + }); + // ADR 0014: a settled diff publishes refs minted from the stored settled + // tree, so the response activates a partial frame and carries the generation + // those refs are pinned to — the same dance the touch route runs. + const refsGeneration = issueSettleObservationRefs(session, observation); + return { + settle: refsGeneration === undefined ? observation : { ...observation, refsGeneration }, + }; +} diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index fa530dd8ca..9a91fdf5c1 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -38,7 +38,7 @@ import { type DirectIosSelectorTarget, } from '../direct-ios-selector.ts'; import { expireRefFrame } from '../ref-frame.ts'; -import { markSessionPartialRefsIssued, resolveRefStalenessWarning } from '../session-snapshot.ts'; +import { issueSettleObservationRefs, resolveRefStalenessWarning } from '../session-snapshot.ts'; import type { DaemonResponse, SessionState } from '../types.ts'; import { assertAndroidPressStayedInApp, @@ -326,41 +326,10 @@ async function buildTargetedTouchResponsePayloads(params: { referenceFrame, extra, staleRefsWarning: params.staleRefsWarning, - settleRefsGeneration: settleRefsGenerationIssue(session, result), + settleRefsGeneration: issueSettleObservationRefs(session, result.settle), }); } -/** - * #1101 `--settle`: a settle observation carrying a diff hands the client refs - * minted from the freshly stored settled tree (added lines carry them), which - * makes the response ref-issuing like snapshot/find: it activates a PARTIAL - * frame (ADR 0014) authorizing exactly those bodies, and the stored tree's - * generation rides inside the settle payload for MCP per-ref pinning. Without a - * diff — never captured, or sparse-quality capture that was not stored — nothing - * was issued and the frame is left as the press's leaf seam expired it. - */ -function settleRefsGenerationIssue( - session: SessionState, - result: PressCommandResult | FillCommandResult | LongPressCommandResult, -): number | undefined { - if (!result.settle?.diff) return undefined; - // ADR 0014: a settled diff publishes the refs it exposed, so it activates a - // PARTIAL frame authorizing exactly those bodies (not the whole tree). - markSessionPartialRefsIssued(session, collectSettleIssuedRefBodies(result.settle)); - return session.snapshotGeneration; -} - -/** The reusable refs a settled diff exposed: added diff lines, `refs`, `tail`. */ -function collectSettleIssuedRefBodies(settle: NonNullable): string[] { - const bodies: string[] = []; - for (const line of settle.diff?.lines ?? []) { - if (line.ref) bodies.push(line.ref); - } - for (const entry of settle.refs ?? []) bodies.push(entry.ref); - for (const entry of settle.tail ?? []) bodies.push(entry.ref); - return bodies; -} - function readLongPressResultDuration(result: TargetedTouchResult): number | undefined { return 'durationMs' in result ? result.durationMs : undefined; } @@ -787,7 +756,7 @@ function buildFillResponsePayloads(params: { referenceFrame, extra: { text: params.text, ...maestroFallback.extra }, staleRefsWarning: params.staleRefsWarning, - settleRefsGeneration: settleRefsGenerationIssue(session, result), + settleRefsGeneration: issueSettleObservationRefs(session, result.settle), }); } diff --git a/src/daemon/request-generic-dispatch.ts b/src/daemon/request-generic-dispatch.ts index 828aad2c9a..f53f2f262b 100644 --- a/src/daemon/request-generic-dispatch.ts +++ b/src/daemon/request-generic-dispatch.ts @@ -1,5 +1,7 @@ import type { CommandFlags } from '@agent-device/contracts/command'; +import type { SettleObservation, SettleParams } from '@agent-device/contracts/interaction'; import { dispatchCommand } from '../core/dispatch.ts'; +import { commandSupportsSettleObservation } from '../core/command-descriptor/registry.ts'; import { requireCommandSupported } from './handlers/response.ts'; import { SessionStore } from './session-store.ts'; import type { DaemonCommandContext } from './context.ts'; @@ -51,11 +53,19 @@ export async function dispatchGenericCommand(params: { const readinessResponse = await ensureGenericCommandReady(session, platformCommand); if (readinessResponse) return readinessResponse; + const settleReadiness = await readGenericSettleRequest(platformCommand, req.flags); + if ('response' in settleReadiness) return settleReadiness.response; + const settleRequest = settleReadiness.settle; const preflightReadiness = await ensureNoAndroidBlockingDialogReady(session, platformCommand); if ('response' in preflightReadiness) return preflightReadiness.response; const { resolvedPositionals, resolvedOut, recordedPositionals, recordedFlags } = resolveCommandPositionals(req); + // #1638 `--settle` baseline: the STORED pre-action tree, captured before the + // action replaces it. Unlike press — whose baseline is the tree it just + // resolved its target against — this can be several commands old; that is the + // honest reading of the #1101 contract, and the diff says so by construction. + const preActionNodes = session.snapshot?.nodes ?? []; const actionStartedAt = Date.now(); const dispatchContext = { @@ -117,7 +127,42 @@ export async function dispatchGenericCommand(params: { flags: req.flags, }); - return { ok: true, data: data ?? {} }; + // Ordering is load-bearing: settle runs AFTER markDeferredInteractionOutcome + // so its first capture is the one that folds the #1542 post-gesture + // stabilization, and after the Android blocking-dialog postflight so a + // recovered dialog is not what the settled diff describes. + const settleData = settleRequest + ? await observeGenericSettle({ + ...params, + settle: settleRequest, + baselineNodes: preActionNodes, + }) + : undefined; + + return { ok: true, data: { ...(data ?? {}), ...(settleData ?? {}) } }; +} + +/** + * The generic route's `--settle` seam, kept behind a lazy import on purpose: + * only `scroll` and `back` carry the descriptor post-action observation trait, + * and observing pulls in the whole interaction runtime. Every other generic leaf + * answers here without loading it, and reaches the rest of dispatch with no + * settle request. + */ +async function readGenericSettleRequest( + command: string, + flags: CommandFlags | undefined, +): Promise<{ settle: SettleParams | undefined } | { response: DaemonResponse }> { + if (!commandSupportsSettleObservation(command)) return { settle: undefined }; + const settleObservation = await import('./generic-settle-observation.ts'); + return settleObservation.readGenericSettleRequest(command, flags); +} + +async function observeGenericSettle( + params: Parameters<(typeof import('./generic-settle-observation.ts'))['observeGenericSettle']>[0], +): Promise<{ settle: SettleObservation }> { + const settleObservation = await import('./generic-settle-observation.ts'); + return await settleObservation.observeGenericSettle(params); } async function ensureNoAndroidBlockingDialogReady( diff --git a/src/daemon/session-event-action.ts b/src/daemon/session-event-action.ts index 121119cf07..ea6e1be641 100644 --- a/src/daemon/session-event-action.ts +++ b/src/daemon/session-event-action.ts @@ -285,6 +285,13 @@ const COMMON_SAFE_FLAG_SPEC = { ], } as const satisfies SafeFlagSpec; +// The `--settle` (#1101) request shape, shown for every command that accepts it: +// the touch commands, and the generic-route `scroll`/`back` (#1638). +const SETTLE_SAFE_FLAGS = { + booleans: [{ source: 'settle' }], + numbers: [{ source: 'settleQuietMs' }], +} as const satisfies Required>; + const SAFE_ACTION_FLAG_SPECS: Record = { [PUBLIC_COMMANDS.open]: { booleans: [{ source: 'relaunch' }, { source: 'testIme' }], @@ -296,7 +303,8 @@ const SAFE_ACTION_FLAG_SPECS: Record = { [PUBLIC_COMMANDS.fill]: textEntrySafeFlagSpec(), [PUBLIC_COMMANDS.type]: textEntrySafeFlagSpec(), [PUBLIC_COMMANDS.scroll]: { - numbers: [{ source: 'pixels' }, { source: 'durationMs' }], + booleans: SETTLE_SAFE_FLAGS.booleans, + numbers: [{ source: 'pixels' }, { source: 'durationMs' }, ...SETTLE_SAFE_FLAGS.numbers], }, [PUBLIC_COMMANDS.gesture]: gestureSafeFlagSpec(), [PUBLIC_COMMANDS.swipe]: gestureSafeFlagSpec(), @@ -322,6 +330,7 @@ const SAFE_ACTION_FLAG_SPECS: Record = { numbers: [{ source: 'snapshotDepth', output: 'depth' }], }, [PUBLIC_COMMANDS.back]: { + ...SETTLE_SAFE_FLAGS, enums: [{ source: 'backMode', output: 'mode', values: BACK_MODES }], }, [PUBLIC_COMMANDS.record]: { @@ -353,13 +362,13 @@ function buildSafeActionFlags(action: SessionAction): Record | function touchSafeFlagSpec(): SafeFlagSpec { return { - booleans: [{ source: 'doubleTap' }, { source: 'settle' }], + booleans: [{ source: 'doubleTap' }, ...SETTLE_SAFE_FLAGS.booleans], numbers: [ { source: 'count' }, { source: 'intervalMs' }, { source: 'holdMs' }, { source: 'jitterPx' }, - { source: 'settleQuietMs' }, + ...SETTLE_SAFE_FLAGS.numbers, ], enums: [{ source: 'clickButton', values: CLICK_BUTTONS }], }; @@ -367,8 +376,8 @@ function touchSafeFlagSpec(): SafeFlagSpec { function textEntrySafeFlagSpec(): SafeFlagSpec { return { - booleans: [{ source: 'settle' }], - numbers: [{ source: 'delayMs' }, { source: 'settleQuietMs' }], + booleans: SETTLE_SAFE_FLAGS.booleans, + numbers: [{ source: 'delayMs' }, ...SETTLE_SAFE_FLAGS.numbers], }; } diff --git a/src/daemon/session-snapshot.ts b/src/daemon/session-snapshot.ts index 8e77f6f316..28c60564b9 100644 --- a/src/daemon/session-snapshot.ts +++ b/src/daemon/session-snapshot.ts @@ -1,4 +1,5 @@ import { randomInt } from 'node:crypto'; +import type { SettleObservation } from '@agent-device/contracts/interaction'; import type { SnapshotState } from '@agent-device/kernel/snapshot'; import { activatePartialRefFrame, refFrameEpoch, refFrameState } from './ref-frame.ts'; import type { SessionState } from './types.ts'; @@ -120,6 +121,40 @@ export function markSessionPartialRefsIssued(session: SessionState, refs: Iterab activatePartialRefFrame(session, scope); } +/** + * #1101 `--settle`: a settle observation carrying a diff hands the client refs + * minted from the freshly stored settled tree (added lines carry them), which + * makes the response ref-issuing like snapshot/find: it activates a PARTIAL + * frame (ADR 0014) authorizing exactly those bodies, and returns the stored + * tree's generation to ride inside the settle payload for MCP per-ref pinning. + * Without a diff — never captured, or a sparse-quality capture that was not + * stored — nothing was issued, so the frame is left exactly as the action's + * own side-effect seam expired it. + * + * Every route that can attach a settle observation goes through here (touch + * commands and the generic scroll/back route, #1638), so the issuance rule has + * one implementation rather than one per handler. + */ +export function issueSettleObservationRefs( + session: SessionState, + settle: SettleObservation | undefined, +): number | undefined { + if (!settle?.diff) return undefined; + markSessionPartialRefsIssued(session, collectSettleIssuedRefBodies(settle)); + return session.snapshotGeneration; +} + +/** The reusable refs a settled diff exposed: added diff lines, `refs`, `tail`. */ +function collectSettleIssuedRefBodies(settle: SettleObservation): string[] { + const bodies: string[] = []; + for (const line of settle.diff?.lines ?? []) { + if (line.ref) bodies.push(line.ref); + } + for (const entry of settle.refs ?? []) bodies.push(entry.ref); + for (const entry of settle.tail ?? []) bodies.push(entry.ref); + return bodies; +} + /** * Warning for a ref pinned to a generation (`@e12~s3`) whose epoch no longer * matches the session's current ref-frame epoch (`refFrameEpoch`) — NOT the