Skip to content

Commit 7142eab

Browse files
committed
fix: preserve gesture execution profiles
1 parent 5c52cf6 commit 7142eab

20 files changed

Lines changed: 668 additions & 25 deletions

File tree

apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -795,7 +795,7 @@ extension RunnerTests {
795795
let frame = context.referenceFrame
796796
let start = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: frame, interfaceOrientation: orientation)
797797
let end = nativeSynthesizedPoint(orientedX: x2, orientedY: y2, in: frame, interfaceOrientation: orientation)
798-
let message = if semantics == .swipe {
798+
let message = if usesFastSynthesizedDragProfile(semantics) {
799799
RunnerSynthesizedGesture.synthesizeSwipe(
800800
withApplication: app,
801801
x: Double(start.x),
@@ -834,6 +834,15 @@ extension RunnerTests {
834834
#endif
835835
}
836836

837+
func usesFastSynthesizedDragProfile(_ semantics: SynthesizedDragSemantics?) -> Bool {
838+
switch semantics {
839+
case .swipe, .fling:
840+
return true
841+
case .pan, .none:
842+
return false
843+
}
844+
}
845+
837846
func synthesizedTapAt(
838847
app: XCUIApplication,
839848
x: Double,
@@ -1428,6 +1437,13 @@ extension RunnerTests {
14281437
)
14291438
}
14301439

1440+
func testFastSynthesizedDragProfileCoversFlingAndSwipeButNotTimedPan() {
1441+
XCTAssertTrue(usesFastSynthesizedDragProfile(.swipe))
1442+
XCTAssertTrue(usesFastSynthesizedDragProfile(.fling))
1443+
XCTAssertFalse(usesFastSynthesizedDragProfile(.pan))
1444+
XCTAssertFalse(usesFastSynthesizedDragProfile(nil))
1445+
}
1446+
14311447
func testPlannedMultiTouchGestureAcceptsMatchingInBoundsTrajectories() throws {
14321448
let plan = try JSONDecoder().decode(
14331449
RunnerGesturePlan.self,

docs/adr/0013-unified-gesture-plans.md

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,9 @@ from motion:
3333

3434
`swipe` is public sugar for a fixed-duration fling. Its historical optional duration remains a
3535
thin compatibility alias to pan and reports a deprecation. The same rule applies to the historical
36-
fling duration. Pinch fixes translation and rotation at zero; rotate fixes translation at zero and
36+
fling duration. Normalization also preserves an internal single-pointer execution profile so those
37+
aliases retain their established swipe motion while a genuine timed pan remains continuous. Pinch
38+
fixes translation and rotation at zero; rotate fixes translation at zero and
3739
scale at one; two-finger pan fixes scale at one and rotation at zero; transform can apply all three
3840
components atomically. Intent remains on the plan even when aliases share an executor.
3941

@@ -59,9 +61,11 @@ Platform adapters consume the canonical plan:
5961
two-contact plan never falls back to `adb input swipe`; issue #690 separately owns removal of the
6062
existing one-contact fallback. The snapshot helper is stopped before local gesture
6163
instrumentation because Android permits only one instrumentation owner of `UiAutomation`.
62-
- iOS converts every planned point to native orientation and feeds the exact arrays to the existing
63-
private XCTest event bridge. macOS lowers a one-contact plan to its drag executor and tvOS lowers
64-
it to remote direction; multi-touch remains capability-gated to iOS simulators.
64+
- iOS lowers one-contact plans through the established synthesized-drag executor, preserving its
65+
screenshot-based coordinate frame, keyboard/fallback policy, and swipe-versus-pan profile.
66+
Two-contact plans convert every planned point to native orientation and feed the exact arrays to
67+
the private XCTest event bridge. macOS lowers a one-contact plan to its drag executor and tvOS
68+
lowers it to remote direction; multi-touch remains capability-gated to iOS simulators.
6569
- WebDriver lowers a supported plan to synchronized W3C pointer action sources. Multi-touch remains
6670
capability-gated until a provider proves it.
6771

@@ -84,6 +88,7 @@ selectors or refs and therefore cannot claim element-targeting guarantees.
8488

8589
- CLI, Node.js, MCP, runtime, and platform adapters share one normalization and planning model.
8690
- Adding an ergonomic gesture alias does not add a platform implementation.
91+
- Public compatibility aliases retain executor behavior without changing canonical pan/fling intent.
8792
- One-finger pan remains the default and explicit two-finger pan retains pan intent.
8893
- The active viewport is resolved for each gesture, so rotation, keyboard, and window changes do
8994
not use stale geometry.

src/commands/cli-grammar/flag-definitions-action.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { RESPONSE_LEVELS } from '../../kernel/contracts.ts';
22
import { RECORDING_SCOPE_VALUES } from '../../contracts/recording-scope.ts';
3+
import { SWIPE_PAUSE_MAX_MS, SWIPE_REPEAT_COUNT_MAX } from '../../contracts/scroll-gesture.ts';
34
import type { FlagDefinition } from './flag-types.ts';
45

56
export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [
@@ -8,7 +9,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [
89
names: ['--count'],
910
type: 'int',
1011
min: 1,
11-
max: 200,
12+
max: SWIPE_REPEAT_COUNT_MAX,
1213
usageLabel: '--count <n>',
1314
usageDescription: 'Repeat count for press/swipe series',
1415
},
@@ -172,7 +173,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [
172173
names: ['--pause-ms'],
173174
type: 'int',
174175
min: 0,
175-
max: 10_000,
176+
max: SWIPE_PAUSE_MAX_MS,
176177
usageLabel: '--pause-ms <ms>',
177178
usageDescription: 'Delay between swipe iterations',
178179
},

src/commands/interaction/metadata.ts

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,10 @@ import { CLICK_BUTTONS } from '../../core/click-button.ts';
2424
import { SCROLL_DURATION_MAX_MS } from '../../contracts/scroll-command.ts';
2525
import {
2626
SCROLL_DIRECTIONS,
27+
SWIPE_PAUSE_MAX_MS,
2728
SWIPE_PATTERNS,
2829
SWIPE_PRESETS,
30+
SWIPE_REPEAT_COUNT_MAX,
2931
} from '../../contracts/scroll-gesture.ts';
3032
import { SCROLL_INPUT_DIRECTIONS } from './runtime/gestures.ts';
3133
import { FIND_LOCATORS } from '../../selectors/find.ts';
@@ -138,8 +140,8 @@ const swipeFields = {
138140
from: requiredField(pointField('Swipe start point.')),
139141
to: requiredField(pointField('Swipe end point.')),
140142
durationMs: integerField('Deprecated: timed movement is a pan; omit for swipe.', { min: 0 }),
141-
count: integerField('Number of swipe repetitions.', { min: 1 }),
142-
pauseMs: integerField('Pause between repeated swipes.', { min: 0 }),
143+
count: integerField('Number of swipe repetitions.', { min: 1, max: SWIPE_REPEAT_COUNT_MAX }),
144+
pauseMs: integerField('Pause between repeated swipes.', { min: 0, max: SWIPE_PAUSE_MAX_MS }),
143145
pattern: enumField(SWIPE_PATTERNS),
144146
};
145147

src/commands/interaction/runtime/gesture-command.ts

Lines changed: 43 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ import { buildGesturePlan } from '../../../contracts/gesture-plan.ts';
44
import type { NormalizedGestureInput } from '../../../contracts/gesture-normalization.ts';
55
import { buildSwipePresetGesturePlan } from '../../../contracts/scroll-gesture.ts';
66
import type { Point, Rect } from '../../../kernel/snapshot.ts';
7-
import { AppError } from '../../../kernel/errors.ts';
7+
import { AppError, normalizeError } from '../../../kernel/errors.ts';
8+
import { emitDiagnostic } from '../../../utils/diagnostics.ts';
89
import { successText } from '../../../utils/success-text.ts';
910
import { toBackendContext } from '../../runtime-common.ts';
1011
import {
@@ -91,14 +92,50 @@ async function captureGestureViewport(
9192
runtime: AgentDeviceRuntime,
9293
options: GestureCommandOptions,
9394
): Promise<Rect> {
94-
const backendViewport = await runtime.backend.resolveGestureViewport?.(
95-
toBackendContext(runtime, options),
96-
);
97-
if (backendViewport) return backendViewport;
95+
let backendViewport: unknown;
96+
try {
97+
backendViewport = await runtime.backend.resolveGestureViewport?.(
98+
toBackendContext(runtime, options),
99+
);
100+
} catch (error) {
101+
emitDiagnostic({
102+
level: 'warn',
103+
phase: 'gesture_viewport_probe_failed',
104+
data: {
105+
platform: runtime.backend.platform,
106+
error: normalizeError(error).message,
107+
},
108+
});
109+
}
110+
if (isUsableGestureViewport(backendViewport)) return backendViewport;
111+
if (backendViewport !== undefined) {
112+
emitDiagnostic({
113+
level: 'warn',
114+
phase: 'gesture_viewport_probe_unusable',
115+
data: { platform: runtime.backend.platform },
116+
});
117+
}
98118
const capture = await captureInteractionSnapshot(runtime, options, false);
99119
return resolveVisibleSnapshotViewport(capture.snapshot.nodes, 'gesture');
100120
}
101121

122+
function isUsableGestureViewport(value: unknown): value is Rect {
123+
if (!value || typeof value !== 'object') return false;
124+
const candidate = value as Record<string, unknown>;
125+
return (
126+
isFiniteNumber(candidate.x) &&
127+
isFiniteNumber(candidate.y) &&
128+
isFiniteNumber(candidate.width) &&
129+
isFiniteNumber(candidate.height) &&
130+
candidate.width > 0 &&
131+
candidate.height > 0
132+
);
133+
}
134+
135+
function isFiniteNumber(value: unknown): value is number {
136+
return typeof value === 'number' && Number.isFinite(value);
137+
}
138+
102139
function resolvePresetGesture(input: NormalizedGestureInput, viewport: Rect): GestureSemanticInput {
103140
if (!('preset' in input)) return input;
104141
const relative = buildSwipePresetGesturePlan(input.preset, {
@@ -113,6 +150,7 @@ function resolvePresetGesture(input: NormalizedGestureInput, viewport: Rect): Ge
113150
origin: from,
114151
delta: { x: to.x - from.x, y: to.y - from.y },
115152
durationMs: input.durationMs,
153+
...(input.executionProfile ? { executionProfile: input.executionProfile } : {}),
116154
};
117155
}
118156

src/commands/interaction/runtime/gestures.test.ts

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import { AppError } from '../../../kernel/errors.ts';
55
import { makeSnapshotState } from '../../../__tests__/test-utils/index.ts';
66
import type { AgentDeviceBackend } from '../../../backend.ts';
77
import type { GesturePlan } from '../../../contracts/gesture-plan.ts';
8+
import type { Rect } from '../../../kernel/snapshot.ts';
9+
import { parseAndroidDisplayGeometry } from '../../../platforms/android/display-geometry.ts';
810
import {
911
createInteractionDevice,
1012
runtimeScrollSnapshot,
@@ -345,6 +347,114 @@ test('runtime directional swipe uses the visible viewport instead of off-screen
345347
]);
346348
});
347349

350+
test('runtime coordinate swipe uses backend viewport geometry without accessibility capture', async () => {
351+
let capturedViewport: unknown;
352+
let executionCount = 0;
353+
const device = createInteractionDevice(selectorSnapshot(), {
354+
platform: 'android',
355+
captureSnapshot: async () => {
356+
throw new Error('coordinate swipe must not capture Android accessibility state');
357+
},
358+
resolveGestureViewport: async () => ({ x: 0, y: 0, width: 400, height: 800 }),
359+
performGesture: async (_context, plan) => {
360+
capturedViewport = plan.viewport;
361+
executionCount += 1;
362+
},
363+
});
364+
365+
const result = await device.interactions.swipe({
366+
from: { x: 300, y: 400 },
367+
to: { x: 100, y: 400 },
368+
session: 'default',
369+
});
370+
371+
assert.equal(result.kind, 'fling');
372+
assert.deepEqual(capturedViewport, { x: 0, y: 0, width: 400, height: 800 });
373+
assert.equal(executionCount, 1);
374+
});
375+
376+
test('runtime coordinate swipe keeps deterministic viewport bounds validation on the fast path', async () => {
377+
let executionCount = 0;
378+
const device = createInteractionDevice(selectorSnapshot(), {
379+
platform: 'android',
380+
captureSnapshot: async () => {
381+
throw new Error('coordinate swipe must not capture Android accessibility state');
382+
},
383+
resolveGestureViewport: async () => ({ x: 0, y: 0, width: 400, height: 800 }),
384+
performGesture: async () => {
385+
executionCount += 1;
386+
},
387+
});
388+
389+
await assert.rejects(
390+
() =>
391+
device.interactions.swipe({
392+
from: { x: 400, y: 400 },
393+
to: { x: 100, y: 400 },
394+
session: 'default',
395+
}),
396+
(error: unknown) =>
397+
error instanceof AppError &&
398+
error.code === 'INVALID_ARGS' &&
399+
error.details?.reason === 'GESTURE_TRAJECTORY_OUT_OF_BOUNDS',
400+
);
401+
assert.equal(executionCount, 0);
402+
});
403+
404+
test('runtime coordinate swipe falls back to the snapshot viewport when display probing fails', async () => {
405+
let snapshotCaptureCount = 0;
406+
let capturedViewport: unknown;
407+
const snapshot = selectorSnapshot();
408+
const device = createInteractionDevice(snapshot, {
409+
platform: 'android',
410+
resolveGestureViewport: async () => {
411+
throw parseAndroidDisplayGeometry('Displays:\n');
412+
},
413+
captureSnapshot: async () => {
414+
snapshotCaptureCount += 1;
415+
return { snapshot };
416+
},
417+
performGesture: async (_context, plan) => {
418+
capturedViewport = plan.viewport;
419+
},
420+
});
421+
422+
const result = await device.interactions.swipe({
423+
from: { x: 20, y: 30 },
424+
to: { x: 90, y: 30 },
425+
session: 'default',
426+
});
427+
428+
assert.equal(result.kind, 'fling');
429+
assert.equal(snapshotCaptureCount, 1);
430+
assert.deepEqual(capturedViewport, { x: 10, y: 20, width: 100, height: 40 });
431+
});
432+
433+
test('runtime gesture falls back when the backend returns an unusable viewport shape', async () => {
434+
let snapshotCaptureCount = 0;
435+
let capturedViewport: unknown;
436+
const snapshot = selectorSnapshot();
437+
const device = createInteractionDevice(snapshot, {
438+
platform: 'android',
439+
resolveGestureViewport: async () => ({ x: 0, y: 0, width: 0, height: 800 }) as unknown as Rect,
440+
captureSnapshot: async () => {
441+
snapshotCaptureCount += 1;
442+
return { snapshot };
443+
},
444+
performGesture: async (_context, plan) => {
445+
capturedViewport = plan.viewport;
446+
},
447+
});
448+
449+
await device.interactions.pan({
450+
origin: { x: 20, y: 30 },
451+
delta: { x: 10, y: 0 },
452+
});
453+
454+
assert.equal(snapshotCaptureCount, 1);
455+
assert.deepEqual(capturedViewport, { x: 10, y: 20, width: 100, height: 40 });
456+
});
457+
348458
test('runtime gesture swipe presets use stable viewport lanes', async () => {
349459
const calls: unknown[] = [];
350460
const device = createInteractionDevice(snapshotWithOffscreenContent(), {

src/contracts/gesture-normalization.test.ts

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,12 @@ test('swipe is fling sugar unless legacy duration requests a pan', () => {
2424
deprecations: [],
2525
});
2626
assert.deepEqual(normalizePublicGesture({ kind: 'swipe', preset: 'left', durationMs: 400 }), {
27-
gesture: { intent: 'pan', preset: 'left', durationMs: 400 },
27+
gesture: {
28+
intent: 'pan',
29+
preset: 'left',
30+
durationMs: 400,
31+
executionProfile: 'swipe',
32+
},
2833
deprecations: [{ rule: 'swipe-duration', replacement: 'Use gesture pan for timed movement.' }],
2934
});
3035
});
@@ -44,6 +49,7 @@ test('duration-bearing fling is an explicit compatibility alias for pan', () =>
4449
origin: { x: 200, y: 300 },
4550
delta: { x: -80, y: 0 },
4651
durationMs: 500,
52+
executionProfile: 'swipe',
4753
},
4854
deprecations: [
4955
{ rule: 'fling-duration', replacement: 'Use gesture pan for timed movement.' },

src/contracts/gesture-normalization.ts

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import type { Point } from '../kernel/snapshot.ts';
22
import type { SwipePreset } from './scroll-gesture.ts';
33
import { readGesturePayload, type GesturePayload } from './gesture-input.ts';
4-
import type { GestureSemanticInput } from './gesture-plan-types.ts';
4+
import type { GestureExecutionProfile, GestureSemanticInput } from './gesture-plan-types.ts';
55

66
export type GestureCompatibilityRule = 'swipe-duration' | 'fling-duration' | 'rotate-velocity';
77

@@ -12,8 +12,16 @@ export type GestureDeprecation = {
1212

1313
export type NormalizedGestureInput =
1414
| GestureSemanticInput
15-
| { intent: 'fling'; preset: SwipePreset }
16-
| { intent: 'pan'; preset: SwipePreset; durationMs: number };
15+
| {
16+
intent: 'fling';
17+
preset: SwipePreset;
18+
}
19+
| {
20+
intent: 'pan';
21+
preset: SwipePreset;
22+
durationMs: number;
23+
executionProfile?: GestureExecutionProfile;
24+
};
1725

1826
export type NormalizedPublicGesture = {
1927
gesture: NormalizedGestureInput;
@@ -100,6 +108,7 @@ export function normalizePublicGesture(input: GesturePayload): NormalizedPublicG
100108
origin: input.origin,
101109
delta: directionDelta(input.direction, input.distance ?? 180),
102110
durationMs: input.durationMs,
111+
executionProfile: 'swipe',
103112
},
104113
deprecations: [
105114
{ rule: 'fling-duration', replacement: 'Use gesture pan for timed movement.' },
@@ -120,7 +129,12 @@ export function normalizePublicGesture(input: GesturePayload): NormalizedPublicG
120129
return input.durationMs === undefined
121130
? { gesture: { intent: 'fling', preset: input.preset }, deprecations: [] }
122131
: {
123-
gesture: { intent: 'pan', preset: input.preset, durationMs: input.durationMs },
132+
gesture: {
133+
intent: 'pan',
134+
preset: input.preset,
135+
durationMs: input.durationMs,
136+
executionProfile: 'swipe',
137+
},
124138
deprecations: [
125139
{ rule: 'swipe-duration', replacement: 'Use gesture pan for timed movement.' },
126140
],
@@ -175,6 +189,7 @@ export function normalizePublicSwipeMotion(input: {
175189
origin: input.from,
176190
delta: { x: input.to.x - input.from.x, y: input.to.y - input.from.y },
177191
durationMs: input.durationMs,
192+
executionProfile: 'swipe',
178193
},
179194
deprecations: [{ rule: 'swipe-duration', replacement: 'Use gesture pan for timed movement.' }],
180195
};

0 commit comments

Comments
 (0)